Changeset 2504
- Timestamp:
- 13/04/2009 19:40:50 (3 years ago)
- Location:
- box/trunk
- Files:
-
- 1 added
- 7 edited
- 4 copied
-
bin/s3simulator (added)
-
bin/s3simulator/s3simulator.cpp (copied) (copied from box/trunk/bin/bbackupd/bbackupd.cpp) (3 diffs)
-
infrastructure/makebuildenv.pl.in (modified) (1 diff)
-
lib/httpserver/HTTPRequest.cpp (modified) (4 diffs)
-
lib/httpserver/HTTPRequest.h (modified) (4 diffs)
-
lib/httpserver/HTTPServer.cpp (modified) (5 diffs)
-
lib/httpserver/HTTPServer.h (modified) (1 diff)
-
lib/httpserver/S3Simulator.cpp (copied) (copied from box/trunk/lib/httpserver/S3Client.cpp) (5 diffs)
-
lib/httpserver/S3Simulator.h (copied) (copied from box/trunk/lib/httpserver/S3Client.h) (1 diff)
-
modules.txt (modified) (1 diff)
-
test/httpserver/testfiles/s3simulator.conf (copied) (copied from box/trunk/test/httpserver/testfiles/httpserver.conf) (1 diff)
-
test/httpserver/testhttpserver.cpp (modified) (9 diffs)
Legend:
- Unmodified
- Added
- Removed
-
box/trunk/bin/s3simulator/s3simulator.cpp
r2180 r2504 2 2 // 3 3 // File 4 // Name: bbackupd.cpp5 // Purpose: main file for backupdaemon4 // Name: s3simulator.cpp 5 // Purpose: main file for S3 simulator daemon 6 6 // Created: 2003/10/11 7 7 // … … 9 9 10 10 #include "Box.h" 11 #include " BackupDaemon.h"11 #include "S3Simulator.h" 12 12 #include "MainHelper.h" 13 #include "BoxPortsAndFiles.h"14 #include "BackupStoreException.h"15 #include "Logging.h"16 13 17 14 #include "MemLeakFindOn.h" 18 19 #ifdef WIN3220 #include "Win32ServiceFunctions.h"21 #include "Win32BackupService.h"22 23 extern Win32BackupService* gpDaemonService;24 #endif25 15 26 16 int main(int argc, const char *argv[]) … … 30 20 MAINHELPER_START 31 21 32 Logging::SetProgramName(" bbackupd");22 Logging::SetProgramName("s3simulator"); 33 23 Logging::ToConsole(true); 34 24 Logging::ToSyslog (true); 35 25 36 #ifdef WIN32 37 38 EnableBackupRights(); 39 40 gpDaemonService = new Win32BackupService(); 41 ExitCode = gpDaemonService->Daemon::Main( 42 BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE, 43 argc, argv); 44 delete gpDaemonService; 45 46 #else // !WIN32 47 48 BackupDaemon daemon; 49 ExitCode = daemon.Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); 50 51 #endif // WIN32 26 S3Simulator daemon; 27 ExitCode = daemon.Main("s3simulator.conf", argc, argv); 52 28 53 29 MAINHELPER_END -
box/trunk/infrastructure/makebuildenv.pl.in
r2415 r2504 432 432 kill_process bbackupd 433 433 kill_process bbstored 434 kill_process httpserver 435 kill_process s3simulator 434 436 __E 435 437 } -
box/trunk/lib/httpserver/HTTPRequest.cpp
r2453 r2504 536 536 ++dataStart; 537 537 } 538 539 if(p == sizeof("Content-Length")-1 540 && ::strncasecmp(h, "Content-Length", sizeof("Content-Length")-1) == 0) 538 539 std::string header_name(ToLowerCase(std::string(h, 540 p))); 541 542 if (header_name == "content-length") 541 543 { 542 544 // Decode number … … 546 548 mContentLength = len; 547 549 } 548 else if(p == sizeof("Content-Type")-1 549 && ::strncasecmp(h, "Content-Type", sizeof("Content-Type")-1) == 0) 550 else if (header_name == "content-type") 550 551 { 551 552 // Store rest of string as content type 552 553 mContentType = h + dataStart; 553 554 } 554 else if(p == sizeof("Host")-1 555 && ::strncasecmp(h, "Host", sizeof("Host")-1) == 0) 555 else if (header_name == "host") 556 556 { 557 557 // Store host header … … 573 573 } 574 574 } 575 else if(p == sizeof("Cookie")-1 576 && ::strncasecmp(h, "Cookie", sizeof("Cookie")-1) == 0) 575 else if (header_name == "cookie") 577 576 { 578 577 // Parse cookies 579 578 ParseCookies(header, dataStart); 580 579 } 581 else if(p == sizeof("Connection")-1 582 && ::strncasecmp(h, "Connection", sizeof("Connection")-1) == 0) 580 else if (header_name == "connection") 583 581 { 584 582 // Connection header, what is required? … … 596 594 else 597 595 { 598 std::string name = header.substr(0, p); 599 mExtraHeaders.push_back(Header(name, 596 mExtraHeaders.push_back(Header(header_name, 600 597 h + dataStart)); 601 598 } -
box/trunk/lib/httpserver/HTTPRequest.h
r2442 r2504 98 98 bool GetHeader(const std::string& rName, std::string* pValueOut) const 99 99 { 100 std::string header = ToLowerCase(rName); 101 100 102 for (std::vector<Header>::const_iterator 101 103 i = mExtraHeaders.begin(); 102 104 i != mExtraHeaders.end(); i++) 103 105 { 104 if (i->first == rName)106 if (i->first == header) 105 107 { 106 108 *pValueOut = i->second; … … 108 110 } 109 111 } 112 110 113 return false; 111 114 } … … 129 132 void AddHeader(const std::string& rName, const std::string& rValue) 130 133 { 131 mExtraHeaders.push_back(Header( rName, rValue));134 mExtraHeaders.push_back(Header(ToLowerCase(rName), rValue)); 132 135 } 133 136 bool IsExpectingContinue() const { return mExpectContinue; } … … 169 172 IOStream* mpStreamToReadFrom; 170 173 std::string mHttpVerb; 174 175 std::string ToLowerCase(const std::string& rInput) const 176 { 177 std::string output = rInput; 178 for (std::string::iterator c = output.begin(); 179 c != output.end(); c++) 180 { 181 *c = tolower(*c); 182 } 183 return output; 184 } 171 185 }; 172 186 -
box/trunk/lib/httpserver/HTTPServer.cpp
r2440 r2504 154 154 // Generate a response 155 155 HTTPResponse response(&rStream); 156 156 157 try 157 158 { … … 160 161 catch(BoxException &e) 161 162 { 162 char exceptionCode[ 64];163 ::sprintf(exceptionCode, " (%d/%d)", e.GetType(), e.GetSubType());164 SendInternalErrorResponse(exceptionCode, rStream);165 return;163 char exceptionCode[256]; 164 ::sprintf(exceptionCode, "%s (%d/%d)", e.what(), 165 e.GetType(), e.GetSubType()); 166 SendInternalErrorResponse(exceptionCode, response); 166 167 } 167 168 catch(...) 168 169 { 169 SendInternalErrorResponse("unknown", rStream); 170 return; 170 SendInternalErrorResponse("unknown", response); 171 171 } 172 172 … … 187 187 } 188 188 189 // Notify derived cla ases189 // Notify derived classes 190 190 HTTPConnectionClosing(); 191 191 } … … 195 195 // 196 196 // Function 197 // Name: HTTPServer::SendInternalErrorResponse(const char *, SocketStream &) 198 // Purpose: Sends an error response to the remote side 199 // Created: 26/3/04 200 // 201 // -------------------------------------------------------------------------- 202 void HTTPServer::SendInternalErrorResponse(const char *Error, SocketStream &rStream) 197 // Name: HTTPServer::SendInternalErrorResponse(const char*, 198 // HTTPResponse&) 199 // Purpose: Generates an error message in the provided response 200 // Created: 26/3/04 201 // 202 // -------------------------------------------------------------------------- 203 void HTTPServer::SendInternalErrorResponse(const std::string& rErrorMsg, 204 HTTPResponse& rResponse) 203 205 { 204 206 #define ERROR_HTML_1 "<html><head><title>Internal Server Error</title></head>\n" \ … … 210 212 211 213 // Generate the error page 212 HTTPResponse response(&rStream); 213 response.SetResponseCode(HTTPResponse::Code_InternalServerError); 214 response.SetContentType("text/html"); 215 response.Write(ERROR_HTML_1, sizeof(ERROR_HTML_1) - 1); 216 response.Write(Error, ::strlen(Error)); 217 response.Write(ERROR_HTML_2, sizeof(ERROR_HTML_2) - 1); 218 219 // Send the error response 220 response.Send(); 214 // rResponse.SetResponseCode(HTTPResponse::Code_InternalServerError); 215 rResponse.SetContentType("text/html"); 216 rResponse.Write(ERROR_HTML_1, sizeof(ERROR_HTML_1) - 1); 217 rResponse.IOStream::Write(rErrorMsg.c_str()); 218 rResponse.Write(ERROR_HTML_2, sizeof(ERROR_HTML_2) - 1); 221 219 } 222 220 -
box/trunk/lib/httpserver/HTTPServer.h
r2440 r2504 53 53 virtual void HTTPConnectionClosing(); 54 54 55 protected: 56 void SendInternalErrorResponse(const std::string& rErrorMsg, 57 HTTPResponse& rResponse); 58 int GetTimeout() { return mTimeout; } 59 55 60 private: 61 int mTimeout; // Timeout for read operations 56 62 const char *DaemonName() const; 57 63 const ConfigurationVerify *GetConfigVerify() const; 58 64 void Run(); 59 65 void Connection(SocketStream &rStream); 60 void SendInternalErrorResponse(const char *Error, SocketStream &rStream);61 62 private:63 int mTimeout; // Timeout for read operations64 66 }; 65 67 -
box/trunk/lib/httpserver/S3Simulator.cpp
r2447 r2504 19 19 #include "HTTPRequest.h" 20 20 #include "HTTPResponse.h" 21 #include "HTTPServer.h"22 21 #include "autogen_HTTPException.h" 23 22 #include "IOStream.h" 24 23 #include "Logging.h" 25 #include "S3 Client.h"24 #include "S3Simulator.h" 26 25 #include "decode.h" 27 26 #include "encode.h" … … 32 31 // 33 32 // Function 34 // Name: S3Client::GetObject(const std::string& rObjectURI) 35 // Purpose: Retrieve the object with the specified URI (key) 36 // from your S3 bucket. 33 // Name: HTTPServer::GetConfigVerify() 34 // Purpose: Returns additional configuration options for the 35 // S3 simulator. Currently the access key, secret key 36 // and store directory can be configured. 37 37 // Created: 09/01/09 38 38 // 39 39 // -------------------------------------------------------------------------- 40 41 HTTPResponse S3Client::GetObject(const std::string& rObjectURI) 40 const ConfigurationVerify* S3Simulator::GetConfigVerify() const 42 41 { 43 return FinishAndSendRequest(HTTPRequest::Method_GET, rObjectURI); 42 static ConfigurationVerifyKey verifyserverkeys[] = 43 { 44 HTTPSERVER_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) // no default addresses 45 }; 46 47 static ConfigurationVerify verifyserver[] = 48 { 49 { 50 "Server", 51 0, 52 verifyserverkeys, 53 ConfigTest_Exists | ConfigTest_LastEntry, 54 0 55 } 56 }; 57 58 static ConfigurationVerifyKey verifyrootkeys[] = 59 { 60 ConfigurationVerifyKey("AccessKey", ConfigTest_Exists), 61 ConfigurationVerifyKey("SecretKey", ConfigTest_Exists), 62 ConfigurationVerifyKey("StoreDirectory", ConfigTest_Exists), 63 HTTPSERVER_VERIFY_ROOT_KEYS 64 }; 65 66 static ConfigurationVerify verify = 67 { 68 "root", 69 verifyserver, 70 verifyrootkeys, 71 ConfigTest_Exists | ConfigTest_LastEntry, 72 0 73 }; 74 75 return &verify; 44 76 } 45 77 … … 47 79 // 48 80 // Function 49 // Name: S3 Client::PutObject(const std::string& rObjectURI,50 // IOStream& rStreamToSend, const char* pContentType)51 // Purpose: Upload the stream to S3, creating or overwriting the52 // object with the specified URI (key) in your S353 // bucket.81 // Name: S3Simulator::Handle(HTTPRequest &rRequest, 82 // HTTPResponse &rResponse) 83 // Purpose: Handles any incoming S3 request, by checking 84 // authorization and then dispatching to one of the 85 // private Handle* methods. 54 86 // Created: 09/01/09 55 87 // 56 88 // -------------------------------------------------------------------------- 57 89 58 HTTPResponse S3Client::PutObject(const std::string& rObjectURI, 59 IOStream& rStreamToSend, const char* pContentType) 90 void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) 60 91 { 61 return FinishAndSendRequest(HTTPRequest::Method_PUT, rObjectURI, 62 &rStreamToSend, pContentType); 92 // if anything goes wrong, return a 500 error 93 rResponse.SetResponseCode(HTTPResponse::Code_InternalServerError); 94 rResponse.SetContentType("text/plain"); 95 96 try 97 { 98 const Configuration& rConfig(GetConfiguration()); 99 std::string access_key = rConfig.GetKeyValue("AccessKey"); 100 std::string secret_key = rConfig.GetKeyValue("SecretKey"); 101 102 std::string md5, date, bucket; 103 rRequest.GetHeader("content-md5", &md5); 104 rRequest.GetHeader("date", &date); 105 106 std::string host = rRequest.GetHostName(); 107 std::string s3suffix = ".s3.amazonaws.com"; 108 if (host.size() > s3suffix.size()) 109 { 110 std::string suffix = host.substr(host.size() - 111 s3suffix.size(), s3suffix.size()); 112 if (suffix == s3suffix) 113 { 114 bucket = host.substr(0, host.size() - 115 s3suffix.size()); 116 } 117 } 118 119 std::ostringstream data; 120 data << rRequest.GetVerb() << "\n"; 121 data << md5 << "\n"; 122 data << rRequest.GetContentType() << "\n"; 123 data << date << "\n"; 124 125 // header names are already in lower case, i.e. canonical form 126 127 std::vector<HTTPRequest::Header> headers = rRequest.GetHeaders(); 128 sort(headers.begin(), headers.end()); 129 130 for (std::vector<HTTPRequest::Header>::iterator 131 i = headers.begin(); i != headers.end(); i++) 132 { 133 if (i->first.substr(0, 5) == "x-amz") 134 { 135 data << i->first << ":" << i->second << "\n"; 136 } 137 } 138 139 if (! bucket.empty()) 140 { 141 data << "/" << bucket; 142 } 143 144 data << rRequest.GetRequestURI(); 145 std::string data_string = data.str(); 146 147 unsigned char digest_buffer[EVP_MAX_MD_SIZE]; 148 unsigned int digest_size = sizeof(digest_buffer); 149 /* unsigned char* mac = */ HMAC(EVP_sha1(), 150 secret_key.c_str(), secret_key.size(), 151 (const unsigned char*)data_string.c_str(), 152 data_string.size(), digest_buffer, &digest_size); 153 std::string digest((const char *)digest_buffer, digest_size); 154 155 base64::encoder encoder; 156 std::string expectedAuth = "AWS " + access_key + ":" + 157 encoder.encode(digest); 158 159 if (expectedAuth[expectedAuth.size() - 1] == '\n') 160 { 161 expectedAuth = expectedAuth.substr(0, 162 expectedAuth.size() - 1); 163 } 164 165 std::string actualAuth; 166 if (!rRequest.GetHeader("authorization", &actualAuth) || 167 actualAuth != expectedAuth) 168 { 169 rResponse.SetResponseCode(HTTPResponse::Code_Unauthorized); 170 SendInternalErrorResponse("Authentication Failed", 171 rResponse); 172 } 173 else if (rRequest.GetMethod() == HTTPRequest::Method_GET) 174 { 175 HandleGet(rRequest, rResponse); 176 } 177 else if (rRequest.GetMethod() == HTTPRequest::Method_PUT) 178 { 179 HandlePut(rRequest, rResponse); 180 } 181 else 182 { 183 rResponse.SetResponseCode(HTTPResponse::Code_MethodNotAllowed); 184 SendInternalErrorResponse("Unsupported Method", 185 rResponse); 186 } 187 } 188 catch (CommonException &ce) 189 { 190 SendInternalErrorResponse(ce.what(), rResponse); 191 } 192 catch (std::exception &e) 193 { 194 SendInternalErrorResponse(e.what(), rResponse); 195 } 196 catch (...) 197 { 198 SendInternalErrorResponse("Unknown exception", rResponse); 199 } 200 201 if (rResponse.GetResponseCode() != 200 && 202 rResponse.GetSize() == 0) 203 { 204 // no error message written, provide a default 205 std::ostringstream s; 206 s << rResponse.GetResponseCode(); 207 SendInternalErrorResponse(s.str().c_str(), rResponse); 208 } 209 210 return; 63 211 } 64 212 … … 66 214 // 67 215 // Function 68 // Name: S3Client::FinishAndSendRequest( 69 // HTTPRequest::Method Method, 70 // const std::string& rRequestURI, 71 // IOStream* pStreamToSend, 72 // const char* pStreamContentType) 73 // Purpose: Internal method which creates an HTTP request to S3, 74 // populates the date and authorization header fields, 75 // and sends it to S3 (or the simulator), attaching 76 // the specified stream if any to the request. Opens a 77 // connection to the server if necessary, which may 78 // throw a ConnectionException. Returns the HTTP 79 // response returned by S3, which may be a 500 error. 216 // Name: S3Simulator::HandleGet(HTTPRequest &rRequest, 217 // HTTPResponse &rResponse) 218 // Purpose: Handles an S3 GET request, i.e. downloading an 219 // existing object. 80 220 // Created: 09/01/09 81 221 // 82 222 // -------------------------------------------------------------------------- 83 223 84 HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, 85 const std::string& rRequestURI, IOStream* pStreamToSend, 86 const char* pStreamContentType) 224 void S3Simulator::HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse) 87 225 { 88 HTTPRequest request(Method, rRequestURI); 89 request.SetHostName(mHostName); 90 91 std::ostringstream date; 92 time_t tt = time(NULL); 93 struct tm *tp = gmtime(&tt); 94 if (!tp) 95 { 96 BOX_ERROR("Failed to get current time"); 97 THROW_EXCEPTION(HTTPException, Internal); 98 } 99 const char *dow[] = {"Sun","Mon","Tue","Wed","Thu","Fri","Sat"}; 100 date << dow[tp->tm_wday] << ", "; 101 const char *month[] = {"Jan","Feb","Mar","Apr","May","Jun", 102 "Jul","Aug","Sep","Oct","Nov","Dec"}; 103 date << std::internal << std::setfill('0') << 104 std::setw(2) << tp->tm_mday << " " << 105 month[tp->tm_mon] << " " << 106 (tp->tm_year + 1900) << " "; 107 date << std::setw(2) << tp->tm_hour << ":" << 108 std::setw(2) << tp->tm_min << ":" << 109 std::setw(2) << tp->tm_sec << " GMT"; 110 request.AddHeader("Date", date.str()); 111 112 if (pStreamContentType) 113 { 114 request.AddHeader("Content-Type", pStreamContentType); 115 } 116 117 std::string s3suffix = ".s3.amazonaws.com"; 118 std::string bucket; 119 if (mHostName.size() > s3suffix.size()) 120 { 121 std::string suffix = mHostName.substr(mHostName.size() - 122 s3suffix.size(), s3suffix.size()); 123 if (suffix == s3suffix) 124 { 125 bucket = mHostName.substr(0, mHostName.size() - 126 s3suffix.size()); 127 } 128 } 129 130 std::ostringstream data; 131 data << request.GetVerb() << "\n"; 132 data << "\n"; /* Content-MD5 */ 133 data << request.GetContentType() << "\n"; 134 data << date.str() << "\n"; 135 136 if (! bucket.empty()) 137 { 138 data << "/" << bucket; 139 } 140 141 data << request.GetRequestURI(); 142 std::string data_string = data.str(); 143 144 unsigned char digest_buffer[EVP_MAX_MD_SIZE]; 145 unsigned int digest_size = sizeof(digest_buffer); 146 /* unsigned char* mac = */ HMAC(EVP_sha1(), 147 mSecretKey.c_str(), mSecretKey.size(), 148 (const unsigned char*)data_string.c_str(), 149 data_string.size(), digest_buffer, &digest_size); 150 std::string digest((const char *)digest_buffer, digest_size); 151 152 base64::encoder encoder; 153 std::string auth_code = "AWS " + mAccessKey + ":" + 154 encoder.encode(digest); 155 156 if (auth_code[auth_code.size() - 1] == '\n') 157 { 158 auth_code = auth_code.substr(0, auth_code.size() - 1); 159 } 160 161 request.AddHeader("Authorization", auth_code); 162 163 if (mpSimulator) 164 { 165 if (pStreamToSend) 166 { 167 pStreamToSend->CopyStreamTo(request); 168 } 169 170 request.SetForReading(); 171 CollectInBufferStream response_buffer; 172 HTTPResponse response(&response_buffer); 173 174 mpSimulator->Handle(request, response); 175 return response; 176 } 177 else 178 { 179 try 180 { 181 if (!mapClientSocket.get()) 182 { 183 mapClientSocket.reset(new SocketStream()); 184 mapClientSocket->Open(Socket::TypeINET, 185 mHostName, mPort); 186 } 187 return SendRequest(request, pStreamToSend, 188 pStreamContentType); 189 } 190 catch (ConnectionException &ce) 191 { 192 if (ce.GetType() == ConnectionException::SocketWriteError) 193 { 194 // server may have disconnected us, 195 // try to reconnect, just once 196 mapClientSocket->Open(Socket::TypeINET, 197 mHostName, mPort); 198 return SendRequest(request, pStreamToSend, 199 pStreamContentType); 200 } 201 else 202 { 203 throw; 204 } 205 } 206 } 226 std::string path = GetConfiguration().GetKeyValue("StoreDirectory"); 227 path += rRequest.GetRequestURI(); 228 std::auto_ptr<FileStream> apFile; 229 230 try 231 { 232 apFile.reset(new FileStream(path)); 233 } 234 catch (CommonException &ce) 235 { 236 if (ce.GetSubType() == CommonException::OSFileOpenError) 237 { 238 rResponse.SetResponseCode(HTTPResponse::Code_NotFound); 239 } 240 else if (ce.GetSubType() == CommonException::AccessDenied) 241 { 242 rResponse.SetResponseCode(HTTPResponse::Code_Forbidden); 243 } 244 throw; 245 } 246 247 // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingRESTOperations.html 248 apFile->CopyStreamTo(rResponse); 249 rResponse.AddHeader("x-amz-id-2", "qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY"); 250 rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); 251 rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); 252 rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); 253 rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\""); 254 rResponse.AddHeader("Server", "AmazonS3"); 255 rResponse.SetResponseCode(HTTPResponse::Code_OK); 207 256 } 208 257 … … 210 259 // 211 260 // Function 212 // Name: S3Client::SendRequest(HTTPRequest& rRequest, 213 // IOStream* pStreamToSend, 214 // const char* pStreamContentType) 215 // Purpose: Internal method which sends a pre-existing HTTP 216 // request to S3. Attaches the specified stream if any 217 // to the request. Opens a connection to the server if 218 // necessary, which may throw a ConnectionException. 219 // Returns the HTTP response returned by S3, which may 220 // be a 500 error. 261 // Name: S3Simulator::HandlePut(HTTPRequest &rRequest, 262 // HTTPResponse &rResponse) 263 // Purpose: Handles an S3 PUT request, i.e. uploading data to 264 // create or replace an object. 221 265 // Created: 09/01/09 222 266 // 223 267 // -------------------------------------------------------------------------- 224 268 225 HTTPResponse S3Client::SendRequest(HTTPRequest& rRequest, 226 IOStream* pStreamToSend, const char* pStreamContentType) 227 { 228 HTTPResponse response; 229 230 if (pStreamToSend) 231 { 232 rRequest.SendWithStream(*mapClientSocket, 233 30000 /* milliseconds */, 234 pStreamToSend, response); 235 } 236 else 237 { 238 rRequest.Send(*mapClientSocket, 30000 /* milliseconds */); 239 response.Receive(*mapClientSocket, 30000 /* milliseconds */); 240 } 241 242 return response; 243 } 269 void S3Simulator::HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse) 270 { 271 std::string path = GetConfiguration().GetKeyValue("StoreDirectory"); 272 path += rRequest.GetRequestURI(); 273 std::auto_ptr<FileStream> apFile; 274 275 try 276 { 277 apFile.reset(new FileStream(path, O_CREAT | O_WRONLY)); 278 } 279 catch (CommonException &ce) 280 { 281 if (ce.GetSubType() == CommonException::OSFileOpenError) 282 { 283 rResponse.SetResponseCode(HTTPResponse::Code_NotFound); 284 } 285 else if (ce.GetSubType() == CommonException::AccessDenied) 286 { 287 rResponse.SetResponseCode(HTTPResponse::Code_Forbidden); 288 } 289 throw; 290 } 291 292 if (rRequest.IsExpectingContinue()) 293 { 294 rResponse.SendContinue(); 295 } 296 297 rRequest.ReadContent(*apFile); 298 299 // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTObjectPUT.html 300 rResponse.AddHeader("x-amz-id-2", "LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7"); 301 rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); 302 rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); 303 rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); 304 rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\""); 305 rResponse.SetContentType(""); 306 rResponse.AddHeader("Server", "AmazonS3"); 307 rResponse.SetResponseCode(HTTPResponse::Code_OK); 308 } -
box/trunk/lib/httpserver/S3Simulator.h
r2447 r2504 2 2 // 3 3 // File 4 // Name: S3 Client.h5 // Purpose: Amazon S3 client helper implementation class4 // Name: S3Simulator.h 5 // Purpose: Amazon S3 simulation HTTP server for S3 testing 6 6 // Created: 09/01/2009 7 7 // 8 8 // -------------------------------------------------------------------------- 9 9 10 #ifndef S3 CLIENT__H11 #define S3 CLIENT__H10 #ifndef S3SIMULATOR__H 11 #define S3SIMULATOR__H 12 12 13 #include <string> 14 #include <map> 13 #include "HTTPServer.h" 15 14 16 #include "HTTPRequest.h" 17 #include "SocketStream.h" 18 15 class ConfigurationVerify; 16 class HTTPRequest; 19 17 class HTTPResponse; 20 class HTTPServer;21 class IOStream;22 18 23 19 // -------------------------------------------------------------------------- 24 20 // 25 21 // Class 26 // Name: S3 Client27 // Purpose: Amazon S3 client helper implementation class22 // Name: S3Simulator 23 // Purpose: Amazon S3 simulation HTTP server for S3 testing 28 24 // Created: 09/01/2009 29 25 // 30 26 // -------------------------------------------------------------------------- 31 class S3 Client27 class S3Simulator : public HTTPServer 32 28 { 33 public: 34 S3Client(HTTPServer* pSimulator, const std::string& rHostName, 35 const std::string& rAccessKey, const std::string& rSecretKey) 36 : mpSimulator(pSimulator), 37 mHostName(rHostName), 38 mAccessKey(rAccessKey), 39 mSecretKey(rSecretKey) 40 { } 41 42 S3Client(std::string HostName, int Port, const std::string& rAccessKey, 43 const std::string& rSecretKey) 44 : mpSimulator(NULL), 45 mHostName(HostName), 46 mPort(Port), 47 mAccessKey(rAccessKey), 48 mSecretKey(rSecretKey) 49 { } 50 51 HTTPResponse GetObject(const std::string& rObjectURI); 52 HTTPResponse PutObject(const std::string& rObjectURI, 53 IOStream& rStreamToSend, const char* pContentType = NULL); 29 public: 30 S3Simulator() { } 31 ~S3Simulator() { } 54 32 55 private: 56 HTTPServer* mpSimulator; 57 std::string mHostName; 58 int mPort; 59 std::auto_ptr<SocketStream> mapClientSocket; 60 std::string mAccessKey, mSecretKey; 61 62 HTTPResponse FinishAndSendRequest(HTTPRequest::Method Method, 63 const std::string& rRequestURI, 64 IOStream* pStreamToSend = NULL, 65 const char* pStreamContentType = NULL); 66 HTTPResponse SendRequest(HTTPRequest& rRequest, 67 IOStream* pStreamToSend = NULL, 68 const char* pStreamContentType = NULL); 33 const ConfigurationVerify* GetConfigVerify() const; 34 virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse); 35 virtual void HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse); 36 virtual void HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse); 69 37 }; 70 38 71 #endif // S3 CLIENT__H39 #endif // S3SIMULATOR__H 72 40 -
box/trunk/modules.txt
r2422 r2504 45 45 lib/httpserver lib/server 46 46 test/httpserver lib/httpserver 47 bin/s3simulator lib/httpserver 47 48 48 49 # END_IF_DISTRIBUTION -
box/trunk/test/httpserver/testfiles/s3simulator.conf
r926 r2504 1 1 AccessKey = 0PN5J17HBGZHT7JJ3X82 2 SecretKey = uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o 3 StoreDirectory = testfiles 2 4 AddressPrefix = http://localhost:1080 3 5 4 6 Server 5 7 { 6 PidFile = testfiles/ httpserver.pid8 PidFile = testfiles/s3simulator.pid 7 9 ListenAddresses = inet:localhost:1080 8 10 } -
box/trunk/test/httpserver/testhttpserver.cpp
r2502 r2504 27 27 #include "IOStreamGetLine.h" 28 28 #include "S3Client.h" 29 #include "S3Simulator.h" 29 30 #include "ServerControl.h" 30 31 #include "Test.h" … … 126 127 TestWebServer::~TestWebServer() {} 127 128 128 class S3Simulator : public HTTPServer129 {130 public:131 S3Simulator() { }132 ~S3Simulator() { }133 134 virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse);135 virtual void HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse);136 virtual void HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse);137 };138 139 void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse)140 {141 // if anything goes wrong, return a 500 error142 rResponse.SetResponseCode(HTTPResponse::Code_InternalServerError);143 rResponse.SetContentType("text/plain");144 145 try146 {147 // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAuthentication.html148 std::string access_key = "0PN5J17HBGZHT7JJ3X82";149 std::string secret_key = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o";150 151 std::string md5, date, bucket;152 rRequest.GetHeader("Content-MD5", &md5);153 rRequest.GetHeader("Date", &date);154 155 std::string host = rRequest.GetHostName();156 std::string s3suffix = ".s3.amazonaws.com";157 if (host.size() > s3suffix.size())158 {159 std::string suffix = host.substr(host.size() -160 s3suffix.size(), s3suffix.size());161 if (suffix == s3suffix)162 {163 bucket = host.substr(0, host.size() -164 s3suffix.size());165 }166 }167 168 std::ostringstream data;169 data << rRequest.GetVerb() << "\n";170 data << md5 << "\n";171 data << rRequest.GetContentType() << "\n";172 data << date << "\n";173 174 std::vector<HTTPRequest::Header> headers = rRequest.GetHeaders();175 176 for (std::vector<HTTPRequest::Header>::iterator177 i = headers.begin(); i != headers.end(); i++)178 {179 std::string& rHeaderName = i->first;180 181 for (std::string::iterator c = rHeaderName.begin();182 c != rHeaderName.end() && *c != ':'; c++)183 {184 *c = tolower(*c);185 }186 }187 188 sort(headers.begin(), headers.end());189 190 for (std::vector<HTTPRequest::Header>::iterator191 i = headers.begin(); i != headers.end(); i++)192 {193 if (i->first.substr(0, 5) == "x-amz")194 {195 data << i->first << ":" << i->second << "\n";196 }197 }198 199 if (! bucket.empty())200 {201 data << "/" << bucket;202 }203 204 data << rRequest.GetRequestURI();205 std::string data_string = data.str();206 207 unsigned char digest_buffer[EVP_MAX_MD_SIZE];208 unsigned int digest_size = sizeof(digest_buffer);209 /* unsigned char* mac = */ HMAC(EVP_sha1(),210 secret_key.c_str(), secret_key.size(),211 (const unsigned char*)data_string.c_str(),212 data_string.size(), digest_buffer, &digest_size);213 std::string digest((const char *)digest_buffer, digest_size);214 215 base64::encoder encoder;216 std::string expectedAuth = "AWS " + access_key + ":" +217 encoder.encode(digest);218 219 if (expectedAuth[expectedAuth.size() - 1] == '\n')220 {221 expectedAuth = expectedAuth.substr(0,222 expectedAuth.size() - 1);223 }224 225 std::string actualAuth;226 if (!rRequest.GetHeader("Authorization", &actualAuth) ||227 actualAuth != expectedAuth)228 {229 rResponse.SetResponseCode(HTTPResponse::Code_Unauthorized);230 }231 else if (rRequest.GetMethod() == HTTPRequest::Method_GET)232 {233 HandleGet(rRequest, rResponse);234 }235 else if (rRequest.GetMethod() == HTTPRequest::Method_PUT)236 {237 HandlePut(rRequest, rResponse);238 }239 else240 {241 rResponse.SetResponseCode(HTTPResponse::Code_MethodNotAllowed);242 }243 }244 catch (CommonException &ce)245 {246 rResponse.IOStream::Write(ce.what());247 }248 catch (std::exception &e)249 {250 rResponse.IOStream::Write(e.what());251 }252 catch (...)253 {254 rResponse.IOStream::Write("Unknown error");255 }256 257 return;258 }259 260 void S3Simulator::HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse)261 {262 std::string path = "testfiles";263 path += rRequest.GetRequestURI();264 std::auto_ptr<FileStream> apFile;265 266 try267 {268 apFile.reset(new FileStream(path));269 }270 catch (CommonException &ce)271 {272 if (ce.GetSubType() == CommonException::OSFileOpenError)273 {274 rResponse.SetResponseCode(HTTPResponse::Code_NotFound);275 }276 else if (ce.GetSubType() == CommonException::AccessDenied)277 {278 rResponse.SetResponseCode(HTTPResponse::Code_Forbidden);279 }280 throw;281 }282 283 // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingRESTOperations.html284 apFile->CopyStreamTo(rResponse);285 rResponse.AddHeader("x-amz-id-2", "qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY");286 rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D");287 rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT");288 rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT");289 rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\"");290 rResponse.AddHeader("Server", "AmazonS3");291 rResponse.SetResponseCode(HTTPResponse::Code_OK);292 }293 294 void S3Simulator::HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse)295 {296 std::string path = "testfiles";297 path += rRequest.GetRequestURI();298 std::auto_ptr<FileStream> apFile;299 300 try301 {302 apFile.reset(new FileStream(path, O_CREAT | O_WRONLY));303 }304 catch (CommonException &ce)305 {306 if (ce.GetSubType() == CommonException::OSFileOpenError)307 {308 rResponse.SetResponseCode(HTTPResponse::Code_NotFound);309 }310 else if (ce.GetSubType() == CommonException::AccessDenied)311 {312 rResponse.SetResponseCode(HTTPResponse::Code_Forbidden);313 }314 throw;315 }316 317 if (rRequest.IsExpectingContinue())318 {319 rResponse.SendContinue();320 }321 322 rRequest.ReadContent(*apFile);323 324 // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTObjectPUT.html325 rResponse.AddHeader("x-amz-id-2", "LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7");326 rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D");327 rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT");328 rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT");329 rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\"");330 rResponse.SetContentType("");331 rResponse.AddHeader("Server", "AmazonS3");332 rResponse.SetResponseCode(HTTPResponse::Code_OK);333 }334 335 129 int test(int argc, const char *argv[]) 336 130 { … … 447 241 TestRemoteProcessMemLeaks("generic-httpserver.memleaks"); 448 242 449 // correct, official signature should succeed 243 // correct, official signature should succeed, with lower-case header 450 244 { 451 245 // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAuthentication.html 452 246 HTTPRequest request(HTTPRequest::Method_GET, "/photos/puppy.jpg"); 453 247 request.SetHostName("johnsmith.s3.amazonaws.com"); 454 request.AddHeader(" Date", "Tue, 27 Mar 2007 19:36:42 +0000");455 request.AddHeader(" Authorization",248 request.AddHeader("date", "Tue, 27 Mar 2007 19:36:42 +0000"); 249 request.AddHeader("authorization", 456 250 "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA="); 457 251 458 252 S3Simulator simulator; 253 simulator.Configure("testfiles/s3simulator.conf"); 459 254 460 255 CollectInBufferStream response_buffer; … … 474 269 HTTPRequest request(HTTPRequest::Method_GET, "/photos/puppy.jpg"); 475 270 request.SetHostName("johnsmith.s3.amazonaws.com"); 476 request.AddHeader(" Date", "Tue, 27 Mar 2007 19:36:42 +0000");477 request.AddHeader(" Authorization",271 request.AddHeader("date", "Tue, 27 Mar 2007 19:36:42 +0000"); 272 request.AddHeader("authorization", 478 273 "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbB="); 479 274 480 275 S3Simulator simulator; 276 simulator.Configure("testfiles/s3simulator.conf"); 481 277 482 278 CollectInBufferStream response_buffer; … … 488 284 std::string response_data((const char *)response.GetBuffer(), 489 285 response.GetSize()); 490 TEST_EQUAL("", response_data); 286 TEST_EQUAL("<html><head>" 287 "<title>Internal Server Error</title></head>\n" 288 "<h1>Internal Server Error</h1>\n" 289 "<p>An error, type Authentication Failed occured " 290 "when processing the request.</p>" 291 "<p>Please try again later.</p></body>\n" 292 "</html>\n", response_data); 491 293 } 492 294 … … 494 296 { 495 297 S3Simulator simulator; 298 simulator.Configure("testfiles/s3simulator.conf"); 496 299 S3Client client(&simulator, "johnsmith.s3.amazonaws.com", 497 300 "0PN5J17HBGZHT7JJ3X82", … … 528 331 "/newfile"); 529 332 request.SetHostName("quotes.s3.amazonaws.com"); 530 request.AddHeader(" Date", "Wed, 01 Mar 2006 12:00:00 GMT");531 request.AddHeader(" Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:XtMYZf0hdOo4TdPYQknZk0Lz7rw=");333 request.AddHeader("date", "Wed, 01 Mar 2006 12:00:00 GMT"); 334 request.AddHeader("authorization", "AWS 0PN5J17HBGZHT7JJ3X82:XtMYZf0hdOo4TdPYQknZk0Lz7rw="); 532 335 request.AddHeader("Content-Type", "text/plain"); 533 336 … … 540 343 541 344 S3Simulator simulator; 345 simulator.Configure("testfiles/s3simulator.conf"); 542 346 simulator.Handle(request, response); 543 347 … … 559 363 560 364 // Start the S3Simulator server 561 pid = LaunchServer("./test s3server testfiles/ httpserver.conf",562 "testfiles/ httpserver.pid");365 pid = LaunchServer("./test s3server testfiles/s3simulator.conf", 366 "testfiles/s3simulator.pid"); 563 367 TEST_THAT(pid != -1 && pid != 0); 564 368 if(pid <= 0)
Note: See TracChangeset
for help on using the changeset viewer.
