Changeset 2428

Timestamp:

03/01/2009 08:59:47 (3 years ago)

Author:

chris

Message:

Add ability to send an HTTPRequest to a socket and to parse an
HTTPResponse from a socket, to create a simple HTTP client.

Location:

box/trunk

Files:

• lib/httpserver/HTTPException.txt (modified) (1 diff)
• lib/httpserver/HTTPQueryDecoder.cpp (modified) (1 diff)
• lib/httpserver/HTTPRequest.cpp (modified) (3 diffs)
• lib/httpserver/HTTPRequest.h (modified) (6 diffs)
• lib/httpserver/HTTPResponse.cpp (modified) (14 diffs)
• lib/httpserver/HTTPResponse.h (modified) (4 diffs)
• test/httpserver/testhttpserver.cpp (modified) (3 diffs)

box/trunk/lib/httpserver/HTTPException.txt

@@ -2 +2 @@
 
 Internal                                                        0
-RequestReadFailed                                       1
-RequestAlreadyBeenRead                          2
+RequestReadFailed                                               1
+RequestAlreadyBeenRead                                          2
 BadRequest                                                      3
-UnknownResponseCodeUsed                         4
-NoContentTypeSet                                        5
-POSTContentTooLong                                      6
-CannotSetRedirectIfReponseHasData       7
-CannotSetNotFoundIfReponseHasData       8
-NotImplemented                                          9
+UnknownResponseCodeUsed                                         4
+NoContentTypeSet                                                5
+POSTContentTooLong                                              6
+CannotSetRedirectIfReponseHasData                               7
+CannotSetNotFoundIfReponseHasData                               8
+NotImplemented                                                  9
+RequestNotInitialised                                           10
+BadResponse                                                     11
+ResponseReadFailed                                              12

box/trunk/lib/httpserver/HTTPQueryDecoder.cpp

@@ -20 +20 @@
 //
 // Function
-//              Name:    HTTPQueryDecoder::HTTPQueryDecoder(HTTPRequest::Query_t &)
-//              Purpose: Constructor. Pass in the query contents you want to decode
-//                               the query string into.
+//              Name:    HTTPQueryDecoder::HTTPQueryDecoder(
+//                       HTTPRequest::Query_t &)
+//              Purpose: Constructor. Pass in the query contents you want
+//                       to decode the query string into.
 //              Created: 26/3/04
 //

box/trunk/lib/httpserver/HTTPRequest.cpp

@@ -53 +53 @@
 //
 // Function
+//              Name:    HTTPRequest::HTTPRequest(enum Method,
+//                       const std::string&)
+//              Purpose: Alternate constructor for hand-crafted requests
+//              Created: 03/01/09
+//
+// --------------------------------------------------------------------------
+HTTPRequest::HTTPRequest(enum Method method, const std::string& rURI)
+        : mMethod(method),
+          mRequestURI(rURI),
+          mHostPort(80),        // default if not specified
+          mHTTPVersion(HTTPVersion_1_1),
+          mContentLength(-1),
+          mpCookies(0),
+          mClientKeepAliveRequested(false)
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
 //              Name:    HTTPRequest::~HTTPRequest()
 //              Purpose: Destructor
@@ -73 +95 @@
 // Function
 //              Name:    HTTPRequest::Read(IOStreamGetLine &, int)
-//              Purpose: Read the request from an IOStreamGetLine (and attached stream)
-//                               Returns false if there was no valid request, probably due to 
-//                               a kept-alive connection closing.
+//              Purpose: Read the request from an IOStreamGetLine (and
+//                       attached stream).
+//                       Returns false if there was no valid request,
+//                       probably due to a kept-alive connection closing.
 //              Created: 26/3/04
 //
@@ -284 +307 @@
         }
         
+        return true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    HTTPRequest::Write(IOStream &, int)
+//              Purpose: Write the request to an IOStream using HTTP.
+//              Created: 03/01/09
+//
+// --------------------------------------------------------------------------
+bool HTTPRequest::Write(IOStream &rStream, int Timeout)
+{
+        switch (mMethod)
+        {
+        case Method_UNINITIALISED:
+                THROW_EXCEPTION(HTTPException, RequestNotInitialised); break;
+        case Method_UNKNOWN:
+                THROW_EXCEPTION(HTTPException, BadRequest); break;
+        case Method_GET:
+                rStream.Write("GET"); break;
+        case Method_HEAD:
+                rStream.Write("HEAD"); break;
+        case Method_POST:
+                rStream.Write("POST"); break;
+        }
+
+        rStream.Write(" ");
+        rStream.Write(mRequestURI.c_str());
+        rStream.Write(" ");
+
+        switch (mHTTPVersion)
+        {
+        case HTTPVersion_0_9: rStream.Write("HTTP/0.9"); break;
+        case HTTPVersion_1_0: rStream.Write("HTTP/1.0"); break;
+        case HTTPVersion_1_1: rStream.Write("HTTP/1.1"); break;
+        default:
+                THROW_EXCEPTION(HTTPException, NotImplemented);
+        }
+
+        rStream.Write("\n");
+        std::ostringstream oss;
+
+        if (mContentLength != -1)
+        {
+                oss << "Content-Length: " << mContentLength << "\n";
+        }
+
+        if (mContentType != "")
+        {
+                oss << "Content-Type: " << mContentType << "\n";
+        }
+
+        if (mHostName != "")
+        {
+                if (mHostPort != 80)
+                {
+                        oss << "Host: " << mHostName << ":" << mHostPort <<
+                                "\n";
+                }
+                else
+                {
+                        oss << "Host: " << mHostName << "\n";
+                }
+        }
+
+        if (mpCookies)
+        {
+                THROW_EXCEPTION(HTTPException, NotImplemented);
+        }
+
+        if (mClientKeepAliveRequested)
+        {
+                oss << "Connection: keep-alive\n";
+        }
+        else
+        {
+                oss << "Connection: close\n";
+        }
+
+        rStream.Write(oss.str().c_str());
+        rStream.Write("\n");
+
         return true;
 }

box/trunk/lib/httpserver/HTTPRequest.h

@@ -28 +28 @@
 {
 public:
+        enum Method
+        {
+                Method_UNINITIALISED = -1,
+                Method_UNKNOWN = 0,
+                Method_GET = 1,
+                Method_HEAD = 2,
+                Method_POST = 3
+        };
+        
         HTTPRequest();
+        HTTPRequest(enum Method method, const std::string& rURI);
         ~HTTPRequest();
 private:
@@ -41 +51 @@
         enum
         {
-                Method_UNINITIALISED = -1,
-                Method_UNKNOWN = 0,
-                Method_GET = 1,
-                Method_HEAD = 2,
-                Method_POST = 3
-        };
-        
-        enum
-        {
                 HTTPVersion__MajorMultiplier = 1000,
                 HTTPVersion_0_9 = 9,
@@ -57 +58 @@
 
         bool Read(IOStreamGetLine &rGetLine, int Timeout);
+        bool Write(IOStream &rStream, int Timeout);
 
         typedef std::map<std::string, std::string> CookieJar_t;
@@ -68 +70 @@
         //
         // --------------------------------------------------------------------------
-        int GetMethod() const {return mMethod;}
+        enum Method GetMethod() const {return mMethod;}
         const std::string &GetRequestURI() const {return mRequestURI;}
         const std::string &GetHostName() const {return mHostName;}      // note: request does splitting of Host: header
@@ -92 +94 @@
         // --------------------------------------------------------------------------
         bool GetClientKeepAliveRequested() const {return mClientKeepAliveRequested;}
+        void SetClientKeepAliveRequested(bool keepAlive)
+        {
+                mClientKeepAliveRequested = keepAlive;
+        }
 
 private:
@@ -98 +104 @@
 
 private:
-        int mMethod;
+        enum Method mMethod;
         std::string mRequestURI;
         std::string mHostName;

box/trunk/lib/httpserver/HTTPResponse.cpp

@@ -14 +14 @@
 
 #include "HTTPResponse.h"
+#include "IOStreamGetLine.h"
 #include "autogen_HTTPException.h"
 
@@ -33 +34 @@
         : mResponseCode(HTTPResponse::Code_NoContent),
           mResponseIsDynamicContent(true),
-          mKeepAlive(false)
+          mKeepAlive(false),
+          mContentLength(-1)
 {
 }
@@ -55 +57 @@
 // Function
 //              Name:    HTTPResponse::ResponseCodeToString(int)
-//              Purpose: Return string equivalent of the response code, suitable for Status: headers
+//              Purpose: Return string equivalent of the response code,
+//                       suitable for Status: headers
 //              Created: 26/3/04
 //
@@ -115 +118 @@
 // Function
 //              Name:    HTTPResponse::Send(IOStream &, bool)
-//              Purpose: Build the response, and send via the stream. Optionally omitting
-//                               the content.
+//              Purpose: Build the response, and send via the stream.
+//                       Optionally omitting the content.
 //              Created: 26/3/04
 //
@@ -140 +143 @@
                 }
                 // Extra headers...
-                for(std::vector<std::string>::const_iterator i(mExtraHeaders.begin()); i != mExtraHeaders.end(); ++i)
+                for(std::vector<std::pair<std::string, std::string> >::const_iterator i(mExtraHeaders.begin()); i != mExtraHeaders.end(); ++i)
                 {
                         header += "\r\n";
-                        header += *i;
+                        header += i->first + ": " + i->second;
                 }
                 // NOTE: a line ending must be included here in all cases
@@ -182 +185 @@
 //
 // Function
+//              Name:    HTTPResponse::ParseHeaders(IOStreamGetLine &, int)
+//              Purpose: Private. Parse the headers of the response
+//              Created: 26/3/04
+//
+// --------------------------------------------------------------------------
+void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout)
+{
+        std::string header;
+        bool haveHeader = false;
+        while(true)
+        {
+                if(rGetLine.IsEOF())
+                {
+                        // Header terminates unexpectedly
+                        THROW_EXCEPTION(HTTPException, BadRequest)              
+                }
+
+                std::string currentLine;        
+                if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout))
+                {
+                        // Timeout
+                        THROW_EXCEPTION(HTTPException, RequestReadFailed)
+                }
+                
+                // Is this a continuation of the previous line?
+                bool processHeader = haveHeader;
+                if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t'))
+                {
+                        // A continuation, don't process anything yet
+                        processHeader = false;
+                }
+                //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str());
+                
+                // Parse the header -- this will actually process the header
+                // from the previous run around the loop.
+                if(processHeader)
+                {
+                        // Find where the : is in the line
+                        const char *h = header.c_str();
+                        int p = 0;
+                        while(h[p] != '\0' && h[p] != ':')
+                        {
+                                ++p;
+                        }
+                        // Skip white space
+                        int dataStart = p + 1;
+                        while(h[dataStart] == ' ' || h[dataStart] == '\t')
+                        {
+                                ++dataStart;
+                        }
+                
+                        if(p == sizeof("Content-Length")-1
+                                && ::strncasecmp(h, "Content-Length", sizeof("Content-Length")-1) == 0)
+                        {
+                                // Decode number
+                                long len = ::strtol(h + dataStart, NULL, 10);   // returns zero in error case, this is OK
+                                if(len < 0) len = 0;
+                                // Store
+                                mContentLength = len;
+                        }
+                        else if(p == sizeof("Content-Type")-1
+                                && ::strncasecmp(h, "Content-Type", sizeof("Content-Type")-1) == 0)
+                        {
+                                // Store rest of string as content type
+                                mContentType = h + dataStart;
+                        }
+                        else if(p == sizeof("Cookie")-1
+                                && ::strncasecmp(h, "Cookie", sizeof("Cookie")-1) == 0)
+                        {
+                                THROW_EXCEPTION(HTTPException, NotImplemented);
+                                /*
+                                // Parse cookies
+                                ParseCookies(header, dataStart);
+                                */
+                        }
+                        else if(p == sizeof("Connection")-1
+                                && ::strncasecmp(h, "Connection", sizeof("Connection")-1) == 0)
+                        {
+                                // Connection header, what is required?
+                                const char *v = h + dataStart;
+                                if(::strcasecmp(v, "close") == 0)
+                                {
+                                        mKeepAlive = false;
+                                }
+                                else if(::strcasecmp(v, "keep-alive") == 0)
+                                {
+                                        mKeepAlive = true;
+                                }
+                                // else don't understand, just assume default for protocol version
+                        }
+                        else
+                        {
+                                std::string headerName = header.substr(0, p);
+                                AddHeader(headerName, h + dataStart);
+                        }
+                        
+                        // Unset have header flag, as it's now been processed
+                        haveHeader = false;
+                }
+
+                // Store the chunk of header the for next time round
+                if(haveHeader)
+                {
+                        header += currentLine;
+                }
+                else
+                {
+                        header = currentLine;
+                        haveHeader = true;
+                }
+
+                // End of headers?
+                if(currentLine.empty())
+                {
+                        // All done!
+                        break;
+                }               
+        }
+}
+
+void HTTPResponse::Receive(IOStream& rStream, int Timeout)
+{
+        IOStreamGetLine rGetLine(rStream);
+
+        if(rGetLine.IsEOF())
+        {
+                // Connection terminated unexpectedly
+                THROW_EXCEPTION(HTTPException, BadResponse)             
+        }
+
+        std::string statusLine; 
+        if(!rGetLine.GetLine(statusLine, false /* no preprocess */, Timeout))
+        {
+                // Timeout
+                THROW_EXCEPTION(HTTPException, ResponseReadFailed)
+        }
+
+        if (statusLine.substr(0, 7) != "HTTP/1." ||
+                statusLine[8] != ' ')
+        {
+                // Status line terminated unexpectedly
+                BOX_ERROR("Bad response status line: " << statusLine);
+                THROW_EXCEPTION(HTTPException, BadResponse)             
+        }
+
+        if (statusLine[5] == '1' && statusLine[7] == '1')
+        {
+                // HTTP/1.1 default is to keep alive
+                mKeepAlive = true;
+        }
+                
+        // Decode the status code
+        long status = ::strtol(statusLine.substr(9, 3).c_str(), NULL, 10);
+        // returns zero in error case, this is OK
+        if(status < 0) status = 0;
+        // Store
+        mResponseCode = status;
+
+        ParseHeaders(rGetLine, Timeout);
+
+        // push back whatever bytes we have left
+        // rGetLine.DetachFile();
+        if (mContentLength > 0)
+        {
+                if (mContentLength < rGetLine.GetSizeOfBufferedData())
+                {
+                        // very small response, not good!
+                        THROW_EXCEPTION(HTTPException, NotImplemented);
+                }
+
+                Write(rGetLine.GetBufferedData(),
+                        rGetLine.GetSizeOfBufferedData());
+        }
+
+        while (mContentLength != 0) // could be -1 as well
+        {
+                char buffer[4096];
+                int readSize = sizeof(buffer);
+                if (mContentLength > 0 && mContentLength < readSize)
+                {
+                        readSize = mContentLength;
+                }
+                readSize = rStream.Read(buffer, readSize, Timeout);
+                if (readSize == 0)
+                {
+                        break;
+                }
+                mContentLength -= readSize;
+                Write(buffer, readSize);
+        }
+
+        SetForReading();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
 //              Name:    HTTPResponse::AddHeader(const char *)
 //              Purpose: Add header, given entire line
@@ -187 +387 @@
 //
 // --------------------------------------------------------------------------
+/*
 void HTTPResponse::AddHeader(const char *EntireHeaderLine)
 {
         mExtraHeaders.push_back(std::string(EntireHeaderLine));
 }
-
+*/
 
 // --------------------------------------------------------------------------
@@ -201 +402 @@
 //
 // --------------------------------------------------------------------------
+/*
 void HTTPResponse::AddHeader(const std::string &rEntireHeaderLine)
 {
         mExtraHeaders.push_back(rEntireHeaderLine);
 }
-
+*/
 
 // --------------------------------------------------------------------------
@@ -215 +417 @@
 //
 // --------------------------------------------------------------------------
-void HTTPResponse::AddHeader(const char *Header, const char *Value)
-{
-        std::string h(Header);
-        h += ": ";
-        h += Value;
-        mExtraHeaders.push_back(h);
+void HTTPResponse::AddHeader(const char *pHeader, const char *pValue)
+{
+        mExtraHeaders.push_back(Header(pHeader, pValue));
 }
 
@@ -232 +431 @@
 //
 // --------------------------------------------------------------------------
-void HTTPResponse::AddHeader(const char *Header, const std::string &rValue)
-{
-        std::string h(Header);
-        h += ": ";
-        h += rValue;
-        mExtraHeaders.push_back(h);
+void HTTPResponse::AddHeader(const char *pHeader, const std::string &rValue)
+{
+        mExtraHeaders.push_back(Header(pHeader, rValue));
 }
 
@@ -251 +447 @@
 void HTTPResponse::AddHeader(const std::string &rHeader, const std::string &rValue)
 {
-        mExtraHeaders.push_back(rHeader + ": " + rValue);
+        mExtraHeaders.push_back(Header(rHeader, rValue));
 }
 
@@ -280 +476 @@
         h += "\"";
 */
-        std::string h("Set-Cookie: ");
+        std::string h;
         h += Name;
         h += "=";
@@ -287 +483 @@
         h += Path;
 
-        mExtraHeaders.push_back(h);
+        mExtraHeaders.push_back(Header("Set-Cookie", h));
 }
 
@@ -313 +509 @@
 
         // Set location to redirect to
-        std::string header("Location: ");
+        std::string header;
         if(IsLocalURI) header += msDefaultURIPrefix;
         header += RedirectTo;
-        mExtraHeaders.push_back(header);
+        mExtraHeaders.push_back(Header("Location", header));
         
         // Set up some default content

box/trunk/lib/httpserver/HTTPResponse.h

@@ -15 +15 @@
 
 #include "CollectInBufferStream.h"
+
+class IOStreamGetLine;
 
 // --------------------------------------------------------------------------
@@ -36 +38 @@
 
         void SetResponseCode(int Code);
+        int GetResponseCode() { return mResponseCode; }
         void SetContentType(const char *ContentType);
+        const std::string& GetContentType() { return mContentType; }
 
         void SetAsRedirect(const char *RedirectTo, bool IsLocalURI = true);
@@ -42 +46 @@
 
         void Send(IOStream &rStream, bool OmitContent = false);
+        void Receive(IOStream& rStream, int Timeout = IOStream::TimeOutInfinite);
 
-        void AddHeader(const char *EntireHeaderLine);
-        void AddHeader(const std::string &rEntireHeaderLine);
+        // void AddHeader(const char *EntireHeaderLine);
+        // void AddHeader(const std::string &rEntireHeaderLine);
         void AddHeader(const char *Header, const char *Value);
         void AddHeader(const char *Header, const std::string &rValue);
@@ -107 +112 @@
         bool mKeepAlive;
         std::string mContentType;
-        std::vector<std::string> mExtraHeaders;
+        typedef std::pair<std::string, std::string> Header;
+        std::vector<Header> mExtraHeaders;
+        int mContentLength; // only used when reading response from stream
         
         static std::string msDefaultURIPrefix;
+
+        void ParseHeaders(IOStreamGetLine &rGetLine, int Timeout);
 };
 

box/trunk/test/httpserver/testhttpserver.cpp

@@ -17 +17 @@
 #include "HTTPRequest.h"
 #include "HTTPResponse.h"
+#include "IOStreamGetLine.h"
 #include "ServerControl.h"
 
@@ -72 +73 @@
                 case HTTPRequest::Method_HEAD: m = "HEAD"; break;
                 case HTTPRequest::Method_POST: m = "POST"; break;
+                default: m = "UNKNOWN";
                 }
                 rResponse.Write(m, 4);
@@ -126 +128 @@
         int pid = LaunchServer("./test server testfiles/httpserver.conf", "testfiles/httpserver.pid");
         TEST_THAT(pid != -1 && pid != 0);
-        if(pid > 0)
-        {
-                // Run the request script
-                TEST_THAT(::system("perl testfiles/testrequests.pl") == 0);
+        if(pid <= 0)
+        {
+                return 0;
+        }
+
+        // Run the request script
+        TEST_THAT(::system("perl testfiles/testrequests.pl") == 0);
+
+        signal(SIGPIPE, SIG_IGN);
+
+        SocketStream sock;
+        sock.Open(Socket::TypeINET, "localhost", 1080);
+
+        for (int i = 0; i < 4; i++)
+        {
+                HTTPRequest request(HTTPRequest::Method_GET,
+                        "/test-one/34/341s/234?p1=vOne&p2=vTwo");
+
+                if (i >= 2)
+                {
+                        // first set of passes has keepalive off by default,
+                        // so when i == 1 the socket has already been closed
+                        // by the server, and we'll get -EPIPE when we try
+                        // to send the request.
+                        request.SetClientKeepAliveRequested(true);
+                }
+
+                if (i == 1)
+                {
+                        TEST_CHECK_THROWS(request.Write(sock,
+                                IOStream::TimeOutInfinite),
+                                ConnectionException, SocketWriteError);
+                        sock.Close();
+                        sock.Open(Socket::TypeINET, "localhost", 1080);
+                        continue;
+                }
+                else
+                {
+                        request.Write(sock, IOStream::TimeOutInfinite);
+                }
+
+                HTTPResponse response;
+                response.Receive(sock);
+                
+                TEST_THAT(response.GetResponseCode() == HTTPResponse::Code_OK);
+                TEST_THAT(response.GetContentType() == "text/html");
+
+                IOStreamGetLine getline(response);
+                std::string line;
+
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("<html>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("<head><title>TEST SERVER RESPONSE</title></head>",
+                        line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("<body><h1>Test response</h1>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("<p><b>URI:</b> /test-one/34/341s/234</p>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("<p><b>Query string:</b> p1=vOne&p2=vTwo</p>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("<p><b>Method:</b> GET </p>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("<p><b>Decoded query:</b><br>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("PARAM:p1=vOne<br>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("PARAM:p2=vTwo<br></p>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("<p><b>Content type:</b> </p>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("<p><b>Content length:</b> -1</p>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("<p><b>Cookies:</b><br>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("</p>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("</body>", line);
+                TEST_THAT(getline.GetLine(line));
+                TEST_EQUAL("</html>", line);
+        }
         
-                // Kill it
-                TEST_THAT(KillServer(pid));
-                TestRemoteProcessMemLeaks("generic-httpserver.memleaks");
-        }
+        // Kill it
+        TEST_THAT(KillServer(pid));
+        TestRemoteProcessMemLeaks("generic-httpserver.memleaks");
 
         return 0;