Changeset 353

Timestamp:

30/01/2006 20:04:53 (6 years ago)

Author:

ben

Message:

Merge chris/bb-save-state, resolving conflicts

Location:

box/trunk

Files:

• BUGS.txt (modified) (1 diff)
• bin/bbackupd/BackupClientDirectoryRecord.cpp (modified) (2 diffs)
• bin/bbackupd/BackupClientDirectoryRecord.h (modified) (2 diffs)
• bin/bbackupd/BackupDaemon.cpp (modified) (7 diffs)
• bin/bbackupd/BackupDaemon.h (modified) (4 diffs)
• bin/bbackupd/bbackupd-config (modified) (1 diff)
• configure.ac (modified) (1 diff)
• lib/backupclient/BackupDaemonConfigVerify.cpp (modified) (2 diffs)
• lib/common/Archive.h (copied) (copied from box/chris/bb-save-state/lib/common/Archive.h)
• lib/common/CommonException.txt (modified) (1 diff)
• lib/common/ExcludeList.cpp (modified) (3 diffs)
• lib/common/ExcludeList.h (modified) (3 diffs)
• lib/server/Daemon.cpp (modified) (13 diffs)
• lib/server/Daemon.h (modified) (3 diffs)

box/trunk/BUGS.txt

@@ -9 +9 @@
 * bbackupquery restore, if not root, then won't do file ownership properly, but won't alert the user to this fact
 * empty (real) directories in the store aren't deleted when they're empty (and will never be used again) -- uses up disc space unnecessarily
+* need unit tests for SSL keepalives and state saving (serialisation)
+* make Archive derive from Protocol
+* more automated tests for win32
+* change off_t to box_off_t in preparation for win32 large file support
+* support large files on win32 by using native *i64 functions instead of posix

box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp

@@ -26 +26 @@
 #include "BackupDaemon.h"
 #include "BackupStoreException.h"
+#include "Archive.h"
 
 #include "MemLeakFindOn.h"
@@ -1227 +1228 @@
 {
 }
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    BackupClientDirectoryRecord::Deserialize(Archive & rArchive)
+//              Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction.
+//
+//              Created: 2005/04/11
+//
+// --------------------------------------------------------------------------
+void BackupClientDirectoryRecord::Deserialize(Archive & rArchive)
+{
+        // Make deletion recursive
+        DeleteSubDirectories();
+
+        // Delete maps
+        if(mpPendingEntries != 0)
+        {
+                delete mpPendingEntries;
+                mpPendingEntries = 0;
+        }
+
+        //
+        //
+        //
+        rArchive.Read(mObjectID);
+        rArchive.Read(mSubDirName);
+        rArchive.Read(mInitialSyncDone);
+        rArchive.Read(mSyncDone);
+
+        //
+        //
+        //
+        int64_t iCount = 0;
+        rArchive.Read(iCount);
+
+        if (iCount != sizeof(mStateChecksum)/sizeof(mStateChecksum[0]))
+        {
+                // we have some kind of internal system representation change: throw for now
+                THROW_EXCEPTION(CommonException, Internal)
+        }
+
+        for (int v = 0; v < iCount; v++)
+        {
+                // Load each checksum entry
+                rArchive.Read(mStateChecksum[v]);
+        }
+
+        //
+        //
+        //
+        iCount = 0;
+        rArchive.Read(iCount);
+
+        if (iCount > 0)
+        {
+                // load each pending entry
+                mpPendingEntries = new std::map<std::string, box_time_t>;
+                if (!mpPendingEntries)
+                {
+                        throw std::bad_alloc();
+                }
+
+                for (int v = 0; v < iCount; v++)
+                {
+                        std::string strItem;
+                        box_time_t btItem;
+
+                        rArchive.Read(strItem);
+                        rArchive.Read(btItem);
+                        (*mpPendingEntries)[strItem] = btItem;
+                }
+        }
+
+        //
+        //
+        //
+        iCount = 0;
+        rArchive.Read(iCount);
+
+        if (iCount > 0)
+        {
+                for (int v = 0; v < iCount; v++)
+                {
+                        std::string strItem;
+                        rArchive.Read(strItem);
+
+                        BackupClientDirectoryRecord* pSubDirRecord = 
+                                new BackupClientDirectoryRecord(0, ""); 
+                        // will be deserialized anyway, give it id 0 for now
+
+                        if (!pSubDirRecord)
+                        {
+                                throw std::bad_alloc();
+                        }
+
+                        /***** RECURSE *****/
+                        pSubDirRecord->Deserialize(rArchive);
+                        mSubDirectories[strItem] = pSubDirRecord;
+                }
+        }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    BackupClientDirectoryRecord::Serialize(Archive & rArchive)
+//              Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction.
+//
+//              Created: 2005/04/11
+//
+// --------------------------------------------------------------------------
+void BackupClientDirectoryRecord::Serialize(Archive & rArchive) const
+{
+        //
+        //
+        //
+        rArchive.Write(mObjectID);
+        rArchive.Write(mSubDirName);
+        rArchive.Write(mInitialSyncDone);
+        rArchive.Write(mSyncDone);
+
+        //
+        //
+        //
+        int64_t iCount = 0;
+
+        // when reading back the archive, we will 
+        // need to know how many items there are.
+        iCount = sizeof(mStateChecksum) / sizeof(mStateChecksum[0]);
+        rArchive.Write(iCount); 
+
+        for (int v = 0; v < iCount; v++)
+        {
+                rArchive.Write(mStateChecksum[v]);
+        }
+
+        //
+        //
+        //
+        if (!mpPendingEntries)
+        {
+                iCount = 0;
+                rArchive.Write(iCount);
+        }
+        else
+        {
+                iCount = mpPendingEntries->size();
+                rArchive.Write(iCount);
+
+                for (std::map<std::string, box_time_t>::const_iterator
+                        i =  mpPendingEntries->begin(); 
+                        i != mpPendingEntries->end(); i++)
+                {
+                        rArchive.Write(i->first);
+                        rArchive.Write(i->second);
+                }
+        }
+        //
+        //
+        //
+        iCount = mSubDirectories.size();
+        rArchive.Write(iCount);
+
+        for (std::map<std::string, BackupClientDirectoryRecord*>::const_iterator
+                i =  mSubDirectories.begin(); 
+                i != mSubDirectories.end(); i++)
+        {
+                const BackupClientDirectoryRecord* pSubItem = i->second;
+                ASSERT(pSubItem);
+
+                rArchive.Write(i->first);
+                pSubItem->Serialize(rArchive);
+        }
+}

box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h

@@ -19 +19 @@
 #include "MD5Digest.h"
 
+class Archive;
 class BackupClientContext;
 class BackupDaemon;
@@ -35 +36 @@
         BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName);
         ~BackupClientDirectoryRecord();
+
+        void Deserialize(Archive & rArchive);
+        void Serialize(Archive & rArchive) const;
 private:
         BackupClientDirectoryRecord(const BackupClientDirectoryRecord &);

box/trunk/bin/bbackupd/BackupDaemon.cpp

@@ -11 +11 @@
 
 #include <stdio.h>
+#include <string.h>
 #include <unistd.h>
 
-#ifndef WIN32
+#ifdef HAVE_SIGNAL_H
         #include <signal.h>
+#endif
+#ifdef HAVE_SYSLOG_H
         #include <syslog.h>
+#endif
+#ifdef HAVE_SYS_PARAM_H
         #include <sys/param.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
         #include <sys/wait.h>
 #endif
@@ -62 +69 @@
 #include "IOStreamGetLine.h"
 #include "Conversion.h"
+#include "Archive.h"
 
 #include "MemLeakFindOn.h"
@@ -468 +476 @@
         box_time_t lastSyncTime = 0;
 
+        // --------------------------------------------------------------------------------------------
+ 
+        // And what's the current client store marker?
+        int64_t clientStoreMarker = 
+                BackupClientContext::ClientStoreMarker_NotKnown;
+        // haven't contacted the store yet
+
+        DeserializeStoreObjectInfo(clientStoreMarker, lastSyncTime, 
+                nextSyncTime);
+ 
         // --------------------------------------------------------------------------------------------
         
-        // And what's the current client store marker?
-        int64_t clientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;            // haven't contacted the store yet
 
         // Set state
@@ -675 +691 @@
                                 // Log
                                 ::syslog(LOG_INFO, "Finished scan of local files");
+
+                                // --------------------------------------------------------------------------------------------
+
+                                // We had a successful backup, save the store info
+                                SerializeStoreObjectInfo(clientStoreMarker, lastSyncTime, nextSyncTime);
+
+                                // --------------------------------------------------------------------------------------------
                         }
                         catch(BoxException &e)
@@ -1832 +1855 @@
 }
 
+// --------------------------------------------------------------------------
+
+typedef struct
+{
+        int32_t mMagicValue;    // also the version number
+        int32_t mNumEntries;
+        int64_t mObjectID;              // this object ID
+        int64_t mContainerID;   // ID of container
+        uint64_t mAttributesModTime;
+        int32_t mOptionsPresent;        // bit mask of optional sections / features present
+
+} loc_StreamFormat;
 
 // --------------------------------------------------------------------------
@@ -1871 +1906 @@
 }
 
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    BackupDaemon::Location::Deserialize(Archive & rArchive)
+//              Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction.
+//
+//              Created: 2005/04/11
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::Location::Deserialize(Archive &rArchive)
+{
+        //
+        //
+        //
+        mpDirectoryRecord.reset(NULL);
+        if (mpExcludeFiles)
+        {
+                delete mpExcludeFiles;
+                mpExcludeFiles = NULL;
+        }
+        if (mpExcludeDirs)
+        {
+                delete mpExcludeDirs;
+                mpExcludeDirs = NULL;
+        }
+
+        //
+        //
+        //
+        rArchive.Read(mName);
+        rArchive.Read(mPath);
+        rArchive.Read(mIDMapIndex);
+
+        //
+        //
+        //
+        int64_t aMagicMarker = 0;
+        rArchive.Read(aMagicMarker);
+
+        if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP)
+        {
+                // NOOP
+        }
+        else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE)
+        {
+                BackupClientDirectoryRecord *pSubRecord = new BackupClientDirectoryRecord(0, "");
+                if (!pSubRecord)
+                        throw std::bad_alloc();
+
+                mpDirectoryRecord.reset(pSubRecord);
+                mpDirectoryRecord->Deserialize(rArchive);
+        }
+        else
+        {
+                // there is something going on here
+                THROW_EXCEPTION(CommonException, Internal)
+        }
+
+        //
+        //
+        //
+        rArchive.Read(aMagicMarker);
+
+        if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP)
+        {
+                // NOOP
+        }
+        else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE)
+        {
+                mpExcludeFiles = new ExcludeList;
+                if (!mpExcludeFiles)
+                        throw std::bad_alloc();
+
+                mpExcludeFiles->Deserialize(rArchive);
+        }
+        else
+        {
+                // there is something going on here
+                THROW_EXCEPTION(CommonException, Internal)
+        }
+
+        //
+        //
+        //
+        rArchive.Read(aMagicMarker);
+
+        if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP)
+        {
+                // NOOP
+        }
+        else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE)
+        {
+                mpExcludeDirs = new ExcludeList;
+                if (!mpExcludeDirs)
+                        throw std::bad_alloc();
+
+                mpExcludeDirs->Deserialize(rArchive);
+        }
+        else
+        {
+                // there is something going on here
+                THROW_EXCEPTION(CommonException, Internal)
+        }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    BackupDaemon::Location::Serialize(Archive & rArchive)
+//              Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction.
+//
+//              Created: 2005/04/11
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::Location::Serialize(Archive & rArchive) const
+{
+        //
+        //
+        //
+        rArchive.Write(mName);
+        rArchive.Write(mPath);
+        rArchive.Write(mIDMapIndex);
+
+        //
+        //
+        //
+        if (mpDirectoryRecord.get() == NULL)
+        {
+                int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP;
+                rArchive.Write(aMagicMarker);
+        }
+        else
+        {
+                int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows
+                rArchive.Write(aMagicMarker);
+
+                mpDirectoryRecord->Serialize(rArchive);
+        }
+
+        //
+        //
+        //
+        if (!mpExcludeFiles)
+        {
+                int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP;
+                rArchive.Write(aMagicMarker);
+        }
+        else
+        {
+                int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows
+                rArchive.Write(aMagicMarker);
+
+                mpExcludeFiles->Serialize(rArchive);
+        }
+
+        //
+        //
+        //
+        if (!mpExcludeDirs)
+        {
+                int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP;
+                rArchive.Write(aMagicMarker);
+        }
+        else
+        {
+                int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows
+                rArchive.Write(aMagicMarker);
+
+                mpExcludeDirs->Serialize(rArchive);
+        }
+}
 
 // --------------------------------------------------------------------------
@@ -1902 +2108 @@
         }
 }
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime)
+//              Purpose: Serializes remote directory and file information into a stream of bytes, using an Archive abstraction.
+//
+//              Created: 2005/04/11
+//
+// --------------------------------------------------------------------------
+
+static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F;
+static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE";
+static const int STOREOBJECTINFO_VERSION = 1;
+
+void BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const
+{
+        if(!GetConfiguration().KeyExists("StoreObjectInfoFile"))
+        {
+                return;
+        }
+
+        std::string StoreObjectInfoFile = 
+                GetConfiguration().GetKeyValue("StoreObjectInfoFile");
+
+        if (StoreObjectInfoFile.size() <= 0)
+        {
+                return;
+        }
+
+        try
+        {
+                FileStream aFile(StoreObjectInfoFile.c_str(), 
+                        O_WRONLY | O_CREAT | O_TRUNC);
+                Archive anArchive(aFile, 0);
+
+                anArchive.Write(STOREOBJECTINFO_MAGIC_ID_VALUE);
+                anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING); 
+                anArchive.Write(STOREOBJECTINFO_VERSION);
+                anArchive.Write(GetLoadedConfigModifiedTime());
+                anArchive.Write(aClientStoreMarker);
+                anArchive.Write(theLastSyncTime);
+                anArchive.Write(theNextSyncTime);
+
+                //
+                //
+                //
+                int64_t iCount = mLocations.size();
+                anArchive.Write(iCount);
+
+                for (int v = 0; v < iCount; v++)
+                {
+                        ASSERT(mLocations[v]);
+                        mLocations[v]->Serialize(anArchive);
+                }
+
+                //
+                //
+                //
+                iCount = mIDMapMounts.size();
+                anArchive.Write(iCount);
+
+                for (int v = 0; v < iCount; v++)
+                        anArchive.Write(mIDMapMounts[v]);
+
+                //
+                //
+                //
+                aFile.Close();
+                ::syslog(LOG_INFO, "Saved store object info file '%s'", 
+                        StoreObjectInfoFile.c_str());
+        }
+        catch (...)
+        {
+                ::syslog(LOG_WARNING, "Requested store object info file '%s' "
+                        "not accessible or could not be created", 
+                        StoreObjectInfoFile.c_str());
+        }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime)
+//              Purpose: Deserializes remote directory and file information from a stream of bytes, using an Archive abstraction.
+//
+//              Created: 2005/04/11
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime)
+{
+        //
+        //
+        //
+        DeleteAllLocations();
+
+        //
+        //
+        //
+        if(!GetConfiguration().KeyExists("StoreObjectInfoFile"))
+        {
+                return;
+        }
+
+        std::string StoreObjectInfoFile = 
+                GetConfiguration().GetKeyValue("StoreObjectInfoFile");
+
+        if (StoreObjectInfoFile.size() <= 0)
+        {
+                return;
+        }
+
+        try
+        {
+                FileStream aFile(StoreObjectInfoFile.c_str(), O_RDONLY);
+                Archive anArchive(aFile, 0);
+
+                //
+                // see if the content looks like a valid serialised archive
+                //
+                int iMagicValue = 0;
+                anArchive.Read(iMagicValue);
+
+                if (iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE)
+                {
+                        ::syslog(LOG_WARNING, "Store object info file '%s' "
+                                "is not a valid or compatible serialised "
+                                "archive. Will re-cache from store.", 
+                                StoreObjectInfoFile.c_str());
+                        return;
+                }
+
+                //
+                // get a bit optimistic and read in a string identifier
+                //
+                std::string strMagicValue;
+                anArchive.Read(strMagicValue);
+
+                if (strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING)
+                {
+                        ::syslog(LOG_WARNING, "Store object info file '%s' "
+                                "is not a valid or compatible serialised "
+                                "archive. Will re-cache from store.", 
+                                StoreObjectInfoFile.c_str());
+                        return;
+                }
+
+                //
+                // check if we are loading some future format
+                // version by mistake
+                //
+                int iVersion = 0;
+                anArchive.Read(iVersion);
+
+                if (iVersion != STOREOBJECTINFO_VERSION)
+                {
+                        ::syslog(LOG_WARNING, "Store object info file '%s' "
+                                "version [%d] unsupported. "
+                                "Will re-cache from store.", 
+                                StoreObjectInfoFile.c_str(), 
+                                iVersion);
+                        return;
+                }
+
+                //
+                // check if this state file is even valid 
+                // for the loaded bbackupd.conf file
+                //
+                box_time_t lastKnownConfigModTime;
+                anArchive.Read(lastKnownConfigModTime);
+
+                if (lastKnownConfigModTime != GetLoadedConfigModifiedTime())
+                {
+                        ::syslog(LOG_WARNING, "Store object info file '%s' "
+                                "out of date. Will re-cache from store", 
+                                StoreObjectInfoFile.c_str());
+                        return;
+                }
+
+                //
+                // this is it, go at it
+                //
+                anArchive.Read(aClientStoreMarker);
+                anArchive.Read(theLastSyncTime);
+                anArchive.Read(theNextSyncTime);
+
+                //
+                //
+                //
+                int64_t iCount = 0;
+                anArchive.Read(iCount);
+
+                for (int v = 0; v < iCount; v++)
+                {
+                        Location* pLocation = new Location;
+                        if (!pLocation)
+                                throw std::bad_alloc();
+
+                        pLocation->Deserialize(anArchive);
+                        mLocations.push_back(pLocation);
+                }
+
+                //
+                //
+                //
+                iCount = 0;
+                anArchive.Read(iCount);
+
+                for (int v = 0; v < iCount; v++)
+                {
+                        std::string strItem;
+                        anArchive.Read(strItem);
+
+                        mIDMapMounts.push_back(strItem);
+                }
+
+                //
+                //
+                //
+                aFile.Close();
+                ::syslog(LOG_INFO, "Loaded store object info file '%s', "
+                        "version [%d]", StoreObjectInfoFile.c_str(), 
+                        iVersion);
+
+                if (::unlink(StoreObjectInfoFile.c_str()) != 0)
+                {
+                        ::syslog(LOG_ERR, "Failed to delete the old "
+                                "store object info file '%s': %s",
+                                StoreObjectInfoFile.c_str(), strerror(errno));
+                }
+        }
+        catch (...)
+        {
+                DeleteAllLocations();
+
+                aClientStoreMarker = 
+                        BackupClientContext::ClientStoreMarker_NotKnown;
+                theLastSyncTime = 0;
+                theNextSyncTime = 0;
+
+                ::syslog(LOG_WARNING, "Requested store object info file '%s' "
+                        "does not exist, not accessible, or inconsistent. "
+                        "Will re-cache from store.", 
+                        StoreObjectInfoFile.c_str());
+        }
+}

box/trunk/bin/bbackupd/BackupDaemon.h

@@ -15 +15 @@
 #include <memory>
 
+#include "BoxTime.h"
 #include "Daemon.h"
-#include "BoxTime.h"
 #include "Socket.h"
 #include "SocketListen.h"
@@ -28 +28 @@
 class ExcludeList;
 class IOStreamGetLine;
+class Archive;
 
 // --------------------------------------------------------------------------
@@ -42 +43 @@
         BackupDaemon();
         ~BackupDaemon();
+
+        // methods below do partial (specialized) serialization of client state only
+        void SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const;
+        void DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime);
 private:
         BackupDaemon(const BackupDaemon &);
@@ -118 +123 @@
                 Location();
                 ~Location();
+
+                void Deserialize(Archive & rArchive);
+                void Serialize(Archive & rArchive) const;
         private:
                 Location(const Location &);     // copy not allowed

box/trunk/bin/bbackupd/bbackupd-config

@@ -383 +383 @@
 CommandSocket = /var/run/bbackupd.sock
 
+# Uncomment the StoreObjectInfoFile to enable the experimental archiving
+# of the daemon's state (including client store marker and configuration)
+# between backup runs. This saves time and increases efficiency when
+# bbackupd is frequently stopped and started, since it removes the need
+# to rescan all directories on the remote server. However, it is new and
+# not yet heavily tested, so use with caution.
+
+# StoreObjectInfoFile = $working_dir/bbackupd.state
 
 Server

box/trunk/configure.ac

@@ -75 +75 @@
 AC_HEADER_STDC
 AC_HEADER_SYS_WAIT
-AC_CHECK_HEADERS([execinfo.h netinet/in.h regex.h sys/types.h sys/xattr.h])
-AC_CHECK_HEADERS([sys/endian.h asm/byteorder.h syslog.h signal.h sys/time.h])
-AC_CHECK_HEADERS([time.h])
+AC_CHECK_HEADERS([execinfo.h regex.h signal.h syslog.h time.h])
+AC_CHECK_HEADERS([asm/byteorder.h])
+AC_CHECK_HEADERS([netinet/in.h])
+AC_CHECK_HEADERS([sys/endian.h sys/param.h sys/types.h sys/wait.h sys/xattr.h sys/time.h])
 
 

box/trunk/lib/backupclient/BackupDaemonConfigVerify.cpp

@@ -86 +86 @@
         {"CommandSocket", 0, 0, 0},                             // not compulsory to have this
         {"KeepAliveTime", 0, ConfigTest_IsInt, 0},                              // optional
+        {"StoreObjectInfoFile", 0, 0, 0},                               // optional
 
         {"NotifyScript", 0, 0, 0},                              // optional script to run when backup needs attention, eg store full
@@ -104 +105 @@
         0
 };
-

box/trunk/lib/common/CommonException.txt

@@ -44 +44 @@
 IOStreamGetLineNotEnoughDataToIgnore    37      Bad value passed to IOStreamGetLine::IgnoreBufferedData()
 TempDirPathTooLong                              38      Your temporary directory path is too long. Check the TMP and TEMP environment variables.
+ArchiveBlockIncompleteRead              39      The Store Object Info File is too short or corrupted, and will be rewritten automatically when the next backup completes.

box/trunk/lib/common/ExcludeList.cpp

@@ -18 +18 @@
 #include "Utils.h"
 #include "Configuration.h"
+#include "Archive.h"
 
 #include "MemLeakFindOn.h"
@@ -131 +132 @@
                                 // Store in list of regular expressions
                                 mRegex.push_back(pregex);
+                                // Store in list of regular expression string for Serialize
+                                mRegexStr.push_back(i->c_str());
                         }
                         catch(...)
@@ -214 +217 @@
 }
 
-
-        
-
-
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    ExcludeList::Deserialize(Archive & rArchive)
+//              Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction.
+//
+//              Created: 2005/04/11
+//
+// --------------------------------------------------------------------------
+void ExcludeList::Deserialize(Archive & rArchive)
+{
+        //
+        //
+        //
+        mDefinite.clear();
+
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+        // free regex memory
+        while(mRegex.size() > 0)
+        {
+                regex_t *pregex = mRegex.back();
+                mRegex.pop_back();
+                // Free regex storage, and the structure itself
+                ::regfree(pregex);
+                delete pregex;
+        }
+
+        mRegexStr.clear();
+#endif
+
+        // Clean up exceptions list
+        if(mpAlwaysInclude != 0)
+        {
+                delete mpAlwaysInclude;
+                mpAlwaysInclude = 0;
+        }
+
+        //
+        //
+        //
+        int64_t iCount = 0;
+        rArchive.Read(iCount);
+
+        if (iCount > 0)
+        {
+                for (int v = 0; v < iCount; v++)
+                {
+                        // load each one
+                        std::string strItem;
+                        rArchive.Read(strItem);
+                        mDefinite.insert(strItem);
+                }
+        }
+
+        //
+        //
+        //
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+        rArchive.Read(iCount);
+
+        if (iCount > 0)
+        {
+                for (int v = 0; v < iCount; v++)
+                {
+                        std::string strItem;
+                        rArchive.Read(strItem);
+
+                        // Allocate memory
+                        regex_t* pregex = new regex_t;
+                        
+                        try
+                        {
+                                // Compile
+                                if(::regcomp(pregex, strItem.c_str(), 
+                                        REG_EXTENDED | REG_NOSUB) != 0)
+                                {
+                                        THROW_EXCEPTION(CommonException, 
+                                                BadRegularExpression)
+                                }
+                                
+                                // Store in list of regular expressions
+                                mRegex.push_back(pregex);
+
+                                // Store in list of regular expression strings
+                                // for Serialize
+                                mRegexStr.push_back(strItem);
+                        }
+                        catch(...)
+                        {
+                                delete pregex;
+                                throw;
+                        }
+                }
+        }
+#endif // PLATFORM_REGEX_NOT_SUPPORTED
+
+        //
+        //
+        //
+        int64_t aMagicMarker = 0;
+        rArchive.Read(aMagicMarker);
+
+        if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP)
+        {
+                // NOOP
+        }
+        else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE)
+        {
+                mpAlwaysInclude = new ExcludeList;
+                if (!mpAlwaysInclude)
+                {
+                        throw std::bad_alloc();
+                }
+
+                mpAlwaysInclude->Deserialize(rArchive);
+        }
+        else
+        {
+                // there is something going on here
+                THROW_EXCEPTION(CommonException, Internal)
+        }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    ExcludeList::Serialize(Archive & rArchive)
+//              Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction.
+//
+//              Created: 2005/04/11
+//
+// --------------------------------------------------------------------------
+void ExcludeList::Serialize(Archive & rArchive) const
+{
+        //
+        //
+        //
+        int64_t iCount = mDefinite.size();
+        rArchive.Write(iCount);
+
+        for (std::set<std::string>::const_iterator i = mDefinite.begin(); 
+                i != mDefinite.end(); i++)
+        {
+                rArchive.Write(*i);
+        }
+
+        //
+        //
+        //
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+        // don't even try to save compiled regular expressions,
+        // use string copies instead.
+        ASSERT(mRegex.size() == mRegexStr.size());      
+
+        iCount = mRegexStr.size();
+        rArchive.Write(iCount);
+
+        for (std::vector<std::string>::const_iterator i = mRegexStr.begin(); 
+                i != mRegexStr.end(); i++)
+        {
+                rArchive.Write(*i);
+        }
+#endif // PLATFORM_REGEX_NOT_SUPPORTED
+
+        //
+        //
+        //
+        if (!mpAlwaysInclude)
+        {
+                int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP;
+                rArchive.Write(aMagicMarker);
+        }
+        else
+        {
+                int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows
+                rArchive.Write(aMagicMarker);
+
+                mpAlwaysInclude->Serialize(rArchive);
+        }
+}

box/trunk/lib/common/ExcludeList.h

@@ -20 +20 @@
 #endif
 
+class Archive;
+
 // --------------------------------------------------------------------------
 //
@@ -33 +35 @@
         ExcludeList();
         ~ExcludeList();
+
+        void Deserialize(Archive & rArchive);
+        void Serialize(Archive & rArchive) const;
 
         void AddDefiniteEntries(const std::string &rEntries);
@@ -56 +61 @@
 #ifdef HAVE_REGEX_H
         std::vector<regex_t *> mRegex;
+        std::vector<std::string> mRegexStr;     // save original regular expression string-based source for Serialize
 #endif
 

box/trunk/lib/server/Daemon.cpp

@@ -10 +10 @@
 #include "Box.h"
 
+#include <errno.h>
 #include <stdio.h>
 #include <unistd.h>
@@ -16 +17 @@
 #include <stdarg.h>
 
-#ifndef WIN32
-#include <syslog.h>
+#ifdef HAVE_SYSLOG_H
+        #include <syslog.h>
 #endif
 
@@ -25 +26 @@
 #include "Guards.h"
 #include "UnixUser.h"
+#include "FileModificationTime.h"
 
 #include "MemLeakFindOn.h"
@@ -93 +95 @@
 
         std::string pidFileName;
-        const char *configfile = 0;
 
         try
         {
                 // Find filename of config file
-                configfile = DefaultConfigFile;
+                mConfigFileName = DefaultConfigFile;
                 if(argc >= 2)
                 {
@@ -104 +105 @@
                         if(::strcmp(argv[1], "-c") == 0 && argc >= 3)
                         {
-                                configfile = argv[2];
+                                mConfigFileName = argv[2];
                         }
                         else
                         {
-                                configfile = argv[1];
+                                mConfigFileName = argv[1];
                         }
                 }
@@ -124 +125 @@
                 // Load the configuration file.
                 std::string errors;
-                std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(configfile, GetConfigVerify(), errors);
+                std::auto_ptr<Configuration> pconfig = 
+                        Configuration::LoadAndVerify(
+                                mConfigFileName.c_str(), 
+                                GetConfigVerify(), errors);
 
                 // Got errors?
@@ -130 +134 @@
                 {
                         // Tell user about errors
-                        fprintf(stderr, "%s: Errors in config file %s:\n%s", DaemonName(), configfile, errors.c_str());
+                        fprintf(stderr, "%s: Errors in config file %s:\n%s", 
+                                DaemonName(), mConfigFileName.c_str(), 
+                                errors.c_str());
                         // And give up
                         return 1;
@@ -137 +143 @@
                 // Store configuration
                 mpConfiguration = pconfig.release();
+                mLoadedConfigModifiedTime = GetConfigFileModifiedTime();
                 
                 // Server configuration
@@ -229 +236 @@
                 ::openlog(DaemonName(), LOG_PID, LOG_LOCAL6);
                 // Log the start message
-                ::syslog(LOG_INFO, "Starting daemon (config: %s) (version " BOX_VERSION ")", configfile);
+                ::syslog(LOG_INFO, "Starting daemon (config: %s) (version " 
+                        BOX_VERSION ")", mConfigFileName.c_str());
 
 #ifndef WIN32
@@ -307 +315 @@
                         {
                                 // Need to reload that config file...
-                                ::syslog(LOG_INFO, "Reloading configuration (config: %s)", configfile);
+                                ::syslog(LOG_INFO, "Reloading configuration "
+                                        "(config: %s)", 
+                                        mConfigFileName.c_str());
                                 std::string errors;
-                                std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(configfile, GetConfigVerify(), errors);
+                                std::auto_ptr<Configuration> pconfig = 
+                                        Configuration::LoadAndVerify(
+                                                mConfigFileName.c_str(),
+                                                GetConfigVerify(), errors);
 
                                 // Got errors?
@@ -315 +328 @@
                                 {
                                         // Tell user about errors
-                                        ::syslog(LOG_ERR, "Errors in config file %s:\n%s", configfile, errors.c_str());
+                                        ::syslog(LOG_ERR, "Errors in config "
+                                                "file %s:\n%s", 
+                                                mConfigFileName.c_str(),
+                                                errors.c_str());
                                         // And give up
                                         return 1;
@@ -326 +342 @@
                                 // Store configuration
                                 mpConfiguration = pconfig.release();
+                                mLoadedConfigModifiedTime =
+                                        GetConfigFileModifiedTime();
                                 
                                 // Stop being marked for loading config again
@@ -548 +566 @@
 #endif // HAVE_SETPROCTITLE
 }
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    Daemon::GetConfigFileModifiedTime()
+//              Purpose: Returns the timestamp when the configuration file
+//                       was last modified
+//
+//              Created: 2006/01/29
+//
+// --------------------------------------------------------------------------
+
+box_time_t Daemon::GetConfigFileModifiedTime() const
+{
+        struct stat st;
+
+        if(::stat(GetConfigFileName().c_str(), &st) != 0)
+        {
+                if (errno == ENOENT)
+                {
+                        return 0;
+                }
+                THROW_EXCEPTION(CommonException, OSFileError)
+        }
+        
+        return FileModificationTime(st);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    Daemon::GetLoadedConfigModifiedTime()
+//              Purpose: Returns the timestamp when the configuration file
+//                       had been last modified, at the time when it was 
+//                       loaded
+//
+//              Created: 2006/01/29
+//
+// --------------------------------------------------------------------------
+
+box_time_t Daemon::GetLoadedConfigModifiedTime() const
+{
+        return mLoadedConfigModifiedTime;
+}
+

box/trunk/lib/server/Daemon.h

@@ -16 +16 @@
 #ifndef DAEMON__H
 #define DAEMON__H
+
+#include <string>
+
+#include "BoxTime.h"
 
 class Configuration;
@@ -41 +45 @@
         virtual void Run();
         const Configuration &GetConfiguration() const;
-        
+        const std::string &GetConfigFileName() const {return mConfigFileName;}
+
         virtual const char *DaemonName() const;
         virtual const char *DaemonBanner() const;
@@ -58 +63 @@
         
         static void SetProcessTitle(const char *format, ...);
+
+protected:
+        box_time_t GetLoadedConfigModifiedTime() const;
         
 private:
         static void SignalHandler(int sigraised);
+        box_time_t GetConfigFileModifiedTime() const;
         
 private:
+        std::string mConfigFileName;
         Configuration *mpConfiguration;
+        box_time_t mLoadedConfigModifiedTime;
         bool mReloadConfigWanted;
         bool mTerminateWanted;