Changeset 2181

Timestamp:

28/05/2008 15:24:05 (2 years ago)

Author:

chris

Message:

Track and log file deletions by name.

Split crypto init and file sync process into its own method, to reduce
call depth and facilitate calling in process from tests.

Differentiate between 3 uses of stat in BackupClientDirectoryRecord? by
renaming the structures.

Use stat instead of lstat when checking the filesystem that's holding an
entity, in case it's a symbolic link to a different filesystem.

Location:

box/trunk/bin/bbackupd

Files:

• BackupClientContext.cpp (modified) (2 diffs)
• BackupClientContext.h (modified) (5 diffs)
• BackupClientDeleteList.cpp (modified) (8 diffs)
• BackupClientDeleteList.h (modified) (1 diff)
• BackupClientDirectoryRecord.cpp (modified) (63 diffs)
• BackupClientDirectoryRecord.h (modified) (5 diffs)
• BackupDaemon.cpp (modified) (24 diffs)
• BackupDaemon.h (modified) (5 diffs)

box/trunk/bin/bbackupd/BackupClientContext.cpp

@@ -49 +49 @@
         bool ExtendedLogging,
         bool ExtendedLogToFile,
-        std::string ExtendedLogFile
+        std::string ExtendedLogFile,
+        ProgressNotifier& rProgressNotifier
 )
         : mrDaemon(rDaemon),
@@ -70 +71 @@
           mpExcludeDirs(0),
           mKeepAliveTimer(0),
-          mbIsManaged(false)
+          mbIsManaged(false),
+          mrProgressNotifier(rProgressNotifier)
 {
 }

box/trunk/bin/bbackupd/BackupClientContext.h

@@ -13 +13 @@
 #include "BoxTime.h"
 #include "BackupClientDeleteList.h"
+#include "BackupClientDirectoryRecord.h"
 #include "BackupStoreFile.h"
 #include "ExcludeList.h"
@@ -46 +47 @@
                 bool ExtendedLogging,
                 bool ExtendedLogToFile,
-                std::string ExtendedLogFile
+                std::string ExtendedLogFile,
+                ProgressNotifier &rProgressNotifier
         );
         virtual ~BackupClientContext();
@@ -71 +73 @@
         
         bool StorageLimitExceeded() {return mStorageLimitExceeded;}
+        void SetStorageLimitExceeded() {mStorageLimitExceeded = true;}
 
         // --------------------------------------------------------------------------
@@ -199 +202 @@
         virtual bool   IsManaged() { return mbIsManaged; }
         
+        ProgressNotifier& GetProgressNotifier() const 
+        { 
+                return mrProgressNotifier;
+        }
+
 private:
         BackupDaemon &mrDaemon;
@@ -222 +230 @@
         int mKeepAliveTime;
         int mMaximumDiffingTime;
+        ProgressNotifier &mrProgressNotifier;
 };
 

box/trunk/bin/bbackupd/BackupClientDeleteList.cpp

@@ -43 +43 @@
 }
 
-// --------------------------------------------------------------------------
-//
-// Function
-//              Name:    BackupClientDeleteList::AddDirectoryDelete(int64_t)
+BackupClientDeleteList::FileToDelete::FileToDelete(int64_t DirectoryID,
+        const BackupStoreFilename& rFilename,
+        const std::string& rLocalPath)
+: mDirectoryID(DirectoryID),
+  mFilename(rFilename),
+  mLocalPath(rLocalPath)
+{ }
+
+BackupClientDeleteList::DirToDelete::DirToDelete(int64_t ObjectID,
+        const std::string& rLocalPath)
+: mObjectID(ObjectID),
+  mLocalPath(rLocalPath)
+{ }
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    BackupClientDeleteList::AddDirectoryDelete(int64_t,
+//                       const BackupStoreFilename&)
 //              Purpose: Add a directory to the list of directories to be deleted.
 //              Created: 10/11/03
 //
 // --------------------------------------------------------------------------
-void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID)
+void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID,
+        const std::string& rLocalPath)
 {
         // Only add the delete to the list if it's not in the "no delete" set
-        if(mDirectoryNoDeleteList.find(ObjectID) == mDirectoryNoDeleteList.end())
+        if(mDirectoryNoDeleteList.find(ObjectID) ==
+                mDirectoryNoDeleteList.end())
         {
                 // Not in the list, so should delete it
-                mDirectoryList.push_back(ObjectID);
-        }
-}
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-//              Name:    BackupClientDeleteList::AddFileDelete(int64_t, BackupStoreFilenameClear &)
+                mDirectoryList.push_back(DirToDelete(ObjectID, rLocalPath));
+        }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    BackupClientDeleteList::AddFileDelete(int64_t,
+//                       const BackupStoreFilename &)
 //              Purpose: 
 //              Created: 10/11/03
 //
 // --------------------------------------------------------------------------
-void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID, const BackupStoreFilename &rFilename)
+void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID,
+        const BackupStoreFilename &rFilename, const std::string& rLocalPath)
 {
         // Try to find it in the no delete list
-        std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator delEntry(mFileNoDeleteList.begin());
+        std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator
+                delEntry(mFileNoDeleteList.begin());
         while(delEntry != mFileNoDeleteList.end())
         {
-                if((delEntry)->first == DirectoryID && (delEntry)->second == rFilename)
+                if((delEntry)->first == DirectoryID 
+                        && (delEntry)->second == rFilename)
                 {
                         // Found!
@@ -87 +108 @@
         if(delEntry == mFileNoDeleteList.end())
         {
-                mFileList.push_back(std::pair<int64_t, BackupStoreFilename>(DirectoryID, rFilename));
+                mFileList.push_back(FileToDelete(DirectoryID, rFilename,
+                        rLocalPath));
         }
 }
@@ -114 +136 @@
         
         // Do the deletes
-        for(std::vector<int64_t>::iterator i(mDirectoryList.begin()); i != mDirectoryList.end(); ++i)
-        {
-                connection.QueryDeleteDirectory(*i);
+        for(std::vector<DirToDelete>::iterator i(mDirectoryList.begin());
+                i != mDirectoryList.end(); ++i)
+        {
+                connection.QueryDeleteDirectory(i->mObjectID);
+                rContext.GetProgressNotifier().NotifyDirectoryDeleted(
+                        i->mObjectID, i->mLocalPath);
         }
         
@@ -123 +148 @@
         
         // Delete the files
-        for(std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator i(mFileList.begin()); i != mFileList.end(); ++i)
-        {
-                connection.QueryDeleteFile(i->first, i->second);
+        for(std::vector<FileToDelete>::iterator i(mFileList.begin());
+                i != mFileList.end(); ++i)
+        {
+                connection.QueryDeleteFile(i->mDirectoryID, i->mFilename);
+                rContext.GetProgressNotifier().NotifyFileDeleted(
+                        i->mDirectoryID, i->mLocalPath);
         }
 }
@@ -141 +169 @@
 {
         // First of all, is it in the delete vector?
-        std::vector<int64_t>::iterator delEntry(std::find(mDirectoryList.begin(), mDirectoryList.end(), ObjectID));
+        std::vector<DirToDelete>::iterator delEntry(mDirectoryList.begin());
+        for(; delEntry != mDirectoryList.end(); delEntry++)
+        {
+                if(delEntry->mObjectID == ObjectID)
+                {
+                        // Found!
+                        break;
+                }
+        }
         if(delEntry != mDirectoryList.end())
         {
@@ -149 +185 @@
         else
         {
-                // Haven't been asked to delete it yet, put it in the no delete list
+                // Haven't been asked to delete it yet, put it in the
+                // no delete list
                 mDirectoryNoDeleteList.insert(ObjectID);
         }
@@ -163 +200 @@
 //
 // --------------------------------------------------------------------------
-void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID, const BackupStoreFilename &rFilename)
+void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID,
+        const BackupStoreFilename &rFilename)
 {
         // Find this in the delete list
-        std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator delEntry(mFileList.begin());
+        std::vector<FileToDelete>::iterator delEntry(mFileList.begin());
         while(delEntry != mFileList.end())
         {
-                if((delEntry)->first == DirectoryID && (delEntry)->second == rFilename)
+                if(delEntry->mDirectoryID == DirectoryID
+                        && delEntry->mFilename == rFilename)
                 {
                         // Found!
@@ -187 +226 @@
                 mFileNoDeleteList.push_back(std::pair<int64_t, BackupStoreFilename>(DirectoryID, rFilename));
         }
-
-}
-
-
-
-
-
+}
+

box/trunk/bin/bbackupd/BackupClientDeleteList.h

@@ -29 +29 @@
 class BackupClientDeleteList
 {
+private:
+        class FileToDelete
+        {
+                public:
+                int64_t mDirectoryID;
+                BackupStoreFilename mFilename;
+                std::string mLocalPath;
+                FileToDelete(int64_t DirectoryID, 
+                        const BackupStoreFilename& rFilename,
+                        const std::string& rLocalPath);
+        };
+
+        class DirToDelete
+        {
+                public:
+                int64_t mObjectID;
+                std::string mLocalPath;
+                DirToDelete(int64_t ObjectID, const std::string& rLocalPath);
+        };
+
 public:
         BackupClientDeleteList();
         ~BackupClientDeleteList();
         
-        void AddDirectoryDelete(int64_t ObjectID);
-        void AddFileDelete(int64_t DirectoryID, const BackupStoreFilename &rFilename);
+        void AddDirectoryDelete(int64_t ObjectID,
+                const std::string& rLocalPath);
+        void AddFileDelete(int64_t DirectoryID,
+                const BackupStoreFilename &rFilename,
+                const std::string& rLocalPath);
 
         void StopDirectoryDeletion(int64_t ObjectID);
-        void StopFileDeletion(int64_t DirectoryID, const BackupStoreFilename &rFilename);
+        void StopFileDeletion(int64_t DirectoryID,
+                const BackupStoreFilename &rFilename);
         
         void PerformDeletions(BackupClientContext &rContext);
         
 private:
-        std::vector<int64_t> mDirectoryList;
+        std::vector<DirToDelete> mDirectoryList;
         std::set<int64_t> mDirectoryNoDeleteList;       // note: things only get in this list if they're not present in mDirectoryList when they are 'added'
-        std::vector<std::pair<int64_t, BackupStoreFilename> > mFileList;
+        std::vector<FileToDelete> mFileList;
         std::vector<std::pair<int64_t, BackupStoreFilename> > mFileNoDeleteList;
 };

box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp

@@ -3 +3 @@
 // File
 //              Name:    BackupClientDirectoryRecord.cpp
-//              Purpose: Implementation of record about directory for backup client
+//              Purpose: Implementation of record about directory for
+//                       backup client
 //              Created: 2003/10/08
 //
@@ -101 +102 @@
 //
 // Function
-//              Name:    BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &, int64_t, const std::string &, bool)
-//              Purpose: Syncronise, recusively, a local directory with the server.
+//              Name:    BackupClientDirectoryRecord::SyncDirectory(i
+//                       BackupClientDirectoryRecord::SyncParams &,
+//                       int64_t, const std::string &,
+//                       const std::string &, bool)
+//              Purpose: Recursively synchronise a local directory
+//                       with the server.
 //              Created: 2003/10/08
 //
 // --------------------------------------------------------------------------
-void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &rParams, int64_t ContainingDirectoryID,
-        const std::string &rLocalPath, bool ThisDirHasJustBeenCreated)
+void BackupClientDirectoryRecord::SyncDirectory(
+        BackupClientDirectoryRecord::SyncParams &rParams,
+        int64_t ContainingDirectoryID,
+        const std::string &rLocalPath,
+        const std::string &rRemotePath,
+        bool ThisDirHasJustBeenCreated)
 {
+        BackupClientContext& rContext(rParams.mrContext);
+        ProgressNotifier& rNotifier(rContext.GetProgressNotifier());
+
         // Signal received by daemon?
         if(rParams.mrDaemon.StopRun())
@@ -119 +131 @@
         // and on the immediate sub directories.
         mSyncDone = false;
-        for(std::map<std::string, BackupClientDirectoryRecord *>::iterator i = mSubDirectories.begin();
+        for(std::map<std::string, BackupClientDirectoryRecord *>::iterator
+                i  = mSubDirectories.begin();
                 i != mSubDirectories.end(); ++i)
         {
@@ -125 +138 @@
         }
 
-        // Work out the time in the future after which the file should be uploaded regardless.
-        // This is a simple way to avoid having too many problems with file servers when they have
-        // clients with badly out of sync clocks.
-        rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + rParams.mMaxFileTimeInFuture;
-        
-        // Build the current state checksum to compare against while getting info from dirs
-        // Note checksum is used locally only, so byte order isn't considered.
+        // Work out the time in the future after which the file should
+        // be uploaded regardless. This is a simple way to avoid having
+        // too many problems with file servers when they have clients
+        // with badly out of sync clocks.
+        rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() +
+                rParams.mMaxFileTimeInFuture;
+        
+        // Build the current state checksum to compare against while
+        // getting info from dirs. Note checksum is used locally only,
+        // so byte order isn't considered.
         MD5Digest currentStateChecksum;
         
+        struct stat dest_st;
         // Stat the directory, to get attribute info
-        {
-                struct stat st;
-                if(::stat(rLocalPath.c_str(), &st) != 0)
-                {
-                        // The directory has probably been deleted, so just ignore this error.
-                        // In a future scan, this deletion will be noticed, deleted from server, and this object deleted.
-                        rParams.GetProgressNotifier().NotifyDirStatFailed(
-                                this, rLocalPath, strerror(errno));
+        // If it's a symbolic link, we want the link target here
+        // (as we're about to back up the contents of the directory)
+        {
+                if(::stat(rLocalPath.c_str(), &dest_st) != 0)
+                {
+                        // The directory has probably been deleted, so
+                        // just ignore this error. In a future scan, this
+                        // deletion will be noticed, deleted from server,
+                        // and this object deleted.
+                        rNotifier.NotifyDirStatFailed(this, rLocalPath,
+                                strerror(errno));
                         return;
                 }
-                // Store inode number in map so directories are tracked in case they're renamed
-                {
-                        BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap());
-                        idMap.AddToMap(st.st_ino, mObjectID, ContainingDirectoryID);
+                // Store inode number in map so directories are tracked
+                // in case they're renamed
+                {
+                        BackupClientInodeToIDMap &idMap(
+                                rParams.mrContext.GetNewIDMap());
+                        idMap.AddToMap(dest_st.st_ino, mObjectID,
+                                ContainingDirectoryID);
                 }
                 // Add attributes to checksum
-                currentStateChecksum.Add(&st.st_mode, sizeof(st.st_mode));
-                currentStateChecksum.Add(&st.st_uid, sizeof(st.st_uid));
-                currentStateChecksum.Add(&st.st_gid, sizeof(st.st_gid));
+                currentStateChecksum.Add(&dest_st.st_mode,
+                        sizeof(dest_st.st_mode));
+                currentStateChecksum.Add(&dest_st.st_uid,
+                        sizeof(dest_st.st_uid));
+                currentStateChecksum.Add(&dest_st.st_gid,
+                        sizeof(dest_st.st_gid));
                 // Inode to be paranoid about things moving around
-                currentStateChecksum.Add(&st.st_ino, sizeof(st.st_ino));
+                currentStateChecksum.Add(&dest_st.st_ino,
+                        sizeof(dest_st.st_ino));
 #ifdef HAVE_STRUCT_STAT_ST_FLAGS
-                currentStateChecksum.Add(&st.st_flags, sizeof(st.st_flags));
+                currentStateChecksum.Add(&dest_st.st_flags,
+                        sizeof(dest_st.st_flags));
 #endif
 
                 StreamableMemBlock xattr;
-                BackupClientFileAttributes::FillExtendedAttr(xattr, rLocalPath.c_str());
+                BackupClientFileAttributes::FillExtendedAttr(xattr,
+                        rLocalPath.c_str());
                 currentStateChecksum.Add(xattr.GetBuffer(), xattr.GetSize());
         }
@@ -171 +200 @@
         bool downloadDirectoryRecordBecauseOfFutureFiles = false;
 
-        struct stat dir_st;
-        if(::lstat(rLocalPath.c_str(), &dir_st) != 0)
+        struct stat link_st;
+        if(::lstat(rLocalPath.c_str(), &link_st) != 0)
         {
                 // Report the error (logs and 
                 // eventual email to administrator)
-                rParams.GetProgressNotifier().NotifyFileStatFailed(this, 
-                        rLocalPath, strerror(errno));
+                rNotifier.NotifyFileStatFailed(this, rLocalPath,
+                        strerror(errno));
                 
                 // FIXME move to NotifyFileStatFailed()
@@ -193 +222 @@
                 try
                 {
-                        rParams.GetProgressNotifier().NotifyScanDirectory(
-                                this, rLocalPath);
+                        rNotifier.NotifyScanDirectory(this, rLocalPath);
 
                         dirHandle = ::opendir(rLocalPath.c_str());
@@ -203 +231 @@
                                 if (errno == EACCES)
                                 {
-                                        rParams.GetProgressNotifier().NotifyDirListFailed(
-                                                this, rLocalPath, "Access denied");
+                                        rNotifier.NotifyDirListFailed(this,
+                                                rLocalPath, "Access denied");
                                 }
                                 else
                                 {
-                                        rParams.GetProgressNotifier().NotifyDirListFailed(this, 
+                                        rNotifier.NotifyDirListFailed(this, 
                                                 rLocalPath, strerror(errno));
                                 }
                                 
-                                // Report the error (logs and eventual email to administrator)
-                                SetErrorWhenReadingFilesystemObject(rParams, rLocalPath.c_str());
+                                // Report the error (logs and eventual email
+                                // to administrator)
+                                SetErrorWhenReadingFilesystemObject(rParams,
+                                        rLocalPath.c_str());
                                 // Ignore this directory for now.
                                 return;
@@ -229 +259 @@
         
                         struct dirent *en = 0;
-                        struct stat st;
+                        struct stat file_st;
                         std::string filename;
                         while((en = ::readdir(dirHandle)) != 0)
@@ -235 +265 @@
                                 rParams.mrContext.DoKeepAlive();
                                 
-                                // Don't need to use LinuxWorkaround_FinishDirentStruct(en, rLocalPath.c_str());
-                                // on Linux, as a stat is performed to get all this info
+                                // Don't need to use
+                                // LinuxWorkaround_FinishDirentStruct(en,
+                                // rLocalPath.c_str());
+                                // on Linux, as a stat is performed to
+                                // get all this info
 
                                 if(en->d_name[0] == '.' && 
@@ -260 +293 @@
                                 int type = en->d_type;
                                 #else
-                                if(::lstat(filename.c_str(), &st) != 0)
+                                if(::lstat(filename.c_str(), &file_st) != 0)
                                 {
                                         // Report the error (logs and 
                                         // eventual email to administrator)
-                                        rParams.GetProgressNotifier().NotifyFileStatFailed(this, 
+                                        rNotifier.NotifyFileStatFailed(this, 
                                                 filename, strerror(errno));
                                         
@@ -275 +308 @@
                                 }
 
-                                if(st.st_dev != dir_st.st_dev)
+                                if(file_st.st_dev != dest_st.st_dev)
                                 {
                                         if(!(rParams.mrContext.ExcludeDir(
                                                 filename)))
                                         {
-                                                rParams.GetProgressNotifier()
-                                                        .NotifyMountPointSkipped(
-                                                                this, filename);
+                                                rNotifier.NotifyMountPointSkipped(
+                                                        this, filename);
                                         }
                                         continue;
                                 }
 
-                                int type = st.st_mode & S_IFMT;
+                                int type = file_st.st_mode & S_IFMT;
                                 #endif
 
@@ -297 +329 @@
                                         if(rParams.mrContext.ExcludeFile(filename))
                                         {
-                                                rParams.GetProgressNotifier()
-                                                        .NotifyFileExcluded(
+                                                rNotifier.NotifyFileExcluded(
                                                                 this, 
                                                                 filename);
@@ -316 +347 @@
                                         if(rParams.mrContext.ExcludeDir(filename))
                                         {
-                                                rParams.GetProgressNotifier()
-                                                        .NotifyDirExcluded(
+                                                rNotifier.NotifyDirExcluded(
                                                                 this, 
                                                                 filename);
@@ -332 +362 @@
                                         if(rParams.mrContext.ExcludeFile(filename))
                                         {
-                                                rParams.GetProgressNotifier()
-                                                        .NotifyFileExcluded(
+                                                rNotifier.NotifyFileExcluded(
                                                                 this, 
                                                                 filename);
@@ -339 +368 @@
                                         else
                                         {
-                                                rParams.GetProgressNotifier()
-                                                        .NotifyUnsupportedFileType(
+                                                rNotifier.NotifyUnsupportedFileType(
                                                                 this, filename);
                                                 SetErrorWhenReadingFilesystemObject(
@@ -355 +383 @@
                                 // We didn't stat the file before,
                                 // but now we need the information.
-                                if(::lstat(filename.c_str(), &st) != 0)
-                                {
-                                        rParams.GetProgressNotifier()
-                                                .NotifyFileStatFailed(this, 
+                                if(::lstat(filename.c_str(), &file_st) != 0)
+                                {
+                                        rNotifier.NotifyFileStatFailed(this, 
                                                         filename, 
                                                         strerror(errno));
@@ -371 +398 @@
                                 }
 
-                                if(st.st_dev != dir_st.st_dev)
-                                {
-                                        rParams.GetProgressNotifier()
-                                                .NotifyMountPointSkipped(this, 
+                                if(file_st.st_dev != link_st.st_dev)
+                                {
+                                        rNotifier.NotifyMountPointSkipped(this, 
                                                         filename);
                                         continue;
@@ -380 +406 @@
                                 #endif
 
-                                checksum_info.mModificationTime = FileModificationTime(st);
-                                checksum_info.mAttributeModificationTime = FileAttrModificationTime(st);
-                                checksum_info.mSize = st.st_size;
+                                checksum_info.mModificationTime = FileModificationTime(file_st);
+                                checksum_info.mAttributeModificationTime = FileAttrModificationTime(file_st);
+                                checksum_info.mSize = file_st.st_size;
                                 currentStateChecksum.Add(&checksum_info, sizeof(checksum_info));
                                 currentStateChecksum.Add(en->d_name, strlen(en->d_name));
@@ -395 +421 @@
                                         if(!rParams.mHaveLoggedWarningAboutFutureFileTimes)
                                         {
-                                                rParams.GetProgressNotifier().NotifyFileModifiedInFuture(
+                                                rNotifier.NotifyFileModifiedInFuture(
                                                         this, filename);
                                                 rParams.mHaveLoggedWarningAboutFutureFileTimes = true;
@@ -469 +495 @@
                 
                 // Do the directory reading
-                bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, pdirOnStore, entriesLeftOver, files, dirs);
+                bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath,
+                        rRemotePath, pdirOnStore, entriesLeftOver, files, dirs);
                 
                 // LAST THING! (think exception safety)
@@ -605 +632 @@
 //
 // --------------------------------------------------------------------------
-bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &rParams,
-        const std::string &rLocalPath, BackupStoreDirectory *pDirOnStore,
+bool BackupClientDirectoryRecord::UpdateItems(
+        BackupClientDirectoryRecord::SyncParams &rParams,
+        const std::string &rLocalPath,
+        const std::string &rRemotePath,
+        BackupStoreDirectory *pDirOnStore,
         std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver,
-        std::vector<std::string> &rFiles, const std::vector<std::string> &rDirs)
+        std::vector<std::string> &rFiles,
+        const std::vector<std::string> &rDirs)
 {
+        BackupClientContext& rContext(rParams.mrContext);
+        ProgressNotifier& rNotifier(rContext.GetProgressNotifier());
+
         bool allUpdatedSuccessfully = true;
 
@@ -635 +669 @@
         {
                 // Send keep-alive message if needed
-                rParams.mrContext.DoKeepAlive();
+                rContext.DoKeepAlive();
                 
                 // Filename of this file
@@ -652 +686 @@
                         if(::lstat(filename.c_str(), &st) != 0)
                         {
-                                rParams.GetProgressNotifier().NotifyFileStatFailed(this, 
+                                rNotifier.NotifyFileStatFailed(this, 
                                         filename, strerror(errno));
 
@@ -690 +724 @@
                 {
                         // Directory exists in the place of this file -- sort it out
-                        RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, en->GetObjectID(), *f);
+                        RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore,
+                                en, *f);
                         en = 0;
                 }
@@ -702 +737 @@
                         
                         // Do we know about the inode number?
-                        const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap());
+                        const BackupClientInodeToIDMap &idMap(rContext.GetCurrentIDMap());
                         int64_t renameObjectID = 0, renameInDirectory = 0;
                         if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory))
@@ -712 +747 @@
                                 box_time_t srvModTime = 0, srvAttributesHash = 0;
                                 BackupStoreFilenameClear oldLeafname;
-                                if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname))
+                                if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname))
                                 {       
                                         // Only interested if it's a file and the latest version
@@ -725 +760 @@
 
                                                         // Get the connection to the server 
-                                                        BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+                                                        BackupProtocolClient &connection(rContext.GetConnection());
 
                                                         // Only do this step if there is room on the server.
                                                         // This step will be repeated later when there is space available
-                                                        if(!rParams.mrContext.StorageLimitExceeded())
+                                                        if(!rContext.StorageLimitExceeded())
                                                         {
                                                                 // Rename the existing files (ie include old versions) on the server
@@ -737 +772 @@
                                                                         
                                                                 // Stop the attempt to delete the file in the original location
-                                                                BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList());
+                                                                BackupClientDeleteList &rdelList(rContext.GetDeleteList());
                                                                 rdelList.StopFileDeletion(renameInDirectory, oldLeafname);
                                                                 
@@ -872 +907 @@
                 }
 
+                bool fileSynced = true;
+
                 if (doUpload)
                 {
+                        // Upload needed, don't mark sync success until
+                        // we've actually done it
+                        fileSynced = false;
+
                         // Make sure we're connected -- must connect here so we know whether
                         // the storage limit has been exceeded, and hence whether or not
                         // to actually upload the file.
-                        rParams.mrContext.GetConnection();
+                        rContext.GetConnection();
 
                         // Only do this step if there is room on the server.
                         // This step will be repeated later when there is space available
-                        if(!rParams.mrContext.StorageLimitExceeded())
+                        if(!rContext.StorageLimitExceeded())
                         {
                                 // Upload the file to the server, recording the object ID it returns
@@ -891 +932 @@
                                 {
                                         latestObjectID = UploadFile(rParams, filename, storeFilename, fileSize, modTime, attributesHash, noPreviousVersionOnServer);
-                                        uploadSuccess = true;
+                                        if (latestObjectID == 0)
+                                        {
+                                                // storage limit exceeded
+                                                rParams.mrContext.SetStorageLimitExceeded();
+                                                uploadSuccess = false;
+                                                allUpdatedSuccessfully = false;
+                                        }
+                                        else
+                                        {
+                                                uploadSuccess = true;
+                                        }
                                 }
                                 catch(ConnectionException &e)
@@ -897 +948 @@
                                         // Connection errors should just be passed on to the main handler, retries
                                         // would probably just cause more problems.
-                                        rParams.GetProgressNotifier()
-                                                .NotifyFileUploadException(
-                                                        this, filename, e);
+                                        // StorageLimitExceeded never gets here.
+                                        
+                                        rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull);
+                                        rNotifier.NotifyFileUploadException(
+                                                this, filename, e);
                                         throw;
                                 }
@@ -908 +961 @@
                                         // Log it.
                                         SetErrorWhenReadingFilesystemObject(rParams, filename.c_str());
-                                        rParams.GetProgressNotifier()
-                                                .NotifyFileUploadException(
-                                                        this, filename, e);
+                                        rNotifier.NotifyFileUploadException(
+                                                this, filename, e);
                                 }
 
@@ -916 +968 @@
                                 if(uploadSuccess)
                                 {
+                                        fileSynced = true;
+
                                         // delete from pending entries
                                         if(pendingFirstSeenTime != 0 && mpPendingEntries != 0)
@@ -925 +979 @@
                         else
                         {
-                                rParams.GetProgressNotifier().NotifyFileSkippedServerFull(this,
+                                rNotifier.NotifyFileSkippedServerFull(this,
                                         filename);
                         }
@@ -932 +986 @@
                 {
                         // Attributes have probably changed, upload them again.
-                        // If the attributes have changed enough, the directory hash will have changed too,
-                        // and so the dir will have been downloaded, and the entry will be available.
+                        // If the attributes have changed enough, the directory
+                        // hash will have changed too, and so the dir will have
+                        // been downloaded, and the entry will be available.
 
                         // Get connection
-                        BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+                        BackupProtocolClient &connection(rContext.GetConnection());
 
                         // Only do this step if there is room on the server.
-                        // This step will be repeated later when there is space available
-                        if(!rParams.mrContext.StorageLimitExceeded())
+                        // This step will be repeated later when there is
+                        // space available
+                        if(!rContext.StorageLimitExceeded())
                         {
                                 // Update store
@@ -947 +1003 @@
                                 MemBlockStream attrStream(attr);
                                 connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream);
+                                fileSynced = true;
                         }
                 }
@@ -982 +1039 @@
                 {
                         // Get the map
-                        BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap());
+                        BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap());
                 
                         // Need to get an ID from somewhere...
@@ -994 +1051 @@
                                 // Don't know it -- haven't sent anything to the store, and didn't get a listing.
                                 // Look it up in the current map, and if it's there, use that.
-                                const BackupClientInodeToIDMap &currentIDMap(rParams.mrContext.GetCurrentIDMap());
+                                const BackupClientInodeToIDMap &currentIDMap(rContext.GetCurrentIDMap());
                                 int64_t objid = 0, dirid = 0;
                                 if(currentIDMap.Lookup(inodeNum, objid, dirid))
@@ -1003 +1060 @@
                                         // or there is a problem somewhere. If this happened on a short test run, look
                                         // into it. However, in a long running process this may happen occasionally and
-                                        // not indiciate anything wrong.
+                                        // not indicate anything wrong.
                                         // Run the release version for real life use, where this check is not made.
                                         idMap.AddToMap(inodeNum, objid, mObjectID /* containing directory */);                          
@@ -1010 +1067 @@
                 }
                 
-                rParams.GetProgressNotifier().NotifyFileSynchronised(this, 
-                        filename, fileSize);
+                if (fileSynced)
+                {
+                        rNotifier.NotifyFileSynchronised(this, filename,
+                                fileSize);
+                }
         }
 
@@ -1031 +1091 @@
         {
                 // Send keep-alive message if needed
-                rParams.mrContext.DoKeepAlive();
+                rContext.DoKeepAlive();
                 
                 // Get the local filename
@@ -1051 +1111 @@
                 if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == 0))
                 {
-                        // Entry exists, but is not a directory. Bad. Get rid of it.
-                        BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+                        // Entry exists, but is not a directory. Bad.
+                        // Get rid of it.
+                        BackupProtocolClient &connection(rContext.GetConnection());
                         connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename);
+                        rNotifier.NotifyFileDeleted(en->GetObjectID(),
+                                storeFilename.GetClearFilename());
                         
                         // Nothing found
@@ -1059 +1122 @@
                 }
 
-                // Flag for having created directory, so can optimise the recusive call not to
-                // read it again, because we know it's empty.
+                // Flag for having created directory, so can optimise the
+                // recusive call not to read it again, because we know
+                // it's empty.
                 bool haveJustCreatedDirOnServer = false;
 
@@ -1087 +1151 @@
                                 subDirObjectID = en->GetObjectID();
                         }
-                        else if(rParams.mrContext.StorageLimitExceeded())       
+                        else if(rContext.StorageLimitExceeded())        
                         // know we've got a connection if we get this far,
                         // as dir will have been modified.
@@ -1113 +1177 @@
                                 int64_t renameObjectID = 0, renameInDirectory = 0;
                                 bool renameDir = false;
-                                const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap());
+                                const BackupClientInodeToIDMap &idMap(
+                                        rContext.GetCurrentIDMap());
                                 if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory))
                                 {
@@ -1120 +1185 @@
                                         bool isDir = false;
                                         bool isCurrentVersion = false;
-                                        if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion))
+                                        if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion))
                                         {       
                                                 // Only interested if it's a directory
@@ -1138 +1203 @@
 
                                 // Get connection
-                                BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+                                BackupProtocolClient &connection(rContext.GetConnection());
                                 
                                 // Don't do a check for storage limit exceeded here, because if we get to this
@@ -1158 +1223 @@
 
                                         // Stop it being deleted later
-                                        BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList());
+                                        BackupClientDeleteList &rdelList(
+                                                rContext.GetDeleteList());
                                         rdelList.StopDirectoryDeletion(renameObjectID);
 
@@ -1195 +1261 @@
                 }
                 
-                ASSERT(psubDirRecord != 0 || rParams.mrContext.StorageLimitExceeded());
+                ASSERT(psubDirRecord != 0 || rContext.StorageLimitExceeded());
                 
                 if(psubDirRecord)
                 {
                         // Sync this sub directory too
-                        psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, haveJustCreatedDirOnServer);
+                        psubDirRecord->SyncDirectory(rParams, mObjectID,
+                                dirname, rRemotePath + "/" + *d,
+                                haveJustCreatedDirOnServer);
                 }
 
@@ -1229 +1297 @@
                         // If there's an error during the process, it doesn't matter if things
                         // aren't actually deleted, as the whole state will be reset anyway.
-                        BackupClientDeleteList &rdel(rParams.mrContext.GetDeleteList());
+                        BackupClientDeleteList &rdel(rContext.GetDeleteList());
+
+                        std::string localName = MakeFullPath(rLocalPath,
+                                en->GetName());
                         
                         // Delete this entry -- file or directory?
@@ -1235 +1306 @@
                         {
                                 // Set a pending deletion for the file
-                                rdel.AddFileDelete(mObjectID, en->GetName());                           
+                                rdel.AddFileDelete(mObjectID, en->GetName(),
+                                        localName);
                         }
                         else if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0)
                         {
                                 // Set as a pending deletion for the directory
-                                rdel.AddDirectoryDelete(en->GetObjectID());
+                                rdel.AddDirectoryDelete(en->GetObjectID(),
+                                        localName);
                                 
-                                // If there's a directory record for it in the sub directory map, delete it now
+                                // If there's a directory record for it in 
+                                // the sub directory map, delete it now
                                 BackupStoreFilenameClear dirname(en->GetName());
                                 std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(dirname.GetClearFilename()));
@@ -1277 +1351 @@
 //
 // --------------------------------------------------------------------------
-void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, int64_t ObjectID, const std::string &rFilename)
+void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(
+        SyncParams &rParams,
+        BackupStoreDirectory* pDirOnStore,
+        BackupStoreDirectory::Entry* pEntry,
+        const std::string &rFilename)
 {
         // First, delete the directory
         BackupProtocolClient &connection(rParams.mrContext.GetConnection());
-        connection.QueryDeleteDirectory(ObjectID);
+        connection.QueryDeleteDirectory(pEntry->GetObjectID());
+
+        BackupStoreFilenameClear clear(pEntry->GetName());
+        rParams.mrContext.GetProgressNotifier().NotifyDirectoryDeleted(
+                pEntry->GetObjectID(), clear.GetClearFilename());
 
         // Then, delete any directory record
-        std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(rFilename));
+        std::map<std::string, BackupClientDirectoryRecord *>::iterator
+                e(mSubDirectories.find(rFilename));
+
         if(e != mSubDirectories.end())
         {
@@ -1301 +1385 @@
 //
 // Function
-//              Name:    BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &, const std::string &, const BackupStoreFilename &, int64_t, box_time_t, box_time_t, bool)
-//              Purpose: Private. Upload a file to the server -- may send a patch instead of the whole thing
+//              Name:    BackupClientDirectoryRecord::UploadFile(
+//                       BackupClientDirectoryRecord::SyncParams &,
+//                       const std::string &,
+//                       const BackupStoreFilename &,
+//                       int64_t, box_time_t, box_time_t, bool)
+//              Purpose: Private. Upload a file to the server. May send
+//                       a patch instead of the whole thing
 //              Created: 20/1/04
 //
 // --------------------------------------------------------------------------
-int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &rParams, const std::string &rFilename, const BackupStoreFilename &rStoreFilename,
-                        int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer)
+int64_t BackupClientDirectoryRecord::UploadFile(
+        BackupClientDirectoryRecord::SyncParams &rParams,
+        const std::string &rFilename,
+        const BackupStoreFilename &rStoreFilename,
+        int64_t FileSize,
+        box_time_t ModificationTime,
+        box_time_t AttributesHash,
+        bool NoPreviousVersionOnServer)
 {
+        BackupClientContext& rContext(rParams.mrContext);
+        ProgressNotifier& rNotifier(rContext.GetProgressNotifier());
+
         // Get the connection
-        BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+        BackupProtocolClient &connection(rContext.GetConnection());
 
         // Info
@@ -1319 +1417 @@
         try
         {
-                // Might an old version be on the server, and is the file size over the diffing threshold?
-                if(!NoPreviousVersionOnServer && FileSize >= rParams.mDiffingUploadSizeThreshold)
+                // Might an old version be on the server, and is the file
+                // size over the diffing threshold?
+                if(!NoPreviousVersionOnServer &&
+                        FileSize >= rParams.mDiffingUploadSizeThreshold)
                 {
                         // YES -- try to do diff, if possible
@@ -1330 +1430 @@
                         {
                                 // Found an old version
-                                rParams.GetProgressNotifier().NotifyFileUploadingPatch(this, 
+                                rNotifier.NotifyFileUploadingPatch(this, 
                                         rFilename);
 
@@ -1340 +1440 @@
                                 //
 
-                                rParams.mrContext.ManageDiffProcess();
+                                rContext.ManageDiffProcess();
 
                                 bool isCompletelyDifferent = false;
@@ -1349 +1449 @@
                                                 rStoreFilename, diffFromID, *blockIndexStream,
                                                 connection.GetTimeout(), 
-                                                &rParams.mrContext, // DiffTimer implementation
+                                                &rContext, // DiffTimer implementation
                                                 0 /* not interested in the modification time */, 
                                                 &isCompletelyDifferent));
         
-                                rParams.mrContext.UnManageDiffProcess();
+                                rContext.UnManageDiffProcess();
 
                                 //
@@ -1361 +1461 @@
                                                 AttributesHash, isCompletelyDifferent?(0):(diffFromID), rStoreFilename, *patchStream));
                                 
+                                // Get object ID from the result                
+                                objID = stored->GetObjectID();
+
                                 // Don't attempt to upload it again!
                                 doNormalUpload = false;
@@ -1369 +1472 @@
                 {
                         // below threshold or nothing to diff from, so upload whole
-                        rParams.GetProgressNotifier().NotifyFileUploading(this, 
-                                rFilename);
+                        rNotifier.NotifyFileUploading(this, rFilename);
                         
                         // Prepare to upload, getting a stream which will encode the file as we go along
@@ -1391 +1493 @@
         catch(BoxException &e)
         {
-                rParams.mrContext.UnManageDiffProcess();
+                rContext.UnManageDiffProcess();
 
                 if(e.GetType() == ConnectionException::ExceptionType && e.GetSubType() == ConnectionException::Protocol_UnexpectedReply)
@@ -1405 +1507 @@
                                         // The hard limit was exceeded on the server, notify!
                                         rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull);
-                                }
-                                rParams.GetProgressNotifier()
-                                        .NotifyFileUploadServerError(
-                                                this, rFilename, type, subtype);
+                                        // return an error code instead of
+                                        // throwing an exception that we
+                                        // can't debug.
+                                        return 0;
+                                }
+                                rNotifier.NotifyFileUploadServerError(this,
+                                        rFilename, type, subtype);
                         }
                 }
@@ -1416 +1521 @@
         }
 
-        rParams.GetProgressNotifier().NotifyFileUploaded(this, rFilename, FileSize);
+        rNotifier.NotifyFileUploaded(this, rFilename, FileSize);
 
         // Return the new object ID of this file
@@ -1458 +1563 @@
 // --------------------------------------------------------------------------
 BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, 
-        ProgressNotifier &rProgressNotifier, BackupClientContext &rContext)
-        : mrProgressNotifier(rProgressNotifier),
-          mSyncPeriodStart(0),
+        BackupClientContext &rContext)
+        : mSyncPeriodStart(0),
           mSyncPeriodEnd(0),
           mMaxUploadWait(0),

box/trunk/bin/bbackupd/BackupClientDirectoryRecord.h

@@ -97 +97 @@
                 const std::string& rLocalPath,
                 int64_t FileSize) = 0;
+        virtual void NotifyDirectoryDeleted(
+                int64_t ObjectID,
+                const std::string& rRemotePath) = 0;
+        virtual void NotifyFileDeleted(
+                int64_t ObjectID,
+                const std::string& rRemotePath) = 0;
 };
 
@@ -138 +144 @@
                 SyncParams(
                         BackupDaemon &rDaemon,
-                        ProgressNotifier &rProgressNotifier,
                         BackupClientContext &rContext);
                 ~SyncParams();
@@ -145 +150 @@
                 SyncParams(const SyncParams&);
                 SyncParams &operator=(const SyncParams&);
-                ProgressNotifier &mrProgressNotifier;
                 
         public:
@@ -162 +166 @@
                 box_time_t mUploadAfterThisTimeInTheFuture;
                 bool mHaveLoggedWarningAboutFutureFileTimes;
-        
-                ProgressNotifier& GetProgressNotifier() const 
-                { 
-                        return mrProgressNotifier;
-                }
         };
 
-        void SyncDirectory(SyncParams &rParams, int64_t ContainingDirectoryID, const std::string &rLocalPath,
+        void SyncDirectory(SyncParams &rParams,
+                int64_t ContainingDirectoryID,
+                const std::string &rLocalPath,
+                const std::string &rRemotePath,
                 bool ThisDirHasJustBeenCreated = false);
 
@@ -175 +177 @@
         void DeleteSubDirectories();
         BackupStoreDirectory *FetchDirectoryListing(SyncParams &rParams);
-        void UpdateAttributes(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, const std::string &rLocalPath);
-        bool UpdateItems(SyncParams &rParams, const std::string &rLocalPath, BackupStoreDirectory *pDirOnStore,
+        void UpdateAttributes(SyncParams &rParams,
+                BackupStoreDirectory *pDirOnStore,
+                const std::string &rLocalPath);
+        bool UpdateItems(SyncParams &rParams, const std::string &rLocalPath,
+                const std::string &rRemotePath,
+                BackupStoreDirectory *pDirOnStore,
                 std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver,
-                std::vector<std::string> &rFiles, const std::vector<std::string> &rDirs);
-        int64_t UploadFile(SyncParams &rParams, const std::string &rFilename, const BackupStoreFilename &rStoreFilename,
-                int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer);
-        void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, const char *Filename);
-        void RemoveDirectoryInPlaceOfFile(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, int64_t ObjectID, const std::string &rFilename);
+                std::vector<std::string> &rFiles,
+                const std::vector<std::string> &rDirs);
+        int64_t UploadFile(SyncParams &rParams,
+                const std::string &rFilename,
+                const BackupStoreFilename &rStoreFilename,
+                int64_t FileSize, box_time_t ModificationTime,
+                box_time_t AttributesHash, bool NoPreviousVersionOnServer);
+        void SetErrorWhenReadingFilesystemObject(SyncParams &rParams,
+                const char *Filename);
+        void RemoveDirectoryInPlaceOfFile(SyncParams &rParams,
+                BackupStoreDirectory* pDirOnStore,
+                BackupStoreDirectory::Entry* pEntry,
+                const std::string &rFilename);
 
 private:
-        int64_t                 mObjectID;
+        int64_t         mObjectID;
         std::string     mSubDirName;
-        bool                    mInitialSyncDone;
-        bool                    mSyncDone;
+        bool            mInitialSyncDone;
+        bool            mSyncDone;
 
         // Checksum of directory contents and attributes, used to detect changes
         uint8_t mStateChecksum[MD5Digest::DigestLength];
 
-        std::map<std::string, box_time_t>                                               *mpPendingEntries;
-        std::map<std::string, BackupClientDirectoryRecord *>    mSubDirectories;
+        std::map<std::string, box_time_t> *mpPendingEntries;
+        std::map<std::string, BackupClientDirectoryRecord *> mSubDirectories;
         // mpPendingEntries is a pointer rather than simple a member
-        // variables, because most of the time it'll be empty. This would waste a lot
-        // of memory because of STL allocation policies.
+        // variable, because most of the time it'll be empty. This would
+        // waste a lot of memory because of STL allocation policies.
 };
 

box/trunk/bin/bbackupd/BackupDaemon.cpp

@@ -48 +48 @@
 
 #include "SSLLib.h"
-#include "TLSContext.h"
 
 #include "BackupDaemon.h"
@@ -125 +124 @@
           mpCommandSocketInfo(0),
           mDeleteUnusedRootDirEntriesAfter(0),
+          mClientStoreMarker(BackupClientContext::ClientStoreMarker_NotKnown),
+          mStorageLimitExceeded(false),
+          mReadErrorsOnFilesystemObjects(false),
+          mLastSyncTime(0),
           mLogAllFileAccess(false)
         #ifdef WIN32
@@ -295 +298 @@
         mLocations.clear();
         
-        // And delete everything from the assoicated mount vector
+        // And delete everything from the associated mount vector
         mIDMapMounts.clear();
 }
@@ -686 +689 @@
 }
 
-// --------------------------------------------------------------------------
-//
-// Function
-//              Name:    BackupDaemon::Run2()
-//              Purpose: Run function for daemon (second stage)
-//              Created: 2003/10/08
-//
-// --------------------------------------------------------------------------
-void BackupDaemon::Run2()
+void BackupDaemon::InitCrypto()
 {
         // Read in the certificates creating a TLS context
-        TLSContext tlsContext;
         const Configuration &conf(GetConfiguration());
         std::string certFile(conf.GetKeyValue("CertificateFile"));
         std::string keyFile(conf.GetKeyValue("PrivateKeyFile"));
         std::string caFile(conf.GetKeyValue("TrustedCAsFile"));
-        tlsContext.Initialise(false /* as client */, certFile.c_str(), keyFile.c_str(), caFile.c_str());
+        mTlsContext.Initialise(false /* as client */, certFile.c_str(),
+                keyFile.c_str(), caFile.c_str());
         
         // Set up the keys for various things
         BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str());
-
-        // Setup various timings
-        int maximumDiffingTime = 600;
-        int keepAliveTime = 60;
-
-        // max diffing time, keep-alive time
-        if(conf.KeyExists("MaximumDiffingTime"))
-        {
-                maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime");
-        }
-        if(conf.KeyExists("KeepAliveTime"))
-        {
-                keepAliveTime = conf.GetKeyValueInt("KeepAliveTime");
-        }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    BackupDaemon::Run2()
+//              Purpose: Run function for daemon (second stage)
+//              Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::Run2()
+{
+        InitCrypto();
+
+        const Configuration &conf(GetConfiguration());
 
         // How often to connect to the store (approximate)
-        box_time_t updateStoreInterval = SecondsToBoxTime(conf.GetKeyValueInt("UpdateStoreInterval"));
+        box_time_t updateStoreInterval = SecondsToBoxTime(
+                conf.GetKeyValueInt("UpdateStoreInterval"));
 
         // But are we connecting automatically?
         bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup");
         
-        // The minimum age a file needs to be before it will be considered for uploading
-        box_time_t minimumFileAge = SecondsToBoxTime(conf.GetKeyValueInt("MinimumFileAge"));
-
-        // The maximum time we'll wait to upload a file, regardless of how often it's modified
-        box_time_t maxUploadWait = SecondsToBoxTime(conf.GetKeyValueInt("MaxUploadWait"));
-        // Adjust by subtracting the minimum file age, so is relative to sync period end in comparisons
-        maxUploadWait = (maxUploadWait > minimumFileAge)?(maxUploadWait - minimumFileAge):(0);
-
         // When the next sync should take place -- which is ASAP
         box_time_t nextSyncTime = 0;
 
         // When the last sync started (only updated if the store was not full when the sync ended)
-        box_time_t lastSyncTime = 0;
+        mLastSyncTime = 0;
 
         // --------------------------------------------------------------------------------------------
  
-        // And what's the current client store marker?
-        int64_t clientStoreMarker = 
-                BackupClientContext::ClientStoreMarker_NotKnown;
-        // haven't contacted the store yet
-
         bool deleteStoreObjectInfoFile = DeserializeStoreObjectInfo(
-                clientStoreMarker, lastSyncTime, nextSyncTime);
+                mLastSyncTime, nextSyncTime);
  
         // --------------------------------------------------------------------------------------------
@@ -756 +740 @@
         // Set state
         SetState(State_Idle);
+
+        bool doSyncForcedByPreviousSyncError = false;
 
         // Loop around doing backups
@@ -766 +752 @@
 
                 // Is a delay necessary?
-                {
-                        box_time_t currentTime;
-                        do
-                        {
-                                // Check whether we should be stopping,
-                                // and don't run a sync if so.
-                                if(StopRun()) break;
-                                
-                                currentTime = GetCurrentBoxTime();
-        
-                                // Pause a while, but no more than 
-                                // MAX_SLEEP_TIME seconds (use the conditional
-                                // because times are unsigned)
-                                box_time_t requiredDelay = 
-                                        (nextSyncTime < currentTime)
-                                        ? (0)
-                                        : (nextSyncTime - currentTime);
-
-                                // If there isn't automatic backup happening, 
-                                // set a long delay. And limit delays at the 
-                                // same time.
-                                if(!automaticBackup || requiredDelay > 
-                                        SecondsToBoxTime(MAX_SLEEP_TIME))
-                                {
-                                        requiredDelay = SecondsToBoxTime(
-                                                MAX_SLEEP_TIME);
-                                }
-
-                                // Only delay if necessary
-                                if(requiredDelay > 0)
-                                {
-                                        // Sleep somehow. There are choices 
-                                        // on how this should be done,
-                                        // depending on the state of the 
-                                        // control connection
-                                        if(mpCommandSocketInfo != 0)
-                                        {
-                                                // A command socket exists, 
-                                                // so sleep by waiting on it
-                                                WaitOnCommandSocket(
-                                                        requiredDelay, doSync, 
-                                                        doSyncForcedByCommand);
-                                        }
-                                        else
-                                        {
-                                                // No command socket or 
-                                                // connection, just do a 
-                                                // normal sleep
-                                                time_t sleepSeconds = 
-                                                        BoxTimeToSeconds(
-                                                        requiredDelay);
-                                                ::sleep((sleepSeconds <= 0)
-                                                        ? 1
-                                                        : sleepSeconds);
-                                        }
-                                }
-                                
-                        } while((!automaticBackup || (currentTime < nextSyncTime)) && !doSync && !StopRun());
-                }
+                box_time_t currentTime;
+
+                do
+                {
+                        // Check whether we should be stopping,
+                        // and don't run a sync if so.
+                        if(StopRun()) break;
+                        
+                        currentTime = GetCurrentBoxTime();
+
+                        // Pause a while, but no more than 
+                        // MAX_SLEEP_TIME seconds (use the conditional
+                        // because times are unsigned)
+                        box_time_t requiredDelay = 
+                                (nextSyncTime < currentTime)
+                                ? (0)
+                                : (nextSyncTime - currentTime);
+
+                        // If there isn't automatic backup happening, 
+                        // set a long delay. And limit delays at the 
+                        // same time.
+                        if(!automaticBackup && !doSyncForcedByPreviousSyncError)
+                        {
+                                requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME);
+                        }
+                        else if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME))
+                        {
+                                requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME);
+                        }
+
+                        // Only delay if necessary
+                        if(requiredDelay > 0)
+                        {
+                                // Sleep somehow. There are choices 
+                                // on how this should be done,
+                                // depending on the state of the 
+                                // control connection
+                                if(mpCommandSocketInfo != 0)
+                                {
+                                        // A command socket exists, 
+                                        // so sleep by waiting on it
+                                        WaitOnCommandSocket(requiredDelay,
+                                                doSync, doSyncForcedByCommand);
+                                }
+                                else
+                                {
+                                        // No command socket or 
+                                        // connection, just do a 
+                                        // normal sleep
+                                        time_t sleepSeconds = 
+                                                BoxTimeToSeconds(requiredDelay);
+                                        ::sleep((sleepSeconds <= 0)
+                                                ? 1 : sleepSeconds);
+                                }
+                        }
+                        
+                        if ((automaticBackup || doSyncForcedByPreviousSyncError)
+                                && currentTime >= nextSyncTime)
+                        {
+                                doSync = true;
+                        }
+                }
+                while(!doSync && !StopRun());
 
                 // Time of sync start, and if it's time for another sync 
                 // (and we're doing automatic syncs), set the flag
                 box_time_t currentSyncStartTime = GetCurrentBoxTime();
-                if(automaticBackup && currentSyncStartTime >= nextSyncTime)
+                if((automaticBackup || doSyncForcedByPreviousSyncError) &&
+                        currentSyncStartTime >= nextSyncTime)
                 {
                         doSync = true;
@@ -860 +851 @@
                         BackupStoreFile::ResetStats();
                         
-                        // Calculate the sync period of files to examine
-                        box_time_t syncPeriodStart = lastSyncTime;
-                        box_time_t syncPeriodEnd = currentSyncStartTime - 
-                                minimumFileAge;
-
-                        if(syncPeriodStart >= syncPeriodEnd &&
-                                syncPeriodStart - syncPeriodEnd < minimumFileAge)
-                        {
-                                // This can happen if we receive a force-sync
-                                // command less than minimumFileAge after
-                                // the last sync. Deal with it by moving back
-                                // syncPeriodStart, which should not do any
-                                // damage.
-                                syncPeriodStart = syncPeriodEnd -
-                                        SecondsToBoxTime(1);
-                        }
-
-                        if(syncPeriodStart >= syncPeriodEnd)
-                        {
-                                BOX_ERROR("Invalid (negative) sync period: "
-                                        "perhaps your clock is going "
-                                        "backwards (" << syncPeriodStart <<
-                                        " to " << syncPeriodEnd << ")");
-                                THROW_EXCEPTION(ClientException,
-                                        ClockWentBackwards);
-                        }
-
-                        // Check logic
-                        ASSERT(syncPeriodEnd > syncPeriodStart);
-                        // Paranoid check on sync times
-                        if(syncPeriodStart >= syncPeriodEnd) continue;
-                        
-                        // Adjust syncPeriodEnd to emulate snapshot 
-                        // behaviour properly
-                        box_time_t syncPeriodEndExtended = syncPeriodEnd;
-                        // Using zero min file age?
-                        if(minimumFileAge == 0)
-                        {
-                                // Add a year on to the end of the end time,
-                                // to make sure we sync files which are 
-                                // modified after the scan run started.
-                                // Of course, they may be eligible to be 
-                                // synced again the next time round,
-                                // but this should be OK, because the changes 
-                                // only upload should upload no data.
-                                syncPeriodEndExtended += SecondsToBoxTime(
-                                        (time_t)(356*24*3600));
-                        }
-
                         // Delete the serialised store object file,
                         // so that we don't try to reload it after a
@@ -934 +876 @@
                         try
                         {
-                                // Set state and log start
-                                SetState(State_Connected);
-                                BOX_NOTICE("Beginning scan of local files");
-
-                                std::string extendedLogFile;
-                                if (conf.KeyExists("ExtendedLogFile"))
-                                {
-                                        extendedLogFile = conf.GetKeyValue(
-                                                "ExtendedLogFile");
-                                }
-                                
-                                if (conf.KeyExists("LogAllFileAccess"))
-                                {
-                                        mLogAllFileAccess = 
-                                                conf.GetKeyValueBool(
-                                                        "LogAllFileAccess");
-                                }
-                                
-                                // Then create a client context object (don't 
-                                // just connect, as this may be unnecessary)
-                                BackupClientContext clientContext
-                                (
-                                        *this, 
-                                        tlsContext, 
-                                        conf.GetKeyValue("StoreHostname"),
-                                        conf.GetKeyValueInt("StorePort"),
-                                        conf.GetKeyValueInt("AccountNumber"), 
-                                        conf.GetKeyValueBool("ExtendedLogging"),
-                                        conf.KeyExists("ExtendedLogFile"),
-                                        extendedLogFile
-                                );
-                                        
-                                // Set up the sync parameters
-                                BackupClientDirectoryRecord::SyncParams params(
-                                        *this, *this, clientContext);
-                                params.mSyncPeriodStart = syncPeriodStart;
-                                params.mSyncPeriodEnd = syncPeriodEndExtended;
-                                // use potentially extended end time
-                                params.mMaxUploadWait = maxUploadWait;
-                                params.mFileTrackingSizeThreshold = 
-                                        conf.GetKeyValueInt(
-                                        "FileTrackingSizeThreshold");
-                                params.mDiffingUploadSizeThreshold = 
-                                        conf.GetKeyValueInt(
-                                        "DiffingUploadSizeThreshold");
-                                params.mMaxFileTimeInFuture = 
-                                        SecondsToBoxTime(
-                                                conf.GetKeyValueInt(
-                                                        "MaxFileTimeInFuture"));
-                                mDeleteRedundantLocationsAfter =
-                                        conf.GetKeyValueInt(
-                                                "DeleteRedundantLocationsAfter");
-
-                                clientContext.SetMaximumDiffingTime(maximumDiffingTime);
-                                clientContext.SetKeepAliveTime(keepAliveTime);
-                                
-                                // Set store marker
-                                clientContext.SetClientStoreMarker(clientStoreMarker);
-                                
-                                // Set up the locations, if necessary -- 
-                                // need to do it here so we have a 
-                                // (potential) connection to use
-                                if(mLocations.empty())
-                                {
-                                        const Configuration &locations(
-                                                conf.GetSubConfiguration(
-                                                        "BackupLocations"));
-                                        
-                                        // Make sure all the directory records
-                                        // are set up
-                                        SetupLocations(clientContext, locations);
-                                }
-                                
-                                // Get some ID maps going
-                                SetupIDMapsForSync();
-
-                                // Delete any unused directories?
-                                DeleteUnusedRootDirEntries(clientContext);
-                                                                
                                 // Notify administrator
                                 NotifySysadmin(NotifyEvent_BackupStart);
 
-                                // Go through the records, syncing them
-                                for(std::vector<Location *>::const_iterator 
-                                        i(mLocations.begin()); 
-                                        i != mLocations.end(); ++i)
-                                {
-                                        // Set current and new ID map pointers
-                                        // in the context
-                                        clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], mNewIDMaps[(*i)->mIDMapIndex]);
-                                
-                                        // Set exclude lists (context doesn't
-                                        // take ownership)
-                                        clientContext.SetExcludeLists(
-                                                (*i)->mpExcludeFiles,
-                                                (*i)->mpExcludeDirs);
-
-                                        // Sync the directory
-                                        (*i)->mpDirectoryRecord->SyncDirectory(
-                                                params,
-                                                BackupProtocolClientListDirectory::RootDirectory,
-                                                (*i)->mPath);
-
-                                        // Unset exclude lists (just in case)
-                                        clientContext.SetExcludeLists(0, 0);
-                                }
-                                
+                                RunSyncNow();
+
                                 // Errors reading any files?
-                                if(params.mReadErrorsOnFilesystemObjects)
+                                if(mReadErrorsOnFilesystemObjects)
                                 {
                                         // Notify administrator
@@ -1054 +894 @@
                                 }
                                 
-                                // Perform any deletions required -- these are
-                                // delayed until the end to allow renaming to 
-                                // happen neatly.
-                                clientContext.PerformDeletions();
-
-                                // Close any open connection
-                                clientContext.CloseAnyOpenConnection();
-                                
-                                // Get the new store marker
-                                clientStoreMarker = clientContext.GetClientStoreMarker();
-                                
                                 // Check the storage limit
-                                if(clientContext.StorageLimitExceeded())
+                                if(mStorageLimitExceeded)
                                 {
                                         // Tell the sysadmin about this
@@ -1073 +902 @@
                                 else
                                 {
-                                        // The start time of the next run is
-                                        // the end time of this run.
-                                        // This is only done if the storage
-                                        // limit wasn't exceeded (as things
-                                        // won't have been done properly if
-                                        // it was)
-                                        lastSyncTime = syncPeriodEnd;
-
                                         // unflag the storage full notify flag
                                         // so that next time the store is full,
@@ -1093 +914 @@
                                         SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY);
                         
-                                // Commit the ID Maps
-                                CommitIDMapsAfterSync();
-
-                                // Log
-                                BOX_NOTICE("Finished scan of local files");
-
                                 // Notify administrator
                                 NotifySysadmin(NotifyEvent_BackupFinish);
@@ -1109 +924 @@
 
                                 deleteStoreObjectInfoFile = 
-                                        SerializeStoreObjectInfo(
-                                                clientStoreMarker, 
-                                                lastSyncTime, nextSyncTime);
+                                        SerializeStoreObjectInfo(mLastSyncTime,
+                                                nextSyncTime);
 
                                 // --------------------------------------------------------------------------------------------
+
+                                // If we were retrying after an error,
+                                // now would be a good time to stop :-)
+                                doSyncForcedByPreviousSyncError = false;
                         }
                         catch(BoxException &e)
@@ -1154 +972 @@
 
                                 // Clear state data
-                                syncPeriodStart = 0;
-                                // go back to beginning of time
-                                clientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;    // no store marker, so download everything
+                                // Go back to beginning of time
+                                mLastSyncTime = 0;
+                                mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;   // no store marker, so download everything
                                 DeleteAllLocations();
                                 DeleteAllIDMaps();
@@ -1169 +987 @@
                                 }
 
-                                // If the Berkely db files get corrupted, delete them and try again immediately
+                                // If the Berkely db files get corrupted,
+                                // delete them and try again immediately.
                                 if(isBerkelyDbFailure)
                                 {
@@ -1189 +1008 @@
                                         ::sleep(10);
                                         nextSyncTime = currentSyncStartTime + 
-                                                SecondsToBoxTime(90) +
+                                                SecondsToBoxTime(100) +
                                                 Random::RandomInt(
                                                         updateStoreInterval >> 
                                                         SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY);
+                                        doSyncForcedByPreviousSyncError = true;
                                 }
                         }
@@ -1222 +1042 @@
 }
 
+void BackupDaemon::RunSyncNow()
+{
+        // Set state and log start
+        SetState(State_Connected);
+        BOX_NOTICE("Beginning scan of local files");
+
+        const Configuration &conf(GetConfiguration());
+
+        std::string extendedLogFile;
+        if (conf.KeyExists("ExtendedLogFile"))
+        {
+                extendedLogFile = conf.GetKeyValue("ExtendedLogFile");
+        }
+        
+        if (conf.KeyExists("LogAllFileAccess"))
+        {
+                mLogAllFileAccess = conf.GetKeyValueBool("LogAllFileAccess");
+        }
+        
+        // Then create a client context object (don't 
+        // just connect, as this may be unnecessary)
+        BackupClientContext clientContext
+        (
+                *this, 
+                mTlsContext, 
+                conf.GetKeyValue("StoreHostname"),
+                conf.GetKeyValueInt("StorePort"),
+                conf.GetKeyValueInt("AccountNumber"), 
+                conf.GetKeyValueBool("ExtendedLogging"),
+                conf.KeyExists("ExtendedLogFile"),
+                extendedLogFile, *this
+        );
+                
+        // The minimum age a file needs to be before it will be
+        // considered for uploading
+        box_time_t minimumFileAge = SecondsToBoxTime(
+                conf.GetKeyValueInt("MinimumFileAge"));
+
+        // The maximum time we'll wait to upload a file, regardless
+        // of how often it's modified
+        box_time_t maxUploadWait = SecondsToBoxTime(
+                conf.GetKeyValueInt("MaxUploadWait"));
+        // Adjust by subtracting the minimum file age, so is relative
+        // to sync period end in comparisons
+        if (maxUploadWait > minimumFileAge)
+        {
+                maxUploadWait -= minimumFileAge;
+        }
+        else
+        {
+                maxUploadWait = 0;
+        }
+
+        // Calculate the sync period of files to examine
+        box_time_t syncPeriodStart = mLastSyncTime;
+        box_time_t syncPeriodEnd = GetCurrentBoxTime() - minimumFileAge;
+
+        if(syncPeriodStart >= syncPeriodEnd &&
+                syncPeriodStart - syncPeriodEnd < minimumFileAge)
+        {
+                // This can happen if we receive a force-sync
+                // command less than minimumFileAge after
+                // the last sync. Deal with it by moving back
+                // syncPeriodStart, which should not do any
+                // damage.
+                syncPeriodStart = syncPeriodEnd -
+                        SecondsToBoxTime(1);
+        }
+
+        if(syncPeriodStart >= syncPeriodEnd)
+        {
+                BOX_ERROR("Invalid (negative) sync period: "
+                        "perhaps your clock is going "
+                        "backwards (" << syncPeriodStart <<
+                        " to " << syncPeriodEnd << ")");
+                THROW_EXCEPTION(ClientException,
+                        ClockWentBackwards);
+        }
+
+        // Check logic
+        ASSERT(syncPeriodEnd > syncPeriodStart);
+        // Paranoid check on sync times
+        if(syncPeriodStart >= syncPeriodEnd) return;
+        
+        // Adjust syncPeriodEnd to emulate snapshot 
+        // behaviour properly
+        box_time_t syncPeriodEndExtended = syncPeriodEnd;
+
+        // Using zero min file age?
+        if(minimumFileAge == 0)
+        {
+                // Add a year on to the end of the end time,
+                // to make sure we sync files which are 
+                // modified after the scan run started.
+                // Of course, they may be eligible to be 
+                // synced again the next time round,
+                // but this should be OK, because the changes 
+                // only upload should upload no data.
+                syncPeriodEndExtended += SecondsToBoxTime(
+                        (time_t)(356*24*3600));
+        }
+
+        // Set up the sync parameters
+        BackupClientDirectoryRecord::SyncParams params(
+                *this, clientContext);
+        params.mSyncPeriodStart = syncPeriodStart;
+        params.mSyncPeriodEnd = syncPeriodEndExtended;
+        // use potentially extended end time
+        params.mMaxUploadWait = maxUploadWait;
+        params.mFileTrackingSizeThreshold = 
+                conf.GetKeyValueInt(
+                "FileTrackingSizeThreshold");
+        params.mDiffingUploadSizeThreshold = 
+                conf.GetKeyValueInt(
+                "DiffingUploadSizeThreshold");
+        params.mMaxFileTimeInFuture = 
+                SecondsToBoxTime(
+                        conf.GetKeyValueInt(
+                                "MaxFileTimeInFuture"));
+        mDeleteRedundantLocationsAfter =
+                conf.GetKeyValueInt(
+                        "DeleteRedundantLocationsAfter");
+        mStorageLimitExceeded = false;
+        mReadErrorsOnFilesystemObjects = false;
+
+        // Setup various timings
+        int maximumDiffingTime = 600;
+        int keepAliveTime = 60;
+
+        // max diffing time, keep-alive time
+        if(conf.KeyExists("MaximumDiffingTime"))
+        {
+                maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime");
+        }
+        if(conf.KeyExists("KeepAliveTime"))
+        {
+                keepAliveTime = conf.GetKeyValueInt("KeepAliveTime");
+        }
+
+        clientContext.SetMaximumDiffingTime(maximumDiffingTime);
+        clientContext.SetKeepAliveTime(keepAliveTime);
+        
+        // Set store marker
+        clientContext.SetClientStoreMarker(mClientStoreMarker);
+        
+        // Set up the locations, if necessary -- 
+        // need to do it here so we have a 
+        // (potential) connection to use
+        if(mLocations.empty())
+        {
+                const Configuration &locations(
+                        conf.GetSubConfiguration(
+                                "BackupLocations"));
+                
+                // Make sure all the directory records
+                // are set up
+                SetupLocations(clientContext, locations);
+        }
+        
+        // Get some ID maps going
+        SetupIDMapsForSync();
+
+        // Delete any unused directories?
+        DeleteUnusedRootDirEntries(clientContext);
+                                        
+        // Go through the records, syncing them
+        for(std::vector<Location *>::const_iterator 
+                i(mLocations.begin()); 
+                i != mLocations.end(); ++i)
+        {
+                // Set current and new ID map pointers
+                // in the context
+                clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex],
+                        mNewIDMaps[(*i)->mIDMapIndex]);
+        
+                // Set exclude lists (context doesn't
+                // take ownership)
+                clientContext.SetExcludeLists(
+                        (*i)->mpExcludeFiles,
+                        (*i)->mpExcludeDirs);
+
+                // Sync the directory
+                (*i)->mpDirectoryRecord->SyncDirectory(
+                        params,
+                        BackupProtocolClientListDirectory::RootDirectory,
+                        (*i)->mPath, std::string("/") + (*i)->mName);
+
+                // Unset exclude lists (just in case)
+                clientContext.SetExcludeLists(0, 0);
+        }
+        
+        // Errors reading any files?
+        if(params.mReadErrorsOnFilesystemObjects)
+        {
+                // Notify administrator
+                NotifySysadmin(NotifyEvent_ReadError);
+        }
+        else
+        {
+                // Unset the read error flag, so the                                            // error is reported again if it
+                // happens again
+                mNotificationsSent[NotifyEvent_ReadError] = false;
+        }
+        
+        // Perform any deletions required -- these are
+        // delayed until the end to allow renaming to 
+        // happen neatly.
+        clientContext.PerformDeletions();
+
+        // Close any open connection
+        clientContext.CloseAnyOpenConnection();
+        
+        // Get the new store marker
+        mClientStoreMarker = clientContext.GetClientStoreMarker();
+        mStorageLimitExceeded = clientContext.StorageLimitExceeded();
+        mReadErrorsOnFilesystemObjects =
+                params.mReadErrorsOnFilesystemObjects;
+
+        if(!mStorageLimitExceeded)
+        {
+                // The start time of the next run is the end time of this
+                // run. This is only done if the storage limit wasn't
+                // exceeded (as things won't have been done properly if
+                // it was)
+                mLastSyncTime = syncPeriodEnd;
+        }
+
+        // Commit the ID Maps
+        CommitIDMapsAfterSync();
+
+        // Log
+        BOX_NOTICE("Finished scan of local files");
+}
 
 // --------------------------------------------------------------------------
@@ -2461 +2514 @@
         BOX_NOTICE("Deleting unused locations from store root...");
         BackupProtocolClient &connection(rContext.GetConnection());
-        for(std::vector<std::pair<int64_t,std::string> >::iterator i(mUnusedRootDirEntries.begin()); i != mUnusedRootDirEntries.end(); ++i)
+        for(std::vector<std::pair<int64_t,std::string> >::iterator
+                i(mUnusedRootDirEntries.begin());
+                i != mUnusedRootDirEntries.end(); ++i)
         {
                 connection.QueryDeleteDirectory(i->first);
-                
-                // Log this
-                BOX_NOTICE("Deleted " << i->second << " (ID " << i->first
-                        << ") from store root");
+                rContext.GetProgressNotifier().NotifyFileDeleted(
+                        i->first, i->second);
         }
 
@@ -2738 +2791 @@
 //
 // Function
-//              Name:    BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime)
-//              Purpose: Serializes remote directory and file information into a stream of bytes, using an Archive abstraction.
-//
+//              Name:    BackupDaemon::SerializeStoreObjectInfo(
+//                       box_time_t theLastSyncTime,
+//                       box_time_t theNextSyncTime)
+//              Purpose: Serializes remote directory and file information
+//                       into a stream of bytes, using an Archive
+//                       abstraction.
 //              Created: 2005/04/11
 //
@@ -2749 +2805 @@
 static const int STOREOBJECTINFO_VERSION = 2;
 
-bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const
+bool BackupDaemon::SerializeStoreObjectInfo(box_time_t theLastSyncTime,
+        box_time_t theNextSyncTime) const
 {
         if(!GetConfiguration().KeyExists("StoreObjectInfoFile"))
@@ -2778 +2835 @@
                 anArchive.Write(STOREOBJECTINFO_VERSION);
                 anArchive.Write(GetLoadedConfigModifiedTime());
-                anArchive.Write(aClientStoreMarker);
+                anArchive.Write(mClientStoreMarker);
                 anArchive.Write(theLastSyncTime);
                 anArchive.Write(theNextSyncTime);
@@ -2830 +2887 @@
         catch(std::exception &e)
         {
-                BOX_ERROR("Internal error writing store object "
-                        "info file (" << StoreObjectInfoFile << "): "
-                        << e.what());
+                BOX_ERROR("Failed to write StoreObjectInfoFile: " <<
+                        StoreObjectInfoFile << ": " << e.what());
         }
         catch(...)
         {
-                BOX_ERROR("Internal error writing store object "
-                        "info file (" << StoreObjectInfoFile << "): "
-                        "unknown error");
+                BOX_ERROR("Failed to write StoreObjectInfoFile: " <<
+                        StoreObjectInfoFile << ": unknown error");
         }
 
@@ -2847 +2902 @@
 //
 // Function
-//              Name:    BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime)
-//              Purpose: Deserializes remote directory and file information from a stream of bytes, using an Archive abstraction.
-//
+//              Name:    BackupDaemon::DeserializeStoreObjectInfo(
+//                       box_time_t & theLastSyncTime,
+//                       box_time_t & theNextSyncTime)
+//              Purpose: Deserializes remote directory and file information
+//                       from a stream of bytes, using an Archive
+//                       abstraction.
 //              Created: 2005/04/11
 //
 // --------------------------------------------------------------------------
-bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime)
+bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime,
+        box_time_t & theNextSyncTime)
 {
         //
@@ -2945 +3004 @@
                 // this is it, go at it
                 //
-                anArchive.Read(aClientStoreMarker);
+                anArchive.Read(mClientStoreMarker);
                 anArchive.Read(theLastSyncTime);
                 anArchive.Read(theNextSyncTime);
@@ -3023 +3082 @@
         DeleteAllLocations();
 
-        aClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;
+        mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;
         theLastSyncTime = 0;
         theNextSyncTime = 0;

box/trunk/bin/bbackupd/BackupDaemon.h

@@ -15 +15 @@
 #include <memory>
 
+#include "BackupClientDirectoryRecord.h"
 #include "BoxTime.h"
 #include "Daemon.h"
-#include "BackupClientDirectoryRecord.h"
+#include "Logging.h"
 #include "Socket.h"
 #include "SocketListen.h"
 #include "SocketStream.h"
-#include "Logging.h"
+#include "TLSContext.h"
+
 #include "autogen_BackupProtocolClient.h"
 
@@ -53 +55 @@
         // methods below do partial (specialized) serialization of 
         // client state only
-        bool SerializeStoreObjectInfo(int64_t aClientStoreMarker, 
-                box_time_t theLastSyncTime, box_time_t theNextSyncTime) const;
-        bool DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, 
-                box_time_t & theLastSyncTime, box_time_t & theNextSyncTime);
+        bool SerializeStoreObjectInfo(box_time_t theLastSyncTime,
+                box_time_t theNextSyncTime) const;
+        bool DeserializeStoreObjectInfo(box_time_t & theLastSyncTime,
+                box_time_t & theNextSyncTime);
         bool DeleteStoreObjectInfo() const;
         BackupDaemon(const BackupDaemon &);
@@ -112 +114 @@
         void Run2();
 
+public:
+        void InitCrypto();
+        void RunSyncNow();
+
+private:
         void DeleteAllLocations();
         void SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf);
@@ -206 +213 @@
         std::vector<std::pair<int64_t,std::string> > mUnusedRootDirEntries;
 
+        int64_t mClientStoreMarker;
+        bool mStorageLimitExceeded;
+        bool mReadErrorsOnFilesystemObjects;
+        box_time_t mLastSyncTime;
+        TLSContext mTlsContext;
+
 public:
         bool StopRun() { return this->Daemon::StopRun(); }
+        bool StorageLimitExceeded() { return mStorageLimitExceeded; }
  
 private:
@@ -428 +442 @@
                 } 
         }
+        virtual void NotifyDirectoryDeleted(
+                int64_t ObjectID,
+                const std::string& rRemotePath)
+        {
+                if (mLogAllFileAccess)
+                {
+                        BOX_NOTICE("Deleted directory: " << rRemotePath << 
+                                " (ID " << BOX_FORMAT_OBJECTID(ObjectID) <<
+                                ")");
+                }
+        }
+        virtual void NotifyFileDeleted(
+                int64_t ObjectID,
+                const std::string& rRemotePath)
+        {
+                if (mLogAllFileAccess)
+                {
+                        BOX_NOTICE("Deleted file: " << rRemotePath << 
+                                " (ID " << BOX_FORMAT_OBJECTID(ObjectID) <<
+                                ")");
+                }
+        }
 
 #ifdef WIN32