Changeset 2663

Timestamp:

08/03/2010 22:00:55 (2 years ago)

Author:

chris

Message:

Merge [2604] [2612] [2613] [2614] [2618] [2633] [2634] [2635] [2636]
from trunk into 0.11rc7. Fix updating of changed file attributes
(creation time) on Windows, fixes #64 in 0.11rc7.

Location:

box/RELEASE/0.11rc7

Files:

• bin/bbackupd/BackupClientDirectoryRecord.cpp (modified) (2 diffs)
• bin/bbackupd/BackupDaemon.h (modified) (1 diff)
• bin/bbackupd/BackupDaemonInterface.h (modified) (1 diff)
• bin/bbackupquery/BackupQueries.cpp (modified) (5 diffs)
• bin/bbackupquery/bbackupquery.cpp (modified) (5 diffs)
• lib/backupclient/BackupClientFileAttributes.cpp (modified) (9 diffs)
• lib/backupclient/BackupClientFileAttributes.h (modified) (1 diff)
• lib/common/Box.h (modified) (1 diff)
• lib/common/FileModificationTime.cpp (copied) (copied from box/trunk/lib/common/FileModificationTime.cpp)
• lib/common/FileModificationTime.h (modified) (1 diff)
• lib/common/Logging.cpp (modified) (1 diff)
• lib/common/Logging.h (modified) (3 diffs)

box/RELEASE/0.11rc7/bin/bbackupd/BackupClientDirectoryRecord.cpp

@@ -614 +614 @@
                 // Explict decryption
                 BackupClientFileAttributes storeAttr(storeAttrEnc);
+                
                 // Compare the attributes
-                if(attr.Compare(storeAttr, true, true /* ignore both modification times */))
+                if(attr.Compare(storeAttr, true,
+                        true /* ignore both modification times */))
                 {
                         // No update necessary
@@ -1049 +1051 @@
                                 try
                                 {
+                                        rNotifier.NotifyFileUploadingAttributes(
+                                                this, filename);
+                                        
                                         // Update store
                                         BackupClientFileAttributes attr;

box/RELEASE/0.11rc7/bin/bbackupd/BackupDaemon.h

@@ -445 +445 @@
                 } 
         }
+        virtual void NotifyFileUploadingAttributes(
+                const BackupClientDirectoryRecord* pDirRecord,
+                const std::string& rLocalPath) 
+        {
+                if (mLogAllFileAccess)
+                {
+                        BOX_NOTICE("Uploading new file attributes: " << 
+                                rLocalPath);
+                } 
+        }
         virtual void NotifyFileUploaded(
                 const BackupClientDirectoryRecord* pDirRecord,

box/RELEASE/0.11rc7/bin/bbackupd/BackupDaemonInterface.h

@@ -124 +124 @@
                 const BackupClientDirectoryRecord* pDirRecord,
                 const std::string& rLocalPath) = 0;
+        virtual void NotifyFileUploadingAttributes(
+                const BackupClientDirectoryRecord* pDirRecord,
+                const std::string& rLocalPath) = 0;
         virtual void NotifyFileUploaded(
                 const BackupClientDirectoryRecord* pDirRecord,

box/RELEASE/0.11rc7/bin/bbackupquery/BackupQueries.cpp

@@ -213 +213 @@
                 { "quit", "" },
                 { "exit", "" },
-                { "list", "rodIFtTsh", },
+                { "list", "rodIFtTash", },
                 { "pwd",  "" },
                 { "cd",   "od" },
@@ -398 +398 @@
         #define LIST_OPTION_TIMES_LOCAL         't'
         #define LIST_OPTION_TIMES_UTC           'T'
+        #define LIST_OPTION_TIMES_ATTRIBS       'a'
         #define LIST_OPTION_SIZEINBLOCKS        's'
         #define LIST_OPTION_DISPLAY_HASH        'h'
@@ -436 +437 @@
 }
 
+static std::string GetTimeString(BackupStoreDirectory::Entry& en,
+        bool useLocalTime, bool showAttrModificationTimes)
+{
+        std::ostringstream out;
+        box_time_t originalTime, newAttributesTime;
+
+        // there is no attribute modification time in the directory
+        // entry, unfortunately, so we can't display it.
+        originalTime = en.GetModificationTime();
+        out << BoxTimeToISO8601String(originalTime, useLocalTime);
+
+        if(en.HasAttributes())
+        {
+                const StreamableMemBlock &storeAttr(en.GetAttributes());
+                BackupClientFileAttributes attr(storeAttr);
+                
+                box_time_t NewModificationTime, NewAttrModificationTime;
+                attr.GetModificationTimes(&NewModificationTime,
+                        &NewAttrModificationTime);
+                
+                if (showAttrModificationTimes)
+                {
+                        newAttributesTime = NewAttrModificationTime;
+                }
+                else
+                {
+                        newAttributesTime = NewModificationTime;
+                }
+                
+                if (newAttributesTime == originalTime)
+                {
+                        out << "*";
+                }
+                else
+                {
+                        out << "~" << BoxTimeToISO8601String(newAttributesTime,
+                                useLocalTime);
+                }
+        }
+        else
+        {
+                out << " ";
+        }
+        
+        return out.str();
+}
 
 // --------------------------------------------------------------------------
@@ -535 +582 @@
                 {
                         // Show UTC times...
-                        std::string time = BoxTimeToISO8601String(
-                                en->GetModificationTime(), false);
-                        printf("%s ", time.c_str());
+                        printf("%s ", GetTimeString(*en, false,
+                                opts[LIST_OPTION_TIMES_ATTRIBS]).c_str());
                 }
 
@@ -543 +589 @@
                 {
                         // Show local times...
-                        std::string time = BoxTimeToISO8601String(
-                                en->GetModificationTime(), true);
-                        printf("%s ", time.c_str());
+                        printf("%s ", GetTimeString(*en, true,
+                                opts[LIST_OPTION_TIMES_ATTRIBS]).c_str());
                 }
                 

box/RELEASE/0.11rc7/bin/bbackupquery/bbackupquery.cpp

@@ -61 +61 @@
 void PrintUsageAndExit()
 {
-        printf("Usage: bbackupquery [-q] [-w] "
+        printf("Usage: bbackupquery [-q*|v*|V|W<level>] [-w] "
 #ifdef WIN32
         "[-u] "
@@ -124 +124 @@
 
 #ifdef WIN32
-        const char* validOpts = "qvwuc:l:o:O:W:";
+        const char* validOpts = "qvVwuc:l:o:O:W:";
         bool unicodeConsole = false;
 #else
-        const char* validOpts = "qvwc:l:o:O:W:";
+        const char* validOpts = "qvVwc:l:o:O:W:";
 #endif
 
@@ -139 +139 @@
                 switch(c)
                 {
-                        case 'q':
+                case 'q':
                         {
                                 if(masterLevel == Log::NOTHING)
@@ -152 +152 @@
                         break;
 
-                        case 'v':
+                case 'v':
                         {
                                 if(masterLevel == Log::EVERYTHING)
@@ -162 +162 @@
                                 }
                                 masterLevel++;
+                        }
+                        break;
+
+                case 'V':
+                        {
+                                masterLevel = Log::EVERYTHING;
                         }
                         break;

box/RELEASE/0.11rc7/lib/backupclient/BackupClientFileAttributes.cpp

@@ -79 +79 @@
 {
         int32_t uid, gid, mode;
+        #ifdef WIN32
+        int64_t fileCreationTime;
+        #endif
 } attributeHashData;
 
@@ -222 +225 @@
 // Function
 //              Name:    BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool)
-//              Purpose: Compare, optionally ignoring the attribute modification time and/or modification time, and some data which is
-//                               irrelevant in practise (eg file generation number)
+//              Purpose: Compare, optionally ignoring the attribute
+//                       modification time and/or modification time, and some
+//                       data which is irrelevant in practise (eg file
+//                       generation number)
 //              Created: 10/12/03
 //
 // --------------------------------------------------------------------------
-bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime, bool IgnoreModTime) const
+bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr,
+        bool IgnoreAttrModTime, bool IgnoreModTime) const
 {
         EnsureClearAvailable();
@@ -235 +241 @@
         if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize())
         {
+                BOX_TRACE("Attribute Compare: Attributes objects are "
+                        "different sizes, cannot compare them: local " <<
+                        mpClearAttributes->GetSize() << " bytes, remote " <<
+                        rAttr.mpClearAttributes->GetSize() << " bytes");
                 return false;
         }
@@ -242 +252 @@
         attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
         attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer();
-        
-        if(a1->AttributeType != a2->AttributeType
-                || a1->UID != a2->UID
-                || a1->GID != a2->GID
-                || a1->UserDefinedFlags != a2->UserDefinedFlags
-                || a1->Mode != a2->Mode)
-        {
-                return false;
-        }
-        
+
+        #define COMPARE(attribute, message) \
+        if (a1->attribute != a2->attribute) \
+        { \
+                BOX_TRACE("Attribute Compare: " << message << " differ: " \
+                        "local "  << ntoh(a1->attribute) << ", " \
+                        "remote " << ntoh(a2->attribute)); \
+                return false; \
+        }
+        COMPARE(AttributeType, "Attribute types");
+        COMPARE(UID, "UIDs");
+        COMPARE(GID, "GIDs");
+        COMPARE(UserDefinedFlags, "User-defined flags");
+        COMPARE(Mode, "Modes");
+
         if(!IgnoreModTime)
         {
-                int t1 = a1->ModificationTime / 1000000;
-                int t2 = a2->ModificationTime / 1000000;
-                if(t1 != t2)
-                {
+                uint64_t t1 = box_ntoh64(a1->ModificationTime);
+                uint64_t t2 = box_ntoh64(a2->ModificationTime);
+                time_t s1 = BoxTimeToSeconds(t1);
+                time_t s2 = BoxTimeToSeconds(t2);
+                if(s1 != s2)
+                {
+                        BOX_TRACE("Attribute Compare: File modification "
+                                "times differ: local " <<
+                                FormatTime(t1, true) << " (" << s1 << "), "
+                                "remote " <<
+                                FormatTime(t2, true) << " (" << s2 << ")");
                         return false;
                 }
         }
-
+        
         if(!IgnoreAttrModTime)
         {
-                int t1 = a1->AttrModificationTime / 1000000;
-                int t2 = a2->AttrModificationTime / 1000000;
-                if(t1 != t2)
-                {
+                uint64_t t1 = box_ntoh64(a1->AttrModificationTime);
+                uint64_t t2 = box_ntoh64(a2->AttrModificationTime);
+                time_t s1 = BoxTimeToSeconds(t1);
+                time_t s2 = BoxTimeToSeconds(t2);
+                if(s1 != s2)
+                {
+                        BOX_TRACE("Attribute Compare: Attribute modification "
+                                "times differ: local " <<
+                                FormatTime(t1, true) << " (" << s1 << "), "
+                                "remote " <<
+                                FormatTime(t2, true) << " (" << s2 << ")");
                         return false;
                 }
@@ -277 +306 @@
         {
                 // Symlink strings don't match. This also compares xattrs
-                if(::memcmp(a1 + 1, a2 + 1, size - sizeof(attr_StreamFormat)) != 0)
-                {
+                int datalen = size - sizeof(attr_StreamFormat);
+
+                if(::memcmp(a1 + 1, a2 + 1, datalen) != 0)
+                {
+                        std::string s1((char *)(a1 + 1), datalen);
+                        std::string s2((char *)(a2 + 1), datalen);
+                        BOX_TRACE("Attribute Compare: Symbolic link target "
+                                "or extended attributes differ: "
+                                "local "  << PrintEscapedBinaryData(s1) << ", "
+                                "remote " << PrintEscapedBinaryData(s2));
                         return false;
                 }
@@ -604 +641 @@
 }
 
+// --------------------------------------------------------------------------
+//
+// Function
+//              Name:    BackupClientFileAttributes::GetModificationTimes()
+//              Purpose: Returns the modification time embedded in the
+//                       attributes.
+//              Created: 2010/02/24
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::GetModificationTimes(
+        box_time_t *pModificationTime,
+        box_time_t *pAttrModificationTime) const
+{
+        // Got something loaded
+        if(GetSize() <= 0)
+        {
+                THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+        }
+        
+        // Make sure there are clear attributes to use
+        EnsureClearAvailable();
+        ASSERT(mpClearAttributes != 0);
+
+        // Check if the decrypted attributes are small enough, and the type of attributes stored
+        if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+        {
+                THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+        }
+        int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+        ASSERT(type != 0);
+        if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX)
+        {
+                // Don't know what to do with these
+                THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+        }
+        
+        // Check there is enough space for an attributes block
+        if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat))
+        {
+                // Too small
+                THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+        }
+
+        // Get pointer to structure
+        attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+
+        if(pModificationTime)
+        {
+                *pModificationTime = box_ntoh64(pattr->ModificationTime);
+        }
+        
+        if(pAttrModificationTime)
+        {
+                *pAttrModificationTime = box_ntoh64(pattr->AttrModificationTime);
+        }
+}
 
 // --------------------------------------------------------------------------
@@ -1033 +1126 @@
 //
 // Function
-//              Name:    BackupClientFileAttributes::GenerateAttributeHash(struct stat &, const std::string &, const std::string &)
-//              Purpose: Generate a 64 bit hash from the attributes, used to detect changes.
-//                               Include filename in the hash, so that it changes from one file to another,
-//                               so don't reveal identical attributes.
+//              Name:    BackupClientFileAttributes::GenerateAttributeHash(
+//                       struct stat &, const std::string &,
+//                       const std::string &)
+//              Purpose: Generate a 64 bit hash from the attributes, used to
+//                       detect changes. Include filename in the hash, so
+//                       that it changes from one file to another, so don't
+//                       reveal identical attributes.
 //              Created: 25/4/04
 //
 // --------------------------------------------------------------------------
-uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname)
+uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st,
+        const std::string &filename, const std::string &leafname)
 {
         if(sAttributeHashSecretLength == 0)
@@ -1055 +1152 @@
         hashData.mode = htonl(st.st_mode);
 
+        #ifdef WIN32
+        // On Windows, the "file attribute modification time" is the
+        // file creation time, and we want to back this up, restore
+        // it and compare it.
+        //
+        // On other platforms, it's not very important and can't
+        // reliably be set to anything other than the current time.
+        hashData.fileCreationTime = box_hton64(st.st_ctime);
+        #endif
+
         StreamableMemBlock xattr;
         FillExtendedAttr(xattr, filename.c_str());
@@ -1063 +1170 @@
         digest.Add(xattr.GetBuffer(), xattr.GetSize());
         digest.Add(leafname.c_str(), leafname.size());
-        digest.Add(sAttributeHashSecret, sAttributeHashSecretLength);
+        digest.Add(sAttributeHashSecret, sAttributeHashSecretLength);   
         digest.Finish();
         

box/RELEASE/0.11rc7/lib/backupclient/BackupClientFileAttributes.h

@@ -48 +48 @@
         void WriteAttributes(const char *Filename, 
                 bool MakeUserWritable = false) const;
-
+        void GetModificationTimes(box_time_t *pModificationTime,
+                box_time_t *pAttrModificationTime) const;
+        
         bool IsSymLink() const;
 

box/RELEASE/0.11rc7/lib/common/Box.h

@@ -164 +164 @@
 #endif
 
+// overloaded auto-conversion functions
+inline uint64_t hton(uint64_t in) { return box_hton64(in); }
+inline uint32_t hton(uint32_t in) { return htonl(in); }
+inline uint16_t hton(uint16_t in) { return htons(in); }
+inline uint8_t  hton(uint8_t in)  { return in; }
+inline int64_t  hton(int64_t in)  { return box_hton64(in); }
+inline int32_t  hton(int32_t in)  { return htonl(in); }
+inline int16_t  hton(int16_t in)  { return htons(in); }
+inline int8_t   hton(int8_t in)   { return in; }
+inline uint64_t ntoh(uint64_t in) { return box_ntoh64(in); }
+inline uint32_t ntoh(uint32_t in) { return ntohl(in); }
+inline uint16_t ntoh(uint16_t in) { return ntohs(in); }
+inline uint8_t  ntoh(uint8_t in)  { return in; }
+inline int64_t  ntoh(int64_t in)  { return box_ntoh64(in); }
+inline int32_t  ntoh(int32_t in)  { return ntohl(in); }
+inline int16_t  ntoh(int16_t in)  { return ntohs(in); }
+inline int8_t   ntoh(int8_t in)   { return in; }
+
 #endif // BOX__H
 

box/RELEASE/0.11rc7/lib/common/FileModificationTime.h

@@ -15 +15 @@
 #include "BoxTime.h"
 
-inline box_time_t FileModificationTime(EMU_STRUCT_STAT &st)
-{
-#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC
-        box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL);
-#else
-        box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL)
-                        + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL));
-#endif
-        
-        return datamodified;
-}
-
-inline box_time_t FileAttrModificationTime(EMU_STRUCT_STAT &st)
-{
-        box_time_t statusmodified =
-#ifdef HAVE_STRUCT_STAT_ST_MTIMESPEC
-                (((int64_t)st.st_ctimespec.tv_nsec) / (NANO_SEC_IN_USEC_LL)) +
-                (((int64_t)st.st_ctimespec.tv_sec)  * (MICRO_SEC_IN_SEC_LL));
-#elif defined HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC
-                (((int64_t)st.st_ctim.tv_nsec) / (NANO_SEC_IN_USEC_LL)) +
-                (((int64_t)st.st_ctim.tv_sec)  * (MICRO_SEC_IN_SEC_LL));
-#elif defined HAVE_STRUCT_STAT_ST_ATIMENSEC
-                (((int64_t)st.st_ctimensec) / (NANO_SEC_IN_USEC_LL)) +
-                (((int64_t)st.st_ctime)     * (MICRO_SEC_IN_SEC_LL));
-#else // no nanoseconds anywhere
-                (((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL));
-#endif
-        
-        return statusmodified;
-}
-
-inline box_time_t FileModificationTimeMaxModAndAttr(EMU_STRUCT_STAT &st)
-{
-#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC
-        box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL);
-        box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL);
-#else
-        box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL)
-                        + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL));
-        box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL)
-                        + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL));
-#endif
-        
-        return (datamodified > statusmodified)?datamodified:statusmodified;
-}
+box_time_t FileModificationTime(EMU_STRUCT_STAT &st);
+box_time_t FileAttrModificationTime(EMU_STRUCT_STAT &st);
+box_time_t FileModificationTimeMaxModAndAttr(EMU_STRUCT_STAT &st);
 
 #endif // FILEMODIFICATIONTIME__H

box/RELEASE/0.11rc7/lib/common/Logging.cpp

@@ -496 +496 @@
         return true;
 }
+
+std::string PrintEscapedBinaryData(const std::string& rInput)
+{
+        std::ostringstream output;
+
+        for (size_t i = 0; i < rInput.length(); i++)
+        {
+                if (isprint(rInput[i]))
+                {
+                        output << rInput[i];
+                }
+                else
+                {
+                        output << "\\x" << std::hex << std::setw(2) <<
+                                std::setfill('0') << (int) rInput[i] <<
+                                std::dec;
+                }
+        }
+
+        return output.str();
+}

box/RELEASE/0.11rc7/lib/common/Logging.h

@@ -18 +18 @@
 
 #include "FileStream.h"
-
-/*
-#define BOX_LOG(level, stuff) \
-{ \
-    if(Log::sMaxLoggingLevelForAnyOutput >= level) \
-        std::ostringstream line; \
-        line << stuff; \
-        Log::Write(level, __FILE__, __LINE__, line.str()); \
-    } \
-}
-*/
 
 #define BOX_LOG(level, stuff) \
@@ -53 +42 @@
         { BOX_LOG(Log::TRACE, stuff) }
 
+#define BOX_SYS_ERROR(stuff) \
+        stuff << ": " << std::strerror(errno) << " (" << errno << ")"
+
 #define BOX_LOG_SYS_WARNING(stuff) \
-        BOX_WARNING(stuff << ": " << std::strerror(errno) << " (" << errno << ")")
+        BOX_WARNING(BOX_SYS_ERROR(stuff))
 #define BOX_LOG_SYS_ERROR(stuff) \
-        BOX_ERROR(stuff << ": " << std::strerror(errno) << " (" << errno << ")")
+        BOX_ERROR(BOX_SYS_ERROR(stuff))
 #define BOX_LOG_SYS_FATAL(stuff) \
-        BOX_FATAL(stuff << ": " << std::strerror(errno) << " (" << errno << ")")
+        BOX_FATAL(BOX_SYS_ERROR(stuff))
+
+#define LOG_AND_THROW_ERROR(message, filename, exception, subtype) \
+        BOX_LOG_SYS_ERROR(message << ": " << filename); \
+        THROW_EXCEPTION_MESSAGE(exception, subtype, \
+                BOX_SYS_ERROR(message << ": " << filename));
 
 inline std::string GetNativeErrorMessage()
@@ -340 +337 @@
 };
 
+std::string PrintEscapedBinaryData(const std::string& rInput);
+
 #endif // LOGGING__H