Changeset 2176

Timestamp:

28/05/2008 15:27:21 (4 years ago)

Author:

chris

Message:

Reformat for readability.

Log removal of directories by housekeeping.

Location:

box/trunk/bin/bbstored

Files:

• HousekeepStoreAccount.cpp (modified) (17 diffs)
• HousekeepStoreAccount.h (modified) (2 diffs)

box/trunk/bin/bbstored/HousekeepStoreAccount.cpp

@@ -86 +86 @@
         // Attempt to lock the account
         std::string writeLockFilename;
-        StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFilename);
+        StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet,
+                writeLockFilename);
         NamedLock writeLock;
-        if(!writeLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */))
+        if(!writeLock.TryAndGetLock(writeLockFilename.c_str(),
+                0600 /* restrictive file permissions */))
         {
                 // Couldn't lock the account -- just stop now
@@ -95 +97 @@
 
         // Load the store info to find necessary info for the housekeeping
-        std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet, false /* Read/Write */));
+        std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(mAccountID,
+                mStoreRoot, mStoreDiscSet, false /* Read/Write */));
 
         // Calculate how much should be deleted
@@ -105 +108 @@
 
         // Scan the directory for potential things to delete
-        // This will also remove elegiable items marked with RemoveASAP
+        // This will also remove eligible items marked with RemoveASAP
         bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
 
-        // If scan directory stopped for some reason, probably parent instructed to teminate, stop now.
+        // If scan directory stopped for some reason, probably parent
+        // instructed to terminate, stop now.
         if(!continueHousekeeping)
         {
-                // If any files were marked "delete now", then update the size of the store.
-                if(mBlocksUsedDelta != 0 || mBlocksInOldFilesDelta != 0 || mBlocksInDeletedFilesDelta != 0)
+                // If any files were marked "delete now", then update
+                // the size of the store.
+                if(mBlocksUsedDelta != 0 || 
+                        mBlocksInOldFilesDelta != 0 ||
+                        mBlocksInDeletedFilesDelta != 0)
                 {
                         info->ChangeBlocksUsed(mBlocksUsedDelta);
@@ -125 +132 @@
         }
 
-        // Log any difference in opinion between the values recorded in the store info, and
-        // the values just calculated for space usage.
+        // Log any difference in opinion between the values recorded in
+        // the store info, and the values just calculated for space usage.
         // BLOCK
         {
@@ -134 +141 @@
                 int64_t usedDirectories = info->GetBlocksInDirectories();
 
-                // If the counts were wrong, taking into account RemoveASAP items deleted, log a message
-                if((used + mBlocksUsedDelta) != mBlocksUsed || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
-                        || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles || usedDirectories != mBlocksInDirectories)
+                // If the counts were wrong, taking into account RemoveASAP 
+                // items deleted, log a message
+                if((used + mBlocksUsedDelta) != mBlocksUsed 
+                        || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
+                        || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles 
+                        || usedDirectories != mBlocksInDirectories)
                 {
                         // Log this
@@ -154 +164 @@
                 
                 // If the current values don't match, store them
-                if(used != mBlocksUsed || usedOld != mBlocksInOldFiles
-                        || usedDeleted != mBlocksInDeletedFiles || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
+                if(used != mBlocksUsed 
+                        || usedOld != mBlocksInOldFiles
+                        || usedDeleted != mBlocksInDeletedFiles 
+                        || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
                 {       
                         // Set corrected values in store info
-                        info->CorrectAllUsedValues(mBlocksUsed, mBlocksInOldFiles, mBlocksInDeletedFiles, mBlocksInDirectories + mBlocksInDirectoriesDelta);
+                        info->CorrectAllUsedValues(mBlocksUsed,
+                                mBlocksInOldFiles, mBlocksInDeletedFiles,
+                                mBlocksInDirectories + mBlocksInDirectoriesDelta);
                         info->Save();
                 }
         }
         
-        // Reset the delta counts for files, as they will include RemoveASAP flagged files deleted
-        // during the initial scan.
-        int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;   // keep for reporting
+        // Reset the delta counts for files, as they will include 
+        // RemoveASAP flagged files deleted during the initial scan.
+
+        // keep for reporting
+        int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
+
         mBlocksUsedDelta = 0;
         mBlocksInOldFilesDelta = 0;
@@ -173 +190 @@
         bool deleteInterrupted = DeleteFiles();
         
-        // If that wasn't interrupted, remove any empty directories which are also marked as deleted in their containing directory
+        // If that wasn't interrupted, remove any empty directories which 
+        // are also marked as deleted in their containing directory
         if(!deleteInterrupted)
         {
@@ -191 +209 @@
         }
         
-        // Make sure the delta's won't cause problems if the counts are really wrong, and
-        // it wasn't fixed because the store was updated during the scan.
+        // Make sure the delta's won't cause problems if the counts are 
+        // really wrong, and it wasn't fixed because the store was 
+        // updated during the scan.
         if(mBlocksUsedDelta < (0 - info->GetBlocksUsed()))
         {
@@ -219 +238 @@
         info->Save();
         
-        // Explicity release the lock (would happen automatically on going out of scope, included for code clarity)
+        // Explicity release the lock (would happen automatically on 
+        // going out of scope, included for code clarity)
         writeLock.ReleaseLock();
 }
@@ -244 +264 @@
 // Function
 //              Name:    HousekeepStoreAccount::ScanDirectory(int64_t)
-//              Purpose: Private. Scan a directory for potenitally deleteable items, and
-//                               add them to the list. Returns true if the scan should continue.
+//              Purpose: Private. Scan a directory for potentially deleteable
+//                       items, and add them to the list. Returns true if the
+//                       scan should continue.
 //              Created: 11/12/03
 //
@@ -254 +275 @@
         if((--mCountUntilNextInterprocessMsgCheck) <= 0)
         {
-                mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+                mCountUntilNextInterprocessMsgCheck =
+                        POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+
                 // Check for having to stop
-                if(mrDaemon.CheckForInterProcessMsg(mAccountID))        // include account ID here as the specified account is locked
+                // Include account ID here as the specified account is locked
+                if(mrDaemon.CheckForInterProcessMsg(mAccountID))
                 {
                         // Need to abort now
@@ -269 +293 @@
 
         // Open it.
-        std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, objectFilename));
+        std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet,
+                objectFilename));
         
         // Add the size of the directory on disc to the size being calculated
@@ -291 +316 @@
         // BLOCK
         {
-                // Remove any files which are marked for removal as soon as they become old
-                // or deleted.
+                // Remove any files which are marked for removal as soon
+                // as they become old or deleted.
                 bool deletedSomething = false;
                 do
@@ -365 +390 @@
                                 d.mMarkNumber = en->GetMarkNumber();
                                 d.mVersionAgeWithinMark = enVersionAge;
+                                d.mIsFlagDeleted = (enFlags &
+                                        BackupStoreDirectory::Entry::Flags_Deleted)
+                                        ? true : false;
                                 
                                 // Add it to the list
@@ -542 +570 @@
                 // Delete the file
                 DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig);
+                BOX_INFO("Housekeeping removed " <<
+                        (i->mIsFlagDeleted ? "deleted" : "old") <<
+                        " file " << BOX_FORMAT_OBJECTID(i->mObjectID) <<
+                        " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory));
                 
                 // Stop if the deletion target has been matched or exceeded
@@ -787 +819 @@
                         }
 
-                        // Load up the directory to potentially delete
-                        std::string dirFilename;
-                        BackupStoreDirectory dir;
-                        int64_t dirSizeInBlocks = 0;
-                        {
-                                MakeObjectFilename(*i, dirFilename);
-                                // Check it actually exists (just in case it gets added twice to the list)
-                                if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename))
-                                {
-                                        // doesn't exist, next!
-                                        continue;
-                                }
-                                // load
-                                std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename));
-                                dirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
-                                dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
-                        }
-
-                        // Make sure this directory is actually empty
-                        if(dir.GetNumberOfEntries() != 0)
-                        {
-                                // Not actually empty, try next one
-                                continue;
-                        }
-
-                        // Candiate for deletion... open containing directory
-                        std::string containingDirFilename;
-                        BackupStoreDirectory containingDir;
-                        int64_t containingDirSizeInBlocksOrig = 0;
-                        {
-                                MakeObjectFilename(dir.GetContainerID(), containingDirFilename);
-                                std::auto_ptr<RaidFileRead> containingDirStream(RaidFileRead::Open(mStoreDiscSet, containingDirFilename));
-                                containingDirSizeInBlocksOrig = containingDirStream->GetDiscUsageInBlocks();
-                                containingDir.ReadFromStream(*containingDirStream, IOStream::TimeOutInfinite);
-                        }
-
-                        // Find the entry
-                        BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID());
-                        if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0))
-                        {
-                                // Should be deleted
-                                containingDir.DeleteEntry(dir.GetObjectID());
-
-                                // Is the containing dir now a candidate for deletion?
-                                if(containingDir.GetNumberOfEntries() == 0)
-                                {
-                                        toExamine.push_back(containingDir.GetObjectID());
-                                }
-
-                                // Write revised parent directory
-                                RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename);
-                                writeDir.Open(true /* allow overwriting */);
-                                containingDir.WriteToStream(writeDir);
-
-                                // get the disc usage (must do this before commiting it)
-                                int64_t dirSize = writeDir.GetDiscUsageInBlocks();
-
-                                // Commit directory
-                                writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
-
-                                // adjust usage counts for this directory
-                                if(dirSize > 0)
-                                {
-                                        int64_t adjust = dirSize - containingDirSizeInBlocksOrig;
-                                        mBlocksUsedDelta += adjust;
-                                        mBlocksInDirectoriesDelta += adjust;
-                                }
-
-                                // Delete the directory itself
-                                {
-                                        RaidFileWrite del(mStoreDiscSet, dirFilename);
-                                        del.Delete();
-                                }
-
-                                // And adjust usage counts for the directory that's just been deleted
-                                mBlocksUsedDelta -= dirSizeInBlocks;
-                                mBlocksInDirectoriesDelta -= dirSizeInBlocks;
-
-                                // Update count
-                                ++mEmptyDirectoriesDeleted;
-                        }
+                        DeleteEmptyDirectory(*i, toExamine);
                 }
 
@@ -880 +832 @@
 }
 
-
-
-
+void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
+        std::vector<int64_t>& rToExamine)
+{
+        // Load up the directory to potentially delete
+        std::string dirFilename;
+        BackupStoreDirectory dir;
+        int64_t dirSizeInBlocks = 0;
+
+        // BLOCK
+        {
+                MakeObjectFilename(dirId, dirFilename);
+                // Check it actually exists (just in case it gets
+                // added twice to the list)
+                if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename))
+                {
+                        // doesn't exist, next!
+                        return;
+                }
+                // load
+                std::auto_ptr<RaidFileRead> dirStream(
+                        RaidFileRead::Open(mStoreDiscSet, dirFilename));
+                dirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
+                dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+        }
+
+        // Make sure this directory is actually empty
+        if(dir.GetNumberOfEntries() != 0)
+        {
+                // Not actually empty, try next one
+                return;
+        }
+
+        // Candidate for deletion... open containing directory
+        std::string containingDirFilename;
+        BackupStoreDirectory containingDir;
+        int64_t containingDirSizeInBlocksOrig = 0;
+        {
+                MakeObjectFilename(dir.GetContainerID(), containingDirFilename);
+                std::auto_ptr<RaidFileRead> containingDirStream(
+                        RaidFileRead::Open(mStoreDiscSet,
+                                containingDirFilename));
+                containingDirSizeInBlocksOrig = 
+                        containingDirStream->GetDiscUsageInBlocks();
+                containingDir.ReadFromStream(*containingDirStream,
+                        IOStream::TimeOutInfinite);
+        }
+
+        // Find the entry
+        BackupStoreDirectory::Entry *pdirentry = 
+                containingDir.FindEntryByID(dir.GetObjectID());
+        if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0))
+        {
+                // Should be deleted
+                containingDir.DeleteEntry(dir.GetObjectID());
+
+                // Is the containing dir now a candidate for deletion?
+                if(containingDir.GetNumberOfEntries() == 0)
+                {
+                        rToExamine.push_back(containingDir.GetObjectID());
+                }
+
+                // Write revised parent directory
+                RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename);
+                writeDir.Open(true /* allow overwriting */);
+                containingDir.WriteToStream(writeDir);
+
+                // get the disc usage (must do this before commiting it)
+                int64_t dirSize = writeDir.GetDiscUsageInBlocks();
+
+                // Commit directory
+                writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+
+                // adjust usage counts for this directory
+                if(dirSize > 0)
+                {
+                        int64_t adjust = dirSize - containingDirSizeInBlocksOrig;
+                        mBlocksUsedDelta += adjust;
+                        mBlocksInDirectoriesDelta += adjust;
+                }
+
+                // Delete the directory itself
+                {
+                        RaidFileWrite del(mStoreDiscSet, dirFilename);
+                        del.Delete();
+                }
+
+                BOX_INFO("Housekeeping removed empty deleted dir " <<
+                        BOX_FORMAT_OBJECTID(dirId));
+
+                // And adjust usage counts for the directory that's
+                // just been deleted
+                mBlocksUsedDelta -= dirSizeInBlocks;
+                mBlocksInDirectoriesDelta -= dirSizeInBlocks;
+
+                // Update count
+                ++mEmptyDirectoriesDeleted;
+        }
+}
+

box/trunk/bin/bbstored/HousekeepStoreAccount.h

@@ -43 +43 @@
         bool DeleteFiles();
         bool DeleteEmptyDirectories();
+        void DeleteEmptyDirectory(int64_t dirId,
+                std::vector<int64_t>& rToExamine);
         void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks);
 
@@ -53 +55 @@
                 int32_t mMarkNumber;
                 int32_t mVersionAgeWithinMark;  // 0 == current, 1 latest old version, etc
+                bool    mIsFlagDeleted; // false for files flagged "Old"
         } DelEn;