Ticket #45: BackupFileDiff.patch

lib/backupclient/BackupStoreFile.h

@@ -206 +206 @@
         static void ResetStats();
         static BackupStoreFileStats msStats;
         
-        // For debug
-#ifndef NDEBUG
-        static bool TraceDetailsOfDiffProcess;
-#endif
-
         // For decoding encoded files
         static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile);
 };

test/backupdiff/testbackupdiff.cpp

@@ -376 +376 @@
         // Want to trace out all the details
         #ifndef NDEBUG
         #ifndef WIN32
-        BackupStoreFile::TraceDetailsOfDiffProcess = true;
+        Logging::SetGlobalLevel(Log::TRACE);
         #endif
         #endif
 

lib/backupclient/BackupStoreFileDiff.cpp

@@ -36 +36 @@
 using namespace BackupStoreFileCryptVar;
 using namespace BackupStoreFileCreation;
 
-// By default, don't trace out details of the diff as we go along -- would fill up logs significantly.
-// But it's useful for the test.
-#ifndef NDEBUG
-        bool BackupStoreFile::TraceDetailsOfDiffProcess = false;
-#endif
-
 static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis);
-static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]);
-static void SearchForMatchingBlocks(IOStream &rFile, 
-        std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, 
-        int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES],
-        DiffTimer *pDiffTimer);
+static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, int64_t NumBlocks, DiffTimer *pDiffTimer);
 static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable);
-static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChecksum &fastSum, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber,
-BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks);
+static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChecksum *pFastSum, 
+                                                                uint8_t *pLow, int32_t LowSize, uint8_t *pHigh, int32_t HighSize, int32_t BlockSize, int64_t FileOffset,
+                                                                BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks);
 static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile);
 
 // --------------------------------------------------------------------------
@@ -166 +157 @@
         
         try
         {
-                // Find which sizes should be scanned
-                int32_t sizesToScan[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
-                FindMostUsedSizes(pindex, blocksInIndex, sizesToScan);
-                
                 // Flag for reporting to the user
                 bool completelyDifferent;
                         
@@ -184 +171 @@
                                 // Get size of file
                                 sizeOfInputFile = file.BytesLeftToRead();
                                 // Find all those lovely matching blocks
-                                SearchForMatchingBlocks(file, foundBlocks, pindex, 
-                                        blocksInIndex, sizesToScan, pDiffTimer);
+                                SearchForMatchingBlocks(file, foundBlocks, pindex, blocksInIndex, pDiffTimer);
                                 
                                 // Is it completely different?
                                 completelyDifferent = (foundBlocks.size() == 0);
@@ -361 +347 @@
         }
 }
 
-
 // --------------------------------------------------------------------------
 //
 // Function
-//              Name:    static FindMostUsedSizes(BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
-//              Purpose: Finds the most commonly used block sizes in the index
+//              Name:    static SearchForMatchingBlocks(IOStream &, std::map<int64_t, int64_t> &, BlocksAvailableEntry *, int64_t, DiffTimer *)
+//              Purpose: Find the matching blocks within the file.
 //              Created: 12/1/04
 //
 // --------------------------------------------------------------------------
-static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks,
+                                                                        BlocksAvailableEntry *pIndex, int64_t NumBlocks, DiffTimer *pDiffTimer)
 {
-        // Array for lengths
-        int64_t sizeCounts[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+        Timer maximumDiffingTime(0);
 
-        // Set arrays to lots of zeros (= unused entries)
-        for(int l = 0; l < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++l)
+        if(pDiffTimer && pDiffTimer->IsManaged())
         {
-                Sizes[l] = 0;
-                sizeCounts[l] = 0;
+                maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime());
         }
 
-        // Array for collecting sizes
-        std::map<int32_t, int64_t> foundSizes;
+        // Flag to abort the run, if too many blocks are found or the diffing
+        // timeout expires
+        bool abortSearch = false;
+        
+        // Buffers used during both phases of search
+        uint8_t *pbuffer0 = 0;
+        uint8_t *pbuffer1 = 0;
+        
+        // Track offsets that already have block matches. We don't really care
+        // if its sorted, and this actually produces a performance issue, so 
+        // see pfitBitmap for a workaround
+        std::map<int64_t, int32_t> goodnessOfFit;
         
-        // Run through blocks and make a count of the entries
-        for(int64_t b = 0; b < NumBlocks; ++b)
+        // Collect sizes that aren't found in the file at their old offset
+        std::map<int32_t, int64_t> unmatchedSizes;
+        
+        // Our arrays of block sizes during second search pass
+        int32_t scanSizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+        int64_t scanSizesCount[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+        int32_t maxScanBlockSize = 0;
+        ::memset(scanSizes, 0, (sizeof(int32_t) * BACKUP_FILE_DIFF_MAX_BLOCK_SIZES));
+        ::memset(scanSizesCount, 0, (sizeof(int64_t) * BACKUP_FILE_DIFF_MAX_BLOCK_SIZES));
+
+        // We need to keep separate rolling checksums for each block size in second search
+        RollingChecksum *rollingSums[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+        ::memset(rollingSums, 0, (sizeof(RollingChecksum *) * BACKUP_FILE_DIFF_MAX_BLOCK_SIZES));
+
+        // And a hash lookup table per block size in seacond search
+        BlocksAvailableEntry **hashTables[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+        ::memset(hashTables, 0, (sizeof(BlocksAvailableEntry **) * BACKUP_FILE_DIFF_MAX_BLOCK_SIZES));
+        
+        // We allow second search pass to short-circuit off rare blocks
+        bool scanThisSize[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+        ::memset(scanThisSize, 0, (sizeof(bool) * BACKUP_FILE_DIFF_MAX_BLOCK_SIZES));
+        
+        // During block read we have a bitmap of prefit locations to avoid std::map 
+        // performance problems
+        uint8_t *pfitBitmap = 0;
+
+        
+        // First search pass...
+        //
+        // For many files (especially large ones) most of the file is unchanged.
+        // The RollingChecksum process requires us to read every byte of the file looking for
+        // blocks that have moved. However, we can make that process more efficient by
+        // quickly rolling over areas that match a different block. We can also use
+        // this to eliminate entirely the rolling checksum for block sizes that only exist
+        // at one location in the file.
+        //
+        // Thus we start by looking for blocks that have not moved. Only if a block
+        // cannot be found at its previous location do we consider scanning for it by
+        // rolling checksum size.
+        // 
+        // This strategy has some disadvantages for files with lots of repeating content
+        // that happens to align with our block size, but the reduction in diff time
+        // for more typical files is worth it.
+        //
+        // Note also that in this pass we consider _all_ block sizes (smaller than 
+        // BACKUP_FILE_MAX_BLOCK_SIZE). Any block size, no matter how small or rare
+        // is cheap for us to find in this pass.
+        //
+        pbuffer0 = (uint8_t *)::malloc(BACKUP_FILE_MAX_BLOCK_SIZE);
+        try
         {
-                // Only if the block size is bigger than the minimum size we'll scan for
-                if(pIndex[b].mSize > BACKUP_FILE_DIFF_MIN_BLOCK_SIZE)
+                if(pbuffer0 == 0)
+                {
+                        throw std::bad_alloc();
+                }
+                
+                // We have to track file offset since the read may fail
+                int64_t fileOffset = 0;
+                
+                // Walk the blocks
+                for(int64_t b = 0; b < NumBlocks; ++b)
                 {
-                        // Find entry?
-                        std::map<int32_t, int64_t>::const_iterator f(foundSizes.find(pIndex[b].mSize));
-                        if(f != foundSizes.end())
+                        // Check diffing timeout
+                        if(maximumDiffingTime.HasExpired())
+                        {
+                                ASSERT(pDiffTimer != NULL);
+                                BOX_INFO("MaximumDiffingTime reached - suspending file diff");
+                                abortSearch = true;
+                                break;
+                        }
+                        
+                        // Send keep alive
+                        if(pDiffTimer) pDiffTimer->DoKeepAlive();
+                        
+                        // Skip blocks too large for our buffer
+                        if(pIndex[b].mSize > BACKUP_FILE_MAX_BLOCK_SIZE) {
+                                fileOffset += pIndex[b].mSize;
+                                continue;
+                        }
+
+                        // Have to guard the seek operation, it could throw. We don't know size of the 
+                        // current file, and checking is pointless anyway since there's a race.
+                        // In reality on Unix this is implemented with lseek which will mean we can
+                        // seek past the EOF, but I don't want to make assumptions about Win32.
+                        int32_t readSize = 0;
+                        try
+                        {
+                                rFile.Seek(fileOffset, IOStream::SeekType_Absolute);
+                                readSize = rFile.Read(pbuffer0, pIndex[b].mSize);
+                        }
+                        catch(BoxException &e)
+                        {
+                                if(e.GetType() != CommonException::ExceptionType || e.GetSubType() != CommonException::OSFileError)
+                                {
+                                        // Not what we expected, rethrow
+                                        throw;
+                                }
+                        }
+                        
+                        // Check for a match
+                        bool blockMatched = false;
+                        if(readSize == pIndex[b].mSize)
                         {
-                                // Increment existing entry
-                                foundSizes[pIndex[b].mSize] = foundSizes[pIndex[b].mSize] + 1;
+                                // We don't have a rolling checksum to this point, so all we can do is MD5. If you
+                                // worry this is expensive just remember that prior versions of this code
+                                // re-read the file BACKUP_FILE_DIFF_MAX_BLOCK_SIZES times and calculated
+                                // rolling checksums every time.
+                                MD5Digest strong;
+                                strong.Add(pbuffer0, pIndex[b].mSize);
+                                strong.Finish();
+                                
+                                // Do we have a match?
+                                if(strong.DigestMatches(pIndex[b].mStrongChecksum))
+                                {
+#ifndef NDEBUG
+                                        BOX_TRACE("Found unchanged block of size " << pIndex[b].mSize << " at offset " << fileOffset);
+#endif                                                  
+                                        rFoundBlocks[fileOffset] = b;
+                                        goodnessOfFit[fileOffset] = pIndex[b].mSize;
+                                        blockMatched = true;
+                                }
                         }
-                        else
+                        
+                        // We're done with the file offset, so increment now
+                        fileOffset += pIndex[b].mSize;
+                        
+                        // If the block didn't match then this is a size we'll have to scan for
+                        if(!blockMatched && (pIndex[b].mSize >= BACKUP_FILE_DIFF_MIN_BLOCK_SIZE))
                         {
-                                // New entry
-                                foundSizes[pIndex[b].mSize] = 1;
+                                // Find entry?
+                                std::map<int32_t, int64_t>::const_iterator f(unmatchedSizes.find(pIndex[b].mSize));
+                                if(f != unmatchedSizes.end())
+                                {
+                                        // Increment existing entry
+                                        unmatchedSizes[pIndex[b].mSize] = unmatchedSizes[pIndex[b].mSize] + 1;
+                                }
+                                else
+                                {
+                                        // New entry
+                                        unmatchedSizes[pIndex[b].mSize] = 1;
+                                }
                         }
                 }
+                
+                // Cleanup
+                ::free(pbuffer0);
+                pbuffer0 = 0;
         }
-        
-        // Make the block sizes
-        for(std::map<int32_t, int64_t>::const_iterator i(foundSizes.begin()); i != foundSizes.end(); ++i)
+        catch(...)
+        {
+                // Cleanup and rethrow
+                if(pbuffer0 != 0) ::free(pbuffer0);
+                throw;
+        }
+                
+        // Are we already out of time?
+        if(abortSearch)
         {
-                // Find the position of the size in the array
+#ifndef NDEBUG
+                goto dumpDiffList;
+#endif
+                return;
+        }
+        
+        
+        // Second search pass...
+        //
+        // In our second phase, having matched all unchanged blocks we now need
+        // to scan for moved blocks. This involves looping across all unmatched
+        // block sizes and using the rolling checksum to look for relocations. To keep
+        // this from being too expensive we cap at BACKUP_FILE_DIFF_MAX_BLOCK_SIZES
+        // for the number of sizes to scan. We also scan for the blocks in order of
+        // their relative probability, a block size that occurs frequently is scanned
+        // first.
+        //
+        
+        // Loop all sizes inserting higher usages into the array
+        for(std::map<int32_t, int64_t>::const_iterator i(unmatchedSizes.begin()); i != unmatchedSizes.end(); ++i)
+        {
+                // TODO: Scanning for any block size isn't cheap, and realistically in many cases
+                // it would be less expensive to upload a few thousand bytes rather than do the scan.
+                // Here would be a good place to filter block sizes that aren't worth the effort
+                // once a suitable set of heuristics is found.
+                
                 for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t)
                 {
-                        // Instead of sorting on the raw count of blocks,
-                        // take the file area covered by this block size.
-                        if(i->second * i->first > sizeCounts[t] * Sizes[t])
+                        // Instead of sorting on the raw count of blocks, take the file area covered by this
+                        // block size. This helps avoid favoring low numbers of large blocks over many
+                        // small blocks.
+                        if((i->second * i->first) > (scanSizesCount[t] * scanSizes[t]))
                         {
                                 // Then this size belong before this entry -- shuffle them up
                                 for(int s = (BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1); s >= t; --s)
                                 {
-                                        Sizes[s] = Sizes[s-1];
-                                        sizeCounts[s] = sizeCounts[s-1];
+                                        scanSizes[s] = scanSizes[s-1];
+                                        scanSizesCount[s] = scanSizesCount[s-1];
                                 }
                                 
                                 // Insert this size
-                                Sizes[t] = i->first;
-                                sizeCounts[t] = i->second;
+                                scanSizes[t] = i->first;
+                                scanSizesCount[t] = i->second;
                                 
-                                // Shouldn't do any more searching
+                                // Update max size
+                                if(maxScanBlockSize < i->first) maxScanBlockSize = i->first;
+                                
+                                // Shouldn't do any more searching for this size
                                 break;
                         }
                 }
         }
-        
-        // trace the size table in debug builds
+        // Dump the size table in debug builds
 #ifndef NDEBUG
-        if(BackupStoreFile::TraceDetailsOfDiffProcess)
+        for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t)
         {
-                for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t)
-                {
-                        TRACE3("Diff block size %d: %d (count = %lld)\n", t, Sizes[t], sizeCounts[t]);
-                }
+                if(scanSizes[t] != 0) BOX_TRACE("Scan block size " << t << ": " << scanSizes[t] << " count: " << scanSizesCount[t]);
         }
 #endif
-}
-
-
-
-// --------------------------------------------------------------------------
-//
-// Function
-//              Name:    static SearchForMatchingBlocks(IOStream &, std::map<int64_t, int64_t> &, BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
-//              Purpose: Find the matching blocks within the file.
-//              Created: 12/1/04
-//
-// --------------------------------------------------------------------------
-static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks,
-        BlocksAvailableEntry *pIndex, int64_t NumBlocks, 
-        int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], DiffTimer *pDiffTimer)
-{
-        Timer maximumDiffingTime(0);
 
-        if(pDiffTimer && pDiffTimer->IsManaged())
+        // If we didn't find any sizes (could happen if all were found in the first matching
+        // phase) we're done.
+        if(maxScanBlockSize == 0)
         {
-                maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime());
+#ifndef NDEBUG
+                BOX_TRACE("No scan block sizes, skip rolling checksum");
+                goto dumpDiffList;
+#endif
+                return;
         }
+        ASSERT(maxScanBlockSize <= BACKUP_FILE_MAX_BLOCK_SIZE);
         
-        std::map<int64_t, int32_t> goodnessOfFit;
-
-        // Allocate the hash lookup table
-        BlocksAvailableEntry **phashTable = (BlocksAvailableEntry **)::malloc(sizeof(BlocksAvailableEntry *) * (64*1024));
-
-        // Choose a size for the buffer, just a little bit more than the maximum block size
-        int32_t bufSize = Sizes[0];
-        for(int z = 1; z < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++z)
-        {
-                if(Sizes[z] > bufSize) bufSize = Sizes[z];
-        }
-        bufSize += 4;
-        ASSERT(bufSize > Sizes[0]);
-        ASSERT(bufSize > 0);
-        if(bufSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
-        {
-                THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
-        }
+        // Allocate two buffers we'll toggle between at the max scan block size
+        // There sizes are doubled to make final block at the end easier (we're cheating)
+        pbuffer0 = (uint8_t *)::malloc(maxScanBlockSize * 2);
+        pbuffer1 = (uint8_t *)::malloc(maxScanBlockSize * 2);
+        
+        // Allocate a bitmap buffer to optimize goodnessOfFit access
+        pfitBitmap = (uint8_t *)::malloc(maxScanBlockSize * 2 / 8);
         
-        // TODO: Because we read in the file a scanned block size at a time,
-        // it is likely to be inefficient. Probably will be much better to
-        // calculate checksums for all block sizes in a single pass.
-
-        // Allocate the buffers.
-        uint8_t *pbuffer0 = (uint8_t *)::malloc(bufSize);
-        uint8_t *pbuffer1 = (uint8_t *)::malloc(bufSize);
         try
         {
                 // Check buffer allocation
-                if(pbuffer0 == 0 || pbuffer1 == 0 || phashTable == 0)
+                if(pbuffer0 == 0 || pbuffer1 == 0 || pfitBitmap == 0)
                 {
                         // If a buffer got allocated, it will be cleaned up in the catch block
                         throw std::bad_alloc();
                 }
                 
-                // Flag to abort the run, if too many blocks are found -- avoid using
-                // huge amounts of processor time when files contain many similar blocks.
-                bool abortSearch = false;
-                
-                // Search for each block size in turn
-                // NOTE: Do the smallest size first, so that the scheme for adding
-                // entries in the found list works as expected and replaces smallers block
-                // with larger blocks when it finds matches at the same offset in the file.
-                for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s)
-                {
-                        ASSERT(Sizes[s] <= bufSize);
-                        BOX_TRACE("Diff pass " << s << ", for block size " <<
-                                Sizes[s]);
-                        
-                        // Check we haven't finished
-                        if(Sizes[s] == 0)
-                        {
-                                // empty entry, try next size
-                                continue;
-                        }
-                        
-                        // Set up the hash table entries
-                        SetupHashTable(pIndex, NumBlocks, Sizes[s], phashTable);
+                // Shift file position back to beginning
+                int64_t bufferFileOffset = 0;
+                rFile.Seek(0, IOStream::SeekType_Absolute);
                 
-                        // Shift file position to beginning
-                        rFile.Seek(0, IOStream::SeekType_Absolute);
-                        
-                        // Read first block
-                        if(rFile.Read(pbuffer0, Sizes[s]) != Sizes[s])
+                // We're going to be flipping back and forth between two buffers, the low and high
+                uint8_t *lowBuffer = pbuffer0;
+                int32_t lowBufferBytes = 0;
+                uint8_t *highBuffer = pbuffer1;
+                int32_t highBufferBytes = 0;
+
+                // In some cases we need to carry over reads from prior buffers
+                int32_t carryOverBytes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+                ::memset(carryOverBytes, 0, (sizeof(int32_t) * BACKUP_FILE_DIFF_MAX_BLOCK_SIZES));
+                
+                // Read the first buffer's worth of data
+                lowBufferBytes = rFile.Read(lowBuffer, maxScanBlockSize);
+                // Fill the second buffer if appropriate
+                if(lowBufferBytes == maxScanBlockSize)
+                {
+                        highBufferBytes = rFile.Read(highBuffer, maxScanBlockSize);
+                }
+                
+                // For every block size, initialize our scan tracking
+                for(int z = 0; z < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++z)
+                {
+                        ASSERT(scanSizes[z] <= maxScanBlockSize);
+
+                        // The sizes array may be mostly empty, in those cases we have no
+                        // state to maintain.
+                        if(scanSizes[z] != 0)
                         {
-                                // Size of file too short to match -- do next size
-                                continue;
-                        }
-                        
-                        // Setup block pointers
-                        uint8_t *beginnings = pbuffer0;
-                        uint8_t *endings = pbuffer1;
-                        int offset = 0;
-                        
-                        // Calculate the first checksum, ready for rolling
-                        RollingChecksum rolling(beginnings, Sizes[s]);
-                        
-                        // Then roll, until the file is exhausted
-                        int64_t fileBlockNumber = 0;
-                        int64_t fileOffset = 0;
-                        int rollOverInitialBytes = 0;
-                        while(true)
-                        {
-                                if(maximumDiffingTime.HasExpired())
-                                {
-                                        ASSERT(pDiffTimer != NULL);
-                                        BOX_INFO("MaximumDiffingTime reached - "
-                                                "suspending file diff");
-                                        abortSearch = true;
-                                        break;
-                                }
+                                // Mark for scan
+                                scanThisSize[z] = true;
                                 
-                                if(pDiffTimer)
+                                // Set up the hash table for this size
+                                hashTables[z] = (BlocksAvailableEntry **)::malloc(sizeof(BlocksAvailableEntry *) * (64*1024));
+                                if(hashTables[z] == 0)
                                 {
-                                        pDiffTimer->DoKeepAlive();
+                                        throw std::bad_alloc();
                                 }
-                                
-                                // Load in another block of data, and record how big it is
-                                int bytesInEndings = rFile.Read(endings, Sizes[s]);
-                                int tmp;
+                                SetupHashTable(pIndex, NumBlocks, scanSizes[z], hashTables[z]);
 
-                                // Skip any bytes from a previous matched block
-                                if(rollOverInitialBytes > 0 && offset < bytesInEndings)
+                                // Set up a rolling checksum, but only if there is enough data to start with
+                                // (file may now be shorter than some block sizes previously used)
+                                if(scanSizes[z] <= lowBufferBytes)
                                 {
-                                        int spaceLeft = bytesInEndings - offset;
-                                        int thisRoll = (rollOverInitialBytes > spaceLeft) ? spaceLeft : rollOverInitialBytes;
+                                        rollingSums[z] = new RollingChecksum(lowBuffer, scanSizes[z]);
+                                }
+                        }
+                }
+                
+                // Read loop while we can get full maxScanBlockSize reads
+                while(highBufferBytes == maxScanBlockSize)
+                {
+                        // Oh happy day! We have maxScanBlockSize * 2 bytes available to us across
+                        // the two buffers, which means we can walk every block size across all offsets
+                        // in lowBuffer without any concern about overrunning the data available in highBuffer.
 
-                                        rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll);
+                        // Check diffing timeout