| 1 | // -------------------------------------------------------------------------- |
|---|
| 2 | // |
|---|
| 3 | // File |
|---|
| 4 | // Name: testbackupdiff.cpp |
|---|
| 5 | // Purpose: Test diffing routines for backup store files |
|---|
| 6 | // Created: 12/1/04 |
|---|
| 7 | // |
|---|
| 8 | // -------------------------------------------------------------------------- |
|---|
| 9 | |
|---|
| 10 | #include "Box.h" |
|---|
| 11 | |
|---|
| 12 | #include <stdio.h> |
|---|
| 13 | #include <string.h> |
|---|
| 14 | |
|---|
| 15 | #include "Test.h" |
|---|
| 16 | #include "BackupClientCryptoKeys.h" |
|---|
| 17 | #include "BackupStoreFile.h" |
|---|
| 18 | #include "BackupStoreFilenameClear.h" |
|---|
| 19 | #include "FileStream.h" |
|---|
| 20 | #include "BackupStoreFileWire.h" |
|---|
| 21 | #include "BackupStoreObjectMagic.h" |
|---|
| 22 | #include "BackupStoreFileCryptVar.h" |
|---|
| 23 | #include "BackupStoreException.h" |
|---|
| 24 | #include "CollectInBufferStream.h" |
|---|
| 25 | |
|---|
| 26 | #include "MemLeakFindOn.h" |
|---|
| 27 | |
|---|
| 28 | using namespace BackupStoreFileCryptVar; |
|---|
| 29 | |
|---|
| 30 | |
|---|
| 31 | // from another file |
|---|
| 32 | void create_test_files(); |
|---|
| 33 | |
|---|
| 34 | bool files_identical(const char *file1, const char *file2) |
|---|
| 35 | { |
|---|
| 36 | FileStream f1(file1); |
|---|
| 37 | FileStream f2(file2); |
|---|
| 38 | |
|---|
| 39 | if(f1.BytesLeftToRead() != f2.BytesLeftToRead()) |
|---|
| 40 | { |
|---|
| 41 | return false; |
|---|
| 42 | } |
|---|
| 43 | |
|---|
| 44 | while(f1.StreamDataLeft()) |
|---|
| 45 | { |
|---|
| 46 | char buffer1[2048]; |
|---|
| 47 | char buffer2[2048]; |
|---|
| 48 | int s = f1.Read(buffer1, sizeof(buffer1)); |
|---|
| 49 | if(f2.Read(buffer2, s) != s) |
|---|
| 50 | { |
|---|
| 51 | return false; |
|---|
| 52 | } |
|---|
| 53 | if(::memcmp(buffer1, buffer2, s) != 0) |
|---|
| 54 | { |
|---|
| 55 | return false; |
|---|
| 56 | } |
|---|
| 57 | } |
|---|
| 58 | |
|---|
| 59 | if(f2.StreamDataLeft()) |
|---|
| 60 | { |
|---|
| 61 | return false; |
|---|
| 62 | } |
|---|
| 63 | |
|---|
| 64 | return true; |
|---|
| 65 | } |
|---|
| 66 | |
|---|
| 67 | bool make_file_of_zeros(const char *filename, size_t size) |
|---|
| 68 | { |
|---|
| 69 | #ifdef WIN32 |
|---|
| 70 | HANDLE handle = openfile(filename, O_WRONLY | O_CREAT | O_EXCL, 0); |
|---|
| 71 | TEST_THAT(handle != INVALID_HANDLE_VALUE); |
|---|
| 72 | TEST_THAT(SetFilePointer(handle, size, NULL, FILE_BEGIN) |
|---|
| 73 | != INVALID_SET_FILE_POINTER); |
|---|
| 74 | TEST_THAT(GetLastError() == NO_ERROR); |
|---|
| 75 | BOOL result = SetEndOfFile(handle); |
|---|
| 76 | if (result != TRUE) |
|---|
| 77 | { |
|---|
| 78 | BOX_ERROR("Failed to create large file " << filename << |
|---|
| 79 | " (" << (size >> 20) << " MB): " << |
|---|
| 80 | GetErrorMessage(GetLastError())); |
|---|
| 81 | } |
|---|
| 82 | TEST_THAT(result == TRUE); |
|---|
| 83 | TEST_THAT(CloseHandle(handle) == TRUE); |
|---|
| 84 | #else |
|---|
| 85 | int fd = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0600); |
|---|
| 86 | if (fd < 0) perror(filename); |
|---|
| 87 | TEST_THAT(fd >= 0); |
|---|
| 88 | TEST_THAT(ftruncate(fd, size) == 0); |
|---|
| 89 | TEST_THAT(close(fd) == 0); |
|---|
| 90 | #endif |
|---|
| 91 | |
|---|
| 92 | bool correct_size = ((size_t)TestGetFileSize(filename) == size); |
|---|
| 93 | TEST_THAT(correct_size); |
|---|
| 94 | if (!correct_size) |
|---|
| 95 | { |
|---|
| 96 | BOX_ERROR("Failed to create large file " << filename << |
|---|
| 97 | " (" << (size >> 20) << " MB): " << |
|---|
| 98 | "got " << (TestGetFileSize(filename) >> 20) << |
|---|
| 99 | " MB instead"); |
|---|
| 100 | } |
|---|
| 101 | return correct_size; |
|---|
| 102 | } |
|---|
| 103 | |
|---|
| 104 | |
|---|
| 105 | void check_encoded_file(const char *filename, int64_t OtherFileID, int new_blocks_expected, int old_blocks_expected) |
|---|
| 106 | { |
|---|
| 107 | FileStream enc(filename); |
|---|
| 108 | |
|---|
| 109 | // Use the interface verify routine |
|---|
| 110 | int64_t otherIDFromFile = 0; |
|---|
| 111 | TEST_THAT(BackupStoreFile::VerifyEncodedFileFormat(enc, &otherIDFromFile)); |
|---|
| 112 | TEST_THAT(otherIDFromFile == OtherFileID); |
|---|
| 113 | |
|---|
| 114 | // Now do our own reading |
|---|
| 115 | enc.Seek(0, IOStream::SeekType_Absolute); |
|---|
| 116 | BackupStoreFile::MoveStreamPositionToBlockIndex(enc); |
|---|
| 117 | // Read in header to check magic value is as expected |
|---|
| 118 | file_BlockIndexHeader hdr; |
|---|
| 119 | TEST_THAT(enc.ReadFullBuffer(&hdr, sizeof(hdr), 0)); |
|---|
| 120 | TEST_THAT(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)); |
|---|
| 121 | TEST_THAT((uint64_t)box_ntoh64(hdr.mOtherFileID) == (uint64_t)OtherFileID); |
|---|
| 122 | // number of blocks |
|---|
| 123 | int64_t nblocks = box_ntoh64(hdr.mNumBlocks); |
|---|
| 124 | BOX_TRACE("Reading index from '" << filename << "', has " << |
|---|
| 125 | nblocks << " blocks"); |
|---|
| 126 | BOX_TRACE("======== ===== ========== ======== ========"); |
|---|
| 127 | BOX_TRACE(" Index Where EncSz/Idx Size WChcksm"); |
|---|
| 128 | // Read them all in |
|---|
| 129 | int64_t nnew = 0, nold = 0; |
|---|
| 130 | for(int64_t b = 0; b < nblocks; ++b) |
|---|
| 131 | { |
|---|
| 132 | file_BlockIndexEntry en; |
|---|
| 133 | TEST_THAT(enc.ReadFullBuffer(&en, sizeof(en), 0)); |
|---|
| 134 | int64_t s = box_ntoh64(en.mEncodedSize); |
|---|
| 135 | |
|---|
| 136 | // Decode the rest |
|---|
| 137 | uint64_t iv = box_ntoh64(hdr.mEntryIVBase); |
|---|
| 138 | iv += b; |
|---|
| 139 | sBlowfishDecryptBlockEntry.SetIV(&iv); |
|---|
| 140 | file_BlockIndexEntryEnc entryEnc; |
|---|
| 141 | sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, |
|---|
| 142 | sizeof(entryEnc), en.mEnEnc, sizeof(en.mEnEnc)); |
|---|
| 143 | |
|---|
| 144 | |
|---|
| 145 | if(s > 0) |
|---|
| 146 | { |
|---|
| 147 | nnew++; |
|---|
| 148 | BOX_TRACE(std::setw(8) << b << " this s=" << |
|---|
| 149 | std::setw(8) << s << " " << |
|---|
| 150 | std::setw(8) << ntohl(entryEnc.mSize) << " " << |
|---|
| 151 | std::setw(8) << std::setfill('0') << |
|---|
| 152 | std::hex << ntohl(entryEnc.mWeakChecksum)); |
|---|
| 153 | } |
|---|
| 154 | else |
|---|
| 155 | { |
|---|
| 156 | nold++; |
|---|
| 157 | BOX_TRACE(std::setw(8) << b << " other i=" << |
|---|
| 158 | std::setw(8) << (0-s) << " " << |
|---|
| 159 | std::setw(8) << ntohl(entryEnc.mSize) << " " << |
|---|
| 160 | std::setw(8) << std::setfill('0') << |
|---|
| 161 | std::hex << ntohl(entryEnc.mWeakChecksum)); |
|---|
| 162 | } |
|---|
| 163 | } |
|---|
| 164 | BOX_TRACE("======== ===== ========== ======== ========"); |
|---|
| 165 | TEST_THAT(new_blocks_expected == nnew); |
|---|
| 166 | TEST_THAT(old_blocks_expected == nold); |
|---|
| 167 | } |
|---|
| 168 | |
|---|
| 169 | void test_diff(int from, int to, int new_blocks_expected, int old_blocks_expected, bool expect_completely_different = false) |
|---|
| 170 | { |
|---|
| 171 | // First, get the block index of the thing it's comparing against |
|---|
| 172 | char from_encoded[256]; |
|---|
| 173 | sprintf(from_encoded, "testfiles/f%d.encoded", from); |
|---|
| 174 | FileStream blockindex(from_encoded); |
|---|
| 175 | BackupStoreFile::MoveStreamPositionToBlockIndex(blockindex); |
|---|
| 176 | |
|---|
| 177 | // make filenames |
|---|
| 178 | char from_orig[256]; |
|---|
| 179 | sprintf(from_orig, "testfiles/f%d", from); |
|---|
| 180 | char to_encoded[256]; |
|---|
| 181 | sprintf(to_encoded, "testfiles/f%d.encoded", to); |
|---|
| 182 | char to_diff[256]; |
|---|
| 183 | sprintf(to_diff, "testfiles/f%d.diff", to); |
|---|
| 184 | char to_orig[256]; |
|---|
| 185 | sprintf(to_orig, "testfiles/f%d", to); |
|---|
| 186 | char rev_diff[256]; |
|---|
| 187 | sprintf(rev_diff, "testfiles/f%d.revdiff", to); |
|---|
| 188 | char from_rebuild[256]; |
|---|
| 189 | sprintf(from_rebuild, "testfiles/f%d.rebuilt", to); |
|---|
| 190 | char from_rebuild_dec[256]; |
|---|
| 191 | sprintf(from_rebuild_dec, "testfiles/f%d.rebuilt_dec", to); |
|---|
| 192 | |
|---|
| 193 | // Then call the encode varient for diffing files |
|---|
| 194 | bool completelyDifferent = !expect_completely_different; // oposite of what we want |
|---|
| 195 | { |
|---|
| 196 | BackupStoreFilenameClear f1name("filename"); |
|---|
| 197 | FileStream out(to_diff, O_WRONLY | O_CREAT | O_EXCL); |
|---|
| 198 | std::auto_ptr<IOStream> encoded( |
|---|
| 199 | BackupStoreFile::EncodeFileDiff( |
|---|
| 200 | to_orig, |
|---|
| 201 | 1 /* dir ID */, |
|---|
| 202 | f1name, |
|---|
| 203 | 1000 + from /* object ID of the file diffing from */, |
|---|
| 204 | blockindex, |
|---|
| 205 | IOStream::TimeOutInfinite, |
|---|
| 206 | NULL, // DiffTimer interface |
|---|
| 207 | 0, |
|---|
| 208 | &completelyDifferent)); |
|---|
| 209 | encoded->CopyStreamTo(out); |
|---|
| 210 | } |
|---|
| 211 | TEST_THAT(completelyDifferent == expect_completely_different); |
|---|
| 212 | |
|---|
| 213 | // Test that the number of blocks in the file match what's expected |
|---|
| 214 | check_encoded_file(to_diff, expect_completely_different?(0):(1000 + from), new_blocks_expected, old_blocks_expected); |
|---|
| 215 | |
|---|
| 216 | // filename |
|---|
| 217 | char to_testdec[256]; |
|---|
| 218 | sprintf(to_testdec, "testfiles/f%d.testdec", to); |
|---|
| 219 | |
|---|
| 220 | if(!completelyDifferent) |
|---|
| 221 | { |
|---|
| 222 | // Then produce a combined file |
|---|
| 223 | { |
|---|
| 224 | FileStream diff(to_diff); |
|---|
| 225 | FileStream diff2(to_diff); |
|---|
| 226 | FileStream from(from_encoded); |
|---|
| 227 | FileStream out(to_encoded, O_WRONLY | O_CREAT | O_EXCL); |
|---|
| 228 | BackupStoreFile::CombineFile(diff, diff2, from, out); |
|---|
| 229 | } |
|---|
| 230 | |
|---|
| 231 | // And check it |
|---|
| 232 | check_encoded_file(to_encoded, 0, new_blocks_expected + old_blocks_expected, 0); |
|---|
| 233 | } |
|---|
| 234 | else |
|---|
| 235 | { |
|---|
| 236 | #ifdef WIN32 |
|---|
| 237 | // Emulate the above stage! |
|---|
| 238 | char src[256], dst[256]; |
|---|
| 239 | sprintf(src, "testfiles\\f%d.diff", to); |
|---|
| 240 | sprintf(dst, "testfiles\\f%d.encoded", to); |
|---|
| 241 | TEST_THAT(CopyFile(src, dst, FALSE) != 0) |
|---|
| 242 | #else |
|---|
| 243 | // Emulate the above stage! |
|---|
| 244 | char cmd[256]; |
|---|
| 245 | sprintf(cmd, "cp testfiles/f%d.diff testfiles/f%d.encoded", to, to); |
|---|
| 246 | ::system(cmd); |
|---|
| 247 | #endif |
|---|
| 248 | } |
|---|
| 249 | |
|---|
| 250 | // Decode it |
|---|
| 251 | { |
|---|
| 252 | FileStream enc(to_encoded); |
|---|
| 253 | BackupStoreFile::DecodeFile(enc, to_testdec, IOStream::TimeOutInfinite); |
|---|
| 254 | TEST_THAT(files_identical(to_orig, to_testdec)); |
|---|
| 255 | } |
|---|
| 256 | |
|---|
| 257 | // Then do some comparisons against the block index |
|---|
| 258 | { |
|---|
| 259 | FileStream index(to_encoded); |
|---|
| 260 | BackupStoreFile::MoveStreamPositionToBlockIndex(index); |
|---|
| 261 | TEST_THAT(BackupStoreFile::CompareFileContentsAgainstBlockIndex(to_orig, index, IOStream::TimeOutInfinite) == true); |
|---|
| 262 | } |
|---|
| 263 | { |
|---|
| 264 | char from_orig[256]; |
|---|
| 265 | sprintf(from_orig, "testfiles/f%d", from); |
|---|
| 266 | FileStream index(to_encoded); |
|---|
| 267 | BackupStoreFile::MoveStreamPositionToBlockIndex(index); |
|---|
| 268 | TEST_THAT(BackupStoreFile::CompareFileContentsAgainstBlockIndex(from_orig, index, IOStream::TimeOutInfinite) == files_identical(from_orig, to_orig)); |
|---|
| 269 | } |
|---|
| 270 | |
|---|
| 271 | // Check that combined index creation works as expected |
|---|
| 272 | { |
|---|
| 273 | // Load a combined index into memory |
|---|
| 274 | FileStream diff(to_diff); |
|---|
| 275 | FileStream from(from_encoded); |
|---|
| 276 | std::auto_ptr<IOStream> indexCmbStr(BackupStoreFile::CombineFileIndices(diff, from)); |
|---|
| 277 | CollectInBufferStream indexCmb; |
|---|
| 278 | indexCmbStr->CopyStreamTo(indexCmb); |
|---|
| 279 | // Then check that it's as expected! |
|---|
| 280 | FileStream result(to_encoded); |
|---|
| 281 | BackupStoreFile::MoveStreamPositionToBlockIndex(result); |
|---|
| 282 | CollectInBufferStream index; |
|---|
| 283 | result.CopyStreamTo(index); |
|---|
| 284 | TEST_THAT(indexCmb.GetSize() == index.GetSize()); |
|---|
| 285 | TEST_THAT(::memcmp(indexCmb.GetBuffer(), index.GetBuffer(), index.GetSize()) == 0); |
|---|
| 286 | } |
|---|
| 287 | |
|---|
| 288 | // Check that reverse delta can be made, and that it decodes OK |
|---|
| 289 | { |
|---|
| 290 | // Create reverse delta |
|---|
| 291 | { |
|---|
| 292 | bool reversedCompletelyDifferent = !completelyDifferent; |
|---|
| 293 | FileStream diff(to_diff); |
|---|
| 294 | FileStream from(from_encoded); |
|---|
| 295 | FileStream from2(from_encoded); |
|---|
| 296 | FileStream reversed(rev_diff, O_WRONLY | O_CREAT); |
|---|
| 297 | BackupStoreFile::ReverseDiffFile(diff, from, from2, reversed, to, &reversedCompletelyDifferent); |
|---|
| 298 | TEST_THAT(reversedCompletelyDifferent == completelyDifferent); |
|---|
| 299 | } |
|---|
| 300 | // Use it to combine a file |
|---|
| 301 | { |
|---|
| 302 | FileStream diff(rev_diff); |
|---|
| 303 | FileStream diff2(rev_diff); |
|---|
| 304 | FileStream from(to_encoded); |
|---|
| 305 | FileStream out(from_rebuild, O_WRONLY | O_CREAT | O_EXCL); |
|---|
| 306 | BackupStoreFile::CombineFile(diff, diff2, from, out); |
|---|
| 307 | } |
|---|
| 308 | // And then confirm that this file is actually the one we want |
|---|
| 309 | { |
|---|
| 310 | FileStream enc(from_rebuild); |
|---|
| 311 | BackupStoreFile::DecodeFile(enc, from_rebuild_dec, IOStream::TimeOutInfinite); |
|---|
| 312 | TEST_THAT(files_identical(from_orig, from_rebuild_dec)); |
|---|
| 313 | } |
|---|
| 314 | // Do some extra checking |
|---|
| 315 | { |
|---|
| 316 | TEST_THAT(files_identical(from_rebuild, from_encoded)); |
|---|
| 317 | } |
|---|
| 318 | } |
|---|
| 319 | } |
|---|
| 320 | |
|---|
| 321 | void test_combined_diff(int version1, int version2, int serial) |
|---|
| 322 | { |
|---|
| 323 | char combined_file[256]; |
|---|
| 324 | char last_diff[256]; |
|---|
| 325 | sprintf(last_diff, "testfiles/f%d.diff", version1 + 1); // ie from version1 to version1 + 1 |
|---|
| 326 | |
|---|
| 327 | for(int v = version1 + 2; v <= version2; ++v) |
|---|
| 328 | { |
|---|
| 329 | FileStream diff1(last_diff); |
|---|
| 330 | char next_diff[256]; |
|---|
| 331 | sprintf(next_diff, "testfiles/f%d.diff", v); |
|---|
| 332 | FileStream diff2(next_diff); |
|---|
| 333 | FileStream diff2b(next_diff); |
|---|
| 334 | sprintf(combined_file, "testfiles/comb%d_%d.cmbdiff", version1, v); |
|---|
| 335 | FileStream out(combined_file, O_WRONLY | O_CREAT); |
|---|
| 336 | BackupStoreFile::CombineDiffs(diff1, diff2, diff2b, out); |
|---|
| 337 | strcpy(last_diff, combined_file); |
|---|
| 338 | } |
|---|
| 339 | |
|---|
| 340 | // Then do a combine on it, and check that it decodes to the right thing |
|---|
| 341 | char orig_enc[256]; |
|---|
| 342 | sprintf(orig_enc, "testfiles/f%d.encoded", version1); |
|---|
| 343 | char combined_out[256]; |
|---|
| 344 | sprintf(combined_out, "testfiles/comb%d_%d.out", version1, version2); |
|---|
| 345 | |
|---|
| 346 | { |
|---|
| 347 | FileStream diff(combined_file); |
|---|
| 348 | FileStream diff2(combined_file); |
|---|
| 349 | FileStream from(orig_enc); |
|---|
| 350 | FileStream out(combined_out, O_WRONLY | O_CREAT); |
|---|
| 351 | BackupStoreFile::CombineFile(diff, diff2, from, out); |
|---|
| 352 | } |
|---|
| 353 | |
|---|
| 354 | char combined_out_dec[256]; |
|---|
| 355 | sprintf(combined_out_dec, "testfiles/comb%d_%d_s%d.dec", version1, version2, serial); |
|---|
| 356 | char to_orig[256]; |
|---|
| 357 | sprintf(to_orig, "testfiles/f%d", version2); |
|---|
| 358 | |
|---|
| 359 | { |
|---|
| 360 | FileStream enc(combined_out); |
|---|
| 361 | BackupStoreFile::DecodeFile(enc, combined_out_dec, IOStream::TimeOutInfinite); |
|---|
| 362 | TEST_THAT(files_identical(to_orig, combined_out_dec)); |
|---|
| 363 | } |
|---|
| 364 | |
|---|
| 365 | } |
|---|
| 366 | |
|---|
| 367 | #define MAX_DIFF 9 |
|---|
| 368 | void test_combined_diffs() |
|---|
| 369 | { |
|---|
| 370 | int serial = 0; |
|---|
| 371 | |
|---|
| 372 | // Number of items to combine at once |
|---|
| 373 | for(int stages = 2; stages <= 4; ++stages) |
|---|
| 374 | { |
|---|
| 375 | // Offset to get complete coverage |
|---|
| 376 | for(int offset = 0; offset < stages; ++offset) |
|---|
| 377 | { |
|---|
| 378 | // And then actual end file number |
|---|
| 379 | for(int f = 0; f <= (MAX_DIFF - stages - offset); ++f) |
|---|
| 380 | { |
|---|
| 381 | // And finally, do something! |
|---|
| 382 | test_combined_diff(offset + f, offset + f + stages, ++serial); |
|---|
| 383 | } |
|---|
| 384 | } |
|---|
| 385 | } |
|---|
| 386 | } |
|---|
| 387 | |
|---|
| 388 | int test(int argc, const char *argv[]) |
|---|
| 389 | { |
|---|
| 390 | // Want to trace out all the details |
|---|
| 391 | #ifndef BOX_RELEASE_BUILD |
|---|
| 392 | #ifndef WIN32 |
|---|
| 393 | BackupStoreFile::TraceDetailsOfDiffProcess = true; |
|---|
| 394 | #endif |
|---|
| 395 | #endif |
|---|
| 396 | |
|---|
| 397 | // Create all the test files |
|---|
| 398 | create_test_files(); |
|---|
| 399 | |
|---|
| 400 | // Setup the crypto |
|---|
| 401 | BackupClientCryptoKeys_Setup("testfiles/backup.keys"); |
|---|
| 402 | |
|---|
| 403 | // Encode the first file |
|---|
| 404 | { |
|---|
| 405 | BackupStoreFilenameClear f0name("f0"); |
|---|
| 406 | FileStream out("testfiles/f0.encoded", O_WRONLY | O_CREAT | O_EXCL); |
|---|
| 407 | std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/f0", 1 /* dir ID */, f0name)); |
|---|
| 408 | encoded->CopyStreamTo(out); |
|---|
| 409 | out.Close(); |
|---|
| 410 | check_encoded_file("testfiles/f0.encoded", 0, 33, 0); |
|---|
| 411 | } |
|---|
| 412 | |
|---|
| 413 | // Check the "seek to index" code |
|---|
| 414 | { |
|---|
| 415 | FileStream enc("testfiles/f0.encoded"); |
|---|
| 416 | BackupStoreFile::MoveStreamPositionToBlockIndex(enc); |
|---|
| 417 | // Read in header to check magic value is as expected |
|---|
| 418 | file_BlockIndexHeader hdr; |
|---|
| 419 | TEST_THAT(enc.ReadFullBuffer(&hdr, sizeof(hdr), 0)); |
|---|
| 420 | TEST_THAT(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)); |
|---|
| 421 | } |
|---|
| 422 | |
|---|
| 423 | // Diff some files -- parameters are from number, to number, |
|---|
| 424 | // then the number of new blocks expected, and the number of old blocks expected. |
|---|
| 425 | |
|---|
| 426 | // Diff the original file to a copy of itself, and check that there is no data in the file |
|---|
| 427 | // This checks that the hash table is constructed properly, because two of the blocks share |
|---|
| 428 | // the same weak checksum. |
|---|
| 429 | test_diff(0, 1, 0, 33); |
|---|
| 430 | |
|---|
| 431 | // Insert some new data |
|---|
| 432 | // Blocks from old file moved whole, but put in different order |
|---|
| 433 | test_diff(1, 2, 7, 32); |
|---|
| 434 | |
|---|
| 435 | // Delete some data, but not on block boundaries |
|---|
| 436 | test_diff(2, 3, 1, 29); |
|---|
| 437 | |
|---|
| 438 | // Add a very small amount of data, not on block boundary |
|---|
| 439 | // delete a little data |
|---|
| 440 | test_diff(3, 4, 3, 25); |
|---|
| 441 | |
|---|
| 442 | // 1 byte insertion between two blocks |
|---|
| 443 | test_diff(4, 5, 1, 28); |
|---|
| 444 | |
|---|
| 445 | // a file with some new content at the very beginning |
|---|
| 446 | // NOTE: You might expect the last numbers to be 2, 29, but the small 1 byte block isn't searched for |
|---|
| 447 | test_diff(5, 6, 3, 28); |
|---|
| 448 | |
|---|
| 449 | // some new content at the very end |
|---|
| 450 | // NOTE: 1 byte block deleted, so number aren't what you'd initial expect. |
|---|
| 451 | test_diff(6, 7, 2, 30); |
|---|
| 452 | |
|---|
| 453 | // a completely different file, with no blocks matching. |
|---|
| 454 | test_diff(7, 8, 14, 0, true /* completely different expected */); |
|---|
| 455 | |
|---|
| 456 | // diff to zero sized file |
|---|
| 457 | test_diff(8, 9, 0, 0, true /* completely different expected */); |
|---|
| 458 | |
|---|
| 459 | // Test that combining diffs works |
|---|
| 460 | test_combined_diffs(); |
|---|
| 461 | |
|---|
| 462 | // Check zero sized file works OK to encode on its own, using normal encoding |
|---|
| 463 | { |
|---|
| 464 | { |
|---|
| 465 | // Encode |
|---|
| 466 | BackupStoreFilenameClear fn("filename"); |
|---|
| 467 | FileStream out("testfiles/f9.zerotest", O_WRONLY | O_CREAT | O_EXCL); |
|---|
| 468 | std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/f9", 1 /* dir ID */, fn)); |
|---|
| 469 | encoded->CopyStreamTo(out); |
|---|
| 470 | out.Close(); |
|---|
| 471 | check_encoded_file("testfiles/f9.zerotest", 0, 0, 0); |
|---|
| 472 | } |
|---|
| 473 | { |
|---|
| 474 | // Decode |
|---|
| 475 | FileStream enc("testfiles/f9.zerotest"); |
|---|
| 476 | BackupStoreFile::DecodeFile(enc, "testfiles/f9.testdec.zero", IOStream::TimeOutInfinite); |
|---|
| 477 | TEST_THAT(files_identical("testfiles/f9", "testfiles/f9.testdec.zero")); |
|---|
| 478 | } |
|---|
| 479 | } |
|---|
| 480 | |
|---|
| 481 | #ifndef WIN32 |
|---|
| 482 | // Check that symlinks aren't diffed |
|---|
| 483 | TEST_THAT(::symlink("f2", "testfiles/f2.symlink") == 0) |
|---|
| 484 | // And go and diff it against the previous encoded file |
|---|
| 485 | { |
|---|
| 486 | bool completelyDifferent = false; |
|---|
| 487 | { |
|---|
| 488 | FileStream blockindex("testfiles/f1.encoded"); |
|---|
| 489 | BackupStoreFile::MoveStreamPositionToBlockIndex(blockindex); |
|---|
| 490 | |
|---|
| 491 | BackupStoreFilenameClear f1name("filename"); |
|---|
| 492 | FileStream out("testfiles/f2.symlink.diff", O_WRONLY | O_CREAT | O_EXCL); |
|---|
| 493 | std::auto_ptr<IOStream> encoded( |
|---|
| 494 | BackupStoreFile::EncodeFileDiff( |
|---|
| 495 | "testfiles/f2.symlink", |
|---|
| 496 | 1 /* dir ID */, |
|---|
| 497 | f1name, |
|---|
| 498 | 1001 /* object ID of the file diffing from */, |
|---|
| 499 | blockindex, |
|---|
| 500 | IOStream::TimeOutInfinite, |
|---|
| 501 | NULL, // DiffTimer interface |
|---|
| 502 | 0, |
|---|
| 503 | &completelyDifferent)); |
|---|
| 504 | encoded->CopyStreamTo(out); |
|---|
| 505 | } |
|---|
| 506 | TEST_THAT(completelyDifferent == true); |
|---|
| 507 | check_encoded_file("testfiles/f2.symlink.diff", 0, 0, 0); |
|---|
| 508 | } |
|---|
| 509 | #endif |
|---|
| 510 | |
|---|
| 511 | // Check that diffing against a file which isn't "complete" and |
|---|
| 512 | // references another isn't allowed |
|---|
| 513 | { |
|---|
| 514 | FileStream blockindex("testfiles/f1.diff"); |
|---|
| 515 | BackupStoreFile::MoveStreamPositionToBlockIndex(blockindex); |
|---|
| 516 | |
|---|
| 517 | BackupStoreFilenameClear f1name("filename"); |
|---|
| 518 | FileStream out("testfiles/f2.testincomplete", O_WRONLY | O_CREAT | O_EXCL); |
|---|
| 519 | TEST_CHECK_THROWS(BackupStoreFile::EncodeFileDiff("testfiles/f2", 1 /* dir ID */, f1name, |
|---|
| 520 | 1001 /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite, |
|---|
| 521 | 0, 0), BackupStoreException, CannotDiffAnIncompleteStoreFile); |
|---|
| 522 | } |
|---|
| 523 | |
|---|
| 524 | // Found a nasty case where files of lots of the same thing |
|---|
| 525 | // suck up lots of processor time -- because of lots of matches |
|---|
| 526 | // found. Check this out! |
|---|
| 527 | |
|---|
| 528 | #ifdef WIN32 |
|---|
| 529 | BOX_WARNING("Testing diffing two large streams, may take a while!"); |
|---|
| 530 | ::fflush(stderr); |
|---|
| 531 | #endif |
|---|
| 532 | |
|---|
| 533 | if (!make_file_of_zeros("testfiles/zero.0", 20*1024*1024)) |
|---|
| 534 | { |
|---|
| 535 | return 1; |
|---|
| 536 | } |
|---|
| 537 | |
|---|
| 538 | if (!make_file_of_zeros("testfiles/zero.1", 200*1024*1024)) |
|---|
| 539 | { |
|---|
| 540 | remove("testfiles/zero.0"); |
|---|
| 541 | return 1; |
|---|
| 542 | } |
|---|
| 543 | |
|---|
| 544 | // Generate a first encoded file |
|---|
| 545 | { |
|---|
| 546 | BackupStoreFilenameClear f0name("zero.0"); |
|---|
| 547 | FileStream out("testfiles/zero.0.enc", O_WRONLY | O_CREAT | O_EXCL); |
|---|
| 548 | std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/zero.0", 1 /* dir ID */, f0name)); |
|---|
| 549 | encoded->CopyStreamTo(out); |
|---|
| 550 | } |
|---|
| 551 | // Then diff from it -- time how long it takes... |
|---|
| 552 | { |
|---|
| 553 | int beginTime = time(0); |
|---|
| 554 | FileStream blockindex("testfiles/zero.0.enc"); |
|---|
| 555 | BackupStoreFile::MoveStreamPositionToBlockIndex(blockindex); |
|---|
| 556 | |
|---|
| 557 | BackupStoreFilenameClear f1name("zero.1"); |
|---|
| 558 | FileStream out("testfiles/zero.1.enc", O_WRONLY | O_CREAT | O_EXCL); |
|---|
| 559 | std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFileDiff("testfiles/zero.1", 1 /* dir ID */, f1name, |
|---|
| 560 | 2000 /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite, |
|---|
| 561 | 0, 0)); |
|---|
| 562 | encoded->CopyStreamTo(out); |
|---|
| 563 | |
|---|
| 564 | printf("Time taken: %d seconds\n", (int)(time(0) - beginTime)); |
|---|
| 565 | |
|---|
| 566 | #ifdef WIN32 |
|---|
| 567 | TEST_THAT(time(0) < (beginTime + 300)); |
|---|
| 568 | #else |
|---|
| 569 | TEST_THAT(time(0) < (beginTime + 40)); |
|---|
| 570 | #endif |
|---|
| 571 | } |
|---|
| 572 | // Remove zero-files to save disk space |
|---|
| 573 | remove("testfiles/zero.0"); |
|---|
| 574 | remove("testfiles/zero.1"); |
|---|
| 575 | |
|---|
| 576 | #if 0 |
|---|
| 577 | // Code for a nasty real world example! (16Mb files, won't include them in the distribution |
|---|
| 578 | // for obvious reasons...) |
|---|
| 579 | // Generate a first encoded file |
|---|
| 580 | { |
|---|
| 581 | BackupStoreFilenameClear f0name("0000000000000000.old"); |
|---|
| 582 | FileStream out("testfiles/0000000000000000.enc.0", O_WRONLY | O_CREAT | O_EXCL); |
|---|
| 583 | std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("/Users/ben/Desktop/0000000000000000.old", 1 /* dir ID */, f0name)); |
|---|
| 584 | encoded->CopyStreamTo(out); |
|---|
| 585 | } |
|---|
| 586 | // Then diff from it -- time how long it takes... |
|---|
| 587 | { |
|---|
| 588 | int beginTime = time(0); |
|---|
| 589 | FileStream blockindex("testfiles/0000000000000000.enc.0"); |
|---|
| 590 | BackupStoreFile::MoveStreamPositionToBlockIndex(blockindex); |
|---|
| 591 | |
|---|
| 592 | BackupStoreFilenameClear f1name("0000000000000000.new"); |
|---|
| 593 | FileStream out("testfiles/0000000000000000.enc.1", O_WRONLY | O_CREAT | O_EXCL); |
|---|
| 594 | std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFileDiff("/Users/ben/Desktop/0000000000000000.new", 1 /* dir ID */, f1name, |
|---|
| 595 | 2000 /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite, |
|---|
| 596 | 0, 0)); |
|---|
| 597 | encoded->CopyStreamTo(out); |
|---|
| 598 | TEST_THAT(time(0) < (beginTime + 20)); |
|---|
| 599 | } |
|---|
| 600 | #endif // 0 |
|---|
| 601 | |
|---|
| 602 | return 0; |
|---|
| 603 | } |
|---|
| 604 | |
|---|
| 605 | |
|---|