Context Navigation

testbackupstore.cpp @ 2983

Revision 2983, 75.7 KB checked in by chris, 9 months ago (diff)

Combine client and server protocols to make way for an offline/local protocol.

Property svn:eol-style set to native

Line
1	// --------------------------------------------------------------------------
2	//
3	// File
4	// Name: testbackupstore.cpp
5	// Purpose: Test backup store server
6	// Created: 2003/08/20
7	//
8	// --------------------------------------------------------------------------
9
10	#include "Box.h"
11
12	#include <stdlib.h>
13	#include <string.h>
14
15	#include "Test.h"
16	#include "autogen_BackupProtocol.h"
17	#include "SSLLib.h"
18	#include "TLSContext.h"
19	#include "SocketStreamTLS.h"
20	#include "BoxPortsAndFiles.h"
21	#include "BackupStoreConstants.h"
22	#include "Socket.h"
23	#include "BackupStoreFilenameClear.h"
24	#include "CollectInBufferStream.h"
25	#include "BackupStoreDirectory.h"
26	#include "BackupStoreFile.h"
27	#include "FileStream.h"
28	#include "RaidFileController.h"
29	#include "RaidFileWrite.h"
30	#include "BackupStoreInfo.h"
31	#include "BackupStoreException.h"
32	#include "RaidFileException.h"
33	#include "MemBlockStream.h"
34	#include "BackupClientFileAttributes.h"
35	#include "BackupClientCryptoKeys.h"
36	#include "ServerControl.h"
37	#include "BackupStoreAccountDatabase.h"
38	#include "BackupStoreRefCountDatabase.h"
39	#include "BackupStoreAccounts.h"
40	#include "HousekeepStoreAccount.h"
41
42	#include "MemLeakFindOn.h"
43
44
45	#define ENCFILE_SIZE 2765
46
47	typedef struct
48	{
49	BackupStoreFilenameClear fn;
50	box_time_t mod;
51	int64_t id;
52	int64_t size;
53	int16_t flags;
54	box_time_t attrmod;
55	} dirtest;
56
57	static dirtest ens[] =
58	{
59	{BackupStoreFilenameClear(), 324324, 3432, 324, BackupStoreDirectory::Entry::Flags_File, 458763243422LL},
60	{BackupStoreFilenameClear(), 3432, 32443245645LL, 78, BackupStoreDirectory::Entry::Flags_Dir, 3248972347LL},
61	{BackupStoreFilenameClear(), 544435, 234234, 23324, BackupStoreDirectory::Entry::Flags_File \| BackupStoreDirectory::Entry::Flags_Deleted, 2348974782LL},
62	{BackupStoreFilenameClear(), 234, 235436, 6523, BackupStoreDirectory::Entry::Flags_File, 32458923175634LL},
63	{BackupStoreFilenameClear(), 0x3242343532144LL, 8978979789LL, 21345, BackupStoreDirectory::Entry::Flags_File, 329483243432LL},
64	{BackupStoreFilenameClear(), 324265765734LL, 12312312321LL, 324987324329874LL, BackupStoreDirectory::Entry::Flags_File \| BackupStoreDirectory::Entry::Flags_Deleted, 32489747234LL},
65	{BackupStoreFilenameClear(), 3452134, 7868578768LL, 324243, BackupStoreDirectory::Entry::Flags_Dir, 34786457432LL},
66	{BackupStoreFilenameClear(), 43543543, 324234, 21432, BackupStoreDirectory::Entry::Flags_Dir \| BackupStoreDirectory::Entry::Flags_Deleted, 3489723478327LL},
67	{BackupStoreFilenameClear(), 325654765874324LL, 4353543, 1, BackupStoreDirectory::Entry::Flags_File \| BackupStoreDirectory::Entry::Flags_OldVersion, 32489734789237LL},
68	{BackupStoreFilenameClear(), 32144325, 436547657, 9, BackupStoreDirectory::Entry::Flags_File \| BackupStoreDirectory::Entry::Flags_OldVersion, 234897347234LL}
69	};
70	static const char *ens_filenames[] = {"obj1ertewt", "obj2", "obj3", "obj4dfedfg43", "obj5", "obj6dfgs", "obj7", "obj8xcvbcx", "obj9", "obj10fgjhfg"};
71	#define DIR_NUM 10
72	#define DIR_DIRS 3
73	#define DIR_FILES 7
74	#define DIR_OLD 2
75	#define DIR_DELETED 3
76
77	typedef struct
78	{
79	const char *fnextra;
80	BackupStoreFilenameClear name;
81	int seed;
82	int size;
83	box_time_t mod_time;
84	int64_t allocated_objid;
85	bool should_be_old_version;
86	bool delete_file;
87	} uploadtest;
88
89	#define TEST_FILE_FOR_PATCHING "testfiles/test2"
90	// a few bytes will be inserted at this point:
91	#define TEST_FILE_FOR_PATCHING_PATCH_AT ((64*1024)-128)
92	#define TEST_FILE_FOR_PATCHING_SIZE ((128*1024)+2564)
93	#define UPLOAD_PATCH_EN 2
94
95	uploadtest uploads[] =
96	{
97	{"0", BackupStoreFilenameClear(), 324, 455, 0, 0, false, false},
98	{"1", BackupStoreFilenameClear(), 3232432, 2674, 0, 0, true, false}, // old ver
99	{"2", BackupStoreFilenameClear(), 234, TEST_FILE_FOR_PATCHING_SIZE, 0, 0, false, false},
100	{"3", BackupStoreFilenameClear(), 324324, 6763, 0, 0, false, false},
101	{"4", BackupStoreFilenameClear(), 23456, 124, 0, 0, true, false}, // old ver
102	{"5", BackupStoreFilenameClear(), 675745, 1, 0, 0, false, false}, // will upload new attrs for this one!
103	{"6", BackupStoreFilenameClear(), 345213, 0, 0, 0, false, false},
104	{"7", BackupStoreFilenameClear(), 12313, 3246, 0, 0, true, true}, // old ver, will get deleted
105	{"8", BackupStoreFilenameClear(), 457, 3434, 0, 0, false, false}, // overwrites
106	{"9", BackupStoreFilenameClear(), 12315, 446, 0, 0, false, false},
107	{"a", BackupStoreFilenameClear(), 3476, 2466, 0, 0, false, false},
108	{"b", BackupStoreFilenameClear(), 124334, 4562, 0, 0, false, false},
109	{"c", BackupStoreFilenameClear(), 45778, 234, 0, 0, false, false}, // overwrites
110	{"d", BackupStoreFilenameClear(), 2423425, 435, 0, 0, false, true} // overwrites, will be deleted
111	};
112	static const char *uploads_filenames[] = {"49587fds", "cvhjhj324", "sdfcscs324", "dsfdsvsdc3214", "XXsfdsdf2342", "dsfdsc232",
113	"sfdsdce2345", "YYstfbdtrdf76", "cvhjhj324", "fbfd098.ycy", "dfs98732hj", "svd987kjsad", "XXsfdsdf2342", "YYstfbdtrdf76"};
114	#define UPLOAD_NUM 14
115	#define UPLOAD_LATEST_FILES 12
116	// file we'll upload some new attributes for
117	#define UPLOAD_ATTRS_EN 5
118	#define UPLOAD_DELETE_EN 13
119	// file which will be moved (as well as it's old version)
120	#define UPLOAD_FILE_TO_MOVE 8
121
122
123	// Nice random data for testing written files
124	class R250 {
125	public:
126	// Set up internal state table with 32-bit random numbers.
127	// The bizarre bit-twiddling is because rand() returns 16 bits of which
128	// the bottom bit is always zero! Hence, I use only some of the bits.
129	// You might want to do something better than this....
130
131	R250(int seed) : posn1(0), posn2(103)
132	{
133	// populate the state and incr tables
134	srand(seed);
135
136	for (int i = 0; i != stateLen; ++i) {
137	state[i] = ((rand() >> 2) << 19) ^ ((rand() >> 2) << 11) ^ (rand() >> 2);
138	incrTable[i] = i == stateLen - 1 ? 0 : i + 1;
139	}
140
141	// stir up the numbers to ensure they're random
142
143	for (int j = 0; j != stateLen * 4; ++j)
144	(void) next();
145	}
146
147	// Returns the next random number. Xor together two elements separated
148	// by 103 mod 250, replacing the first element with the result. Then
149	// increment the two indices mod 250.
150	inline int next()
151	{
152	int ret = (state[posn1] ^= state[posn2]); // xor and replace element
153
154	posn1 = incrTable[posn1]; // increment indices using lookup table
155	posn2 = incrTable[posn2];
156
157	return ret;
158	}
159	private:
160	enum { stateLen = 250 }; // length of the state table
161	int state[stateLen]; // holds the random number state
162	int incrTable[stateLen]; // lookup table: maps i to (i+1) % stateLen
163	int posn1, posn2; // indices into the state table
164	};
165
166
167	int SkipEntries(int e, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet)
168	{
169	if(e >= DIR_NUM) return e;
170
171	bool skip = false;
172	do
173	{
174	skip = false;
175
176	if(FlagsMustBeSet != BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING)
177	{
178	if((ens[e].flags & FlagsMustBeSet) != FlagsMustBeSet)
179	{
180	skip = true;
181	}
182	}
183	if((ens[e].flags & FlagsNotToBeSet) != 0)
184	{
185	skip = true;
186	}
187
188	if(skip)
189	{
190	++e;
191	}
192	} while(skip && e < DIR_NUM);
193
194	return e;
195	}
196
197	void CheckEntries(BackupStoreDirectory &rDir, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet)
198	{
199	int e = 0;
200
201	BackupStoreDirectory::Iterator i(rDir);
202	BackupStoreDirectory::Entry *en = 0;
203	while((en = i.Next()) != 0)
204	{
205	TEST_THAT(e < DIR_NUM);
206
207	// Skip to entry in the ens array which matches
208	e = SkipEntries(e, FlagsMustBeSet, FlagsNotToBeSet);
209
210	// Does it match?
211	TEST_THAT(en->GetName() == ens[e].fn && en->GetModificationTime() == ens[e].mod && en->GetObjectID() == ens[e].id && en->GetFlags() == ens[e].flags && en->GetSizeInBlocks() == ens[e].size);
212
213	// next
214	++e;
215	}
216
217	// Got them all?
218	TEST_THAT(en == 0);
219	TEST_THAT(DIR_NUM == SkipEntries(e, FlagsMustBeSet, FlagsNotToBeSet));
220	}
221
222	int test1(int argc, const char *argv[])
223	{
224	// Initialise the raid file controller
225	RaidFileController &rcontroller = RaidFileController::GetController();
226	rcontroller.Initialise("testfiles/raidfile.conf");
227
228	// test some basics -- encoding and decoding filenames
229	{
230	// Make some filenames in various ways
231	BackupStoreFilenameClear fn1;
232	fn1.SetClearFilename(std::string("filenameXYZ"));
233	BackupStoreFilenameClear fn2(std::string("filenameXYZ"));
234	BackupStoreFilenameClear fn3(fn1);
235	TEST_THAT(fn1 == fn2);
236	TEST_THAT(fn1 == fn3);
237
238	// Check that it's been encrypted
239	std::string name(fn2.GetEncodedFilename());
240	TEST_THAT(name.find("name") == name.npos);
241
242	// Bung it in a stream, get it out in a Clear filename
243	{
244	CollectInBufferStream stream;
245	fn1.WriteToStream(stream);
246	stream.SetForReading();
247	BackupStoreFilenameClear fn4;
248	fn4.ReadFromStream(stream, IOStream::TimeOutInfinite);
249	TEST_THAT(fn4.GetClearFilename() == "filenameXYZ");
250	TEST_THAT(fn4 == fn1);
251	}
252	// Bung it in a stream, get it out in a server non-Clear filename (two of them into the same var)
253	{
254	BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf ");
255	CollectInBufferStream stream;
256	fn1.WriteToStream(stream);
257	fno.WriteToStream(stream);
258	stream.SetForReading();
259	BackupStoreFilename fn5;
260	fn5.ReadFromStream(stream, IOStream::TimeOutInfinite);
261	TEST_THAT(fn5 == fn1);
262	fn5.ReadFromStream(stream, IOStream::TimeOutInfinite);
263	TEST_THAT(fn5 == fno);
264	}
265	// Same again with clear strings
266	{
267	BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf ");
268	CollectInBufferStream stream;
269	fn1.WriteToStream(stream);
270	fno.WriteToStream(stream);
271	stream.SetForReading();
272	BackupStoreFilenameClear fn5;
273	fn5.ReadFromStream(stream, IOStream::TimeOutInfinite);
274	TEST_THAT(fn5.GetClearFilename() == "filenameXYZ");
275	fn5.ReadFromStream(stream, IOStream::TimeOutInfinite);
276	TEST_THAT(fn5.GetClearFilename() == "pinglet dksfnsf jksjdf ");
277	}
278	// Test a very big filename
279	{
280	const char *fnr = "01234567890123456789012345678901234567890123456789"
281	"01234567890123456789012345678901234567890123456789"
282	"01234567890123456789012345678901234567890123456789"
283	"01234567890123456789012345678901234567890123456789"
284	"01234567890123456789012345678901234567890123456789"
285	"01234567890123456789012345678901234567890123456789"
286	"01234567890123456789012345678901234567890123456789"
287	"01234567890123456789012345678901234567890123456789";
288	BackupStoreFilenameClear fnLong(fnr);
289	CollectInBufferStream stream;
290	fnLong.WriteToStream(stream);
291	stream.SetForReading();
292	BackupStoreFilenameClear fn9;
293	fn9.ReadFromStream(stream, IOStream::TimeOutInfinite);
294	TEST_THAT(fn9.GetClearFilename() == fnr);
295	TEST_THAT(fn9 == fnLong);
296	}
297	// Test a filename which went wrong once
298	{
299	BackupStoreFilenameClear dodgy("content-negotiation.html");
300	}
301	}
302	return 0;
303	}
304
305	int test2(int argc, const char *argv[])
306	{
307	{
308	// Now play with directories
309
310	// Fill in...
311	BackupStoreDirectory dir1(12, 98);
312	for(int e = 0; e < DIR_NUM; ++e)
313	{
314	dir1.AddEntry(ens[e].fn, ens[e].mod, ens[e].id, ens[e].size, ens[e].flags, ens[e].attrmod);
315	}
316	// Got the right number
317	TEST_THAT(dir1.GetNumberOfEntries() == DIR_NUM);
318
319	// Stick it into a stream and get it out again
320	{
321	CollectInBufferStream stream;
322	dir1.WriteToStream(stream);
323	stream.SetForReading();
324	BackupStoreDirectory dir2;
325	dir2.ReadFromStream(stream, IOStream::TimeOutInfinite);
326	TEST_THAT(dir2.GetNumberOfEntries() == DIR_NUM);
327	TEST_THAT(dir2.GetObjectID() == 12);
328	TEST_THAT(dir2.GetContainerID() == 98);
329	CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING);
330	}
331
332	// Then do selective writes and reads
333	{
334	CollectInBufferStream stream;
335	dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File);
336	stream.SetForReading();
337	BackupStoreDirectory dir2;
338	dir2.ReadFromStream(stream, IOStream::TimeOutInfinite);
339	TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES);
340	CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING);
341	}
342	{
343	CollectInBufferStream stream;
344	dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_File);
345	stream.SetForReading();
346	BackupStoreDirectory dir2;
347	dir2.ReadFromStream(stream, IOStream::TimeOutInfinite);
348	TEST_THAT(dir2.GetNumberOfEntries() == DIR_DIRS);
349	CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_Dir, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING);
350	}
351	{
352	CollectInBufferStream stream;
353	dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion);
354	stream.SetForReading();
355	BackupStoreDirectory dir2;
356	dir2.ReadFromStream(stream, IOStream::TimeOutInfinite);
357	TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - DIR_OLD);
358	CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion);
359	}
360
361	// Finally test deleting items
362	{
363	dir1.DeleteEntry(12312312321LL);
364	// Verify
365	TEST_THAT(dir1.GetNumberOfEntries() == DIR_NUM - 1);
366	CollectInBufferStream stream;
367	dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File);
368	stream.SetForReading();
369	BackupStoreDirectory dir2;
370	dir2.ReadFromStream(stream, IOStream::TimeOutInfinite);
371	TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - 1);
372	}
373
374	// Check attributes
375	{
376	int attrI[4] = {1, 2, 3, 4};
377	StreamableMemBlock attr(attrI, sizeof(attrI));
378	BackupStoreDirectory d1(16, 546);
379	d1.SetAttributes(attr, 56234987324232LL);
380	TEST_THAT(d1.GetAttributes() == attr);
381	TEST_THAT(d1.GetAttributesModTime() == 56234987324232LL);
382	CollectInBufferStream stream;
383	d1.WriteToStream(stream);
384	stream.SetForReading();
385	BackupStoreDirectory d2;
386	d2.ReadFromStream(stream, IOStream::TimeOutInfinite);
387	TEST_THAT(d2.GetAttributes() == attr);
388	TEST_THAT(d2.GetAttributesModTime() == 56234987324232LL);
389	}
390	}
391	return 0;
392	}
393
394	void write_test_file(int t)
395	{
396	std::string filename("testfiles/test");
397	filename += uploads[t].fnextra;
398	printf("%s\n", filename.c_str());
399
400	FileStream write(filename.c_str(), O_WRONLY \| O_CREAT);
401
402	R250 r(uploads[t].seed);
403
404	unsigned char data = (unsigned char)malloc(uploads[t].size);
405	for(int l = 0; l < uploads[t].size; ++l)
406	{
407	data[l] = r.next() & 0xff;
408	}
409	write.Write(data, uploads[t].size);
410
411	free(data);
412	}
413
414	void test_test_file(int t, IOStream &rStream)
415	{
416	// Decode to a file
417	BackupStoreFile::DecodeFile(rStream, "testfiles/test_download", IOStream::TimeOutInfinite);
418
419	// Compare...
420	FileStream in("testfiles/test_download");
421	TEST_THAT(in.BytesLeftToRead() == uploads[t].size);
422
423	R250 r(uploads[t].seed);
424
425	unsigned char data = (unsigned char)malloc(uploads[t].size);
426	TEST_THAT(in.ReadFullBuffer(data, uploads[t].size, 0 /* not interested in bytes read if this fails */));
427
428	for(int l = 0; l < uploads[t].size; ++l)
429	{
430	TEST_THAT(data[l] == (r.next() & 0xff));
431	}
432
433	free(data);
434	in.Close();
435	TEST_THAT(unlink("testfiles/test_download") == 0);
436	}
437
438	void test_everything_deleted(BackupProtocolClient &protocol, int64_t DirID)
439	{
440	printf("Test for del: %llx\n", DirID);
441
442	// Command
443	std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory(
444	DirID,
445	BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING,
446	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
447	// Stream
448	BackupStoreDirectory dir;
449	std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
450	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
451
452	BackupStoreDirectory::Iterator i(dir);
453	BackupStoreDirectory::Entry *en = 0;
454	int files = 0;
455	int dirs = 0;
456	while((en = i.Next()) != 0)
457	{
458	if(en->GetFlags() & BackupProtocolListDirectory::Flags_Dir)
459	{
460	dirs++;
461	// Recurse
462	test_everything_deleted(protocol, en->GetObjectID());
463	}
464	else
465	{
466	files++;
467	}
468	// Check it's deleted
469	TEST_THAT(en->GetFlags() & BackupProtocolListDirectory::Flags_Deleted);
470	}
471
472	// Check there were the right number of files and directories
473	TEST_THAT(files == 3);
474	TEST_THAT(dirs == 0 \|\| dirs == 2);
475	}
476
477	std::vector<uint32_t> ExpectedRefCounts;
478
479	void set_refcount(int64_t ObjectID, uint32_t RefCount = 1)
480	{
481	if (ExpectedRefCounts.size() <= ObjectID);
482	{
483	ExpectedRefCounts.resize(ObjectID + 1, 0);
484	}
485	ExpectedRefCounts[ObjectID] = RefCount;
486	}
487
488	void create_file_in_dir(std::string name, std::string source, int64_t parentId,
489	BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount)
490	{
491	BackupStoreFilenameClear name_encoded("file_One");
492	std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(
493	source.c_str(), parentId, name_encoded));
494	std::auto_ptr<BackupProtocolSuccess> stored(
495	protocol.QueryStoreFile(
496	parentId,
497	0x123456789abcdefLL, /* modification time */
498	0x7362383249872dfLL, /* attr hash */
499	0, /* diff from ID */
500	name_encoded,
501	*upload));
502	int64_t objectId = stored->GetObjectID();
503	TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed());
504	TEST_EQUAL(1, rRefCount.GetRefCount(objectId))
505	set_refcount(objectId, 1);
506	}
507
508	int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir,
509	const char *name, int depth, BackupStoreRefCountDatabase& rRefCount)
510	{
511	// Create a directory
512	int64_t subdirid = 0;
513	BackupStoreFilenameClear dirname(name);
514	{
515	// Create with dummy attributes
516	int attrS = 0;
517	MemBlockStream attr(&attrS, sizeof(attrS));
518	std::auto_ptr<BackupProtocolSuccess> dirCreate(protocol.QueryCreateDirectory(
519	indir,
520	9837429842987984LL, dirname, attr));
521	subdirid = dirCreate->GetObjectID();
522	}
523
524	printf("Create subdirs, depth = %d, dirid = %llx\n", depth, subdirid);
525
526	TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed());
527	TEST_EQUAL(1, rRefCount.GetRefCount(subdirid))
528	set_refcount(subdirid, 1);
529
530	// Put more directories in it, if we haven't gone down too far
531	if(depth > 0)
532	{
533	create_test_data_subdirs(protocol, subdirid, "dir_One",
534	depth - 1, rRefCount);
535	create_test_data_subdirs(protocol, subdirid, "dir_Two",
536	depth - 1, rRefCount);
537	}
538
539	// Stick some files in it
540	create_file_in_dir("file_One", "testfiles/file1", subdirid, protocol,
541	rRefCount);
542	create_file_in_dir("file_Two", "testfiles/file1", subdirid, protocol,
543	rRefCount);
544	create_file_in_dir("file_Three", "testfiles/file1", subdirid, protocol,
545	rRefCount);
546	return subdirid;
547	}
548
549
550	void check_dir_after_uploads(BackupProtocolClient &protocol, const StreamableMemBlock &Attributes)
551	{
552	// Command
553	std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory(
554	BackupProtocolListDirectory::RootDirectory,
555	BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING,
556	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
557	TEST_THAT(dirreply->GetObjectID() == BackupProtocolListDirectory::RootDirectory);
558	// Stream
559	BackupStoreDirectory dir;
560	std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
561	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
562	TEST_THAT(dir.GetNumberOfEntries() == UPLOAD_NUM + 1 /* for the first test file */);
563	TEST_THAT(!dir.HasAttributes());
564
565	// Check them!
566	BackupStoreDirectory::Iterator i(dir);
567	// Discard first
568	BackupStoreDirectory::Entry *en = i.Next();
569	TEST_THAT(en != 0);
570
571	for(int t = 0; t < UPLOAD_NUM; ++t)
572	{
573	en = i.Next();
574	TEST_THAT(en != 0);
575	TEST_THAT(en->GetName() == uploads[t].name);
576	TEST_THAT(en->GetObjectID() == uploads[t].allocated_objid);
577	TEST_THAT(en->GetModificationTime() == uploads[t].mod_time);
578	int correct_flags = BackupProtocolListDirectory::Flags_File;
579	if(uploads[t].should_be_old_version) correct_flags \|= BackupProtocolListDirectory::Flags_OldVersion;
580	if(uploads[t].delete_file) correct_flags \|= BackupProtocolListDirectory::Flags_Deleted;
581	TEST_THAT(en->GetFlags() == correct_flags);
582	if(t == UPLOAD_ATTRS_EN)
583	{
584	TEST_THAT(en->HasAttributes());
585	TEST_THAT(en->GetAttributesHash() == 32498749832475LL);
586	TEST_THAT(en->GetAttributes() == Attributes);
587	}
588	else
589	{
590	// No attributes on this one
591	TEST_THAT(!en->HasAttributes());
592	}
593	}
594	en = i.Next();
595	TEST_THAT(en == 0);
596	}
597
598
599	typedef struct
600	{
601	int objectsNotDel;
602	int deleted;
603	int old;
604	} recursive_count_objects_results;
605
606	void recursive_count_objects_r(BackupProtocolClient &protocol, int64_t id, recursive_count_objects_results &results)
607	{
608	// Command
609	std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory(
610	id,
611	BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING,
612	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
613	// Stream
614	BackupStoreDirectory dir;
615	std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
616	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
617
618	// Check them!
619	BackupStoreDirectory::Iterator i(dir);
620	// Discard first
621	BackupStoreDirectory::Entry *en = 0;
622
623	while((en = i.Next()) != 0)
624	{
625	if((en->GetFlags() & (BackupStoreDirectory::Entry::Flags_Deleted \| BackupStoreDirectory::Entry::Flags_OldVersion)) == 0) results.objectsNotDel++;
626	if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) results.deleted++;
627	if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) results.old++;
628
629	if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir)
630	{
631	recursive_count_objects_r(protocol, en->GetObjectID(), results);
632	}
633	}
634	}
635
636	void recursive_count_objects(const char *hostname, int64_t id, recursive_count_objects_results &results)
637	{
638	// Context
639	TLSContext context;
640	context.Initialise(false /* client */,
641	"testfiles/clientCerts.pem",
642	"testfiles/clientPrivKey.pem",
643	"testfiles/clientTrustedCAs.pem");
644
645	// Get a connection
646	SocketStreamTLS connReadOnly;
647	connReadOnly.Open(context, Socket::TypeINET, hostname,
648	BOX_PORT_BBSTORED_TEST);
649	BackupProtocolClient protocolReadOnly(connReadOnly);
650
651	{
652	std::auto_ptr<BackupProtocolVersion> serverVersion(protocolReadOnly.QueryVersion(BACKUP_STORE_SERVER_VERSION));
653	TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
654	std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocolReadOnly.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly));
655	}
656
657	// Count objects
658	recursive_count_objects_r(protocolReadOnly, id, results);
659
660	// Close it
661	protocolReadOnly.QueryFinished();
662	}
663
664	bool check_block_index(const char *encoded_file, IOStream &rBlockIndex)
665	{
666	// Open file, and move to the right position
667	FileStream enc(encoded_file);
668	BackupStoreFile::MoveStreamPositionToBlockIndex(enc);
669
670	bool same = true;
671
672	// Now compare the two...
673	while(enc.StreamDataLeft())
674	{
675	char buffer1[2048];
676	char buffer2[2048];
677	int s = enc.Read(buffer1, sizeof(buffer1));
678	if(rBlockIndex.Read(buffer2, s) != s)
679	{
680	same = false;
681	break;
682	}
683	if(::memcmp(buffer1, buffer2, s) != 0)
684	{
685	same = false;
686	break;
687	}
688	}
689
690	if(rBlockIndex.StreamDataLeft())
691	{
692	same = false;
693
694	// Absorb all this excess data so procotol is in the first state
695	char buffer[2048];
696	while(rBlockIndex.StreamDataLeft())
697	{
698	rBlockIndex.Read(buffer, sizeof(buffer));
699	}
700	}
701
702	return same;
703	}
704
705	bool check_files_same(const char f1, const char f2)
706	{
707	// Open file, and move to the right position
708	FileStream f1s(f1);
709	FileStream f2s(f2);
710
711	bool same = true;
712
713	// Now compare the two...
714	while(f1s.StreamDataLeft())
715	{
716	char buffer1[2048];
717	char buffer2[2048];
718	int s = f1s.Read(buffer1, sizeof(buffer1));
719	if(f2s.Read(buffer2, s) != s)
720	{
721	same = false;
722	break;
723	}
724	if(::memcmp(buffer1, buffer2, s) != 0)
725	{
726	same = false;
727	break;
728	}
729	}
730
731	if(f2s.StreamDataLeft())
732	{
733	same = false;
734	}
735
736	return same;
737	}
738
739
740	void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protocolReadOnly)
741	{
742	int encfile[ENCFILE_SIZE];
743	{
744	for(int l = 0; l < ENCFILE_SIZE; ++l)
745	{
746	encfile[l] = l * 173;
747	}
748
749	// Write this to a file
750	{
751	FileStream f("testfiles/file1", O_WRONLY \| O_CREAT \| O_EXCL);
752	f.Write(encfile, sizeof(encfile));
753	}
754
755	}
756
757	// Read the root directory a few times (as it's cached, so make sure it doesn't hurt anything)
758	for(int l = 0; l < 3; ++l)
759	{
760	// Command
761	std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory(
762	BackupProtocolListDirectory::RootDirectory,
763	BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING,
764	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
765	// Stream
766	BackupStoreDirectory dir;
767	std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
768	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
769	TEST_THAT(dir.GetNumberOfEntries() == 0);
770	}
771
772	// Read the dir from the readonly connection (make sure it gets in the cache)
773	{
774	// Command
775	std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory(
776	BackupProtocolListDirectory::RootDirectory,
777	BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING,
778	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
779	// Stream
780	BackupStoreDirectory dir;
781	std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
782	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
783	TEST_THAT(dir.GetNumberOfEntries() == 0);
784	}
785
786	// Store a file -- first make the encoded file
787	BackupStoreFilenameClear store1name("testfiles/file1");
788	{
789	FileStream out("testfiles/file1_upload1", O_WRONLY \| O_CREAT \| O_EXCL);
790	std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/file1", BackupProtocolListDirectory::RootDirectory, store1name));
791	encoded->CopyStreamTo(out);
792	}
793
794	// printf("SKIPPING\n");
795	// goto skip; {
796	// Then send it
797	int64_t store1objid = 0;
798	{
799	FileStream upload("testfiles/file1_upload1");
800	std::auto_ptr<BackupProtocolSuccess> stored(protocol.QueryStoreFile(
801	BackupProtocolListDirectory::RootDirectory,
802	0x123456789abcdefLL, /* modification time */
803	0x7362383249872dfLL, /* attr hash */
804	0, /* diff from ID */
805	store1name,
806	upload));
807	store1objid = stored->GetObjectID();
808	TEST_THAT(store1objid == 2);
809	}
810	set_refcount(store1objid, 1);
811	// And retrieve it
812	{
813	// Retrieve as object
814	std::auto_ptr<BackupProtocolSuccess> getfile(protocol.QueryGetObject(store1objid));
815	TEST_THAT(getfile->GetObjectID() == store1objid);
816	// BLOCK
817	{
818	// Get stream
819	std::auto_ptr<IOStream> filestream(protocol.ReceiveStream());
820	// Need to put it in another stream, because it's not in stream order
821	CollectInBufferStream f;
822	filestream->CopyStreamTo(f);
823	f.SetForReading();
824	// Get and decode
825	BackupStoreFile::DecodeFile(f, "testfiles/file1_upload_retrieved", IOStream::TimeOutInfinite);
826	}
827
828	// Retrieve as file
829	std::auto_ptr<BackupProtocolSuccess> getobj(protocol.QueryGetFile(BackupProtocolListDirectory::RootDirectory, store1objid));
830	TEST_THAT(getobj->GetObjectID() == store1objid);
831	// BLOCK
832	{
833	// Get stream
834	std::auto_ptr<IOStream> filestream(protocol.ReceiveStream());
835	// Get and decode
836	BackupStoreFile::DecodeFile(*filestream, "testfiles/file1_upload_retrieved_str", IOStream::TimeOutInfinite);
837	}
838
839	// Read in rebuilt original, and compare contents
840	{
841	FileStream in("testfiles/file1_upload_retrieved");
842	int encfile_i[ENCFILE_SIZE];
843	in.Read(encfile_i, sizeof(encfile_i));
844	TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0);
845	}
846	{
847	FileStream in("testfiles/file1_upload_retrieved_str");
848	int encfile_i[ENCFILE_SIZE];
849	in.Read(encfile_i, sizeof(encfile_i));
850	TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0);
851	}
852
853	// Retrieve the block index, by ID
854	{
855	std::auto_ptr<BackupProtocolSuccess> getblockindex(protocol.QueryGetBlockIndexByID(store1objid));
856	TEST_THAT(getblockindex->GetObjectID() == store1objid);
857	std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream());
858	// Check against uploaded file
859	TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream));
860	}
861	// and again, by name
862	{
863	std::auto_ptr<BackupProtocolSuccess> getblockindex(protocol.QueryGetBlockIndexByName(BackupProtocolListDirectory::RootDirectory, store1name));
864	TEST_THAT(getblockindex->GetObjectID() == store1objid);
865	std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream());
866	// Check against uploaded file
867	TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream));
868	}
869	}
870	// Get the directory again, and see if the entry is in it
871	{
872	// Command
873	std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory(
874	BackupProtocolListDirectory::RootDirectory,
875	BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING,
876	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
877	// Stream
878	BackupStoreDirectory dir;
879	std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
880	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
881	TEST_THAT(dir.GetNumberOfEntries() == 1);
882	BackupStoreDirectory::Iterator i(dir);
883	BackupStoreDirectory::Entry *en = i.Next();
884	TEST_THAT(en != 0);
885	TEST_THAT(i.Next() == 0);
886	if(en != 0)
887	{
888	TEST_THAT(en->GetName() == store1name);
889	TEST_THAT(en->GetModificationTime() == 0x123456789abcdefLL);
890	TEST_THAT(en->GetAttributesHash() == 0x7362383249872dfLL);
891	TEST_THAT(en->GetObjectID() == store1objid);
892	TEST_THAT(en->GetSizeInBlocks() < ((ENCFILE_SIZE * 4 * 3) / 2 / 2048)+2);
893	TEST_THAT(en->GetFlags() == BackupStoreDirectory::Entry::Flags_File);
894	}
895	}
896
897	// Try using GetFile on a directory
898	{
899	TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolSuccess> getFile(protocol.QueryGetFile(BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::RootDirectory)),
900	ConnectionException, Conn_Protocol_UnexpectedReply);
901	}
902	}
903
904	void init_context(TLSContext& rContext)
905	{
906	rContext.Initialise(false /* client */,
907	"testfiles/clientCerts.pem",
908	"testfiles/clientPrivKey.pem",
909	"testfiles/clientTrustedCAs.pem");
910	}
911
912	std::auto_ptr<SocketStreamTLS> open_conn(const char *hostname,
913	TLSContext& rContext)
914	{
915	init_context(rContext);
916	std::auto_ptr<SocketStreamTLS> conn(new SocketStreamTLS);
917	conn->Open(rContext, Socket::TypeINET, hostname,
918	BOX_PORT_BBSTORED_TEST);
919	return conn;
920	}
921
922	std::auto_ptr<BackupProtocolClient> test_server_login(const char *hostname,
923	TLSContext& rContext, std::auto_ptr<SocketStreamTLS>& rapConn)
924	{
925	rapConn = open_conn(hostname, rContext);
926
927	// Make a protocol
928	std::auto_ptr<BackupProtocolClient> protocol(new
929	BackupProtocolClient(*rapConn));
930
931	// Check the version
932	std::auto_ptr<BackupProtocolVersion> serverVersion(
933	protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION));
934	TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
935
936	// Login
937	std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(
938	protocol->QueryLogin(0x01234567, 0));
939
940	return protocol;
941	}
942
943	void run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount)
944	{
945	std::string rootDir = BackupStoreAccounts::GetAccountRoot(rAccount);
946	int discSet = rAccount.GetDiscSet();
947
948	// Do housekeeping on this account
949	HousekeepStoreAccount housekeeping(rAccount.GetID(), rootDir,
950	discSet, NULL);
951	housekeeping.DoHousekeeping(true /* keep trying forever */);
952	}
953
954	// Run housekeeping (for which we need to disconnect ourselves) and check
955	// that it doesn't change the numbers of files.
956	//
957	// Also check that bbstoreaccounts doesn't change anything
958
959	void run_housekeeping_and_check_account(const char *hostname,
960	TLSContext& rContext, std::auto_ptr<SocketStreamTLS>& rapConn,
961	std::auto_ptr<BackupProtocolClient>& rapProtocol)
962	{
963	rapProtocol->QueryFinished();
964	std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
965	BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
966	BackupStoreAccountDatabase::Entry account =
967	apAccounts->GetEntry(0x1234567);
968	run_housekeeping(account);
969
970	TEST_THAT(::system(BBSTOREACCOUNTS
971	" -c testfiles/bbstored.conf check 01234567 fix") == 0);
972	TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
973
974	rapProtocol = test_server_login(hostname, rContext, rapConn);
975	}
976
977	int test_server(const char *hostname)
978	{
979	TLSContext context;
980	std::auto_ptr<SocketStreamTLS> conn;
981	std::auto_ptr<BackupProtocolClient> apProtocol =
982	test_server_login(hostname, context, conn);
983
984	// Make some test attributes
985	#define ATTR1_SIZE 245
986	#define ATTR2_SIZE 23
987	#define ATTR3_SIZE 122
988	int attr1[ATTR1_SIZE];
989	int attr2[ATTR2_SIZE];
990	int attr3[ATTR3_SIZE];
991	{
992	R250 r(3465657);
993	for(int l = 0; l < ATTR1_SIZE; ++l) {attr1[l] = r.next();}
994	for(int l = 0; l < ATTR2_SIZE; ++l) {attr2[l] = r.next();}
995	for(int l = 0; l < ATTR3_SIZE; ++l) {attr3[l] = r.next();}
996	}
997
998	// BLOCK
999	{
1000	// Get it logging
1001	FILE *protocolLog = ::fopen("testfiles/protocol.log", "w");
1002	TEST_THAT(protocolLog != 0);
1003	apProtocol->SetLogToFile(protocolLog);
1004
1005	#ifndef WIN32
1006	// Check that we can't open a new connection which requests write permissions
1007	{
1008	SocketStreamTLS conn;
1009	conn.Open(context, Socket::TypeINET, hostname,
1010	BOX_PORT_BBSTORED_TEST);
1011	BackupProtocolClient protocol(conn);
1012	std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
1013	TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
1014	TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)),
1015	ConnectionException, Conn_Protocol_UnexpectedReply);
1016	protocol.QueryFinished();
1017	}
1018	#endif
1019
1020	// Set the client store marker
1021	apProtocol->QuerySetClientStoreMarker(0x8732523ab23aLL);
1022
1023	#ifndef WIN32
1024	// Open a new connection which is read only
1025	SocketStreamTLS connReadOnly;
1026	connReadOnly.Open(context, Socket::TypeINET, hostname,
1027	BOX_PORT_BBSTORED_TEST);
1028	BackupProtocolClient protocolReadOnly(connReadOnly);
1029
1030	// Get it logging
1031	FILE *protocolReadOnlyLog = ::fopen("testfiles/protocolReadOnly.log", "w");
1032	TEST_THAT(protocolReadOnlyLog != 0);
1033	protocolReadOnly.SetLogToFile(protocolReadOnlyLog);
1034
1035	{
1036	std::auto_ptr<BackupProtocolVersion> serverVersion(protocolReadOnly.QueryVersion(BACKUP_STORE_SERVER_VERSION));
1037	TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
1038	std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocolReadOnly.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly));
1039
1040	// Check client store marker
1041	TEST_THAT(loginConf->GetClientStoreMarker() == 0x8732523ab23aLL);
1042	}
1043	#else // WIN32
1044	BackupProtocolClient& protocolReadOnly(*apProtocol);
1045	#endif
1046
1047	test_server_1(*apProtocol, protocolReadOnly);
1048
1049	#define TEST_NUM_FILES(files, old, deleted, dirs) \
1050	{ \
1051	std::auto_ptr<BackupStoreInfo> apInfo = \
1052	BackupStoreInfo::Load(0x1234567, \
1053	"backup/01234567/", 0, true); \
1054	TEST_EQUAL_LINE(files, apInfo->GetNumFiles(), \
1055	"num files"); \
1056	TEST_EQUAL_LINE(old, apInfo->GetNumOldFiles(), \
1057	"old files"); \
1058	TEST_EQUAL_LINE(deleted, apInfo->GetNumDeletedFiles(), \
1059	"deleted files"); \
1060	TEST_EQUAL_LINE(dirs, apInfo->GetNumDirectories(), \
1061	"directories"); \
1062	}
1063
1064	TEST_NUM_FILES(1, 0, 0, 1);
1065	run_housekeeping_and_check_account(hostname, context,
1066	conn, apProtocol);
1067	TEST_NUM_FILES(1, 0, 0, 1);
1068
1069	// sleep to ensure that the timestamp on the file will change
1070	::safe_sleep(1);
1071
1072	// Create and upload some test files
1073	int64_t maxID = 0;
1074	for(int t = 0; t < UPLOAD_NUM; ++t)
1075	{
1076	write_test_file(t);
1077
1078	std::string filename("testfiles/test");
1079	filename += uploads[t].fnextra;
1080	int64_t modtime = 0;
1081
1082	std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename.c_str(), BackupProtocolListDirectory::RootDirectory, uploads[t].name, &modtime));
1083	TEST_THAT(modtime != 0);
1084
1085	std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile(
1086	BackupProtocolListDirectory::RootDirectory,
1087	modtime,
1088	modtime, /* use it for attr hash too */
1089	0, /* diff from ID */
1090	uploads[t].name,
1091	*upload));
1092	uploads[t].allocated_objid = stored->GetObjectID();
1093	uploads[t].mod_time = modtime;
1094	if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID();
1095	set_refcount(stored->GetObjectID(), 1);
1096	BOX_TRACE("wrote file " << filename << " to server "
1097	"as object " <<
1098	BOX_FORMAT_OBJECTID(stored->GetObjectID()));
1099	TEST_NUM_FILES(t + 2, 0, 0, 1);
1100
1101	run_housekeeping_and_check_account(hostname, context,
1102	conn, apProtocol);
1103	TEST_NUM_FILES(t + 2, 0, 0, 1);
1104	}
1105
1106	// Add some attributes onto one of them
1107	{
1108	TEST_NUM_FILES(UPLOAD_NUM + 1, 0, 0, 1);
1109	MemBlockStream attrnew(attr3, sizeof(attr3));
1110	std::auto_ptr<BackupProtocolSuccess> set(apProtocol->QuerySetReplacementFileAttributes(
1111	BackupProtocolListDirectory::RootDirectory,
1112	32498749832475LL,
1113	uploads[UPLOAD_ATTRS_EN].name,
1114	attrnew));
1115	TEST_THAT(set->GetObjectID() == uploads[UPLOAD_ATTRS_EN].allocated_objid);
1116	TEST_NUM_FILES(UPLOAD_NUM + 1, 0, 0, 1);
1117	}
1118
1119	// Delete one of them (will implicitly delete an old version)
1120	{
1121	std::auto_ptr<BackupProtocolSuccess> del(apProtocol->QueryDeleteFile(
1122	BackupProtocolListDirectory::RootDirectory,
1123	uploads[UPLOAD_DELETE_EN].name));
1124	TEST_THAT(del->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid);
1125	TEST_NUM_FILES(UPLOAD_NUM, 0, 1, 1);
1126	}
1127
1128	// Check that the block index can be obtained by name even though it's been deleted
1129	{
1130	// Fetch the raw object
1131	{
1132	FileStream out("testfiles/downloaddelobj", O_WRONLY \| O_CREAT);
1133	std::auto_ptr<BackupProtocolSuccess> getobj(apProtocol->QueryGetObject(uploads[UPLOAD_DELETE_EN].allocated_objid));
1134	std::auto_ptr<IOStream> objstream(apProtocol->ReceiveStream());
1135	objstream->CopyStreamTo(out);
1136	}
1137	// query index and test
1138	std::auto_ptr<BackupProtocolSuccess> getblockindex(apProtocol->QueryGetBlockIndexByName(
1139	BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_DELETE_EN].name));
1140	TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid);
1141	std::auto_ptr<IOStream> blockIndexStream(apProtocol->ReceiveStream());
1142	TEST_THAT(check_block_index("testfiles/downloaddelobj", *blockIndexStream));
1143	}
1144
1145	// Download them all... (even deleted files)
1146	for(int t = 0; t < UPLOAD_NUM; ++t)
1147	{
1148	printf("%d\n", t);
1149	std::auto_ptr<BackupProtocolSuccess> getFile(apProtocol->QueryGetFile(BackupProtocolListDirectory::RootDirectory, uploads[t].allocated_objid));
1150	TEST_THAT(getFile->GetObjectID() == uploads[t].allocated_objid);
1151	std::auto_ptr<IOStream> filestream(apProtocol->ReceiveStream());
1152	test_test_file(t, *filestream);
1153	}
1154
1155	{
1156	StreamableMemBlock attrtest(attr3, sizeof(attr3));
1157
1158	// Use the read only connection to verify that the directory is as we expect
1159	printf("\n\n==== Reading directory using read-only connection\n");
1160	check_dir_after_uploads(protocolReadOnly, attrtest);
1161	printf("done.\n\n");
1162	// And on the read/write one
1163	check_dir_after_uploads(*apProtocol, attrtest);
1164	}
1165
1166	// sleep to ensure that the timestamp on the file will change
1167	::safe_sleep(1);
1168
1169	// Check diffing and rsync like stuff...
1170	// Build a modified file
1171	{
1172	// Basically just insert a bit in the middle
1173	TEST_THAT(TestGetFileSize(TEST_FILE_FOR_PATCHING) == TEST_FILE_FOR_PATCHING_SIZE);
1174	FileStream in(TEST_FILE_FOR_PATCHING);
1175	void *buf = ::malloc(TEST_FILE_FOR_PATCHING_SIZE);
1176	FileStream out(TEST_FILE_FOR_PATCHING ".mod", O_WRONLY \| O_CREAT \| O_EXCL);
1177	TEST_THAT(in.Read(buf, TEST_FILE_FOR_PATCHING_PATCH_AT) == TEST_FILE_FOR_PATCHING_PATCH_AT);
1178	out.Write(buf, TEST_FILE_FOR_PATCHING_PATCH_AT);
1179	char insert[13] = "INSERTINSERT";
1180	out.Write(insert, sizeof(insert));
1181	TEST_THAT(in.Read(buf, TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT) == TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT);
1182	out.Write(buf, TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT);
1183	::free(buf);
1184	}
1185
1186	TEST_NUM_FILES(UPLOAD_NUM, 0, 1, 1);
1187
1188	// Run housekeeping (for which we need to disconnect
1189	// ourselves) and check that it doesn't change the numbers
1190	// of files
1191	apProtocol->QueryFinished();
1192	std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
1193	BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
1194	BackupStoreAccountDatabase::Entry account =
1195	apAccounts->GetEntry(0x1234567);
1196	run_housekeeping(account);
1197
1198	// Also check that bbstoreaccounts doesn't change anything
1199	TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS
1200	" -c testfiles/bbstored.conf check 01234567 fix") == 0);
1201	TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
1202
1203	apProtocol = test_server_login(hostname, context, conn);
1204
1205	TEST_NUM_FILES(UPLOAD_NUM, 0, 1, 1);
1206
1207	{
1208	// Fetch the block index for this one
1209	std::auto_ptr<BackupProtocolSuccess> getblockindex(apProtocol->QueryGetBlockIndexByName(
1210	BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_PATCH_EN].name));
1211	TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_PATCH_EN].allocated_objid);
1212	std::auto_ptr<IOStream> blockIndexStream(apProtocol->ReceiveStream());
1213
1214	// Do the patching
1215	bool isCompletelyDifferent = false;
1216	int64_t modtime;
1217	std::auto_ptr<IOStream> patchstream(
1218	BackupStoreFile::EncodeFileDiff(
1219	TEST_FILE_FOR_PATCHING ".mod",
1220	BackupProtocolListDirectory::RootDirectory,
1221	uploads[UPLOAD_PATCH_EN].name,
1222	uploads[UPLOAD_PATCH_EN].allocated_objid,
1223	*blockIndexStream,
1224	IOStream::TimeOutInfinite,
1225	NULL, // pointer to DiffTimer impl
1226	&modtime, &isCompletelyDifferent));
1227	TEST_THAT(isCompletelyDifferent == false);
1228	// Sent this to a file, so we can check the size, rather than uploading it directly
1229	{
1230	FileStream patch(TEST_FILE_FOR_PATCHING ".patch", O_WRONLY \| O_CREAT \| O_EXCL);
1231	patchstream->CopyStreamTo(patch);
1232	}
1233	// Make sure the stream is a plausible size for a patch containing only one new block
1234	TEST_THAT(TestGetFileSize(TEST_FILE_FOR_PATCHING ".patch") < (8*1024));
1235	// Upload it
1236	int64_t patchedID = 0;
1237	{
1238	FileStream uploadpatch(TEST_FILE_FOR_PATCHING ".patch");
1239	std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile(
1240	BackupProtocolListDirectory::RootDirectory,
1241	modtime,
1242	modtime, /* use it for attr hash too */
1243	uploads[UPLOAD_PATCH_EN].allocated_objid, /* diff from ID */
1244	uploads[UPLOAD_PATCH_EN].name,
1245	uploadpatch));
1246	TEST_THAT(stored->GetObjectID() > 0);
1247	if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID();
1248	patchedID = stored->GetObjectID();
1249	}
1250
1251	set_refcount(patchedID, 1);
1252
1253	// Then download it to check it's OK
1254	std::auto_ptr<BackupProtocolSuccess> getFile(apProtocol->QueryGetFile(BackupProtocolListDirectory::RootDirectory, patchedID));
1255	TEST_THAT(getFile->GetObjectID() == patchedID);
1256	std::auto_ptr<IOStream> filestream(apProtocol->ReceiveStream());
1257	BackupStoreFile::DecodeFile(*filestream, TEST_FILE_FOR_PATCHING ".downloaded", IOStream::TimeOutInfinite);
1258	// Check it's the same
1259	TEST_THAT(check_files_same(TEST_FILE_FOR_PATCHING ".downloaded", TEST_FILE_FOR_PATCHING ".mod"));
1260
1261	TEST_NUM_FILES(UPLOAD_NUM, 1, 1, 1);
1262	}
1263
1264	// Create a directory
1265	int64_t subdirid = 0;
1266	BackupStoreFilenameClear dirname("lovely_directory");
1267	{
1268	// Attributes
1269	MemBlockStream attr(attr1, sizeof(attr1));
1270	std::auto_ptr<BackupProtocolSuccess> dirCreate(apProtocol->QueryCreateDirectory(
1271	BackupProtocolListDirectory::RootDirectory,
1272	9837429842987984LL, dirname, attr));
1273	subdirid = dirCreate->GetObjectID();
1274	TEST_THAT(subdirid == maxID + 1);
1275
1276	TEST_NUM_FILES(UPLOAD_NUM, 1, 1, 2);
1277	}
1278
1279	set_refcount(subdirid, 1);
1280
1281	// Stick a file in it
1282	int64_t subdirfileid = 0;
1283	{
1284	std::string filename("testfiles/test0");
1285	int64_t modtime;
1286	std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename.c_str(), subdirid, uploads[0].name, &modtime));
1287
1288	std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile(
1289	subdirid,
1290	modtime,
1291	modtime, /* use for attr hash too */
1292	0, /* diff from ID */
1293	uploads[0].name,
1294	*upload));
1295	subdirfileid = stored->GetObjectID();
1296
1297	TEST_NUM_FILES(UPLOAD_NUM + 1, 1, 1, 2);
1298	}
1299
1300	set_refcount(subdirfileid, 1);
1301
1302	printf("\n==== Checking upload using read-only connection\n");
1303	// Check the directories on the read only connection
1304	{
1305	// Command
1306	std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory(
1307	BackupProtocolListDirectory::RootDirectory,
1308	BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING,
1309	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */)); // Stream
1310	BackupStoreDirectory dir;
1311	std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
1312	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
1313	TEST_THAT(dir.GetNumberOfEntries() == UPLOAD_NUM + 3 /* for the first test file, the patched upload, and this new dir */);
1314
1315	// Check the last one...
1316	BackupStoreDirectory::Iterator i(dir);
1317	BackupStoreDirectory::Entry *en = 0;
1318	BackupStoreDirectory::Entry *t = 0;
1319	while((t = i.Next()) != 0)
1320	{
1321	if(en != 0)
1322	{
1323	// here for all but last object
1324	TEST_THAT(en->GetObjectID() != subdirid);
1325	TEST_THAT(en->GetName() != dirname);
1326	}
1327	en = t;
1328	}
1329	// Does it look right?
1330	TEST_THAT(en->GetName() == dirname);
1331	TEST_THAT(en->GetFlags() == BackupProtocolListDirectory::Flags_Dir);
1332	TEST_THAT(en->GetObjectID() == subdirid);
1333	TEST_THAT(en->GetModificationTime() == 0); // dirs don't have modification times.
1334	}
1335
1336	{
1337	// Command
1338	std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory(
1339	subdirid,
1340	BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING,
1341	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, true /* get attributes */));
1342	TEST_THAT(dirreply->GetObjectID() == subdirid);
1343	// Stream
1344	BackupStoreDirectory dir;
1345	std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
1346	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
1347	TEST_THAT(dir.GetNumberOfEntries() == 1);
1348
1349	// Check the last one...
1350	BackupStoreDirectory::Iterator i(dir);
1351	// Discard first
1352	BackupStoreDirectory::Entry *en = i.Next();
1353	TEST_THAT(en != 0);
1354	// Does it look right?
1355	TEST_THAT(en->GetName() == uploads[0].name);
1356	TEST_THAT(en->GetFlags() == BackupProtocolListDirectory::Flags_File);
1357	TEST_THAT(en->GetObjectID() == subdirfileid);
1358	TEST_THAT(en->GetModificationTime() != 0);
1359
1360	// Attributes
1361	TEST_THAT(dir.HasAttributes());
1362	TEST_THAT(dir.GetAttributesModTime() == 9837429842987984LL);
1363	StreamableMemBlock attr(attr1, sizeof(attr1));
1364	TEST_THAT(dir.GetAttributes() == attr);
1365	}
1366	printf("done.\n\n");
1367
1368	// Check that we don't get attributes if we don't ask for them
1369	{
1370	// Command
1371	std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory(
1372	subdirid,
1373	BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING,
1374	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */));
1375	// Stream
1376	BackupStoreDirectory dir;
1377	std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
1378	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
1379	TEST_THAT(!dir.HasAttributes());
1380	}
1381
1382	// sleep to ensure that the timestamp on the file will change
1383	::safe_sleep(1);
1384
1385	// Change attributes on the directory
1386	{
1387	MemBlockStream attrnew(attr2, sizeof(attr2));
1388	std::auto_ptr<BackupProtocolSuccess> changereply(apProtocol->QueryChangeDirAttributes(
1389	subdirid,
1390	329483209443598LL,
1391	attrnew));
1392	TEST_THAT(changereply->GetObjectID() == subdirid);
1393	}
1394	// Check the new attributes
1395	{
1396	// Command
1397	std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory(
1398	subdirid,
1399	0, // no flags
1400	BackupProtocolListDirectory::Flags_EXCLUDE_EVERYTHING, true /* get attributes */));
1401	// Stream
1402	BackupStoreDirectory dir;
1403	std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
1404	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
1405	TEST_THAT(dir.GetNumberOfEntries() == 0);
1406
1407	// Attributes
1408	TEST_THAT(dir.HasAttributes());
1409	TEST_THAT(dir.GetAttributesModTime() == 329483209443598LL);
1410	StreamableMemBlock attrtest(attr2, sizeof(attr2));
1411	TEST_THAT(dir.GetAttributes() == attrtest);
1412	}
1413
1414	// sleep to ensure that the timestamp on the file will change
1415	::safe_sleep(1);
1416
1417	// Test moving a file
1418	{
1419	BackupStoreFilenameClear newName("moved-files");
1420
1421	std::auto_ptr<BackupProtocolSuccess> rep(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid,
1422	BackupProtocolListDirectory::RootDirectory,
1423	subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName));
1424	TEST_THAT(rep->GetObjectID() == uploads[UPLOAD_FILE_TO_MOVE].allocated_objid);
1425	}
1426
1427	// Try some dodgy renames
1428	{
1429	BackupStoreFilenameClear newName("moved-files");
1430	TEST_CHECK_THROWS(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid,
1431	BackupProtocolListDirectory::RootDirectory,
1432	subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName),
1433	ConnectionException, Conn_Protocol_UnexpectedReply);
1434	TEST_CHECK_THROWS(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid,
1435	subdirid,
1436	subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName),
1437	ConnectionException, Conn_Protocol_UnexpectedReply);
1438	}
1439
1440	// Rename within a directory
1441	{
1442	BackupStoreFilenameClear newName("moved-files-x");
1443	apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid,
1444	subdirid,
1445	subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName);
1446	}
1447
1448	// Check it's all gone from the root directory...
1449	{
1450	// Command
1451	std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory(
1452	BackupProtocolListDirectory::RootDirectory,
1453	BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING,
1454	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
1455	// Stream
1456	BackupStoreDirectory dir;
1457	std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
1458	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
1459	// Read all entries
1460	BackupStoreDirectory::Iterator i(dir);
1461	BackupStoreDirectory::Entry *en = 0;
1462	while((en = i.Next()) != 0)
1463	{
1464	TEST_THAT(en->GetName() != uploads[UPLOAD_FILE_TO_MOVE].name);
1465	}
1466	}
1467
1468	// Check the old and new versions are in the other directory
1469	{
1470	BackupStoreFilenameClear lookFor("moved-files-x");
1471
1472	// Command
1473	std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory(
1474	subdirid,
1475	BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING,
1476	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
1477	// Stream
1478	BackupStoreDirectory dir;
1479	std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
1480	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
1481	// Check entries
1482	BackupStoreDirectory::Iterator i(dir);
1483	BackupStoreDirectory::Entry *en = 0;
1484	bool foundCurrent = false;
1485	bool foundOld = false;
1486	while((en = i.Next()) != 0)
1487	{
1488	if(en->GetName() == lookFor)
1489	{
1490	if(en->GetFlags() == (BackupStoreDirectory::Entry::Flags_File)) foundCurrent = true;
1491	if(en->GetFlags() == (BackupStoreDirectory::Entry::Flags_File \| BackupStoreDirectory::Entry::Flags_OldVersion)) foundOld = true;
1492	}
1493	}
1494	TEST_THAT(foundCurrent);
1495	TEST_THAT(foundOld);
1496	}
1497
1498	// sleep to ensure that the timestamp on the file will change
1499	::safe_sleep(1);
1500
1501	// make a little bit more of a thing to look at
1502	int64_t subsubdirid = 0;
1503	int64_t subsubfileid = 0;
1504	{
1505	BackupStoreFilenameClear nd("sub2");
1506	// Attributes
1507	MemBlockStream attr(attr1, sizeof(attr1));
1508	std::auto_ptr<BackupProtocolSuccess> dirCreate(apProtocol->QueryCreateDirectory(
1509	subdirid,
1510	9837429842987984LL, nd, attr));
1511	subsubdirid = dirCreate->GetObjectID();
1512
1513	FileStream upload("testfiles/file1_upload1");
1514	BackupStoreFilenameClear nf("file2");
1515	std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile(
1516	subsubdirid,
1517	0x123456789abcdefLL, /* modification time */
1518	0x7362383249872dfLL, /* attr hash */
1519	0, /* diff from ID */
1520	nf,
1521	upload));
1522	subsubfileid = stored->GetObjectID();
1523	}
1524
1525	set_refcount(subsubdirid, 1);
1526	set_refcount(subsubfileid, 1);
1527
1528	// Query names -- test that invalid stuff returns not found OK
1529	{
1530	std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(3248972347823478927LL, subsubdirid));
1531	TEST_THAT(nameRep->GetNumNameElements() == 0);
1532	}
1533	{
1534	std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(subsubfileid, 2342378424LL));
1535	TEST_THAT(nameRep->GetNumNameElements() == 0);
1536	}
1537	{
1538	std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(38947234789LL, 2342378424LL));
1539	TEST_THAT(nameRep->GetNumNameElements() == 0);
1540	}
1541	{
1542	std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(BackupProtocolGetObjectName::ObjectID_DirectoryOnly, 2234342378424LL));
1543	TEST_THAT(nameRep->GetNumNameElements() == 0);
1544	}
1545
1546	// Query names... first, get info for the file
1547	{
1548	std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(subsubfileid, subsubdirid));
1549	std::auto_ptr<IOStream> namestream(apProtocol->ReceiveStream());
1550
1551	TEST_THAT(nameRep->GetNumNameElements() == 3);
1552	TEST_THAT(nameRep->GetFlags() == BackupProtocolListDirectory::Flags_File);
1553	TEST_THAT(nameRep->GetModificationTime() == 0x123456789abcdefLL);
1554	TEST_THAT(nameRep->GetAttributesHash() == 0x7362383249872dfLL);
1555	static const char *testnames[] = {"file2","sub2","lovely_directory"};
1556	for(int l = 0; l < nameRep->GetNumNameElements(); ++l)
1557	{
1558	BackupStoreFilenameClear fn;
1559	fn.ReadFromStream(*namestream, 10000);
1560	TEST_THAT(fn.GetClearFilename() == testnames[l]);
1561	}
1562	}
1563
1564	// Query names... secondly, for the directory
1565	{
1566	std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(BackupProtocolGetObjectName::ObjectID_DirectoryOnly, subsubdirid));
1567	std::auto_ptr<IOStream> namestream(apProtocol->ReceiveStream());
1568
1569	TEST_THAT(nameRep->GetNumNameElements() == 2);
1570	TEST_THAT(nameRep->GetFlags() == BackupProtocolListDirectory::Flags_Dir);
1571	static const char *testnames[] = {"sub2","lovely_directory"};
1572	for(int l = 0; l < nameRep->GetNumNameElements(); ++l)
1573	{
1574	BackupStoreFilenameClear fn;
1575	fn.ReadFromStream(*namestream, 10000);
1576	TEST_THAT(fn.GetClearFilename() == testnames[l]);
1577	}
1578	}
1579
1580	//} skip:
1581
1582	std::auto_ptr<BackupStoreRefCountDatabase> apRefCount(
1583	BackupStoreRefCountDatabase::Load(
1584	apAccounts->GetEntry(0x1234567), true));
1585
1586	// Create some nice recursive directories
1587	int64_t dirtodelete = create_test_data_subdirs(*apProtocol,
1588	BackupProtocolListDirectory::RootDirectory,
1589	"test_delete", 6 /* depth /, apRefCount);
1590
1591	// And delete them
1592	{
1593	std::auto_ptr<BackupProtocolSuccess> dirdel(apProtocol->QueryDeleteDirectory(
1594	dirtodelete));
1595	TEST_THAT(dirdel->GetObjectID() == dirtodelete);
1596	}
1597
1598	// Get the root dir, checking for deleted items
1599	{
1600	// Command
1601	std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory(
1602	BackupProtocolListDirectory::RootDirectory,
1603	BackupProtocolListDirectory::Flags_Dir \| BackupProtocolListDirectory::Flags_Deleted,
1604	BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
1605	// Stream
1606	BackupStoreDirectory dir;
1607	std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
1608	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
1609
1610	// Check there's only that one entry
1611	TEST_THAT(dir.GetNumberOfEntries() == 1);
1612
1613	BackupStoreDirectory::Iterator i(dir);
1614	BackupStoreDirectory::Entry *en = i.Next();
1615	TEST_THAT(en != 0);
1616	if(en)
1617	{
1618	TEST_THAT(en->GetObjectID() == dirtodelete);
1619	BackupStoreFilenameClear n("test_delete");
1620	TEST_THAT(en->GetName() == n);
1621	}
1622
1623	// Then... check everything's deleted
1624	test_everything_deleted(protocolReadOnly, dirtodelete);
1625	}
1626
1627	// Finish the connections
1628	#ifndef WIN32
1629	protocolReadOnly.QueryFinished();
1630	#endif
1631	apProtocol->QueryFinished();
1632
1633	// Close logs
1634	#ifndef WIN32
1635	::fclose(protocolReadOnlyLog);
1636	#endif
1637	::fclose(protocolLog);
1638	}
1639
1640	return 0;
1641	}
1642
1643	int test3(int argc, const char *argv[])
1644	{
1645	// Now test encoded files
1646	// TODO: This test needs to check failure situations as well as everything working,
1647	// but this will be saved for the full implementation.
1648	int encfile[ENCFILE_SIZE];
1649	{
1650	for(int l = 0; l < ENCFILE_SIZE; ++l)
1651	{
1652	encfile[l] = l * 173;
1653	}
1654
1655	// Encode and decode a small block (shouldn't be compressed)
1656	{
1657	#define SMALL_BLOCK_SIZE 251
1658	int encBlockSize = BackupStoreFile::MaxBlockSizeForChunkSize(SMALL_BLOCK_SIZE);
1659	TEST_THAT(encBlockSize > SMALL_BLOCK_SIZE);
1660	BackupStoreFile::EncodingBuffer encoded;
1661	encoded.Allocate(encBlockSize / 8); // make sure reallocation happens
1662
1663	// Encode!
1664	int encSize = BackupStoreFile::EncodeChunk(encfile, SMALL_BLOCK_SIZE, encoded);
1665	// Check the header says it's not been compressed
1666	TEST_THAT((encoded.mpBuffer[0] & 1) == 0);
1667	// Check the output size has been inflated (no compression)
1668	TEST_THAT(encSize > SMALL_BLOCK_SIZE);
1669
1670	// Decode it
1671	int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(SMALL_BLOCK_SIZE);
1672	TEST_THAT(decBlockSize > SMALL_BLOCK_SIZE);
1673	uint8_t decoded = (uint8_t)malloc(decBlockSize);
1674	int decSize = BackupStoreFile::DecodeChunk(encoded.mpBuffer, encSize, decoded, decBlockSize);
1675	TEST_THAT(decSize < decBlockSize);
1676	TEST_THAT(decSize == SMALL_BLOCK_SIZE);
1677
1678	// Check it came out of the wash the same
1679	TEST_THAT(::memcmp(encfile, decoded, SMALL_BLOCK_SIZE) == 0);
1680
1681	free(decoded);
1682	}
1683
1684	// Encode and decode a big block (should be compressed)
1685	{
1686	int encBlockSize = BackupStoreFile::MaxBlockSizeForChunkSize(ENCFILE_SIZE);
1687	TEST_THAT(encBlockSize > ENCFILE_SIZE);
1688	BackupStoreFile::EncodingBuffer encoded;
1689	encoded.Allocate(encBlockSize / 8); // make sure reallocation happens
1690
1691	// Encode!
1692	int encSize = BackupStoreFile::EncodeChunk(encfile, ENCFILE_SIZE, encoded);
1693	// Check the header says it's compressed
1694	TEST_THAT((encoded.mpBuffer[0] & 1) == 1);
1695	// Check the output size make it likely that it's compressed (is very compressible data)
1696	TEST_THAT(encSize < ENCFILE_SIZE);
1697
1698	// Decode it
1699	int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(ENCFILE_SIZE);
1700	TEST_THAT(decBlockSize > ENCFILE_SIZE);
1701	uint8_t decoded = (uint8_t)malloc(decBlockSize);
1702	int decSize = BackupStoreFile::DecodeChunk(encoded.mpBuffer, encSize, decoded, decBlockSize);
1703	TEST_THAT(decSize < decBlockSize);
1704	TEST_THAT(decSize == ENCFILE_SIZE);
1705
1706	// Check it came out of the wash the same
1707	TEST_THAT(::memcmp(encfile, decoded, ENCFILE_SIZE) == 0);
1708
1709	free(decoded);
1710	}
1711
1712	// The test block to a file
1713	{
1714	FileStream f("testfiles/testenc1", O_WRONLY \| O_CREAT \| O_EXCL);
1715	f.Write(encfile, sizeof(encfile));
1716	}
1717
1718	// Encode it
1719	{
1720	FileStream out("testfiles/testenc1_enc", O_WRONLY \| O_CREAT \| O_EXCL);
1721	BackupStoreFilenameClear name("testfiles/testenc1");
1722
1723	std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testenc1", 32, name));
1724	encoded->CopyStreamTo(out);
1725	}
1726
1727	// Verify it
1728	{
1729	FileStream enc("testfiles/testenc1_enc");
1730	TEST_THAT(BackupStoreFile::VerifyEncodedFileFormat(enc) == true);
1731	}
1732
1733	// Decode it
1734	{
1735	FileStream enc("testfiles/testenc1_enc");
1736	BackupStoreFile::DecodeFile(enc, "testfiles/testenc1_orig", IOStream::TimeOutInfinite);
1737	}
1738
1739	// Read in rebuilt original, and compare contents
1740	{
1741	TEST_THAT(TestGetFileSize("testfiles/testenc1_orig") == sizeof(encfile));
1742	FileStream in("testfiles/testenc1_orig");
1743	int encfile_i[ENCFILE_SIZE];
1744	in.Read(encfile_i, sizeof(encfile_i));
1745	TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0);
1746	}
1747
1748	// Check how many blocks it had, and test the stream based interface
1749	{
1750	FileStream enc("testfiles/testenc1_enc");
1751	std::auto_ptr<BackupStoreFile::DecodedStream> decoded(BackupStoreFile::DecodeFileStream(enc, IOStream::TimeOutInfinite));
1752	CollectInBufferStream d;
1753	decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 971 /* buffer block size */);
1754	d.SetForReading();
1755	TEST_THAT(d.GetSize() == sizeof(encfile));
1756	TEST_THAT(memcmp(encfile, d.GetBuffer(), sizeof(encfile)) == 0);
1757
1758	TEST_THAT(decoded->GetNumBlocks() == 3);
1759	}
1760
1761	// Test that the last block in a file, if less than 256 bytes, gets put into the last block
1762	{
1763	#define FILE_SIZE_JUST_OVER ((4096*2)+58)
1764	FileStream f("testfiles/testenc2", O_WRONLY \| O_CREAT \| O_EXCL);
1765	f.Write(encfile + 2, FILE_SIZE_JUST_OVER);
1766	BackupStoreFilenameClear name("testenc2");
1767	std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testenc2", 32, name));
1768	CollectInBufferStream e;
1769	encoded->CopyStreamTo(e);
1770	e.SetForReading();
1771	std::auto_ptr<BackupStoreFile::DecodedStream> decoded(BackupStoreFile::DecodeFileStream(e, IOStream::TimeOutInfinite));
1772	CollectInBufferStream d;
1773	decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 879 /* buffer block size */);
1774	d.SetForReading();
1775	TEST_THAT(d.GetSize() == FILE_SIZE_JUST_OVER);
1776	TEST_THAT(memcmp(encfile + 2, d.GetBuffer(), FILE_SIZE_JUST_OVER) == 0);
1777
1778	TEST_THAT(decoded->GetNumBlocks() == 2);
1779	}
1780
1781	// Test that reordered streams work too
1782	{
1783	FileStream enc("testfiles/testenc1_enc");
1784	std::auto_ptr<IOStream> reordered(BackupStoreFile::ReorderFileToStreamOrder(&enc, false));
1785	std::auto_ptr<BackupStoreFile::DecodedStream> decoded(BackupStoreFile::DecodeFileStream(*reordered, IOStream::TimeOutInfinite));
1786	CollectInBufferStream d;
1787	decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 971 /* buffer block size */);
1788	d.SetForReading();
1789	TEST_THAT(d.GetSize() == sizeof(encfile));
1790	TEST_THAT(memcmp(encfile, d.GetBuffer(), sizeof(encfile)) == 0);
1791
1792	TEST_THAT(decoded->GetNumBlocks() == 3);
1793	}
1794
1795	#ifndef WIN32 // no symlinks on Win32
1796	// Try out doing this on a symlink
1797	{
1798	TEST_THAT(::symlink("does/not/exist", "testfiles/testsymlink") == 0);
1799	BackupStoreFilenameClear name("testsymlink");
1800	std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testsymlink", 32, name));
1801	// Can't decode it from the stream, because it's in file order, and doesn't have the
1802	// required properties to be able to reorder it. So buffer it...
1803	CollectInBufferStream b;
1804	encoded->CopyStreamTo(b);
1805	b.SetForReading();
1806	// Decode it
1807	BackupStoreFile::DecodeFile(b, "testfiles/testsymlink_2", IOStream::TimeOutInfinite);
1808	}
1809	#endif
1810	}
1811
1812	// Store info
1813	{
1814	RaidFileWrite::CreateDirectory(0, "test-info");
1815	BackupStoreInfo::CreateNew(76, "test-info/", 0, 3461231233455433LL, 2934852487LL);
1816	TEST_CHECK_THROWS(BackupStoreInfo::CreateNew(76, "test-info/", 0, 0, 0), RaidFileException, CannotOverwriteExistingFile);
1817	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(76, "test-info/", 0, true));
1818	TEST_CHECK_THROWS(info->Save(), BackupStoreException, StoreInfoIsReadOnly);
1819	TEST_CHECK_THROWS(info->ChangeBlocksUsed(1), BackupStoreException, StoreInfoIsReadOnly);
1820	TEST_CHECK_THROWS(info->ChangeBlocksInOldFiles(1), BackupStoreException, StoreInfoIsReadOnly);
1821	TEST_CHECK_THROWS(info->ChangeBlocksInDeletedFiles(1), BackupStoreException, StoreInfoIsReadOnly);
1822	TEST_CHECK_THROWS(info->RemovedDeletedDirectory(2), BackupStoreException, StoreInfoIsReadOnly);
1823	TEST_CHECK_THROWS(info->AddDeletedDirectory(2), BackupStoreException, StoreInfoIsReadOnly);
1824	TEST_CHECK_THROWS(info->SetAccountName("hello"), BackupStoreException, StoreInfoIsReadOnly);
1825	}
1826	{
1827	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(76, "test-info/", 0, false));
1828	info->ChangeBlocksUsed(8);
1829	info->ChangeBlocksInOldFiles(9);
1830	info->ChangeBlocksInDeletedFiles(10);
1831	info->ChangeBlocksUsed(-1);
1832	info->ChangeBlocksInOldFiles(-4);
1833	info->ChangeBlocksInDeletedFiles(-9);
1834	TEST_CHECK_THROWS(info->ChangeBlocksUsed(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative);
1835	TEST_CHECK_THROWS(info->ChangeBlocksInOldFiles(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative);
1836	TEST_CHECK_THROWS(info->ChangeBlocksInDeletedFiles(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative);
1837	info->AddDeletedDirectory(2);
1838	info->AddDeletedDirectory(3);
1839	info->AddDeletedDirectory(4);
1840	info->RemovedDeletedDirectory(3);
1841	info->SetAccountName("whee");
1842	TEST_CHECK_THROWS(info->RemovedDeletedDirectory(9), BackupStoreException, StoreInfoDirNotInList);
1843	info->Save();
1844	}
1845	{
1846	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(76, "test-info/", 0, true));
1847	TEST_THAT(info->GetBlocksUsed() == 7);
1848	TEST_THAT(info->GetBlocksInOldFiles() == 5);
1849	TEST_THAT(info->GetBlocksInDeletedFiles() == 1);
1850	TEST_THAT(info->GetBlocksSoftLimit() == 3461231233455433LL);
1851	TEST_THAT(info->GetBlocksHardLimit() == 2934852487LL);
1852	TEST_THAT(info->GetAccountName() == "whee");
1853	const std::vector<int64_t> &delfiles(info->GetDeletedDirectories());
1854	TEST_THAT(delfiles.size() == 2);
1855	TEST_THAT(delfiles[0] == 2);
1856	TEST_THAT(delfiles[1] == 4);
1857	}
1858
1859	//printf("SKIPPINGTESTS---------\n");
1860	//return 0;
1861
1862	// Context
1863	TLSContext context;
1864	context.Initialise(false /* client */,
1865	"testfiles/clientCerts.pem",
1866	"testfiles/clientPrivKey.pem",
1867	"testfiles/clientTrustedCAs.pem");
1868
1869	// First, try logging in without an account having been created... just make sure login fails.
1870
1871	std::string cmd = BBSTORED " " + bbstored_args +
1872	" testfiles/bbstored.conf";
1873	int pid = LaunchServer(cmd.c_str(), "testfiles/bbstored.pid");
1874
1875	TEST_THAT(pid != -1 && pid != 0);
1876	if(pid > 0)
1877	{
1878	::sleep(1);
1879	TEST_THAT(ServerIsAlive(pid));
1880
1881	// BLOCK
1882	{
1883	// Open a connection to the server
1884	SocketStreamTLS conn;
1885	conn.Open(context, Socket::TypeINET, "localhost",
1886	BOX_PORT_BBSTORED_TEST);
1887
1888	// Make a protocol
1889	BackupProtocolClient protocol(conn);
1890
1891	// Check the version
1892	std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
1893	TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
1894
1895	// Login
1896	TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)),
1897	ConnectionException, Conn_Protocol_UnexpectedReply);
1898
1899	// Finish the connection
1900	protocol.QueryFinished();
1901	}
1902
1903	// Create an account for the test client
1904	TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS
1905	" -c testfiles/bbstored.conf create 01234567 0 "
1906	"10000B 20000B") == 0);
1907
1908	TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
1909
1910	TEST_THAT(TestDirExists("testfiles/0_0/backup/01234567"));
1911	TEST_THAT(TestDirExists("testfiles/0_1/backup/01234567"));
1912	TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567"));
1913	TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8);
1914	// make sure something is written to it
1915
1916	std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
1917	BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
1918
1919	std::auto_ptr<BackupStoreRefCountDatabase> apReferences(
1920	BackupStoreRefCountDatabase::Load(
1921	apAccounts->GetEntry(0x1234567), true));
1922	TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID,
1923	apReferences->GetLastObjectIDUsed());
1924	TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID))
1925	apReferences.reset();
1926
1927	// Delete the refcount database and log in again,
1928	// check that it is recreated automatically but with
1929	// no objects in it, to ensure seamless upgrade.
1930	TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw"));
1931
1932	TLSContext context;
1933	std::auto_ptr<SocketStreamTLS> conn;
1934	test_server_login("localhost", context, conn)->QueryFinished();
1935
1936	BackupStoreAccountDatabase::Entry account =
1937	apAccounts->GetEntry(0x1234567);
1938	apReferences = BackupStoreRefCountDatabase::Load(account, true);
1939	TEST_EQUAL(0, apReferences->GetLastObjectIDUsed());
1940
1941	TEST_THAT(ServerIsAlive(pid));
1942
1943	run_housekeeping(account);
1944
1945	// Check that housekeeping fixed the ref counts
1946	TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID,
1947	apReferences->GetLastObjectIDUsed());
1948	TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID))
1949
1950	TEST_THAT(ServerIsAlive(pid));
1951
1952	set_refcount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1);
1953
1954	TEST_THAT(test_server("localhost") == 0);
1955
1956	// test that all object reference counts have the
1957	// expected values
1958	TEST_EQUAL(ExpectedRefCounts.size() - 1,
1959	apReferences->GetLastObjectIDUsed());
1960	for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID;
1961	i < ExpectedRefCounts.size(); i++)
1962	{
1963	TEST_EQUAL_LINE(ExpectedRefCounts[i],
1964	apReferences->GetRefCount(i),
1965	"object " << BOX_FORMAT_OBJECTID(i));
1966	}
1967
1968	// Delete the refcount database again, and let
1969	// housekeeping recreate it and fix the ref counts.
1970	// This could also happen after upgrade, if a housekeeping
1971	// runs before the user logs in.
1972	apReferences.reset();
1973	TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw"));
1974	run_housekeeping(account);
1975	apReferences = BackupStoreRefCountDatabase::Load(account, true);
1976
1977	TEST_EQUAL(ExpectedRefCounts.size() - 1,
1978	apReferences->GetLastObjectIDUsed());
1979	for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID;
1980	i < ExpectedRefCounts.size(); i++)
1981	{
1982	TEST_EQUAL_LINE(ExpectedRefCounts[i],
1983	apReferences->GetRefCount(i),
1984	"object " << BOX_FORMAT_OBJECTID(i));
1985	}
1986
1987	// Test the deletion of objects by the housekeeping system
1988	// First, things as they are now.
1989	recursive_count_objects_results before = {0,0,0};
1990
1991	recursive_count_objects("localhost", BackupProtocolListDirectory::RootDirectory, before);
1992
1993	TEST_THAT(before.objectsNotDel != 0);
1994	TEST_THAT(before.deleted != 0);
1995	TEST_THAT(before.old != 0);
1996
1997	// Kill it
1998	TEST_THAT(KillServer(pid));
1999	::sleep(1);
2000	TEST_THAT(!ServerIsAlive(pid));
2001
2002	#ifdef WIN32
2003	TEST_THAT(unlink("testfiles/bbstored.pid") == 0);
2004	#else
2005	TestRemoteProcessMemLeaks("bbstored.memleaks");
2006	#endif
2007
2008	// Set a new limit on the account -- leave the hard limit
2009	// high to make sure the target for freeing space is the
2010	// soft limit.
2011	TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS
2012	" -c testfiles/bbstored.conf setlimit 01234567 "
2013	"10B 20000B") == 0);
2014	TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
2015
2016	// Start things up
2017	pid = LaunchServer(BBSTORED " testfiles/bbstored.conf",
2018	"testfiles/bbstored.pid");
2019
2020	::sleep(1);
2021	TEST_THAT(ServerIsAlive(pid));
2022
2023	// wait for housekeeping to happen
2024	printf("waiting for housekeeping:\n");
2025	for(int l = 0; l < 30; ++l)
2026	{
2027	::sleep(1);
2028	printf(".");
2029	fflush(stdout);
2030	}
2031	printf("\n");
2032
2033	// Count the objects again
2034	recursive_count_objects_results after = {0,0,0};
2035	recursive_count_objects("localhost",
2036	BackupProtocolListDirectory::RootDirectory,
2037	after);
2038
2039	// If these tests fail then try increasing the timeout above
2040	TEST_THAT(after.objectsNotDel == before.objectsNotDel);
2041	TEST_THAT(after.deleted == 0);
2042	TEST_THAT(after.old == 0);
2043
2044	// Set a really small hard limit
2045	TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS
2046	" -c testfiles/bbstored.conf setlimit 01234567 "
2047	"10B 20B") == 0);
2048	TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
2049
2050	// Try to upload a file and create a directory, and check an error is generated
2051	{
2052	// Open a connection to the server
2053	SocketStreamTLS conn;
2054	conn.Open(context, Socket::TypeINET, "localhost",
2055	BOX_PORT_BBSTORED_TEST);
2056
2057	// Make a protocol
2058	BackupProtocolClient protocol(conn);
2059
2060	// Check the version
2061	std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
2062	TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
2063
2064	// Login
2065	std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0));
2066
2067	int64_t modtime = 0;
2068
2069	BackupStoreFilenameClear fnx("exceed-limit");
2070	std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/test3", BackupProtocolListDirectory::RootDirectory, fnx, &modtime));
2071	TEST_THAT(modtime != 0);
2072
2073	TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolSuccess> stored(protocol.QueryStoreFile(
2074	BackupProtocolListDirectory::RootDirectory,
2075	modtime,
2076	modtime, /* use it for attr hash too */
2077	0, /* diff from ID */
2078	fnx,
2079	*upload)),
2080	ConnectionException, Conn_Protocol_UnexpectedReply);
2081
2082	MemBlockStream attr(&modtime, sizeof(modtime));
2083	BackupStoreFilenameClear fnxd("exceed-limit-dir");
2084	TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolSuccess> dirCreate(protocol.QueryCreateDirectory(
2085	BackupProtocolListDirectory::RootDirectory,
2086	9837429842987984LL, fnxd, attr)),
2087	ConnectionException, Conn_Protocol_UnexpectedReply);
2088
2089
2090	// Finish the connection
2091	protocol.QueryFinished();
2092	}
2093
2094	// Kill it again
2095	TEST_THAT(KillServer(pid));
2096	::sleep(1);
2097	TEST_THAT(!ServerIsAlive(pid));
2098
2099	#ifdef WIN32
2100	TEST_THAT(unlink("testfiles/bbstored.pid") == 0);
2101	#else
2102	TestRemoteProcessMemLeaks("bbstored.memleaks");
2103	#endif
2104	}
2105
2106	return 0;
2107	}
2108
2109	int multi_server()
2110	{
2111	printf("Starting server for connection from remote machines...\n");
2112
2113	// Create an account for the test client
2114	TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS
2115	" -c testfiles/bbstored.conf create 01234567 0 "
2116	"30000B 40000B") == 0);
2117	TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
2118
2119	// First, try logging in without an account having been created... just make sure login fails.
2120
2121	int pid = LaunchServer(BBSTORED " testfiles/bbstored_multi.conf",
2122	"testfiles/bbstored.pid");
2123
2124	TEST_THAT(pid != -1 && pid != 0);
2125	if(pid > 0)
2126	{
2127	::sleep(1);
2128	TEST_THAT(ServerIsAlive(pid));
2129
2130	// Wait for a keypress
2131	printf("Press ENTER to terminate the server\n");
2132	char line[512];
2133	fgets(line, 512, stdin);
2134	printf("Terminating server...\n");
2135
2136	// Kill it
2137	TEST_THAT(KillServer(pid));
2138	::sleep(1);
2139	TEST_THAT(!ServerIsAlive(pid));
2140
2141	#ifdef WIN32
2142	TEST_THAT(unlink("testfiles/bbstored.pid") == 0);
2143	#else
2144	TestRemoteProcessMemLeaks("bbstored.memleaks");
2145	#endif
2146	}
2147
2148
2149	return 0;
2150	}
2151
2152	#ifdef WIN32
2153	WCHAR* ConvertUtf8ToWideString(const char* pString);
2154	std::string ConvertPathToAbsoluteUnicode(const char *pFileName);
2155	#endif
2156
2157	int test(int argc, const char *argv[])
2158	{
2159	#ifdef WIN32
2160	// this had better work, or bbstored will die when combining diffs
2161	char* file = "foo";
2162	std::string abs = ConvertPathToAbsoluteUnicode(file);
2163	WCHAR* wfile = ConvertUtf8ToWideString(abs.c_str());
2164
2165	DWORD accessRights = FILE_READ_ATTRIBUTES \|
2166	FILE_LIST_DIRECTORY \| FILE_READ_EA \| FILE_WRITE_ATTRIBUTES \|
2167	FILE_WRITE_DATA \| FILE_WRITE_EA /\| FILE_ALL_ACCESS/;
2168	DWORD shareMode = FILE_SHARE_READ \| FILE_SHARE_WRITE;
2169
2170	HANDLE h1 = CreateFileW(wfile, accessRights, shareMode,
2171	NULL, OPEN_ALWAYS, FILE_FLAG_BACKUP_SEMANTICS, NULL);
2172	assert(h1 != INVALID_HANDLE_VALUE);
2173	TEST_THAT(h1 != INVALID_HANDLE_VALUE);
2174
2175	accessRights = FILE_READ_ATTRIBUTES \|
2176	FILE_LIST_DIRECTORY \| FILE_READ_EA;
2177
2178	HANDLE h2 = CreateFileW(wfile, accessRights, shareMode,
2179	NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
2180	assert(h2 != INVALID_HANDLE_VALUE);
2181	TEST_THAT(h2 != INVALID_HANDLE_VALUE);
2182
2183	CloseHandle(h2);
2184	CloseHandle(h1);
2185	delete [] wfile;
2186
2187	h1 = openfile("foo", O_CREAT \| O_RDWR, 0);
2188	TEST_THAT(h1 != INVALID_HANDLE_VALUE);
2189	h2 = openfile("foo", O_RDWR, 0);
2190	TEST_THAT(h2 != INVALID_HANDLE_VALUE);
2191	CloseHandle(h2);
2192	CloseHandle(h1);
2193	#endif
2194
2195	// SSL library
2196	SSLLib::Initialise();
2197
2198	// Give a test key for the filenames
2199	// BackupStoreFilenameClear::SetBlowfishKey(FilenameEncodingKey, sizeof(FilenameEncodingKey));
2200	// And set the encoding to blowfish
2201	// BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish);
2202
2203	// And for directory attributes -- need to set it, as used in file encoding
2204	// BackupClientFileAttributes::SetBlowfishKey(AttributesEncodingKey, sizeof(AttributesEncodingKey));
2205
2206	// And finally for file encoding
2207	// BackupStoreFile::SetBlowfishKeys(FileEncodingKey, sizeof(FileEncodingKey), FileBlockEntryEncodingKey, sizeof(FileBlockEntryEncodingKey));
2208
2209	// Use the setup crypto command to set up all these keys, so that the bbackupquery command can be used
2210	// for seeing what's going on.
2211	BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys");
2212
2213	// encode in some filenames -- can't do static initialisation
2214	// because the key won't be set up when these are initialised
2215	{
2216	MEMLEAKFINDER_NO_LEAKS
2217
2218	for(unsigned int l = 0; l < sizeof(ens_filenames) / sizeof(ens_filenames[0]); ++l)
2219	{
2220	ens[l].fn = BackupStoreFilenameClear(ens_filenames[l]);
2221	}
2222	for(unsigned int l = 0; l < sizeof(uploads_filenames) / sizeof(uploads_filenames[0]); ++l)
2223	{
2224	uploads[l].name = BackupStoreFilenameClear(uploads_filenames[l]);
2225	}
2226	}
2227
2228	// Trace errors out
2229	SET_DEBUG_SSLLIB_TRACE_ERRORS
2230
2231	if(argc == 2 && strcmp(argv[1], "server") == 0)
2232	{
2233	return multi_server();
2234	}
2235	if(argc == 3 && strcmp(argv[1], "client") == 0)
2236	{
2237	return test_server(argv[2]);
2238	}
2239	// large file test
2240	/* {
2241	int64_t modtime = 0;
2242	std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("/Users/ben/temp/large.tar",
2243	BackupProtocolListDirectory::RootDirectory, uploads[0].name, &modtime));
2244	TEST_THAT(modtime != 0);
2245	FileStream write("testfiles/large.enc", O_WRONLY \| O_CREAT);
2246	upload->CopyStreamTo(write);
2247	}
2248	printf("SKIPPING TESTS ------------------------------------------------------\n");
2249	return 0;*/
2250	int r = 0;
2251	r = test1(argc, argv);
2252	if(r != 0) return r;
2253	r = test2(argc, argv);
2254	if(r != 0) return r;
2255	r = test3(argc, argv);
2256	if(r != 0) return r;
2257	return 0;
2258	}
2259

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: box/trunk/test/backupstore/testbackupstore.cpp @ 2983

Download in other formats: