Context Navigation

BackupDaemon.cpp @ 3102

Revision 3102, 89.5 KB checked in by chris, 4 weeks ago (diff)
Allow BackupDaemon? user to reset state for testing.
Property svn:eol-style set to `native`

Line
1	// --------------------------------------------------------------------------
2	//
3	// File
4	// Name: BackupDaemon.cpp
5	// Purpose: Backup daemon
6	// Created: 2003/10/08
7	//
8	// --------------------------------------------------------------------------
9
10	#include "Box.h"
11
12	#include <stdio.h>
13	#include <stdlib.h>
14	#include <string.h>
15
16	#ifdef HAVE_UNISTD_H
17	#include <unistd.h>
18	#endif
19	#ifdef HAVE_SIGNAL_H
20	#include <signal.h>
21	#endif
22	#ifdef HAVE_SYS_PARAM_H
23	#include <sys/param.h>
24	#endif
25	#ifdef HAVE_SYS_WAIT_H
26	#include <sys/wait.h>
27	#endif
28	#ifdef HAVE_SYS_MOUNT_H
29	#include <sys/mount.h>
30	#endif
31	#ifdef HAVE_MNTENT_H
32	#include <mntent.h>
33	#endif
34	#ifdef HAVE_SYS_MNTTAB_H
35	#include <cstdio>
36	#include <sys/mnttab.h>
37	#endif
38	#ifdef HAVE_PROCESS_H
39	#include <process.h>
40	#endif
41
42	#include <iostream>
43	#include <set>
44	#include <sstream>
45
46	#include "Configuration.h"
47	#include "IOStream.h"
48	#include "MemBlockStream.h"
49	#include "CommonException.h"
50	#include "BoxPortsAndFiles.h"
51
52	#include "SSLLib.h"
53
54	#include "autogen_BackupProtocol.h"
55	#include "autogen_ClientException.h"
56	#include "autogen_ConversionException.h"
57	#include "Archive.h"
58	#include "BackupClientContext.h"
59	#include "BackupClientCryptoKeys.h"
60	#include "BackupClientDirectoryRecord.h"
61	#include "BackupClientFileAttributes.h"
62	#include "BackupClientInodeToIDMap.h"
63	#include "BackupClientMakeExcludeList.h"
64	#include "BackupDaemon.h"
65	#include "BackupDaemonConfigVerify.h"
66	#include "BackupStoreConstants.h"
67	#include "BackupStoreDirectory.h"
68	#include "BackupStoreException.h"
69	#include "BackupStoreFile.h"
70	#include "BackupStoreFilenameClear.h"
71	#include "BannerText.h"
72	#include "Conversion.h"
73	#include "ExcludeList.h"
74	#include "FileStream.h"
75	#include "IOStreamGetLine.h"
76	#include "LocalProcessStream.h"
77	#include "Logging.h"
78	#include "Random.h"
79	#include "Timer.h"
80	#include "Utils.h"
81
82	#ifdef WIN32
83	#include "Win32ServiceFunctions.h"
84	#include "Win32BackupService.h"
85
86	extern Win32BackupService* gpDaemonService;
87
88	# ifdef ENABLE_VSS
89	# include <comdef.h>
90	# include <Vss.h>
91	# include <VsWriter.h>
92	# include <VsBackup.h>
93
94	// http://www.flounder.com/cstring.htm
95	std::string GetMsgForHresult(HRESULT hr)
96	{
97	std::ostringstream buf;
98
99	if(hr == VSS_S_ASYNC_CANCELLED)
100	{
101	buf << "VSS async operation cancelled";
102	}
103	else if(hr == VSS_S_ASYNC_FINISHED)
104	{
105	buf << "VSS async operation finished";
106	}
107	else if(hr == VSS_S_ASYNC_PENDING)
108	{
109	buf << "VSS async operation pending";
110	}
111	else
112	{
113	buf << _com_error(hr).ErrorMessage();
114	}
115
116	buf << " (" << BOX_FORMAT_HEX32(hr) << ")";
117	return buf.str();
118	}
119
120	std::string WideStringToString(WCHAR *buf)
121	{
122	if (buf == NULL)
123	{
124	return "(null)";
125	}
126
127	char* pStr = ConvertFromWideString(buf, CP_UTF8);
128
129	if(pStr == NULL)
130	{
131	return "(conversion failed)";
132	}
133
134	std::string result(pStr);
135	free(pStr);
136	return result;
137	}
138
139	std::string GuidToString(GUID guid)
140	{
141	wchar_t buf[64];
142	StringFromGUID2(guid, buf, sizeof(buf));
143	return WideStringToString(buf);
144	}
145
146	std::string BstrToString(const BSTR arg)
147	{
148	if(arg == NULL)
149	{
150	return std::string("(null)");
151	}
152	else
153	{
154	// Extract the long before where the arg points to
155	long len = ((long *)arg)[-1] / 2;
156	std::wstring wstr((WCHAR *)arg, len);
157	std::string str;
158	if(!ConvertFromWideString(wstr, &str, CP_UTF8))
159	{
160	throw std::exception("string conversion failed");
161	}
162	return str;
163	}
164	}
165	# endif
166	#endif
167
168	#include "MemLeakFindOn.h"
169
170	static const time_t MAX_SLEEP_TIME = 1024;
171
172	// Make the actual sync period have a little bit of extra time, up to a 64th of the main sync period.
173	// This prevents repetative cycles of load on the server
174	#define SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY 6
175
176	// --------------------------------------------------------------------------
177	//
178	// Function
179	// Name: BackupDaemon::BackupDaemon()
180	// Purpose: constructor
181	// Created: 2003/10/08
182	//
183	// --------------------------------------------------------------------------
184	BackupDaemon::BackupDaemon()
185	: mState(BackupDaemon::State_Initialising),
186	mDeleteRedundantLocationsAfter(0),
187	mLastNotifiedEvent(SysadminNotifier::MAX),
188	mDeleteUnusedRootDirEntriesAfter(0),
189	mClientStoreMarker(BackupClientContext::ClientStoreMarker_NotKnown),
190	mStorageLimitExceeded(false),
191	mReadErrorsOnFilesystemObjects(false),
192	mLastSyncTime(0),
193	mNextSyncTime(0),
194	mCurrentSyncStartTime(0),
195	mUpdateStoreInterval(0),
196	mDeleteStoreObjectInfoFile(false),
197	mDoSyncForcedByPreviousSyncError(false),
198	mNumFilesUploaded(-1),
199	mNumDirsCreated(-1),
200	mMaxBandwidthFromSyncAllowScript(0),
201	mLogAllFileAccess(false),
202	mpProgressNotifier(this),
203	mpLocationResolver(this),
204	mpRunStatusProvider(this),
205	mpSysadminNotifier(this)
206	#ifdef WIN32
207	, mInstallService(false),
208	mRemoveService(false),
209	mRunAsService(false),
210	mServiceName("bbackupd")
211	#endif
212	#ifdef ENABLE_VSS
213	, mpVssBackupComponents(NULL)
214	#endif
215	{
216	// Only ever one instance of a daemon
217	SSLLib::Initialise();
218	}
219
220	// --------------------------------------------------------------------------
221	//
222	// Function
223	// Name: BackupDaemon::~BackupDaemon()
224	// Purpose: Destructor
225	// Created: 2003/10/08
226	//
227	// --------------------------------------------------------------------------
228	BackupDaemon::~BackupDaemon()
229	{
230	DeleteAllLocations();
231	DeleteAllIDMaps();
232	}
233
234	// --------------------------------------------------------------------------
235	//
236	// Function
237	// Name: BackupDaemon::DaemonName()
238	// Purpose: Get name of daemon
239	// Created: 2003/10/08
240	//
241	// --------------------------------------------------------------------------
242	const char *BackupDaemon::DaemonName() const
243	{
244	return "bbackupd";
245	}
246
247
248	// --------------------------------------------------------------------------
249	//
250	// Function
251	// Name: BackupDaemon::DaemonBanner()
252	// Purpose: Daemon banner
253	// Created: 1/1/04
254	//
255	// --------------------------------------------------------------------------
256	std::string BackupDaemon::DaemonBanner() const
257	{
258	return BANNER_TEXT("Backup Client");
259	}
260
261	void BackupDaemon::Usage()
262	{
263	this->Daemon::Usage();
264
265	#ifdef WIN32
266	std::cout <<
267	" -s Run as a Windows Service, for internal use only\n"
268	" -i Install Windows Service (you may want to specify a config file)\n"
269	" -r Remove Windows Service\n"
270	" -S <name> Service name for -i and -r options\n";
271	#endif
272	}
273
274
275	// --------------------------------------------------------------------------
276	//
277	// Function
278	// Name: BackupDaemon::GetConfigVerify()
279	// Purpose: Get configuration specification
280	// Created: 2003/10/08
281	//
282	// --------------------------------------------------------------------------
283	const ConfigurationVerify *BackupDaemon::GetConfigVerify() const
284	{
285	// Defined elsewhere
286	return &BackupDaemonConfigVerify;
287	}
288
289	#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET
290	// --------------------------------------------------------------------------
291	//
292	// Function
293	// Name: BackupDaemon::SetupInInitialProcess()
294	// Purpose: Platforms with non-checkable credentials on
295	// local sockets only.
296	// Prints a warning if the command socket is used.
297	// Created: 25/2/04
298	//
299	// --------------------------------------------------------------------------
300	void BackupDaemon::SetupInInitialProcess()
301	{
302	// Print a warning on this platform if the CommandSocket is used.
303	if(GetConfiguration().KeyExists("CommandSocket"))
304	{
305	BOX_WARNING(
306	"==============================================================================\n"
307	"SECURITY WARNING: This platform cannot check the credentials of connections to\n"
308	"the command socket. This is a potential DoS security problem.\n"
309	"Remove the CommandSocket directive from the bbackupd.conf file if bbackupctl\n"
310	"is not used.\n"
311	"==============================================================================\n"
312	);
313	}
314	}
315	#endif
316
317
318	// --------------------------------------------------------------------------
319	//
320	// Function
321	// Name: BackupDaemon::DeleteAllLocations()
322	// Purpose: Deletes all records stored
323	// Created: 2003/10/08
324	//
325	// --------------------------------------------------------------------------
326	void BackupDaemon::DeleteAllLocations()
327	{
328	// Run through, and delete everything
329	for(Locations::iterator i = mLocations.begin();
330	i != mLocations.end(); ++i)
331	{
332	delete *i;
333	}
334
335	// Clear the contents of the map, so it is empty
336	mLocations.clear();
337
338	// And delete everything from the associated mount vector
339	mIDMapMounts.clear();
340	}
341
342	#ifdef WIN32
343	std::string BackupDaemon::GetOptionString()
344	{
345	std::string oldOpts = this->Daemon::GetOptionString();
346	ASSERT(oldOpts.find("s") == std::string::npos);
347	ASSERT(oldOpts.find("S") == std::string::npos);
348	ASSERT(oldOpts.find("i") == std::string::npos);
349	ASSERT(oldOpts.find("r") == std::string::npos);
350	return oldOpts + "sS:ir";
351	}
352
353	int BackupDaemon::ProcessOption(signed int option)
354	{
355	switch(option)
356	{
357	case 's':
358	{
359	mRunAsService = true;
360	return 0;
361	}
362
363	case 'S':
364	{
365	mServiceName = optarg;
366	Logging::SetProgramName(mServiceName);
367	return 0;
368	}
369
370	case 'i':
371	{
372	mInstallService = true;
373	return 0;
374	}
375
376	case 'r':
377	{
378	mRemoveService = true;
379	return 0;
380	}
381
382	default:
383	{
384	return this->Daemon::ProcessOption(option);
385	}
386	}
387	}
388
389	int BackupDaemon::Main(const std::string &rConfigFileName)
390	{
391	if (mInstallService)
392	{
393	return InstallService(rConfigFileName.c_str(), mServiceName);
394	}
395
396	if (mRemoveService)
397	{
398	return RemoveService(mServiceName);
399	}
400
401	#ifdef ENABLE_VSS
402	HRESULT result = CoInitialize(NULL);
403	if(result != S_OK)
404	{
405	BOX_ERROR("VSS: Failed to initialize COM: " <<
406	GetMsgForHresult(result));
407	return 1;
408	}
409	#endif
410
411	int returnCode;
412
413	if (mRunAsService)
414	{
415	// We will be called reentrantly by the Service Control
416	// Manager, and we had better not call OurService again!
417	mRunAsService = false;
418
419	BOX_INFO("Box Backup service starting");
420	returnCode = OurService(rConfigFileName.c_str());
421	BOX_INFO("Box Backup service shut down");
422	}
423	else
424	{
425	returnCode = this->Daemon::Main(rConfigFileName);
426	}
427
428	return returnCode;
429	}
430	#endif
431
432	// --------------------------------------------------------------------------
433	//
434	// Function
435	// Name: BackupDaemon::Run()
436	// Purpose: Run function for daemon
437	// Created: 18/2/04
438	//
439	// --------------------------------------------------------------------------
440	void BackupDaemon::Run()
441	{
442	// initialise global timer mechanism
443	Timers::Init();
444
445	#ifndef WIN32
446	// Ignore SIGPIPE so that if a command connection is broken,
447	// the daemon doesn't terminate.
448	::signal(SIGPIPE, SIG_IGN);
449	#endif
450
451	// Create a command socket?
452	const Configuration &conf(GetConfiguration());
453	if(conf.KeyExists("CommandSocket"))
454	{
455	// Yes, create a local UNIX socket
456	mapCommandSocketInfo.reset(new CommandSocketInfo);
457	const char *socketName =
458	conf.GetKeyValue("CommandSocket").c_str();
459	#ifdef WIN32
460	mapCommandSocketInfo->mListeningSocket.Listen(
461	socketName);
462	#else
463	::unlink(socketName);
464	mapCommandSocketInfo->mListeningSocket.Listen(
465	Socket::TypeUNIX, socketName);
466	#endif
467	}
468
469	// Handle things nicely on exceptions
470	try
471	{
472	Run2();
473	}
474	catch(...)
475	{
476	if(mapCommandSocketInfo.get())
477	{
478	try
479	{
480	mapCommandSocketInfo.reset();
481	}
482	catch(std::exception &e)
483	{
484	BOX_WARNING("Internal error while "
485	"closing command socket after "
486	"another exception: " << e.what());
487	}
488	catch(...)
489	{
490	BOX_WARNING("Error closing command socket "
491	"after exception, ignored.");
492	}
493	}
494
495	Timers::Cleanup();
496
497	throw;
498	}
499
500	// Clean up
501	mapCommandSocketInfo.reset();
502	Timers::Cleanup();
503	}
504
505	void BackupDaemon::InitCrypto()
506	{
507	// Read in the certificates creating a TLS context
508	const Configuration &conf(GetConfiguration());
509	std::string certFile(conf.GetKeyValue("CertificateFile"));
510	std::string keyFile(conf.GetKeyValue("PrivateKeyFile"));
511	std::string caFile(conf.GetKeyValue("TrustedCAsFile"));
512	mTlsContext.Initialise(false /* as client */, certFile.c_str(),
513	keyFile.c_str(), caFile.c_str());
514
515	// Set up the keys for various things
516	BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile"));
517	}
518
519	// --------------------------------------------------------------------------
520	//
521	// Function
522	// Name: BackupDaemon::Run2()
523	// Purpose: Run function for daemon (second stage)
524	// Created: 2003/10/08
525	//
526	// --------------------------------------------------------------------------
527	void BackupDaemon::Run2()
528	{
529	InitCrypto();
530
531	const Configuration &conf(GetConfiguration());
532
533	// How often to connect to the store (approximate)
534	mUpdateStoreInterval = SecondsToBoxTime(
535	conf.GetKeyValueInt("UpdateStoreInterval"));
536
537	// But are we connecting automatically?
538	bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup");
539
540	// When the next sync should take place -- which is ASAP
541	mNextSyncTime = 0;
542
543	// When the last sync started (only updated if the store was not full when the sync ended)
544	mLastSyncTime = 0;
545
546	// --------------------------------------------------------------------------------------------
547
548	mDeleteStoreObjectInfoFile = DeserializeStoreObjectInfo(
549	mLastSyncTime, mNextSyncTime);
550
551	// --------------------------------------------------------------------------------------------
552
553
554	// Set state
555	SetState(State_Idle);
556
557	mDoSyncForcedByPreviousSyncError = false;
558
559	// Loop around doing backups
560	do
561	{
562	// Flags used below
563	bool storageLimitExceeded = false;
564	bool doSync = false;
565	bool mDoSyncForcedByCommand = false;
566
567	// Is a delay necessary?
568	box_time_t currentTime;
569
570	do
571	{
572	// Check whether we should be stopping,
573	// and don't run a sync if so.
574	if(StopRun()) break;
575
576	currentTime = GetCurrentBoxTime();
577
578	// Pause a while, but no more than
579	// MAX_SLEEP_TIME seconds (use the conditional
580	// because times are unsigned)
581	box_time_t requiredDelay =
582	(mNextSyncTime < currentTime)
583	? (0)
584	: (mNextSyncTime - currentTime);
585
586	// If there isn't automatic backup happening,
587	// set a long delay. And limit delays at the
588	// same time.
589	if(!automaticBackup && !mDoSyncForcedByPreviousSyncError)
590	{
591	requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME);
592	}
593	else if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME))
594	{
595	requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME);
596	}
597
598	// Only delay if necessary
599	if(requiredDelay > 0)
600	{
601	// Sleep somehow. There are choices
602	// on how this should be done,
603	// depending on the state of the
604	// control connection
605	if(mapCommandSocketInfo.get() != 0)
606	{
607	// A command socket exists,
608	// so sleep by waiting on it
609	WaitOnCommandSocket(requiredDelay,
610	doSync, mDoSyncForcedByCommand);
611	}
612	else
613	{
614	// No command socket or
615	// connection, just do a
616	// normal sleep
617	time_t sleepSeconds =
618	BoxTimeToSeconds(requiredDelay);
619	::sleep((sleepSeconds <= 0)
620	? 1 : sleepSeconds);
621	}
622	}
623
624	if ((automaticBackup \|\| mDoSyncForcedByPreviousSyncError)
625	&& currentTime >= mNextSyncTime)
626	{
627	doSync = true;
628	}
629	}
630	while(!doSync && !StopRun());
631
632	// Time of sync start, and if it's time for another sync
633	// (and we're doing automatic syncs), set the flag
634	mCurrentSyncStartTime = GetCurrentBoxTime();
635	if((automaticBackup \|\| mDoSyncForcedByPreviousSyncError) &&
636	mCurrentSyncStartTime >= mNextSyncTime)
637	{
638	doSync = true;
639	}
640
641	// Use a script to see if sync is allowed now?
642	if(!mDoSyncForcedByCommand && doSync && !StopRun())
643	{
644	int d = UseScriptToSeeIfSyncAllowed();
645	if(d > 0)
646	{
647	// Script has asked for a delay
648	mNextSyncTime = GetCurrentBoxTime() +
649	SecondsToBoxTime(d);
650	doSync = false;
651	}
652	}
653
654	// Ready to sync? (but only if we're not supposed
655	// to be stopping)
656	if(doSync && !StopRun())
657	{
658	RunSyncNowWithExceptionHandling();
659	}
660
661	// Set state
662	SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle);
663
664	} while(!StopRun());
665
666	// Make sure we have a clean start next time round (if restart)
667	DeleteAllLocations();
668	DeleteAllIDMaps();
669	}
670
671	void BackupDaemon::RunSyncNowWithExceptionHandling()
672	{
673	bool errorOccurred = false;
674	int errorCode = 0, errorSubCode = 0;
675	const char* errorString = "unknown";
676
677	try
678	{
679	OnBackupStart();
680	// Do sync
681	RunSyncNow();
682	}
683	catch(BoxException &e)
684	{
685	errorOccurred = true;
686	errorString = e.what();
687	errorCode = e.GetType();
688	errorSubCode = e.GetSubType();
689	}
690	catch(std::exception &e)
691	{
692	BOX_ERROR("Internal error during backup run: " << e.what());
693	errorOccurred = true;
694	errorString = e.what();
695	}
696	catch(...)
697	{
698	// TODO: better handling of exceptions here...
699	// need to be very careful
700	errorOccurred = true;
701	}
702
703	// do not retry immediately without a good reason
704	mDoSyncForcedByPreviousSyncError = false;
705
706	// Notify system administrator about the final state of the backup
707	if(errorOccurred)
708	{
709	// Is it a berkely db failure?
710	bool isBerkelyDbFailure = false;
711
712	if (errorCode == BackupStoreException::ExceptionType
713	&& errorSubCode == BackupStoreException::BerkelyDBFailure)
714	{
715	isBerkelyDbFailure = true;
716	}
717
718	if(isBerkelyDbFailure)
719	{
720	// Delete corrupt files
721	DeleteCorruptBerkelyDbFiles();
722	}
723
724	ResetCachedState();
725
726	// Handle restart?
727	if(StopRun())
728	{
729	BOX_NOTICE("Exception (" << errorCode
730	<< "/" << errorSubCode
731	<< ") due to signal");
732	OnBackupFinish();
733	return;
734	}
735
736	NotifySysadmin(SysadminNotifier::BackupError);
737
738	// If the Berkely db files get corrupted,
739	// delete them and try again immediately.
740	if(isBerkelyDbFailure)
741	{
742	BOX_ERROR("Berkely db inode map files corrupted, "
743	"deleting and restarting scan. Renamed files "
744	"and directories will not be tracked until "
745	"after this scan.");
746	::sleep(1);
747	}
748	else
749	{
750	// Not restart/terminate, pause and retry
751	// Notify administrator
752	SetState(State_Error);
753	BOX_ERROR("Exception caught (" << errorString <<
754	" " << errorCode << "/" << errorSubCode <<
755	"), reset state and waiting to retry...");
756	::sleep(10);
757	mNextSyncTime = mCurrentSyncStartTime +
758	SecondsToBoxTime(100) +
759	Random::RandomInt(mUpdateStoreInterval >>
760	SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY);
761	}
762	}
763
764	if(mReadErrorsOnFilesystemObjects)
765	{
766	NotifySysadmin(SysadminNotifier::ReadError);
767	}
768
769	if(mStorageLimitExceeded)
770	{
771	NotifySysadmin(SysadminNotifier::StoreFull);
772	}
773
774	if (!errorOccurred && !mReadErrorsOnFilesystemObjects &&
775	!mStorageLimitExceeded)
776	{
777	NotifySysadmin(SysadminNotifier::BackupOK);
778	}
779
780	// If we were retrying after an error, and this backup succeeded,
781	// then now would be a good time to stop :-)
782	mDoSyncForcedByPreviousSyncError = errorOccurred;
783
784	OnBackupFinish();
785	}
786
787	void BackupDaemon::ResetCachedState()
788	{
789	// Clear state data
790	// Go back to beginning of time
791	mLastSyncTime = 0;
792	mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything
793	DeleteAllLocations();
794	DeleteAllIDMaps();
795	}
796
797	void BackupDaemon::RunSyncNow()
798	{
799	// Delete the serialised store object file,
800	// so that we don't try to reload it after a
801	// partially completed backup
802	if(mDeleteStoreObjectInfoFile &&
803	!DeleteStoreObjectInfo())
804	{
805	BOX_ERROR("Failed to delete the StoreObjectInfoFile, "
806	"backup cannot continue safely.");
807	THROW_EXCEPTION(ClientException,
808	FailedToDeleteStoreObjectInfoFile);
809	}
810
811	// In case the backup throws an exception,
812	// we should not try to delete the store info
813	// object file again.
814	mDeleteStoreObjectInfoFile = false;
815
816	const Configuration &conf(GetConfiguration());
817
818	std::auto_ptr<FileLogger> fileLogger;
819
820	if (conf.KeyExists("LogFile"))
821	{
822	Log::Level level = Log::INFO;
823	if (conf.KeyExists("LogFileLevel"))
824	{
825	level = Logging::GetNamedLevel(
826	conf.GetKeyValue("LogFileLevel"));
827	}
828	fileLogger.reset(new FileLogger(conf.GetKeyValue("LogFile"),
829	level));
830	}
831
832	std::string extendedLogFile;
833	if (conf.KeyExists("ExtendedLogFile"))
834	{
835	extendedLogFile = conf.GetKeyValue("ExtendedLogFile");
836	}
837
838	if (conf.KeyExists("LogAllFileAccess"))
839	{
840	mLogAllFileAccess = conf.GetKeyValueBool("LogAllFileAccess");
841	}
842
843	// Then create a client context object (don't
844	// just connect, as this may be unnecessary)
845	BackupClientContext clientContext
846	(
847	*mpLocationResolver,
848	mTlsContext,
849	conf.GetKeyValue("StoreHostname"),
850	conf.GetKeyValueInt("StorePort"),
851	conf.GetKeyValueUint32("AccountNumber"),
852	conf.GetKeyValueBool("ExtendedLogging"),
853	conf.KeyExists("ExtendedLogFile"),
854	extendedLogFile,
855	*mpProgressNotifier,
856	conf.GetKeyValueBool("TcpNice")
857	);
858
859	// The minimum age a file needs to be before it will be
860	// considered for uploading
861	box_time_t minimumFileAge = SecondsToBoxTime(
862	conf.GetKeyValueInt("MinimumFileAge"));
863
864	// The maximum time we'll wait to upload a file, regardless
865	// of how often it's modified
866	box_time_t maxUploadWait = SecondsToBoxTime(
867	conf.GetKeyValueInt("MaxUploadWait"));
868	// Adjust by subtracting the minimum file age, so is relative
869	// to sync period end in comparisons
870	if (maxUploadWait > minimumFileAge)
871	{
872	maxUploadWait -= minimumFileAge;
873	}
874	else
875	{
876	maxUploadWait = 0;
877	}
878
879	// Calculate the sync period of files to examine
880	box_time_t syncPeriodStart = mLastSyncTime;
881	box_time_t syncPeriodEnd = GetCurrentBoxTime() - minimumFileAge;
882
883	if(syncPeriodStart >= syncPeriodEnd &&
884	syncPeriodStart - syncPeriodEnd < minimumFileAge)
885	{
886	// This can happen if we receive a force-sync command less
887	// than minimumFileAge after the last sync. Deal with it by
888	// moving back syncPeriodStart, which should not do any
889	// damage.
890	syncPeriodStart = syncPeriodEnd -
891	SecondsToBoxTime(1);
892	}
893
894	if(syncPeriodStart >= syncPeriodEnd)
895	{
896	BOX_ERROR("Invalid (negative) sync period: "
897	"perhaps your clock is going "
898	"backwards (" << syncPeriodStart <<
899	" to " << syncPeriodEnd << ")");
900	THROW_EXCEPTION(ClientException,
901	ClockWentBackwards);
902	}
903
904	// Check logic
905	ASSERT(syncPeriodEnd > syncPeriodStart);
906	// Paranoid check on sync times
907	if(syncPeriodStart >= syncPeriodEnd) return;
908
909	// Adjust syncPeriodEnd to emulate snapshot
910	// behaviour properly
911	box_time_t syncPeriodEndExtended = syncPeriodEnd;
912
913	// Using zero min file age?
914	if(minimumFileAge == 0)
915	{
916	// Add a year on to the end of the end time,
917	// to make sure we sync files which are
918	// modified after the scan run started.
919	// Of course, they may be eligible to be
920	// synced again the next time round,
921	// but this should be OK, because the changes
922	// only upload should upload no data.
923	syncPeriodEndExtended += SecondsToBoxTime(
924	(time_t)(356243600));
925	}
926
927	// Set up the sync parameters
928	BackupClientDirectoryRecord::SyncParams params(*mpRunStatusProvider,
929	mpSysadminNotifier, mpProgressNotifier, clientContext);
930	params.mSyncPeriodStart = syncPeriodStart;
931	params.mSyncPeriodEnd = syncPeriodEndExtended;
932	// use potentially extended end time
933	params.mMaxUploadWait = maxUploadWait;
934	params.mFileTrackingSizeThreshold =
935	conf.GetKeyValueInt("FileTrackingSizeThreshold");
936	params.mDiffingUploadSizeThreshold =
937	conf.GetKeyValueInt("DiffingUploadSizeThreshold");
938	params.mMaxFileTimeInFuture =
939	SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture"));
940	mNumFilesUploaded = 0;
941	mNumDirsCreated = 0;
942
943	if(conf.KeyExists("MaxUploadRate"))
944	{
945	params.mMaxUploadRate = conf.GetKeyValueInt("MaxUploadRate");
946	}
947
948	if(mMaxBandwidthFromSyncAllowScript != 0)
949	{
950	params.mMaxUploadRate = mMaxBandwidthFromSyncAllowScript;
951	}
952
953	mDeleteRedundantLocationsAfter =
954	conf.GetKeyValueInt("DeleteRedundantLocationsAfter");
955	mStorageLimitExceeded = false;
956	mReadErrorsOnFilesystemObjects = false;
957
958	// Setup various timings
959	int maximumDiffingTime = 600;
960	int keepAliveTime = 60;
961
962	// max diffing time, keep-alive time
963	if(conf.KeyExists("MaximumDiffingTime"))
964	{
965	maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime");
966	}
967	if(conf.KeyExists("KeepAliveTime"))
968	{
969	keepAliveTime = conf.GetKeyValueInt("KeepAliveTime");
970	}
971
972	clientContext.SetMaximumDiffingTime(maximumDiffingTime);
973	clientContext.SetKeepAliveTime(keepAliveTime);
974
975	// Set store marker
976	clientContext.SetClientStoreMarker(mClientStoreMarker);
977
978	// Set up the locations, if necessary --
979	// need to do it here so we have a
980	// (potential) connection to use
981	{
982	const Configuration &locations(
983	conf.GetSubConfiguration(
984	"BackupLocations"));
985
986	// Make sure all the directory records
987	// are set up
988	SetupLocations(clientContext, locations);
989	}
990
991	mpProgressNotifier->NotifyIDMapsSetup(clientContext);
992
993	// Get some ID maps going
994	SetupIDMapsForSync();
995
996	// Delete any unused directories?
997	DeleteUnusedRootDirEntries(clientContext);
998
999	#ifdef ENABLE_VSS
1000	CreateVssBackupComponents();
1001	#endif
1002
1003	// Go through the records, syncing them
1004	for(Locations::const_iterator
1005	i(mLocations.begin());
1006	i != mLocations.end(); ++i)
1007	{
1008	// Set current and new ID map pointers
1009	// in the context
1010	clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex],
1011	mNewIDMaps[(*i)->mIDMapIndex]);
1012
1013	// Set exclude lists (context doesn't
1014	// take ownership)
1015	clientContext.SetExcludeLists(
1016	(*i)->mpExcludeFiles,
1017	(*i)->mpExcludeDirs);
1018
1019	// Sync the directory
1020	std::string locationPath = (*i)->mPath;
1021	#ifdef ENABLE_VSS
1022	if((*i)->mIsSnapshotCreated)
1023	{
1024	locationPath = (*i)->mSnapshotPath;
1025	}
1026	#endif
1027
1028	(*i)->mpDirectoryRecord->SyncDirectory(params,
1029	BackupProtocolListDirectory::RootDirectory,
1030	locationPath, std::string("/") + (i)->mName, *i);
1031
1032	// Unset exclude lists (just in case)
1033	clientContext.SetExcludeLists(0, 0);
1034	}
1035
1036	// Perform any deletions required -- these are
1037	// delayed until the end to allow renaming to
1038	// happen neatly.
1039	clientContext.PerformDeletions();
1040
1041	// Close any open connection
1042	clientContext.CloseAnyOpenConnection();
1043
1044	#ifdef ENABLE_VSS
1045	CleanupVssBackupComponents();
1046	#endif
1047
1048	// Get the new store marker
1049	mClientStoreMarker = clientContext.GetClientStoreMarker();
1050	mStorageLimitExceeded = clientContext.StorageLimitExceeded();
1051	mReadErrorsOnFilesystemObjects \|=
1052	params.mReadErrorsOnFilesystemObjects;
1053
1054	if(!mStorageLimitExceeded)
1055	{
1056	// The start time of the next run is the end time of this
1057	// run. This is only done if the storage limit wasn't
1058	// exceeded (as things won't have been done properly if
1059	// it was)
1060	mLastSyncTime = syncPeriodEnd;
1061	}
1062
1063	// Commit the ID Maps
1064	CommitIDMapsAfterSync();
1065
1066	// Calculate when the next sync run should be
1067	mNextSyncTime = mCurrentSyncStartTime +
1068	mUpdateStoreInterval +
1069	Random::RandomInt(mUpdateStoreInterval >>
1070	SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY);
1071
1072	// --------------------------------------------------------------------------------------------
1073
1074	// We had a successful backup, save the store
1075	// info. If we save successfully, we must
1076	// delete the file next time we start a backup
1077
1078	mDeleteStoreObjectInfoFile =
1079	SerializeStoreObjectInfo(mLastSyncTime,
1080	mNextSyncTime);
1081
1082	// --------------------------------------------------------------------------------------------
1083	}
1084
1085	#ifdef ENABLE_VSS
1086	bool BackupDaemon::WaitForAsync(IVssAsync *pAsync,
1087	const std::string& description)
1088	{
1089	BOX_INFO("VSS: waiting for " << description << " to complete");
1090	HRESULT result;
1091
1092	do
1093	{
1094	result = pAsync->Wait(1000);
1095	if(result != S_OK)
1096	{
1097	BOX_ERROR("VSS: Failed to wait for " << description <<
1098	" to complete: " << GetMsgForHresult(result));
1099	break;
1100	}
1101
1102	HRESULT result2;
1103	result = pAsync->QueryStatus(&result2, NULL);
1104	if(result != S_OK)
1105	{
1106	BOX_ERROR("VSS: Failed to query " << description <<
1107	" status: " << GetMsgForHresult(result));
1108	break;
1109	}
1110
1111	result = result2;
1112	BOX_INFO("VSS: " << description << " status: " <<
1113	GetMsgForHresult(result));
1114	}
1115	while(result == VSS_S_ASYNC_PENDING);
1116
1117	pAsync->Release();
1118
1119	return (result == VSS_S_ASYNC_FINISHED);
1120	}
1121
1122	#define CALL_MEMBER_FN(object, method) ((object).*(method))
1123
1124	bool BackupDaemon::CallAndWaitForAsync(AsyncMethod method,
1125	const std::string& description)
1126	{
1127	IVssAsync *pAsync;
1128	HRESULT result = CALL_MEMBER_FN(*mpVssBackupComponents, method)(&pAsync);
1129	if(result != S_OK)
1130	{
1131	BOX_ERROR("VSS: " << description << " failed: " <<
1132	GetMsgForHresult(result));
1133	return false;
1134	}
1135
1136	return WaitForAsync(pAsync, description);
1137	}
1138
1139	void FreeSnapshotProp(VSS_SNAPSHOT_PROP *pSnap)
1140	{
1141	CoTaskMemFree(pSnap->m_pwszSnapshotDeviceObject);
1142	CoTaskMemFree(pSnap->m_pwszOriginalVolumeName);
1143	CoTaskMemFree(pSnap->m_pwszOriginatingMachine);
1144	CoTaskMemFree(pSnap->m_pwszServiceMachine);
1145	CoTaskMemFree(pSnap->m_pwszExposedName);
1146	CoTaskMemFree(pSnap->m_pwszExposedPath);
1147	}
1148
1149	void BackupDaemon::CreateVssBackupComponents()
1150	{
1151	std::map<char, VSS_ID> volumesIncluded;
1152
1153	HRESULT result = ::CreateVssBackupComponents(&mpVssBackupComponents);
1154	if(result != S_OK)
1155	{
1156	BOX_ERROR("VSS: Failed to create backup components: " <<
1157	GetMsgForHresult(result));
1158	return;
1159	}
1160
1161	result = mpVssBackupComponents->InitializeForBackup(NULL);
1162	if(result != S_OK)
1163	{
1164	std::string message = GetMsgForHresult(result);
1165
1166	if (result == VSS_E_UNEXPECTED)
1167	{
1168	message = "Check the Application Log for details, and ensure "
1169	"that the Volume Shadow Copy, COM+ System Application, "
1170	"and Distributed Transaction Coordinator services "
1171	"are running";
1172	}
1173
1174	BOX_ERROR("VSS: Failed to initialize for backup: " << message);
1175	return;
1176	}
1177
1178	result = mpVssBackupComponents->SetContext(VSS_CTX_BACKUP);
1179	if(result == E_NOTIMPL)
1180	{
1181	BOX_INFO("VSS: Failed to set context to VSS_CTX_BACKUP: "
1182	"not implemented, probably Windows XP, ignored.");
1183	}
1184	else if(result != S_OK)
1185	{
1186	BOX_ERROR("VSS: Failed to set context to VSS_CTX_BACKUP: " <<
1187	GetMsgForHresult(result));
1188	return;
1189	}
1190
1191	result = mpVssBackupComponents->SetBackupState(
1192	false, /* no components for now */
1193	true, /* might as well ask for a bootable backup */
1194	VSS_BT_FULL,
1195	false /* what is Partial File Support? */);
1196	if(result != S_OK)
1197	{
1198	BOX_ERROR("VSS: Failed to set backup state: " <<
1199	GetMsgForHresult(result));
1200	return;
1201	}
1202
1203	if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterMetadata,
1204	"GatherWriterMetadata()"))
1205	{
1206	goto CreateVssBackupComponents_cleanup_WriterMetadata;
1207	}
1208
1209	UINT writerCount;
1210	result = mpVssBackupComponents->GetWriterMetadataCount(&writerCount);
1211	if(result != S_OK)
1212	{
1213	BOX_ERROR("VSS: Failed to get writer count: " <<
1214	GetMsgForHresult(result));
1215	goto CreateVssBackupComponents_cleanup_WriterMetadata;
1216	}
1217
1218	for(UINT iWriter = 0; iWriter < writerCount; iWriter++)
1219	{
1220	BOX_INFO("VSS: Getting metadata from writer " << iWriter);
1221	VSS_ID writerInstance;
1222	IVssExamineWriterMetadata* pMetadata;
1223	result = mpVssBackupComponents->GetWriterMetadata(iWriter,
1224	&writerInstance, &pMetadata);
1225	if(result != S_OK)
1226	{
1227	BOX_ERROR("Failed to get VSS metadata from writer " << iWriter <<
1228	": " << GetMsgForHresult(result));
1229	continue;
1230	}
1231
1232	UINT includeFiles, excludeFiles, numComponents;
1233	result = pMetadata->GetFileCounts(&includeFiles, &excludeFiles,
1234	&numComponents);
1235	if(result != S_OK)
1236	{
1237	BOX_ERROR("VSS: Failed to get metadata file counts from "
1238	"writer " << iWriter << ": " <<
1239	GetMsgForHresult(result));
1240	pMetadata->Release();
1241	continue;
1242	}
1243
1244	for(UINT iComponent = 0; iComponent < numComponents; iComponent++)
1245	{
1246	IVssWMComponent* pComponent;
1247	result = pMetadata->GetComponent(iComponent, &pComponent);
1248	if(result != S_OK)
1249	{
1250	BOX_ERROR("VSS: Failed to get metadata component " <<
1251	iComponent << " from writer " << iWriter << ": " <<
1252	GetMsgForHresult(result));
1253	continue;
1254	}
1255
1256	PVSSCOMPONENTINFO pComponentInfo;
1257	result = pComponent->GetComponentInfo(&pComponentInfo);
1258	if(result != S_OK)
1259	{
1260	BOX_ERROR("VSS: Failed to get metadata component " <<
1261	iComponent << " info from writer " << iWriter << ": " <<
1262	GetMsgForHresult(result));
1263	pComponent->Release();
1264	continue;
1265	}
1266
1267	BOX_TRACE("VSS: writer " << iWriter << " component " <<
1268	iComponent << " info:");
1269	switch(pComponentInfo->type)
1270	{
1271	case VSS_CT_UNDEFINED: BOX_TRACE("VSS: type: undefined"); break;
1272	case VSS_CT_DATABASE: BOX_TRACE("VSS: type: database"); break;
1273	case VSS_CT_FILEGROUP: BOX_TRACE("VSS: type: filegroup"); break;
1274	default:
1275	BOX_WARNING("VSS: type: unknown (" << pComponentInfo->type << ")");
1276	}
1277
1278	BOX_TRACE("VSS: logical path: " <<
1279	BstrToString(pComponentInfo->bstrLogicalPath));
1280	BOX_TRACE("VSS: component name: " <<
1281	BstrToString(pComponentInfo->bstrComponentName));
1282	BOX_TRACE("VSS: caption: " <<
1283	BstrToString(pComponentInfo->bstrCaption));
1284	BOX_TRACE("VSS: restore metadata: " <<
1285	pComponentInfo->bRestoreMetadata);
1286	BOX_TRACE("VSS: notify on complete: " <<
1287	pComponentInfo->bRestoreMetadata);
1288	BOX_TRACE("VSS: selectable: " <<
1289	pComponentInfo->bSelectable);
1290	BOX_TRACE("VSS: selectable for restore: " <<
1291	pComponentInfo->bSelectableForRestore);
1292	BOX_TRACE("VSS: component flags: " <<
1293	BOX_FORMAT_HEX32(pComponentInfo->dwComponentFlags));
1294	BOX_TRACE("VSS: file count: " <<
1295	pComponentInfo->cFileCount);
1296	BOX_TRACE("VSS: databases: " <<
1297	pComponentInfo->cDatabases);
1298	BOX_TRACE("VSS: log files: " <<
1299	pComponentInfo->cLogFiles);
1300	BOX_TRACE("VSS: dependencies: " <<
1301	pComponentInfo->cDependencies);
1302
1303	pComponent->FreeComponentInfo(pComponentInfo);
1304	pComponent->Release();
1305	}
1306
1307	pMetadata->Release();
1308	}
1309
1310	VSS_ID snapshotSetId;
1311	result = mpVssBackupComponents->StartSnapshotSet(&snapshotSetId);
1312	if(result != S_OK)
1313	{
1314	BOX_ERROR("VSS: Failed to start snapshot set: " <<
1315	GetMsgForHresult(result));
1316	goto CreateVssBackupComponents_cleanup_WriterMetadata;
1317	}
1318
1319	// Add all volumes included as backup locations to the snapshot set
1320	for(std::vector<Location *>::iterator
1321	iLocation = mLocations.begin();
1322	iLocation != mLocations.end();
1323	iLocation++)
1324	{
1325	Location& rLocation(**iLocation);
1326	std::string path = rLocation.mPath;
1327	// convert to absolute and remove Unicode prefix
1328	path = ConvertPathToAbsoluteUnicode(path.c_str()).substr(4);
1329
1330	if(path.length() >= 3 && path[1] == ':' && path[2] == '\\')
1331	{
1332	std::string volumeRoot = path.substr(0, 3);
1333
1334	std::map<char, VSS_ID>::iterator i =
1335	volumesIncluded.find(path[0]);
1336
1337	if(i == volumesIncluded.end())
1338	{
1339	std::wstring volumeRootWide;
1340	volumeRootWide.push_back((WCHAR) path[0]);
1341	volumeRootWide.push_back((WCHAR) ':');
1342	volumeRootWide.push_back((WCHAR) '\\');
1343	VSS_ID newVolumeId;
1344	result = mpVssBackupComponents->AddToSnapshotSet(
1345	(VSS_PWSZ)(volumeRootWide.c_str()), GUID_NULL,
1346	&newVolumeId);
1347	if(result == S_OK)
1348	{
1349	BOX_TRACE("VSS: Added volume " << volumeRoot <<
1350	" for backup location " << path <<
1351	" to snapshot set");
1352	volumesIncluded[path[0]] = newVolumeId;
1353	rLocation.mSnapshotVolumeId = newVolumeId;
1354	rLocation.mIsSnapshotCreated = true;
1355
1356	// If the snapshot path starts with the volume root
1357	// (drive letter), because the path is absolute (as
1358	// it should be), then remove it so that the
1359	// resulting snapshot path can be appended to the
1360	// snapshot device object to make a real path,
1361	// without a spurious drive letter in it.
1362
1363	if (path.substr(0, volumeRoot.length()) == volumeRoot)
1364	{
1365	path = path.substr(volumeRoot.length());
1366	}
1367
1368	rLocation.mSnapshotPath = path;
1369	}
1370	else
1371	{
1372	BOX_ERROR("VSS: Failed to add volume " <<
1373	volumeRoot << " to snapshot set: " <<
1374	GetMsgForHresult(result));
1375	goto CreateVssBackupComponents_cleanup_WriterMetadata;
1376	}
1377	}
1378	else
1379	{
1380	BOX_TRACE("VSS: Skipping already included volume " <<
1381	volumeRoot << " for backup location " << path);
1382	rLocation.mSnapshotVolumeId = i->second;
1383	rLocation.mIsSnapshotCreated = true;
1384	}
1385	}
1386	else
1387	{
1388	BOX_WARNING("VSS: Skipping backup location " << path <<
1389	" which does not start with a volume specification");
1390	}
1391	}
1392
1393	if(!CallAndWaitForAsync(&IVssBackupComponents::PrepareForBackup,
1394	"PrepareForBackup()"))
1395	{
1396	goto CreateVssBackupComponents_cleanup_WriterMetadata;
1397	}
1398
1399	if(!CallAndWaitForAsync(&IVssBackupComponents::DoSnapshotSet,
1400	"DoSnapshotSet()"))
1401	{
1402	goto CreateVssBackupComponents_cleanup_WriterMetadata;
1403	}
1404
1405	if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterStatus,
1406	"GatherWriterStatus()"))
1407	{
1408	goto CreateVssBackupComponents_cleanup_WriterStatus;
1409	}
1410
1411	result = mpVssBackupComponents->GetWriterStatusCount(&writerCount);
1412	if(result != S_OK)
1413	{
1414	BOX_ERROR("VSS: Failed to get writer status count: " <<
1415	GetMsgForHresult(result));
1416	goto CreateVssBackupComponents_cleanup_WriterStatus;
1417	}
1418
1419	for(UINT iWriter = 0; iWriter < writerCount; iWriter++)
1420	{
1421	VSS_ID instance, writer;
1422	BSTR writerNameBstr;
1423	VSS_WRITER_STATE writerState;
1424	HRESULT writerResult;
1425
1426	result = mpVssBackupComponents->GetWriterStatus(iWriter,
1427	&instance, &writer, &writerNameBstr, &writerState,
1428	&writerResult);
1429	if(result != S_OK)
1430	{
1431	BOX_ERROR("VSS: Failed to query writer " << iWriter <<
1432	" status: " << GetMsgForHresult(result));
1433	goto CreateVssBackupComponents_cleanup_WriterStatus;
1434	}
1435
1436	std::string writerName = BstrToString(writerNameBstr);
1437	::SysFreeString(writerNameBstr);
1438
1439	if(writerResult != S_OK)
1440	{
1441	BOX_ERROR("VSS: Writer " << iWriter << " (" <<
1442	writerName << ") failed: " <<
1443	GetMsgForHresult(writerResult));
1444	continue;
1445	}
1446
1447	std::string stateName;
1448
1449	switch(writerState)
1450	{
1451	#define WRITER_STATE(code) \
1452	case code: stateName = #code; break;
1453	WRITER_STATE(VSS_WS_UNKNOWN);
1454	WRITER_STATE(VSS_WS_STABLE);
1455	WRITER_STATE(VSS_WS_WAITING_FOR_FREEZE);
1456	WRITER_STATE(VSS_WS_WAITING_FOR_THAW);
1457	WRITER_STATE(VSS_WS_WAITING_FOR_POST_SNAPSHOT);
1458	WRITER_STATE(VSS_WS_WAITING_FOR_BACKUP_COMPLETE);
1459	WRITER_STATE(VSS_WS_FAILED_AT_IDENTIFY);
1460	WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_BACKUP);
1461	WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_SNAPSHOT);
1462	WRITER_STATE(VSS_WS_FAILED_AT_FREEZE);
1463	WRITER_STATE(VSS_WS_FAILED_AT_THAW);
1464	WRITER_STATE(VSS_WS_FAILED_AT_POST_SNAPSHOT);
1465	WRITER_STATE(VSS_WS_FAILED_AT_BACKUP_COMPLETE);
1466	WRITER_STATE(VSS_WS_FAILED_AT_PRE_RESTORE);
1467	WRITER_STATE(VSS_WS_FAILED_AT_POST_RESTORE);
1468	WRITER_STATE(VSS_WS_FAILED_AT_BACKUPSHUTDOWN);
1469	#undef WRITER_STATE
1470	default:
1471	std::ostringstream o;
1472	o << "unknown (" << writerState << ")";
1473	stateName = o.str();
1474	}
1475
1476	BOX_TRACE("VSS: Writer " << iWriter << " (" <<
1477	writerName << ") is in state " << stateName);
1478	}
1479
1480	// lookup new snapshot volume for each location that has a snapshot
1481	for(std::vector<Location *>::iterator
1482	iLocation = mLocations.begin();
1483	iLocation != mLocations.end();
1484	iLocation++)
1485	{
1486	Location& rLocation(**iLocation);
1487	if(rLocation.mIsSnapshotCreated)
1488	{
1489	VSS_SNAPSHOT_PROP prop;
1490	result = mpVssBackupComponents->GetSnapshotProperties(
1491	rLocation.mSnapshotVolumeId, &prop);
1492	if(result != S_OK)
1493	{
1494	BOX_ERROR("VSS: Failed to get snapshot properties "
1495	"for volume " << GuidToString(rLocation.mSnapshotVolumeId) <<
1496	" for location " << rLocation.mPath << ": " <<
1497	GetMsgForHresult(result));
1498	rLocation.mIsSnapshotCreated = false;
1499	continue;
1500	}
1501
1502	rLocation.mSnapshotPath =
1503	WideStringToString(prop.m_pwszSnapshotDeviceObject) +
1504	DIRECTORY_SEPARATOR + rLocation.mSnapshotPath;
1505	FreeSnapshotProp(&prop);
1506
1507	BOX_INFO("VSS: Location " << rLocation.mPath << " using "
1508	"snapshot path " << rLocation.mSnapshotPath);
1509	}
1510	}
1511
1512	IVssEnumObject *pEnum;
1513	result = mpVssBackupComponents->Query(GUID_NULL, VSS_OBJECT_NONE,
1514	VSS_OBJECT_SNAPSHOT, &pEnum);
1515	if(result != S_OK)
1516	{
1517	BOX_ERROR("VSS: Failed to query snapshot list: " <<
1518	GetMsgForHresult(result));
1519	goto CreateVssBackupComponents_cleanup_WriterStatus;
1520	}
1521
1522	while(result == S_OK)
1523	{
1524	VSS_OBJECT_PROP rgelt;
1525	ULONG count;
1526	result = pEnum->Next(1, &rgelt, &count);
1527
1528	if(result == S_FALSE)
1529	{
1530	// end of list, break out of the loop
1531	break;
1532	}
1533	else if(result != S_OK)
1534	{
1535	BOX_ERROR("VSS: Failed to enumerate snapshot: " <<
1536	GetMsgForHresult(result));
1537	}
1538	else if(count != 1)
1539	{
1540	BOX_ERROR("VSS: Failed to enumerate snapshot: " <<
1541	"Next() returned " << count << " objects instead of 1");
1542	}
1543	else if(rgelt.Type != VSS_OBJECT_SNAPSHOT)
1544	{
1545	BOX_ERROR("VSS: Failed to enumerate snapshot: " <<
1546	"Next() returned a type " << rgelt.Type << " object "
1547	"instead of VSS_OBJECT_SNAPSHOT");
1548	}
1549	else
1550	{
1551	VSS_SNAPSHOT_PROP *pSnap = &rgelt.Obj.Snap;
1552	BOX_TRACE("VSS: Snapshot ID: " <<
1553	GuidToString(pSnap->m_SnapshotId));
1554	BOX_TRACE("VSS: Snapshot set ID: " <<
1555	GuidToString(pSnap->m_SnapshotSetId));
1556	BOX_TRACE("VSS: Number of volumes: " <<
1557	pSnap->m_lSnapshotsCount);
1558	BOX_TRACE("VSS: Snapshot device object: " <<
1559	WideStringToString(pSnap->m_pwszSnapshotDeviceObject));
1560	BOX_TRACE("VSS: Original volume name: " <<
1561	WideStringToString(pSnap->m_pwszOriginalVolumeName));
1562	BOX_TRACE("VSS: Originating machine: " <<
1563	WideStringToString(pSnap->m_pwszOriginatingMachine));
1564	BOX_TRACE("VSS: Service machine: " <<
1565	WideStringToString(pSnap->m_pwszServiceMachine));
1566	BOX_TRACE("VSS: Exposed name: " <<
1567	WideStringToString(pSnap->m_pwszExposedName));
1568	BOX_TRACE("VSS: Exposed path: " <<
1569	WideStringToString(pSnap->m_pwszExposedPath));
1570	BOX_TRACE("VSS: Provider ID: " <<
1571	GuidToString(pSnap->m_ProviderId));
1572	BOX_TRACE("VSS: Snapshot attributes: " <<
1573	BOX_FORMAT_HEX32(pSnap->m_lSnapshotAttributes));
1574	BOX_TRACE("VSS: Snapshot creation time: " <<
1575	BOX_FORMAT_HEX32(pSnap->m_tsCreationTimestamp));
1576
1577	std::string status;
1578	switch(pSnap->m_eStatus)
1579	{
1580	case VSS_SS_UNKNOWN: status = "Unknown (error)"; break;
1581	case VSS_SS_PREPARING: status = "Preparing"; break;
1582	case VSS_SS_PROCESSING_PREPARE: status = "Preparing (processing)"; break;
1583	case VSS_SS_PREPARED: status = "Prepared"; break;
1584	case VSS_SS_PROCESSING_PRECOMMIT: status = "Precommitting"; break;
1585	case VSS_SS_PRECOMMITTED: status = "Precommitted"; break;
1586	case VSS_SS_PROCESSING_COMMIT: status = "Commiting"; break;
1587	case VSS_SS_COMMITTED: status = "Committed"; break;
1588	case VSS_SS_PROCESSING_POSTCOMMIT: status = "Postcommitting"; break;
1589	case VSS_SS_PROCESSING_PREFINALCOMMIT: status = "Pre final committing"; break;
1590	case VSS_SS_PREFINALCOMMITTED: status = "Pre final committed"; break;
1591	case VSS_SS_PROCESSING_POSTFINALCOMMIT: status = "Post final committing"; break;
1592	case VSS_SS_CREATED: status = "Created"; break;
1593	case VSS_SS_ABORTED: status = "Aborted"; break;
1594	case VSS_SS_DELETED: status = "Deleted"; break;
1595	case VSS_SS_POSTCOMMITTED: status = "Postcommitted"; break;
1596	default:
1597	std::ostringstream buf;
1598	buf << "Unknown code: " << pSnap->m_eStatus;
1599	status = buf.str();
1600	}
1601
1602	BOX_TRACE("VSS: Snapshot status: " << status);
1603	FreeSnapshotProp(pSnap);
1604	}
1605	}
1606
1607	pEnum->Release();
1608
1609	CreateVssBackupComponents_cleanup_WriterStatus:
1610	result = mpVssBackupComponents->FreeWriterStatus();
1611	if(result != S_OK)
1612	{
1613	BOX_ERROR("VSS: Failed to free writer status: " <<
1614	GetMsgForHresult(result));
1615	}
1616
1617	CreateVssBackupComponents_cleanup_WriterMetadata:
1618	result = mpVssBackupComponents->FreeWriterMetadata();
1619	if(result != S_OK)
1620	{
1621	BOX_ERROR("VSS: Failed to free writer metadata: " <<
1622	GetMsgForHresult(result));
1623	}
1624	}
1625
1626	void BackupDaemon::CleanupVssBackupComponents()
1627	{
1628	if(mpVssBackupComponents == NULL)
1629	{
1630	return;
1631	}
1632
1633	CallAndWaitForAsync(&IVssBackupComponents::BackupComplete,
1634	"BackupComplete()");
1635
1636	mpVssBackupComponents->Release();
1637	mpVssBackupComponents = NULL;
1638	}
1639	#endif
1640
1641	void BackupDaemon::OnBackupStart()
1642	{
1643	// Touch a file to record times in filesystem
1644	TouchFileInWorkingDir("last_sync_start");
1645
1646	// Reset statistics on uploads
1647	BackupStoreFile::ResetStats();
1648
1649	// Tell anything connected to the command socket
1650	SendSyncStartOrFinish(true /* start */);
1651
1652	// Notify administrator
1653	NotifySysadmin(SysadminNotifier::BackupStart);
1654
1655	// Set state and log start
1656	SetState(State_Connected);
1657	BOX_NOTICE("Beginning scan of local files");
1658	}
1659
1660	void BackupDaemon::OnBackupFinish()
1661	{
1662	try
1663	{
1664	// Log
1665	BOX_NOTICE("Finished scan of local files");
1666
1667	// Log the stats
1668	BOX_NOTICE("File statistics: total file size uploaded "
1669	<< BackupStoreFile::msStats.mBytesInEncodedFiles
1670	<< ", bytes already on server "
1671	<< BackupStoreFile::msStats.mBytesAlreadyOnServer
1672	<< ", encoded size "
1673	<< BackupStoreFile::msStats.mTotalFileStreamSize
1674	<< ", " << mNumFilesUploaded << " files uploaded, "
1675	<< mNumDirsCreated << " dirs created");
1676
1677	// Reset statistics again
1678	BackupStoreFile::ResetStats();
1679
1680	// Notify administrator
1681	NotifySysadmin(SysadminNotifier::BackupFinish);
1682
1683	// Tell anything connected to the command socket
1684	SendSyncStartOrFinish(false /* finish */);
1685
1686	// Touch a file to record times in filesystem
1687	TouchFileInWorkingDir("last_sync_finish");
1688	}
1689	catch (std::exception &e)
1690	{
1691	BOX_ERROR("Failed to perform backup finish actions: " << e.what());
1692	}
1693	}
1694
1695	// --------------------------------------------------------------------------
1696	//
1697	// Function
1698	// Name: BackupDaemon::UseScriptToSeeIfSyncAllowed()
1699	// Purpose: Private. Use a script to see if the sync should be
1700	// allowed now (if configured). Returns -1 if it's
1701	// allowed, time in seconds to wait otherwise.
1702	// Created: 21/6/04
1703	//
1704	// --------------------------------------------------------------------------
1705	int BackupDaemon::UseScriptToSeeIfSyncAllowed()
1706	{
1707	const Configuration &conf(GetConfiguration());
1708
1709	// Got a script to run?
1710	if(!conf.KeyExists("SyncAllowScript"))
1711	{
1712	// No. Do sync.
1713	return -1;
1714	}
1715
1716	// If there's no result, try again in five minutes
1717	int waitInSeconds = (60*5);
1718
1719	std::string script(conf.GetKeyValue("SyncAllowScript") +
1720	" \"" + GetConfigFileName() + "\"");
1721
1722	// Run it?
1723	pid_t pid = 0;
1724	try
1725	{
1726	std::auto_ptr<IOStream> pscript(LocalProcessStream(script,
1727	pid));
1728
1729	// Read in the result
1730	IOStreamGetLine getLine(*pscript);
1731	std::string line;
1732	if(getLine.GetLine(line, true, 30000)) // 30 seconds should be enough
1733	{
1734	waitInSeconds = BackupDaemon::ParseSyncAllowScriptOutput(script, line);
1735	}
1736	else
1737	{
1738	BOX_ERROR("SyncAllowScript output nothing within "
1739	"30 seconds, waiting 5 minutes to try again"
1740	" (" << script << ")");
1741	}
1742	}
1743	catch(std::exception &e)
1744	{
1745	BOX_ERROR("Internal error running SyncAllowScript: "
1746	<< e.what() << " (" << script << ")");
1747	}
1748	catch(...)
1749	{
1750	// Ignore any exceptions
1751	// Log that something bad happened
1752	BOX_ERROR("Unknown error running SyncAllowScript (" <<
1753	script << ")");
1754	}
1755
1756	// Wait and then cleanup child process, if any
1757	if(pid != 0)
1758	{
1759	int status = 0;
1760	::waitpid(pid, &status, 0);
1761	}
1762
1763	return waitInSeconds;
1764	}
1765
1766	int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script,
1767	const std::string& output)
1768	{
1769	int waitInSeconds = (60*5);
1770	std::istringstream iss(output);
1771
1772	std::string delay;
1773	iss >> delay;
1774
1775	if(delay == "")
1776	{
1777	BOX_ERROR("SyncAllowScript output an empty line");
1778	return waitInSeconds;
1779	}
1780
1781	// Got a string, interpret
1782	if(delay == "now")
1783	{
1784	// Script says do it now. Obey.
1785	waitInSeconds = -1;
1786
1787	BOX_NOTICE("SyncAllowScript requested a backup now "
1788	<< "(" << script << ")");
1789	}
1790	else
1791	{
1792	try
1793	{
1794	// How many seconds to wait?
1795	waitInSeconds = BoxConvert::Convert<int32_t, const std::string&>(delay);
1796	}
1797	catch(ConversionException &e)
1798	{
1799	BOX_ERROR("SyncAllowScript output an invalid "
1800	"number: '" << output << "' (" <<
1801	script << ")");
1802	throw;
1803	}
1804
1805	BOX_NOTICE("SyncAllowScript requested a delay of " <<
1806	waitInSeconds << " seconds due to SyncAllowScript "
1807	<< "(" << script << ")");
1808	}
1809
1810	if(iss.eof())
1811	{
1812	// No bandwidth limit requested
1813	mMaxBandwidthFromSyncAllowScript = 0;
1814	BOX_NOTICE("SyncAllowScript did not set a maximum bandwidth "
1815	"(" << script << ")");
1816	}
1817	else
1818	{
1819	std::string maxBandwidth;
1820	iss >> maxBandwidth;
1821
1822	try
1823	{
1824	// How many seconds to wait?
1825	mMaxBandwidthFromSyncAllowScript =
1826	BoxConvert::Convert<int32_t, const std::string&>(maxBandwidth);
1827	}
1828	catch(ConversionException &e)
1829	{
1830	BOX_ERROR("Invalid maximum bandwidth from "
1831	"SyncAllowScript: '" <<
1832	output << "' (" << script << ")");
1833	throw;
1834	}
1835
1836	BOX_NOTICE("SyncAllowScript set maximum bandwidth to " <<
1837	mMaxBandwidthFromSyncAllowScript << " kB/s (" <<
1838	script << ")");
1839	}
1840
1841	return waitInSeconds;
1842	}
1843
1844
1845	// --------------------------------------------------------------------------
1846	//
1847	// Function
1848	// Name: BackupDaemon::WaitOnCommandSocket(box_time_t, bool &, bool &)
1849	// Purpose: Waits on a the command socket for a time of UP TO the required time
1850	// but may be much less, and handles a command if necessary.
1851	// Created: 18/2/04
1852	//
1853	// --------------------------------------------------------------------------
1854	void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut)
1855	{
1856	ASSERT(mapCommandSocketInfo.get());
1857	if(!mapCommandSocketInfo.get())
1858	{
1859	// failure case isn't too bad
1860	::sleep(1);
1861	return;
1862	}
1863
1864	BOX_TRACE("Wait on command socket, delay = " << RequiredDelay);
1865
1866	try
1867	{
1868	// Timeout value for connections and things
1869	int timeout = ((int)BoxTimeToMilliSeconds(RequiredDelay)) + 1;
1870	// Handle bad boundary cases
1871	if(timeout <= 0) timeout = 1;
1872	if(timeout == INFTIM) timeout = 100000;
1873
1874	// Wait for socket connection, or handle a command?
1875	if(mapCommandSocketInfo->mpConnectedSocket.get() == 0)
1876	{
1877	// No connection, listen for a new one
1878	mapCommandSocketInfo->mpConnectedSocket.reset(mapCommandSocketInfo->mListeningSocket.Accept(timeout).release());
1879
1880	if(mapCommandSocketInfo->mpConnectedSocket.get() == 0)
1881	{
1882	// If a connection didn't arrive, there was a timeout, which means we've
1883	// waited long enough and it's time to go.
1884	return;
1885	}
1886	else
1887	{
1888	#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET
1889	bool uidOK = true;
1890	BOX_WARNING("On this platform, no security check can be made on the credentials of peers connecting to the command socket. (bbackupctl)");
1891	#else
1892	// Security check -- does the process connecting to this socket have
1893	// the same UID as this process?
1894	bool uidOK = false;
1895	// BLOCK
1896	{
1897	uid_t remoteEUID = 0xffff;
1898	gid_t remoteEGID = 0xffff;
1899	if(mapCommandSocketInfo->mpConnectedSocket->GetPeerCredentials(remoteEUID, remoteEGID))
1900	{
1901	// Credentials are available -- check UID
1902	if(remoteEUID == ::getuid())
1903	{
1904	// Acceptable
1905	uidOK = true;
1906	}
1907	}
1908	}
1909	#endif // PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET
1910
1911	// Is this an acceptable connection?
1912	if(!uidOK)
1913	{
1914	// Dump the connection
1915	BOX_ERROR("Incoming command connection from peer had different user ID than this process, or security check could not be completed.");
1916	mapCommandSocketInfo->mpConnectedSocket.reset();
1917	return;
1918	}
1919	else
1920	{
1921	// Log
1922	BOX_INFO("Connection from command socket");
1923
1924	// Send a header line summarising the configuration and current state
1925	const Configuration &conf(GetConfiguration());
1926	char summary[256];
1927	int summarySize = sprintf(summary, "bbackupd: %d %d %d %d\nstate %d\n",
1928	conf.GetKeyValueBool("AutomaticBackup"),
1929	conf.GetKeyValueInt("UpdateStoreInterval"),
1930	conf.GetKeyValueInt("MinimumFileAge"),
1931	conf.GetKeyValueInt("MaxUploadWait"),
1932	mState);
1933	mapCommandSocketInfo->mpConnectedSocket->Write(summary, summarySize);
1934
1935	// Set the timeout to something very small, so we don't wait too long on waiting
1936	// for any incoming data
1937	timeout = 10; // milliseconds
1938	}
1939	}
1940	}
1941
1942	// So there must be a connection now.
1943	ASSERT(mapCommandSocketInfo->mpConnectedSocket.get() != 0);
1944
1945	// Is there a getline object ready?
1946	if(mapCommandSocketInfo->mpGetLine == 0)
1947	{
1948	// Create a new one
1949	mapCommandSocketInfo->mpGetLine = new IOStreamGetLine(*(mapCommandSocketInfo->mpConnectedSocket.get()));
1950	}
1951
1952	// Ping the remote side, to provide errors which will mean the socket gets closed
1953	mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5);
1954
1955	// Wait for a command or something on the socket
1956	std::string command;
1957	while(mapCommandSocketInfo->mpGetLine != 0 && !mapCommandSocketInfo->mpGetLine->IsEOF()
1958	&& mapCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout))
1959	{
1960	BOX_TRACE("Receiving command '" << command
1961	<< "' over command socket");
1962
1963	bool sendOK = false;
1964	bool sendResponse = true;
1965
1966	// Command to process!
1967	if(command == "quit" \|\| command == "")
1968	{
1969	// Close the socket.
1970	CloseCommandConnection();
1971	sendResponse = false;
1972	}
1973	else if(command == "sync")
1974	{
1975	// Sync now!
1976	DoSyncFlagOut = true;
1977	SyncIsForcedOut = false;
1978	sendOK = true;
1979	}
1980	else if(command == "force-sync")
1981	{
1982	// Sync now (forced -- overrides any SyncAllowScript)
1983	DoSyncFlagOut = true;
1984	SyncIsForcedOut = true;
1985	sendOK = true;
1986	}
1987	else if(command == "reload")
1988	{
1989	// Reload the configuration
1990	SetReloadConfigWanted();
1991	sendOK = true;
1992	}
1993	else if(command == "terminate")
1994	{
1995	// Terminate the daemon cleanly
1996	SetTerminateWanted();
1997	sendOK = true;
1998	}
1999
2000	// Send a response back?
2001	if(sendResponse)
2002	{
2003	mapCommandSocketInfo->mpConnectedSocket->Write(sendOK?"ok\n":"error\n", sendOK?3:6);
2004	}
2005
2006	// Set timeout to something very small, so this just checks for data which is waiting
2007	timeout = 1;
2008	}
2009
2010	// Close on EOF?
2011	if(mapCommandSocketInfo->mpGetLine != 0 && mapCommandSocketInfo->mpGetLine->IsEOF())
2012	{
2013	CloseCommandConnection();
2014	}
2015	}
2016	catch(ConnectionException &ce)
2017	{
2018	BOX_NOTICE("Failed to write to command socket: " << ce.what());
2019
2020	// If an error occurs, and there is a connection active,
2021	// just close that connection and continue. Otherwise,
2022	// let the error propagate.
2023
2024	if(mapCommandSocketInfo->mpConnectedSocket.get() == 0)
2025	{
2026	throw; // thread will die
2027	}
2028	else
2029	{
2030	// Close socket and ignore error
2031	CloseCommandConnection();
2032	}
2033	}
2034	catch(std::exception &e)
2035	{
2036	BOX_ERROR("Failed to write to command socket: " <<
2037	e.what());
2038
2039	// If an error occurs, and there is a connection active,
2040	// just close that connection and continue. Otherwise,
2041	// let the error propagate.
2042
2043	if(mapCommandSocketInfo->mpConnectedSocket.get() == 0)
2044	{
2045	throw; // thread will die
2046	}
2047	else
2048	{
2049	// Close socket and ignore error
2050	CloseCommandConnection();
2051	}
2052	}
2053	catch(...)
2054	{
2055	BOX_ERROR("Failed to write to command socket: unknown error");
2056
2057	// If an error occurs, and there is a connection active,
2058	// just close that connection and continue. Otherwise,
2059	// let the error propagate.
2060
2061	if(mapCommandSocketInfo->mpConnectedSocket.get() == 0)
2062	{
2063	throw; // thread will die
2064	}
2065	else
2066	{
2067	// Close socket and ignore error
2068	CloseCommandConnection();
2069	}
2070	}
2071	}
2072
2073
2074	// --------------------------------------------------------------------------
2075	//
2076	// Function
2077	// Name: BackupDaemon::CloseCommandConnection()
2078	// Purpose: Close the command connection, ignoring any errors
2079	// Created: 18/2/04
2080	//
2081	// --------------------------------------------------------------------------
2082	void BackupDaemon::CloseCommandConnection()
2083	{
2084	try
2085	{
2086	BOX_TRACE("Closing command connection");
2087
2088	if(mapCommandSocketInfo->mpGetLine)
2089	{
2090	delete mapCommandSocketInfo->mpGetLine;
2091	mapCommandSocketInfo->mpGetLine = 0;
2092	}
2093	mapCommandSocketInfo->mpConnectedSocket.reset();
2094	}
2095	catch(std::exception &e)
2096	{
2097	BOX_ERROR("Internal error while closing command "
2098	"socket: " << e.what());
2099	}
2100	catch(...)
2101	{
2102	// Ignore any errors
2103	}
2104	}
2105
2106
2107	// --------------------------------------------------------------------------
2108	//
2109	// File
2110	// Name: BackupDaemon.cpp
2111	// Purpose: Send a start or finish sync message to the command socket, if it's connected.
2112	//
2113	// Created: 18/2/04
2114	//
2115	// --------------------------------------------------------------------------
2116	void BackupDaemon::SendSyncStartOrFinish(bool SendStart)
2117	{
2118	// The bbackupctl program can't rely on a state change, because it
2119	// may never change if the server doesn't need to be contacted.
2120
2121	if(mapCommandSocketInfo.get() &&
2122	mapCommandSocketInfo->mpConnectedSocket.get() != 0)
2123	{
2124	std::string message = SendStart ? "start-sync" : "finish-sync";
2125	try
2126	{
2127	message += "\n";
2128	mapCommandSocketInfo->mpConnectedSocket->Write(
2129	message.c_str(), message.size());
2130	}
2131	catch(std::exception &e)
2132	{
2133	BOX_ERROR("Internal error while sending to "
2134	"command socket client: " << e.what());
2135	CloseCommandConnection();
2136	}
2137	catch(...)
2138	{
2139	CloseCommandConnection();
2140	}
2141	}
2142	}
2143
2144
2145
2146
2147	#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_NMTONNAME)
2148	// string comparison ordering for when mount points are handled
2149	// by code, rather than the OS.
2150	typedef struct
2151	{
2152	bool operator()(const std::string &s1, const std::string &s2)
2153	{
2154	if(s1.size() == s2.size())
2155	{
2156	// Equal size, sort according to natural sort order
2157	return s1 < s2;
2158	}
2159	else
2160	{
2161	// Make sure longer strings go first
2162	return s1.size() > s2.size();
2163	}
2164	}
2165	} mntLenCompare;
2166	#endif
2167
2168	// --------------------------------------------------------------------------
2169	//
2170	// Function
2171	// Name: BackupDaemon::SetupLocations(BackupClientContext &, const Configuration &)
2172	// Purpose: Makes sure that the list of directories records is correctly set up
2173	// Created: 2003/10/08
2174	//
2175	// --------------------------------------------------------------------------
2176	void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf)
2177	{
2178	// Going to need a copy of the root directory. Get a connection,
2179	// and fetch it.
2180	BackupProtocolCallable& connection(rClientContext.GetConnection());
2181
2182	// Ask server for a list of everything in the root directory,
2183	// which is a directory itself
2184	std::auto_ptr<BackupProtocolSuccess> dirreply(
2185	connection.QueryListDirectory(
2186	BackupProtocolListDirectory::RootDirectory,
2187	// only directories
2188	BackupProtocolListDirectory::Flags_Dir,
2189	// exclude old/deleted stuff
2190	BackupProtocolListDirectory::Flags_Deleted \|
2191	BackupProtocolListDirectory::Flags_OldVersion,
2192	false /* no attributes */));
2193
2194	// Retrieve the directory from the stream following
2195	BackupStoreDirectory dir;
2196	std::auto_ptr<IOStream> dirstream(connection.ReceiveStream());
2197	dir.ReadFromStream(*dirstream, connection.GetTimeout());
2198
2199	// Map of mount names to ID map index
2200	std::map<std::string, int> mounts;
2201	int numIDMaps = 0;
2202
2203	#ifdef HAVE_MOUNTS
2204	#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_MNTONNAME)
2205	// Linux and others can't tell you where a directory is mounted. So we
2206	// have to read the mount entries from /etc/mtab! Bizarre that the OS
2207	// itself can't tell you, but there you go.
2208	std::set<std::string, mntLenCompare> mountPoints;
2209	// BLOCK
2210	FILE *mountPointsFile = 0;
2211
2212	#ifdef HAVE_STRUCT_MNTENT_MNT_DIR
2213	// Open mounts file
2214	mountPointsFile = ::setmntent("/proc/mounts", "r");
2215	if(mountPointsFile == 0)
2216	{
2217	mountPointsFile = ::setmntent("/etc/mtab", "r");
2218	}
2219	if(mountPointsFile == 0)
2220	{
2221	THROW_EXCEPTION(CommonException, OSFileError);
2222	}
2223
2224	try
2225	{
2226	// Read all the entries, and put them in the set
2227	struct mntent *entry = 0;
2228	while((entry = ::getmntent(mountPointsFile)) != 0)
2229	{
2230	BOX_TRACE("Found mount point at " << entry->mnt_dir);
2231	mountPoints.insert(std::string(entry->mnt_dir));
2232	}
2233
2234	// Close mounts file
2235	::endmntent(mountPointsFile);
2236	}
2237	catch(...)
2238	{
2239	::endmntent(mountPointsFile);
2240	throw;
2241	}
2242	#else // ! HAVE_STRUCT_MNTENT_MNT_DIR
2243	// Open mounts file
2244	mountPointsFile = ::fopen("/etc/mnttab", "r");
2245	if(mountPointsFile == 0)
2246	{
2247	THROW_EXCEPTION(CommonException, OSFileError);
2248	}
2249
2250	try
2251	{
2252	// Read all the entries, and put them in the set
2253	struct mnttab entry;
2254	while(getmntent(mountPointsFile, &entry) == 0)
2255	{
2256	BOX_TRACE("Found mount point at " << entry.mnt_mountp);
2257	mountPoints.insert(std::string(entry.mnt_mountp));
2258	}
2259
2260	// Close mounts file
2261	::fclose(mountPointsFile);
2262	}
2263	catch(...)
2264	{
2265	::fclose(mountPointsFile);
2266	throw;
2267	}
2268	#endif // HAVE_STRUCT_MNTENT_MNT_DIR
2269	// Check sorting and that things are as we expect
2270	ASSERT(mountPoints.size() > 0);
2271	#ifndef BOX_RELEASE_BUILD
2272	{
2273	std::set<std::string, mntLenCompare>::reverse_iterator i(mountPoints.rbegin());
2274	ASSERT(*i == "/");
2275	}
2276	#endif // n BOX_RELEASE_BUILD
2277	#endif // n HAVE_STRUCT_STATFS_F_MNTONNAME \|\| n HAVE_STRUCT_STATVFS_F_MNTONNAME
2278	#endif // HAVE_MOUNTS
2279
2280	// Then... go through each of the entries in the configuration,
2281	// making sure there's a directory created for it.
2282	std::vector<std::string> locNames =
2283	rLocationsConf.GetSubConfigurationNames();
2284
2285	// We only want completely configured locations to be in the list
2286	// when this function exits, so move them all to a temporary list.
2287	// Entries matching a properly configured location will be moved
2288	// back to mLocations. Anything left in this list after the loop
2289	// finishes will be deleted.
2290	Locations tmpLocations = mLocations;
2291	mLocations.clear();
2292
2293	// The ID map list will be repopulated automatically by this loop
2294	mIDMapMounts.clear();
2295
2296	for(std::vector<std::string>::iterator
2297	pLocName = locNames.begin();
2298	pLocName != locNames.end();
2299	pLocName++)
2300	{
2301	Location* pLoc = NULL;
2302
2303	// Try to find and reuse an existing Location object
2304	for(Locations::const_iterator
2305	i = tmpLocations.begin();
2306	i != tmpLocations.end(); i++)
2307	{
2308	if ((i)->mName == pLocName)
2309	{
2310	BOX_TRACE("Location already configured: " << *pLocName);
2311	pLoc = *i;
2312	break;
2313	}
2314	}
2315
2316	const Configuration& rConfig(
2317	rLocationsConf.GetSubConfiguration(*pLocName));
2318	std::auto_ptr<Location> apLoc;
2319
2320	try
2321	{
2322	if(pLoc == NULL)
2323	{
2324	// Create a record for it
2325	BOX_TRACE("New location: " << *pLocName);
2326	pLoc = new Location;
2327
2328	// ensure deletion if setup fails
2329	apLoc.reset(pLoc);
2330
2331	// Setup names in the location record
2332	pLoc->mName = *pLocName;
2333	pLoc->mPath = rConfig.GetKeyValue("Path");
2334
2335	// Read the exclude lists from the Configuration
2336	pLoc->mpExcludeFiles = BackupClientMakeExcludeList_Files(rConfig);
2337	pLoc->mpExcludeDirs = BackupClientMakeExcludeList_Dirs(rConfig);
2338	}
2339
2340	// Does this exist on the server?
2341	// Remove from dir object early, so that if we fail
2342	// to stat the local directory, we still don't
2343	// consider to remote one for deletion.
2344	BackupStoreDirectory::Iterator iter(dir);
2345	BackupStoreFilenameClear dirname(pLoc->mName); // generate the filename
2346	BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname);
2347	int64_t oid = 0;
2348	if(en != 0)
2349	{
2350	oid = en->GetObjectID();
2351
2352	// Delete the entry from the directory, so we get a list of
2353	// unused root directories at the end of this.
2354	dir.DeleteEntry(oid);
2355	}
2356
2357	// Do a fsstat on the pathname to find out which mount it's on
2358	{
2359
2360	#if defined HAVE_STRUCT_STATFS_F_MNTONNAME \|\| defined HAVE_STRUCT_STATVFS_F_MNTONNAME \|\| defined WIN32
2361
2362	// BSD style statfs -- includes mount point, which is nice.
2363	#ifdef HAVE_STRUCT_STATVFS_F_MNTONNAME
2364	struct statvfs s;
2365	if(::statvfs(pLoc->mPath.c_str(), &s) != 0)
2366	#else // HAVE_STRUCT_STATVFS_F_MNTONNAME
2367	struct statfs s;
2368	if(::statfs(pLoc->mPath.c_str(), &s) != 0)
2369	#endif // HAVE_STRUCT_STATVFS_F_MNTONNAME
2370	{
2371	THROW_SYS_ERROR("Failed to stat path "
2372	"'" << pLoc->mPath << "' "
2373	"for location "
2374	"'" << pLoc->mName << "'",
2375	CommonException, OSFileError);
2376	}
2377
2378	// Where the filesystem is mounted
2379	std::string mountName(s.f_mntonname);
2380
2381	#else // !HAVE_STRUCT_STATFS_F_MNTONNAME && !WIN32
2382
2383	// Warn in logs if the directory isn't absolute
2384	if(pLoc->mPath[0] != '/')
2385	{
2386	BOX_WARNING("Location path '"
2387	<< pLoc->mPath
2388	<< "' is not absolute");
2389	}
2390	// Go through the mount points found, and find a suitable one
2391	std::string mountName("/");
2392	{
2393	std::set<std::string, mntLenCompare>::const_iterator i(mountPoints.begin());
2394	BOX_TRACE(mountPoints.size()
2395	<< " potential mount points");
2396	for(; i != mountPoints.end(); ++i)
2397	{
2398	// Compare first n characters with the filename
2399	// If it matches, the file belongs in that mount point
2400	// (sorting order ensures this)
2401	BOX_TRACE("checking against mount point " << *i);
2402	if(::strncmp(i->c_str(), pLoc->mPath.c_str(), i->size()) == 0)
2403	{
2404	// Match
2405	mountName = *i;
2406	break;
2407	}
2408	}
2409	BOX_TRACE("mount point chosen for "
2410	<< pLoc->mPath << " is "
2411	<< mountName);
2412	}
2413
2414	#endif
2415
2416	// Got it?
2417	std::map<std::string, int>::iterator f(mounts.find(mountName));
2418	if(f != mounts.end())
2419	{
2420	// Yes -- store the index
2421	pLoc->mIDMapIndex = f->second;
2422	}
2423	else
2424	{
2425	// No -- new index
2426	pLoc->mIDMapIndex = numIDMaps;
2427	mounts[mountName] = numIDMaps;
2428
2429	// Store the mount name
2430	mIDMapMounts.push_back(mountName);
2431
2432	// Increment number of maps
2433	++numIDMaps;
2434	}
2435	}
2436
2437	// Does this exist on the server?
2438	if(en == 0)
2439	{
2440	// Doesn't exist, so it has to be created on the server. Let's go!
2441	// First, get the directory's attributes and modification time
2442	box_time_t attrModTime = 0;
2443	BackupClientFileAttributes attr;
2444	try
2445	{
2446	attr.ReadAttributes(pLoc->mPath.c_str(),
2447	true /* directories have zero mod times */,
2448	0 /* not interested in mod time */,
2449	&attrModTime /* get the attribute modification time */);
2450	}
2451	catch (BoxException &e)
2452	{
2453	BOX_ERROR("Failed to get attributes "
2454	"for path '" << pLoc->mPath
2455	<< "', skipping location '" <<
2456	pLoc->mName << "'");
2457	throw;
2458	}
2459
2460	// Execute create directory command
2461	try
2462	{
2463	MemBlockStream attrStream(attr);
2464	std::auto_ptr<BackupProtocolSuccess>
2465	dirCreate(connection.QueryCreateDirectory(
2466	BackupProtocolListDirectory::RootDirectory,
2467	attrModTime, dirname, attrStream));
2468
2469	// Object ID for later creation
2470	oid = dirCreate->GetObjectID();
2471	}
2472	catch (BoxException &e)
2473	{
2474	BOX_ERROR("Failed to create remote "
2475	"directory '/" << pLoc->mName <<
2476	"', skipping location '" <<
2477	pLoc->mName << "'");
2478	throw;
2479	}
2480
2481	}
2482
2483	// Create and store the directory object for the root of this location
2484	ASSERT(oid != 0);
2485	if(pLoc->mpDirectoryRecord.get() == NULL)
2486	{
2487	BackupClientDirectoryRecord *precord =
2488	new BackupClientDirectoryRecord(oid, *pLocName);
2489	pLoc->mpDirectoryRecord.reset(precord);
2490	}
2491
2492	// Remove it from the temporary list to avoid deletion
2493	tmpLocations.remove(pLoc);
2494
2495	// Push it back on the vector of locations
2496	mLocations.push_back(pLoc);
2497
2498	if(apLoc.get() != NULL)
2499	{
2500	// Don't delete it now!
2501	apLoc.release();
2502	}
2503	}
2504	catch (std::exception &e)
2505	{
2506	BOX_ERROR("Failed to configure location '"
2507	<< pLoc->mName << "' path '"
2508	<< pLoc->mPath << "': " << e.what() <<
2509	": please check for previous errors");
2510	mReadErrorsOnFilesystemObjects = true;
2511	}
2512	catch(...)
2513	{
2514	BOX_ERROR("Failed to configure location '"
2515	<< pLoc->mName << "' path '"
2516	<< pLoc->mPath << "': please check for "
2517	"previous errors");
2518	mReadErrorsOnFilesystemObjects = true;
2519	}
2520	}
2521
2522	// Now remove any leftovers
2523	for(BackupDaemon::Locations::iterator
2524	i = tmpLocations.begin();
2525	i != tmpLocations.end(); i++)
2526	{
2527	BOX_INFO("Removing obsolete location from memory: " <<
2528	(*i)->mName);
2529	delete *i;
2530	}
2531
2532	tmpLocations.clear();
2533
2534	// Any entries in the root directory which need deleting?
2535	if(dir.GetNumberOfEntries() > 0 &&
2536	mDeleteRedundantLocationsAfter == 0)
2537	{
2538	BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations "
2539	"in root directory found, but will not delete because "
2540	"DeleteRedundantLocationsAfter = 0");
2541	}
2542	else if(dir.GetNumberOfEntries() > 0)
2543	{
2544	box_time_t now = GetCurrentBoxTime();
2545
2546	// This should reset the timer if the list of unused
2547	// locations changes, but it will not if the number of
2548	// unused locations does not change, but the locations
2549	// do change, e.g. one mysteriously appears and another
2550	// mysteriously appears. (FIXME)
2551	if (dir.GetNumberOfEntries() != mUnusedRootDirEntries.size() \|\|
2552	mDeleteUnusedRootDirEntriesAfter == 0)
2553	{
2554	mDeleteUnusedRootDirEntriesAfter = now +
2555	SecondsToBoxTime(mDeleteRedundantLocationsAfter);
2556	}
2557
2558	int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter
2559	- now);
2560
2561	BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations "
2562	"in root directory found, will delete from store "
2563	"after " << secs << " seconds.");
2564
2565	// Store directories in list of things to delete
2566	mUnusedRootDirEntries.clear();
2567	BackupStoreDirectory::Iterator iter(dir);
2568	BackupStoreDirectory::Entry *en = 0;
2569	while((en = iter.Next()) != 0)
2570	{
2571	// Add name to list
2572	BackupStoreFilenameClear clear(en->GetName());
2573	const std::string &name(clear.GetClearFilename());
2574	mUnusedRootDirEntries.push_back(
2575	std::pair<int64_t,std::string>
2576	(en->GetObjectID(), name));
2577	// Log this
2578	BOX_INFO("Unused location in root: " << name);
2579	}
2580	ASSERT(mUnusedRootDirEntries.size() > 0);
2581	}
2582	}
2583
2584
2585	// --------------------------------------------------------------------------
2586	//
2587	// Function
2588	// Name: BackupDaemon::SetupIDMapsForSync()
2589	// Purpose: Sets up ID maps for the sync process -- make sure they're all there
2590	// Created: 11/11/03
2591	//
2592	// --------------------------------------------------------------------------
2593	void BackupDaemon::SetupIDMapsForSync()
2594	{
2595	// Make sure we have some blank, empty ID maps
2596	DeleteIDMapVector(mNewIDMaps);
2597	FillIDMapVector(mNewIDMaps, true /* new maps */);
2598	DeleteIDMapVector(mCurrentIDMaps);
2599	FillIDMapVector(mCurrentIDMaps, false /* new maps */);
2600	}
2601
2602
2603	// --------------------------------------------------------------------------
2604	//
2605	// Function
2606	// Name: BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &)
2607	// Purpose: Fills the vector with the right number of empty ID maps
2608	// Created: 11/11/03
2609	//
2610	// --------------------------------------------------------------------------
2611	void BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector, bool NewMaps)
2612	{
2613	ASSERT(rVector.size() == 0);
2614	rVector.reserve(mIDMapMounts.size());
2615
2616	for(unsigned int l = 0; l < mIDMapMounts.size(); ++l)
2617	{
2618	// Create the object
2619	BackupClientInodeToIDMap *pmap = new BackupClientInodeToIDMap();
2620	try
2621	{
2622	// Get the base filename of this map
2623	std::string filename;
2624	MakeMapBaseName(l, filename);
2625
2626	// If it's a new one, add a suffix
2627	if(NewMaps)
2628	{
2629	filename += ".n";
2630	}
2631
2632	// The new map file should not exist yet. If there's
2633	// one left over from a previous failed run, it's not
2634	// useful to us because we never read from it and will
2635	// overwrite the entries of all files that still
2636	// exist, so we should just delete it and start afresh.
2637	if(NewMaps && FileExists(filename.c_str()))
2638	{
2639	BOX_NOTICE("Found an incomplete ID map "
2640	"database, deleting it to start "
2641	"afresh: " << filename);
2642	if(unlink(filename.c_str()) != 0)
2643	{
2644	BOX_LOG_NATIVE_ERROR(BOX_FILE_MESSAGE(
2645	filename, "Failed to delete "
2646	"incomplete ID map database"));
2647	}
2648	}
2649
2650	// If it's not a new map, it may not exist in which case an empty map should be created
2651	if(!NewMaps && !FileExists(filename.c_str()))
2652	{
2653	pmap->OpenEmpty();
2654	}
2655	else
2656	{
2657	// Open the map
2658	pmap->Open(filename.c_str(), !NewMaps /* read only /, NewMaps / create new */);
2659	}
2660
2661	// Store on vector
2662	rVector.push_back(pmap);
2663	}
2664	catch(...)
2665	{
2666	delete pmap;
2667	throw;
2668	}
2669	}
2670	}
2671
2672
2673	// --------------------------------------------------------------------------
2674	//
2675	// Function
2676	// Name: BackupDaemon::DeleteCorruptBerkelyDbFiles()
2677	// Purpose: Delete the Berkely db files from disc after they have been corrupted.
2678	// Created: 14/9/04
2679	//
2680	// --------------------------------------------------------------------------
2681	void BackupDaemon::DeleteCorruptBerkelyDbFiles()
2682	{
2683	for(unsigned int l = 0; l < mIDMapMounts.size(); ++l)
2684	{
2685	// Get the base filename of this map
2686	std::string filename;
2687	MakeMapBaseName(l, filename);
2688
2689	// Delete the file
2690	BOX_TRACE("Deleting " << filename);
2691	::unlink(filename.c_str());
2692
2693	// Add a suffix for the new map
2694	filename += ".n";
2695
2696	// Delete that too
2697	BOX_TRACE("Deleting " << filename);
2698	::unlink(filename.c_str());
2699	}
2700	}
2701
2702
2703	// --------------------------------------------------------------------------
2704	//
2705	// Function
2706	// Name: MakeMapBaseName(unsigned int, std::string &)
2707	// Purpose: Makes the base name for a inode map
2708	// Created: 20/11/03
2709	//
2710	// --------------------------------------------------------------------------
2711	void BackupDaemon::MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const
2712	{
2713	// Get the directory for the maps
2714	const Configuration &config(GetConfiguration());
2715	std::string dir(config.GetKeyValue("DataDirectory"));
2716
2717	// Make a leafname
2718	std::string leaf(mIDMapMounts[MountNumber]);
2719	for(unsigned int z = 0; z < leaf.size(); ++z)
2720	{
2721	if(leaf[z] == DIRECTORY_SEPARATOR_ASCHAR)
2722	{
2723	leaf[z] = '_';
2724	}
2725	}
2726
2727	// Build the final filename
2728	rNameOut = dir + DIRECTORY_SEPARATOR "mnt" + leaf;
2729	}
2730
2731
2732
2733
2734	// --------------------------------------------------------------------------
2735	//
2736	// Function
2737	// Name: BackupDaemon::CommitIDMapsAfterSync()
2738	// Purpose: Commits the new ID maps, so the 'new' maps are now the 'current' maps.
2739	// Created: 11/11/03
2740	//
2741	// --------------------------------------------------------------------------
2742	void BackupDaemon::CommitIDMapsAfterSync()
2743	{
2744	// Get rid of the maps in memory (leaving them on disc of course)
2745	DeleteIDMapVector(mCurrentIDMaps);
2746	DeleteIDMapVector(mNewIDMaps);
2747
2748	// Then move the old maps into the new places
2749	for(unsigned int l = 0; l < mIDMapMounts.size(); ++l)
2750	{
2751	std::string target;
2752	MakeMapBaseName(l, target);
2753	std::string newmap(target + ".n");
2754
2755	// Try to rename
2756	#ifdef WIN32
2757	// win32 rename doesn't overwrite existing files
2758	::remove(target.c_str());
2759	#endif
2760	if(::rename(newmap.c_str(), target.c_str()) != 0)
2761	{
2762	BOX_LOG_SYS_ERROR("Failed to rename ID map: " <<
2763	newmap << " to " << target);
2764	THROW_EXCEPTION(CommonException, OSFileError)
2765	}
2766	}
2767	}
2768
2769
2770
2771	// --------------------------------------------------------------------------
2772	//
2773	// Function
2774	// Name: BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &)
2775	// Purpose: Deletes the contents of a vector of ID maps
2776	// Created: 11/11/03
2777	//
2778	// --------------------------------------------------------------------------
2779	void BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector)
2780	{
2781	while(!rVector.empty())
2782	{
2783	// Pop off list
2784	BackupClientInodeToIDMap *toDel = rVector.back();
2785	rVector.pop_back();
2786
2787	// Close and delete
2788	delete toDel;
2789	}
2790	ASSERT(rVector.size() == 0);
2791	}
2792
2793
2794	// --------------------------------------------------------------------------
2795	//
2796	// Function
2797	// Name: BackupDaemon::FindLocationPathName(const std::string &, std::string &) const
2798	// Purpose: Tries to find the path of the root of a backup location. Returns true (and path in rPathOut)
2799	// if it can be found, false otherwise.
2800	// Created: 12/11/03
2801	//
2802	// --------------------------------------------------------------------------
2803	bool BackupDaemon::FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const
2804	{
2805	// Search for the location
2806	for(Locations::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i)
2807	{
2808	if((*i)->mName == rLocationName)
2809	{
2810	rPathOut = (*i)->mPath;
2811	return true;
2812	}
2813	}
2814
2815	// Didn't find it
2816	return false;
2817	}
2818
2819
2820	// --------------------------------------------------------------------------
2821	//
2822	// Function
2823	// Name: BackupDaemon::SetState(int)
2824	// Purpose: Record current action of daemon, and update process title to reflect this
2825	// Created: 11/12/03
2826	//
2827	// --------------------------------------------------------------------------
2828	void BackupDaemon::SetState(int State)
2829	{
2830	// Two little checks
2831	if(State == mState) return;
2832	if(State < 0) return;
2833
2834	// Update
2835	mState = State;
2836
2837	// Set process title
2838	const static char *stateText[] = {"idle", "connected", "error -- waiting for retry", "over limit on server -- not backing up"};
2839	SetProcessTitle(stateText[State]);
2840
2841	// If there's a command socket connected, then inform it -- disconnecting from the
2842	// command socket if there's an error
2843
2844	char newState[64];
2845	sprintf(newState, "state %d", State);
2846	std::string message = newState;
2847
2848	message += "\n";
2849
2850	if(!mapCommandSocketInfo.get())
2851	{
2852	return;
2853	}
2854
2855	if(mapCommandSocketInfo->mpConnectedSocket.get() == 0)
2856	{
2857	return;
2858	}
2859
2860	// Something connected to the command socket, tell it about the new state
2861	try
2862	{
2863	mapCommandSocketInfo->mpConnectedSocket->Write(message.c_str(),
2864	message.length());
2865	}
2866	catch(ConnectionException &ce)
2867	{
2868	BOX_NOTICE("Failed to write state to command socket: " <<
2869	ce.what());
2870	CloseCommandConnection();
2871	}
2872	catch(std::exception &e)
2873	{
2874	BOX_ERROR("Failed to write state to command socket: " <<
2875	e.what());
2876	CloseCommandConnection();
2877	}
2878	catch(...)
2879	{
2880	BOX_ERROR("Failed to write state to command socket: "
2881	"unknown error");
2882	CloseCommandConnection();
2883	}
2884	}
2885
2886
2887	// --------------------------------------------------------------------------
2888	//
2889	// Function
2890	// Name: BackupDaemon::TouchFileInWorkingDir(const char *)
2891	// Purpose: Make sure a zero length file of the name exists in the working directory.
2892	// Use for marking times of events in the filesystem.
2893	// Created: 21/2/04
2894	//
2895	// --------------------------------------------------------------------------
2896	void BackupDaemon::TouchFileInWorkingDir(const char *Filename)
2897	{
2898	// Filename
2899	const Configuration &config(GetConfiguration());
2900	std::string fn(config.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR);
2901	fn += Filename;
2902
2903	// Open and close it to update the timestamp
2904	try
2905	{
2906	FileStream touch(fn, O_WRONLY \| O_CREAT \| O_TRUNC,
2907	S_IRUSR \| S_IWUSR);
2908	}
2909	catch (std::exception &e)
2910	{
2911	BOX_ERROR("Failed to write to timestamp file: " << fn << ": " <<
2912	e.what());
2913	}
2914	}
2915
2916
2917	// --------------------------------------------------------------------------
2918	//
2919	// Function
2920	// Name: BackupDaemon::NotifySysadmin(int)
2921	// Purpose: Run the script to tell the sysadmin about events
2922	// which need attention.
2923	// Created: 25/2/04
2924	//
2925	// --------------------------------------------------------------------------
2926	void BackupDaemon::NotifySysadmin(SysadminNotifier::EventCode Event)
2927	{
2928	static const char *sEventNames[] =
2929	{
2930	"store-full",
2931	"read-error",
2932	"backup-error",
2933	"backup-start",
2934	"backup-finish",
2935	"backup-ok",
2936	0
2937	};
2938
2939	// BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames));
2940	// BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames));
2941	// BOX_TRACE("NotifyEvent__MAX == " << NotifyEvent__MAX);
2942	ASSERT((sizeof(sEventNames)/sizeof(*sEventNames)) == SysadminNotifier::MAX + 1);
2943
2944	if(Event < 0 \|\| Event >= SysadminNotifier::MAX)
2945	{
2946	BOX_ERROR("BackupDaemon::NotifySysadmin() called for "
2947	"invalid event code " << Event);
2948	THROW_EXCEPTION(BackupStoreException,
2949	BadNotifySysadminEventCode);
2950	}
2951
2952	BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " <<
2953	sEventNames[Event]);
2954
2955	if(!GetConfiguration().KeyExists("NotifyAlways") \|\|
2956	!GetConfiguration().GetKeyValueBool("NotifyAlways"))
2957	{
2958	// Don't send lots of repeated messages
2959	// Note: backup-start and backup-finish will always be
2960	// logged, because mLastNotifiedEvent is never set to
2961	// these values and therefore they are never "duplicates".
2962	if(mLastNotifiedEvent == Event)
2963	{
2964	if(Event == SysadminNotifier::BackupOK)
2965	{
2966	BOX_INFO("Suppressing duplicate notification "
2967	"about " << sEventNames[Event]);
2968	}
2969	else
2970	{
2971	BOX_WARNING("Suppressing duplicate notification "
2972	"about " << sEventNames[Event]);
2973	}
2974	return;
2975	}
2976	}
2977
2978	// Is there a notification script?
2979	const Configuration &conf(GetConfiguration());
2980	if(!conf.KeyExists("NotifyScript"))
2981	{
2982	// Log, and then return
2983	if(Event != SysadminNotifier::BackupStart &&
2984	Event != SysadminNotifier::BackupFinish)
2985	{
2986	BOX_INFO("Not notifying administrator about event "
2987	<< sEventNames[Event] << ", set NotifyScript "
2988	"to do this in future");
2989	}
2990	return;
2991	}
2992
2993	// Script to run
2994	std::string script(conf.GetKeyValue("NotifyScript") + " " +
2995	sEventNames[Event] + " \"" + GetConfigFileName() + "\"");
2996
2997	// Log what we're about to do
2998	BOX_INFO("About to notify administrator about event "
2999	<< sEventNames[Event] << ", running script '" << script << "'");
3000
3001	// Then do it
3002	int returnCode = ::system(script.c_str());
3003	if(returnCode != 0)
3004	{
3005	BOX_WARNING("Notify script returned error code: " <<
3006	returnCode << " (" << script << ")");
3007	}
3008	else if(Event != SysadminNotifier::BackupStart &&
3009	Event != SysadminNotifier::BackupFinish)
3010	{
3011	mLastNotifiedEvent = Event;
3012	}
3013	}
3014
3015
3016	// --------------------------------------------------------------------------
3017	//
3018	// Function
3019	// Name: BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &)
3020	// Purpose: Deletes any unused entries in the root directory, if they're scheduled to be deleted.
3021	// Created: 13/5/04
3022	//
3023	// --------------------------------------------------------------------------
3024	void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext)
3025	{
3026	if(mUnusedRootDirEntries.empty())
3027	{
3028	BOX_INFO("Not deleting unused entries - none in list");
3029	return;
3030	}
3031
3032	if(mDeleteUnusedRootDirEntriesAfter == 0)
3033	{
3034	BOX_INFO("Not deleting unused entries - "
3035	"zero delete time (bad)");
3036	return;
3037	}
3038
3039	// Check time
3040	box_time_t now = GetCurrentBoxTime();
3041	if(now < mDeleteUnusedRootDirEntriesAfter)
3042	{
3043	int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter
3044	- now);
3045	BOX_INFO("Not deleting unused entries - too early ("
3046	<< secs << " seconds remaining)");
3047	return;
3048	}
3049
3050	// Entries to delete, and it's the right time to do so...
3051	BOX_NOTICE("Deleting unused locations from store root...");
3052	BackupProtocolCallable &connection(rContext.GetConnection());
3053	for(std::vector<std::pair<int64_t,std::string> >::iterator
3054	i(mUnusedRootDirEntries.begin());
3055	i != mUnusedRootDirEntries.end(); ++i)
3056	{
3057	connection.QueryDeleteDirectory(i->first);
3058	rContext.GetProgressNotifier().NotifyFileDeleted(
3059	i->first, i->second);
3060	}
3061
3062	// Reset state
3063	mDeleteUnusedRootDirEntriesAfter = 0;
3064	mUnusedRootDirEntries.clear();
3065	}
3066
3067	// --------------------------------------------------------------------------
3068
3069	typedef struct
3070	{
3071	int32_t mMagicValue; // also the version number
3072	int32_t mNumEntries;
3073	int64_t mObjectID; // this object ID
3074	int64_t mContainerID; // ID of container
3075	uint64_t mAttributesModTime;
3076	int32_t mOptionsPresent; // bit mask of optional sections / features present
3077
3078	} loc_StreamFormat;
3079
3080	// --------------------------------------------------------------------------
3081	//
3082	// Function
3083	// Name: BackupDaemon::CommandSocketInfo::CommandSocketInfo()
3084	// Purpose: Constructor
3085	// Created: 18/2/04
3086	//
3087	// --------------------------------------------------------------------------
3088	BackupDaemon::CommandSocketInfo::CommandSocketInfo()
3089	: mpGetLine(0)
3090	{
3091	}
3092
3093
3094	// --------------------------------------------------------------------------
3095	//
3096	// Function
3097	// Name: BackupDaemon::CommandSocketInfo::~CommandSocketInfo()
3098	// Purpose: Destructor
3099	// Created: 18/2/04
3100	//
3101	// --------------------------------------------------------------------------
3102	BackupDaemon::CommandSocketInfo::~CommandSocketInfo()
3103	{
3104	if(mpGetLine)
3105	{
3106	delete mpGetLine;
3107	mpGetLine = 0;
3108	}
3109	}
3110
3111	// --------------------------------------------------------------------------
3112	//
3113	// Function
3114	// Name: BackupDaemon::SerializeStoreObjectInfo(
3115	// box_time_t theLastSyncTime,
3116	// box_time_t theNextSyncTime)
3117	// Purpose: Serializes remote directory and file information
3118	// into a stream of bytes, using an Archive
3119	// abstraction.
3120	// Created: 2005/04/11
3121	//
3122	// --------------------------------------------------------------------------
3123
3124	static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F;
3125	static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE";
3126	static const int STOREOBJECTINFO_VERSION = 2;
3127
3128	bool BackupDaemon::SerializeStoreObjectInfo(box_time_t theLastSyncTime,
3129	box_time_t theNextSyncTime) const
3130	{
3131	if(!GetConfiguration().KeyExists("StoreObjectInfoFile"))
3132	{
3133	return false;
3134	}
3135
3136	std::string StoreObjectInfoFile =
3137	GetConfiguration().GetKeyValue("StoreObjectInfoFile");
3138
3139	if(StoreObjectInfoFile.size() <= 0)
3140	{
3141	return false;
3142	}
3143
3144	bool created = false;
3145
3146	try
3147	{
3148	FileStream aFile(StoreObjectInfoFile.c_str(),
3149	O_WRONLY \| O_CREAT \| O_TRUNC);
3150	created = true;
3151
3152	Archive anArchive(aFile, 0);
3153
3154	anArchive.Write(STOREOBJECTINFO_MAGIC_ID_VALUE);
3155	anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING);
3156	anArchive.Write(STOREOBJECTINFO_VERSION);
3157	anArchive.Write(GetLoadedConfigModifiedTime());
3158	anArchive.Write(mClientStoreMarker);
3159	anArchive.Write(theLastSyncTime);
3160	anArchive.Write(theNextSyncTime);
3161
3162	//
3163	//
3164	//
3165	int64_t iCount = mLocations.size();
3166	anArchive.Write(iCount);
3167
3168	for(Locations::const_iterator i = mLocations.begin();
3169	i != mLocations.end(); i++)
3170	{
3171	ASSERT(*i);
3172	(*i)->Serialize(anArchive);
3173	}
3174
3175	//
3176	//
3177	//
3178	iCount = mIDMapMounts.size();
3179	anArchive.Write(iCount);
3180
3181	for(int v = 0; v < iCount; v++)
3182	anArchive.Write(mIDMapMounts[v]);
3183
3184	//
3185	//
3186	//
3187	iCount = mUnusedRootDirEntries.size();
3188	anArchive.Write(iCount);
3189
3190	for(int v = 0; v < iCount; v++)
3191	{
3192	anArchive.Write(mUnusedRootDirEntries[v].first);
3193	anArchive.Write(mUnusedRootDirEntries[v].second);
3194	}
3195
3196	if (iCount > 0)
3197	{
3198	anArchive.Write(mDeleteUnusedRootDirEntriesAfter);
3199	}
3200
3201	//
3202	//
3203	//
3204	aFile.Close();
3205	BOX_INFO("Saved store object info file version " <<
3206	STOREOBJECTINFO_VERSION << " (" <<
3207	StoreObjectInfoFile << ")");
3208	}
3209	catch(std::exception &e)
3210	{
3211	BOX_ERROR("Failed to write StoreObjectInfoFile: " <<
3212	StoreObjectInfoFile << ": " << e.what());
3213	}
3214	catch(...)
3215	{
3216	BOX_ERROR("Failed to write StoreObjectInfoFile: " <<
3217	StoreObjectInfoFile << ": unknown error");
3218	}
3219
3220	return created;
3221	}
3222
3223	// --------------------------------------------------------------------------
3224	//
3225	// Function
3226	// Name: BackupDaemon::DeserializeStoreObjectInfo(
3227	// box_time_t & theLastSyncTime,
3228	// box_time_t & theNextSyncTime)
3229	// Purpose: Deserializes remote directory and file information
3230	// from a stream of bytes, using an Archive
3231	// abstraction.
3232	// Created: 2005/04/11
3233	//
3234	// --------------------------------------------------------------------------
3235	bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime,
3236	box_time_t & theNextSyncTime)
3237	{
3238	//
3239	//
3240	//
3241	DeleteAllLocations();
3242
3243	//
3244	//
3245	//
3246	if(!GetConfiguration().KeyExists("StoreObjectInfoFile"))
3247	{
3248	return false;
3249	}
3250
3251	std::string StoreObjectInfoFile =
3252	GetConfiguration().GetKeyValue("StoreObjectInfoFile");
3253
3254	if(StoreObjectInfoFile.size() <= 0)
3255	{
3256	return false;
3257	}
3258
3259	try
3260	{
3261	FileStream aFile(StoreObjectInfoFile.c_str(), O_RDONLY);
3262	Archive anArchive(aFile, 0);
3263
3264	//
3265	// see if the content looks like a valid serialised archive
3266	//
3267	int iMagicValue = 0;
3268	anArchive.Read(iMagicValue);
3269
3270	if(iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE)
3271	{
3272	BOX_WARNING("Store object info file "
3273	"is not a valid or compatible serialised "
3274	"archive. Will re-cache from store. "
3275	"(" << StoreObjectInfoFile << ")");
3276	return false;
3277	}
3278
3279	//
3280	// get a bit optimistic and read in a string identifier
3281	//
3282	std::string strMagicValue;
3283	anArchive.Read(strMagicValue);
3284
3285	if(strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING)
3286	{
3287	BOX_WARNING("Store object info file "
3288	"is not a valid or compatible serialised "
3289	"archive. Will re-cache from store. "
3290	"(" << StoreObjectInfoFile << ")");
3291	return false;
3292	}
3293
3294	//
3295	// check if we are loading some future format
3296	// version by mistake
3297	//
3298	int iVersion = 0;
3299	anArchive.Read(iVersion);
3300
3301	if(iVersion != STOREOBJECTINFO_VERSION)
3302	{
3303	BOX_WARNING("Store object info file "
3304	"version " << iVersion << " unsupported. "
3305	"Will re-cache from store. "
3306	"(" << StoreObjectInfoFile << ")");
3307	return false;
3308	}
3309
3310	//
3311	// check if this state file is even valid
3312	// for the loaded bbackupd.conf file
3313	//
3314	box_time_t lastKnownConfigModTime;
3315	anArchive.Read(lastKnownConfigModTime);
3316
3317	if(lastKnownConfigModTime != GetLoadedConfigModifiedTime())
3318	{
3319	BOX_WARNING("Store object info file "
3320	"out of date. Will re-cache from store. "
3321	"(" << StoreObjectInfoFile << ")");
3322	return false;
3323	}
3324
3325	//
3326	// this is it, go at it
3327	//
3328	anArchive.Read(mClientStoreMarker);
3329	anArchive.Read(theLastSyncTime);
3330	anArchive.Read(theNextSyncTime);
3331
3332	//
3333	//
3334	//
3335	int64_t iCount = 0;
3336	anArchive.Read(iCount);
3337
3338	for(int v = 0; v < iCount; v++)
3339	{
3340	Location* pLocation = new Location;
3341	if(!pLocation)
3342	{
3343	throw std::bad_alloc();
3344	}
3345
3346	pLocation->Deserialize(anArchive);
3347	mLocations.push_back(pLocation);
3348	}
3349
3350	//
3351	//
3352	//
3353	iCount = 0;
3354	anArchive.Read(iCount);
3355
3356	for(int v = 0; v < iCount; v++)
3357	{
3358	std::string strItem;
3359	anArchive.Read(strItem);
3360
3361	mIDMapMounts.push_back(strItem);
3362	}
3363
3364	//
3365	//
3366	//
3367	iCount = 0;
3368	anArchive.Read(iCount);
3369
3370	for(int v = 0; v < iCount; v++)
3371	{
3372	int64_t anId;
3373	anArchive.Read(anId);
3374
3375	std::string aName;
3376	anArchive.Read(aName);
3377
3378	mUnusedRootDirEntries.push_back(std::pair<int64_t, std::string>(anId, aName));
3379	}
3380
3381	if (iCount > 0)
3382	anArchive.Read(mDeleteUnusedRootDirEntriesAfter);
3383
3384	//
3385	//
3386	//
3387	aFile.Close();
3388	BOX_INFO("Loaded store object info file version " << iVersion
3389	<< " (" << StoreObjectInfoFile << ")");
3390
3391	return true;
3392	}
3393	catch(std::exception &e)
3394	{
3395	BOX_ERROR("Internal error reading store object info file: "
3396	<< StoreObjectInfoFile << ": " << e.what());
3397	}
3398	catch(...)
3399	{
3400	BOX_ERROR("Internal error reading store object info file: "
3401	<< StoreObjectInfoFile << ": unknown error");
3402	}
3403
3404	DeleteAllLocations();
3405
3406	mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;
3407	theLastSyncTime = 0;
3408	theNextSyncTime = 0;
3409
3410	BOX_WARNING("Store object info file is missing, not accessible, "
3411	"or inconsistent. Will re-cache from store. "
3412	"(" << StoreObjectInfoFile << ")");
3413
3414	return false;
3415	}
3416
3417	// --------------------------------------------------------------------------
3418	//
3419	// Function
3420	// Name: BackupDaemon::DeleteStoreObjectInfo()
3421	// Purpose: Deletes the serialised state file, to prevent us
3422	// from using it again if a backup is interrupted.
3423	//
3424	// Created: 2006/02/12
3425	//
3426	// --------------------------------------------------------------------------
3427
3428	bool BackupDaemon::DeleteStoreObjectInfo() const
3429	{
3430	if(!GetConfiguration().KeyExists("StoreObjectInfoFile"))
3431	{
3432	return false;
3433	}
3434
3435	std::string storeObjectInfoFile(GetConfiguration().GetKeyValue("StoreObjectInfoFile"));
3436
3437	// Check to see if the file exists
3438	if(!FileExists(storeObjectInfoFile.c_str()))
3439	{
3440	// File doesn't exist -- so can't be deleted. But something
3441	// isn't quite right, so log a message
3442	BOX_WARNING("StoreObjectInfoFile did not exist when it "
3443	"was supposed to: " << storeObjectInfoFile);
3444
3445	// Return true to stop things going around in a loop
3446	return true;
3447	}
3448
3449	// Actually delete it
3450	if(::unlink(storeObjectInfoFile.c_str()) != 0)
3451	{
3452	BOX_LOG_SYS_ERROR("Failed to delete the old "
3453	"StoreObjectInfoFile: " << storeObjectInfoFile);
3454	return false;
3455	}
3456
3457	return true;
3458	}

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: box/trunk/bin/bbackupd/BackupDaemon.cpp @ 3102

Download in other formats: