source: box/trunk/contrib/bbreporter/bbreporter.py @ 2352

Revision 2352, 20.6 KB checked in by cbkm, 4 years ago (diff)
  • fix issue with log format change, but stay backwards compatible.
  • Property svn:executable set to *
Line 
1#!/usr/bin/env python
2# BoxBackupReporter - Simple script to report on backups that have been
3#                     performed using BoxBackup.
4#
5# Copyright: (C) 2007 Three A IT Limited
6# Author: Kenny Millington <kenny.millington@3ait.co.uk>
7#
8# Credit: This script is based on the ideas of BoxReport.pl by Matt Brown of
9#         Three A IT Support Limited.
10#
11################################################################################
12# !! Important !!
13# To make use of this script you need to run the boxbackup client with the -v
14# commandline option and set LogAllFileAccess = yes in your bbackupd.conf file.
15#
16# Notes on lazy mode:
17# If reporting on lazy mode backups you absolutely must ensure that
18# logrotate (or similar) rotates the log files at the same rate at
19# which you run this reporting script or you will report on the same
20# backup sessions on each execution.
21#
22# Notes on --rotate and log rotation in general:
23# The use-case for --rotate that I imagine is that you'll add a line like the
24# following into your syslog.conf file:-
25#
26# local6.* -/var/log/box
27#
28# Then specifying --rotate to this script will make it rotate the logs
29# each time you report on the backup so that you don't risk a backup session
30# being spread across two log files (e.g. syslog and syslog.0).
31#
32# NB: To do this you'll need to prevent logrotate/syslog from rotating your
33#     /var/log/box file. On Debian based distros you'll need to edit two files.
34#     
35#     First: /etc/cron.daily/sysklogd, find the following line and make the
36#            the required change:
37#            Change: for LOG in `syslogd-listfiles`
38#                To: for LOG in `syslogd-listfiles -s box`
39#                                 
40#     Second: /etc/cron.weekly/sysklogd, find the following line and make the
41#            the required change:
42#            Change: for LOG in `syslogd-listfiles --weekly`
43#                To: for LOG in `syslogd-listfiles --weekly -s box`
44#
45# Alternatively, if suitable just ensure the backups stop before the
46# /etc/cron.daily/sysklogd file runs (usually 6:25am) and report on it
47# before the files get rotated. (If going for this option I'd just use
48# the main syslog file instead of creating a separate log file for box
49# backup since you know for a fact the syslog will get rotated daily.)
50#
51################################################################################
52# This program is free software: you can redistribute it and/or modify
53# it under the terms of the GNU General Public License as published by
54# the Free Software Foundation, either version 3 of the License, or
55# (at your option) any later version.
56#
57# This program is distributed in the hope that it will be useful,
58# but WITHOUT ANY WARRANTY; without even the implied warranty of
59# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
60# GNU General Public License for more details.
61#
62# You should have received a copy of the GNU General Public License
63# along with this program.  If not, see <http://www.gnu.org/licenses/>. 
64#
65
66# If sendmail is not in one of these paths, add the path.
67SENDMAIL_PATHS = ["/usr/sbin/", "/usr/bin/", "/bin/" , "/sbin/"]
68
69# The name of the sendmail binary, you probably won't need to change this.
70SENDMAIL_BIN = "sendmail"
71
72# Number of files to rotate around
73ROTATE_COUNT = 7
74
75# Import the required libraries
76import sys, os, re, getopt, shutil, gzip
77
78class BoxBackupReporter:
79    class BoxBackupReporterError(Exception):
80        pass
81   
82    def __init__(self, config_file="/etc/box/bbackupd.conf", 
83                 log_file="/var/log/syslog", email_to=None, 
84                 email_from="report@boxbackup", rotate=False, 
85                 verbose=False, stats=False, sort=False, debug=False):
86       
87        # Config options
88        self.config_file = config_file
89        self.log_file = log_file
90        self.email_to = email_to
91        self.email_from = email_from
92        self.rotate_log_file = rotate
93        self.verbose_report = verbose
94        self.usage_stats = stats
95        self.sort_files = sort
96        self.debug = debug
97
98        # Regex's
99        self.re_automatic_backup = re.compile(" *AutomaticBackup *= *no", re.I)
100        self.re_syslog = re.compile("(\S+) +(\S+) +([\d:]+) +(\S+) +([^:]+): +"+
101                                    "(?:[A-Z]+:)? *([^:]+): *(.*)") 
102       
103        # Initialise report
104        self.reset()
105       
106    def _debug(self, msg):
107        if self.debug:
108            sys.stderr.write("[bbreporter.py Debug]: %s\n" % msg)
109
110    def reset(self):
111        # Reset report data to default values
112        self.hostname = ""
113        self.patched_files = []
114        self.synced_files = []
115        self.uploaded_files = []
116        self.warnings = []
117        self.errors = []
118        self.stats = None
119        self.start_datetime = "Unknown"
120        self.end_datetime = "Unfinished"
121        self.report = "No report generated"
122   
123    def run(self):
124        try:
125            self._determine_operating_mode()
126           
127            if self.lazy_mode:
128                self._debug("Operating in LAZY MODE.")
129            else:
130                self._debug("Operating in SNAPSHOT MODE.")
131
132        except IOError:
133            raise BoxBackupReporter.BoxBackupReporterError("Error: "+\
134                  "Config file \"%s\" could not be read." % self.config_file)
135           
136        try:
137            self._parse_syslog()
138        except IOError:
139            raise BoxBackupReporter.BoxBackupReporterError("Error: "+\
140                  "Log file \"%s\" could not be read." % self.log_file)
141       
142        self._parse_stats()
143        self._generate_report()
144       
145    def deliver(self):
146        # If we're not e-mailing the report then just dump it to stdout
147        # and return.
148        if self.email_to is None:
149            print self.report
150            # Now that we've delivered the report it's time to rotate the logs
151            # if we're requested to do so.
152            self._rotate_log()
153            return
154       
155        # Locate the sendmail binary
156        sendmail = self._locate_sendmail()
157        if(sendmail is None):
158            raise BoxBackupReporter.BoxBackupReporterError("Error: "+\
159                  "Could not find sendmail binary - Unable to send e-mail!")
160           
161       
162        # Set the subject based on whether we think we failed or not.
163        # (suffice it to say I consider getting an error and backing up
164        #  no files a failure or indeed not finding a start time in the logs).
165        subject = "BoxBackup Reporter (%s) - " % self.hostname
166        if self.start_datetime == "Unknown" or\
167            (len(self.patched_files)  == 0 and len(self.synced_files) == 0 and\
168             len(self.uploaded_files) == 0):
169            subject = subject + "FAILED"
170        else:
171            subject = subject + "SUCCESS"
172
173            if len(self.errors) > 0:
174                subject = subject + " (with errors)"
175           
176        # Prepare the e-mail message.
177        mail = []
178        mail.append("To: " + self.email_to)
179        mail.append("From: " + self.email_from)
180        mail.append("Subject: " + subject)
181        mail.append("")
182        mail.append(self.report)
183       
184        # Send the mail.
185        p = os.popen(sendmail + " -t", "w")
186        p.write("\r\n".join(mail))
187        p.close()
188       
189        # Now that we've delivered the report it's time to rotate the logs
190        # if we're requested to do so.
191        self._rotate_log()
192         
193    def _determine_operating_mode(self):
194        # Scan the config file and determine if we're running in lazy or
195        # snapshot mode.
196        cfh = open(self.config_file)
197
198        for line in cfh:
199            if not line.startswith("#"):
200                if self.re_automatic_backup.match(line):
201                    self.lazy_mode = False
202                    cfh.close()
203                    return
204       
205        self.lazy_mode = True
206        cfh.close()
207   
208    def _parse_syslog(self):
209        lfh = open(self.log_file)
210       
211        patched_files = {}
212        uploaded_files = {}
213        synced_files = {}
214       
215        for line in lfh:
216            # Only run the regex if we find a box backup entry.
217            if line.find("Box Backup") > -1 or line.find("bbackupd") > -1:
218                raw_data = self.re_syslog.findall(line)
219                try:
220                    data = raw_data[0]
221                except IndexError:
222                    # If the regex didn't match it's not a message that we're
223                    # interested in so move to the next line.
224                    continue
225               
226                # Set the hostname, it shouldn't change in a log file
227                self.hostname = data[3]
228               
229                # If we find the backup-start event then set the start_datetime.
230                if data[6].find("backup-start") > -1:
231                    # If we're not in lazy mode or the start_datetime hasn't
232                    # been set then reset the data and set it.
233                    #
234                    # If we're in lazy mode and encounter a second backup-start
235                    # we don't want to change the start_datetime likewise if
236                    # we're not in lazy mode we do want to and we want to reset
237                    # so we only capture the most recent session.
238                    if not self.lazy_mode or self.start_datetime == "Unknown":
239                        self._debug("Reset start dtime with old time: %s." % 
240                                    self.start_datetime)
241                       
242                        # Reset ourselves
243                        self.reset()
244                       
245                        # Reset our temporary variables which we store
246                        # the files in.
247                        patched_files = {}
248                        uploaded_files = {}
249                        synced_files = {}
250
251                        self.start_datetime = data[1]+" "+data[0]+ " "+data[2]
252                        self._debug("Reset start dtime with new time %s." %
253                                    self.start_datetime)
254                                             
255                # If we find the backup-finish event then set the end_datetime.
256                elif data[6].find("backup-finish") > -1:
257                    self.end_datetime = data[1] + " " + data[0] + " " + data[2]
258                    self._debug("Set end dtime: %s" % self.end_datetime)
259               
260                # Only log the events if we have our start time.
261                elif self.start_datetime != "Unknown":
262                    # We found a patch event, add the file to the patched_files.
263                    if data[5] == "Uploading patch to file":
264                        patched_files[data[6]] = ""
265
266                    # We found an upload event, add to uploaded files.
267                    elif data[5] == "Uploading complete file":
268                        uploaded_files[data[6]] = ""
269                   
270                    # We found another upload event.
271                    elif data[5] == "Uploaded file":
272                        uploaded_files[data[6]] = ""
273                   
274                    # We found a sync event, add the file to the synced_files.
275                    elif data[5] == "Synchronised file":
276                        synced_files[data[6]] = ""
277               
278                    # We found a warning, add the warning to the warnings.
279                    elif data[5] == "WARNING":
280                        self.warnings.append(data[6])
281
282                    # We found an error, add the error to the errors.
283                    elif data[5] == "ERROR":
284                        self.errors.append(data[6])
285       
286       
287        self.patched_files = patched_files.keys()
288        self.uploaded_files = uploaded_files.keys()
289        self.synced_files = synced_files.keys()
290       
291        # There's no point running the sort functions if we're not going
292        # to display the resultant lists.
293        if self.sort_files and self.verbose_report:
294            self.patched_files.sort()
295            self.uploaded_files.sort()
296           
297       
298        lfh.close()
299   
300    def _parse_stats(self):
301        if(not self.usage_stats):
302            return
303       
304        # Grab the stats from bbackupquery
305        sfh = os.popen("bbackupquery usage quit", "r")
306        raw_stats = sfh.read()
307        sfh.close()
308       
309        # Parse the stats
310        stats_re = re.compile("commands.[\n ]*\n(.*)\n+", re.S)
311        stats = stats_re.findall(raw_stats)
312       
313        try:
314            self.stats = stats[0]
315        except IndexError:
316            self.stats = "Unable to retrieve usage information."
317           
318    def _generate_report(self):
319        if self.start_datetime == "Unknown":
320            self.report = "No report data has been found."
321            return
322       
323        total_files = len(self.patched_files) + len(self.uploaded_files)
324       
325        report = []
326        report.append("--------------------------------------------------")
327        report.append("Report Title  : Box Backup - Backup Statistics")
328        report.append("Report Period : %s - %s" % (self.start_datetime, 
329                                                   self.end_datetime))
330        report.append("--------------------------------------------------")
331        report.append("")
332        report.append("This is your box backup report, in summary:")
333        report.append("")
334        report.append("%d file(s) have been backed up." % total_files)
335        report.append("%d file(s) were uploaded." % len(self.uploaded_files))
336        report.append("%d file(s) were patched." % len(self.patched_files))
337        report.append("%d file(s) were synchronised." % len(self.synced_files))
338       
339        report.append("")
340        report.append("%d warning(s) occurred." % len(self.warnings))
341        report.append("%d error(s) occurred." % len(self.errors))
342        report.append("")
343        report.append("")
344       
345        # If we asked for the backup stats and they're available
346        # show them.
347        if(self.stats is not None and self.stats != ""):
348            report.append("Your backup usage information follows:")
349            report.append("")
350            report.append(self.stats)
351            report.append("")
352            report.append("")
353           
354        # List the files if we've been asked for a verbose report.
355        if(self.verbose_report):
356            if len(self.uploaded_files) > 0:
357                report.append("Uploaded Files (%d)" % len(self.uploaded_files))
358                report.append("---------------------")
359                for file in self.uploaded_files:
360                    report.append(file)
361                report.append("")
362                report.append("")
363           
364            if len(self.patched_files) > 0:
365                report.append("Patched Files (%d)" % len(self.patched_files))
366                report.append("---------------------")
367                for file in self.patched_files:
368                    report.append(file)
369                report.append("")
370                report.append("")
371   
372        # Always output the warnings/errors.
373        if len(self.warnings) > 0:
374            report.append("Warnings (%d)" % len(self.warnings))
375            report.append("---------------------")
376            for warning in self.warnings:
377                report.append(warning)
378            report.append("")
379            report.append("")
380       
381        if len(self.errors) > 0:
382            report.append("Errors (%d)" % len(self.errors))
383            report.append("---------------------")
384            for error in self.errors:
385                report.append(error)
386            report.append("")
387            report.append("")
388
389        self.report = "\r\n".join(report)
390
391    def _locate_sendmail(self):
392        for path in SENDMAIL_PATHS:
393            sendmail = os.path.join(path, SENDMAIL_BIN)
394            if os.path.isfile(sendmail):
395                return sendmail
396
397        return None
398   
399    def _rotate_log(self):
400        # If we're not configured to rotate then abort.
401        if(not self.rotate_log_file):
402            return
403       
404        # So we have these files to possibly account for while we process the
405        # rotation:-
406        # self.log_file, self.log_file.0, self.log_file.1.gz, self.log_file.2.gz
407        # self.log_file.3.gz....self.log_file.(ROTATE_COUNT-1).gz
408        #
409        # Algorithm:-
410        # * Delete last file.
411        # * Work backwards moving 5->6, 4->5, 3->4, etc... but stop at .0
412        # * For .0 move it to .1 then gzip it.
413        # * Move self.log_file to .0
414        # * Done.
415       
416        # If it exists, remove the oldest file.
417        if(os.path.isfile(self.log_file + ".%d.gz" % (ROTATE_COUNT - 1))):
418            os.unlink(self.log_file + ".%d.gz" % (ROTATE_COUNT - 1))   
419
420        # Copy through the other gzipped log files.
421        for i in range(ROTATE_COUNT - 1, 1, -1):
422            src_file = self.log_file + ".%d.gz" % (i - 1)
423            dst_file = self.log_file + ".%d.gz" % i
424           
425            # If the source file exists move/rename it.
426            if(os.path.isfile(src_file)):
427                shutil.move(src_file, dst_file)
428       
429        # Now we need to handle the .0 -> .1.gz case.
430        if(os.path.isfile(self.log_file + ".0")):
431            # Move .0 to .1
432            shutil.move(self.log_file + ".0", self.log_file + ".1")
433           
434            # gzip the file.
435            fh = open(self.log_file + ".1", "r")
436            zfh = gzip.GzipFile(self.log_file + ".1.gz", "w")
437            zfh.write(fh.read())
438            zfh.flush()
439            zfh.close()
440            fh.close()
441           
442            # If gzip worked remove the original .1 file.
443            if(os.path.isfile(self.log_file + ".1.gz")):
444                os.unlink(self.log_file + ".1")
445       
446        # Finally move the current logfile to .0
447        shutil.move(self.log_file, self.log_file + ".0") 
448
449
450def stderr(text):
451    sys.stderr.write("%s\n" % text)
452
453def usage():
454    stderr("Usage: %s [OPTIONS]\n" % sys.argv[0])
455    stderr("Valid Options:-")
456    stderr("  --logfile=LOGFILE\t\t\tSpecify the logfile to process,\n"+\
457           "\t\t\t\t\tdefault: /var/log/syslog\n")
458   
459    stderr("  --configfile=CONFIGFILE\t\tSpecify the bbackupd config file,\n "+\
460           "\t\t\t\t\tdefault: /etc/box/bbackupd.conf\n")
461   
462    stderr("  --email-to=user@example.com\t\tSpecify the e-mail address(es)\n"+\
463           "\t\t\t\t\tto send the report to, default is to\n"+\
464           "\t\t\t\t\tdisplay the report on the console.\n")
465   
466    stderr("  --email-from=user@example.com\t\tSpecify the e-mail address(es)"+\
467           "\n\t\t\t\t\tto set the From: address to,\n "+\
468           "\t\t\t\t\tdefault: report@boxbackup\n")
469   
470    stderr("  --stats\t\t\t\tIncludes the usage stats retrieved from \n"+\
471           "\t\t\t\t\t'bbackupquery usage' in the report.\n")
472   
473    stderr("  --sort\t\t\t\tSorts the file lists in verbose mode.\n")
474   
475    stderr("  --debug\t\t\t\tEnables debug output.\n")
476
477    stderr("  --verbose\t\t\t\tList every file that was backed up to\n"+\
478           "\t\t\t\t\tthe server, default is to just display\n"+\
479           "\t\t\t\t\tthe summary.\n")
480   
481    stderr("  --rotate\t\t\t\tRotates the log files like logrotate\n"+\
482           "\t\t\t\t\twould, see the comments for a use-case.\n")
483
484def main():
485    # The defaults
486    logfile = "/var/log/syslog"
487    configfile = "/etc/box/bbackupd.conf"
488    email_to = None
489    email_from = "report@boxbackup"
490    rotate = False
491    verbose = False 
492    stats = False
493    sort = False
494    debug = False
495    # Parse the options
496    try:
497        opts, args = getopt.getopt(sys.argv[1:], "dosrvhl:c:t:f:", 
498                        ["help", "logfile=", "configfile=","email-to=", 
499                         "email-from=","rotate","verbose","stats","sort",
500                         "debug"])
501    except getopt.GetoptError:
502        usage()
503        return
504   
505    for opt, arg in opts:
506        if(opt in ("--logfile","-l")):
507            logfile = arg
508        elif(opt in ("--configfile", "-c")):
509            configfile = arg
510        elif(opt in ("--email-to", "-t")):
511            email_to = arg
512        elif(opt in ("--email-from", "-f")):
513            email_from = arg
514        elif(opt in ("--rotate", "-r")):
515            rotate = True
516        elif(opt in ("--verbose", "-v")):
517            verbose = True
518        elif(opt in ("--stats", "-s")):
519            stats = True
520        elif(opt in ("--sort", "-o")):
521            sort = True
522        elif(opt in ("--debug", "-d")):
523            debug = True
524        elif(opt in ("--help", "-h")):
525            usage()
526            return
527   
528    # Run the reporter
529    bbr = BoxBackupReporter(configfile, logfile, email_to, email_from, 
530                            rotate, verbose, stats, sort, debug)
531    try:
532        bbr.run()
533        bbr.deliver()
534    except BoxBackupReporter.BoxBackupReporterError, error_msg:
535        print error_msg
536
537if __name__ == "__main__":
538    main()
Note: See TracBrowser for help on using the repository browser.