rossigee-backups 2.9.0__py3-none-any.whl

backups/compress.py

@@ -0,0 +1,19 @@
+import subprocess
+import logging
+
+from backups.exceptions import BackupException
+
+def compress(filename, tracer=None):
+    logging.info("Compressing '%s'..." % filename)
+    compfilename = '%s.gz' % filename
+    comperrsname = '%s.err' % filename
+    comperrs = open(comperrsname, 'wb')
+    compargs = ['gzip', '-f', '--fast', filename]
+    compproc1 = subprocess.Popen(compargs, stdin=subprocess.PIPE, stderr=comperrs)
+    compproc1.wait()
+    comperrs.close()
+    exitcode = compproc1.returncode
+    if exitcode != 0:
+        errmsg = open(comperrsname, 'rb').read()
+        raise BackupException("Error while compressing: %s" % errmsg)
+    return compfilename

backups/destinations/__init__.py

@@ -0,0 +1,12 @@
+# Manage list of destinations as they are initialised
+handlers = {}
+
+import logging
+
+def backupdestination(id):
+    def register(handler_class):
+        logging.debug("Adding destination '%s' as '%s'" % (handler_class, id))
+        handlers[id] = handler_class
+        return handler_class
+
+    return register

backups/destinations/b2.py

@@ -0,0 +1,66 @@
+import os, os.path
+import datetime
+import logging
+
+from b2sdk.v2 import InMemoryAccountInfo, B2Api
+
+from backups.exceptions import BackupException
+from backups.destinations import backupdestination
+from backups.destinations.destination import BackupDestination
+
+@backupdestination('b2')
+class B2(BackupDestination):
+    def __init__(self, config):
+        BackupDestination.__init__(self, config)
+        self.bucket_name = config['bucket']
+        self.application_key_id = config['credentials']['application_key_id']
+        self.application_key = config['credentials']['application_key']
+
+    def _get_bucket(self):
+        info = InMemoryAccountInfo()
+        api = B2Api(info)
+        api.authorize_account("production", self.application_key_id, self.application_key)
+        return api.get_bucket_by_name(self.bucket_name)
+
+    def send(self, id, name, filename):
+        remote_name = "%s/%s/%s" % (id, self.runtime.strftime("%Y%m%d%H%M%S"), os.path.basename(filename))
+        logging.info("Uploading '%s' backup for '%s' to B2 (%s/%s)..." % (name, self.id, self.bucket_name, remote_name))
+        try:
+            bucket = self._get_bucket()
+            bucket.upload_local_file(local_file=filename, file_name=remote_name)
+        except Exception as e:
+            raise BackupException("Error while uploading (%s): %s" % (self.id, str(e)))
+        return "b2://%s/%s" % (self.bucket_name, remote_name)
+
+    def cleanup(self, id, name):
+        logging.info("Clearing down older '%s' backups for '%s' from B2..." % (name, self.id))
+        try:
+            bucket = self._get_bucket()
+            candidates = []
+            for file_version, _ in bucket.ls(folder_to_list="%s/" % id, recursive=True):
+                candidates.append([file_version.upload_timestamp, file_version])
+            candidates.sort(key=lambda x: x[0])
+        except Exception as e:
+            raise BackupException("Error listing B2 bucket (%s): %s" % (self.id, str(e)))
+
+        removable = []
+        retained = []
+        if self.retention_copies > 0:
+            items = [f for d, f in candidates]
+            if len(items) > self.retention_copies:
+                removable = items[0:(len(items) - self.retention_copies)]
+            retained = [f.file_name for f in items[(len(items) - self.retention_copies):]]
+        if self.retention_days > 0:
+            now_ms = datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000
+            for ts, file_version in candidates:
+                age_days = (now_ms - ts) / (1000 * 86400)
+                if age_days > self.retention_days:
+                    removable.append(file_version)
+        for file_version in removable:
+            logging.info("Removing '%s'..." % file_version.file_name)
+            try:
+                file_version.delete()
+            except Exception as e:
+                logging.warning("Failed to remove '%s': %s" % (file_version.file_name, str(e)))
+
+        return retained

backups/destinations/destination.py

@@ -0,0 +1,14 @@
+import datetime
+
+# Abstract
+class BackupDestination:
+    def __init__(self, config):
+        self.id = config['id']
+        self.runtime = datetime.datetime.now()
+
+        self.retention_copies = 0
+        if 'retention_copies' in config:
+            self.retention_copies = int(config['retention_copies'])
+        self.retention_days = 0
+        if 'retention_days' in config:
+            self.retention_days = int(config['retention_days'])

backups/destinations/dropbox.py

@@ -0,0 +1,72 @@
+import os, os.path
+import datetime
+import logging
+
+import dropbox
+from dropbox.exceptions import ApiError
+
+from backups.exceptions import BackupException
+from backups.destinations import backupdestination
+from backups.destinations.destination import BackupDestination
+
+@backupdestination('dropbox')
+class Dropbox(BackupDestination):
+    def __init__(self, config):
+        BackupDestination.__init__(self, config)
+        self.access_token = config['access_token']
+        self.folder = config.get('folder', '/backups')
+
+    def _get_client(self):
+        return dropbox.Dropbox(self.access_token)
+
+    def send(self, id, name, filename):
+        remote_path = "%s/%s/%s/%s" % (
+            self.folder.rstrip('/'),
+            id,
+            self.runtime.strftime("%Y%m%d%H%M%S"),
+            os.path.basename(filename))
+        logging.info("Uploading '%s' backup for '%s' to Dropbox (%s)..." % (name, self.id, remote_path))
+        try:
+            dbx = self._get_client()
+            with open(filename, 'rb') as f:
+                dbx.files_upload(f.read(), remote_path)
+        except ApiError as e:
+            raise BackupException("Error while uploading (%s): %s" % (self.id, str(e)))
+        return remote_path
+
+    def cleanup(self, id, name):
+        folder_path = "%s/%s" % (self.folder.rstrip('/'), id)
+        logging.info("Clearing down older '%s' backups for '%s' from Dropbox (%s)..." % (name, self.id, folder_path))
+        try:
+            dbx = self._get_client()
+            result = dbx.files_list_folder(folder_path)
+            entries = result.entries
+            while result.has_more:
+                result = dbx.files_list_folder_continue(result.cursor)
+                entries.extend(result.entries)
+        except ApiError as e:
+            if e.error.is_path() and e.error.get_path().is_not_found():
+                return []
+            raise BackupException("Error listing Dropbox folder (%s): %s" % (self.id, str(e)))
+
+        candidates = []
+        for entry in entries:
+            if isinstance(entry, dropbox.files.FolderMetadata):
+                candidates.append([entry.name, entry.path_lower])
+        candidates.sort()
+
+        removable = []
+        retained = []
+        if self.retention_copies > 0:
+            paths = [p for n, p in candidates]
+            if len(paths) > self.retention_copies:
+                removable = paths[0:(len(paths) - self.retention_copies)]
+            retained = paths[(len(paths) - self.retention_copies):]
+        for path in removable:
+            logging.info("Removing '%s'..." % path)
+            try:
+                dbx.files_delete_v2(path)
+            except ApiError as e:
+                logging.warning("Failed to remove '%s': %s" % (path, str(e)))
+
+        return retained

backups/destinations/gdrive.py

@@ -0,0 +1,133 @@
+import os, os.path
+import datetime
+import logging
+import contextlib
+
+from google.oauth2 import service_account
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload
+from googleapiclient.errors import HttpError
+
+from backups.exceptions import BackupException
+from backups.destinations import backupdestination
+from backups.destinations.destination import BackupDestination
+
+SCOPES = ['https://www.googleapis.com/auth/drive']
+
+@backupdestination('gdrive')
+class GDrive(BackupDestination):
+    def __init__(self, config):
+        BackupDestination.__init__(self, config)
+        self.creds_file = config['creds_file']
+        self.folder_id = config['folder_id']
+
+    def _get_tracer(self):
+        if hasattr(self, 'tracer') and self.tracer is not None:
+            return self.tracer
+        class _NoOpTracer:
+            @contextlib.contextmanager
+            def start_as_current_span(self, name, **kwargs):
+                class _NoOpSpan:
+                    def set_attribute(self, *a, **k): pass
+                    def record_exception(self, *a, **k): pass
+                    def set_status(self, *a, **k): pass
+                yield _NoOpSpan()
+        return _NoOpTracer()
+
+    def _get_service(self):
+        credentials = service_account.Credentials.from_service_account_file(
+            self.creds_file, scopes=SCOPES)
+        return build('drive', 'v3', credentials=credentials)
+
+    def _get_or_create_folder(self, service, parent_id, folder_name):
+        query = "name='%s' and '%s' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false" % (folder_name, parent_id)
+        results = service.files().list(q=query, fields="files(id)").execute()
+        files = results.get('files', [])
+        if files:
+            return files[0]['id']
+        metadata = {
+            'name': folder_name,
+            'mimeType': 'application/vnd.google-apps.folder',
+            'parents': [parent_id]
+        }
+        folder = service.files().create(body=metadata, fields='id').execute()
+        return folder['id']
+
+    def send(self, id, name, filename):
+        with self._get_tracer().start_as_current_span("gdrive_upload") as span:
+            span.set_attribute("destination.id", self.id)
+            span.set_attribute("gdrive.folder_id", self.folder_id)
+            span.set_attribute("source.id", id)
+            span.set_attribute("upload.file", filename)
+            
+            logging.info("Uploading '%s' backup for '%s' to Google Drive..." % (name, self.id))
+            try:
+                service = self._get_service()
+                source_folder_id = self._get_or_create_folder(service, self.folder_id, id)
+                ts_folder_id = self._get_or_create_folder(service, source_folder_id, self.runtime.strftime("%Y%m%d%H%M%S"))
+                file_metadata = {
+                    'name': os.path.basename(filename),
+                    'parents': [ts_folder_id]
+                }
+                media = MediaFileUpload(filename, resumable=True)
+                uploaded = service.files().create(
+                    body=file_metadata, media_body=media, fields='id,name').execute()
+                span.set_attribute("gdrive.file_id", uploaded.get('id'))
+                span.set_attribute("gdrive.file_name", uploaded.get('name'))
+            except HttpError as e:
+                span.record_exception(BackupException("Error while uploading (%s): %s" % (self.id, str(e))))
+                raise BackupException("Error while uploading (%s): %s" % (self.id, str(e)))
+            return "gdrive://%s/%s" % (self.folder_id, uploaded.get('name'))
+
+    def cleanup(self, id, name):
+        with self._get_tracer().start_as_current_span("gdrive_cleanup") as span:
+            span.set_attribute("destination.id", self.id)
+            span.set_attribute("gdrive.folder_id", self.folder_id)
+            span.set_attribute("source.id", id)
+            span.set_attribute("retention.copies", self.retention_copies)
+            span.set_attribute("retention.days", self.retention_days)
+            
+            logging.info("Clearing down older '%s' backups for '%s' from Google Drive..." % (name, self.id))
+            try:
+                service = self._get_service()
+                query = "name='%s' and '%s' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false" % (id, self.folder_id)
+                results = service.files().list(q=query, fields="files(id)").execute()
+                source_folders = results.get('files', [])
+                if not source_folders:
+                    span.set_attribute("source_folder.count", 0)
+                    return []
+                source_folder_id = source_folders[0]['id']
+
+                query = "'%s' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false" % source_folder_id
+                results = service.files().list(q=query, fields="files(id,name,createdTime)", orderBy="createdTime").execute()
+                candidates = [(f['createdTime'], f['id'], f['name']) for f in results.get('files', [])]
+                candidates.sort()
+                span.set_attribute("object.count", len(candidates))
+            except HttpError as e:
+                span.record_exception(BackupException("Error listing Google Drive folder (%s): %s" % (self.id, str(e))))
+                raise BackupException("Error listing Google Drive folder (%s): %s" % (self.id, str(e)))
+
+            removable = []
+            retained = []
+            if self.retention_copies > 0:
+                items = candidates
+                if len(items) > self.retention_copies:
+                    removable = items[0:(len(items) - self.retention_copies)]
+                retained = [n for d, fid, n in items[(len(items) - self.retention_copies):]]
+            if self.retention_days > 0:
+                now = datetime.datetime.now(datetime.timezone.utc)
+                for created, fid, fname in candidates:
+                    from dateutil import parser as dtparser
+                    age = (now - dtparser.parse(created)).days
+                    if age > self.retention_days:
+                        removable.append((created, fid, fname))
+            for created, fid, fname in removable:
+                logging.info("Removing '%s'..." % fname)
+                try:
+                    service.files().delete(fileId=fid).execute()
+                except HttpError as e:
+                    logging.warning("Failed to remove '%s': %s" % (fname, str(e)))
+            span.set_attribute("removed.count", len(removable))
+            span.set_attribute("retained.count", len(retained))
+
+            return retained

backups/destinations/gs.py

@@ -0,0 +1,74 @@
+import os, os.path
+import subprocess
+import logging
+import datetime
+
+from google.cloud import storage
+from google.api_core.exceptions import BadRequest
+
+from dateutil import parser, tz
+
+from backups.exceptions import BackupException
+from backups.destinations import backupdestination
+from backups.destinations.destination import BackupDestination
+
+@backupdestination('gs')
+class GS(BackupDestination):
+    def __init__(self, config):
+        BackupDestination.__init__(self, config)
+        # TODO: Finish this off...
+        self.gcs_creds_path = config['gcs_creds_path']
+        self.bucket = config['bucket']
+
+    def send(self, id, name, filename):
+        gslocation = "gs://%s/%s/%s/%s" % (
+            self.bucket,
+            id,
+            self.runtime.strftime("%Y%m%d%H%M%S"),
+            os.path.basename(filename))
+        logging.info("Uploading '%s' backup for '%s' to GS (%s)..." % (name, self.id, gslocation))
+        uploadargs = ['gsutil', '-q', 'cp', filename, gslocation]
+        uploadenv = os.environ.copy()
+        uploadproc = subprocess.Popen(uploadargs, stderr=subprocess.PIPE, env=uploadenv)
+        uploadproc.wait()
+        exitcode = uploadproc.returncode
+        errmsg = uploadproc.stderr.read()
+        if exitcode != 0:
+            raise BackupException("Error while uploading (%s): %s" % (self.id, errmsg))
+
+        return gslocation
+
+    def cleanup(self, id, name):
+        gslocation = "gs://%s/%s" % (self.bucket, id)
+        logging.info("Clearing down older '%s' backups for '%s' from GS (%s)..." % (name, self.id, gslocation))
+
+        client = storage.Client.from_service_account_json(self.gcs_creds_path)
+        bucket = client.get_bucket(self.bucket)
+
+        # Gather list of potentials first
+        candidates = []
+        for key in bucket.list(prefix="%s/" % id):
+            parsed_date = parser.parse(key.last_modified)
+            candidates.append([parsed_date, key.name])
+        candidates.sort()
+
+        # Loop and purge unretainable copies
+        removable_names = []
+        retained_copies = []
+        if self.retention_copies > 0:
+            names = [name for d, name in candidates]
+            if len(names) > self.retention_copies:
+                removable_names = names[0:(len(names) - self.retention_copies)]
+            else:
+                retained_copies = names[(len(names) - self.retention_copies):]
+        if self.retention_days > 0:
+            for d, name in candidates:
+                days = (datetime.datetime.now(tz.tzutc()) - d).days
+                if days > self.retention_days:
+                    removable_names.append(name)
+        for name in removable_names:
+            logging.info("Removing '%s'..." % name)
+            bucket.get_key(name).delete()
+
+        # Return list of retained copies
+        return retained_copies

backups/destinations/local.py

@@ -0,0 +1,55 @@
+import os, os.path
+import shutil
+import datetime
+import logging
+
+from backups.exceptions import BackupException
+from backups.destinations import backupdestination
+from backups.destinations.destination import BackupDestination
+
+@backupdestination('local')
+class Local(BackupDestination):
+    def __init__(self, config):
+        BackupDestination.__init__(self, config)
+        self.path = config['path']
+
+    def send(self, id, name, filename):
+        destdir = os.path.join(self.path, id, self.runtime.strftime("%Y%m%d%H%M%S"))
+        os.makedirs(destdir, exist_ok=True)
+        destfile = os.path.join(destdir, os.path.basename(filename))
+        logging.info("Copying '%s' backup for '%s' to local path (%s)..." % (name, self.id, destfile))
+        try:
+            shutil.copy2(filename, destfile)
+        except Exception as e:
+            raise BackupException("Error while copying (%s): %s" % (self.id, str(e)))
+        return destfile
+
+    def cleanup(self, id, name):
+        basedir = os.path.join(self.path, id)
+        if not os.path.isdir(basedir):
+            return []
+
+        candidates = []
+        for entry in os.scandir(basedir):
+            if entry.is_dir():
+                candidates.append([entry.stat().st_mtime, entry.path])
+        candidates.sort()
+
+        removable = []
+        retained = []
+        if self.retention_copies > 0:
+            paths = [p for d, p in candidates]
+            if len(paths) > self.retention_copies:
+                removable = paths[0:(len(paths) - self.retention_copies)]
+            retained = paths[(len(paths) - self.retention_copies):]
+        if self.retention_days > 0:
+            now = datetime.datetime.now()
+            for d, path in candidates:
+                age = (now - datetime.datetime.fromtimestamp(d)).days
+                if age > self.retention_days:
+                    removable.append(path)
+        for path in removable:
+            logging.info("Removing '%s'..." % path)
+            shutil.rmtree(path)
+
+        return retained

backups/destinations/minio.py

@@ -0,0 +1,107 @@
+import os, os.path
+import datetime
+import logging
+import contextlib
+
+from minio import Minio
+from minio.error import S3Error
+
+from backups.exceptions import BackupException
+from backups.destinations import backupdestination
+from backups.destinations.destination import BackupDestination
+
+@backupdestination('minio')
+class MinioDestination(BackupDestination):
+    def __init__(self, config):
+        BackupDestination.__init__(self, config)
+        self.bucket = config['bucket']
+        self.endpoint = config['endpoint']
+        self.access_key = config['credentials']['access_key']
+        self.secret_key = config['credentials']['secret_key']
+        self.secure = bool(config.get('secure', True))
+
+    def _get_tracer(self):
+        if hasattr(self, 'tracer') and self.tracer is not None:
+            return self.tracer
+        class _NoOpTracer:
+            @contextlib.contextmanager
+            def start_as_current_span(self, name, **kwargs):
+                class _NoOpSpan:
+                    def set_attribute(self, *a, **k): pass
+                    def record_exception(self, *a, **k): pass
+                    def set_status(self, *a, **k): pass
+                yield _NoOpSpan()
+        return _NoOpTracer()
+
+    def _get_client(self):
+        return Minio(
+            self.endpoint,
+            access_key=self.access_key,
+            secret_key=self.secret_key,
+            secure=self.secure
+        )
+
+    def send(self, id, name, filename):
+        with self._get_tracer().start_as_current_span("minio_upload") as span:
+            span.set_attribute("destination.id", self.id)
+            span.set_attribute("minio.endpoint", self.endpoint)
+            span.set_attribute("minio.bucket", self.bucket)
+            span.set_attribute("source.id", id)
+            span.set_attribute("upload.file", filename)
+            
+            object_name = "%s/%s/%s" % (id, self.runtime.strftime("%Y%m%d%H%M%S"), os.path.basename(filename))
+            location = "%s/%s/%s" % (self.endpoint, self.bucket, object_name)
+            logging.info("Uploading '%s' backup for '%s' to Minio (%s)..." % (name, self.id, location))
+            try:
+                client = self._get_client()
+                client.fput_object(self.bucket, object_name, filename)
+                span.set_attribute("minio.object", object_name)
+            except S3Error as e:
+                span.record_exception(BackupException("Error while uploading (%s): %s" % (self.id, str(e))))
+                raise BackupException("Error while uploading (%s): %s" % (self.id, str(e)))
+            return location
+
+    def cleanup(self, id, name):
+        with self._get_tracer().start_as_current_span("minio_cleanup") as span:
+            span.set_attribute("destination.id", self.id)
+            span.set_attribute("minio.endpoint", self.endpoint)
+            span.set_attribute("minio.bucket", self.bucket)
+            span.set_attribute("source.id", id)
+            span.set_attribute("retention.copies", self.retention_copies)
+            span.set_attribute("retention.days", self.retention_days)
+            
+            logging.info("Clearing down older '%s' backups for '%s' from Minio..." % (name, self.id))
+            try:
+                client = self._get_client()
+                objects = list(client.list_objects(self.bucket, prefix="%s/" % id, recursive=True))
+            except S3Error as e:
+                span.record_exception(BackupException("Error listing Minio bucket (%s): %s" % (self.id, str(e))))
+                raise BackupException("Error listing Minio bucket (%s): %s" % (self.id, str(e)))
+
+            candidates = [[obj.last_modified, obj.object_name] for obj in objects]
+            candidates.sort()
+            span.set_attribute("object.count", len(candidates))
+
+            removable = []
+            retained = []
+            if self.retention_copies > 0:
+                names = [n for d, n in candidates]
+                if len(names) > self.retention_copies:
+                    removable = names[0:(len(names) - self.retention_copies)]
+                retained = names[(len(names) - self.retention_copies):]
+            if self.retention_days > 0:
+                now = datetime.datetime.now(datetime.timezone.utc)
+                for d, n in candidates:
+                    age = (now - d).days
+                    if age > self.retention_days:
+                        removable.append(n)
+            for object_name in removable:
+                logging.info("Removing '%s'..." % object_name)
+                try:
+                    client.remove_object(self.bucket, object_name)
+                except S3Error as e:
+                    logging.warning("Failed to remove '%s': %s" % (object_name, str(e)))
+            span.set_attribute("removed.count", len(removable))
+            span.set_attribute("retained.count", len(retained))
+
+            return retained