bup 0.11.9__tar.gz

bup-0.11.9/LICENSE

@@ -0,0 +1,15 @@
+bup - AWS and github local backup
+Copyright (C) 2020-2026 James Abel
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

bup-0.11.9/LICENSE.txt

@@ -0,0 +1,15 @@
+bup - AWS and github local backup
+Copyright (C) 2020-2026 James Abel
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

bup-0.11.9/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.4
+Name: bup
+Version: 0.11.9
+Summary: backup all your AWS S3 buckets and DynamoDB tables, and github repos to a backup directory/folder
+Author-email: abel <j@abel.co>
+License-Expression: GPL-3.0-only
+Project-URL: Homepage, https://github.com/jamesabel/bup
+Project-URL: Download, https://github.com/jamesabel/bup
+Keywords: aws,dynamodb,s3,github,backup
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: LICENSE.txt
+Requires-Dist: ismain
+Requires-Dist: balsa
+Requires-Dist: boto3
+Requires-Dist: typeguard
+Requires-Dist: hashy
+Requires-Dist: dictim
+Requires-Dist: awsimple
+Requires-Dist: pressenter2exit
+Requires-Dist: awscli
+Requires-Dist: GitPython
+Requires-Dist: github3.py
+Requires-Dist: sqlitedict
+Requires-Dist: tobool
+Requires-Dist: attrs
+Requires-Dist: appdirs
+Requires-Dist: pref
+Requires-Dist: pyqt5
+Requires-Dist: python-dotenv
+Requires-Dist: sentry-sdk
+Dynamic: license-file
+
+# bup
+
+backup for github repos and AWS S3 and DynamoDB
+
+# Acknowledgements 
+
+<div>Icons made by <a href="https://www.freepik.com" title="Freepik">Freepik</a> from <a href="https://www.flaticon.com/" title="Flaticon">www.flaticon.com</a></div>

bup-0.11.9/README.md

@@ -0,0 +1,7 @@
+# bup
+
+backup for github repos and AWS S3 and DynamoDB
+
+# Acknowledgements 
+
+<div>Icons made by <a href="https://www.freepik.com" title="Freepik">Freepik</a> from <a href="https://www.flaticon.com/" title="Flaticon">www.flaticon.com</a></div>

bup-0.11.9/bup/LICENSE

@@ -0,0 +1,15 @@
+bup - AWS and github local backup
+Copyright (C) 2020-2026 James Abel
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

bup-0.11.9/bup/__init__.py

@@ -0,0 +1,11 @@
+from .__version__ import __application_name__, __version__, __description__, __author__, __url__, __download_url__, __author_url__
+from .robust_os_calls import mkdirs, rmdir
+from .ui_types import UITypes
+from .print_log import print_log
+from .arguments import arguments
+from .preferences import BupPreferences, get_preferences, ExclusionPreferences
+from .backup_types import BackupTypes
+from .bup_base import BupBase
+from .s3_backup import S3Backup
+from .dynamodb_backup import DynamoDBBackup
+from .github_backup import GithubBackup

bup-0.11.9/bup/__main__.py

@@ -0,0 +1,18 @@
+from ismain import is_main
+
+from bup import arguments
+from bup.cli import cli_main
+from bup.gui import gui_main
+
+
+def main():
+    args = arguments()
+    if args is None:
+        gui_main()
+    else:
+        # if we've not been given anything to do on the command line, then it must be CLI
+        cli_main(args)
+
+
+if is_main():
+    main()

bup-0.11.9/bup/__version__.py

@@ -0,0 +1,9 @@
+__version__ = "0.11.9"
+__application_name__ = "bup"
+__title__ = __application_name__
+__author__ = "abel"
+__author_email__ = "j@abel.co"
+__author_url__ = "https://www.abel.co"
+__url__ = "https://github.com/jamesabel/bup"
+__download_url__ = __url__
+__description__ = "backup all your AWS S3 buckets and DynamoDB tables, and github repos to a backup directory/folder"

bup-0.11.9/bup/arguments.py

@@ -0,0 +1,53 @@
+import argparse
+from pathlib import Path
+import sys
+
+from balsa import verbose_arg_string, log_dir_arg_string, delete_existing_arg_string
+
+from bup import __application_name__, __version__, __description__
+
+
+def arguments():
+
+    version_string = "version"
+
+    if len(sys.argv) > 1 and sys.argv[1].lower() == f"--{version_string}":
+        print(__version__)
+        sys.exit()
+
+    if len(sys.argv) > 1:
+
+        parser = argparse.ArgumentParser(
+            prog=__application_name__,
+            description=__description__,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            epilog=f"{__application_name__} V{__version__}, see github.com/jamesabel/bup for LICENSE",
+        )
+        parser.add_argument("path", help="directory to back up to")
+        parser.add_argument("-s", "--s3", action="store_true", default=False, help="backup AWS S3")
+        parser.add_argument("-d", "--dynamodb", action="store_true", default=False, help="backup AWS DynamoDB")
+        parser.add_argument("-a", "--aws", action="store_true", default=False, help="backup both AWS S3 and DynamoDB")
+        parser.add_argument("-g", "--github", action="store_true", default=False, help="backup github")
+        parser.add_argument(
+            "-e",
+            "--exclude",
+            nargs="*",
+            help="exclude these AWS S3 buckets and/or tables (only do one backup type at a time when setting exclusions - these values are saved and used on subsequent runs)",
+        )
+        parser.add_argument("-p", "--profile", help="AWS profile (uses the default AWS profile if not given)")
+        parser.add_argument("-r", "--region", help="AWS region (uses the default AWS region if not given)")
+        parser.add_argument("-t", "--token", help="github token (saved and used on subsequent runs)")
+        parser.add_argument("--dry_run", action="store_true", default=False, help="displays operations that would be performed using the specified command without actually running them")
+        parser.add_argument(f"--{version_string}", action="store_true", default=False, help="display version and exit")
+        parser.add_argument("-v", f"--{verbose_arg_string}", dest=verbose_arg_string, action="store_true", default=False, help="set verbose")
+        parser.add_argument("-l", f"--{log_dir_arg_string}", dest=log_dir_arg_string, help="log dir")
+        parser.add_argument(f"--{delete_existing_arg_string}", dest=delete_existing_arg_string, help="delete existing logs")
+        args = parser.parse_args()
+
+        if args.logdir is None and args.path is not None:
+            # put the logs in the backup dir unless explicitly given
+            args.logdir = Path(args.path, "log")
+    else:
+        args = None
+
+    return args

bup-0.11.9/bup/backup_types.py

@@ -0,0 +1,7 @@
+from enum import Enum
+
+
+class BackupTypes(Enum):
+    S3 = 0
+    DynamoDB = 1
+    github = 2

bup-0.11.9/bup/bup_base.py

@@ -0,0 +1,64 @@
+from typing import Callable
+
+from PyQt5.QtCore import QThread, pyqtSignal
+from typeguard import typechecked
+
+from balsa import get_logger
+
+from bup import __application_name__, UITypes
+
+log = get_logger(__application_name__)
+
+
+class BupBase(QThread):
+
+    backup_type = None
+    info_out_signal = pyqtSignal(str)
+    warning_out_signal = pyqtSignal(str)
+    error_out_signal = pyqtSignal(str)
+
+    @typechecked()
+    def __init__(self, ui_type: UITypes, info_out: Callable, warning_out: Callable, error_out: Callable):
+        super().__init__()
+        self._stop_requested = False
+        self.ui_type = ui_type
+        self.caller_info_out = info_out
+        self.caller_warning_out = warning_out
+        self.caller_error_out = error_out
+        self.info_out_signal.connect(self._info_out)
+        self.warning_out_signal.connect(self._warning_out)
+        self.error_out_signal.connect(self._error_out)
+
+    def request_stop(self):
+        self._stop_requested = True
+
+    @property
+    def stop_requested(self) -> bool:
+        return self._stop_requested
+
+    def info_out(self, s: str):
+        # callees could call emit(), but that seems awkward so provide this method
+        self.info_out_signal.emit(s)
+
+    def _info_out(self, s: str):
+        # hooked up to the signal for threading
+        log.info(s)
+        self.caller_info_out(s)
+
+    def warning_out(self, s: str):
+        # callees could call emit(), but that seems awkward so provide this method
+        self.warning_out_signal.emit(s)
+
+    def _warning_out(self, s: str):
+        # hooked up to the signal for threading
+        log.warning(s)
+        self.caller_warning_out(s)
+
+    def error_out(self, s: str):
+        # callees could call emit(), but that seems awkward so provide this method
+        self.error_out_signal.emit(s)
+
+    def _error_out(self, s: str):
+        # hooked up to the signal for threading
+        log.error(s)
+        self.caller_error_out(s)

bup-0.11.9/bup/cli/__init__.py

@@ -0,0 +1 @@
+from .cli_main import cli_main

bup-0.11.9/bup/cli/cli_main.py

@@ -0,0 +1,61 @@
+from balsa import Balsa, get_logger
+
+from bup import __application_name__, __author__, __version__, S3Backup, DynamoDBBackup, GithubBackup, get_preferences, UITypes, ExclusionPreferences, BackupTypes
+
+log = get_logger(__application_name__)
+
+
+def cli_main(args):
+
+    ui_type = UITypes.cli
+
+    balsa = Balsa(__application_name__, __author__)
+    balsa.log_console_prefix = "\r"
+    balsa.init_logger_from_args(args)
+    log.info(f"__application_name__={__application_name__}")
+    log.info(f"__author__={__author__}")
+    log.info(f"__version__={__version__}")
+
+    try:
+        preferences = get_preferences(ui_type)
+        preferences.backup_directory = args.path  # backup classes will read the preferences DB directly
+        preferences.github_token = args.token
+        preferences.aws_profile = args.profile
+        preferences.dry_run = args.dry_run
+
+        # If setting the exclusions, just do it for one backup type at a time.  The values are stored for subsequent runs.
+        if args.exclude is not None and len(args.exclude) > 0:
+            if args.s3:
+                ExclusionPreferences(BackupTypes.S3.name).set(args.exclude)
+            elif args.dynamodb:
+                ExclusionPreferences(BackupTypes.DynamoDB.name).set(args.exclude)
+            elif args.github:
+                ExclusionPreferences(BackupTypes.github.name).set(args.exclude)
+
+        did_something = False
+        dynamodb_local_backup = None
+        s3_local_backup = None
+        github_local_backup = None
+        if args.s3 or args.aws:
+            s3_local_backup = S3Backup(ui_type, log.info, log.warning, log.error)
+            s3_local_backup.start()
+            did_something = True
+        if args.dynamodb or args.aws:
+            dynamodb_local_backup = DynamoDBBackup(ui_type, log.info, log.warning, log.error)
+            dynamodb_local_backup.start()
+            did_something = True
+        if args.github:
+            github_local_backup = GithubBackup(ui_type, log.info, log.warning, log.error)
+            github_local_backup.start()
+            did_something = True
+        if not did_something:
+            print("nothing to do - please specify a backup to do or -h/--help for help")
+
+        if dynamodb_local_backup is not None:
+            dynamodb_local_backup.join()
+        if s3_local_backup is not None:
+            s3_local_backup.join()
+        if github_local_backup is not None:
+            github_local_backup.join()
+    except Exception as e:
+        log.exception(e)

bup-0.11.9/bup/dynamodb_backup.py

@@ -0,0 +1,74 @@
+import logging
+import pickle
+from datetime import timedelta
+from pathlib import Path
+
+from botocore.exceptions import ClientError
+from awsimple import DynamoDBAccess, dynamodb_to_json
+from balsa import get_logger
+
+# boto3/botocore log responses at DEBUG level; the repr() of large responses can contain
+# characters that cp1252 (Windows console default) cannot encode, causing noisy logging errors.
+logging.getLogger("boto3").setLevel(logging.WARNING)
+logging.getLogger("botocore").setLevel(logging.WARNING)
+
+from bup import BupBase, BackupTypes, get_preferences, ExclusionPreferences, __application_name__
+
+log = get_logger(__application_name__)
+
+
+class DynamoDBBackup(BupBase):
+
+    backup_type = BackupTypes.DynamoDB
+
+    def run(self):
+        preferences = get_preferences(self.ui_type)
+        backup_directory = preferences.backup_directory
+        dry_run = preferences.dry_run
+        exclusions = ExclusionPreferences(self.backup_type.name).get_no_comments()
+
+        dynamodb_access = DynamoDBAccess(
+            profile_name=preferences.aws_profile or None,
+            aws_access_key_id=preferences.aws_access_key_id or None,
+            aws_secret_access_key=preferences.aws_secret_access_key or None,
+            region_name=preferences.aws_region or None,
+        )
+        try:
+            tables = dynamodb_access.get_table_names()
+        except ClientError as e:
+            log.warning(e)
+            tables = []
+        self.info_out(f"found {len(tables)} DynamoDB tables")
+        count = 0
+        for table_name in tables:
+            if self.stop_requested:
+                break
+
+            # awsimple will update immediately if number of table rows changes, but backup from scratch every so often to be safe
+            cache_life = timedelta(days=1).total_seconds()
+
+            if table_name in exclusions:
+                self.info_out(f"excluding {table_name}")
+            elif dry_run:
+                self.info_out(f"dry run {table_name}")
+            else:
+                self.info_out(f"{table_name}")
+                table = DynamoDBAccess(
+                    table_name,
+                    profile_name=preferences.aws_profile or None,
+                    aws_access_key_id=preferences.aws_access_key_id or None,
+                    aws_secret_access_key=preferences.aws_secret_access_key or None,
+                    region_name=preferences.aws_region or None,
+                    cache_life=cache_life,
+                )
+                table_contents = table.scan_table_cached()
+
+                dir_path = Path(backup_directory, "dynamodb")
+                dir_path.mkdir(parents=True, exist_ok=True)
+                with Path(dir_path, f"{table_name}.pickle").open("wb") as f:
+                    pickle.dump(table_contents, f)
+                with Path(dir_path, f"{table_name}.json").open("w", encoding="utf-8") as f:
+                    f.write(dynamodb_to_json(table_contents, indent=4))
+                count += 1
+
+        self.info_out(f"{len(tables)} tables, {count} backed up, {len(exclusions)} excluded")

bup-0.11.9/bup/github_backup.py

@@ -0,0 +1,146 @@
+from pathlib import Path
+import shutil
+import time
+
+import github3
+from github3.exceptions import AuthenticationFailed
+from balsa import get_logger
+from typeguard import typechecked
+
+from bup import __application_name__, BupBase, BackupTypes, get_preferences, ExclusionPreferences, rmdir
+
+log = get_logger(__application_name__)
+
+
+class GithubBackup(BupBase):
+
+    backup_type = BackupTypes.github
+
+    def run(self):
+        if shutil.which("git") is None:
+            self.error_out("git executable not found in PATH - GitHub backup cannot run")
+            return
+
+        from git import Repo
+        from git.exc import GitCommandError
+
+        preferences = get_preferences(self.ui_type)
+        dry_run = preferences.dry_run
+        exclusions = ExclusionPreferences(BackupTypes.github.name).get_no_comments()
+
+        backup_dir = Path(preferences.backup_directory, "github")
+
+        try:
+            gh = github3.login(token=preferences.github_token)
+            if gh is None:
+                log.warning("could not login to github")
+                repositories = []
+            else:
+                repositories = list(gh.repositories())  # this actually throws the authentication error
+        except AuthenticationFailed:
+            log.error("github authentication failed")
+            return
+
+        clone_count = 0
+        pull_count = 0
+
+        for github_repo in repositories:
+            if self.stop_requested:
+                break
+
+            repo_owner_and_name = str(github_repo)
+            repo_name = repo_owner_and_name.split("/")[-1]
+            if any([e == repo_name for e in exclusions]):
+                self.info_out(f"{repo_owner_and_name} excluded")
+            elif dry_run:
+                self.info_out(f"dry run {repo_owner_and_name}")
+            else:
+                repo_dir = Path(backup_dir, repo_owner_and_name).absolute()
+                branches = list(github_repo.branches())
+
+                # if we've cloned previously, just do a pull
+                pull_success = False
+                if repo_dir.exists():
+                    try:
+                        if pull_success := self.pull_branches(repo_owner_and_name, branches, repo_dir):
+                            pull_count += 1
+                    except GitCommandError as e:
+                        self.warning_out(f'could not pull "{repo_dir}" - will try to start over and do a clone of "{repo_owner_and_name},{e}","{__file__}"')
+
+                # new to us - clone the repo
+                if not pull_success:
+                    try:
+                        if repo_dir.exists():
+                            rmdir(repo_dir)
+
+                        if repo_dir.exists():
+                            self.error_out(f'could not remove "{repo_dir}" - may require manual removal')
+                        else:
+                            self.info_out(f'git clone "{repo_owner_and_name}"')
+                            clone_url = github_repo.clone_url
+                            if preferences.github_token:
+                                clone_url = clone_url.replace("https://", f"https://{preferences.github_token}@")
+                            Repo.clone_from(clone_url, repo_dir)
+                            time.sleep(1.0)
+                            self.pull_branches(repo_owner_and_name, branches, repo_dir)
+                            clone_count += 1
+                    except PermissionError as e:
+                        self.warning_out(f'{repo_owner_and_name},"{repo_dir}",{e},"{__file__}"')
+                    except GitCommandError as e:
+                        self.warning_out(f'{repo_owner_and_name},"{repo_dir}",{e},"{__file__}"')
+
+        self.info_out(f"{len(repositories)} repos, {pull_count} pulls, {clone_count} clones, {len(exclusions)} excluded")
+
+    @typechecked()
+    def pull_branches(self, repo_name: str, branches: list, repo_dir: Path) -> bool:
+        from git import Repo
+        from git.exc import GitCommandError, InvalidGitRepositoryError
+
+        try:
+            git_repo = Repo(repo_dir)
+        except InvalidGitRepositoryError as e:
+            self.error_out(f'InvalidGitRepositoryError: {repo_name},"{repo_dir}",{e},"{__file__}"')
+            git_repo = None
+
+        success = False
+        main_branch = None
+        if git_repo is not None:
+            for branch in branches:
+                if self.stop_requested:
+                    break
+                branch_name = branch.name
+
+                # prefer "main" over "master"
+                if branch_name.lower() == "main" or (main_branch is None and branch_name.lower() == "master"):
+                    main_branch = branch
+
+                self.info_out(f'git pull "{repo_name}" branch:"{branch_name}"')
+                try:
+                    git_repo.git.reset("--hard")
+                    git_repo.git.clean("-fd")
+                    git_repo.git.checkout(branch_name)
+                    git_repo.git.pull()
+                    success = True
+                except GitCommandError as e:
+                    if "did not match any file".lower() in str(e).lower():
+                        # new branch with no files yet - skip it and continue with other branches
+                        self.info_out(f"git pull {repo_name} branch:{branch_name} - no files")
+                        continue
+                    elif "would be overwritten by checkout" in str(e).lower():
+                        # typically caused by CRLF line-ending normalization on Windows, not actual user edits
+                        self.warning_out(f"{repo_name} branch:{branch_name} : checkout blocked by apparent local changes (likely line-ending normalization) : {e}")
+                        success = False
+                        break
+                    else:
+                        self.error_out(f"{repo_name} : {e}")
+                        success = False
+                        break
+
+            # if more than one branch, switch to main (or master) branch upon exit
+            if len(branches) > 1 and main_branch is not None:
+                main_branch_name = main_branch.name
+                self.info_out(f'git switch "{repo_name}" branch:"{main_branch_name}"')
+                git_repo.git.reset("--hard")  # clear any phantom changes (e.g. CRLF normalization) before switching
+                git_repo.git.switch(main_branch_name)
+
+        return success