bup 0.12.0__tar.gz → 0.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bup-0.12.0/bup.egg-info → bup-0.13.0}/PKG-INFO +12 -4
- {bup-0.12.0 → bup-0.13.0}/bup/__version__.py +1 -1
- {bup-0.12.0 → bup-0.13.0}/bup/arguments.py +1 -0
- bup-0.13.0/bup/aws_cli.py +121 -0
- {bup-0.12.0 → bup-0.13.0}/bup/bup_base.py +20 -3
- {bup-0.12.0 → bup-0.13.0}/bup/cli/cli_main.py +3 -0
- {bup-0.12.0 → bup-0.13.0}/bup/dynamodb_backup.py +1 -1
- {bup-0.12.0 → bup-0.13.0}/bup/github_backup.py +1 -1
- {bup-0.12.0 → bup-0.13.0}/bup/gui/__init__.py +1 -0
- {bup-0.12.0 → bup-0.13.0}/bup/gui/about.py +14 -0
- {bup-0.12.0 → bup-0.13.0}/bup/gui/bup_dialog.py +5 -1
- {bup-0.12.0 → bup-0.13.0}/bup/gui/gui_main.py +4 -0
- bup-0.13.0/bup/gui/log_widget.py +118 -0
- {bup-0.12.0 → bup-0.13.0}/bup/gui/preferences_widget.py +57 -0
- {bup-0.12.0 → bup-0.13.0}/bup/gui/run_backup_widget.py +2 -0
- bup-0.13.0/bup/log_routing.py +124 -0
- {bup-0.12.0 → bup-0.13.0}/bup/preferences.py +9 -0
- {bup-0.12.0 → bup-0.13.0}/bup/s3_backup.py +36 -38
- {bup-0.12.0 → bup-0.13.0/bup.egg-info}/PKG-INFO +12 -4
- {bup-0.12.0 → bup-0.13.0}/bup.egg-info/SOURCES.txt +3 -0
- {bup-0.12.0 → bup-0.13.0}/bup.egg-info/requires.txt +12 -3
- {bup-0.12.0 → bup-0.13.0}/pyproject.toml +14 -6
- {bup-0.12.0 → bup-0.13.0}/LICENSE +0 -0
- {bup-0.12.0 → bup-0.13.0}/LICENSE.txt +0 -0
- {bup-0.12.0 → bup-0.13.0}/README.md +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup/LICENSE +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup/__init__.py +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup/__main__.py +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup/backup_types.py +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup/bup.ico +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup/bup.png +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup/cli/__init__.py +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup/gpl-3.0.md +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup/gui/gui_icon.py +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup/robust_os_calls.py +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup/ui_types.py +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup.egg-info/dependency_links.txt +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup.egg-info/entry_points.txt +0 -0
- {bup-0.12.0 → bup-0.13.0}/bup.egg-info/top_level.txt +0 -0
- {bup-0.12.0 → bup-0.13.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0
|
|
4
4
|
Summary: backup all your AWS S3 buckets and DynamoDB tables, and github repos to a backup directory/folder
|
|
5
5
|
Author-email: abel <j@abel.co>
|
|
6
6
|
License-Expression: GPL-3.0-only
|
|
@@ -15,10 +15,7 @@ Requires-Dist: ismain
|
|
|
15
15
|
Requires-Dist: balsa
|
|
16
16
|
Requires-Dist: boto3
|
|
17
17
|
Requires-Dist: typeguard
|
|
18
|
-
Requires-Dist: hashy
|
|
19
|
-
Requires-Dist: dictim
|
|
20
18
|
Requires-Dist: awsimple
|
|
21
|
-
Requires-Dist: pressenter2exit
|
|
22
19
|
Requires-Dist: awscli
|
|
23
20
|
Requires-Dist: GitPython
|
|
24
21
|
Requires-Dist: github3.py
|
|
@@ -30,6 +27,17 @@ Requires-Dist: pref
|
|
|
30
27
|
Requires-Dist: pyqt5
|
|
31
28
|
Requires-Dist: python-dotenv
|
|
32
29
|
Requires-Dist: sentry-sdk
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: wheel; extra == "dev"
|
|
32
|
+
Requires-Dist: black; extra == "dev"
|
|
33
|
+
Requires-Dist: pytest; extra == "dev"
|
|
34
|
+
Requires-Dist: coverage; extra == "dev"
|
|
35
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
36
|
+
Requires-Dist: pyship; extra == "dev"
|
|
37
|
+
Requires-Dist: flake8; extra == "dev"
|
|
38
|
+
Requires-Dist: PyQt5-stubs; extra == "dev"
|
|
39
|
+
Requires-Dist: build; extra == "dev"
|
|
40
|
+
Requires-Dist: twine; extra == "dev"
|
|
33
41
|
Dynamic: license-file
|
|
34
42
|
|
|
35
43
|
# bup
|
|
@@ -38,6 +38,7 @@ def arguments():
|
|
|
38
38
|
parser.add_argument("-r", "--region", help="AWS region (uses the default AWS region if not given)")
|
|
39
39
|
parser.add_argument("-t", "--token", help="github token (saved and used on subsequent runs)")
|
|
40
40
|
parser.add_argument("--dry_run", action="store_true", default=False, help="displays operations that would be performed using the specified command without actually running them")
|
|
41
|
+
parser.add_argument("--size_only", action="store_true", default=False, help="AWS S3 sync compares file size only, not timestamps (aws s3 sync --size-only)")
|
|
41
42
|
parser.add_argument(f"--{version_string}", action="store_true", default=False, help="display version and exit")
|
|
42
43
|
parser.add_argument("-v", f"--{verbose_arg_string}", dest=verbose_arg_string, action="store_true", default=False, help="set verbose")
|
|
43
44
|
parser.add_argument("-l", f"--{log_dir_arg_string}", dest=log_dir_arg_string, help="log dir")
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
import shutil
|
|
5
|
+
import subprocess
|
|
6
|
+
import sys
|
|
7
|
+
import urllib.request
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Optional, Tuple
|
|
10
|
+
|
|
11
|
+
from balsa import get_logger
|
|
12
|
+
|
|
13
|
+
from bup import __application_name__
|
|
14
|
+
|
|
15
|
+
log = get_logger(__application_name__)
|
|
16
|
+
|
|
17
|
+
decoding = "utf-8"
|
|
18
|
+
|
|
19
|
+
awscli_pypi_url = "https://pypi.org/pypi/awscli/json"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def find_aws_cli() -> Tuple[Optional[Path], Optional[Path]]:
|
|
23
|
+
"""
|
|
24
|
+
Locate the AWS CLI executable and the python executable it should run with.
|
|
25
|
+
Use sys.executable to reliably locate python and aws CLI in the same directory,
|
|
26
|
+
which works for both local venv and installed app scenarios.
|
|
27
|
+
:return: (aws_cli_path, python_path), or (None, None) if not found
|
|
28
|
+
"""
|
|
29
|
+
python_exe = Path(sys.executable)
|
|
30
|
+
aws_names = ["aws.cmd", "aws.exe", "aws"] if sys.platform == "win32" else ["aws"]
|
|
31
|
+
aws_candidates = (
|
|
32
|
+
[(python_exe, python_exe.parent / name) for name in aws_names] # same dir as python (venv Scripts/)
|
|
33
|
+
+ [(python_exe, python_exe.parent / "Scripts" / name) for name in aws_names] # CLIP layout: python at root, scripts in Scripts/
|
|
34
|
+
+ [(Path("venv", "Scripts", "python.exe").absolute(), Path("venv", "Scripts", name).absolute()) for name in aws_names] # local venv from CWD
|
|
35
|
+
)
|
|
36
|
+
for p, a in aws_candidates:
|
|
37
|
+
if p.exists() and a.exists():
|
|
38
|
+
return a, p
|
|
39
|
+
|
|
40
|
+
# fall back to whatever is on the system PATH
|
|
41
|
+
aws_in_path = shutil.which("aws")
|
|
42
|
+
if aws_in_path:
|
|
43
|
+
return Path(aws_in_path), python_exe
|
|
44
|
+
|
|
45
|
+
log.error(f"AWS CLI executable not found ({aws_candidates=})")
|
|
46
|
+
return None, None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def make_aws_cli_env(python_path: Path) -> dict:
|
|
50
|
+
"""
|
|
51
|
+
Environment for running the AWS CLI. The AWS CLI needs the python executable to be in the PATH if it's not in the same dir, which happens
|
|
52
|
+
when this program is installed. Make the directory of our python.exe the first in the list so it's found and not any of the others that
|
|
53
|
+
may or may not be in the PATH.
|
|
54
|
+
"""
|
|
55
|
+
env_var = os.environ.copy()
|
|
56
|
+
env_var["PATH"] = f"{str(python_path.parent)}{os.pathsep}{env_var.get('PATH', '')}"
|
|
57
|
+
return env_var
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def get_aws_cli_version(aws_cli_path: Path, env_var: dict) -> Optional[str]:
|
|
61
|
+
"""
|
|
62
|
+
Version of the given AWS CLI executable (e.g. "1.42.35"), from "aws --version".
|
|
63
|
+
:return: the version string, or None if it could not be determined
|
|
64
|
+
"""
|
|
65
|
+
command_line = [str(aws_cli_path), "--version"]
|
|
66
|
+
try:
|
|
67
|
+
result = subprocess.run(command_line, env=env_var, capture_output=True, timeout=60.0)
|
|
68
|
+
except (OSError, subprocess.SubprocessError) as e:
|
|
69
|
+
log.warning(f'error executing {" ".join(command_line)} : {e}')
|
|
70
|
+
return None
|
|
71
|
+
# output looks like "aws-cli/1.42.35 Python/3.13.0 Windows/11 botocore/1.40.35" (older AWS CLI versions write it to stderr, newer to stdout)
|
|
72
|
+
output = f"{result.stdout.decode(decoding, errors='replace')} {result.stderr.decode(decoding, errors='replace')}"
|
|
73
|
+
match = re.search(r"aws-cli/(\S+)", output)
|
|
74
|
+
if match is None:
|
|
75
|
+
log.warning(f'could not parse AWS CLI version from "{output.strip()}"')
|
|
76
|
+
return None
|
|
77
|
+
return match.group(1)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def get_latest_awscli_version(timeout: float = 10.0) -> Optional[str]:
|
|
81
|
+
"""
|
|
82
|
+
Latest awscli version published on PyPI (the AWS CLI is pip-installed as the awscli package).
|
|
83
|
+
:return: the version string, or None if it could not be determined (e.g. offline)
|
|
84
|
+
"""
|
|
85
|
+
try:
|
|
86
|
+
with urllib.request.urlopen(awscli_pypi_url, timeout=timeout) as response:
|
|
87
|
+
package_info = json.load(response)
|
|
88
|
+
return package_info["info"]["version"]
|
|
89
|
+
except (OSError, ValueError, KeyError) as e:
|
|
90
|
+
# OSError covers URLError and timeouts, ValueError covers JSONDecodeError
|
|
91
|
+
log.info(f"could not get latest awscli version from {awscli_pypi_url} : {e}")
|
|
92
|
+
return None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def parse_version(version: str) -> Optional[Tuple[int, ...]]:
|
|
96
|
+
"""
|
|
97
|
+
Parse a dotted numeric version string (e.g. "1.42.35") into a comparable tuple of ints.
|
|
98
|
+
:return: the tuple, or None if the string is not a dotted numeric version
|
|
99
|
+
"""
|
|
100
|
+
try:
|
|
101
|
+
return tuple(int(part) for part in version.split("."))
|
|
102
|
+
except ValueError:
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def check_aws_cli_version(installed_version: Optional[str], latest_version: Optional[str]) -> Tuple[str, str]:
|
|
107
|
+
"""
|
|
108
|
+
Check that the installed AWS CLI is the latest available.
|
|
109
|
+
:return: (level, message) where level is "warning" or "info"
|
|
110
|
+
"""
|
|
111
|
+
if installed_version is None:
|
|
112
|
+
return "warning", "could not determine the installed AWS CLI version"
|
|
113
|
+
if latest_version is None:
|
|
114
|
+
return "warning", f"could not determine the latest AWS CLI version (installed: {installed_version})"
|
|
115
|
+
installed = parse_version(installed_version)
|
|
116
|
+
latest = parse_version(latest_version)
|
|
117
|
+
if installed is None or latest is None:
|
|
118
|
+
return "warning", f"could not compare AWS CLI versions (installed: {installed_version}, latest: {latest_version})"
|
|
119
|
+
if installed < latest:
|
|
120
|
+
return "warning", f'AWS CLI {installed_version} is not the latest ({latest_version}) - upgrade with "pip install --upgrade awscli"'
|
|
121
|
+
return "info", f"AWS CLI {installed_version} is up to date (latest: {latest_version})"
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import threading
|
|
1
2
|
from typing import Callable
|
|
2
3
|
|
|
3
4
|
from PyQt5.QtCore import QThread, pyqtSignal
|
|
@@ -36,6 +37,17 @@ class BupBase(QThread):
|
|
|
36
37
|
self.error_count = 0
|
|
37
38
|
super().start(*args, **kwargs)
|
|
38
39
|
|
|
40
|
+
def run(self):
|
|
41
|
+
# QThreads aren't registered with Python's threading module by name, so log records would show a meaningless
|
|
42
|
+
# thread name like "Dummy-25" - name the thread after this backup class (guard so a direct run() call in
|
|
43
|
+
# tests doesn't rename the main thread)
|
|
44
|
+
if threading.current_thread() is not threading.main_thread():
|
|
45
|
+
threading.current_thread().name = type(self).__name__
|
|
46
|
+
self.run_backup()
|
|
47
|
+
|
|
48
|
+
def run_backup(self):
|
|
49
|
+
raise NotImplementedError
|
|
50
|
+
|
|
39
51
|
def request_stop(self):
|
|
40
52
|
self._stop_requested = True
|
|
41
53
|
|
|
@@ -51,9 +63,14 @@ class BupBase(QThread):
|
|
|
51
63
|
else:
|
|
52
64
|
self.info_out_signal.emit(s)
|
|
53
65
|
|
|
66
|
+
def _log_extra(self) -> dict:
|
|
67
|
+
# stamp the record with this engine's backup type so the GUI log tab can route it to the right pane
|
|
68
|
+
# (the record's own source location is this file, not the backup module the message came from)
|
|
69
|
+
return {"backup_type": self.backup_type}
|
|
70
|
+
|
|
54
71
|
def _info_out(self, s: str):
|
|
55
72
|
# hooked up to the signal for threading
|
|
56
|
-
log.info(s)
|
|
73
|
+
log.info(s, extra=self._log_extra())
|
|
57
74
|
self.caller_info_out(s)
|
|
58
75
|
|
|
59
76
|
def warning_out(self, s: str):
|
|
@@ -65,7 +82,7 @@ class BupBase(QThread):
|
|
|
65
82
|
|
|
66
83
|
def _warning_out(self, s: str):
|
|
67
84
|
# hooked up to the signal for threading
|
|
68
|
-
log.warning(s)
|
|
85
|
+
log.warning(s, extra=self._log_extra())
|
|
69
86
|
self.caller_warning_out(s)
|
|
70
87
|
|
|
71
88
|
def error_out(self, s: str):
|
|
@@ -78,5 +95,5 @@ class BupBase(QThread):
|
|
|
78
95
|
def _error_out(self, s: str):
|
|
79
96
|
# hooked up to the signal for threading
|
|
80
97
|
self.error_count += 1
|
|
81
|
-
log.error(s)
|
|
98
|
+
log.error(s, extra=self._log_extra())
|
|
82
99
|
self.caller_error_out(s)
|
|
@@ -3,6 +3,7 @@ import sys
|
|
|
3
3
|
from balsa import Balsa, get_logger
|
|
4
4
|
|
|
5
5
|
from bup import __application_name__, __author__, __version__, S3Backup, DynamoDBBackup, GithubBackup, get_preferences, UITypes, ExclusionPreferences, BackupTypes
|
|
6
|
+
from bup.log_routing import set_detailed_file_logging
|
|
6
7
|
|
|
7
8
|
log = get_logger(__application_name__)
|
|
8
9
|
|
|
@@ -32,6 +33,7 @@ def cli_main(args):
|
|
|
32
33
|
|
|
33
34
|
preferences = get_preferences(ui_type)
|
|
34
35
|
preferences.backup_directory = args.path # backup classes will read the preferences DB directly
|
|
36
|
+
set_detailed_file_logging(preferences.detailed_log_directory, preferences.detailed_log_file_size_limit_mb) # one detailed log file per backup type, if configured
|
|
35
37
|
# only overwrite saved values when explicitly given on the command line
|
|
36
38
|
if args.token is not None:
|
|
37
39
|
preferences.github_token = args.token
|
|
@@ -40,6 +42,7 @@ def cli_main(args):
|
|
|
40
42
|
if args.region is not None:
|
|
41
43
|
preferences.aws_region = args.region
|
|
42
44
|
preferences.dry_run = args.dry_run
|
|
45
|
+
preferences.s3_size_only = args.size_only
|
|
43
46
|
|
|
44
47
|
# Set the exclusions for the selected backup type(s). The values are stored for subsequent runs.
|
|
45
48
|
# An explicitly empty -e (no values) clears the stored exclusions.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from .preferences_widget import PreferencesWidget, get_gui_preferences
|
|
2
2
|
from .about import BupAbout
|
|
3
3
|
from .gui_icon import get_icon_path
|
|
4
|
+
from .log_widget import LogWidget
|
|
4
5
|
from .run_backup_widget import RunBackupWidget
|
|
5
6
|
from .bup_dialog import BupDialog
|
|
6
7
|
from .gui_main import gui_main
|
|
@@ -4,10 +4,23 @@ from PyQt5.QtWidgets import QVBoxLayout, QWidget, QTextBrowser
|
|
|
4
4
|
from balsa import get_logger
|
|
5
5
|
|
|
6
6
|
from bup import __url__, __author_url__, __version__, __application_name__
|
|
7
|
+
from bup.aws_cli import find_aws_cli, make_aws_cli_env, get_aws_cli_version, get_latest_awscli_version
|
|
7
8
|
|
|
8
9
|
log = get_logger(__application_name__)
|
|
9
10
|
|
|
10
11
|
|
|
12
|
+
def get_aws_cli_version_text() -> str:
|
|
13
|
+
aws_cli_path, python_path = find_aws_cli()
|
|
14
|
+
if aws_cli_path is None:
|
|
15
|
+
installed_text = "not found"
|
|
16
|
+
else:
|
|
17
|
+
aws_cli_version = get_aws_cli_version(aws_cli_path, make_aws_cli_env(python_path))
|
|
18
|
+
installed_text = aws_cli_version if aws_cli_version is not None else "unknown"
|
|
19
|
+
latest_version = get_latest_awscli_version()
|
|
20
|
+
latest_text = latest_version if latest_version is not None else "unknown"
|
|
21
|
+
return f"{installed_text} (latest: {latest_text})"
|
|
22
|
+
|
|
23
|
+
|
|
11
24
|
class BupAbout(QWidget):
|
|
12
25
|
def __init__(self):
|
|
13
26
|
super().__init__()
|
|
@@ -28,6 +41,7 @@ class BupAbout(QWidget):
|
|
|
28
41
|
if license_path is not None and license_path.exists() and license_path.is_file():
|
|
29
42
|
|
|
30
43
|
about_text = f"### BUP ({__version__})\n\n"
|
|
44
|
+
about_text += f"AWS CLI version: {get_aws_cli_version_text()}\n\n"
|
|
31
45
|
about_text += license_path.read_text()
|
|
32
46
|
about_text += "\n---\n"
|
|
33
47
|
about_text += f"\nsource code: [{__url__}]({__url__})\n"
|
|
@@ -7,7 +7,7 @@ from PyQt5.QtGui import QCloseEvent, QIcon
|
|
|
7
7
|
from PyQt5.QtCore import QTimer, Qt
|
|
8
8
|
|
|
9
9
|
from bup import __application_name__, __version__, __author__, get_preferences, UITypes
|
|
10
|
-
from bup.gui import PreferencesWidget, RunBackupWidget, BupAbout, get_icon_path
|
|
10
|
+
from bup.gui import PreferencesWidget, RunBackupWidget, LogWidget, BupAbout, get_icon_path
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
class BupDialog(QDialog):
|
|
@@ -32,10 +32,12 @@ class BupDialog(QDialog):
|
|
|
32
32
|
|
|
33
33
|
self.run_backup_widget = RunBackupWidget()
|
|
34
34
|
self.preferences_widget = PreferencesWidget()
|
|
35
|
+
self.log_widget = LogWidget()
|
|
35
36
|
self.about_widget = BupAbout()
|
|
36
37
|
|
|
37
38
|
self.tab_widget.addTab(self.run_backup_widget, "Backup")
|
|
38
39
|
self.tab_widget.addTab(self.preferences_widget, "Preferences")
|
|
40
|
+
self.tab_widget.addTab(self.log_widget, "Log")
|
|
39
41
|
self.tab_widget.addTab(self.about_widget, "About")
|
|
40
42
|
|
|
41
43
|
preferences = get_preferences(UITypes.gui)
|
|
@@ -101,6 +103,8 @@ class BupDialog(QDialog):
|
|
|
101
103
|
self.autobackup_timer.stop()
|
|
102
104
|
self.run_backup_widget.stop()
|
|
103
105
|
self.run_backup_widget.wait_for_threads(5000)
|
|
106
|
+
self.log_widget.detach()
|
|
107
|
+
self.log_widget.save_state()
|
|
104
108
|
self.run_backup_widget.save_state()
|
|
105
109
|
preferences = get_preferences(UITypes.gui)
|
|
106
110
|
preferences.width = self.width()
|
|
@@ -7,6 +7,7 @@ from dotenv import load_dotenv, find_dotenv
|
|
|
7
7
|
from tobool import to_bool_strict
|
|
8
8
|
|
|
9
9
|
from bup import __application_name__, __author__
|
|
10
|
+
from bup.log_routing import set_detailed_file_logging
|
|
10
11
|
from bup.gui import BupDialog, get_gui_preferences
|
|
11
12
|
|
|
12
13
|
load_dotenv(find_dotenv())
|
|
@@ -25,6 +26,9 @@ def gui_main():
|
|
|
25
26
|
balsa.sentry_dsn = sentry_dsn
|
|
26
27
|
balsa.init_logger()
|
|
27
28
|
|
|
29
|
+
gui_preferences = get_gui_preferences()
|
|
30
|
+
set_detailed_file_logging(gui_preferences.detailed_log_directory, gui_preferences.detailed_log_file_size_limit_mb)
|
|
31
|
+
|
|
28
32
|
app = QApplication([])
|
|
29
33
|
bup_gui = BupDialog()
|
|
30
34
|
bup_gui.show()
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
from PyQt5.QtCore import QObject, pyqtSignal, Qt
|
|
4
|
+
from PyQt5.QtGui import QFontDatabase
|
|
5
|
+
from PyQt5.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QPlainTextEdit, QPushButton, QGroupBox, QSplitter
|
|
6
|
+
from balsa import get_logger
|
|
7
|
+
|
|
8
|
+
from bup import __application_name__
|
|
9
|
+
from bup.log_routing import LogSources, get_log_source, log_line_format
|
|
10
|
+
from bup.gui import get_gui_preferences
|
|
11
|
+
|
|
12
|
+
log = get_logger(__application_name__)
|
|
13
|
+
|
|
14
|
+
max_log_lines = 10000
|
|
15
|
+
|
|
16
|
+
minimum_pane_height = 50 # pixels
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class QtLogEmitter(QObject):
|
|
20
|
+
log_line_signal = pyqtSignal(str, str) # (LogSources value, formatted log line)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class QtLogHandler(logging.Handler):
|
|
24
|
+
"""
|
|
25
|
+
logging handler that forwards formatted log records to the GUI via a Qt signal
|
|
26
|
+
(backups log from worker QThreads, and Qt widgets may only be touched from the GUI thread - the signal connection handles the thread hop)
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(self):
|
|
30
|
+
super().__init__()
|
|
31
|
+
self.emitter = QtLogEmitter()
|
|
32
|
+
self.setFormatter(logging.Formatter(log_line_format))
|
|
33
|
+
|
|
34
|
+
def emit(self, record: logging.LogRecord):
|
|
35
|
+
self.emitter.log_line_signal.emit(get_log_source(record).value, self.format(record))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class LogPane(QGroupBox):
|
|
39
|
+
"""
|
|
40
|
+
One log source's pane - a titled, read-only, monospaced, non-wrapping text view with FIFO line trimming.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def __init__(self, name: str):
|
|
44
|
+
super().__init__(name)
|
|
45
|
+
self.setLayout(QVBoxLayout())
|
|
46
|
+
self.text_box = QPlainTextEdit()
|
|
47
|
+
self.text_box.setReadOnly(True)
|
|
48
|
+
self.text_box.setMaximumBlockCount(max_log_lines) # FIFO - drops the oldest lines
|
|
49
|
+
self.text_box.setLineWrapMode(QPlainTextEdit.NoWrap)
|
|
50
|
+
self.text_box.setFont(QFontDatabase.systemFont(QFontDatabase.FixedFont))
|
|
51
|
+
self.layout().addWidget(self.text_box)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class LogWidget(QWidget):
|
|
55
|
+
"""
|
|
56
|
+
"Log" tab - detailed application log (e.g. the full AWS CLI command lines), fed by all log records of this application's logger,
|
|
57
|
+
with a separate pane per backup type plus one for the rest of the application.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
def __init__(self):
|
|
61
|
+
super().__init__()
|
|
62
|
+
self.setLayout(QVBoxLayout())
|
|
63
|
+
|
|
64
|
+
self.controls_widget = QWidget()
|
|
65
|
+
self.controls_layout = QHBoxLayout()
|
|
66
|
+
self.controls_widget.setLayout(self.controls_layout)
|
|
67
|
+
self.clear_button = QPushButton("Clear")
|
|
68
|
+
self.controls_layout.addWidget(self.clear_button)
|
|
69
|
+
self.controls_layout.addStretch()
|
|
70
|
+
self.clear_button.clicked.connect(self.clear)
|
|
71
|
+
|
|
72
|
+
self.splitter = QSplitter(Qt.Vertical)
|
|
73
|
+
self.log_panes = {}
|
|
74
|
+
for log_source in LogSources:
|
|
75
|
+
self.log_panes[log_source] = LogPane(log_source.value)
|
|
76
|
+
self.splitter.addWidget(self.log_panes[log_source])
|
|
77
|
+
|
|
78
|
+
self.layout().addWidget(self.controls_widget)
|
|
79
|
+
self.layout().addWidget(self.splitter)
|
|
80
|
+
|
|
81
|
+
self.log_handler = QtLogHandler()
|
|
82
|
+
self.log_handler.setLevel(logging.DEBUG) # display everything the logger's own level lets through (e.g. DEBUG with the verbose preference)
|
|
83
|
+
self.log_handler.emitter.log_line_signal.connect(self.append_log_line)
|
|
84
|
+
logging.getLogger(__application_name__).addHandler(self.log_handler)
|
|
85
|
+
|
|
86
|
+
self.restore_state()
|
|
87
|
+
|
|
88
|
+
def get_pane_height_key(self, log_source: LogSources) -> str:
|
|
89
|
+
return f"log_pane_{log_source.name}_height"
|
|
90
|
+
|
|
91
|
+
def save_state(self):
|
|
92
|
+
preferences = get_gui_preferences()
|
|
93
|
+
for log_source, pane_height in zip(LogSources, self.splitter.sizes()):
|
|
94
|
+
setattr(preferences, self.get_pane_height_key(log_source), pane_height)
|
|
95
|
+
|
|
96
|
+
def restore_state(self):
|
|
97
|
+
preferences = get_gui_preferences()
|
|
98
|
+
pane_heights = []
|
|
99
|
+
for log_source in LogSources:
|
|
100
|
+
pane_height = getattr(preferences, self.get_pane_height_key(log_source))
|
|
101
|
+
# make sure every pane comes up visible, even if not set or the user has collapsed it to zero
|
|
102
|
+
if pane_height is None or int(pane_height) < minimum_pane_height:
|
|
103
|
+
pane_height = minimum_pane_height
|
|
104
|
+
pane_heights.append(int(pane_height))
|
|
105
|
+
self.splitter.setSizes(pane_heights)
|
|
106
|
+
|
|
107
|
+
def append_log_line(self, log_source_value: str, line: str):
|
|
108
|
+
self.log_panes[LogSources(log_source_value)].text_box.appendPlainText(line)
|
|
109
|
+
|
|
110
|
+
def clear(self):
|
|
111
|
+
for log_pane in self.log_panes.values():
|
|
112
|
+
log_pane.text_box.clear()
|
|
113
|
+
|
|
114
|
+
def detach(self):
|
|
115
|
+
"""
|
|
116
|
+
Stop capturing log records. Call before the widget is destroyed so the handler doesn't write into a dead widget.
|
|
117
|
+
"""
|
|
118
|
+
logging.getLogger(__application_name__).removeHandler(self.log_handler)
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
|
+
from PyQt5.QtCore import QTimer
|
|
3
4
|
from PyQt5.QtWidgets import QHBoxLayout, QVBoxLayout, QWidget, QPushButton, QLabel, QFileDialog, QLineEdit, QCheckBox, QSpinBox
|
|
4
5
|
|
|
5
6
|
from bup import get_preferences, UITypes
|
|
7
|
+
from bup.log_routing import set_detailed_file_logging, default_file_size_limit_mb
|
|
6
8
|
|
|
7
9
|
|
|
8
10
|
def get_gui_preferences():
|
|
@@ -118,11 +120,43 @@ class PreferencesWidget(QWidget):
|
|
|
118
120
|
self.layout().addWidget(QLabel()) # space
|
|
119
121
|
self.layout().addWidget(QLabel()) # space
|
|
120
122
|
|
|
123
|
+
# detailed log directory (optional - one log file per backup type, with e.g. the full AWS CLI calls and their output)
|
|
124
|
+
# debounce applying the directory so a half-typed path doesn't start collecting log files
|
|
125
|
+
self.detailed_log_apply_timer = QTimer(self)
|
|
126
|
+
self.detailed_log_apply_timer.setSingleShot(True)
|
|
127
|
+
self.detailed_log_apply_timer.setInterval(1000) # ms
|
|
128
|
+
self.detailed_log_apply_timer.timeout.connect(self.apply_detailed_log_directory)
|
|
129
|
+
self.detailed_log_widget = QWidget()
|
|
130
|
+
self.detailed_log_widget.setLayout(QHBoxLayout())
|
|
131
|
+
self.detailed_log_directory_line_edit = PreferencesLineEdit()
|
|
132
|
+
self.detailed_log_directory_line_edit.textChanged.connect(self.detailed_log_directory_changed)
|
|
133
|
+
self.select_detailed_log_directory_button = QPushButton("Select Detailed Log Directory")
|
|
134
|
+
self.select_detailed_log_directory_button.clicked.connect(self.select_detailed_log_directory)
|
|
135
|
+
self.detailed_log_widget.layout().addWidget(QLabel("Detailed Log Directory (blank for none):"))
|
|
136
|
+
self.detailed_log_widget.layout().addWidget(self.detailed_log_directory_line_edit)
|
|
137
|
+
self.detailed_log_widget.layout().addWidget(self.select_detailed_log_directory_button)
|
|
138
|
+
self.detailed_log_widget.layout().addWidget(QLabel("File size limit (MB):"))
|
|
139
|
+
self.detailed_log_file_size_limit = QSpinBox()
|
|
140
|
+
self.detailed_log_file_size_limit.setMinimum(1)
|
|
141
|
+
self.detailed_log_file_size_limit.setMaximum(10000)
|
|
142
|
+
self.detailed_log_file_size_limit.setValue(default_file_size_limit_mb)
|
|
143
|
+
self.detailed_log_file_size_limit.valueChanged.connect(self.detailed_log_file_size_limit_changed)
|
|
144
|
+
self.detailed_log_widget.layout().addWidget(self.detailed_log_file_size_limit)
|
|
145
|
+
self.detailed_log_widget.layout().addStretch()
|
|
146
|
+
self.layout().addWidget(self.detailed_log_widget)
|
|
147
|
+
self.layout().addWidget(QLabel()) # space
|
|
148
|
+
self.layout().addWidget(QLabel()) # space
|
|
149
|
+
|
|
121
150
|
# dry run
|
|
122
151
|
self.dry_run_check_box = QCheckBox("Dry run")
|
|
123
152
|
self.dry_run_check_box.clicked.connect(self.dry_run_clicked)
|
|
124
153
|
self.layout().addWidget(self.dry_run_check_box)
|
|
125
154
|
|
|
155
|
+
# S3 sync --size-only
|
|
156
|
+
self.s3_size_only_check_box = QCheckBox("S3 sync compares file size only, not timestamps (--size-only)")
|
|
157
|
+
self.s3_size_only_check_box.clicked.connect(self.s3_size_only_clicked)
|
|
158
|
+
self.layout().addWidget(self.s3_size_only_check_box)
|
|
159
|
+
|
|
126
160
|
# verbose
|
|
127
161
|
self.verbose_check_box = QCheckBox("Verbose")
|
|
128
162
|
self.verbose_check_box.clicked.connect(self.verbose_clicked)
|
|
@@ -140,7 +174,11 @@ class PreferencesWidget(QWidget):
|
|
|
140
174
|
self.aws_secret_access_key_line_edit.setText(preferences.aws_secret_access_key)
|
|
141
175
|
self.aws_region_line_edit.setText(preferences.aws_region)
|
|
142
176
|
self.github_token_line_edit.setText(preferences.github_token)
|
|
177
|
+
self.detailed_log_directory_line_edit.setText(preferences.detailed_log_directory)
|
|
178
|
+
if preferences.detailed_log_file_size_limit_mb is not None:
|
|
179
|
+
self.detailed_log_file_size_limit.setValue(preferences.detailed_log_file_size_limit_mb)
|
|
143
180
|
self.dry_run_check_box.setChecked(bool(preferences.dry_run)) # None translates to False
|
|
181
|
+
self.s3_size_only_check_box.setChecked(bool(preferences.s3_size_only)) # None translates to False
|
|
144
182
|
self.verbose_check_box.setChecked(bool(preferences.verbose)) # None translates to False
|
|
145
183
|
self.automatic_backup_enable_check_box.setChecked(bool(preferences.automatic_backup))
|
|
146
184
|
if preferences.backup_period is not None:
|
|
@@ -177,6 +215,22 @@ class PreferencesWidget(QWidget):
|
|
|
177
215
|
def github_token_changed(self):
|
|
178
216
|
get_gui_preferences().github_token = self.github_token_line_edit.text()
|
|
179
217
|
|
|
218
|
+
def select_detailed_log_directory(self):
|
|
219
|
+
new_detailed_log_directory = QFileDialog.getExistingDirectory(self, "Select Detailed Log Directory")
|
|
220
|
+
if new_detailed_log_directory is not None and len(new_detailed_log_directory) > 0:
|
|
221
|
+
self.detailed_log_directory_line_edit.setText(new_detailed_log_directory)
|
|
222
|
+
|
|
223
|
+
def detailed_log_directory_changed(self):
|
|
224
|
+
get_gui_preferences().detailed_log_directory = self.detailed_log_directory_line_edit.text()
|
|
225
|
+
self.detailed_log_apply_timer.start()
|
|
226
|
+
|
|
227
|
+
def detailed_log_file_size_limit_changed(self):
|
|
228
|
+
get_gui_preferences().detailed_log_file_size_limit_mb = self.detailed_log_file_size_limit.value() # QSpinBox guarantees an int
|
|
229
|
+
self.detailed_log_apply_timer.start()
|
|
230
|
+
|
|
231
|
+
def apply_detailed_log_directory(self):
|
|
232
|
+
set_detailed_file_logging(self.detailed_log_directory_line_edit.text(), self.detailed_log_file_size_limit.value())
|
|
233
|
+
|
|
180
234
|
def github_visible_clicked(self):
|
|
181
235
|
if self.github_token_line_edit.echoMode() == PreferencesLineEdit.Password:
|
|
182
236
|
self.github_show_button.setText("Hide")
|
|
@@ -191,6 +245,9 @@ class PreferencesWidget(QWidget):
|
|
|
191
245
|
def dry_run_clicked(self):
|
|
192
246
|
get_gui_preferences().dry_run = self.dry_run_check_box.isChecked()
|
|
193
247
|
|
|
248
|
+
def s3_size_only_clicked(self):
|
|
249
|
+
get_gui_preferences().s3_size_only = self.s3_size_only_check_box.isChecked()
|
|
250
|
+
|
|
194
251
|
def automatic_backup_changed(self):
|
|
195
252
|
preferences = get_gui_preferences()
|
|
196
253
|
preferences.automatic_backup = self.automatic_backup_enable_check_box.isChecked()
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import threading
|
|
1
2
|
from enum import Enum
|
|
2
3
|
from datetime import datetime
|
|
3
4
|
|
|
@@ -22,6 +23,7 @@ class RunAll(QThread):
|
|
|
22
23
|
super().__init__()
|
|
23
24
|
|
|
24
25
|
def run(self):
|
|
26
|
+
threading.current_thread().name = type(self).__name__ # QThreads otherwise show up in logs as "Dummy-N"
|
|
25
27
|
for backup_type in self.widget.backup_engines:
|
|
26
28
|
self.widget.backup_engines[backup_type].start()
|
|
27
29
|
for backup_type in self.widget.backup_engines:
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from enum import Enum
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Dict, Optional, TextIO, Union
|
|
5
|
+
|
|
6
|
+
from bup import __application_name__, BackupTypes
|
|
7
|
+
|
|
8
|
+
log_line_format = "%(asctime)s %(levelname)s %(filename)s:%(lineno)d %(threadName)s %(message)s"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class LogSources(Enum):
|
|
12
|
+
s3 = "AWS S3"
|
|
13
|
+
dynamodb = "DynamoDB"
|
|
14
|
+
github = "GitHub"
|
|
15
|
+
application = "Application"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# records stamped with a backup type by BupBase (the info_out/warning_out/error_out path, whose records all originate in bup_base.py)
|
|
19
|
+
log_sources_by_backup_type = {
|
|
20
|
+
BackupTypes.S3: LogSources.s3,
|
|
21
|
+
BackupTypes.DynamoDB: LogSources.dynamodb,
|
|
22
|
+
BackupTypes.github: LogSources.github,
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
# records logged directly (not via BupBase) are routed by the source file they were logged from
|
|
26
|
+
# (all of bup logs to the same application logger)
|
|
27
|
+
log_sources_by_filename = {
|
|
28
|
+
"s3_backup.py": LogSources.s3,
|
|
29
|
+
"aws_cli.py": LogSources.s3,
|
|
30
|
+
"dynamodb_backup.py": LogSources.dynamodb,
|
|
31
|
+
"github_backup.py": LogSources.github,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_log_source(record: logging.LogRecord) -> LogSources:
|
|
36
|
+
backup_type = getattr(record, "backup_type", None)
|
|
37
|
+
if backup_type in log_sources_by_backup_type:
|
|
38
|
+
return log_sources_by_backup_type[backup_type]
|
|
39
|
+
return log_sources_by_filename.get(record.filename, LogSources.application)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
bytes_per_mb = 1024 * 1024
|
|
43
|
+
default_file_size_limit_mb = 10
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class DetailedFileLogHandler(logging.Handler):
|
|
47
|
+
"""
|
|
48
|
+
Writes every log record to a file per log source in the given directory (s3.log, dynamodb.log, github.log, application.log),
|
|
49
|
+
so each backup type's full details (e.g. the AWS CLI calls and their stdout/stderr) are captured separately.
|
|
50
|
+
When a file reaches max_bytes it is rotated to <name>.log.1 (replacing any previous rotation), so each source uses
|
|
51
|
+
at most about twice max_bytes of disk.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
def __init__(self, directory: Union[str, Path], max_bytes: int = default_file_size_limit_mb * bytes_per_mb):
|
|
55
|
+
super().__init__()
|
|
56
|
+
self.directory = Path(directory)
|
|
57
|
+
self.max_bytes = max_bytes
|
|
58
|
+
self.setFormatter(logging.Formatter(log_line_format))
|
|
59
|
+
self.files: Dict[LogSources, TextIO] = {}
|
|
60
|
+
|
|
61
|
+
def get_log_file_path(self, log_source: LogSources) -> Path:
|
|
62
|
+
return Path(self.directory, f"{log_source.name}.log")
|
|
63
|
+
|
|
64
|
+
def rotate(self, log_source: LogSources):
|
|
65
|
+
file = self.files.pop(log_source)
|
|
66
|
+
file.close()
|
|
67
|
+
log_file_path = self.get_log_file_path(log_source)
|
|
68
|
+
rotated_path = Path(f"{log_file_path}.1")
|
|
69
|
+
if rotated_path.exists():
|
|
70
|
+
rotated_path.unlink()
|
|
71
|
+
log_file_path.rename(rotated_path)
|
|
72
|
+
|
|
73
|
+
def emit(self, record: logging.LogRecord):
|
|
74
|
+
log_source = get_log_source(record)
|
|
75
|
+
try:
|
|
76
|
+
file = self.files.get(log_source)
|
|
77
|
+
if file is None:
|
|
78
|
+
# open lazily so files and directories only appear once there is something to write
|
|
79
|
+
self.directory.mkdir(parents=True, exist_ok=True)
|
|
80
|
+
file = open(self.get_log_file_path(log_source), "a", encoding="utf-8")
|
|
81
|
+
self.files[log_source] = file
|
|
82
|
+
file.write(f"{self.format(record)}\n")
|
|
83
|
+
file.flush() # the point of this log is diagnosing problems - don't lose lines on a crash
|
|
84
|
+
if file.tell() >= self.max_bytes:
|
|
85
|
+
self.rotate(log_source) # the next record reopens the (fresh) file lazily
|
|
86
|
+
except OSError:
|
|
87
|
+
self.handleError(record)
|
|
88
|
+
|
|
89
|
+
def close(self):
|
|
90
|
+
self.acquire()
|
|
91
|
+
try:
|
|
92
|
+
for file in self.files.values():
|
|
93
|
+
try:
|
|
94
|
+
file.close()
|
|
95
|
+
except OSError:
|
|
96
|
+
pass # closing on teardown - nothing useful to do about a failed close
|
|
97
|
+
self.files = {}
|
|
98
|
+
finally:
|
|
99
|
+
self.release()
|
|
100
|
+
super().close()
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
_detailed_file_log_handler: Optional[DetailedFileLogHandler] = None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def set_detailed_file_logging(directory: Optional[Union[str, Path]], file_size_limit_mb: Optional[int] = None):
|
|
107
|
+
"""
|
|
108
|
+
Write detailed logs to the given directory (one file per log source), replacing any previous detailed log directory.
|
|
109
|
+
None (or blank) turns detailed file logging off. file_size_limit_mb caps each log file's size (rotating to a single
|
|
110
|
+
.1 backup); None or a non-positive value uses the default.
|
|
111
|
+
"""
|
|
112
|
+
global _detailed_file_log_handler
|
|
113
|
+
logger = logging.getLogger(__application_name__)
|
|
114
|
+
if _detailed_file_log_handler is not None:
|
|
115
|
+
logger.removeHandler(_detailed_file_log_handler)
|
|
116
|
+
_detailed_file_log_handler.close()
|
|
117
|
+
_detailed_file_log_handler = None
|
|
118
|
+
directory_string = "" if directory is None else str(directory).strip()
|
|
119
|
+
if len(directory_string) > 0:
|
|
120
|
+
if file_size_limit_mb is None or file_size_limit_mb <= 0:
|
|
121
|
+
file_size_limit_mb = default_file_size_limit_mb
|
|
122
|
+
_detailed_file_log_handler = DetailedFileLogHandler(Path(directory_string), max_bytes=file_size_limit_mb * bytes_per_mb)
|
|
123
|
+
_detailed_file_log_handler.setLevel(logging.DEBUG) # capture everything the logger's own level lets through
|
|
124
|
+
logger.addHandler(_detailed_file_log_handler)
|
|
@@ -25,8 +25,13 @@ class BupPreferences(Pref):
|
|
|
25
25
|
|
|
26
26
|
dry_run: bool = attrib(default=False)
|
|
27
27
|
|
|
28
|
+
s3_size_only: bool = attrib(default=False) # aws s3 sync --size-only (compare file size only, not timestamps)
|
|
29
|
+
|
|
28
30
|
verbose: bool = attrib(default=False)
|
|
29
31
|
|
|
32
|
+
detailed_log_directory: str = attrib(default=None) # None/blank = detailed file logging off
|
|
33
|
+
detailed_log_file_size_limit_mb: int = attrib(default=None) # None = default limit
|
|
34
|
+
|
|
30
35
|
height: int = attrib(default=None)
|
|
31
36
|
width: int = attrib(default=None)
|
|
32
37
|
S3_exclusions_height: int = attrib(default=None)
|
|
@@ -53,6 +58,10 @@ class BupPreferences(Pref):
|
|
|
53
58
|
github_warnings_width: int = attrib(default=None)
|
|
54
59
|
github_errors_height: int = attrib(default=None)
|
|
55
60
|
github_errors_width: int = attrib(default=None)
|
|
61
|
+
log_pane_s3_height: int = attrib(default=None)
|
|
62
|
+
log_pane_dynamodb_height: int = attrib(default=None)
|
|
63
|
+
log_pane_github_height: int = attrib(default=None)
|
|
64
|
+
log_pane_application_height: int = attrib(default=None)
|
|
56
65
|
|
|
57
66
|
|
|
58
67
|
def get_preferences(ui_type: UITypes) -> BupPreferences:
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
import shutil
|
|
3
2
|
import subprocess
|
|
4
|
-
import sys
|
|
5
3
|
import os
|
|
6
4
|
from pathlib import Path
|
|
7
|
-
from typing import
|
|
5
|
+
from typing import Tuple
|
|
8
6
|
|
|
9
7
|
from awsimple import S3Access
|
|
10
8
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
@@ -14,6 +12,7 @@ logging.getLogger("boto3").setLevel(logging.WARNING)
|
|
|
14
12
|
logging.getLogger("botocore").setLevel(logging.WARNING)
|
|
15
13
|
|
|
16
14
|
from bup import __application_name__, BupBase, BackupTypes, get_preferences, ExclusionPreferences
|
|
15
|
+
from bup.aws_cli import find_aws_cli, make_aws_cli_env, get_aws_cli_version, get_latest_awscli_version, check_aws_cli_version
|
|
17
16
|
|
|
18
17
|
log = get_logger(__application_name__)
|
|
19
18
|
|
|
@@ -51,6 +50,21 @@ def get_bucket_size(s3_client, bucket_name: str) -> Tuple[int, int]:
|
|
|
51
50
|
return total_size, object_count
|
|
52
51
|
|
|
53
52
|
|
|
53
|
+
def count_synced_files(sync_stdout: str) -> int:
|
|
54
|
+
"""
|
|
55
|
+
Number of files "aws s3 sync" transferred (or would transfer, for a dry run), from its stdout -
|
|
56
|
+
the CLI emits one "download:" (or "copy:") line per file, and nothing for files already up to date.
|
|
57
|
+
"""
|
|
58
|
+
synced_count = 0
|
|
59
|
+
for line in sync_stdout.splitlines():
|
|
60
|
+
line = line.strip()
|
|
61
|
+
if line.startswith("(dryrun) "):
|
|
62
|
+
line = line[len("(dryrun) ") :]
|
|
63
|
+
if line.startswith(("download:", "copy:")):
|
|
64
|
+
synced_count += 1
|
|
65
|
+
return synced_count
|
|
66
|
+
|
|
67
|
+
|
|
54
68
|
def compare_backup_sizes(s3_total_size: int, local_size: int) -> Tuple[str, str]:
|
|
55
69
|
"""
|
|
56
70
|
Rough check that the sync worked, based on total sizes.
|
|
@@ -66,33 +80,6 @@ def compare_backup_sizes(s3_total_size: int, local_size: int) -> Tuple[str, str]
|
|
|
66
80
|
return "info", "match"
|
|
67
81
|
|
|
68
82
|
|
|
69
|
-
def find_aws_cli() -> Tuple[Optional[Path], Optional[Path]]:
|
|
70
|
-
"""
|
|
71
|
-
Locate the AWS CLI executable and the python executable it should run with.
|
|
72
|
-
Use sys.executable to reliably locate python and aws CLI in the same directory,
|
|
73
|
-
which works for both local venv and installed app scenarios.
|
|
74
|
-
:return: (aws_cli_path, python_path), or (None, None) if not found
|
|
75
|
-
"""
|
|
76
|
-
python_exe = Path(sys.executable)
|
|
77
|
-
aws_names = ["aws.cmd", "aws.exe", "aws"] if sys.platform == "win32" else ["aws"]
|
|
78
|
-
aws_candidates = (
|
|
79
|
-
[(python_exe, python_exe.parent / name) for name in aws_names] # same dir as python (venv Scripts/)
|
|
80
|
-
+ [(python_exe, python_exe.parent / "Scripts" / name) for name in aws_names] # CLIP layout: python at root, scripts in Scripts/
|
|
81
|
-
+ [(Path("venv", "Scripts", "python.exe").absolute(), Path("venv", "Scripts", name).absolute()) for name in aws_names] # local venv from CWD
|
|
82
|
-
)
|
|
83
|
-
for p, a in aws_candidates:
|
|
84
|
-
if p.exists() and a.exists():
|
|
85
|
-
return a, p
|
|
86
|
-
|
|
87
|
-
# fall back to whatever is on the system PATH
|
|
88
|
-
aws_in_path = shutil.which("aws")
|
|
89
|
-
if aws_in_path:
|
|
90
|
-
return Path(aws_in_path), python_exe
|
|
91
|
-
|
|
92
|
-
log.error(f"AWS CLI executable not found ({aws_candidates=})")
|
|
93
|
-
return None, None
|
|
94
|
-
|
|
95
|
-
|
|
96
83
|
class S3Backup(BupBase):
|
|
97
84
|
|
|
98
85
|
backup_type = BackupTypes.S3
|
|
@@ -114,10 +101,11 @@ class S3Backup(BupBase):
|
|
|
114
101
|
break
|
|
115
102
|
return process.returncode, stdout.decode(decoding, errors="replace"), stderr.decode(decoding, errors="replace")
|
|
116
103
|
|
|
117
|
-
def
|
|
104
|
+
def run_backup(self):
|
|
118
105
|
|
|
119
106
|
preferences = get_preferences(self.ui_type)
|
|
120
107
|
dry_run = preferences.dry_run
|
|
108
|
+
size_only = preferences.s3_size_only
|
|
121
109
|
|
|
122
110
|
backup_directory = os.path.join(preferences.backup_directory, "s3")
|
|
123
111
|
|
|
@@ -135,10 +123,11 @@ class S3Backup(BupBase):
|
|
|
135
123
|
self.error_out("AWS CLI executable not found - S3 backup cannot run")
|
|
136
124
|
return
|
|
137
125
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
126
|
+
env_var = make_aws_cli_env(python_path)
|
|
127
|
+
|
|
128
|
+
# check that the AWS CLI doing the backups is the latest available (warn but proceed - an outdated CLI can still back up)
|
|
129
|
+
aws_cli_version_level, aws_cli_version_message = check_aws_cli_version(get_aws_cli_version(aws_cli_path, env_var), get_latest_awscli_version())
|
|
130
|
+
{"warning": self.warning_out, "info": self.info_out}[aws_cli_version_level](aws_cli_version_message)
|
|
142
131
|
|
|
143
132
|
try:
|
|
144
133
|
buckets = s3_access.bucket_list()
|
|
@@ -149,6 +138,7 @@ class S3Backup(BupBase):
|
|
|
149
138
|
|
|
150
139
|
count = 0
|
|
151
140
|
dry_run_count = 0
|
|
141
|
+
total_synced_count = 0
|
|
152
142
|
exclusions_no_comments = ExclusionPreferences(BackupTypes.S3.name).get_no_comments()
|
|
153
143
|
for bucket_name in buckets:
|
|
154
144
|
if self.stop_requested:
|
|
@@ -169,6 +159,8 @@ class S3Backup(BupBase):
|
|
|
169
159
|
s3_bucket_path = f"s3://{bucket_name}"
|
|
170
160
|
# Don't use --delete. We want to keep 'old' files locally.
|
|
171
161
|
sync_command_line = [str(aws_cli_path), "s3", "sync", s3_bucket_path, str(destination.absolute())]
|
|
162
|
+
if size_only:
|
|
163
|
+
sync_command_line.append("--size-only")
|
|
172
164
|
if dry_run:
|
|
173
165
|
sync_command_line.append("--dryrun")
|
|
174
166
|
log.info(subprocess.list2cmdline(sync_command_line))
|
|
@@ -192,9 +184,13 @@ class S3Backup(BupBase):
|
|
|
192
184
|
self.error_out(f"aws s3 sync failed (exit code {sync_returncode}) for {bucket_name}")
|
|
193
185
|
continue # don't count or verify a failed sync
|
|
194
186
|
|
|
187
|
+
synced_count = count_synced_files(sync_stdout)
|
|
188
|
+
total_synced_count += synced_count
|
|
189
|
+
|
|
195
190
|
# check the results (skip during dry run - nothing was synced)
|
|
196
191
|
if dry_run:
|
|
197
192
|
dry_run_count += 1
|
|
193
|
+
self.info_out(f"{bucket_name} : {synced_count} files would be synced")
|
|
198
194
|
continue
|
|
199
195
|
|
|
200
196
|
try:
|
|
@@ -207,9 +203,11 @@ class S3Backup(BupBase):
|
|
|
207
203
|
local_size, local_count = get_dir_size(destination)
|
|
208
204
|
level, message = compare_backup_sizes(s3_total_size, local_size)
|
|
209
205
|
output_routines = {"error": self.error_out, "info": log.info}
|
|
210
|
-
output_routines[level](
|
|
206
|
+
output_routines[level](
|
|
207
|
+
f"{bucket_name} : {message} (synced={synced_count}, s3_count={s3_object_count}, local_count={local_count}; s3_total_size={s3_total_size}, local_size={local_size})"
|
|
208
|
+
)
|
|
211
209
|
|
|
212
210
|
if dry_run:
|
|
213
|
-
self.info_out(f"{len(buckets)} buckets, {dry_run_count} dry run, {len(exclusions_no_comments)} excluded")
|
|
211
|
+
self.info_out(f"{len(buckets)} buckets, {dry_run_count} dry run, {total_synced_count} files would be synced, {len(exclusions_no_comments)} excluded")
|
|
214
212
|
else:
|
|
215
|
-
self.info_out(f"{len(buckets)} buckets, {count} backed up, {len(exclusions_no_comments)} excluded")
|
|
213
|
+
self.info_out(f"{len(buckets)} buckets, {count} backed up, {total_synced_count} files synced, {len(exclusions_no_comments)} excluded")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0
|
|
4
4
|
Summary: backup all your AWS S3 buckets and DynamoDB tables, and github repos to a backup directory/folder
|
|
5
5
|
Author-email: abel <j@abel.co>
|
|
6
6
|
License-Expression: GPL-3.0-only
|
|
@@ -15,10 +15,7 @@ Requires-Dist: ismain
|
|
|
15
15
|
Requires-Dist: balsa
|
|
16
16
|
Requires-Dist: boto3
|
|
17
17
|
Requires-Dist: typeguard
|
|
18
|
-
Requires-Dist: hashy
|
|
19
|
-
Requires-Dist: dictim
|
|
20
18
|
Requires-Dist: awsimple
|
|
21
|
-
Requires-Dist: pressenter2exit
|
|
22
19
|
Requires-Dist: awscli
|
|
23
20
|
Requires-Dist: GitPython
|
|
24
21
|
Requires-Dist: github3.py
|
|
@@ -30,6 +27,17 @@ Requires-Dist: pref
|
|
|
30
27
|
Requires-Dist: pyqt5
|
|
31
28
|
Requires-Dist: python-dotenv
|
|
32
29
|
Requires-Dist: sentry-sdk
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: wheel; extra == "dev"
|
|
32
|
+
Requires-Dist: black; extra == "dev"
|
|
33
|
+
Requires-Dist: pytest; extra == "dev"
|
|
34
|
+
Requires-Dist: coverage; extra == "dev"
|
|
35
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
36
|
+
Requires-Dist: pyship; extra == "dev"
|
|
37
|
+
Requires-Dist: flake8; extra == "dev"
|
|
38
|
+
Requires-Dist: PyQt5-stubs; extra == "dev"
|
|
39
|
+
Requires-Dist: build; extra == "dev"
|
|
40
|
+
Requires-Dist: twine; extra == "dev"
|
|
33
41
|
Dynamic: license-file
|
|
34
42
|
|
|
35
43
|
# bup
|
|
@@ -7,6 +7,7 @@ bup/__init__.py
|
|
|
7
7
|
bup/__main__.py
|
|
8
8
|
bup/__version__.py
|
|
9
9
|
bup/arguments.py
|
|
10
|
+
bup/aws_cli.py
|
|
10
11
|
bup/backup_types.py
|
|
11
12
|
bup/bup.ico
|
|
12
13
|
bup/bup.png
|
|
@@ -14,6 +15,7 @@ bup/bup_base.py
|
|
|
14
15
|
bup/dynamodb_backup.py
|
|
15
16
|
bup/github_backup.py
|
|
16
17
|
bup/gpl-3.0.md
|
|
18
|
+
bup/log_routing.py
|
|
17
19
|
bup/preferences.py
|
|
18
20
|
bup/robust_os_calls.py
|
|
19
21
|
bup/s3_backup.py
|
|
@@ -31,5 +33,6 @@ bup/gui/about.py
|
|
|
31
33
|
bup/gui/bup_dialog.py
|
|
32
34
|
bup/gui/gui_icon.py
|
|
33
35
|
bup/gui/gui_main.py
|
|
36
|
+
bup/gui/log_widget.py
|
|
34
37
|
bup/gui/preferences_widget.py
|
|
35
38
|
bup/gui/run_backup_widget.py
|
|
@@ -2,10 +2,7 @@ ismain
|
|
|
2
2
|
balsa
|
|
3
3
|
boto3
|
|
4
4
|
typeguard
|
|
5
|
-
hashy
|
|
6
|
-
dictim
|
|
7
5
|
awsimple
|
|
8
|
-
pressenter2exit
|
|
9
6
|
awscli
|
|
10
7
|
GitPython
|
|
11
8
|
github3.py
|
|
@@ -17,3 +14,15 @@ pref
|
|
|
17
14
|
pyqt5
|
|
18
15
|
python-dotenv
|
|
19
16
|
sentry-sdk
|
|
17
|
+
|
|
18
|
+
[dev]
|
|
19
|
+
wheel
|
|
20
|
+
black
|
|
21
|
+
pytest
|
|
22
|
+
coverage
|
|
23
|
+
pytest-cov
|
|
24
|
+
pyship
|
|
25
|
+
flake8
|
|
26
|
+
PyQt5-stubs
|
|
27
|
+
build
|
|
28
|
+
twine
|
|
@@ -19,10 +19,7 @@ dependencies = [
|
|
|
19
19
|
"balsa",
|
|
20
20
|
"boto3",
|
|
21
21
|
"typeguard",
|
|
22
|
-
"hashy",
|
|
23
|
-
"dictim",
|
|
24
22
|
"awsimple",
|
|
25
|
-
"pressenter2exit",
|
|
26
23
|
"awscli",
|
|
27
24
|
"GitPython",
|
|
28
25
|
"github3.py",
|
|
@@ -36,6 +33,20 @@ dependencies = [
|
|
|
36
33
|
"sentry-sdk",
|
|
37
34
|
]
|
|
38
35
|
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
dev = [
|
|
38
|
+
"wheel",
|
|
39
|
+
"black",
|
|
40
|
+
"pytest",
|
|
41
|
+
"coverage",
|
|
42
|
+
"pytest-cov",
|
|
43
|
+
"pyship",
|
|
44
|
+
"flake8",
|
|
45
|
+
"PyQt5-stubs",
|
|
46
|
+
"build",
|
|
47
|
+
"twine",
|
|
48
|
+
]
|
|
49
|
+
|
|
39
50
|
[project.urls]
|
|
40
51
|
Homepage = "https://github.com/jamesabel/bup"
|
|
41
52
|
Download = "https://github.com/jamesabel/bup"
|
|
@@ -52,9 +63,6 @@ include = ["bup*"]
|
|
|
52
63
|
[tool.setuptools.package-data]
|
|
53
64
|
bup = ["bup.png", "bup.ico", "LICENSE", "gpl-3.0.md"]
|
|
54
65
|
|
|
55
|
-
[tool.hatch.version]
|
|
56
|
-
path = "bup/__version__.py"
|
|
57
|
-
|
|
58
66
|
[tool.pyship]
|
|
59
67
|
is_gui = true
|
|
60
68
|
upload = true
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|