dar-backup 1.0.0.1__py3-none-any.whl → 1.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dar_backup/Changelog.md +64 -1
- dar_backup/README.md +355 -34
- dar_backup/__about__.py +3 -2
- dar_backup/clean_log.py +102 -63
- dar_backup/cleanup.py +225 -76
- dar_backup/command_runner.py +198 -103
- dar_backup/config_settings.py +158 -1
- dar_backup/dar-backup.conf +18 -0
- dar_backup/dar-backup.conf.j2 +44 -0
- dar_backup/dar_backup.py +806 -131
- dar_backup/demo.py +18 -9
- dar_backup/installer.py +18 -1
- dar_backup/manager.py +304 -91
- dar_backup/util.py +502 -141
- {dar_backup-1.0.0.1.dist-info → dar_backup-1.0.2.dist-info}/METADATA +358 -37
- dar_backup-1.0.2.dist-info/RECORD +25 -0
- {dar_backup-1.0.0.1.dist-info → dar_backup-1.0.2.dist-info}/WHEEL +1 -1
- dar_backup-1.0.0.1.dist-info/RECORD +0 -25
- {dar_backup-1.0.0.1.dist-info → dar_backup-1.0.2.dist-info}/entry_points.txt +0 -0
- {dar_backup-1.0.0.1.dist-info → dar_backup-1.0.2.dist-info}/licenses/LICENSE +0 -0
dar_backup/clean_log.py
CHANGED
|
@@ -23,13 +23,54 @@ import re
|
|
|
23
23
|
import os
|
|
24
24
|
import sys
|
|
25
25
|
|
|
26
|
+
from datetime import datetime
|
|
27
|
+
|
|
26
28
|
from dar_backup import __about__ as about
|
|
27
29
|
from dar_backup.config_settings import ConfigSettings
|
|
30
|
+
from dar_backup.util import send_discord_message, get_logger
|
|
28
31
|
|
|
29
32
|
LICENSE = '''Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
|
|
30
33
|
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW, not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
31
34
|
See section 15 and section 16 in the supplied "LICENSE" file.'''
|
|
32
35
|
|
|
36
|
+
TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}\b")
|
|
37
|
+
CLEAN_MESSAGE_PREFIXES = (
|
|
38
|
+
"Inspecting directory",
|
|
39
|
+
"Finished Inspecting",
|
|
40
|
+
"<File",
|
|
41
|
+
"</File",
|
|
42
|
+
"<Attributes",
|
|
43
|
+
"</Attributes",
|
|
44
|
+
"<Directory",
|
|
45
|
+
"</Directory",
|
|
46
|
+
"<Catalog",
|
|
47
|
+
"</Catalog",
|
|
48
|
+
"<Symlink",
|
|
49
|
+
"</Symlink",
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def _split_level_and_message(line):
|
|
53
|
+
line = line.rstrip("\n")
|
|
54
|
+
if " - " not in line:
|
|
55
|
+
return None, None
|
|
56
|
+
|
|
57
|
+
parts = line.split(" - ")
|
|
58
|
+
if len(parts) >= 3 and TIMESTAMP_RE.match(parts[0].strip()):
|
|
59
|
+
level = parts[1]
|
|
60
|
+
message = " - ".join(parts[2:])
|
|
61
|
+
else:
|
|
62
|
+
level = parts[0]
|
|
63
|
+
message = " - ".join(parts[1:])
|
|
64
|
+
|
|
65
|
+
return level.strip(), message
|
|
66
|
+
|
|
67
|
+
def _should_remove_line(line):
|
|
68
|
+
level, message = _split_level_and_message(line)
|
|
69
|
+
if level != "INFO" or message is None:
|
|
70
|
+
return False
|
|
71
|
+
message = message.lstrip()
|
|
72
|
+
return any(message.startswith(prefix) for prefix in CLEAN_MESSAGE_PREFIXES)
|
|
73
|
+
|
|
33
74
|
def clean_log_file(log_file_path, dry_run=False):
|
|
34
75
|
"""Removes specific log lines from the given file using a memory-efficient streaming approach."""
|
|
35
76
|
|
|
@@ -42,7 +83,7 @@ def clean_log_file(log_file_path, dry_run=False):
|
|
|
42
83
|
print(f"No read permission for '{log_file_path}'")
|
|
43
84
|
sys.exit(1)
|
|
44
85
|
|
|
45
|
-
if not os.access(log_file_path, os.W_OK):
|
|
86
|
+
if not dry_run and not os.access(log_file_path, os.W_OK):
|
|
46
87
|
print(f"Error: No write permission for '{log_file_path}'")
|
|
47
88
|
sys.exit(1)
|
|
48
89
|
|
|
@@ -51,47 +92,20 @@ def clean_log_file(log_file_path, dry_run=False):
|
|
|
51
92
|
print(f"Performing a dry run on: {log_file_path}")
|
|
52
93
|
|
|
53
94
|
temp_file_path = log_file_path + ".tmp"
|
|
54
|
-
|
|
55
|
-
patterns = [
|
|
56
|
-
r"INFO\s*-\s*Inspecting\s*directory",
|
|
57
|
-
r"INFO\s*-\s*Finished\s*Inspecting",
|
|
58
|
-
r"INFO\s*-\s*<File",
|
|
59
|
-
r"INFO\s*-\s*</File",
|
|
60
|
-
r"INFO\s*-\s*<Attributes",
|
|
61
|
-
r"INFO\s*-\s*</Attributes",
|
|
62
|
-
r"INFO\s*-\s*</Directory",
|
|
63
|
-
r"INFO\s*-\s*<Directory",
|
|
64
|
-
r"INFO\s*-\s*<Catalog",
|
|
65
|
-
r"INFO\s*-\s*</Catalog",
|
|
66
|
-
r"INFO\s*-\s*<Symlink",
|
|
67
|
-
r"INFO\s*-\s*</Symlink",
|
|
68
|
-
]
|
|
69
95
|
|
|
70
96
|
try:
|
|
71
|
-
|
|
97
|
+
if dry_run:
|
|
98
|
+
with open(log_file_path, "r", errors="ignore") as infile:
|
|
99
|
+
for line in infile:
|
|
100
|
+
if _should_remove_line(line):
|
|
101
|
+
print(f"Would remove: {line.strip()}")
|
|
102
|
+
return
|
|
72
103
|
|
|
104
|
+
with open(log_file_path, "r", errors="ignore") as infile, open(temp_file_path, "w") as outfile:
|
|
73
105
|
for line in infile:
|
|
74
|
-
|
|
75
|
-
matched = False # Track if a pattern is matched
|
|
76
|
-
|
|
77
|
-
for pattern in patterns:
|
|
78
|
-
if re.search(pattern, line): # Check if the pattern matches
|
|
79
|
-
if dry_run:
|
|
80
|
-
print(f"Would remove: {original_line.strip()}") # Print full line for dry-run
|
|
81
|
-
matched = True # Mark that a pattern matched
|
|
82
|
-
break # No need to check other patterns if one matches
|
|
83
|
-
|
|
84
|
-
if not dry_run and not matched: # In normal mode, only write non-empty lines
|
|
106
|
+
if not _should_remove_line(line):
|
|
85
107
|
outfile.write(line.rstrip() + "\n")
|
|
86
108
|
|
|
87
|
-
if dry_run and matched:
|
|
88
|
-
continue # In dry-run mode, skip writing (since we’re just showing)
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
# Ensure the temp file exists before renaming
|
|
92
|
-
if not os.path.exists(temp_file_path):
|
|
93
|
-
open(temp_file_path, "w").close() # Create an empty file if nothing was written
|
|
94
|
-
|
|
95
109
|
os.replace(temp_file_path, log_file_path)
|
|
96
110
|
print(f"Successfully cleaned log file: {log_file_path}")
|
|
97
111
|
|
|
@@ -131,43 +145,68 @@ def main():
|
|
|
131
145
|
|
|
132
146
|
args = parser.parse_args()
|
|
133
147
|
|
|
134
|
-
|
|
148
|
+
try:
|
|
149
|
+
config_settings = ConfigSettings(os.path.expanduser(os.path.expandvars(args.config_file)))
|
|
150
|
+
except Exception as exc:
|
|
151
|
+
msg = f"Config error: {exc}"
|
|
152
|
+
print(msg, file=sys.stderr)
|
|
153
|
+
ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
|
|
154
|
+
send_discord_message(f"{ts} - clean-log: FAILURE - {msg}")
|
|
155
|
+
sys.exit(127)
|
|
135
156
|
|
|
136
|
-
|
|
137
|
-
args.file
|
|
157
|
+
try:
|
|
158
|
+
files_to_clean = args.file if args.file else [config_settings.logfile_location]
|
|
159
|
+
logfile_dir = os.path.dirname(os.path.realpath(config_settings.logfile_location))
|
|
160
|
+
validated_files = []
|
|
138
161
|
|
|
139
|
-
|
|
162
|
+
for file_path in files_to_clean:
|
|
163
|
+
if not isinstance(file_path, (str, bytes, os.PathLike)):
|
|
164
|
+
print(f"Error: Invalid file path type: {file_path}")
|
|
165
|
+
sys.exit(1)
|
|
140
166
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
167
|
+
file_path = os.fspath(file_path)
|
|
168
|
+
if isinstance(file_path, bytes):
|
|
169
|
+
file_path = os.fsdecode(file_path)
|
|
144
170
|
|
|
145
|
-
|
|
146
|
-
|
|
171
|
+
if file_path.strip() == "":
|
|
172
|
+
print(f"Error: Invalid empty filename '{file_path}'.")
|
|
173
|
+
sys.exit(1)
|
|
147
174
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
175
|
+
if ".." in os.path.normpath(file_path).split(os.sep):
|
|
176
|
+
print(f"Error: Path traversal is not allowed: '{file_path}'")
|
|
177
|
+
sys.exit(1)
|
|
151
178
|
|
|
152
|
-
|
|
153
|
-
if not isinstance(file_path, (str, bytes, os.PathLike)):
|
|
154
|
-
print(f"Error: Invalid file path type: {file_path}")
|
|
155
|
-
sys.exit(1)
|
|
179
|
+
resolved_path = os.path.realpath(file_path)
|
|
156
180
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
181
|
+
if not resolved_path.startswith(logfile_dir + os.sep):
|
|
182
|
+
print(f"Error: File is outside allowed directory: '{file_path}'")
|
|
183
|
+
sys.exit(1)
|
|
160
184
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
185
|
+
if not os.path.exists(file_path):
|
|
186
|
+
print(f"Error: Log file '{file_path}' does not exist.")
|
|
187
|
+
sys.exit(1)
|
|
164
188
|
|
|
165
|
-
|
|
166
|
-
# Run the log file cleaning function
|
|
167
|
-
for log_file in args.file:
|
|
168
|
-
clean_log_file(log_file, dry_run=args.dry_run)
|
|
169
|
-
print(f"Log file '{args.file}' has been cleaned successfully.")
|
|
189
|
+
validated_files.append(file_path)
|
|
170
190
|
|
|
171
191
|
|
|
192
|
+
# Run the log file cleaning function
|
|
193
|
+
for log_file in validated_files:
|
|
194
|
+
clean_log_file(log_file, dry_run=args.dry_run)
|
|
195
|
+
file_list = ", ".join(validated_files)
|
|
196
|
+
if args.dry_run:
|
|
197
|
+
print(f"Dry run complete for: {file_list}")
|
|
198
|
+
else:
|
|
199
|
+
print(f"Log file '{file_list}' has been cleaned successfully.")
|
|
200
|
+
except Exception as e:
|
|
201
|
+
msg = f"Unexpected error during clean-log: {e}"
|
|
202
|
+
logger = get_logger()
|
|
203
|
+
if logger:
|
|
204
|
+
logger.error(msg, exc_info=True)
|
|
205
|
+
else:
|
|
206
|
+
print(msg, file=sys.stderr)
|
|
207
|
+
|
|
208
|
+
ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
|
|
209
|
+
send_discord_message(f"{ts} - clean-log: FAILURE - {msg}", config_settings=config_settings)
|
|
210
|
+
sys.exit(1)
|
|
172
211
|
if __name__ == "__main__":
|
|
173
212
|
main()
|
dar_backup/cleanup.py
CHANGED
|
@@ -24,25 +24,31 @@ import sys
|
|
|
24
24
|
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
|
|
28
27
|
from datetime import datetime, timedelta
|
|
29
28
|
from inputimeout import inputimeout, TimeoutOccurred
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from sys import stderr
|
|
30
31
|
from time import time
|
|
31
32
|
from typing import Dict, List, NamedTuple, Tuple
|
|
33
|
+
import glob
|
|
32
34
|
|
|
33
35
|
|
|
34
36
|
from . import __about__ as about
|
|
35
37
|
from dar_backup.config_settings import ConfigSettings
|
|
36
38
|
from dar_backup.util import list_backups
|
|
37
39
|
from dar_backup.util import setup_logging
|
|
40
|
+
from dar_backup.util import get_config_file
|
|
38
41
|
from dar_backup.util import get_logger
|
|
39
42
|
from dar_backup.util import requirements
|
|
40
43
|
from dar_backup.util import show_version
|
|
41
44
|
from dar_backup.util import get_invocation_command_line
|
|
42
45
|
from dar_backup.util import print_aligned_settings
|
|
43
46
|
from dar_backup.util import backup_definition_completer, list_archive_completer
|
|
47
|
+
from dar_backup.util import is_archive_name_allowed
|
|
44
48
|
from dar_backup.util import is_safe_filename
|
|
49
|
+
from dar_backup.util import safe_remove_file
|
|
45
50
|
from dar_backup.util import show_scriptname
|
|
51
|
+
from dar_backup.util import send_discord_message
|
|
46
52
|
|
|
47
53
|
from dar_backup.command_runner import CommandRunner
|
|
48
54
|
from dar_backup.command_runner import CommandResult
|
|
@@ -50,7 +56,56 @@ from dar_backup.command_runner import CommandResult
|
|
|
50
56
|
logger = None
|
|
51
57
|
runner = None
|
|
52
58
|
|
|
53
|
-
def
|
|
59
|
+
def _delete_par2_files(
|
|
60
|
+
archive_name: str,
|
|
61
|
+
backup_dir: str,
|
|
62
|
+
config_settings: ConfigSettings = None,
|
|
63
|
+
backup_definition: str = None,
|
|
64
|
+
dry_run: bool = False,
|
|
65
|
+
) -> None:
|
|
66
|
+
if config_settings and hasattr(config_settings, "get_par2_config"):
|
|
67
|
+
par2_config = config_settings.get_par2_config(backup_definition)
|
|
68
|
+
else:
|
|
69
|
+
par2_config = {
|
|
70
|
+
"par2_dir": None,
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
par2_dir = par2_config.get("par2_dir") or backup_dir
|
|
74
|
+
par2_dir = os.path.expanduser(os.path.expandvars(par2_dir))
|
|
75
|
+
if not os.path.isdir(par2_dir):
|
|
76
|
+
logger.warning(f"PAR2 directory not found, skipping cleanup: {par2_dir}")
|
|
77
|
+
return
|
|
78
|
+
|
|
79
|
+
par2_glob = os.path.join(par2_dir, f"{archive_name}*.par2")
|
|
80
|
+
targets = set(glob.glob(par2_glob))
|
|
81
|
+
manifest_path = os.path.join(par2_dir, f"{archive_name}.par2.manifest.ini")
|
|
82
|
+
if os.path.exists(manifest_path):
|
|
83
|
+
targets.add(manifest_path)
|
|
84
|
+
|
|
85
|
+
par2_regex = re.compile(rf"^{re.escape(archive_name)}\.[0-9]+\.dar.*\.par2$")
|
|
86
|
+
for entry in os.scandir(par2_dir):
|
|
87
|
+
if not entry.is_file():
|
|
88
|
+
continue
|
|
89
|
+
filename = entry.name
|
|
90
|
+
if par2_regex.match(filename):
|
|
91
|
+
targets.add(entry.path)
|
|
92
|
+
|
|
93
|
+
if not targets:
|
|
94
|
+
logger.info("No par2 files matched the cleanup patterns.")
|
|
95
|
+
return
|
|
96
|
+
|
|
97
|
+
for file_path in sorted(targets):
|
|
98
|
+
try:
|
|
99
|
+
if dry_run:
|
|
100
|
+
logger.info(f"Dry run: would delete PAR2 file: {file_path}")
|
|
101
|
+
else:
|
|
102
|
+
safe_remove_file(file_path, base_dir=Path(par2_dir))
|
|
103
|
+
logger.info(f"Deleted PAR2 file: {file_path}")
|
|
104
|
+
except Exception as e:
|
|
105
|
+
logger.error(f"Error deleting PAR2 file {file_path}: {e}")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def delete_old_backups(backup_dir, age, backup_type, args, backup_definition=None, config_settings: ConfigSettings = None):
|
|
54
109
|
"""
|
|
55
110
|
Delete backups older than the specified age in days.
|
|
56
111
|
Only .dar and .par2 files are considered for deletion.
|
|
@@ -66,8 +121,12 @@ def delete_old_backups(backup_dir, age, backup_type, args, backup_definition=Non
|
|
|
66
121
|
|
|
67
122
|
archives_deleted = {}
|
|
68
123
|
|
|
69
|
-
|
|
70
|
-
|
|
124
|
+
dry_run = getattr(args, "dry_run", False) is True
|
|
125
|
+
for entry in os.scandir(backup_dir):
|
|
126
|
+
if not entry.is_file():
|
|
127
|
+
continue
|
|
128
|
+
filename = entry.name
|
|
129
|
+
if not filename.endswith('.dar'):
|
|
71
130
|
continue
|
|
72
131
|
if backup_definition and not filename.startswith(backup_definition):
|
|
73
132
|
continue
|
|
@@ -80,10 +139,13 @@ def delete_old_backups(backup_dir, age, backup_type, args, backup_definition=Non
|
|
|
80
139
|
raise
|
|
81
140
|
|
|
82
141
|
if file_date < cutoff_date:
|
|
83
|
-
file_path =
|
|
142
|
+
file_path = entry.path
|
|
84
143
|
try:
|
|
85
|
-
|
|
86
|
-
|
|
144
|
+
if dry_run:
|
|
145
|
+
logger.info(f"Dry run: would delete {backup_type} backup: {file_path}")
|
|
146
|
+
else:
|
|
147
|
+
safe_remove_file(file_path, base_dir=Path(backup_dir))
|
|
148
|
+
logger.info(f"Deleted {backup_type} backup: {file_path}")
|
|
87
149
|
archive_name = filename.split('.')[0]
|
|
88
150
|
if not archive_name in archives_deleted:
|
|
89
151
|
logger.debug(f"Archive name: '{archive_name}' added to catalog deletion list")
|
|
@@ -92,10 +154,17 @@ def delete_old_backups(backup_dir, age, backup_type, args, backup_definition=Non
|
|
|
92
154
|
logger.error(f"Error deleting file {file_path}: {e}")
|
|
93
155
|
|
|
94
156
|
for archive_name in archives_deleted.keys():
|
|
95
|
-
|
|
157
|
+
if not is_archive_name_allowed(archive_name):
|
|
158
|
+
raise ValueError(f"Refusing unsafe archive name: {archive_name}")
|
|
159
|
+
archive_definition = archive_name.split('_')[0]
|
|
160
|
+
_delete_par2_files(archive_name, backup_dir, config_settings, archive_definition, dry_run=dry_run)
|
|
161
|
+
if dry_run:
|
|
162
|
+
logger.info(f"Dry run: would run manager to delete archive '{archive_name}'")
|
|
163
|
+
else:
|
|
164
|
+
delete_catalog(archive_name, args)
|
|
96
165
|
|
|
97
166
|
|
|
98
|
-
def delete_archive(backup_dir, archive_name, args):
|
|
167
|
+
def delete_archive(backup_dir, archive_name, args, config_settings: ConfigSettings = None):
|
|
99
168
|
"""
|
|
100
169
|
Delete all .dar and .par2 files in the backup directory for the given archive name.
|
|
101
170
|
|
|
@@ -107,36 +176,33 @@ def delete_archive(backup_dir, archive_name, args):
|
|
|
107
176
|
|
|
108
177
|
# Delete the specified .dar files according to the naming convention
|
|
109
178
|
files_deleted = False
|
|
110
|
-
|
|
179
|
+
dry_run = getattr(args, "dry_run", False) is True
|
|
180
|
+
for entry in os.scandir(backup_dir):
|
|
181
|
+
if not entry.is_file():
|
|
182
|
+
continue
|
|
183
|
+
filename = entry.name
|
|
111
184
|
if archive_regex.match(filename):
|
|
112
|
-
file_path =
|
|
185
|
+
file_path = entry.path
|
|
113
186
|
try:
|
|
114
|
-
|
|
115
|
-
|
|
187
|
+
if dry_run:
|
|
188
|
+
logger.info(f"Dry run: would delete archive slice: {file_path}")
|
|
189
|
+
else:
|
|
190
|
+
is_safe_filename(file_path) and os.remove(file_path)
|
|
191
|
+
logger.info(f"Deleted archive slice: {file_path}")
|
|
116
192
|
files_deleted = True
|
|
117
193
|
except Exception as e:
|
|
118
194
|
logger.error(f"Error deleting archive slice {file_path}: {e}")
|
|
119
195
|
|
|
120
196
|
if files_deleted:
|
|
121
|
-
|
|
197
|
+
if dry_run:
|
|
198
|
+
logger.info(f"Dry run: would run manager to delete archive '{archive_name}'")
|
|
199
|
+
else:
|
|
200
|
+
delete_catalog(archive_name, args)
|
|
122
201
|
else:
|
|
123
202
|
logger.info("No .dar files matched the regex for deletion.")
|
|
124
203
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
files_deleted = False
|
|
128
|
-
for filename in sorted(os.listdir(backup_dir)):
|
|
129
|
-
if par2_regex.match(filename):
|
|
130
|
-
file_path = os.path.join(backup_dir, filename)
|
|
131
|
-
try:
|
|
132
|
-
is_safe_filename(file_path) and os.remove(file_path)
|
|
133
|
-
logger.info(f"Deleted PAR2 file: {file_path}")
|
|
134
|
-
files_deleted = True
|
|
135
|
-
except Exception as e:
|
|
136
|
-
logger.error(f"Error deleting PAR2 file {file_path}: {e}")
|
|
137
|
-
|
|
138
|
-
if not files_deleted:
|
|
139
|
-
logger.info("No .par2 matched the regex for deletion.")
|
|
204
|
+
archive_definition = archive_name.split('_')[0]
|
|
205
|
+
_delete_par2_files(archive_name, backup_dir, config_settings, archive_definition, dry_run=dry_run)
|
|
140
206
|
|
|
141
207
|
|
|
142
208
|
def delete_catalog(catalog_name: str, args: NamedTuple) -> bool:
|
|
@@ -190,42 +256,82 @@ def main():
|
|
|
190
256
|
|
|
191
257
|
parser = argparse.ArgumentParser(description="Cleanup old archives according to AGE configuration.")
|
|
192
258
|
parser.add_argument('-d', '--backup-definition', help="Specific backup definition to cleanup.").completer = backup_definition_completer
|
|
193
|
-
parser.add_argument('-c', '--config-file', '-c', type=str, help="Path to 'dar-backup.conf'", default=
|
|
259
|
+
parser.add_argument('-c', '--config-file', '-c', type=str, help="Path to 'dar-backup.conf'", default=None)
|
|
194
260
|
parser.add_argument('-v', '--version', action='store_true', help="Show version information.")
|
|
195
261
|
parser.add_argument('--alternate-archive-dir', type=str, help="Cleanup in this directory instead of the default one.")
|
|
196
|
-
parser.add_argument(
|
|
262
|
+
parser.add_argument(
|
|
263
|
+
'--cleanup-specific-archives',
|
|
264
|
+
type=str,
|
|
265
|
+
nargs='?',
|
|
266
|
+
const="",
|
|
267
|
+
default=None,
|
|
268
|
+
help="Comma separated list of archives to cleanup",
|
|
269
|
+
).completer = list_archive_completer
|
|
270
|
+
parser.add_argument(
|
|
271
|
+
'cleanup_specific_archives_list',
|
|
272
|
+
nargs='*',
|
|
273
|
+
help=argparse.SUPPRESS,
|
|
274
|
+
).completer = list_archive_completer
|
|
197
275
|
parser.add_argument('-l', '--list', action='store_true', help="List available archives.")
|
|
198
276
|
parser.add_argument('--verbose', action='store_true', help="Print various status messages to screen")
|
|
199
277
|
parser.add_argument('--log-level', type=str, help="`debug` or `trace`, default is `info`", default="info")
|
|
200
278
|
parser.add_argument('--log-stdout', action='store_true', help='also print log messages to stdout')
|
|
201
279
|
parser.add_argument('--test-mode', action='store_true', help='Read envvars in order to run some pytest cases')
|
|
280
|
+
parser.add_argument('--dry-run', action='store_true', help='Show what would be deleted without removing files')
|
|
202
281
|
|
|
203
|
-
|
|
282
|
+
comp_line = os.environ.get("COMP_LINE", "")
|
|
283
|
+
only_archives = "--cleanup-specific-archives" in comp_line
|
|
284
|
+
argcomplete.autocomplete(parser, always_complete_options=not only_archives)
|
|
204
285
|
|
|
205
286
|
args = parser.parse_args()
|
|
206
287
|
|
|
207
|
-
args.config_file = os.path.expanduser(os.path.expandvars(args.config_file))
|
|
208
|
-
|
|
209
|
-
|
|
210
288
|
if args.version:
|
|
211
289
|
show_version()
|
|
212
290
|
sys.exit(0)
|
|
213
291
|
|
|
214
|
-
|
|
292
|
+
config_settings_path = get_config_file(args)
|
|
293
|
+
if not (os.path.isfile(config_settings_path) and os.access(config_settings_path, os.R_OK)):
|
|
294
|
+
if args.test_mode or os.getenv("PYTEST_CURRENT_TEST"):
|
|
295
|
+
args.config_file = config_settings_path
|
|
296
|
+
else:
|
|
297
|
+
print(f"Config file {config_settings_path} must exist and be readable.", file=stderr)
|
|
298
|
+
raise SystemExit(127)
|
|
299
|
+
args.config_file = config_settings_path
|
|
300
|
+
|
|
301
|
+
try:
|
|
302
|
+
config_settings = ConfigSettings(args.config_file)
|
|
303
|
+
except Exception as exc:
|
|
304
|
+
msg = f"Config error: {exc}"
|
|
305
|
+
print(msg, file=stderr)
|
|
306
|
+
ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
|
|
307
|
+
send_discord_message(f"{ts} - cleanup: FAILURE - {msg}")
|
|
308
|
+
sys.exit(127)
|
|
215
309
|
|
|
216
310
|
start_time=int(time())
|
|
217
311
|
|
|
218
312
|
# command_output_log = os.path.join(config_settings.logfile_location.removesuffix("dar-backup.log"), "dar-backup-commands.log")
|
|
219
313
|
command_output_log = config_settings.logfile_location.replace("dar-backup.log", "dar-backup-commands.log")
|
|
220
|
-
logger = setup_logging(
|
|
314
|
+
logger = setup_logging(
|
|
315
|
+
config_settings.logfile_location,
|
|
316
|
+
command_output_log,
|
|
317
|
+
args.log_level,
|
|
318
|
+
args.log_stdout,
|
|
319
|
+
logfile_max_bytes=config_settings.logfile_max_bytes,
|
|
320
|
+
logfile_backup_count=config_settings.logfile_backup_count,
|
|
321
|
+
trace_log_max_bytes=getattr(config_settings, "trace_log_max_bytes", 10485760),
|
|
322
|
+
trace_log_backup_count=getattr(config_settings, "trace_log_backup_count", 1)
|
|
323
|
+
)
|
|
221
324
|
command_logger = get_logger(command_output_logger = True)
|
|
222
|
-
runner = CommandRunner(
|
|
325
|
+
runner = CommandRunner(
|
|
326
|
+
logger=logger,
|
|
327
|
+
command_logger=command_logger,
|
|
328
|
+
default_capture_limit_bytes=getattr(config_settings, "command_capture_max_bytes", None)
|
|
329
|
+
)
|
|
223
330
|
|
|
224
331
|
start_msgs: List[Tuple[str, str]] = []
|
|
225
332
|
|
|
226
333
|
start_msgs.append((f"{show_scriptname()}:", about.__version__))
|
|
227
334
|
|
|
228
|
-
logger.info(f"START TIME: {start_time}")
|
|
229
335
|
logger.debug(f"Command line: {get_invocation_command_line()}")
|
|
230
336
|
logger.debug(f"`args`:\n{args}")
|
|
231
337
|
logger.debug(f"`config_settings`:\n{config_settings}")
|
|
@@ -239,52 +345,95 @@ def main():
|
|
|
239
345
|
args.verbose and start_msgs.append(("Logfile backup count:", config_settings.logfile_backup_count))
|
|
240
346
|
args.verbose and start_msgs.append(("--alternate-archive-dir:", args.alternate_archive_dir))
|
|
241
347
|
args.verbose and start_msgs.append(("--cleanup-specific-archives:", args.cleanup_specific_archives))
|
|
348
|
+
args.verbose and start_msgs.append(("--dry-run:", args.dry_run))
|
|
242
349
|
|
|
243
350
|
dangerous_keywords = ["--cleanup", "_FULL_"] # TODO: add more dangerous keywords
|
|
244
351
|
print_aligned_settings(start_msgs, highlight_keywords=dangerous_keywords, quiet=not args.verbose)
|
|
245
352
|
|
|
246
353
|
# run PREREQ scripts
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
354
|
+
try:
|
|
355
|
+
requirements('PREREQ', config_settings)
|
|
356
|
+
except Exception as exc:
|
|
357
|
+
msg = f"PREREQ failed: {exc}"
|
|
358
|
+
logger.error(msg)
|
|
359
|
+
ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
|
|
360
|
+
send_discord_message(f"{ts} - cleanup: FAILURE - {msg}", config_settings=config_settings)
|
|
361
|
+
sys.exit(1)
|
|
362
|
+
|
|
363
|
+
try:
|
|
364
|
+
if args.alternate_archive_dir:
|
|
365
|
+
if not os.path.exists(args.alternate_archive_dir):
|
|
366
|
+
logger.error(f"Alternate archive directory does not exist: {args.alternate_archive_dir}, exiting")
|
|
367
|
+
sys.exit(1)
|
|
368
|
+
if not os.path.isdir(args.alternate_archive_dir):
|
|
369
|
+
logger.error(f"Alternate archive directory is not a directory, exiting")
|
|
370
|
+
sys.exit(1)
|
|
371
|
+
config_settings.backup_dir = args.alternate_archive_dir
|
|
372
|
+
|
|
373
|
+
if args.cleanup_specific_archives is None and args.test_mode:
|
|
374
|
+
logger.info("No --cleanup-specific-archives provided; skipping specific archive deletion in test mode.")
|
|
375
|
+
|
|
376
|
+
if args.cleanup_specific_archives or args.cleanup_specific_archives_list:
|
|
377
|
+
combined = []
|
|
378
|
+
if args.cleanup_specific_archives:
|
|
379
|
+
combined.extend(args.cleanup_specific_archives.split(','))
|
|
380
|
+
combined.extend(args.cleanup_specific_archives_list or [])
|
|
381
|
+
archive_names = [name.strip() for name in combined if name.strip()]
|
|
382
|
+
logger.info(f"Cleaning up specific archives: {', '.join(archive_names)}")
|
|
383
|
+
for archive_name in archive_names:
|
|
384
|
+
if not is_archive_name_allowed(archive_name):
|
|
385
|
+
logger.error(f"Refusing unsafe archive name: {archive_name}")
|
|
267
386
|
continue
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
backup_definitions.append(args.backup_definition)
|
|
387
|
+
if "_FULL_" in archive_name:
|
|
388
|
+
if not confirm_full_archive_deletion(archive_name, args.test_mode):
|
|
389
|
+
continue
|
|
390
|
+
archive_path = os.path.join(config_settings.backup_dir, archive_name.strip())
|
|
391
|
+
logger.info(f"Deleting archive: {archive_path}")
|
|
392
|
+
delete_archive(config_settings.backup_dir, archive_name.strip(), args, config_settings)
|
|
393
|
+
elif args.list:
|
|
394
|
+
list_backups(config_settings.backup_dir, args.backup_definition)
|
|
277
395
|
else:
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
396
|
+
backup_definitions = []
|
|
397
|
+
if args.backup_definition:
|
|
398
|
+
backup_definitions.append(args.backup_definition)
|
|
399
|
+
else:
|
|
400
|
+
for root, _, files in os.walk(config_settings.backup_d_dir):
|
|
401
|
+
for file in files:
|
|
402
|
+
backup_definitions.append(file.split('.')[0])
|
|
403
|
+
|
|
404
|
+
for definition in backup_definitions:
|
|
405
|
+
delete_old_backups(
|
|
406
|
+
config_settings.backup_dir,
|
|
407
|
+
config_settings.diff_age,
|
|
408
|
+
'DIFF',
|
|
409
|
+
args,
|
|
410
|
+
backup_definition=definition,
|
|
411
|
+
config_settings=config_settings
|
|
412
|
+
)
|
|
413
|
+
delete_old_backups(
|
|
414
|
+
config_settings.backup_dir,
|
|
415
|
+
config_settings.incr_age,
|
|
416
|
+
'INCR',
|
|
417
|
+
args,
|
|
418
|
+
backup_definition=definition,
|
|
419
|
+
config_settings=config_settings
|
|
420
|
+
)
|
|
421
|
+
except Exception as e:
|
|
422
|
+
msg = f"Unexpected error during cleanup: {e}"
|
|
423
|
+
logger.error(msg, exc_info=True)
|
|
424
|
+
ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
|
|
425
|
+
send_discord_message(f"{ts} - cleanup: FAILURE - {msg}", config_settings=config_settings)
|
|
426
|
+
sys.exit(1)
|
|
285
427
|
|
|
286
428
|
# run POST scripts
|
|
287
|
-
|
|
429
|
+
try:
|
|
430
|
+
requirements('POSTREQ', config_settings)
|
|
431
|
+
except Exception as exc:
|
|
432
|
+
msg = f"POSTREQ failed: {exc}"
|
|
433
|
+
logger.error(msg)
|
|
434
|
+
ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
|
|
435
|
+
send_discord_message(f"{ts} - cleanup: FAILURE - {msg}", config_settings=config_settings)
|
|
436
|
+
sys.exit(1)
|
|
288
437
|
|
|
289
438
|
|
|
290
439
|
end_time=int(time())
|