cli-backup 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli_backup/__init__.py +1 -0
- cli_backup/cli.py +209 -0
- cli_backup/config.py +68 -0
- cli_backup/cpu_priority.py +21 -0
- cli_backup/files.py +216 -0
- cli_backup/mysql.py +109 -0
- cli_backup/redis_backup.py +24 -0
- cli_backup/rehearsal.py +79 -0
- cli_backup/scheduled.py +616 -0
- cli_backup/storage.py +386 -0
- cli_backup-0.1.0.dist-info/METADATA +119 -0
- cli_backup-0.1.0.dist-info/RECORD +16 -0
- cli_backup-0.1.0.dist-info/WHEEL +5 -0
- cli_backup-0.1.0.dist-info/entry_points.txt +2 -0
- cli_backup-0.1.0.dist-info/licenses/LICENSE +21 -0
- cli_backup-0.1.0.dist-info/top_level.txt +1 -0
cli_backup/storage.py
ADDED
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
"""Manifest-last backup storage."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import secrets
|
|
9
|
+
import shutil
|
|
10
|
+
import stat
|
|
11
|
+
import sys
|
|
12
|
+
import tempfile
|
|
13
|
+
from dataclasses import asdict, dataclass
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import BinaryIO
|
|
17
|
+
from .config import validate_project
|
|
18
|
+
|
|
19
|
+
_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
|
|
20
|
+
MULTIPART_THRESHOLD = 5 * 1024**3
|
|
21
|
+
MULTIPART_PART_SIZE = 64 * 1024**2
|
|
22
|
+
S3_RANGE_SIZE = 64 * 1024**2
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def valid_name(value: str) -> str:
|
|
26
|
+
if not _NAME.fullmatch(value) or value in (".", ".."):
|
|
27
|
+
raise ValueError("invalid artifact name")
|
|
28
|
+
return value
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def digest(stream: BinaryIO) -> tuple[str, int]:
|
|
32
|
+
h = hashlib.sha256()
|
|
33
|
+
size = 0
|
|
34
|
+
while data := stream.read(1024 * 1024):
|
|
35
|
+
size += len(data)
|
|
36
|
+
h.update(data)
|
|
37
|
+
return h.hexdigest(), size
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class Manifest:
|
|
42
|
+
version: int
|
|
43
|
+
project: str
|
|
44
|
+
source: str
|
|
45
|
+
artifact: str
|
|
46
|
+
timestamp: str
|
|
47
|
+
size: int
|
|
48
|
+
sha256: str
|
|
49
|
+
object_counts: str | None = None
|
|
50
|
+
|
|
51
|
+
def validate(self) -> None:
|
|
52
|
+
if type(self.version) is not int or self.version != 1:
|
|
53
|
+
raise ValueError("unsupported manifest version")
|
|
54
|
+
validate_project(self.project)
|
|
55
|
+
valid_name(self.source)
|
|
56
|
+
valid_name(self.artifact)
|
|
57
|
+
if type(self.size) is not int or self.size < 0 or not isinstance(self.sha256, str) or not re.fullmatch(r"[0-9a-f]{64}", self.sha256):
|
|
58
|
+
raise ValueError("invalid manifest checksum or size")
|
|
59
|
+
if self.object_counts is not None and (not isinstance(self.object_counts, str) or not re.fullmatch(r'\d+(?:\s+\d+){4}', self.object_counts)):
|
|
60
|
+
raise ValueError("invalid object counts")
|
|
61
|
+
if not isinstance(self.timestamp, str):
|
|
62
|
+
raise ValueError("invalid manifest timestamp")
|
|
63
|
+
try:
|
|
64
|
+
timestamp = datetime.fromisoformat(self.timestamp.replace('Z', '+00:00'))
|
|
65
|
+
except ValueError as exc:
|
|
66
|
+
raise ValueError("invalid manifest timestamp") from exc
|
|
67
|
+
if timestamp.utcoffset() is None:
|
|
68
|
+
raise ValueError("manifest timestamp must include a timezone")
|
|
69
|
+
|
|
70
|
+
def encode(self) -> bytes:
|
|
71
|
+
self.validate()
|
|
72
|
+
return json.dumps(asdict(self), sort_keys=True).encode() + b"\n"
|
|
73
|
+
|
|
74
|
+
@classmethod
|
|
75
|
+
def decode(cls, data: bytes) -> Manifest:
|
|
76
|
+
if len(data) > 65536:
|
|
77
|
+
raise ValueError("manifest too large")
|
|
78
|
+
try:
|
|
79
|
+
obj = cls(**json.loads(data))
|
|
80
|
+
obj.validate()
|
|
81
|
+
return obj
|
|
82
|
+
except (TypeError, json.JSONDecodeError) as exc:
|
|
83
|
+
raise ValueError("invalid manifest") from exc
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def make_manifest(path: Path, project: str, source: str, name: str, timestamp: str, object_counts: str | None = None) -> Manifest:
|
|
87
|
+
with path.open("rb") as stream:
|
|
88
|
+
sha, size = digest(stream)
|
|
89
|
+
result = Manifest(1, project, source, name, timestamp, size, sha, object_counts)
|
|
90
|
+
result.validate()
|
|
91
|
+
return result
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class LocalStorage:
|
|
95
|
+
def __init__(self, root: Path):
|
|
96
|
+
self.root = root
|
|
97
|
+
|
|
98
|
+
def directory(self, manifest: Manifest) -> Path:
|
|
99
|
+
manifest.validate()
|
|
100
|
+
return self.root / manifest.project / manifest.source
|
|
101
|
+
|
|
102
|
+
def put(self, path: Path, manifest: Manifest) -> None:
|
|
103
|
+
folder = self.directory(manifest)
|
|
104
|
+
folder.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
105
|
+
dest = folder / manifest.artifact
|
|
106
|
+
metadata = folder / (manifest.artifact + ".json")
|
|
107
|
+
if dest.exists() or metadata.exists():
|
|
108
|
+
raise FileExistsError(dest)
|
|
109
|
+
fd, temp = tempfile.mkstemp(prefix=".partial-", dir=folder)
|
|
110
|
+
try:
|
|
111
|
+
with os.fdopen(fd, "wb") as out, path.open("rb") as source:
|
|
112
|
+
shutil.copyfileobj(source, out)
|
|
113
|
+
out.flush()
|
|
114
|
+
os.fsync(out.fileno())
|
|
115
|
+
with open(temp, "rb") as source:
|
|
116
|
+
if digest(source) != (manifest.sha256, manifest.size):
|
|
117
|
+
raise ValueError("artifact changed during copy")
|
|
118
|
+
os.link(temp, dest)
|
|
119
|
+
meta_fd, meta_temp = tempfile.mkstemp(prefix=".manifest-", dir=folder)
|
|
120
|
+
try:
|
|
121
|
+
with os.fdopen(meta_fd, "wb") as out:
|
|
122
|
+
out.write(manifest.encode())
|
|
123
|
+
out.flush()
|
|
124
|
+
os.fsync(out.fileno())
|
|
125
|
+
os.link(meta_temp, metadata)
|
|
126
|
+
finally:
|
|
127
|
+
os.unlink(meta_temp)
|
|
128
|
+
finally:
|
|
129
|
+
os.unlink(temp)
|
|
130
|
+
|
|
131
|
+
def list(self, project: str, source: str) -> list[Manifest]:
|
|
132
|
+
folder = self.root / validate_project(project) / valid_name(source)
|
|
133
|
+
if not folder.exists():
|
|
134
|
+
return []
|
|
135
|
+
manifests = []
|
|
136
|
+
for path in folder.glob("*.json"):
|
|
137
|
+
try:
|
|
138
|
+
m = Manifest.decode(path.read_bytes())
|
|
139
|
+
if m.project == project and m.source == source and path.name == m.artifact + ".json" and (folder / m.artifact).is_file():
|
|
140
|
+
manifests.append(m)
|
|
141
|
+
except (OSError, ValueError):
|
|
142
|
+
continue
|
|
143
|
+
return sorted(manifests, key=lambda m: datetime.fromisoformat(m.timestamp.replace('Z', '+00:00')))
|
|
144
|
+
|
|
145
|
+
def verify(self, manifest: Manifest) -> bool:
|
|
146
|
+
with (self.directory(manifest) / manifest.artifact).open("rb") as stream:
|
|
147
|
+
return digest(stream) == (manifest.sha256, manifest.size)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class S3Storage:
|
|
151
|
+
def __init__(self, client, bucket: str, prefix: str = ""):
|
|
152
|
+
self.client, self.bucket, self.prefix = client, bucket, prefix.strip("/")
|
|
153
|
+
|
|
154
|
+
def key(self, manifest: Manifest) -> str:
|
|
155
|
+
manifest.validate()
|
|
156
|
+
return "/".join(part for part in (self.prefix, manifest.project, manifest.source, manifest.artifact) if part)
|
|
157
|
+
|
|
158
|
+
def put(self, path: Path, manifest: Manifest) -> None:
|
|
159
|
+
key = self.key(manifest)
|
|
160
|
+
with path.open("rb") as data:
|
|
161
|
+
if digest(data) != (manifest.sha256, manifest.size):
|
|
162
|
+
raise ValueError("artifact changed before upload")
|
|
163
|
+
data.seek(0)
|
|
164
|
+
if manifest.size <= MULTIPART_THRESHOLD:
|
|
165
|
+
self.client.put_object(Bucket=self.bucket, Key=key, Body=data, IfNoneMatch="*")
|
|
166
|
+
else:
|
|
167
|
+
self._multipart_upload(data, key)
|
|
168
|
+
if not self.verify(manifest):
|
|
169
|
+
raise ValueError("S3 verification failed")
|
|
170
|
+
self.client.put_object(Bucket=self.bucket, Key=key + ".json", Body=manifest.encode(), IfNoneMatch="*")
|
|
171
|
+
|
|
172
|
+
def _multipart_upload(self, data: BinaryIO, key: str) -> None:
|
|
173
|
+
upload = self.client.create_multipart_upload(Bucket=self.bucket, Key=key)['UploadId']
|
|
174
|
+
parts = []
|
|
175
|
+
try:
|
|
176
|
+
while chunk := data.read(MULTIPART_PART_SIZE):
|
|
177
|
+
number = len(parts) + 1
|
|
178
|
+
if number > 10000:
|
|
179
|
+
raise ValueError('backup is too large for multipart upload')
|
|
180
|
+
response = self.client.upload_part(Bucket=self.bucket, Key=key, UploadId=upload,
|
|
181
|
+
PartNumber=number, Body=chunk)
|
|
182
|
+
parts.append({'PartNumber': number, 'ETag': response['ETag']})
|
|
183
|
+
self.client.complete_multipart_upload(Bucket=self.bucket, Key=key, UploadId=upload,
|
|
184
|
+
MultipartUpload={'Parts': parts}, IfNoneMatch='*')
|
|
185
|
+
except Exception:
|
|
186
|
+
try:
|
|
187
|
+
self.client.abort_multipart_upload(Bucket=self.bucket, Key=key, UploadId=upload)
|
|
188
|
+
except Exception:
|
|
189
|
+
pass
|
|
190
|
+
raise
|
|
191
|
+
|
|
192
|
+
def verify(self, manifest: Manifest) -> bool:
|
|
193
|
+
if manifest.size > MULTIPART_THRESHOLD:
|
|
194
|
+
try:
|
|
195
|
+
s3_get_ranges(self, manifest)
|
|
196
|
+
return True
|
|
197
|
+
except ValueError:
|
|
198
|
+
return False
|
|
199
|
+
stream = self.client.get_object(Bucket=self.bucket, Key=self.key(manifest))["Body"]
|
|
200
|
+
try:
|
|
201
|
+
return digest(stream) == (manifest.sha256, manifest.size)
|
|
202
|
+
finally:
|
|
203
|
+
stream.close()
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def copy_verified(stream: BinaryIO, target: BinaryIO, manifest: Manifest) -> None:
|
|
207
|
+
"""Copy a stream and reject incomplete or altered data before use."""
|
|
208
|
+
checksum = hashlib.sha256()
|
|
209
|
+
size = 0
|
|
210
|
+
while chunk := stream.read(1024 * 1024):
|
|
211
|
+
target.write(chunk)
|
|
212
|
+
checksum.update(chunk)
|
|
213
|
+
size += len(chunk)
|
|
214
|
+
if (checksum.hexdigest(), size) != (manifest.sha256, manifest.size):
|
|
215
|
+
raise ValueError("backup checksum or size mismatch")
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def local_get(store: LocalStorage, manifest: Manifest, target: BinaryIO) -> None:
|
|
219
|
+
with (store.directory(manifest) / manifest.artifact).open("rb") as stream:
|
|
220
|
+
copy_verified(stream, target, manifest)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def s3_list(store: S3Storage, project: str, source: str) -> list[Manifest]:
|
|
224
|
+
project, source = validate_project(project), valid_name(source)
|
|
225
|
+
prefix = "/".join(p for p in (store.prefix, project, source, "") if p) + "/"
|
|
226
|
+
found = []
|
|
227
|
+
for page in store.client.get_paginator("list_objects_v2").paginate(Bucket=store.bucket, Prefix=prefix):
|
|
228
|
+
for item in page.get("Contents", []):
|
|
229
|
+
key = item["Key"]
|
|
230
|
+
if not key.endswith(".json") or "/" in key[len(prefix):]:
|
|
231
|
+
continue
|
|
232
|
+
body = store.client.get_object(Bucket=store.bucket, Key=key)["Body"]
|
|
233
|
+
try:
|
|
234
|
+
manifest = Manifest.decode(body.read(65537))
|
|
235
|
+
except (ValueError, TypeError, UnicodeError):
|
|
236
|
+
continue
|
|
237
|
+
finally:
|
|
238
|
+
body.close()
|
|
239
|
+
if manifest.project == project and manifest.source == source and store.key(manifest) + ".json" == key:
|
|
240
|
+
from botocore.exceptions import ClientError
|
|
241
|
+
try:
|
|
242
|
+
info = store.client.head_object(Bucket=store.bucket, Key=store.key(manifest))
|
|
243
|
+
except ClientError as exc:
|
|
244
|
+
if exc.response.get('Error', {}).get('Code') not in ('404', 'NoSuchKey', 'NotFound'):
|
|
245
|
+
raise
|
|
246
|
+
print(f'incomplete backup: {manifest.artifact} (data missing)', file=sys.stderr)
|
|
247
|
+
continue
|
|
248
|
+
if info.get('ContentLength') != manifest.size:
|
|
249
|
+
print(f'incomplete backup: {manifest.artifact} (size mismatch)', file=sys.stderr)
|
|
250
|
+
continue
|
|
251
|
+
found.append(manifest)
|
|
252
|
+
return sorted(found, key=lambda m: datetime.fromisoformat(m.timestamp.replace('Z', '+00:00')))
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def s3_get_ranges(store: S3Storage, manifest: Manifest, target: BinaryIO | None = None) -> None:
|
|
256
|
+
"""Read bounded ranges; retry a failed range without publishing partial bytes."""
|
|
257
|
+
manifest.validate()
|
|
258
|
+
checksum = hashlib.sha256()
|
|
259
|
+
for offset in range(0, manifest.size, S3_RANGE_SIZE):
|
|
260
|
+
end = min(offset + S3_RANGE_SIZE, manifest.size) - 1
|
|
261
|
+
for attempt in range(3):
|
|
262
|
+
try:
|
|
263
|
+
response = store.client.get_object(Bucket=store.bucket, Key=store.key(manifest),
|
|
264
|
+
Range=f'bytes={offset}-{end}')
|
|
265
|
+
stream = response['Body']
|
|
266
|
+
try:
|
|
267
|
+
data = stream.read(end - offset + 2)
|
|
268
|
+
finally:
|
|
269
|
+
stream.close()
|
|
270
|
+
if response.get('ContentRange') != f'bytes {offset}-{end}/{manifest.size}' or len(data) != end - offset + 1:
|
|
271
|
+
raise ValueError('invalid S3 range response')
|
|
272
|
+
break
|
|
273
|
+
except Exception:
|
|
274
|
+
if attempt == 2:
|
|
275
|
+
raise
|
|
276
|
+
checksum.update(data)
|
|
277
|
+
if target is not None:
|
|
278
|
+
target.write(data)
|
|
279
|
+
if checksum.hexdigest() != manifest.sha256:
|
|
280
|
+
raise ValueError('artifact checksum mismatch')
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def s3_get(store: S3Storage, manifest: Manifest, target: BinaryIO) -> None:
|
|
284
|
+
if manifest.size > MULTIPART_THRESHOLD:
|
|
285
|
+
s3_get_ranges(store, manifest, target)
|
|
286
|
+
return
|
|
287
|
+
stream = store.client.get_object(Bucket=store.bucket, Key=store.key(manifest))["Body"]
|
|
288
|
+
try:
|
|
289
|
+
copy_verified(stream, target, manifest)
|
|
290
|
+
finally:
|
|
291
|
+
stream.close()
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def prune_local(store: LocalStorage, project: str, source: str, days: int, now) -> int:
|
|
295
|
+
"""Remove indexed expired local copies without following directory symlinks."""
|
|
296
|
+
from datetime import datetime, timedelta, timezone
|
|
297
|
+
if days < 1:
|
|
298
|
+
raise ValueError('local retention days must be positive')
|
|
299
|
+
cutoff = now.astimezone(timezone.utc) - timedelta(days=days)
|
|
300
|
+
folder = (store.root / validate_project(project) / valid_name(source)).absolute()
|
|
301
|
+
fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY)
|
|
302
|
+
try:
|
|
303
|
+
for part in folder.parts[1:]:
|
|
304
|
+
child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
|
305
|
+
os.close(fd)
|
|
306
|
+
fd = child
|
|
307
|
+
removed = 0
|
|
308
|
+
for name in os.listdir(fd):
|
|
309
|
+
if not name.endswith('.json'):
|
|
310
|
+
continue
|
|
311
|
+
try:
|
|
312
|
+
valid_name(name[:-5])
|
|
313
|
+
meta_fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=fd)
|
|
314
|
+
with os.fdopen(meta_fd, 'rb') as metadata:
|
|
315
|
+
if not stat.S_ISREG(os.fstat(metadata.fileno()).st_mode):
|
|
316
|
+
continue
|
|
317
|
+
manifest = Manifest.decode(metadata.read(65537))
|
|
318
|
+
if (manifest.project, manifest.source, manifest.artifact + '.json') != (project, source, name):
|
|
319
|
+
continue
|
|
320
|
+
created = datetime.fromisoformat(manifest.timestamp.replace('Z', '+00:00'))
|
|
321
|
+
if created.tzinfo is None or created >= cutoff:
|
|
322
|
+
continue
|
|
323
|
+
artifact_stat = os.stat(manifest.artifact, dir_fd=fd, follow_symlinks=False)
|
|
324
|
+
if not stat.S_ISREG(artifact_stat.st_mode):
|
|
325
|
+
raise ValueError('refusing to remove non-regular backup')
|
|
326
|
+
os.unlink(name, dir_fd=fd)
|
|
327
|
+
os.unlink(manifest.artifact, dir_fd=fd)
|
|
328
|
+
removed += 1
|
|
329
|
+
except (FileNotFoundError, ValueError, TypeError):
|
|
330
|
+
continue
|
|
331
|
+
return removed
|
|
332
|
+
finally:
|
|
333
|
+
os.close(fd)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def write_private_copy(source: BinaryIO, destination: Path) -> None:
|
|
337
|
+
"""Publish verified bytes privately through a pinned destination directory."""
|
|
338
|
+
parent = destination.parent
|
|
339
|
+
if destination.name in ('', '.', '..'):
|
|
340
|
+
raise ValueError('invalid destination name')
|
|
341
|
+
before = os.lstat(parent)
|
|
342
|
+
dirfd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
|
343
|
+
temp = '.cli-backup-download-' + secrets.token_hex(16)
|
|
344
|
+
created = False
|
|
345
|
+
published = False
|
|
346
|
+
try:
|
|
347
|
+
pinned = os.fstat(dirfd)
|
|
348
|
+
if (before.st_dev, before.st_ino) != (pinned.st_dev, pinned.st_ino):
|
|
349
|
+
raise OSError('destination directory changed')
|
|
350
|
+
writable_by_others = pinned.st_mode & (stat.S_IWGRP | stat.S_IWOTH)
|
|
351
|
+
if pinned.st_uid not in (os.geteuid(), 0) or (writable_by_others and not pinned.st_mode & stat.S_ISVTX):
|
|
352
|
+
raise OSError('destination directory is not protected from other users')
|
|
353
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC
|
|
354
|
+
fd = os.open(temp, flags, 0o600, dir_fd=dirfd)
|
|
355
|
+
created = True
|
|
356
|
+
with os.fdopen(fd, 'wb') as target:
|
|
357
|
+
os.fchmod(target.fileno(), 0o600)
|
|
358
|
+
info = os.fstat(target.fileno())
|
|
359
|
+
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) != 0o600:
|
|
360
|
+
raise OSError('invalid temporary file')
|
|
361
|
+
shutil.copyfileobj(source, target)
|
|
362
|
+
target.flush()
|
|
363
|
+
os.fsync(target.fileno())
|
|
364
|
+
os.link(temp, destination.name, src_dir_fd=dirfd, dst_dir_fd=dirfd, follow_symlinks=False)
|
|
365
|
+
published = True
|
|
366
|
+
linked = os.stat(destination.name, dir_fd=dirfd, follow_symlinks=False)
|
|
367
|
+
if (linked.st_dev, linked.st_ino) != (info.st_dev, info.st_ino):
|
|
368
|
+
raise OSError('temporary file changed during publication')
|
|
369
|
+
current = os.lstat(parent)
|
|
370
|
+
if (current.st_dev, current.st_ino) != (pinned.st_dev, pinned.st_ino):
|
|
371
|
+
raise OSError('destination directory changed')
|
|
372
|
+
os.fsync(dirfd)
|
|
373
|
+
except Exception:
|
|
374
|
+
if published:
|
|
375
|
+
try:
|
|
376
|
+
os.unlink(destination.name, dir_fd=dirfd)
|
|
377
|
+
os.fsync(dirfd)
|
|
378
|
+
except OSError:
|
|
379
|
+
pass
|
|
380
|
+
raise
|
|
381
|
+
finally:
|
|
382
|
+
try:
|
|
383
|
+
if created:
|
|
384
|
+
os.unlink(temp, dir_fd=dirfd)
|
|
385
|
+
finally:
|
|
386
|
+
os.close(dirfd)
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cli-backup
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Host backup CLI for database snapshots and directory archives
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/XakkiDev/cli-backup
|
|
7
|
+
Project-URL: Issues, https://github.com/XakkiDev/cli-backup/issues
|
|
8
|
+
Keywords: backup,mysql,mariadb,sqlite,s3
|
|
9
|
+
Requires-Python: >=3.11
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: boto3==1.43.127
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: ruff==0.6.9; extra == "dev"
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# cli-backup
|
|
18
|
+
|
|
19
|
+
A Python command-line tool for full database backups and directory archives, stored locally, in S3-compatible object storage, or both. Use it directly or include the shared Makefile in multiple projects on the same host.
|
|
20
|
+
|
|
21
|
+
[](https://github.com/XakkiDev/cli-backup/actions/workflows/ci.yml) · [MIT license](LICENSE)
|
|
22
|
+
|
|
23
|
+
## Capabilities
|
|
24
|
+
|
|
25
|
+
| Interface | Sources | Features |
|
|
26
|
+
| --- | --- | --- |
|
|
27
|
+
| `scheduled` and `share.Makefile` | MariaDB, MySQL, SQLite | Date-based daily, weekly and monthly snapshots; Docker or Podman containers, TCP, or SQLite online backup |
|
|
28
|
+
| Generic commands | MariaDB/MySQL, Redis, directories | Full dumps, Redis RDB snapshots, gzip tar archives and exclusion patterns |
|
|
29
|
+
|
|
30
|
+
The CLI runs on the host, supports rootless Docker and Podman for scheduled database backups, and does not require a separate backup service in a project's Compose file. Scheduled Compose targets are discovered from container labels. PostgreSQL, MongoDB and incremental backups are not implemented.
|
|
31
|
+
|
|
32
|
+
**Protect backup data and credentials.** Backups are not encrypted by this tool. Use private local paths, protected environment files and encrypted transport. Backups are not a substitute for an authorized recovery exercise. This CLI does not create buckets, change lifecycle policies, or schedule jobs.
|
|
33
|
+
|
|
34
|
+
## Installation
|
|
35
|
+
|
|
36
|
+
Python 3.11+ on a POSIX host is required. Client binaries are required for direct database or Redis connections. The selected Docker/Podman runtime is required for container sources.
|
|
37
|
+
|
|
38
|
+
```sh
|
|
39
|
+
git clone https://github.com/XakkiDev/cli-backup.git
|
|
40
|
+
cd cli-backup
|
|
41
|
+
make venv install
|
|
42
|
+
.venv/bin/cli-backup --help
|
|
43
|
+
.venv/bin/cli-backup scheduled --help
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
For a user-wide executable, `pipx install .` installs from the checkout. `share.Makefile` is supplied in the repository checkout rather than the Python wheel: keep or download a trusted checkout to use the shared Make targets. On systems without `ensurepip`, install the matching Python venv package first.
|
|
47
|
+
|
|
48
|
+
The Composer package `xakki/cli-backup` distributes the source tree and agent skills for Composer-based projects. Composer does not install Python dependencies or provide a ready-to-run CLI: after `composer require xakki/cli-backup`, install the Python project from `vendor/xakki/cli-backup` with `pip` or `pipx` and include `vendor/xakki/cli-backup/share.Makefile` when needed. The `extra.skills.source` metadata identifies the agent skills for compatible Composer skill managers.
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
## Scheduled database backups
|
|
52
|
+
|
|
53
|
+
Include `share.Makefile` from your checkout. For a Compose service:
|
|
54
|
+
|
|
55
|
+
```make
|
|
56
|
+
CLI_BACKUP_COMPOSE_PROJECT_NAME := example
|
|
57
|
+
CLI_BACKUP_DB_CONTAINER := mariadb
|
|
58
|
+
include /path/to/cli-backup/share.Makefile
|
|
59
|
+
export CLI_BACKUP_COMPOSE_PROJECT_NAME CLI_BACKUP_DB_CONTAINER
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Set `CLI_BACKUP_PROJECT_NAME` if the storage namespace should differ from the Compose project. For a standalone container, set `CLI_BACKUP_PROJECT_NAME` and `CLI_BACKUP_DB_CONTAINER` to its name or ID. Set `CLI_BACKUP_CONTAINER_RUNTIME=podman` to use local rootless Podman; Docker is the default and uses the selected Docker context. For TCP use `CLI_BACKUP_PROJECT_NAME`, `CLI_BACKUP_DB_HOST`, optionally `CLI_BACKUP_DB_PORT`, `CLI_BACKUP_DB_NAME`, `CLI_BACKUP_DB_USER` and `CLI_BACKUP_DB_PASSWORD`.
|
|
63
|
+
|
|
64
|
+
Container mode defaults to root, using `MARIADB_ROOT_PASSWORD` and `MARIADB_DATABASE` from the container; MySQL equivalents are also supported. Set a user and password together to override root. `CLI_BACKUP_DB_NAME` accepts comma-separated database names; each database produces a separate file. Credentials must be supplied in the environment or a protected project file, never as command-line arguments.
|
|
65
|
+
|
|
66
|
+
For SQLite, set `CLI_BACKUP_PROJECT_NAME`, `CLI_BACKUP_DB_SERVICE_NAME=sqlite`, and an absolute `CLI_BACKUP_DB_PATH`. Do not set container, host or Compose variables. The SQLite online backup API includes committed WAL transactions without stopping the writer. Automatic SQLite restoration is not implemented.
|
|
67
|
+
|
|
68
|
+
```sh
|
|
69
|
+
make cli-backup@config-check
|
|
70
|
+
make cli-backup@backup
|
|
71
|
+
make cli-backup@list path=daily where=local
|
|
72
|
+
make cli-backup@verify path=daily where=local file=2026-01-01.example.sql.gz
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Additional targets `cli-backup@restore-test` and `cli-backup@restore` exist for MariaDB/MySQL. A live restore is destructive, requires an explicit file and interactive confirmation, and must target the correct stopped-writer environment. A restore rehearsal creates a disposable isolated container and should run only with appropriate authorization. For an unavailable source container, set `CLI_BACKUP_TEST_MYSQL_IMAGE` to an appropriate disposable test image before a rehearsal. Use `db=<name>` to filter when needed. Source-independent `list` and `verify` work with a stopped database container.
|
|
76
|
+
|
|
77
|
+
### Storage and schedule
|
|
78
|
+
|
|
79
|
+
Set `CLI_BACKUP_LOCAL_ROOT` for local storage and/or the S3 variables `CLI_BACKUP_S3_BUCKET`, `CLI_BACKUP_S3_ENDPOINT`, `CLI_BACKUP_S3_REGION`, `CLI_BACKUP_S3_KEY`, `CLI_BACKUP_S3_SECRET`, and optionally `CLI_BACKUP_S3_PREFIX`. Project environment values override global environment values; explicit scheduled CLI options override project values. `--project-env` accepts literal assignments and does not execute shell code. Period keys are relative to the optional S3 prefix:
|
|
80
|
+
|
|
81
|
+
```text
|
|
82
|
+
daily/<project>/<source>/YYYY-MM-DD.<database>.sql.gz
|
|
83
|
+
weekly/<project>/<source>/YYYY-MM-DD.<database>.sql.gz
|
|
84
|
+
monthly/<project>/<source>/YYYY-MM-DD.<database>.sql.gz
|
|
85
|
+
.state/<project>/<source>/... # schedule metadata
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
SQLite uses `.sqlite.gz`. `CLI_BACKUP_PERIOD_DAILY` and `CLI_BACKUP_PERIOD_MONTHLY` default to `true`; `CLI_BACKUP_PERIOD_WEEKLY` defaults to `false`. With daily enabled, beginning on Wednesday or the third day of the month, the latest eligible daily copy from the preceding week or month is copied to the relevant period if missing. When both select the same Sunday, weekly chooses an earlier snapshot. With daily disabled, a new weekly or monthly snapshot is created on or after the threshold if the current period is missing. Run the command from an external scheduler, normally daily, so missed attempts can be retried. A completed period may not create a new snapshot on a repeated invocation.
|
|
89
|
+
|
|
90
|
+
`CLI_BACKUP_LOCAL_DAYS` sets local retention, with minimum retention for period selection and idempotency. The object storage service controls S3 retention. Required destinations are attempted independently, and errors cause a nonzero exit. S3 publishing reads and verifies the complete uploaded object before publishing its manifest. The default CPU scheduling priority is `nice 10` for the CLI and container dump client; configure `CLI_BACKUP_CPU_NICE` from 0 to 19. This does not limit the database server's CPU or I/O.
|
|
91
|
+
|
|
92
|
+
## Generic CLI
|
|
93
|
+
|
|
94
|
+
Run `cli-backup --help` for global options and commands. Generic CLI options precede the subcommand. Its configuration uses `CLI_BACKUP_PROJECT` and `CLI_BACKUP_DB_MODE` (`compose`, `container`, or `tcp`), which are distinct from the scheduled interface's project and connection variables.
|
|
95
|
+
|
|
96
|
+
```sh
|
|
97
|
+
cli-backup backup mysql
|
|
98
|
+
cli-backup backup redis
|
|
99
|
+
cli-backup backup files --directory /srv/example --exclude '*.tmp' --exclude 'cache/*'
|
|
100
|
+
cli-backup list files --storage local
|
|
101
|
+
cli-backup verify files --storage local --file NAME
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Directory archives are full scans, not point-in-time filesystem snapshots. Exclusion patterns use Python `fnmatch` relative to the source; `*` can match `/`. Symlinks are archived without following them, hard links are archived separately, and unsafe escaping links or special files must be excluded. Extraction requires a new absolute destination and rejects unsafe paths; permissions are restored without setuid/setgid bits, and ownership is not restored. See `cli-backup restore --help` before recovery.
|
|
105
|
+
|
|
106
|
+
MariaDB/MySQL dumps use `--single-transaction`; coordinate schema changes and nontransactional tables. Object-count comparisons do not prove row-level equivalence. The local and S3 files are sensitive; ensure sufficient free temporary space and protect the destination.
|
|
107
|
+
|
|
108
|
+
## Development and AI agent skills
|
|
109
|
+
|
|
110
|
+
```sh
|
|
111
|
+
make venv install
|
|
112
|
+
make test lint
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
CI runs unit tests on Python 3.11–3.13, lint, build and installed-wheel smoke tests. `make test-integration` requires explicitly selected disposable images and may perform database restoration; run it only when authorized. Project-portable skills for agents are in [`.agents/skills/`](.agents/skills/): `cli-backup-setup` covers integration and `cli-backup-develop` covers implementation and testing. They do not authorize storage administration, schedule changes, or restoration.
|
|
116
|
+
|
|
117
|
+
## License
|
|
118
|
+
|
|
119
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
cli_backup/__init__.py,sha256=hN9Q0GdHR-ffScFi5OYQw4sOX_ZDpdMDGa_rFX1cpc8,18
|
|
2
|
+
cli_backup/cli.py,sha256=SCp_BSjEmEBvEu_Yd3lbDNzUD_mTkvB3groZ9lh3DmM,11861
|
|
3
|
+
cli_backup/config.py,sha256=oVwJZekCk01OHwrKSvaJ0jmYKaNSKEEAEbQQMUb53lg,2664
|
|
4
|
+
cli_backup/cpu_priority.py,sha256=SapxQ-mqNEGpmskBSeZ5OrYXUmnD2mmHlaKX33yQHR4,687
|
|
5
|
+
cli_backup/files.py,sha256=68IPkK_osehcubXWAk1P7inRQDQ8tNh0X5CSNllLwDo,9573
|
|
6
|
+
cli_backup/mysql.py,sha256=03zk-clPJJwG7hAGpKosioLpHVuAU6Op-OkmNC-xTdo,5197
|
|
7
|
+
cli_backup/redis_backup.py,sha256=YNRgHytK9jHn7CXMzWUSEvwrOdRZ39Vu6H-qRcKJY8A,1090
|
|
8
|
+
cli_backup/rehearsal.py,sha256=My0DWB4QL9DaI2hJwZpfv4o-deTMMvgPKwUV73Gw5M8,4093
|
|
9
|
+
cli_backup/scheduled.py,sha256=WxTTqJU3xI8zch9ftQUChFZsmy7WYly48cHKFDxTDU8,31206
|
|
10
|
+
cli_backup/storage.py,sha256=1EaPyBa5evUtVaWCcLDQr7iOkPeYzaEFYO6jFOi1p38,16534
|
|
11
|
+
cli_backup-0.1.0.dist-info/licenses/LICENSE,sha256=LT_FggPDXG8pHihapLDPf5IkZoAXBYdF28hrxxf7sCI,1080
|
|
12
|
+
cli_backup-0.1.0.dist-info/METADATA,sha256=KRhjKZlilfKETVWMzcOX2YFh3vjXPypnnNdl_ZnQ9qA,9239
|
|
13
|
+
cli_backup-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
14
|
+
cli_backup-0.1.0.dist-info/entry_points.txt,sha256=IA3yRKrINolVcMFD4_8RB6crrNBWaO1GbCvQ2Pni1Ig,51
|
|
15
|
+
cli_backup-0.1.0.dist-info/top_level.txt,sha256=e45rnGwSfR33PilMDehreTSxQv4vO30aPPO5AS3U-SY,11
|
|
16
|
+
cli_backup-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 cli-backup contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cli_backup
|