vvz-file-backup 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- file_backup/__init__.py +60 -0
- file_backup/_archives.py +100 -0
- file_backup/_catalog.py +87 -0
- file_backup/_configuration.py +40 -0
- file_backup/_namespace.py +16 -0
- file_backup/_operations.py +356 -0
- file_backup/_resources.py +7 -0
- file_backup/_validation.py +82 -0
- file_backup/api/backup/__init__.py +15 -0
- file_backup/api/backup/execute/__init__.py +1 -0
- file_backup/api/backup/execute/implementation.py +12 -0
- file_backup/api/backup/execute/metadata.json +73 -0
- file_backup/api/backup/execute/schema.json +50 -0
- file_backup/api/backup/get_schema/__init__.py +1 -0
- file_backup/api/backup/get_schema/implementation.py +11 -0
- file_backup/api/backup/get_schema/metadata.json +53 -0
- file_backup/api/backup/get_schema/schema.json +6 -0
- file_backup/api/backup/metadata/__init__.py +1 -0
- file_backup/api/backup/metadata/implementation.py +11 -0
- file_backup/api/backup/metadata/metadata.json +53 -0
- file_backup/api/backup/metadata/schema.json +6 -0
- file_backup/api/help/__init__.py +15 -0
- file_backup/api/help/execute/__init__.py +1 -0
- file_backup/api/help/execute/implementation.py +66 -0
- file_backup/api/help/execute/metadata.json +34 -0
- file_backup/api/help/execute/schema.json +23 -0
- file_backup/api/help/get_schema/__init__.py +1 -0
- file_backup/api/help/get_schema/implementation.py +15 -0
- file_backup/api/help/get_schema/metadata.json +20 -0
- file_backup/api/help/get_schema/schema.json +6 -0
- file_backup/api/help/metadata/__init__.py +1 -0
- file_backup/api/help/metadata/implementation.py +15 -0
- file_backup/api/help/metadata/metadata.json +20 -0
- file_backup/api/help/metadata/schema.json +6 -0
- file_backup/api/info/USER_GUIDE.md +101 -0
- file_backup/api/info/__init__.py +15 -0
- file_backup/api/info/execute/__init__.py +1 -0
- file_backup/api/info/execute/implementation.py +79 -0
- file_backup/api/info/execute/metadata.json +34 -0
- file_backup/api/info/execute/schema.json +19 -0
- file_backup/api/info/get_schema/__init__.py +1 -0
- file_backup/api/info/get_schema/implementation.py +15 -0
- file_backup/api/info/get_schema/metadata.json +20 -0
- file_backup/api/info/get_schema/schema.json +6 -0
- file_backup/api/info/metadata/__init__.py +1 -0
- file_backup/api/info/metadata/implementation.py +15 -0
- file_backup/api/info/metadata/metadata.json +20 -0
- file_backup/api/info/metadata/schema.json +6 -0
- file_backup/api/inspect_point/__init__.py +15 -0
- file_backup/api/inspect_point/execute/__init__.py +1 -0
- file_backup/api/inspect_point/execute/implementation.py +12 -0
- file_backup/api/inspect_point/execute/metadata.json +64 -0
- file_backup/api/inspect_point/execute/schema.json +42 -0
- file_backup/api/inspect_point/get_schema/__init__.py +1 -0
- file_backup/api/inspect_point/get_schema/implementation.py +11 -0
- file_backup/api/inspect_point/get_schema/metadata.json +44 -0
- file_backup/api/inspect_point/get_schema/schema.json +6 -0
- file_backup/api/inspect_point/metadata/__init__.py +1 -0
- file_backup/api/inspect_point/metadata/implementation.py +11 -0
- file_backup/api/inspect_point/metadata/metadata.json +44 -0
- file_backup/api/inspect_point/metadata/schema.json +6 -0
- file_backup/api/list_files/__init__.py +15 -0
- file_backup/api/list_files/execute/__init__.py +1 -0
- file_backup/api/list_files/execute/implementation.py +12 -0
- file_backup/api/list_files/execute/metadata.json +64 -0
- file_backup/api/list_files/execute/schema.json +42 -0
- file_backup/api/list_files/get_schema/__init__.py +1 -0
- file_backup/api/list_files/get_schema/implementation.py +11 -0
- file_backup/api/list_files/get_schema/metadata.json +44 -0
- file_backup/api/list_files/get_schema/schema.json +6 -0
- file_backup/api/list_files/metadata/__init__.py +1 -0
- file_backup/api/list_files/metadata/implementation.py +11 -0
- file_backup/api/list_files/metadata/metadata.json +44 -0
- file_backup/api/list_files/metadata/schema.json +6 -0
- file_backup/api/list_points/__init__.py +15 -0
- file_backup/api/list_points/execute/__init__.py +1 -0
- file_backup/api/list_points/execute/implementation.py +12 -0
- file_backup/api/list_points/execute/metadata.json +55 -0
- file_backup/api/list_points/execute/schema.json +37 -0
- file_backup/api/list_points/get_schema/__init__.py +1 -0
- file_backup/api/list_points/get_schema/implementation.py +11 -0
- file_backup/api/list_points/get_schema/metadata.json +35 -0
- file_backup/api/list_points/get_schema/schema.json +6 -0
- file_backup/api/list_points/metadata/__init__.py +1 -0
- file_backup/api/list_points/metadata/implementation.py +11 -0
- file_backup/api/list_points/metadata/metadata.json +35 -0
- file_backup/api/list_points/metadata/schema.json +6 -0
- file_backup/api/recover_catalog/__init__.py +15 -0
- file_backup/api/recover_catalog/execute/__init__.py +1 -0
- file_backup/api/recover_catalog/execute/implementation.py +12 -0
- file_backup/api/recover_catalog/execute/metadata.json +82 -0
- file_backup/api/recover_catalog/execute/schema.json +55 -0
- file_backup/api/recover_catalog/get_schema/__init__.py +1 -0
- file_backup/api/recover_catalog/get_schema/implementation.py +11 -0
- file_backup/api/recover_catalog/get_schema/metadata.json +62 -0
- file_backup/api/recover_catalog/get_schema/schema.json +6 -0
- file_backup/api/recover_catalog/metadata/__init__.py +1 -0
- file_backup/api/recover_catalog/metadata/implementation.py +11 -0
- file_backup/api/recover_catalog/metadata/metadata.json +62 -0
- file_backup/api/recover_catalog/metadata/schema.json +6 -0
- file_backup/api/restore/__init__.py +15 -0
- file_backup/api/restore/execute/__init__.py +1 -0
- file_backup/api/restore/execute/implementation.py +12 -0
- file_backup/api/restore/execute/metadata.json +82 -0
- file_backup/api/restore/execute/schema.json +55 -0
- file_backup/api/restore/get_schema/__init__.py +1 -0
- file_backup/api/restore/get_schema/implementation.py +11 -0
- file_backup/api/restore/get_schema/metadata.json +62 -0
- file_backup/api/restore/get_schema/schema.json +6 -0
- file_backup/api/restore/metadata/__init__.py +1 -0
- file_backup/api/restore/metadata/implementation.py +11 -0
- file_backup/api/restore/metadata/metadata.json +62 -0
- file_backup/api/restore/metadata/schema.json +6 -0
- file_backup/api/restore_file/__init__.py +15 -0
- file_backup/api/restore_file/execute/__init__.py +1 -0
- file_backup/api/restore_file/execute/implementation.py +12 -0
- file_backup/api/restore_file/execute/metadata.json +100 -0
- file_backup/api/restore_file/execute/schema.json +68 -0
- file_backup/api/restore_file/get_schema/__init__.py +1 -0
- file_backup/api/restore_file/get_schema/implementation.py +11 -0
- file_backup/api/restore_file/get_schema/metadata.json +80 -0
- file_backup/api/restore_file/get_schema/schema.json +6 -0
- file_backup/api/restore_file/metadata/__init__.py +1 -0
- file_backup/api/restore_file/metadata/implementation.py +11 -0
- file_backup/api/restore_file/metadata/metadata.json +80 -0
- file_backup/api/restore_file/metadata/schema.json +6 -0
- file_backup/api/verify/__init__.py +15 -0
- file_backup/api/verify/execute/__init__.py +1 -0
- file_backup/api/verify/execute/implementation.py +12 -0
- file_backup/api/verify/execute/metadata.json +64 -0
- file_backup/api/verify/execute/schema.json +42 -0
- file_backup/api/verify/get_schema/__init__.py +1 -0
- file_backup/api/verify/get_schema/implementation.py +11 -0
- file_backup/api/verify/get_schema/metadata.json +44 -0
- file_backup/api/verify/get_schema/schema.json +6 -0
- file_backup/api/verify/metadata/__init__.py +1 -0
- file_backup/api/verify/metadata/implementation.py +11 -0
- file_backup/api/verify/metadata/metadata.json +44 -0
- file_backup/api/verify/metadata/schema.json +6 -0
- file_backup/cli.py +332 -0
- file_backup/errors.py +100 -0
- file_backup/types.py +159 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/01_backup_lifecycle.py +72 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/02_catalog_browse.py +51 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/03_restore_and_preview.py +88 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/04_verify_and_chain_failure.py +58 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/05_typed_errors_and_conflicts.py +78 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/06_catalog_configuration.py +57 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/07_metadata_and_schemas.py +63 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/08_help_and_info.py +45 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/09_thin_cli.py +70 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/10_offline_installed_restore.py +52 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/11_recover_catalog.py +60 -0
- vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/manifest.json +15 -0
- vvz_file_backup-0.1.1.dist-info/METADATA +6 -0
- vvz_file_backup-0.1.1.dist-info/RECORD +159 -0
- vvz_file_backup-0.1.1.dist-info/WHEEL +5 -0
- vvz_file_backup-0.1.1.dist-info/entry_points.txt +2 -0
- vvz_file_backup-0.1.1.dist-info/top_level.txt +1 -0
file_backup/__init__.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Independent backup, saved-point inspection, verification, and restore API."""
|
|
2
|
+
|
|
3
|
+
from .api.backup import Backup
|
|
4
|
+
from .api.list_points import ListPoints
|
|
5
|
+
from .api.inspect_point import InspectPoint
|
|
6
|
+
from .api.list_files import ListFiles
|
|
7
|
+
from .api.restore import Restore
|
|
8
|
+
from .api.restore_file import RestoreFile
|
|
9
|
+
from .api.verify import Verify
|
|
10
|
+
from .api.recover_catalog import RecoverCatalog
|
|
11
|
+
from .api.help import Help
|
|
12
|
+
from .api.info import Info
|
|
13
|
+
from ._configuration import DEFAULT_CATALOG_DIRECTORY
|
|
14
|
+
from .errors import FileBackupError, ArgumentError, PointNotFoundError, FileNotFoundError, ArchiveMissingError, ArchiveCorruptError, ChainError, BackupIOError, CatalogError, ConflictError, SourceChangedError, SectionNotFoundError, CommandNotFoundError
|
|
15
|
+
from .types import BackupCreated, BackupUnchanged, BackupPreview, PointInfo, PointList, FileList, RestoreResult, FileRestoreResult, RestorePreview, FileRestorePreview, VerificationReport, CatalogRecoveryResult, CatalogRecoveryPreview, HelpPage, InfoTOC, InfoSection, CLIResult, CLIError
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
'DEFAULT_CATALOG_DIRECTORY',
|
|
19
|
+
'Backup',
|
|
20
|
+
'ListPoints',
|
|
21
|
+
'InspectPoint',
|
|
22
|
+
'ListFiles',
|
|
23
|
+
'Restore',
|
|
24
|
+
'RestoreFile',
|
|
25
|
+
'Verify',
|
|
26
|
+
'RecoverCatalog',
|
|
27
|
+
'Help',
|
|
28
|
+
'Info',
|
|
29
|
+
'FileBackupError',
|
|
30
|
+
'ArgumentError',
|
|
31
|
+
'PointNotFoundError',
|
|
32
|
+
'FileNotFoundError',
|
|
33
|
+
'ArchiveMissingError',
|
|
34
|
+
'ArchiveCorruptError',
|
|
35
|
+
'ChainError',
|
|
36
|
+
'BackupIOError',
|
|
37
|
+
'CatalogError',
|
|
38
|
+
'ConflictError',
|
|
39
|
+
'SourceChangedError',
|
|
40
|
+
'SectionNotFoundError',
|
|
41
|
+
'CommandNotFoundError',
|
|
42
|
+
'BackupCreated',
|
|
43
|
+
'BackupUnchanged',
|
|
44
|
+
'BackupPreview',
|
|
45
|
+
'PointInfo',
|
|
46
|
+
'PointList',
|
|
47
|
+
'FileList',
|
|
48
|
+
'RestoreResult',
|
|
49
|
+
'FileRestoreResult',
|
|
50
|
+
'RestorePreview',
|
|
51
|
+
'FileRestorePreview',
|
|
52
|
+
'VerificationReport',
|
|
53
|
+
'CatalogRecoveryResult',
|
|
54
|
+
'CatalogRecoveryPreview',
|
|
55
|
+
'HelpPage',
|
|
56
|
+
'InfoTOC',
|
|
57
|
+
'InfoSection',
|
|
58
|
+
'CLIResult',
|
|
59
|
+
'CLIError',
|
|
60
|
+
]
|
file_backup/_archives.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Archive creation and strict catalog-bound integrity checks."""
|
|
2
|
+
import hashlib
|
|
3
|
+
import io
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import tarfile
|
|
7
|
+
|
|
8
|
+
from ._validation import relative_path, safe_absolute
|
|
9
|
+
from .errors import ArchiveMissingError, ArchiveCorruptError, BackupIOError
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def digest(data):
|
|
13
|
+
return hashlib.sha256(data).hexdigest()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def encode(manifest):
|
|
17
|
+
return json.dumps(manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def create(path, manifest, content):
|
|
21
|
+
with tarfile.open(path, "w:gz") as archive:
|
|
22
|
+
for name, data in [("manifest.json", encode(manifest)), *[("files/" + n, b) for n, b in sorted(content.items())]]:
|
|
23
|
+
member = tarfile.TarInfo(name)
|
|
24
|
+
member.size = len(data)
|
|
25
|
+
archive.addfile(member, io.BytesIO(data))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def read(repository, row, identity, operation):
|
|
29
|
+
relative = row["archive"]
|
|
30
|
+
path = Path(repository) / relative
|
|
31
|
+
context = dict(operation=operation, point=row["point"], archive=str(path))
|
|
32
|
+
if not relative_path(relative) or relative != f"archives/{row['point']}.tar.gz" or not safe_absolute(str(path)):
|
|
33
|
+
raise ArchiveCorruptError("Unsafe archive path", expected="safe archive", got=relative, **context)
|
|
34
|
+
try:
|
|
35
|
+
data = path.read_bytes()
|
|
36
|
+
except FileNotFoundError as cause:
|
|
37
|
+
raise ArchiveMissingError("Required archive missing", **context) from cause
|
|
38
|
+
except OSError as cause:
|
|
39
|
+
raise BackupIOError("Cannot read archive", operation=operation, path=str(path), phase="read") from cause
|
|
40
|
+
actual = digest(data)
|
|
41
|
+
if actual != row["archive_sha256"]:
|
|
42
|
+
raise ArchiveCorruptError("Archive checksum mismatch", expected=row["archive_sha256"], got=actual, **context)
|
|
43
|
+
try:
|
|
44
|
+
content = {}
|
|
45
|
+
with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive:
|
|
46
|
+
names = set()
|
|
47
|
+
for member in archive:
|
|
48
|
+
if member.name in names or not member.isfile() or not relative_path(member.name):
|
|
49
|
+
raise ValueError("unsafe or duplicate archive member")
|
|
50
|
+
names.add(member.name)
|
|
51
|
+
handle = archive.extractfile(member)
|
|
52
|
+
payload = handle.read()
|
|
53
|
+
if member.name == "manifest.json":
|
|
54
|
+
manifest = json.loads(payload)
|
|
55
|
+
elif member.name.startswith("files/") and relative_path(member.name[6:]):
|
|
56
|
+
content[member.name[6:]] = payload
|
|
57
|
+
else:
|
|
58
|
+
raise ValueError("unexpected member")
|
|
59
|
+
if "manifest.json" not in names:
|
|
60
|
+
raise ValueError("missing manifest")
|
|
61
|
+
if manifest != row["manifest"] or manifest["repository"] != identity:
|
|
62
|
+
raise ValueError("manifest/catalog binding mismatch")
|
|
63
|
+
for field in ("point", "kind", "base_point", "previous_point", "snapshot_at", "completed_at"):
|
|
64
|
+
if manifest[field] != row[field]:
|
|
65
|
+
raise ValueError("manifest point binding mismatch")
|
|
66
|
+
changed = manifest["changed"]
|
|
67
|
+
if len(changed) != len(set(changed)) or set(changed) != set(content):
|
|
68
|
+
raise ValueError("archive member set mismatch")
|
|
69
|
+
for name, item in manifest["files"].items():
|
|
70
|
+
if not relative_path(name) or item["size"] < 0 or item["version"] != item["sha256"]:
|
|
71
|
+
raise ValueError("invalid file manifest")
|
|
72
|
+
for name, payload in content.items():
|
|
73
|
+
item = manifest["files"][name]
|
|
74
|
+
if digest(payload) != item["sha256"] or len(payload) != item["size"]:
|
|
75
|
+
raise ValueError("file integrity mismatch")
|
|
76
|
+
if len(manifest["deleted"]) != len(set(manifest["deleted"])) or any(not relative_path(n) or n in manifest["files"] for n in manifest["deleted"]):
|
|
77
|
+
raise ValueError("invalid tombstones")
|
|
78
|
+
return content
|
|
79
|
+
except (tarfile.TarError, OSError, ValueError, KeyError, TypeError, UnboundLocalError) as cause:
|
|
80
|
+
raise ArchiveCorruptError("Invalid archive or manifest", expected="catalog-bound safe archive", got=str(cause), **context) from cause
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def materialize(repository, chain, identity, operation):
|
|
84
|
+
live = {}
|
|
85
|
+
for row in chain:
|
|
86
|
+
updates = read(repository, row, identity, operation)
|
|
87
|
+
manifest = row["manifest"]
|
|
88
|
+
if row["kind"] == "full":
|
|
89
|
+
live = {}
|
|
90
|
+
for name in manifest["deleted"]:
|
|
91
|
+
if name not in live:
|
|
92
|
+
raise ArchiveCorruptError("Tombstone has no prior file", operation=operation, point=row["point"],
|
|
93
|
+
archive=row["archive"], expected="live prior path", got=name)
|
|
94
|
+
del live[name]
|
|
95
|
+
live.update(updates)
|
|
96
|
+
expected = manifest["files"]
|
|
97
|
+
if set(live) != set(expected) or any(digest(payload) != expected[name]["sha256"] or len(payload) != expected[name]["size"] for name, payload in live.items()):
|
|
98
|
+
raise ArchiveCorruptError("Saved slice does not match manifest", operation=operation, point=row["point"],
|
|
99
|
+
archive=row["archive"], expected="exact slice", got="mismatch")
|
|
100
|
+
return live
|
file_backup/_catalog.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""SQLite saved-point catalog and exact chain selection."""
|
|
2
|
+
import json
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import sqlite3
|
|
5
|
+
|
|
6
|
+
from .errors import CatalogError, ChainError, PointNotFoundError
|
|
7
|
+
from .types import PointInfo
|
|
8
|
+
|
|
9
|
+
SCHEMA = """
|
|
10
|
+
CREATE TABLE IF NOT EXISTS repository (identity TEXT NOT NULL);
|
|
11
|
+
CREATE TABLE IF NOT EXISTS points (
|
|
12
|
+
sequence INTEGER PRIMARY KEY, point TEXT UNIQUE NOT NULL,
|
|
13
|
+
kind TEXT NOT NULL, base_point TEXT NOT NULL, previous_point TEXT,
|
|
14
|
+
snapshot_at TEXT NOT NULL, completed_at TEXT NOT NULL,
|
|
15
|
+
archive TEXT NOT NULL, archive_sha256 TEXT NOT NULL, manifest TEXT NOT NULL);
|
|
16
|
+
CREATE TABLE IF NOT EXISTS checkpoint (point TEXT NOT NULL);
|
|
17
|
+
CREATE TABLE IF NOT EXISTS objects (path TEXT PRIMARY KEY);
|
|
18
|
+
CREATE TABLE IF NOT EXISTS content_versions (sha256 TEXT PRIMARY KEY, size INTEGER NOT NULL);
|
|
19
|
+
CREATE TABLE IF NOT EXISTS archives (point TEXT PRIMARY KEY, file TEXT NOT NULL, digest TEXT NOT NULL, kind TEXT NOT NULL, state TEXT NOT NULL);
|
|
20
|
+
CREATE TABLE IF NOT EXISTS chains (point TEXT PRIMARY KEY, base_point TEXT NOT NULL, previous_point TEXT);
|
|
21
|
+
CREATE TABLE IF NOT EXISTS point_members (point TEXT NOT NULL, path TEXT NOT NULL, sha256 TEXT NOT NULL, archive TEXT NOT NULL, PRIMARY KEY(point,path));
|
|
22
|
+
CREATE TABLE IF NOT EXISTS tombstones (point TEXT NOT NULL, path TEXT NOT NULL, PRIMARY KEY(point,path));
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def load(repository, operation, catalog=None):
|
|
27
|
+
catalog = Path(catalog) if catalog else Path(repository) / "catalog.sqlite3"
|
|
28
|
+
if catalog.is_symlink():
|
|
29
|
+
raise CatalogError("Catalog must not be a symlink", operation=operation, catalog=str(catalog), phase="open")
|
|
30
|
+
if not catalog.exists():
|
|
31
|
+
archives = Path(repository) / "archives"
|
|
32
|
+
if archives.exists() and any(archives.glob("*.tar.gz")):
|
|
33
|
+
raise CatalogError("Catalog missing while archives exist", operation=operation,
|
|
34
|
+
catalog=str(catalog), phase="open")
|
|
35
|
+
return None, []
|
|
36
|
+
try:
|
|
37
|
+
with sqlite3.connect(catalog.as_uri() + "?mode=ro", uri=True) as connection:
|
|
38
|
+
connection.row_factory = sqlite3.Row
|
|
39
|
+
if connection.execute("PRAGMA integrity_check").fetchone()[0] != "ok":
|
|
40
|
+
raise sqlite3.DatabaseError("catalog integrity check failed")
|
|
41
|
+
identity = connection.execute("SELECT identity FROM repository").fetchone()[0]
|
|
42
|
+
points = []
|
|
43
|
+
for row in connection.execute("SELECT * FROM points ORDER BY sequence"):
|
|
44
|
+
point = dict(row)
|
|
45
|
+
point["manifest"] = json.loads(point["manifest"])
|
|
46
|
+
points.append(point)
|
|
47
|
+
checkpoint = connection.execute("SELECT point FROM checkpoint").fetchall()
|
|
48
|
+
missing_base_rebootstrap = operation == "backup" and not points and len(checkpoint) == 1
|
|
49
|
+
if not missing_base_rebootstrap and (len(checkpoint) != bool(points) or points and checkpoint[0][0] != points[-1]["point"]):
|
|
50
|
+
raise sqlite3.DatabaseError("checkpoint does not match last committed point")
|
|
51
|
+
return identity, points
|
|
52
|
+
except (sqlite3.Error, ValueError, KeyError, IndexError, TypeError) as cause:
|
|
53
|
+
raise CatalogError("Cannot read catalog", operation=operation, catalog=str(catalog), phase="read") from cause
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def chain(points, point, operation):
|
|
57
|
+
by_id = {row["point"]: row for row in points}
|
|
58
|
+
if point not in by_id:
|
|
59
|
+
raise PointNotFoundError("Saved point not found", operation=operation, point=point)
|
|
60
|
+
result, seen = [], set()
|
|
61
|
+
selected = by_id[point]
|
|
62
|
+
current = selected
|
|
63
|
+
while True:
|
|
64
|
+
identifier = current["point"]
|
|
65
|
+
if identifier in seen:
|
|
66
|
+
raise ChainError("Cyclic saved-point chain", operation=operation, point=point, missing_dependency=identifier)
|
|
67
|
+
seen.add(identifier)
|
|
68
|
+
result.append(current)
|
|
69
|
+
if current["kind"] == "full":
|
|
70
|
+
if current["base_point"] != identifier or current["previous_point"] is not None or selected["base_point"] != identifier:
|
|
71
|
+
raise ChainError("Invalid full base", operation=operation, point=point, missing_dependency=identifier)
|
|
72
|
+
return list(reversed(result))
|
|
73
|
+
previous = by_id.get(current["previous_point"])
|
|
74
|
+
if (current["kind"] != "incremental" or previous is None
|
|
75
|
+
or previous["sequence"] >= current["sequence"]
|
|
76
|
+
or previous["base_point"] != current["base_point"]):
|
|
77
|
+
raise ChainError("Missing or unordered dependency", operation=operation, point=point,
|
|
78
|
+
missing_dependency=current["previous_point"])
|
|
79
|
+
current = previous
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def info(points, row, operation):
|
|
83
|
+
members = row["manifest"]["files"]
|
|
84
|
+
return PointInfo(row["point"], row["kind"], row["base_point"], row["previous_point"],
|
|
85
|
+
row["snapshot_at"], row["completed_at"], row["archive"], row["archive_sha256"],
|
|
86
|
+
tuple(item["point"] for item in chain(points, row["point"], operation)),
|
|
87
|
+
len(members), len(row["manifest"]["deleted"]))
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Explicit catalog directory selection without ambient configuration."""
|
|
2
|
+
from collections.abc import Mapping
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from ._validation import invalid
|
|
7
|
+
|
|
8
|
+
DEFAULT_CATALOG_DIRECTORY = "."
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def directory(repository, value, operation, parameter):
|
|
12
|
+
if type(value) is not str:
|
|
13
|
+
invalid(operation, parameter, "type", "nonempty directory string", value)
|
|
14
|
+
if not value or "\0" in value:
|
|
15
|
+
invalid(operation, parameter, "value", "nonempty NUL-free directory", value)
|
|
16
|
+
raw = Path(repository) / value
|
|
17
|
+
# Check lexical components before normalizing .. away.
|
|
18
|
+
for part in (raw, *raw.parents):
|
|
19
|
+
if part.is_symlink() or part.exists() and not part.is_dir():
|
|
20
|
+
invalid(operation, parameter, "value", "directory without symlink components", value)
|
|
21
|
+
return Path(os.path.abspath(raw))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def resolve_catalog(repository, catalog_directory, config, operation):
|
|
25
|
+
if config is not None and not isinstance(config, Mapping):
|
|
26
|
+
invalid(operation, "config", "type", "mapping or None", config)
|
|
27
|
+
settings = {} if config is None else config
|
|
28
|
+
for key in settings:
|
|
29
|
+
if type(key) is not str or key != "catalog_directory":
|
|
30
|
+
invalid(operation, "config", "unknown", ["catalog_directory"], key)
|
|
31
|
+
configured = None
|
|
32
|
+
if "catalog_directory" in settings:
|
|
33
|
+
configured = directory(repository, settings["catalog_directory"], operation, "config.catalog_directory")
|
|
34
|
+
explicit = None
|
|
35
|
+
if catalog_directory is not None:
|
|
36
|
+
explicit = directory(repository, catalog_directory, operation, "catalog_directory")
|
|
37
|
+
selected = explicit if explicit is not None else configured
|
|
38
|
+
if selected is None:
|
|
39
|
+
selected = directory(repository, DEFAULT_CATALOG_DIRECTORY, operation, "catalog_directory")
|
|
40
|
+
return selected / "catalog.sqlite3"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Internal namespace base for classmethod-only operations."""
|
|
2
|
+
|
|
3
|
+
from ._resources import resource
|
|
4
|
+
|
|
5
|
+
class _OperationNamespace:
|
|
6
|
+
"""Operation declarations cannot be instantiated."""
|
|
7
|
+
def __new__(cls, *args: object, **kwargs: object) -> object:
|
|
8
|
+
raise TypeError("operation namespaces cannot be instantiated")
|
|
9
|
+
|
|
10
|
+
def __init_subclass__(cls, **kwargs: object) -> None:
|
|
11
|
+
super().__init_subclass__(**kwargs)
|
|
12
|
+
declaration = resource(cls.__module__.rsplit(".", 1)[-1], "metadata.json")
|
|
13
|
+
for attribute, field in (("name", "name"), ("version", "version"),
|
|
14
|
+
("descr", "description"), ("category", "category"),
|
|
15
|
+
("author", "author"), ("email", "email")):
|
|
16
|
+
setattr(cls, attribute, declaration[field])
|
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
"""Filesystem backup operations over committed catalog points."""
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from datetime import datetime, timezone
|
|
4
|
+
import fcntl
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import shutil
|
|
8
|
+
import sqlite3
|
|
9
|
+
import stat
|
|
10
|
+
import tempfile
|
|
11
|
+
import uuid
|
|
12
|
+
|
|
13
|
+
from . import _archives as archives
|
|
14
|
+
from . import _catalog as catalog
|
|
15
|
+
from ._validation import invalid, safe_absolute
|
|
16
|
+
from .errors import (ArchiveCorruptError, ArchiveMissingError, BackupIOError, CatalogError, ChainError,
|
|
17
|
+
ConflictError, FileNotFoundError as SavedFileNotFoundError, SourceChangedError)
|
|
18
|
+
from .types import (BackupCreated, BackupUnchanged, BackupPreview, PointList, FileList,
|
|
19
|
+
RestoreResult, RestorePreview, FileRestoreResult, FileRestorePreview,
|
|
20
|
+
VerificationReport, CatalogRecoveryResult, CatalogRecoveryPreview)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def now():
|
|
24
|
+
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def inside(path, root):
|
|
28
|
+
return Path(path).is_relative_to(Path(root))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def scan(source, operation):
|
|
32
|
+
root = Path(source)
|
|
33
|
+
if not root.is_dir():
|
|
34
|
+
invalid(operation, "source", "value", "existing directory", source)
|
|
35
|
+
content, signatures = {}, {}
|
|
36
|
+
try:
|
|
37
|
+
for directory, dirs, names in os.walk(root, followlinks=False):
|
|
38
|
+
for name in sorted(dirs + names):
|
|
39
|
+
path = Path(directory) / name
|
|
40
|
+
before = path.lstat()
|
|
41
|
+
if stat.S_ISDIR(before.st_mode):
|
|
42
|
+
continue
|
|
43
|
+
if not stat.S_ISREG(before.st_mode):
|
|
44
|
+
invalid(operation, "source", "value", "regular files only", str(path))
|
|
45
|
+
with path.open("rb") as handle:
|
|
46
|
+
opened = os.fstat(handle.fileno())
|
|
47
|
+
if (opened.st_ino, opened.st_dev) != (before.st_ino, before.st_dev):
|
|
48
|
+
raise SourceChangedError("Source changed before read", operation=operation, path=str(path))
|
|
49
|
+
payload = handle.read()
|
|
50
|
+
after = os.fstat(handle.fileno())
|
|
51
|
+
signature = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns)
|
|
52
|
+
if signature != (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns):
|
|
53
|
+
raise SourceChangedError("Source changed during read", operation=operation, path=str(path))
|
|
54
|
+
relative = path.relative_to(root).as_posix()
|
|
55
|
+
content[relative] = payload
|
|
56
|
+
signatures[relative] = signature
|
|
57
|
+
return content, signatures
|
|
58
|
+
except OSError as cause:
|
|
59
|
+
raise BackupIOError("Cannot scan source", operation=operation, path=str(root), phase="scan") from cause
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@contextmanager
|
|
63
|
+
def writer(repository, catalog_path):
|
|
64
|
+
root = Path(repository)
|
|
65
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
66
|
+
catalog_path.parent.mkdir(parents=True, exist_ok=True)
|
|
67
|
+
with (catalog_path.parent / ".writer.lock").open("a+b") as handle:
|
|
68
|
+
try:
|
|
69
|
+
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
70
|
+
except BlockingIOError as cause:
|
|
71
|
+
raise ConflictError("Another backup writer is active", operation="backup", path=str(root), reason="writer_locked") from cause
|
|
72
|
+
try:
|
|
73
|
+
yield
|
|
74
|
+
finally:
|
|
75
|
+
fcntl.flock(handle, fcntl.LOCK_UN)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def backup(repository, source, dry_run=False, catalog_path=None):
|
|
79
|
+
operation = "backup"
|
|
80
|
+
catalog_path = Path(catalog_path) if catalog_path else Path(repository) / "catalog.sqlite3"
|
|
81
|
+
if inside(repository, source) or inside(source, repository):
|
|
82
|
+
invalid(operation, "source", "value", "disjoint repository and source", source)
|
|
83
|
+
if inside(catalog_path.parent, source) or inside(source, catalog_path.parent):
|
|
84
|
+
invalid(operation, "catalog_directory", "value", "disjoint catalog and source", str(catalog_path.parent))
|
|
85
|
+
initial = scan(source, operation)
|
|
86
|
+
if dry_run:
|
|
87
|
+
return _backup(repository, source, True, initial, catalog_path)
|
|
88
|
+
# Missing selected catalog must fail before creating its directory or lock.
|
|
89
|
+
catalog.load(repository, operation, catalog_path)
|
|
90
|
+
try:
|
|
91
|
+
with writer(repository, catalog_path):
|
|
92
|
+
return _backup(repository, source, False, initial, catalog_path)
|
|
93
|
+
except OSError as cause:
|
|
94
|
+
raise BackupIOError("Cannot write backup", operation=operation, path=repository, phase="write") from cause
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _backup(repository, source, dry_run, initial, catalog_path):
|
|
98
|
+
operation = "backup"
|
|
99
|
+
content, signatures = initial
|
|
100
|
+
identity, points = catalog.load(repository, operation, catalog_path)
|
|
101
|
+
previous = points[-1] if points else None
|
|
102
|
+
base = next((r for r in points if previous and r["point"] == previous["base_point"]), None)
|
|
103
|
+
full = previous is None or base is None or not (Path(repository) / base["archive"]).exists()
|
|
104
|
+
if previous and not full:
|
|
105
|
+
archives.materialize(repository, catalog.chain(points, previous["point"], operation), identity, operation)
|
|
106
|
+
before = previous["manifest"]["files"] if previous else {}
|
|
107
|
+
added = set(content) - set(before)
|
|
108
|
+
modified = {n for n in set(content) & set(before) if archives.digest(content[n]) != before[n]["sha256"]}
|
|
109
|
+
deleted = set(before) - set(content)
|
|
110
|
+
action = "full" if full else "incremental" if added or modified or deleted else "unchanged"
|
|
111
|
+
if dry_run:
|
|
112
|
+
return BackupPreview("preview", action, len(added), len(modified), len(deleted), previous["point"] if previous else None)
|
|
113
|
+
if action == "unchanged":
|
|
114
|
+
return BackupUnchanged("unchanged", previous["point"], None, 0, 0, 0)
|
|
115
|
+
root = Path(repository)
|
|
116
|
+
archive_dir = root / "archives"
|
|
117
|
+
if not safe_absolute(str(archive_dir)):
|
|
118
|
+
invalid(operation, "repository", "value", "archive directory without symlinks", str(archive_dir))
|
|
119
|
+
archive_dir.mkdir(exist_ok=True)
|
|
120
|
+
# A finalized archive without a committed catalog row is a crash orphan.
|
|
121
|
+
known = {r["archive"] for r in points}
|
|
122
|
+
for orphan in archive_dir.glob("*.tar.gz"):
|
|
123
|
+
if orphan.relative_to(root).as_posix() not in known:
|
|
124
|
+
quarantine = root / "quarantine"
|
|
125
|
+
quarantine.mkdir(exist_ok=True)
|
|
126
|
+
os.replace(orphan, quarantine / (orphan.name + "." + uuid.uuid4().hex))
|
|
127
|
+
identity = identity or str(uuid.uuid4())
|
|
128
|
+
marker = root / ".repository-id"
|
|
129
|
+
if not marker.exists():
|
|
130
|
+
marker.write_text(identity, encoding="ascii")
|
|
131
|
+
elif marker.read_text(encoding="ascii") != identity:
|
|
132
|
+
raise CatalogError("Repository identity mismatch", operation=operation, catalog=str(catalog_path), phase="identity")
|
|
133
|
+
point = str(uuid.uuid4())
|
|
134
|
+
stamp = now()
|
|
135
|
+
relative = f"archives/{point}.tar.gz"
|
|
136
|
+
manifest = dict(repository=identity, point=point, kind=action,
|
|
137
|
+
base_point=point if full else previous["base_point"],
|
|
138
|
+
previous_point=None if full else previous["point"],
|
|
139
|
+
snapshot_at=stamp, completed_at=stamp,
|
|
140
|
+
files={name:dict(sha256=archives.digest(data), version=archives.digest(data), size=len(data),
|
|
141
|
+
archive=relative if full or name in added or name in modified else before[name]["archive"])
|
|
142
|
+
for name, data in sorted(content.items())},
|
|
143
|
+
changed=sorted(content if full else added | modified), deleted=[] if full else sorted(deleted))
|
|
144
|
+
temporary = None
|
|
145
|
+
final = root / relative
|
|
146
|
+
connection = None
|
|
147
|
+
committed = False
|
|
148
|
+
catalog_existed = catalog_path.exists()
|
|
149
|
+
try:
|
|
150
|
+
descriptor, temporary = tempfile.mkstemp(prefix=".archive-", dir=archive_dir)
|
|
151
|
+
os.close(descriptor)
|
|
152
|
+
archives.create(temporary, manifest, {n:content[n] for n in manifest["changed"]})
|
|
153
|
+
with open(temporary, "rb") as handle:
|
|
154
|
+
os.fsync(handle.fileno())
|
|
155
|
+
after, after_signatures = scan(source, operation)
|
|
156
|
+
if content != after or signatures != after_signatures:
|
|
157
|
+
raise SourceChangedError("Source changed during backup", operation=operation, path=source)
|
|
158
|
+
archive_hash = archives.digest(Path(temporary).read_bytes())
|
|
159
|
+
os.replace(temporary, final)
|
|
160
|
+
temporary = None
|
|
161
|
+
directory_descriptor = os.open(archive_dir, os.O_RDONLY)
|
|
162
|
+
try:
|
|
163
|
+
os.fsync(directory_descriptor)
|
|
164
|
+
finally:
|
|
165
|
+
os.close(directory_descriptor)
|
|
166
|
+
after, after_signatures = scan(source, operation)
|
|
167
|
+
if content != after or signatures != after_signatures:
|
|
168
|
+
raise SourceChangedError("Source changed before checkpoint", operation=operation, path=source)
|
|
169
|
+
try:
|
|
170
|
+
connection = sqlite3.connect(catalog_path, timeout=0)
|
|
171
|
+
connection.executescript(catalog.SCHEMA)
|
|
172
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
173
|
+
if not connection.execute("SELECT identity FROM repository").fetchone():
|
|
174
|
+
connection.execute("INSERT INTO repository VALUES (?)", (identity,))
|
|
175
|
+
connection.execute("INSERT INTO points(point,kind,base_point,previous_point,snapshot_at,completed_at,archive,archive_sha256,manifest) VALUES (?,?,?,?,?,?,?,?,?)",
|
|
176
|
+
(point, action, manifest["base_point"], manifest["previous_point"], stamp, stamp, relative, archive_hash, archives.encode(manifest).decode("utf-8")))
|
|
177
|
+
connection.execute("INSERT INTO archives VALUES (?,?,?,?,?)", (point, relative, archive_hash, action, "complete"))
|
|
178
|
+
connection.execute("INSERT INTO chains VALUES (?,?,?)", (point, manifest["base_point"], manifest["previous_point"]))
|
|
179
|
+
for name, item in manifest["files"].items():
|
|
180
|
+
connection.execute("INSERT OR IGNORE INTO objects VALUES (?)", (name,))
|
|
181
|
+
connection.execute("INSERT OR IGNORE INTO content_versions VALUES (?,?)", (item["sha256"], item["size"]))
|
|
182
|
+
connection.execute("INSERT INTO point_members VALUES (?,?,?,?)", (point, name, item["sha256"], item["archive"]))
|
|
183
|
+
for name in manifest["deleted"]:
|
|
184
|
+
connection.execute("INSERT INTO tombstones VALUES (?,?)", (point, name))
|
|
185
|
+
connection.execute("DELETE FROM checkpoint")
|
|
186
|
+
connection.execute("INSERT INTO checkpoint VALUES (?)", (point,))
|
|
187
|
+
connection.commit()
|
|
188
|
+
committed = True
|
|
189
|
+
except sqlite3.Error as cause:
|
|
190
|
+
if connection:
|
|
191
|
+
connection.rollback()
|
|
192
|
+
raise CatalogError("Cannot commit saved point", operation=operation, catalog=str(catalog_path), phase="commit") from cause
|
|
193
|
+
return BackupCreated("created", point, action, manifest["base_point"], manifest["previous_point"], str(final), stamp,
|
|
194
|
+
len(added), len(modified), len(deleted))
|
|
195
|
+
finally:
|
|
196
|
+
if connection:
|
|
197
|
+
connection.close()
|
|
198
|
+
if temporary and Path(temporary).exists():
|
|
199
|
+
Path(temporary).unlink()
|
|
200
|
+
if not committed and final.exists():
|
|
201
|
+
final.unlink()
|
|
202
|
+
if not committed and not catalog_existed:
|
|
203
|
+
catalog_path.unlink(missing_ok=True)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def selected(repository, point, operation, catalog_path=None):
|
|
207
|
+
identity, points = catalog.load(repository, operation, catalog_path)
|
|
208
|
+
chain = catalog.chain(points, point, operation)
|
|
209
|
+
try:
|
|
210
|
+
content = archives.materialize(repository, chain, identity, operation)
|
|
211
|
+
except (ArchiveMissingError, ArchiveCorruptError) as cause:
|
|
212
|
+
if operation in ("restore", "restore_file") and cause.point != point:
|
|
213
|
+
raise ChainError("Saved point dependency is unavailable", operation=operation,
|
|
214
|
+
point=point, missing_dependency=cause.point) from cause
|
|
215
|
+
raise
|
|
216
|
+
return identity, points, chain, content
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def list_points(repository, catalog_path=None):
|
|
220
|
+
_, points = catalog.load(repository, "list_points", catalog_path)
|
|
221
|
+
return PointList(tuple(catalog.info(points, row, "list_points") for row in sorted(points, key=lambda r:(r["completed_at"], r["point"]))))
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def inspect_point(repository, point, catalog_path=None):
|
|
225
|
+
_, points = catalog.load(repository, "inspect_point", catalog_path)
|
|
226
|
+
chain = catalog.chain(points, point, "inspect_point")
|
|
227
|
+
return catalog.info(points, chain[-1], "inspect_point")
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def list_files(repository, point, catalog_path=None):
|
|
231
|
+
_, points = catalog.load(repository, "list_files", catalog_path)
|
|
232
|
+
chain = catalog.chain(points, point, "list_files")
|
|
233
|
+
return FileList(tuple(dict(path=name, **item) for name,item in sorted(chain[-1]["manifest"]["files"].items())))
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def destination_check(repository, destination, operation, catalog_path=None):
|
|
237
|
+
if inside(destination, repository) or inside(repository, destination):
|
|
238
|
+
invalid(operation, "destination", "value", "outside repository", destination)
|
|
239
|
+
if catalog_path is not None and operation in ("restore", "restore_file") and inside(destination, Path(catalog_path).parent):
|
|
240
|
+
invalid(operation, "destination", "value", "outside catalog directory", destination)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def restore(repository, point, destination, dry_run=False, catalog_path=None):
|
|
244
|
+
operation = "restore"
|
|
245
|
+
destination_check(repository, destination, operation, catalog_path)
|
|
246
|
+
target = Path(destination)
|
|
247
|
+
if target.exists() and (not target.is_dir() or any(target.iterdir())):
|
|
248
|
+
raise ConflictError("Restore destination must be empty", operation=operation, path=destination, reason="nonempty_destination")
|
|
249
|
+
_, _, _, content = selected(repository, point, operation, catalog_path)
|
|
250
|
+
size = sum(map(len, content.values()))
|
|
251
|
+
if dry_run:
|
|
252
|
+
return RestorePreview("preview", point, destination, len(content), size)
|
|
253
|
+
staging = None
|
|
254
|
+
try:
|
|
255
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
256
|
+
staging = Path(tempfile.mkdtemp(prefix=".restore-", dir=target.parent))
|
|
257
|
+
for name, data in content.items():
|
|
258
|
+
path = staging / name
|
|
259
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
260
|
+
with path.open("wb") as handle:
|
|
261
|
+
handle.write(data)
|
|
262
|
+
handle.flush()
|
|
263
|
+
os.fsync(handle.fileno())
|
|
264
|
+
if target.exists() and any(target.iterdir()):
|
|
265
|
+
raise ConflictError("Destination changed during restore", operation=operation, path=destination, reason="nonempty_destination")
|
|
266
|
+
os.replace(staging, target)
|
|
267
|
+
staging = None
|
|
268
|
+
return RestoreResult("restored", point, destination, len(content), size)
|
|
269
|
+
except OSError as cause:
|
|
270
|
+
raise BackupIOError("Cannot publish restored slice", operation=operation, path=destination, phase="restore") from cause
|
|
271
|
+
finally:
|
|
272
|
+
if staging:
|
|
273
|
+
shutil.rmtree(staging)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def restore_file(repository, point, path, destination, overwrite=False, dry_run=False, catalog_path=None):
|
|
277
|
+
operation = "restore_file"
|
|
278
|
+
destination_check(repository, destination, operation, catalog_path)
|
|
279
|
+
target = Path(destination)
|
|
280
|
+
if not safe_absolute(str(target)):
|
|
281
|
+
invalid(operation, "destination", "value", "no symlink components", str(target))
|
|
282
|
+
if target.exists() and (not overwrite or not target.is_file()):
|
|
283
|
+
raise ConflictError("Destination file exists", operation=operation, path=str(target), reason="file_exists")
|
|
284
|
+
_, _, _, content = selected(repository, point, operation, catalog_path)
|
|
285
|
+
if path not in content:
|
|
286
|
+
raise SavedFileNotFoundError("File is not live at point", operation=operation, point=point, path=path)
|
|
287
|
+
payload = content[path]
|
|
288
|
+
sha = archives.digest(payload)
|
|
289
|
+
if dry_run:
|
|
290
|
+
return FileRestorePreview("preview", point, path, sha, len(payload))
|
|
291
|
+
temporary = None
|
|
292
|
+
try:
|
|
293
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
294
|
+
descriptor, temporary = tempfile.mkstemp(prefix=".restore-file-", dir=target.parent)
|
|
295
|
+
with os.fdopen(descriptor, "wb") as handle:
|
|
296
|
+
handle.write(payload)
|
|
297
|
+
handle.flush()
|
|
298
|
+
os.fsync(handle.fileno())
|
|
299
|
+
if overwrite:
|
|
300
|
+
os.replace(temporary, target)
|
|
301
|
+
else:
|
|
302
|
+
os.link(temporary, target)
|
|
303
|
+
os.unlink(temporary)
|
|
304
|
+
temporary = None
|
|
305
|
+
return FileRestoreResult("restored", point, path, str(target), sha, len(payload))
|
|
306
|
+
except OSError as cause:
|
|
307
|
+
raise BackupIOError("Cannot restore file", operation=operation, path=str(target), phase="restore") from cause
|
|
308
|
+
finally:
|
|
309
|
+
if temporary:
|
|
310
|
+
Path(temporary).unlink(missing_ok=True)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def verify(repository, point, catalog_path=None):
|
|
314
|
+
_, _, chain, content = selected(repository, point, "verify", catalog_path)
|
|
315
|
+
return VerificationReport("verified", point, len(chain), len(content), sum(map(len, content.values())))
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def recover_catalog(repository, destination, catalog_copy, dry_run=False, catalog_path=None):
|
|
319
|
+
operation = "recover_catalog"
|
|
320
|
+
destination_check(repository, destination, operation)
|
|
321
|
+
target = Path(destination)
|
|
322
|
+
if catalog_path is not None and target == Path(catalog_path):
|
|
323
|
+
invalid(operation, "destination", "value", "different from active catalog", destination)
|
|
324
|
+
if target.exists():
|
|
325
|
+
raise ConflictError("Catalog destination exists", operation=operation, path=destination, reason="destination_exists")
|
|
326
|
+
identity, points = catalog.load(repository, operation, catalog_copy)
|
|
327
|
+
marker = Path(repository) / ".repository-id"
|
|
328
|
+
try:
|
|
329
|
+
expected = marker.read_text(encoding="ascii")
|
|
330
|
+
except OSError as cause:
|
|
331
|
+
raise CatalogError("Repository identity is unavailable", operation=operation, catalog=catalog_copy, phase="identity") from cause
|
|
332
|
+
if identity != expected:
|
|
333
|
+
raise CatalogError("Catalog copy belongs to another repository", operation=operation, catalog=catalog_copy, phase="identity")
|
|
334
|
+
for row in points:
|
|
335
|
+
archives.materialize(repository, catalog.chain(points, row["point"], operation), identity, operation)
|
|
336
|
+
if dry_run:
|
|
337
|
+
return CatalogRecoveryPreview("preview", destination, len(points))
|
|
338
|
+
temporary = None
|
|
339
|
+
try:
|
|
340
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
341
|
+
descriptor, temporary = tempfile.mkstemp(prefix=".catalog-", dir=target.parent)
|
|
342
|
+
os.close(descriptor)
|
|
343
|
+
with sqlite3.connect(Path(catalog_copy).as_uri() + "?mode=ro", uri=True) as source:
|
|
344
|
+
with sqlite3.connect(temporary) as output:
|
|
345
|
+
source.backup(output)
|
|
346
|
+
with open(temporary, "rb") as handle:
|
|
347
|
+
os.fsync(handle.fileno())
|
|
348
|
+
os.link(temporary, target)
|
|
349
|
+
os.unlink(temporary)
|
|
350
|
+
temporary = None
|
|
351
|
+
return CatalogRecoveryResult("recovered", destination, len(points))
|
|
352
|
+
except (OSError, sqlite3.Error) as cause:
|
|
353
|
+
raise CatalogError("Cannot publish catalog copy", operation=operation, catalog=destination, phase="recover") from cause
|
|
354
|
+
finally:
|
|
355
|
+
if temporary:
|
|
356
|
+
Path(temporary).unlink(missing_ok=True)
|