vvz-file-backup 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. file_backup/__init__.py +60 -0
  2. file_backup/_archives.py +100 -0
  3. file_backup/_catalog.py +87 -0
  4. file_backup/_configuration.py +40 -0
  5. file_backup/_namespace.py +16 -0
  6. file_backup/_operations.py +356 -0
  7. file_backup/_resources.py +7 -0
  8. file_backup/_validation.py +82 -0
  9. file_backup/api/backup/__init__.py +15 -0
  10. file_backup/api/backup/execute/__init__.py +1 -0
  11. file_backup/api/backup/execute/implementation.py +12 -0
  12. file_backup/api/backup/execute/metadata.json +73 -0
  13. file_backup/api/backup/execute/schema.json +50 -0
  14. file_backup/api/backup/get_schema/__init__.py +1 -0
  15. file_backup/api/backup/get_schema/implementation.py +11 -0
  16. file_backup/api/backup/get_schema/metadata.json +53 -0
  17. file_backup/api/backup/get_schema/schema.json +6 -0
  18. file_backup/api/backup/metadata/__init__.py +1 -0
  19. file_backup/api/backup/metadata/implementation.py +11 -0
  20. file_backup/api/backup/metadata/metadata.json +53 -0
  21. file_backup/api/backup/metadata/schema.json +6 -0
  22. file_backup/api/help/__init__.py +15 -0
  23. file_backup/api/help/execute/__init__.py +1 -0
  24. file_backup/api/help/execute/implementation.py +66 -0
  25. file_backup/api/help/execute/metadata.json +34 -0
  26. file_backup/api/help/execute/schema.json +23 -0
  27. file_backup/api/help/get_schema/__init__.py +1 -0
  28. file_backup/api/help/get_schema/implementation.py +15 -0
  29. file_backup/api/help/get_schema/metadata.json +20 -0
  30. file_backup/api/help/get_schema/schema.json +6 -0
  31. file_backup/api/help/metadata/__init__.py +1 -0
  32. file_backup/api/help/metadata/implementation.py +15 -0
  33. file_backup/api/help/metadata/metadata.json +20 -0
  34. file_backup/api/help/metadata/schema.json +6 -0
  35. file_backup/api/info/USER_GUIDE.md +101 -0
  36. file_backup/api/info/__init__.py +15 -0
  37. file_backup/api/info/execute/__init__.py +1 -0
  38. file_backup/api/info/execute/implementation.py +79 -0
  39. file_backup/api/info/execute/metadata.json +34 -0
  40. file_backup/api/info/execute/schema.json +19 -0
  41. file_backup/api/info/get_schema/__init__.py +1 -0
  42. file_backup/api/info/get_schema/implementation.py +15 -0
  43. file_backup/api/info/get_schema/metadata.json +20 -0
  44. file_backup/api/info/get_schema/schema.json +6 -0
  45. file_backup/api/info/metadata/__init__.py +1 -0
  46. file_backup/api/info/metadata/implementation.py +15 -0
  47. file_backup/api/info/metadata/metadata.json +20 -0
  48. file_backup/api/info/metadata/schema.json +6 -0
  49. file_backup/api/inspect_point/__init__.py +15 -0
  50. file_backup/api/inspect_point/execute/__init__.py +1 -0
  51. file_backup/api/inspect_point/execute/implementation.py +12 -0
  52. file_backup/api/inspect_point/execute/metadata.json +64 -0
  53. file_backup/api/inspect_point/execute/schema.json +42 -0
  54. file_backup/api/inspect_point/get_schema/__init__.py +1 -0
  55. file_backup/api/inspect_point/get_schema/implementation.py +11 -0
  56. file_backup/api/inspect_point/get_schema/metadata.json +44 -0
  57. file_backup/api/inspect_point/get_schema/schema.json +6 -0
  58. file_backup/api/inspect_point/metadata/__init__.py +1 -0
  59. file_backup/api/inspect_point/metadata/implementation.py +11 -0
  60. file_backup/api/inspect_point/metadata/metadata.json +44 -0
  61. file_backup/api/inspect_point/metadata/schema.json +6 -0
  62. file_backup/api/list_files/__init__.py +15 -0
  63. file_backup/api/list_files/execute/__init__.py +1 -0
  64. file_backup/api/list_files/execute/implementation.py +12 -0
  65. file_backup/api/list_files/execute/metadata.json +64 -0
  66. file_backup/api/list_files/execute/schema.json +42 -0
  67. file_backup/api/list_files/get_schema/__init__.py +1 -0
  68. file_backup/api/list_files/get_schema/implementation.py +11 -0
  69. file_backup/api/list_files/get_schema/metadata.json +44 -0
  70. file_backup/api/list_files/get_schema/schema.json +6 -0
  71. file_backup/api/list_files/metadata/__init__.py +1 -0
  72. file_backup/api/list_files/metadata/implementation.py +11 -0
  73. file_backup/api/list_files/metadata/metadata.json +44 -0
  74. file_backup/api/list_files/metadata/schema.json +6 -0
  75. file_backup/api/list_points/__init__.py +15 -0
  76. file_backup/api/list_points/execute/__init__.py +1 -0
  77. file_backup/api/list_points/execute/implementation.py +12 -0
  78. file_backup/api/list_points/execute/metadata.json +55 -0
  79. file_backup/api/list_points/execute/schema.json +37 -0
  80. file_backup/api/list_points/get_schema/__init__.py +1 -0
  81. file_backup/api/list_points/get_schema/implementation.py +11 -0
  82. file_backup/api/list_points/get_schema/metadata.json +35 -0
  83. file_backup/api/list_points/get_schema/schema.json +6 -0
  84. file_backup/api/list_points/metadata/__init__.py +1 -0
  85. file_backup/api/list_points/metadata/implementation.py +11 -0
  86. file_backup/api/list_points/metadata/metadata.json +35 -0
  87. file_backup/api/list_points/metadata/schema.json +6 -0
  88. file_backup/api/recover_catalog/__init__.py +15 -0
  89. file_backup/api/recover_catalog/execute/__init__.py +1 -0
  90. file_backup/api/recover_catalog/execute/implementation.py +12 -0
  91. file_backup/api/recover_catalog/execute/metadata.json +82 -0
  92. file_backup/api/recover_catalog/execute/schema.json +55 -0
  93. file_backup/api/recover_catalog/get_schema/__init__.py +1 -0
  94. file_backup/api/recover_catalog/get_schema/implementation.py +11 -0
  95. file_backup/api/recover_catalog/get_schema/metadata.json +62 -0
  96. file_backup/api/recover_catalog/get_schema/schema.json +6 -0
  97. file_backup/api/recover_catalog/metadata/__init__.py +1 -0
  98. file_backup/api/recover_catalog/metadata/implementation.py +11 -0
  99. file_backup/api/recover_catalog/metadata/metadata.json +62 -0
  100. file_backup/api/recover_catalog/metadata/schema.json +6 -0
  101. file_backup/api/restore/__init__.py +15 -0
  102. file_backup/api/restore/execute/__init__.py +1 -0
  103. file_backup/api/restore/execute/implementation.py +12 -0
  104. file_backup/api/restore/execute/metadata.json +82 -0
  105. file_backup/api/restore/execute/schema.json +55 -0
  106. file_backup/api/restore/get_schema/__init__.py +1 -0
  107. file_backup/api/restore/get_schema/implementation.py +11 -0
  108. file_backup/api/restore/get_schema/metadata.json +62 -0
  109. file_backup/api/restore/get_schema/schema.json +6 -0
  110. file_backup/api/restore/metadata/__init__.py +1 -0
  111. file_backup/api/restore/metadata/implementation.py +11 -0
  112. file_backup/api/restore/metadata/metadata.json +62 -0
  113. file_backup/api/restore/metadata/schema.json +6 -0
  114. file_backup/api/restore_file/__init__.py +15 -0
  115. file_backup/api/restore_file/execute/__init__.py +1 -0
  116. file_backup/api/restore_file/execute/implementation.py +12 -0
  117. file_backup/api/restore_file/execute/metadata.json +100 -0
  118. file_backup/api/restore_file/execute/schema.json +68 -0
  119. file_backup/api/restore_file/get_schema/__init__.py +1 -0
  120. file_backup/api/restore_file/get_schema/implementation.py +11 -0
  121. file_backup/api/restore_file/get_schema/metadata.json +80 -0
  122. file_backup/api/restore_file/get_schema/schema.json +6 -0
  123. file_backup/api/restore_file/metadata/__init__.py +1 -0
  124. file_backup/api/restore_file/metadata/implementation.py +11 -0
  125. file_backup/api/restore_file/metadata/metadata.json +80 -0
  126. file_backup/api/restore_file/metadata/schema.json +6 -0
  127. file_backup/api/verify/__init__.py +15 -0
  128. file_backup/api/verify/execute/__init__.py +1 -0
  129. file_backup/api/verify/execute/implementation.py +12 -0
  130. file_backup/api/verify/execute/metadata.json +64 -0
  131. file_backup/api/verify/execute/schema.json +42 -0
  132. file_backup/api/verify/get_schema/__init__.py +1 -0
  133. file_backup/api/verify/get_schema/implementation.py +11 -0
  134. file_backup/api/verify/get_schema/metadata.json +44 -0
  135. file_backup/api/verify/get_schema/schema.json +6 -0
  136. file_backup/api/verify/metadata/__init__.py +1 -0
  137. file_backup/api/verify/metadata/implementation.py +11 -0
  138. file_backup/api/verify/metadata/metadata.json +44 -0
  139. file_backup/api/verify/metadata/schema.json +6 -0
  140. file_backup/cli.py +332 -0
  141. file_backup/errors.py +100 -0
  142. file_backup/types.py +159 -0
  143. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/01_backup_lifecycle.py +72 -0
  144. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/02_catalog_browse.py +51 -0
  145. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/03_restore_and_preview.py +88 -0
  146. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/04_verify_and_chain_failure.py +58 -0
  147. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/05_typed_errors_and_conflicts.py +78 -0
  148. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/06_catalog_configuration.py +57 -0
  149. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/07_metadata_and_schemas.py +63 -0
  150. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/08_help_and_info.py +45 -0
  151. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/09_thin_cli.py +70 -0
  152. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/10_offline_installed_restore.py +52 -0
  153. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/11_recover_catalog.py +60 -0
  154. vvz_file_backup-0.1.1.data/data/share/vvz-file-backup/examples/manifest.json +15 -0
  155. vvz_file_backup-0.1.1.dist-info/METADATA +6 -0
  156. vvz_file_backup-0.1.1.dist-info/RECORD +159 -0
  157. vvz_file_backup-0.1.1.dist-info/WHEEL +5 -0
  158. vvz_file_backup-0.1.1.dist-info/entry_points.txt +2 -0
  159. vvz_file_backup-0.1.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,60 @@
1
+ """Independent backup, saved-point inspection, verification, and restore API."""
2
+
3
+ from .api.backup import Backup
4
+ from .api.list_points import ListPoints
5
+ from .api.inspect_point import InspectPoint
6
+ from .api.list_files import ListFiles
7
+ from .api.restore import Restore
8
+ from .api.restore_file import RestoreFile
9
+ from .api.verify import Verify
10
+ from .api.recover_catalog import RecoverCatalog
11
+ from .api.help import Help
12
+ from .api.info import Info
13
+ from ._configuration import DEFAULT_CATALOG_DIRECTORY
14
+ from .errors import FileBackupError, ArgumentError, PointNotFoundError, FileNotFoundError, ArchiveMissingError, ArchiveCorruptError, ChainError, BackupIOError, CatalogError, ConflictError, SourceChangedError, SectionNotFoundError, CommandNotFoundError
15
+ from .types import BackupCreated, BackupUnchanged, BackupPreview, PointInfo, PointList, FileList, RestoreResult, FileRestoreResult, RestorePreview, FileRestorePreview, VerificationReport, CatalogRecoveryResult, CatalogRecoveryPreview, HelpPage, InfoTOC, InfoSection, CLIResult, CLIError
16
+
17
+ __all__ = [
18
+ 'DEFAULT_CATALOG_DIRECTORY',
19
+ 'Backup',
20
+ 'ListPoints',
21
+ 'InspectPoint',
22
+ 'ListFiles',
23
+ 'Restore',
24
+ 'RestoreFile',
25
+ 'Verify',
26
+ 'RecoverCatalog',
27
+ 'Help',
28
+ 'Info',
29
+ 'FileBackupError',
30
+ 'ArgumentError',
31
+ 'PointNotFoundError',
32
+ 'FileNotFoundError',
33
+ 'ArchiveMissingError',
34
+ 'ArchiveCorruptError',
35
+ 'ChainError',
36
+ 'BackupIOError',
37
+ 'CatalogError',
38
+ 'ConflictError',
39
+ 'SourceChangedError',
40
+ 'SectionNotFoundError',
41
+ 'CommandNotFoundError',
42
+ 'BackupCreated',
43
+ 'BackupUnchanged',
44
+ 'BackupPreview',
45
+ 'PointInfo',
46
+ 'PointList',
47
+ 'FileList',
48
+ 'RestoreResult',
49
+ 'FileRestoreResult',
50
+ 'RestorePreview',
51
+ 'FileRestorePreview',
52
+ 'VerificationReport',
53
+ 'CatalogRecoveryResult',
54
+ 'CatalogRecoveryPreview',
55
+ 'HelpPage',
56
+ 'InfoTOC',
57
+ 'InfoSection',
58
+ 'CLIResult',
59
+ 'CLIError',
60
+ ]
@@ -0,0 +1,100 @@
1
+ """Archive creation and strict catalog-bound integrity checks."""
2
+ import hashlib
3
+ import io
4
+ import json
5
+ from pathlib import Path
6
+ import tarfile
7
+
8
+ from ._validation import relative_path, safe_absolute
9
+ from .errors import ArchiveMissingError, ArchiveCorruptError, BackupIOError
10
+
11
+
12
+ def digest(data):
13
+ return hashlib.sha256(data).hexdigest()
14
+
15
+
16
+ def encode(manifest):
17
+ return json.dumps(manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
18
+
19
+
20
+ def create(path, manifest, content):
21
+ with tarfile.open(path, "w:gz") as archive:
22
+ for name, data in [("manifest.json", encode(manifest)), *[("files/" + n, b) for n, b in sorted(content.items())]]:
23
+ member = tarfile.TarInfo(name)
24
+ member.size = len(data)
25
+ archive.addfile(member, io.BytesIO(data))
26
+
27
+
28
+ def read(repository, row, identity, operation):
29
+ relative = row["archive"]
30
+ path = Path(repository) / relative
31
+ context = dict(operation=operation, point=row["point"], archive=str(path))
32
+ if not relative_path(relative) or relative != f"archives/{row['point']}.tar.gz" or not safe_absolute(str(path)):
33
+ raise ArchiveCorruptError("Unsafe archive path", expected="safe archive", got=relative, **context)
34
+ try:
35
+ data = path.read_bytes()
36
+ except FileNotFoundError as cause:
37
+ raise ArchiveMissingError("Required archive missing", **context) from cause
38
+ except OSError as cause:
39
+ raise BackupIOError("Cannot read archive", operation=operation, path=str(path), phase="read") from cause
40
+ actual = digest(data)
41
+ if actual != row["archive_sha256"]:
42
+ raise ArchiveCorruptError("Archive checksum mismatch", expected=row["archive_sha256"], got=actual, **context)
43
+ try:
44
+ content = {}
45
+ with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive:
46
+ names = set()
47
+ for member in archive:
48
+ if member.name in names or not member.isfile() or not relative_path(member.name):
49
+ raise ValueError("unsafe or duplicate archive member")
50
+ names.add(member.name)
51
+ handle = archive.extractfile(member)
52
+ payload = handle.read()
53
+ if member.name == "manifest.json":
54
+ manifest = json.loads(payload)
55
+ elif member.name.startswith("files/") and relative_path(member.name[6:]):
56
+ content[member.name[6:]] = payload
57
+ else:
58
+ raise ValueError("unexpected member")
59
+ if "manifest.json" not in names:
60
+ raise ValueError("missing manifest")
61
+ if manifest != row["manifest"] or manifest["repository"] != identity:
62
+ raise ValueError("manifest/catalog binding mismatch")
63
+ for field in ("point", "kind", "base_point", "previous_point", "snapshot_at", "completed_at"):
64
+ if manifest[field] != row[field]:
65
+ raise ValueError("manifest point binding mismatch")
66
+ changed = manifest["changed"]
67
+ if len(changed) != len(set(changed)) or set(changed) != set(content):
68
+ raise ValueError("archive member set mismatch")
69
+ for name, item in manifest["files"].items():
70
+ if not relative_path(name) or item["size"] < 0 or item["version"] != item["sha256"]:
71
+ raise ValueError("invalid file manifest")
72
+ for name, payload in content.items():
73
+ item = manifest["files"][name]
74
+ if digest(payload) != item["sha256"] or len(payload) != item["size"]:
75
+ raise ValueError("file integrity mismatch")
76
+ if len(manifest["deleted"]) != len(set(manifest["deleted"])) or any(not relative_path(n) or n in manifest["files"] for n in manifest["deleted"]):
77
+ raise ValueError("invalid tombstones")
78
+ return content
79
+ except (tarfile.TarError, OSError, ValueError, KeyError, TypeError, UnboundLocalError) as cause:
80
+ raise ArchiveCorruptError("Invalid archive or manifest", expected="catalog-bound safe archive", got=str(cause), **context) from cause
81
+
82
+
83
+ def materialize(repository, chain, identity, operation):
84
+ live = {}
85
+ for row in chain:
86
+ updates = read(repository, row, identity, operation)
87
+ manifest = row["manifest"]
88
+ if row["kind"] == "full":
89
+ live = {}
90
+ for name in manifest["deleted"]:
91
+ if name not in live:
92
+ raise ArchiveCorruptError("Tombstone has no prior file", operation=operation, point=row["point"],
93
+ archive=row["archive"], expected="live prior path", got=name)
94
+ del live[name]
95
+ live.update(updates)
96
+ expected = manifest["files"]
97
+ if set(live) != set(expected) or any(digest(payload) != expected[name]["sha256"] or len(payload) != expected[name]["size"] for name, payload in live.items()):
98
+ raise ArchiveCorruptError("Saved slice does not match manifest", operation=operation, point=row["point"],
99
+ archive=row["archive"], expected="exact slice", got="mismatch")
100
+ return live
@@ -0,0 +1,87 @@
1
+ """SQLite saved-point catalog and exact chain selection."""
2
+ import json
3
+ from pathlib import Path
4
+ import sqlite3
5
+
6
+ from .errors import CatalogError, ChainError, PointNotFoundError
7
+ from .types import PointInfo
8
+
9
+ SCHEMA = """
10
+ CREATE TABLE IF NOT EXISTS repository (identity TEXT NOT NULL);
11
+ CREATE TABLE IF NOT EXISTS points (
12
+ sequence INTEGER PRIMARY KEY, point TEXT UNIQUE NOT NULL,
13
+ kind TEXT NOT NULL, base_point TEXT NOT NULL, previous_point TEXT,
14
+ snapshot_at TEXT NOT NULL, completed_at TEXT NOT NULL,
15
+ archive TEXT NOT NULL, archive_sha256 TEXT NOT NULL, manifest TEXT NOT NULL);
16
+ CREATE TABLE IF NOT EXISTS checkpoint (point TEXT NOT NULL);
17
+ CREATE TABLE IF NOT EXISTS objects (path TEXT PRIMARY KEY);
18
+ CREATE TABLE IF NOT EXISTS content_versions (sha256 TEXT PRIMARY KEY, size INTEGER NOT NULL);
19
+ CREATE TABLE IF NOT EXISTS archives (point TEXT PRIMARY KEY, file TEXT NOT NULL, digest TEXT NOT NULL, kind TEXT NOT NULL, state TEXT NOT NULL);
20
+ CREATE TABLE IF NOT EXISTS chains (point TEXT PRIMARY KEY, base_point TEXT NOT NULL, previous_point TEXT);
21
+ CREATE TABLE IF NOT EXISTS point_members (point TEXT NOT NULL, path TEXT NOT NULL, sha256 TEXT NOT NULL, archive TEXT NOT NULL, PRIMARY KEY(point,path));
22
+ CREATE TABLE IF NOT EXISTS tombstones (point TEXT NOT NULL, path TEXT NOT NULL, PRIMARY KEY(point,path));
23
+ """
24
+
25
+
26
+ def load(repository, operation, catalog=None):
27
+ catalog = Path(catalog) if catalog else Path(repository) / "catalog.sqlite3"
28
+ if catalog.is_symlink():
29
+ raise CatalogError("Catalog must not be a symlink", operation=operation, catalog=str(catalog), phase="open")
30
+ if not catalog.exists():
31
+ archives = Path(repository) / "archives"
32
+ if archives.exists() and any(archives.glob("*.tar.gz")):
33
+ raise CatalogError("Catalog missing while archives exist", operation=operation,
34
+ catalog=str(catalog), phase="open")
35
+ return None, []
36
+ try:
37
+ with sqlite3.connect(catalog.as_uri() + "?mode=ro", uri=True) as connection:
38
+ connection.row_factory = sqlite3.Row
39
+ if connection.execute("PRAGMA integrity_check").fetchone()[0] != "ok":
40
+ raise sqlite3.DatabaseError("catalog integrity check failed")
41
+ identity = connection.execute("SELECT identity FROM repository").fetchone()[0]
42
+ points = []
43
+ for row in connection.execute("SELECT * FROM points ORDER BY sequence"):
44
+ point = dict(row)
45
+ point["manifest"] = json.loads(point["manifest"])
46
+ points.append(point)
47
+ checkpoint = connection.execute("SELECT point FROM checkpoint").fetchall()
48
+ missing_base_rebootstrap = operation == "backup" and not points and len(checkpoint) == 1
49
+ if not missing_base_rebootstrap and (len(checkpoint) != bool(points) or points and checkpoint[0][0] != points[-1]["point"]):
50
+ raise sqlite3.DatabaseError("checkpoint does not match last committed point")
51
+ return identity, points
52
+ except (sqlite3.Error, ValueError, KeyError, IndexError, TypeError) as cause:
53
+ raise CatalogError("Cannot read catalog", operation=operation, catalog=str(catalog), phase="read") from cause
54
+
55
+
56
+ def chain(points, point, operation):
57
+ by_id = {row["point"]: row for row in points}
58
+ if point not in by_id:
59
+ raise PointNotFoundError("Saved point not found", operation=operation, point=point)
60
+ result, seen = [], set()
61
+ selected = by_id[point]
62
+ current = selected
63
+ while True:
64
+ identifier = current["point"]
65
+ if identifier in seen:
66
+ raise ChainError("Cyclic saved-point chain", operation=operation, point=point, missing_dependency=identifier)
67
+ seen.add(identifier)
68
+ result.append(current)
69
+ if current["kind"] == "full":
70
+ if current["base_point"] != identifier or current["previous_point"] is not None or selected["base_point"] != identifier:
71
+ raise ChainError("Invalid full base", operation=operation, point=point, missing_dependency=identifier)
72
+ return list(reversed(result))
73
+ previous = by_id.get(current["previous_point"])
74
+ if (current["kind"] != "incremental" or previous is None
75
+ or previous["sequence"] >= current["sequence"]
76
+ or previous["base_point"] != current["base_point"]):
77
+ raise ChainError("Missing or unordered dependency", operation=operation, point=point,
78
+ missing_dependency=current["previous_point"])
79
+ current = previous
80
+
81
+
82
+ def info(points, row, operation):
83
+ members = row["manifest"]["files"]
84
+ return PointInfo(row["point"], row["kind"], row["base_point"], row["previous_point"],
85
+ row["snapshot_at"], row["completed_at"], row["archive"], row["archive_sha256"],
86
+ tuple(item["point"] for item in chain(points, row["point"], operation)),
87
+ len(members), len(row["manifest"]["deleted"]))
@@ -0,0 +1,40 @@
1
+ """Explicit catalog directory selection without ambient configuration."""
2
+ from collections.abc import Mapping
3
+ import os
4
+ from pathlib import Path
5
+
6
+ from ._validation import invalid
7
+
8
+ DEFAULT_CATALOG_DIRECTORY = "."
9
+
10
+
11
+ def directory(repository, value, operation, parameter):
12
+ if type(value) is not str:
13
+ invalid(operation, parameter, "type", "nonempty directory string", value)
14
+ if not value or "\0" in value:
15
+ invalid(operation, parameter, "value", "nonempty NUL-free directory", value)
16
+ raw = Path(repository) / value
17
+ # Check lexical components before normalizing .. away.
18
+ for part in (raw, *raw.parents):
19
+ if part.is_symlink() or part.exists() and not part.is_dir():
20
+ invalid(operation, parameter, "value", "directory without symlink components", value)
21
+ return Path(os.path.abspath(raw))
22
+
23
+
24
+ def resolve_catalog(repository, catalog_directory, config, operation):
25
+ if config is not None and not isinstance(config, Mapping):
26
+ invalid(operation, "config", "type", "mapping or None", config)
27
+ settings = {} if config is None else config
28
+ for key in settings:
29
+ if type(key) is not str or key != "catalog_directory":
30
+ invalid(operation, "config", "unknown", ["catalog_directory"], key)
31
+ configured = None
32
+ if "catalog_directory" in settings:
33
+ configured = directory(repository, settings["catalog_directory"], operation, "config.catalog_directory")
34
+ explicit = None
35
+ if catalog_directory is not None:
36
+ explicit = directory(repository, catalog_directory, operation, "catalog_directory")
37
+ selected = explicit if explicit is not None else configured
38
+ if selected is None:
39
+ selected = directory(repository, DEFAULT_CATALOG_DIRECTORY, operation, "catalog_directory")
40
+ return selected / "catalog.sqlite3"
@@ -0,0 +1,16 @@
1
+ """Internal namespace base for classmethod-only operations."""
2
+
3
+ from ._resources import resource
4
+
5
+ class _OperationNamespace:
6
+ """Operation declarations cannot be instantiated."""
7
+ def __new__(cls, *args: object, **kwargs: object) -> object:
8
+ raise TypeError("operation namespaces cannot be instantiated")
9
+
10
+ def __init_subclass__(cls, **kwargs: object) -> None:
11
+ super().__init_subclass__(**kwargs)
12
+ declaration = resource(cls.__module__.rsplit(".", 1)[-1], "metadata.json")
13
+ for attribute, field in (("name", "name"), ("version", "version"),
14
+ ("descr", "description"), ("category", "category"),
15
+ ("author", "author"), ("email", "email")):
16
+ setattr(cls, attribute, declaration[field])
@@ -0,0 +1,356 @@
1
+ """Filesystem backup operations over committed catalog points."""
2
+ from contextlib import contextmanager
3
+ from datetime import datetime, timezone
4
+ import fcntl
5
+ import os
6
+ from pathlib import Path
7
+ import shutil
8
+ import sqlite3
9
+ import stat
10
+ import tempfile
11
+ import uuid
12
+
13
+ from . import _archives as archives
14
+ from . import _catalog as catalog
15
+ from ._validation import invalid, safe_absolute
16
+ from .errors import (ArchiveCorruptError, ArchiveMissingError, BackupIOError, CatalogError, ChainError,
17
+ ConflictError, FileNotFoundError as SavedFileNotFoundError, SourceChangedError)
18
+ from .types import (BackupCreated, BackupUnchanged, BackupPreview, PointList, FileList,
19
+ RestoreResult, RestorePreview, FileRestoreResult, FileRestorePreview,
20
+ VerificationReport, CatalogRecoveryResult, CatalogRecoveryPreview)
21
+
22
+
23
+ def now():
24
+ return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
25
+
26
+
27
+ def inside(path, root):
28
+ return Path(path).is_relative_to(Path(root))
29
+
30
+
31
+ def scan(source, operation):
32
+ root = Path(source)
33
+ if not root.is_dir():
34
+ invalid(operation, "source", "value", "existing directory", source)
35
+ content, signatures = {}, {}
36
+ try:
37
+ for directory, dirs, names in os.walk(root, followlinks=False):
38
+ for name in sorted(dirs + names):
39
+ path = Path(directory) / name
40
+ before = path.lstat()
41
+ if stat.S_ISDIR(before.st_mode):
42
+ continue
43
+ if not stat.S_ISREG(before.st_mode):
44
+ invalid(operation, "source", "value", "regular files only", str(path))
45
+ with path.open("rb") as handle:
46
+ opened = os.fstat(handle.fileno())
47
+ if (opened.st_ino, opened.st_dev) != (before.st_ino, before.st_dev):
48
+ raise SourceChangedError("Source changed before read", operation=operation, path=str(path))
49
+ payload = handle.read()
50
+ after = os.fstat(handle.fileno())
51
+ signature = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns)
52
+ if signature != (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns):
53
+ raise SourceChangedError("Source changed during read", operation=operation, path=str(path))
54
+ relative = path.relative_to(root).as_posix()
55
+ content[relative] = payload
56
+ signatures[relative] = signature
57
+ return content, signatures
58
+ except OSError as cause:
59
+ raise BackupIOError("Cannot scan source", operation=operation, path=str(root), phase="scan") from cause
60
+
61
+
62
+ @contextmanager
63
+ def writer(repository, catalog_path):
64
+ root = Path(repository)
65
+ root.mkdir(parents=True, exist_ok=True)
66
+ catalog_path.parent.mkdir(parents=True, exist_ok=True)
67
+ with (catalog_path.parent / ".writer.lock").open("a+b") as handle:
68
+ try:
69
+ fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
70
+ except BlockingIOError as cause:
71
+ raise ConflictError("Another backup writer is active", operation="backup", path=str(root), reason="writer_locked") from cause
72
+ try:
73
+ yield
74
+ finally:
75
+ fcntl.flock(handle, fcntl.LOCK_UN)
76
+
77
+
78
+ def backup(repository, source, dry_run=False, catalog_path=None):
79
+ operation = "backup"
80
+ catalog_path = Path(catalog_path) if catalog_path else Path(repository) / "catalog.sqlite3"
81
+ if inside(repository, source) or inside(source, repository):
82
+ invalid(operation, "source", "value", "disjoint repository and source", source)
83
+ if inside(catalog_path.parent, source) or inside(source, catalog_path.parent):
84
+ invalid(operation, "catalog_directory", "value", "disjoint catalog and source", str(catalog_path.parent))
85
+ initial = scan(source, operation)
86
+ if dry_run:
87
+ return _backup(repository, source, True, initial, catalog_path)
88
+ # Missing selected catalog must fail before creating its directory or lock.
89
+ catalog.load(repository, operation, catalog_path)
90
+ try:
91
+ with writer(repository, catalog_path):
92
+ return _backup(repository, source, False, initial, catalog_path)
93
+ except OSError as cause:
94
+ raise BackupIOError("Cannot write backup", operation=operation, path=repository, phase="write") from cause
95
+
96
+
97
+ def _backup(repository, source, dry_run, initial, catalog_path):
98
+ operation = "backup"
99
+ content, signatures = initial
100
+ identity, points = catalog.load(repository, operation, catalog_path)
101
+ previous = points[-1] if points else None
102
+ base = next((r for r in points if previous and r["point"] == previous["base_point"]), None)
103
+ full = previous is None or base is None or not (Path(repository) / base["archive"]).exists()
104
+ if previous and not full:
105
+ archives.materialize(repository, catalog.chain(points, previous["point"], operation), identity, operation)
106
+ before = previous["manifest"]["files"] if previous else {}
107
+ added = set(content) - set(before)
108
+ modified = {n for n in set(content) & set(before) if archives.digest(content[n]) != before[n]["sha256"]}
109
+ deleted = set(before) - set(content)
110
+ action = "full" if full else "incremental" if added or modified or deleted else "unchanged"
111
+ if dry_run:
112
+ return BackupPreview("preview", action, len(added), len(modified), len(deleted), previous["point"] if previous else None)
113
+ if action == "unchanged":
114
+ return BackupUnchanged("unchanged", previous["point"], None, 0, 0, 0)
115
+ root = Path(repository)
116
+ archive_dir = root / "archives"
117
+ if not safe_absolute(str(archive_dir)):
118
+ invalid(operation, "repository", "value", "archive directory without symlinks", str(archive_dir))
119
+ archive_dir.mkdir(exist_ok=True)
120
+ # A finalized archive without a committed catalog row is a crash orphan.
121
+ known = {r["archive"] for r in points}
122
+ for orphan in archive_dir.glob("*.tar.gz"):
123
+ if orphan.relative_to(root).as_posix() not in known:
124
+ quarantine = root / "quarantine"
125
+ quarantine.mkdir(exist_ok=True)
126
+ os.replace(orphan, quarantine / (orphan.name + "." + uuid.uuid4().hex))
127
+ identity = identity or str(uuid.uuid4())
128
+ marker = root / ".repository-id"
129
+ if not marker.exists():
130
+ marker.write_text(identity, encoding="ascii")
131
+ elif marker.read_text(encoding="ascii") != identity:
132
+ raise CatalogError("Repository identity mismatch", operation=operation, catalog=str(catalog_path), phase="identity")
133
+ point = str(uuid.uuid4())
134
+ stamp = now()
135
+ relative = f"archives/{point}.tar.gz"
136
+ manifest = dict(repository=identity, point=point, kind=action,
137
+ base_point=point if full else previous["base_point"],
138
+ previous_point=None if full else previous["point"],
139
+ snapshot_at=stamp, completed_at=stamp,
140
+ files={name:dict(sha256=archives.digest(data), version=archives.digest(data), size=len(data),
141
+ archive=relative if full or name in added or name in modified else before[name]["archive"])
142
+ for name, data in sorted(content.items())},
143
+ changed=sorted(content if full else added | modified), deleted=[] if full else sorted(deleted))
144
+ temporary = None
145
+ final = root / relative
146
+ connection = None
147
+ committed = False
148
+ catalog_existed = catalog_path.exists()
149
+ try:
150
+ descriptor, temporary = tempfile.mkstemp(prefix=".archive-", dir=archive_dir)
151
+ os.close(descriptor)
152
+ archives.create(temporary, manifest, {n:content[n] for n in manifest["changed"]})
153
+ with open(temporary, "rb") as handle:
154
+ os.fsync(handle.fileno())
155
+ after, after_signatures = scan(source, operation)
156
+ if content != after or signatures != after_signatures:
157
+ raise SourceChangedError("Source changed during backup", operation=operation, path=source)
158
+ archive_hash = archives.digest(Path(temporary).read_bytes())
159
+ os.replace(temporary, final)
160
+ temporary = None
161
+ directory_descriptor = os.open(archive_dir, os.O_RDONLY)
162
+ try:
163
+ os.fsync(directory_descriptor)
164
+ finally:
165
+ os.close(directory_descriptor)
166
+ after, after_signatures = scan(source, operation)
167
+ if content != after or signatures != after_signatures:
168
+ raise SourceChangedError("Source changed before checkpoint", operation=operation, path=source)
169
+ try:
170
+ connection = sqlite3.connect(catalog_path, timeout=0)
171
+ connection.executescript(catalog.SCHEMA)
172
+ connection.execute("BEGIN IMMEDIATE")
173
+ if not connection.execute("SELECT identity FROM repository").fetchone():
174
+ connection.execute("INSERT INTO repository VALUES (?)", (identity,))
175
+ connection.execute("INSERT INTO points(point,kind,base_point,previous_point,snapshot_at,completed_at,archive,archive_sha256,manifest) VALUES (?,?,?,?,?,?,?,?,?)",
176
+ (point, action, manifest["base_point"], manifest["previous_point"], stamp, stamp, relative, archive_hash, archives.encode(manifest).decode("utf-8")))
177
+ connection.execute("INSERT INTO archives VALUES (?,?,?,?,?)", (point, relative, archive_hash, action, "complete"))
178
+ connection.execute("INSERT INTO chains VALUES (?,?,?)", (point, manifest["base_point"], manifest["previous_point"]))
179
+ for name, item in manifest["files"].items():
180
+ connection.execute("INSERT OR IGNORE INTO objects VALUES (?)", (name,))
181
+ connection.execute("INSERT OR IGNORE INTO content_versions VALUES (?,?)", (item["sha256"], item["size"]))
182
+ connection.execute("INSERT INTO point_members VALUES (?,?,?,?)", (point, name, item["sha256"], item["archive"]))
183
+ for name in manifest["deleted"]:
184
+ connection.execute("INSERT INTO tombstones VALUES (?,?)", (point, name))
185
+ connection.execute("DELETE FROM checkpoint")
186
+ connection.execute("INSERT INTO checkpoint VALUES (?)", (point,))
187
+ connection.commit()
188
+ committed = True
189
+ except sqlite3.Error as cause:
190
+ if connection:
191
+ connection.rollback()
192
+ raise CatalogError("Cannot commit saved point", operation=operation, catalog=str(catalog_path), phase="commit") from cause
193
+ return BackupCreated("created", point, action, manifest["base_point"], manifest["previous_point"], str(final), stamp,
194
+ len(added), len(modified), len(deleted))
195
+ finally:
196
+ if connection:
197
+ connection.close()
198
+ if temporary and Path(temporary).exists():
199
+ Path(temporary).unlink()
200
+ if not committed and final.exists():
201
+ final.unlink()
202
+ if not committed and not catalog_existed:
203
+ catalog_path.unlink(missing_ok=True)
204
+
205
+
206
+ def selected(repository, point, operation, catalog_path=None):
207
+ identity, points = catalog.load(repository, operation, catalog_path)
208
+ chain = catalog.chain(points, point, operation)
209
+ try:
210
+ content = archives.materialize(repository, chain, identity, operation)
211
+ except (ArchiveMissingError, ArchiveCorruptError) as cause:
212
+ if operation in ("restore", "restore_file") and cause.point != point:
213
+ raise ChainError("Saved point dependency is unavailable", operation=operation,
214
+ point=point, missing_dependency=cause.point) from cause
215
+ raise
216
+ return identity, points, chain, content
217
+
218
+
219
+ def list_points(repository, catalog_path=None):
220
+ _, points = catalog.load(repository, "list_points", catalog_path)
221
+ return PointList(tuple(catalog.info(points, row, "list_points") for row in sorted(points, key=lambda r:(r["completed_at"], r["point"]))))
222
+
223
+
224
+ def inspect_point(repository, point, catalog_path=None):
225
+ _, points = catalog.load(repository, "inspect_point", catalog_path)
226
+ chain = catalog.chain(points, point, "inspect_point")
227
+ return catalog.info(points, chain[-1], "inspect_point")
228
+
229
+
230
+ def list_files(repository, point, catalog_path=None):
231
+ _, points = catalog.load(repository, "list_files", catalog_path)
232
+ chain = catalog.chain(points, point, "list_files")
233
+ return FileList(tuple(dict(path=name, **item) for name,item in sorted(chain[-1]["manifest"]["files"].items())))
234
+
235
+
236
+ def destination_check(repository, destination, operation, catalog_path=None):
237
+ if inside(destination, repository) or inside(repository, destination):
238
+ invalid(operation, "destination", "value", "outside repository", destination)
239
+ if catalog_path is not None and operation in ("restore", "restore_file") and inside(destination, Path(catalog_path).parent):
240
+ invalid(operation, "destination", "value", "outside catalog directory", destination)
241
+
242
+
243
+ def restore(repository, point, destination, dry_run=False, catalog_path=None):
244
+ operation = "restore"
245
+ destination_check(repository, destination, operation, catalog_path)
246
+ target = Path(destination)
247
+ if target.exists() and (not target.is_dir() or any(target.iterdir())):
248
+ raise ConflictError("Restore destination must be empty", operation=operation, path=destination, reason="nonempty_destination")
249
+ _, _, _, content = selected(repository, point, operation, catalog_path)
250
+ size = sum(map(len, content.values()))
251
+ if dry_run:
252
+ return RestorePreview("preview", point, destination, len(content), size)
253
+ staging = None
254
+ try:
255
+ target.parent.mkdir(parents=True, exist_ok=True)
256
+ staging = Path(tempfile.mkdtemp(prefix=".restore-", dir=target.parent))
257
+ for name, data in content.items():
258
+ path = staging / name
259
+ path.parent.mkdir(parents=True, exist_ok=True)
260
+ with path.open("wb") as handle:
261
+ handle.write(data)
262
+ handle.flush()
263
+ os.fsync(handle.fileno())
264
+ if target.exists() and any(target.iterdir()):
265
+ raise ConflictError("Destination changed during restore", operation=operation, path=destination, reason="nonempty_destination")
266
+ os.replace(staging, target)
267
+ staging = None
268
+ return RestoreResult("restored", point, destination, len(content), size)
269
+ except OSError as cause:
270
+ raise BackupIOError("Cannot publish restored slice", operation=operation, path=destination, phase="restore") from cause
271
+ finally:
272
+ if staging:
273
+ shutil.rmtree(staging)
274
+
275
+
276
+ def restore_file(repository, point, path, destination, overwrite=False, dry_run=False, catalog_path=None):
277
+ operation = "restore_file"
278
+ destination_check(repository, destination, operation, catalog_path)
279
+ target = Path(destination)
280
+ if not safe_absolute(str(target)):
281
+ invalid(operation, "destination", "value", "no symlink components", str(target))
282
+ if target.exists() and (not overwrite or not target.is_file()):
283
+ raise ConflictError("Destination file exists", operation=operation, path=str(target), reason="file_exists")
284
+ _, _, _, content = selected(repository, point, operation, catalog_path)
285
+ if path not in content:
286
+ raise SavedFileNotFoundError("File is not live at point", operation=operation, point=point, path=path)
287
+ payload = content[path]
288
+ sha = archives.digest(payload)
289
+ if dry_run:
290
+ return FileRestorePreview("preview", point, path, sha, len(payload))
291
+ temporary = None
292
+ try:
293
+ target.parent.mkdir(parents=True, exist_ok=True)
294
+ descriptor, temporary = tempfile.mkstemp(prefix=".restore-file-", dir=target.parent)
295
+ with os.fdopen(descriptor, "wb") as handle:
296
+ handle.write(payload)
297
+ handle.flush()
298
+ os.fsync(handle.fileno())
299
+ if overwrite:
300
+ os.replace(temporary, target)
301
+ else:
302
+ os.link(temporary, target)
303
+ os.unlink(temporary)
304
+ temporary = None
305
+ return FileRestoreResult("restored", point, path, str(target), sha, len(payload))
306
+ except OSError as cause:
307
+ raise BackupIOError("Cannot restore file", operation=operation, path=str(target), phase="restore") from cause
308
+ finally:
309
+ if temporary:
310
+ Path(temporary).unlink(missing_ok=True)
311
+
312
+
313
+ def verify(repository, point, catalog_path=None):
314
+ _, _, chain, content = selected(repository, point, "verify", catalog_path)
315
+ return VerificationReport("verified", point, len(chain), len(content), sum(map(len, content.values())))
316
+
317
+
318
+ def recover_catalog(repository, destination, catalog_copy, dry_run=False, catalog_path=None):
319
+ operation = "recover_catalog"
320
+ destination_check(repository, destination, operation)
321
+ target = Path(destination)
322
+ if catalog_path is not None and target == Path(catalog_path):
323
+ invalid(operation, "destination", "value", "different from active catalog", destination)
324
+ if target.exists():
325
+ raise ConflictError("Catalog destination exists", operation=operation, path=destination, reason="destination_exists")
326
+ identity, points = catalog.load(repository, operation, catalog_copy)
327
+ marker = Path(repository) / ".repository-id"
328
+ try:
329
+ expected = marker.read_text(encoding="ascii")
330
+ except OSError as cause:
331
+ raise CatalogError("Repository identity is unavailable", operation=operation, catalog=catalog_copy, phase="identity") from cause
332
+ if identity != expected:
333
+ raise CatalogError("Catalog copy belongs to another repository", operation=operation, catalog=catalog_copy, phase="identity")
334
+ for row in points:
335
+ archives.materialize(repository, catalog.chain(points, row["point"], operation), identity, operation)
336
+ if dry_run:
337
+ return CatalogRecoveryPreview("preview", destination, len(points))
338
+ temporary = None
339
+ try:
340
+ target.parent.mkdir(parents=True, exist_ok=True)
341
+ descriptor, temporary = tempfile.mkstemp(prefix=".catalog-", dir=target.parent)
342
+ os.close(descriptor)
343
+ with sqlite3.connect(Path(catalog_copy).as_uri() + "?mode=ro", uri=True) as source:
344
+ with sqlite3.connect(temporary) as output:
345
+ source.backup(output)
346
+ with open(temporary, "rb") as handle:
347
+ os.fsync(handle.fileno())
348
+ os.link(temporary, target)
349
+ os.unlink(temporary)
350
+ temporary = None
351
+ return CatalogRecoveryResult("recovered", destination, len(points))
352
+ except (OSError, sqlite3.Error) as cause:
353
+ raise CatalogError("Cannot publish catalog copy", operation=operation, catalog=destination, phase="recover") from cause
354
+ finally:
355
+ if temporary:
356
+ Path(temporary).unlink(missing_ok=True)