scalio-backup 0.1.0__py3-none-any.whl

scalio_backup/__init__.py

@@ -0,0 +1,24 @@
+"""Encrypted export and import of a Scalio app instance (backup, restore, self-hosted <-> SaaS move)."""
+from .age import AgeError, WrongPassword
+from .engine import (
+    Bundle,
+    BundleError,
+    IncompatibleBundle,
+    apply_import,
+    blank_paths,
+    check_compatible,
+    check_password,
+    export_instance,
+    open_bundle,
+    purge,
+    recover,
+    rollback,
+    status,
+)
+from .spec import CACHE, CLASSES, CONFIG, DATA, EPHEMERAL, INSTANCE, AppSpec, Preserve
+
+__version__ = "0.1.0"
+__all__ = ["AgeError", "AppSpec", "Bundle", "BundleError", "CACHE", "CLASSES", "CONFIG", "DATA", "EPHEMERAL",
+           "INSTANCE", "IncompatibleBundle", "Preserve", "WrongPassword", "apply_import", "blank_paths",
+           "check_compatible", "check_password", "export_instance", "open_bundle", "purge", "recover",
+           "rollback", "status"]

scalio_backup/age.py

@@ -0,0 +1,237 @@
+"""age v1 file format with a passphrase (scrypt recipient), streaming in both directions.
+
+Spec: https://age-encryption.org/v1 . Files written here open with the `age` tool (`age -d file`) and files
+from `age -p` open here, so a customer can always reach the data without Scalio. Only the scrypt
+recipient is supported, and it must be the only stanza (as the spec requires).
+"""
+import base64
+import hashlib
+import hmac
+import io
+import os
+
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.ciphers.aead import ChaCha20Poly1305
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from cryptography.hazmat.primitives.kdf.scrypt import Scrypt
+
+INTRO = b"age-encryption.org/v1\n"
+SCRYPT_LABEL = b"age-encryption.org/v1/scrypt"
+CHUNK = 64 * 1024
+TAG = 16
+DEFAULT_LOG_N = 18          # same work factor as the age tool on a current machine (256 MiB, about 1 s)
+MAX_LOG_N = 20              # refuse files that would make us spend more memory than that
+
+
+class AgeError(Exception):
+    """The file is not a valid age file, or it was changed."""
+
+
+class WrongPassword(AgeError):
+    """The passphrase does not open this file."""
+
+
+def _b64(data: bytes) -> bytes:
+    return base64.b64encode(data).rstrip(b"=")
+
+
+def _unb64(data: bytes) -> bytes:
+    if b"=" in data or b"\r" in data:
+        raise AgeError("bad base64 in header")
+    try:
+        out = base64.b64decode(data + b"=" * (-len(data) % 4), validate=True)
+    except ValueError as e:
+        raise AgeError("bad base64 in header") from e
+    if _b64(out) != data:
+        raise AgeError("non-canonical base64 in header")
+    return out
+
+
+def _hkdf(ikm: bytes, salt: bytes, info: bytes) -> bytes:
+    return HKDF(algorithm=hashes.SHA256(), length=32, salt=salt, info=info).derive(ikm)
+
+
+def _scrypt(password: bytes, salt: bytes, log_n: int) -> bytes:
+    return Scrypt(salt=SCRYPT_LABEL + salt, length=32, n=1 << log_n, r=8, p=1).derive(password)
+
+
+def _nonce(counter: int, last: bool) -> bytes:
+    if counter >= 1 << 88:
+        raise AgeError("file too large")
+    return counter.to_bytes(11, "big") + (b"\x01" if last else b"\x00")
+
+
+def _pw(password) -> bytes:
+    if isinstance(password, str):
+        password = password.encode("utf-8")
+    if not password:
+        raise ValueError("empty passphrase")
+    return password
+
+
+class EncryptWriter(io.RawIOBase):
+    """File-like object: write() plaintext, close() finishes the file. Writes the age file to `out`."""
+
+    def __init__(self, out, password, log_n: int = DEFAULT_LOG_N):
+        super().__init__()
+        self._out = out
+        file_key = os.urandom(16)
+        salt = os.urandom(16)
+        wrap = ChaCha20Poly1305(_scrypt(_pw(password), salt, log_n)).encrypt(b"\x00" * 12, file_key, None)
+        header = INTRO + b"-> scrypt " + _b64(salt) + b" " + str(log_n).encode() + b"\n" + _b64(wrap) + b"\n---"
+        mac = hmac.new(_hkdf(file_key, b"", b"header"), header, hashlib.sha256).digest()
+        out.write(header + b" " + _b64(mac) + b"\n")
+        nonce = os.urandom(16)
+        out.write(nonce)
+        self._aead = ChaCha20Poly1305(_hkdf(file_key, nonce, b"payload"))
+        self._buf = bytearray()
+        self._counter = 0
+        self._done = False
+
+    def writable(self):
+        return True
+
+    def write(self, data) -> int:
+        if self._done:
+            raise ValueError("write to closed age writer")
+        self._buf += data
+        # keep at least one byte back: the last chunk is only known on close
+        while len(self._buf) > CHUNK:
+            self._emit(bytes(self._buf[:CHUNK]), last=False)
+            del self._buf[:CHUNK]
+        return len(data)
+
+    def _emit(self, chunk: bytes, last: bool):
+        self._out.write(self._aead.encrypt(_nonce(self._counter, last), chunk, None))
+        self._counter += 1
+
+    def close(self):
+        if not self._done:
+            self._emit(bytes(self._buf), last=True)
+            self._buf.clear()
+            self._done = True
+        super().close()
+
+
+def _read_line(src, limit: int = 4096) -> bytes:
+    line = bytearray()
+    while True:
+        c = src.read(1)
+        if not c:
+            raise AgeError("unexpected end of header")
+        line += c
+        if c == b"\n":
+            return bytes(line)
+        if len(line) > limit:
+            raise AgeError("header line too long")
+
+
+class DecryptReader(io.RawIOBase):
+    """File-like object over an age file: read() returns verified plaintext. A changed chunk raises AgeError
+    at the read that reaches it; nothing unverified is ever returned."""
+
+    def __init__(self, src, password, max_log_n: int = MAX_LOG_N):
+        super().__init__()
+        self._src = src
+        if _read_line(src) != INTRO:
+            raise AgeError("not an age v1 file")
+        header = bytearray(INTRO)
+        stanzas = []
+        while True:
+            line = _read_line(src)
+            if line.startswith(b"---"):
+                break
+            if not line.startswith(b"-> "):
+                raise AgeError("bad header")
+            header += line
+            args = line[3:-1].split(b" ")
+            body = bytearray()
+            while True:
+                bl = _read_line(src)
+                header += bl
+                chunk = bl[:-1]
+                if len(chunk) > 64:
+                    raise AgeError("bad stanza body")
+                body += chunk
+                if len(chunk) < 64:
+                    break
+            stanzas.append((args, _unb64(bytes(body))))
+        if not line.startswith(b"--- ") or not line.endswith(b"\n"):
+            raise AgeError("bad header end")
+        header += b"---"
+        mac = _unb64(line[4:-1])
+        if len(stanzas) != 1 or stanzas[0][0][:1] != [b"scrypt"]:
+            raise AgeError("this file is not protected by a passphrase only")
+        args, body = stanzas[0]
+        if len(args) != 3:
+            raise AgeError("bad scrypt stanza")
+        salt = _unb64(args[1])
+        if len(salt) != 16 or not args[2].isdigit() or args[2].startswith(b"0") or len(body) != 32:
+            raise AgeError("bad scrypt stanza")
+        log_n = int(args[2])
+        if log_n > max_log_n:
+            raise AgeError(f"work factor 2^{log_n} is above the allowed 2^{max_log_n}")
+        try:
+            file_key = ChaCha20Poly1305(_scrypt(_pw(password), salt, log_n)).decrypt(b"\x00" * 12, body, None)
+        except InvalidTag as e:
+            raise WrongPassword("wrong password") from e
+        if not hmac.compare_digest(hmac.new(_hkdf(file_key, b"", b"header"), bytes(header), hashlib.sha256).digest(),
+                                   mac):
+            raise AgeError("header was changed")
+        nonce = src.read(16)
+        if len(nonce) != 16:
+            raise AgeError("file is truncated")
+        self._aead = ChaCha20Poly1305(_hkdf(file_key, nonce, b"payload"))
+        self._counter = 0
+        self._plain = b""
+        self._pos = 0
+        self._eof = False
+        self._next = src.read(CHUNK + TAG)  # one chunk read ahead to know which is the last
+
+    def readable(self):
+        return True
+
+    def _decrypt_next(self):
+        cur = self._next
+        if not cur:
+            raise AgeError("file is truncated")
+        nxt = self._src.read(CHUNK + TAG) if len(cur) == CHUNK + TAG else b""
+        last = not nxt
+        try:
+            plain = self._aead.decrypt(_nonce(self._counter, last), cur, None)
+        except InvalidTag as e:
+            raise AgeError("file was changed or is truncated") from e
+        if last and not plain and self._counter > 0:
+            raise AgeError("empty last chunk")
+        self._counter += 1
+        self._next = nxt
+        self._eof = last
+        self._plain, self._pos = plain, 0
+
+    def readinto(self, b) -> int:
+        while self._pos >= len(self._plain):
+            if self._eof:
+                return 0
+            self._decrypt_next()
+        n = min(len(b), len(self._plain) - self._pos)
+        b[:n] = self._plain[self._pos:self._pos + n]
+        self._pos += n
+        return n
+
+    def read_all_and_verify(self):
+        """Read to the end; raises AgeError if anything was changed."""
+        while self.read(CHUNK):
+            pass
+
+
+def encrypt_bytes(data: bytes, password, log_n: int = DEFAULT_LOG_N) -> bytes:
+    out = io.BytesIO()
+    w = EncryptWriter(out, password, log_n)
+    w.write(data)
+    w.close()
+    return out.getvalue()
+
+
+def decrypt_bytes(data: bytes, password, max_log_n: int = MAX_LOG_N) -> bytes:
+    return DecryptReader(io.BytesIO(data), password, max_log_n).read()

scalio_backup/engine.py

@@ -0,0 +1,495 @@
+"""Export a MongoDB database into an encrypted bundle and import one with full overwrite.
+
+Bundle = age file (passphrase) around a tar stream:
+  manifest.json                 first member; format, product, versions, source, options, parts with SHA-256
+  collections/<name>.jsonl      one document per line, MongoDB Extended JSON (canonical, types stay exact)
+  meta.jsonl                    the exported documents of the mixed meta collection
+  files/<dir>.index.jsonl       {path, size, sha256} per file of an exported directory
+  files/<dir>/<path>            the files
+
+Import = open_bundle() (decrypt, verify every hash, check product and version, nothing changed yet), then
+apply_import() (stage, swap, keep a rollback copy) inside the app's maintenance window. rollback(),
+recover() (call on startup) and purge() handle the rollback copy. Concept: https://claude.ai/artifact/Ej8DFrgi4hcFDQ69dvKWZ7
+"""
+import hashlib
+import io
+import json
+import shutil
+import tarfile
+import tempfile
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from pathlib import Path, PurePosixPath
+
+from bson import json_util
+from packaging.version import InvalidVersion, Version
+
+from .age import DEFAULT_LOG_N, MAX_LOG_N, AgeError, DecryptReader, EncryptWriter
+from .spec import JOURNAL, ROLLBACK_PREFIX, STAGING_PREFIX, AppSpec, Preserve
+
+FORMAT = "scalio-backup/1"
+MANIFEST = "manifest.json"
+BATCH = 500
+ROLLBACK_DIR = ".preimport"
+_JSON = json_util.CANONICAL_JSON_OPTIONS
+
+
+class BundleError(Exception):
+    """The bundle is damaged, changed, or not a Scalio backup."""
+
+
+class IncompatibleBundle(BundleError):
+    """The bundle is valid but cannot be imported here (other product, newer version)."""
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _iso(dt: datetime) -> str:
+    return dt.isoformat()
+
+
+# ----------------------------------------------------------------------------------------------- helpers ---
+def _blank(value):
+    if isinstance(value, dict):
+        return {}
+    if isinstance(value, list):
+        return []
+    return ""
+
+
+def blank_paths(doc: dict, paths) -> list[str]:
+    """Blank the dotted paths in doc (in place). Returns the paths that held a value."""
+    hit = []
+    for path in paths:
+        node, parts = doc, path.split(".")
+        for p in parts[:-1]:
+            node = node.get(p) if isinstance(node, dict) else None
+            if node is None:
+                break
+        if isinstance(node, dict) and node.get(parts[-1]):
+            node[parts[-1]] = _blank(node[parts[-1]])
+            hit.append(path)
+    return hit
+
+
+class _HashWriter:
+    def __init__(self, path: Path):
+        self.f = open(path, "wb")
+        self.h = hashlib.sha256()
+        self.size = 0
+
+    def write(self, data: bytes):
+        self.f.write(data)
+        self.h.update(data)
+        self.size += len(data)
+
+    def close(self):
+        self.f.close()
+
+
+def _sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for block in iter(lambda: f.read(1 << 20), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+def _safe_member(name: str) -> PurePosixPath:
+    p = PurePosixPath(name)
+    if p.is_absolute() or ".." in p.parts or not p.parts or "\\" in name or "\x00" in name:
+        raise BundleError(f"unsafe path in bundle: {name!r}")
+    return p
+
+
+async def _existing(db) -> set[str]:
+    return set(await db.list_collection_names())
+
+
+# ------------------------------------------------------------------------------------------------ export ---
+async def export_instance(db, spec: AppSpec, out, password, *, include_data: bool = True,
+                          include_secrets: bool = True, created_by: str = "", source: dict | None = None,
+                          log_n: int = DEFAULT_LOG_N, work_dir: str | None = None) -> dict:
+    """Write the encrypted bundle to the binary file object `out`. Returns the manifest."""
+    with tempfile.TemporaryDirectory(prefix="scalio-backup-", dir=work_dir) as tmp:
+        tmp = Path(tmp)
+        (tmp / "collections").mkdir()
+        (tmp / "files").mkdir()
+        parts, blanked = [], []
+
+        async def dump(name: str, cursor, rel: str, kind: str):
+            w = _HashWriter(tmp / rel)
+            count = 0
+            async for doc in cursor:
+                if spec.export_hook:
+                    doc = await spec.export_hook(name, doc)
+                if not include_secrets and name in spec.secret_fields:
+                    for path in blank_paths(doc, spec.secret_fields[name]):
+                        if f"{name}:{path}" not in blanked:
+                            blanked.append(f"{name}:{path}")
+                w.write(json_util.dumps(doc, json_options=_JSON).encode() + b"\n")
+                count += 1
+            w.close()
+            parts.append({"kind": kind, "name": name, "path": rel, "count": count, "bytes": w.size,
+                          "sha256": w.h.hexdigest()})
+
+        for coll in spec.exported_collections(include_data):
+            await dump(coll, db[coll].find({}), f"collections/{coll}.jsonl", "collection")
+        if spec.meta_collection:
+            ids = spec.exported_meta_ids(include_data)
+            await dump(spec.meta_collection, db[spec.meta_collection].find({spec.meta_key: {"$in": ids}}),
+                       "meta.jsonl", "meta")
+
+        files = []  # (archive path, source path)
+        if include_data:
+            for name, directory in sorted(spec.file_dirs.items()):
+                rows, base = [], Path(directory)
+                if base.is_dir():
+                    for src in sorted(base.rglob("*")):
+                        rel = src.relative_to(base)
+                        if rel.parts[0] == ROLLBACK_DIR or not src.is_file() or src.is_symlink():
+                            continue
+                        arc = f"files/{name}/{rel.as_posix()}"
+                        _safe_member(arc)
+                        rows.append({"path": rel.as_posix(), "size": src.stat().st_size,
+                                     "sha256": _sha256_file(src)})
+                        files.append((arc, src))
+                idx = tmp / "files" / f"{name}.index.jsonl"
+                idx.write_bytes(b"".join(json.dumps(r, sort_keys=True).encode() + b"\n" for r in rows))
+                parts.append({"kind": "files", "name": name, "path": f"files/{name}.index.jsonl",
+                              "count": len(rows), "bytes": sum(r["size"] for r in rows),
+                              "sha256": _sha256_file(idx)})
+
+        manifest = {
+            "format": FORMAT, "product": spec.product, "app_version": spec.app_version,
+            "schema_version": await spec.schema_version(db) if spec.schema_version else None,
+            "created_at": _iso(_now()), "created_by": created_by, "source": source or {},
+            "options": {"include_data": include_data, "include_secrets": include_secrets},
+            "blanked_secrets": blanked, "parts": parts,
+        }
+        writer = EncryptWriter(out, password, log_n)
+        with tarfile.open(fileobj=writer, mode="w|", format=tarfile.PAX_FORMAT) as tar:
+            def add_bytes(name: str, data: bytes):
+                info = tarfile.TarInfo(name)
+                info.size = len(data)
+                info.mtime = 0
+                tar.addfile(info, io.BytesIO(data))
+            add_bytes(MANIFEST, json.dumps(manifest, indent=1, sort_keys=True).encode())
+            for p in parts:
+                tar.add(tmp / p["path"], arcname=p["path"], recursive=False)
+            for arc, src in files:
+                tar.add(src, arcname=arc, recursive=False)
+        writer.close()
+        return manifest
+
+
+# ------------------------------------------------------------------------------------------------ import ---
+def _version(v: str):
+    try:
+        return Version(v)
+    except (InvalidVersion, TypeError):
+        return None
+
+
+@dataclass
+class Bundle:
+    """A decrypted, verified bundle in a work directory. Nothing in the database has changed yet."""
+    manifest: dict
+    path: Path
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def include_data(self) -> bool:
+        return bool(self.manifest["options"].get("include_data"))
+
+    def counts(self) -> dict[str, int]:
+        return {f'{p["kind"]}:{p["name"]}': p["count"] for p in self.manifest["parts"]}
+
+    def preview(self) -> dict:
+        m = self.manifest
+        return {"product": m["product"], "app_version": m["app_version"], "schema_version": m["schema_version"],
+                "created_at": m["created_at"], "created_by": m["created_by"], "source": m["source"],
+                "options": m["options"], "blanked_secrets": m["blanked_secrets"], "counts": self.counts(),
+                "warnings": list(self.warnings)}
+
+    def docs(self, rel: str):
+        with open(self.path / rel, "rb") as f:
+            for line in f:
+                if line.strip():
+                    yield json_util.loads(line, json_options=_JSON)
+
+    def cleanup(self):
+        shutil.rmtree(self.path, ignore_errors=True)
+
+
+def check_compatible(manifest: dict, spec: AppSpec, target_schema: int | None) -> list[str]:
+    """Raise IncompatibleBundle when the bundle cannot go into this app; return warnings otherwise."""
+    if manifest.get("format") != FORMAT:
+        raise IncompatibleBundle(f"unknown bundle format {manifest.get('format')!r}")
+    if manifest.get("product") != spec.product:
+        raise IncompatibleBundle(f"bundle is for {manifest.get('product')!r}, this is {spec.product!r}")
+    src, dst = _version(manifest.get("app_version")), _version(spec.app_version)
+    if src and dst and src > dst:
+        raise IncompatibleBundle(f"bundle comes from version {src}, this instance runs {dst}; update first")
+    sv = manifest.get("schema_version")
+    if sv is not None and target_schema is not None and sv > target_schema:
+        raise IncompatibleBundle(f"bundle has schema {sv}, this instance has {target_schema}; update first")
+    warnings = []
+    if not (src and dst):
+        warnings.append("version could not be compared")
+    if manifest.get("blanked_secrets"):
+        warnings.append("bundle was exported without secrets; enter them again after the import")
+    return warnings
+
+
+def open_bundle(src, password, spec: AppSpec, *, target_schema: int | None = None, work_dir: str | None = None,
+                max_log_n: int = MAX_LOG_N) -> Bundle:
+    """Decrypt the bundle from the binary file object `src` into a new work directory and verify it."""
+    path = Path(tempfile.mkdtemp(prefix="scalio-import-", dir=work_dir))
+    try:
+        reader = DecryptReader(src, password, max_log_n)  # WrongPassword / AgeError
+        seen, manifest = {}, None
+        try:
+            with tarfile.open(fileobj=reader, mode="r|") as tar:
+                for member in tar:
+                    name = _safe_member(member.name)
+                    if manifest is None:
+                        if member.name != MANIFEST or not member.isfile():
+                            raise BundleError("bundle does not start with a manifest")
+                        manifest = json.loads(tar.extractfile(member).read())
+                        if not isinstance(manifest, dict) or not isinstance(manifest.get("parts"), list):
+                            raise BundleError("bad manifest")
+                        continue
+                    allowed = name.parts[0] in ("collections", "files") or member.name == "meta.jsonl"
+                    if not member.isfile() or not allowed:
+                        raise BundleError(f"unexpected entry in bundle: {member.name!r}")
+                    if member.name in seen:
+                        raise BundleError(f"duplicate entry in bundle: {member.name!r}")
+                    dest = path.joinpath(*name.parts)
+                    dest.parent.mkdir(parents=True, exist_ok=True)
+                    h = hashlib.sha256()
+                    with tar.extractfile(member) as f, open(dest, "wb") as out:
+                        for block in iter(lambda: f.read(1 << 20), b""):
+                            h.update(block)
+                            out.write(block)
+                    seen[member.name] = h.hexdigest()
+            reader.read_all_and_verify()  # the tar end is not the file end; verify the last chunk too
+        except tarfile.TarError as e:
+            raise BundleError(f"bundle content is not readable: {e}") from e
+        except AgeError as e:
+            raise BundleError(str(e)) from e
+        if manifest is None:
+            raise BundleError("bundle is empty")
+        expected = {}
+        for p in manifest["parts"]:
+            expected[p["path"]] = p["sha256"]
+            if p["kind"] == "files":
+                with open(path / p["path"], "rb") if p["path"] in seen else io.BytesIO() as f:
+                    for line in f:
+                        row = json.loads(line)
+                        expected[f'files/{p["name"]}/{row["path"]}'] = row["sha256"]
+        if set(seen) != set(expected):
+            missing, extra = sorted(set(expected) - set(seen)), sorted(set(seen) - set(expected))
+            raise BundleError(f"bundle content does not match its manifest (missing {missing[:5]}, extra {extra[:5]})")
+        bad = [k for k, v in expected.items() if seen[k] != v]
+        if bad:
+            raise BundleError(f"checksum mismatch: {bad[:5]}")
+        warnings = check_compatible(manifest, spec, target_schema)
+        return Bundle(manifest=manifest, path=path, warnings=warnings)
+    except BaseException:
+        shutil.rmtree(path, ignore_errors=True)
+        raise
+
+
+async def _journal(db, **fields):
+    await db[JOURNAL].update_one({"_id": "current"}, {"$set": fields}, upsert=True)
+
+
+async def _load(db, spec: AppSpec, name: str, target: str, docs) -> int:
+    await db[target].drop()
+    batch, n = [], 0
+    for doc in docs:
+        if spec.import_hook:
+            doc = await spec.import_hook(name, doc)
+        batch.append(doc)
+        if len(batch) >= BATCH:
+            await db[target].insert_many(batch)
+            n += len(batch)
+            batch = []
+    if batch:
+        await db[target].insert_many(batch)
+        n += len(batch)
+    return n
+
+
+async def apply_import(db, spec: AppSpec, bundle: Bundle, *, preserve: tuple[Preserve, ...] = (),
+                       imported_by: str = "") -> dict:
+    """Replace every collection, meta document and file directory the bundle covers. Run it while the app
+    accepts no writes. Keeps the previous state as a rollback copy until purge()."""
+    include_data = bundle.include_data
+    colls = spec.exported_collections(include_data)
+    meta_ids = spec.exported_meta_ids(include_data) if spec.meta_collection else []
+    dirs = sorted(spec.file_dirs) if include_data else []
+    parts = {(p["kind"], p["name"]): p for p in bundle.manifest["parts"]}
+    current = await db[JOURNAL].find_one({"_id": "current"}) or {}
+    if current.get("state") in ("staging", "swapping"):
+        raise RuntimeError("an import is already running or was interrupted; call recover() first")
+    await _journal(db, state="staging", started_at=_iso(_now()), finished_at=None, imported_by=imported_by,
+                   bundle_created_at=bundle.manifest["created_at"], source=bundle.manifest["source"],
+                   collections=colls, meta_ids=meta_ids, dirs=dirs, had=[], swapped=[], meta_swapped=False,
+                   dirs_swapped=[])
+    # a new import replaces the rollback copy of the previous one
+    for name in await _existing(db):
+        if name.startswith(ROLLBACK_PREFIX) or name.startswith(STAGING_PREFIX):
+            await db[name].drop()
+
+    counts = {}
+    for coll in colls:
+        part = parts.get(("collection", coll))
+        docs = bundle.docs(part["path"]) if part else iter(())
+        counts[coll] = await _load(db, spec, coll, STAGING_PREFIX + coll, docs)
+    for p in preserve:
+        if p.collection not in colls:
+            continue
+        staged = {d.get(p.key) async for d in db[STAGING_PREFIX + p.collection].find({}, {p.key: 1})}
+        async for doc in db[p.collection].find(p.query):
+            if doc.get(p.key) not in staged:
+                doc.pop("_id", None)
+                await db[STAGING_PREFIX + p.collection].insert_one(doc)
+                counts[p.collection] += 1
+
+    await _journal(db, state="swapping")
+    existing = await _existing(db)
+    had = [c for c in colls if c in existing]
+    await _journal(db, had=had)
+    for coll in colls:
+        await db[JOURNAL].update_one({"_id": "current"}, {"$push": {"swapped": coll}})
+        if coll in existing:
+            await db[coll].rename(ROLLBACK_PREFIX + coll, dropTarget=True)
+        else:
+            await db[ROLLBACK_PREFIX + coll].drop()
+        if counts[coll]:
+            await db[STAGING_PREFIX + coll].rename(coll)
+        else:
+            await db[STAGING_PREFIX + coll].drop()
+    if meta_ids:
+        m = spec.meta_collection
+        old = [d async for d in db[m].find({spec.meta_key: {"$in": meta_ids}})]
+        if old:
+            await db[ROLLBACK_PREFIX + m].insert_many(old)
+        await _journal(db, meta_swapped=True)
+        await db[m].delete_many({spec.meta_key: {"$in": meta_ids}})
+        part = parts.get(("meta", m))
+        new = []
+        for doc in bundle.docs(part["path"]) if part else ():
+            if doc.get(spec.meta_key) in meta_ids:
+                new.append(await spec.import_hook(m, doc) if spec.import_hook else doc)
+        if new:
+            await db[m].insert_many(new)
+        counts[m] = len(new)
+    for name in dirs:
+        target = Path(spec.file_dirs[name])
+        target.mkdir(parents=True, exist_ok=True)
+        keep = target / ROLLBACK_DIR
+        shutil.rmtree(keep, ignore_errors=True)
+        await db[JOURNAL].update_one({"_id": "current"}, {"$push": {"dirs_swapped": name}})
+        keep.mkdir()
+        for entry in list(target.iterdir()):
+            if entry.name != ROLLBACK_DIR:
+                shutil.move(str(entry), str(keep / entry.name))
+        src = bundle.path / "files" / name
+        if src.is_dir():
+            for entry in list(src.iterdir()):
+                shutil.move(str(entry), str(target / entry.name))
+        counts[f"files:{name}"] = parts.get(("files", name), {}).get("count", 0)
+    await _journal(db, state="done", finished_at=_iso(_now()))
+    return {"counts": counts}
+
+
+async def rollback(db, spec: AppSpec) -> bool:
+    """Put back the state from before the last import. Returns False when there is nothing to roll back."""
+    j = await db[JOURNAL].find_one({"_id": "current"})
+    if not j or j.get("state") not in ("swapping", "done"):
+        return False
+    existing = await _existing(db)
+    had = set(j.get("had") or [])
+    for coll in j.get("swapped") or []:
+        if ROLLBACK_PREFIX + coll in existing:
+            await db[ROLLBACK_PREFIX + coll].rename(coll, dropTarget=True)
+        elif coll not in had:
+            await db[coll].drop()
+    for coll in j.get("collections") or []:
+        await db[STAGING_PREFIX + coll].drop()
+    ids = j.get("meta_ids") or []
+    if ids and spec.meta_collection and j.get("meta_swapped"):
+        m = spec.meta_collection
+        await db[m].delete_many({spec.meta_key: {"$in": ids}})
+        old = [d async for d in db[ROLLBACK_PREFIX + m].find({})]
+        if old:
+            await db[m].insert_many(old)
+        await db[ROLLBACK_PREFIX + m].drop()
+    for name in j.get("dirs_swapped") or []:
+        target = Path(spec.file_dirs[name])
+        keep = target / ROLLBACK_DIR
+        if not keep.is_dir():
+            continue
+        for entry in list(target.iterdir()):
+            if entry.name != ROLLBACK_DIR:
+                shutil.rmtree(entry) if entry.is_dir() and not entry.is_symlink() else entry.unlink()
+        for entry in list(keep.iterdir()):
+            shutil.move(str(entry), str(target / entry.name))
+        keep.rmdir()
+    await _journal(db, state="rolled_back", rolled_back_at=_iso(_now()))
+    return True
+
+
+async def recover(db, spec: AppSpec) -> str | None:
+    """Call on startup. Cleans up an import that stopped half way. Returns what it did, or None."""
+    j = await db[JOURNAL].find_one({"_id": "current"})
+    if not j:
+        return None
+    if j.get("state") == "staging":
+        for coll in j.get("collections") or []:
+            await db[STAGING_PREFIX + coll].drop()
+        await _journal(db, state="aborted")
+        return "aborted"
+    if j.get("state") == "swapping":
+        await rollback(db, spec)
+        return "rolled_back"
+    return None
+
+
+async def status(db) -> dict | None:
+    """The last import, for the UI (rollback possible while state is 'done')."""
+    j = await db[JOURNAL].find_one({"_id": "current"}, {"_id": 0})
+    return j
+
+
+async def purge(db, spec: AppSpec, older_than: timedelta = timedelta(days=7), now: datetime | None = None) -> bool:
+    """Drop the rollback copy once it is older than `older_than` (E8: 7 days)."""
+    j = await db[JOURNAL].find_one({"_id": "current"})
+    if not j or j.get("state") != "done" or not j.get("finished_at"):
+        return False
+    if datetime.fromisoformat(j["finished_at"]) > (now or _now()) - older_than:
+        return False
+    for coll in (j.get("collections") or []) + ([spec.meta_collection] if spec.meta_collection else []):
+        await db[ROLLBACK_PREFIX + coll].drop()
+    for name in j.get("dirs") or []:
+        shutil.rmtree(Path(spec.file_dirs[name]) / ROLLBACK_DIR, ignore_errors=True)
+    await _journal(db, state="purged", purged_at=_iso(now or _now()))
+    return True
+
+
+def check_password(password: str) -> str | None:
+    """Minimum rule for a bundle password. Returns an error key or None."""
+    if not password or len(password) < 12:
+        return "too_short"
+    if len(set(password)) < 5:
+        return "too_simple"
+    return None
+
+
+__all__ = ["Bundle", "BundleError", "IncompatibleBundle", "apply_import", "check_compatible", "check_password",
+           "export_instance", "open_bundle", "purge", "recover", "rollback", "status", "blank_paths"]

scalio_backup/spec.py

@@ -0,0 +1,60 @@
+"""What an app exports: one class per collection and per meta document, plus the app's hooks."""
+from dataclasses import dataclass, field
+from typing import Any, Awaitable, Callable, Optional
+
+CONFIG = "config"          # always exported
+DATA = "data"              # exported unless "configuration only"
+CACHE = "cache"            # never exported, rebuilt by the app after an import
+EPHEMERAL = "ephemeral"    # never exported (sessions, one-time links, running jobs, logs)
+INSTANCE = "instance"      # never exported, bound to this installation (license, keys, consent)
+CLASSES = (CONFIG, DATA, CACHE, EPHEMERAL, INSTANCE)
+
+# Collections the engine itself creates; apps must not use these prefixes.
+STAGING_PREFIX = "_import_"
+ROLLBACK_PREFIX = "_preimport_"
+JOURNAL = "_backup_journal"
+
+DocHook = Callable[[str, dict], Awaitable[dict]]
+
+
+@dataclass(frozen=True)
+class Preserve:
+    """Keep target documents the bundle does not contain, e.g. the account of the person importing (E7).
+    Documents in `collection` matching `query` are kept when no imported document has the same `key`."""
+    collection: str
+    query: dict
+    key: str = "id"
+
+
+@dataclass
+class AppSpec:
+    product: str                                  # product code, e.g. "scalio-nexus"
+    app_version: str                              # running version, e.g. "1.2.5"
+    collections: dict[str, str]                   # name -> class
+    meta_collection: Optional[str] = "meta"       # mixed collection classified per document
+    meta_key: str = "id"
+    meta_ids: dict[str, str] = field(default_factory=dict)
+    secret_fields: dict[str, tuple[str, ...]] = field(default_factory=dict)   # dotted paths, blanked without secrets
+    file_dirs: dict[str, str] = field(default_factory=dict)  # archive name -> absolute directory
+    schema_version: Optional[Callable[[Any], Awaitable[int]]] = None          # async (db) -> int
+    export_hook: Optional[DocHook] = None   # (collection, doc) -> doc, e.g. decrypt with the instance key
+    import_hook: Optional[DocHook] = None   # (collection, doc) -> doc, e.g. encrypt with the target's key
+
+    def __post_init__(self):
+        bad = {k: v for k, v in {**self.collections, **self.meta_ids}.items() if v not in CLASSES}
+        if bad:
+            raise ValueError(f"unknown backup classes: {bad}")
+        for name in self.collections:
+            if name.startswith((STAGING_PREFIX, ROLLBACK_PREFIX)) or name == JOURNAL:
+                raise ValueError(f"collection name {name!r} uses a reserved prefix")
+
+    def classes_for(self, include_data: bool) -> tuple[str, ...]:
+        return (CONFIG, DATA) if include_data else (CONFIG,)
+
+    def exported_collections(self, include_data: bool = True) -> list[str]:
+        wanted = self.classes_for(include_data)
+        return sorted(c for c, k in self.collections.items() if k in wanted and c != self.meta_collection)
+
+    def exported_meta_ids(self, include_data: bool = True) -> list[str]:
+        wanted = self.classes_for(include_data)
+        return sorted(i for i, k in self.meta_ids.items() if k in wanted)

scalio_backup-0.1.0.dist-info/METADATA

@@ -0,0 +1,80 @@
+Metadata-Version: 2.4
+Name: scalio-backup
+Version: 0.1.0
+Summary: Encrypted export and import (backup, restore, move) shared by the Scalio apps
+License: Proprietary
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: cryptography>=42
+Requires-Dist: pymongo>=4.6
+Requires-Dist: packaging>=23
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
+Requires-Dist: mongomock-motor>=0.0.29; extra == "dev"
+Requires-Dist: typing-extensions>=4.8; extra == "dev"
+Requires-Dist: pyrage>=1.1; extra == "dev"
+Requires-Dist: motor>=3.3; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+
+# scalio-backup
+
+Encrypted export and import of a whole Scalio app instance, shared by the Scalio apps (Nexus first, then
+Foundry, Rosetta, Citadel). Used for backup/restore and for moving an instance between self-hosted and SaaS.
+Concept and decisions E1-E12: https://claude.ai/artifact/Ej8DFrgi4hcFDQ69dvKWZ7
+
+Nothing in this package is secret. The password is the customer's; without it nobody (Scalio included) can
+open a bundle.
+
+## Bundle format (`.scaliobak`)
+
+- Outside: standard [age v1](https://age-encryption.org/v1) with a passphrase (scrypt, work factor 2^18),
+  ChaCha20-Poly1305 in 64 KiB chunks. `age -d backup.scaliobak > backup.tar` opens it without Scalio.
+- Inside: a tar stream. `manifest.json` first (format `scalio-backup/1`, product, app and schema version,
+  source, options, SHA-256 per part), then `collections/<name>.jsonl` (MongoDB Extended JSON, canonical, so
+  dates, Int64 and ObjectIds stay exact), `meta.jsonl`, and for file folders `files/<dir>.index.jsonl` plus the
+  files. The import checks every hash; a changed or incomplete bundle is refused before anything is written.
+
+## Classes
+
+Every collection (and every document of a mixed `meta` collection) has one class: `config` (always exported),
+`data` (exported unless "configuration only"), `cache`, `ephemeral`, `instance` (never exported). Each app keeps
+its classes in its own `backup_spec.py` and a test that fails on an unclassified collection.
+
+## Use in an app
+
+```python
+from scalio_backup import AppSpec, Preserve, apply_import, export_instance, open_bundle, recover, rollback, purge
+
+SPEC = AppSpec(product="scalio-nexus", app_version=VERSION, collections=..., meta_ids=...,
+               secret_fields={"settings": ("entra.clientSecret", ...)}, file_dirs={"hr": HR_UPLOAD_DIR},
+               schema_version=current_schema, export_hook=decrypt_instance_secrets, import_hook=encrypt_again)
+
+# export (background job): writes the encrypted bundle to a file object
+manifest = await export_instance(db, SPEC, out, password, include_data=True, include_secrets=True,
+                                 created_by=user_email, source={"mode": "selfhosted", "host": host, "tenant": tid})
+
+# import, step 1: decrypt and verify into a work folder, nothing changes yet (WrongPassword, BundleError,
+# IncompatibleBundle when the product differs or the bundle is newer than this instance)
+bundle = open_bundle(upload, password, SPEC, target_schema=await current_schema(db))
+show(bundle.preview())          # source, counts, warnings; the app adds its own (other tenant, license limits)
+
+# import, step 2: inside the app's maintenance window (no writes, background jobs paused)
+await apply_import(db, SPEC, bundle, preserve=(Preserve("app_users", {"email": me}),), imported_by=me)
+bundle.cleanup()
+# then: migrations, end all sessions, re-sync caches, show the checklist
+
+await rollback(db, SPEC)        # back to the state before the import (until purge)
+await recover(db, SPEC)         # on startup: cleans up an import that stopped half way
+await purge(db, SPEC)           # daily: drops the rollback copy after 7 days
+```
+
+How the swap works: documents go into `_import_<name>`, then each collection is renamed to
+`_preimport_<name>` and the staging copy into its place. A journal (`_backup_journal`) records each step, so an
+interrupted import is rolled back on the next start. File folders keep their old content in `.preimport/`.
+
+## Development
+
+`pip install -e ".[dev]"`, then `pytest -q` and `ruff check .`. Engine tests use mongomock; with `MONGO_URL` set
+they run against a real MongoDB (CI does both). `tests/vector.scaliobak` is the shared test bundle
+(`tests/make_vector.py`); the age layer is checked against rage (`pyrage`) in both directions.

scalio_backup-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+scalio_backup/__init__.py,sha256=2k8zfeAiI86k4Z6zFPv0XJJz73v_zMskS1zm3hZb-5w,849
+scalio_backup/age.py,sha256=h5NS5gz8pvLX58jjKudLHgnhH8BN3-sk9eEWnVmegWg,8577
+scalio_backup/engine.py,sha256=348YyhjPlobfBDxVhOBzG7xeDU0zGVPL5AuqWo_eBO0,22120
+scalio_backup/spec.py,sha256=skw49bZQ4JoHh4ZqnK1OWoSG4es0RAjk7nCI2Y_3_0c,3087
+scalio_backup-0.1.0.dist-info/METADATA,sha256=zhikMP46PB92gHnVnO92G24TdaRa6pNRKipP5FgesvM,4413
+scalio_backup-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+scalio_backup-0.1.0.dist-info/top_level.txt,sha256=0oiEeRyQVTNh277vtUHnFzD7LNqlhBweyzfll7kexqg,14
+scalio_backup-0.1.0.dist-info/RECORD,,

scalio_backup-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

scalio_backup-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+scalio_backup