gatewaysdk 0.3.2__py3-none-any.whl

gatewaysdk/environment/schema/snapshot.py

@@ -0,0 +1,479 @@
+"""World data snapshots: the validated SQLite, built once and opened at start.
+
+A snapshot is a ``state.sqlite`` the store wrote (contract lock and batch ledger
+inside it, as always) plus a ``state.json`` sidecar naming what it holds::
+
+    {"contractHash": ..., "batchDigest": ..., "entityCounts": {...},
+     "builtAt": ..., "runtime": "<gatewaysdk version>", "fileSha256": ...}
+
+Keyed by ``(contractHash, batchDigest)``: the same key is the same file, never
+rebuilt. ``db/state.sqlite`` is the default snapshot (``data/initial.json``
+applied, plus any ``data/batches/*.json``); ``scenarios/<name>/state.sqlite`` is
+the default with that scenario's ``seed/*.json`` overlays replacing whole
+entities. ``compile`` builds them, ``check`` verifies them, the served world
+copies one to its temp dir and opens it instead of replaying the JSON.
+
+Digests (the contract, section 0):
+
+* contract hash: sha256 of the canonical contract lock, the bytes the store's
+  ``_check_contract`` compares;
+* batch digest: ``sha256(parent + "\\n" + batchId + "\\n" + "\\n".join(sorted(chunk digests)))``,
+  the empty world's parent being sixty-four zeros;
+* the default batch's chunks are the tree's data files (``data/initial.json``
+  and ``data/batches/*.json``), each digested as its bytes; a scenario's batch
+  is ``scenario:<name>`` over the default with its seed files as chunks.
+
+A tree the platform published as a data-only version carries ``db/snapshot.json``,
+the **pin**: the hosted snapshot's ``batchDigest`` (and sidecar fields), which
+replaces the data files as the default batch. The tree's ``data/*.json`` then no
+longer describes the world's data — only that snapshot does — so a pinned tree is
+never rebuilt from its data files; the platform hands the file down
+(``load_world_to_sandbox``, the pod's ``WORLD_SNAPSHOT_*``).
+
+Everything here is standard library, so the api bundle's server can load it.
+"""
+
+from __future__ import annotations
+
+import gzip
+import hashlib
+import json
+import shutil
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterable
+
+try:
+    import tomllib
+except ModuleNotFoundError:  # pragma: no cover - python 3.10
+    import tomli as tomllib
+
+from .compiler import Contract
+from .store import SQLiteStore
+from .validation import ContractError, canonical, fail, read_json
+
+SNAPSHOT = "state.sqlite"
+SIDECAR = "state.json"
+PIN = "snapshot.json"
+DEFAULT_DIR = "db"
+SCENARIOS = "scenarios"
+EMPTY_DIGEST = "0" * 64
+SIDECAR_KEYS = ("contractHash", "batchDigest", "entityCounts", "builtAt", "runtime", "fileSha256")
+#: The hosted pod's snapshot: a presigned GET, the sha256 of the file bytes, the batch digest.
+ENV_URL = "WORLD_SNAPSHOT_URL"
+ENV_SHA256 = "WORLD_SNAPSHOT_SHA256"
+ENV_DIGEST = "WORLD_SNAPSHOT_DIGEST"
+_BLOCK = 1024 * 1024
+
+
+# ---------------------------------------------------------------- digests
+
+
+def contract_hash(contract: Contract) -> str:
+    return hashlib.sha256(canonical(contract.lock).encode("utf-8")).hexdigest()
+
+
+def batch_digest(parent: str, batch_id: str, chunk_digests: Iterable[str]) -> str:
+    text = parent + "\n" + batch_id + "\n" + "\n".join(sorted(chunk_digests))
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def file_sha256(path: str | Path) -> str:
+    digest = hashlib.sha256()
+    with open(path, "rb") as stream:
+        for block in iter(lambda: stream.read(_BLOCK), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def runtime_version() -> str:
+    try:
+        from importlib.metadata import PackageNotFoundError, version
+
+        return version("gatewaysdk")
+    except (ImportError, PackageNotFoundError):  # the api bundle carries the package without its metadata
+        return "unknown"
+
+
+def _data_files(world: Path) -> list[Path]:
+    """The default batch's chunks: the tree's data files, in the order they apply."""
+    files = [world / "data/initial.json"]
+    batches = world / "data/batches"
+    if batches.is_dir():
+        files.extend(path for path in sorted(batches.iterdir()) if path.is_file() and path.suffix == ".json")
+    return files
+
+
+def default_batch_digest(world: str | Path) -> str:
+    """The digest of the tree's data files applied in order (the module header)."""
+    world = Path(world)
+    initial = read_json(world / "data/initial.json")
+    batch_id = initial.get("id") if isinstance(initial, dict) and isinstance(initial.get("id"), str) else ""
+    return batch_digest(EMPTY_DIGEST, batch_id, [file_sha256(path) for path in _data_files(world)])
+
+
+def pin_path(world: str | Path) -> Path:
+    return Path(world) / DEFAULT_DIR / PIN
+
+
+def pinned_default(world: str | Path) -> dict | None:
+    """The platform's pin when the tree is a data-only version: ``db/snapshot.json``
+    naming the hosted snapshot (``batchDigest`` at least); None for an authored tree."""
+    path = pin_path(world)
+    if not path.is_file():
+        return None
+    pin = read_json(path)
+    if not isinstance(pin, dict) or not isinstance(pin.get("batchDigest"), str) or len(pin["batchDigest"]) != 64:
+        fail(
+            "snapshot_pin", f"{DEFAULT_DIR}/{PIN} must name the hosted snapshot's batchDigest", f"/{DEFAULT_DIR}/{PIN}"
+        )
+    return pin
+
+
+def expected_default_digest(world: str | Path) -> str:
+    """The batch digest the default snapshot must carry: the pin's for a data-only
+    version, the data files' for an authored tree."""
+    pin = pinned_default(world)
+    return pin["batchDigest"] if pin is not None else default_batch_digest(world)
+
+
+def scenario_dirs(world: str | Path) -> dict[str, Path]:
+    """A world's authored scenarios: ``scenarios/<name>/scenario.toml``, each a starting
+    state (seed overlays replacing whole entities) plus an instruction."""
+    scenarios = Path(world) / SCENARIOS
+    if not scenarios.is_dir():
+        return {}
+    out: dict[str, Path] = {}
+    for path in sorted(scenarios.iterdir()):
+        toml_path = path / "scenario.toml"
+        if not toml_path.is_file():
+            continue
+        table = tomllib.loads(toml_path.read_text(encoding="utf-8"))
+        if table.get("name", path.name) != path.name:
+            fail(
+                "scenario",
+                f"{path.name}/scenario.toml names {table.get('name')!r}; the directory is the name",
+                f"/{SCENARIOS}/{path.name}",
+            )
+        for overlay in scenario_seed_files(path):
+            if not isinstance(read_json(overlay), dict):
+                fail("scenario", f"{overlay.name} must be an object of entity -> rows", f"/{SCENARIOS}/{path.name}")
+        out[path.name] = path
+    return out
+
+
+def scenario_seed_files(scenario: Path) -> list[Path]:
+    table = tomllib.loads((scenario / "scenario.toml").read_text(encoding="utf-8"))
+    seed_dir = scenario / str(table.get("seed") or "seed")
+    return sorted(seed_dir.glob("*.json")) if seed_dir.is_dir() else []
+
+
+def scenario_overlay(scenario: Path) -> dict[str, list]:
+    """The scenario's seed files merged: later files replace an entity named twice."""
+    overlay: dict[str, list] = {}
+    for path in scenario_seed_files(scenario):
+        overlay.update(read_json(path))
+    return overlay
+
+
+def scenario_batch_digest(scenario: Path, default_digest: str) -> str:
+    return batch_digest(
+        default_digest, f"scenario:{scenario.name}", [file_sha256(p) for p in scenario_seed_files(scenario)]
+    )
+
+
+# ---------------------------------------------------------------- sidecars
+
+
+def sidecar_path(snapshot: Path) -> Path:
+    return snapshot.with_name(SIDECAR)
+
+
+def read_sidecar(snapshot: Path) -> dict | None:
+    path = sidecar_path(snapshot)
+    if not path.is_file():
+        return None
+    try:
+        sidecar = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as error:
+        raise ContractError("snapshot", f"Cannot read the snapshot sidecar {path.name}", str(path)) from error
+    if not isinstance(sidecar, dict) or any(key not in sidecar for key in SIDECAR_KEYS):
+        fail("snapshot", f"The snapshot sidecar {path.name} lacks {', '.join(SIDECAR_KEYS)}", str(path))
+    return sidecar
+
+
+def write_sidecar(snapshot: Path, contract_digest: str, digest: str, counts: dict[str, int]) -> dict:
+    sidecar = {
+        "contractHash": contract_digest,
+        "batchDigest": digest,
+        "entityCounts": counts,
+        "builtAt": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"),
+        "runtime": runtime_version(),
+        "fileSha256": file_sha256(snapshot),
+    }
+    sidecar_path(snapshot).write_text(json.dumps(sidecar, indent=1) + "\n", encoding="utf-8")
+    return sidecar
+
+
+def current(snapshot: Path, contract_digest: str, digest: str) -> dict | None:
+    """The sidecar when this snapshot already is ``(contract_digest, digest)``, else None."""
+    if not snapshot.is_file():
+        return None
+    sidecar = read_sidecar(snapshot)
+    if sidecar and sidecar["contractHash"] == contract_digest and sidecar["batchDigest"] == digest:
+        return sidecar
+    return None
+
+
+def verify(snapshot: Path, contract: Contract) -> dict:
+    """The sidecar of a snapshot whose bytes and contract it names: anything else is refused."""
+    sidecar = read_sidecar(snapshot)
+    if sidecar is None:
+        fail("snapshot", f"{snapshot} has no {SIDECAR} sidecar; recompile", str(snapshot))
+    if sidecar["contractHash"] != contract_hash(contract):
+        fail("snapshot", f"{snapshot} was built for a different contract; recompile", str(snapshot))
+    if not snapshot.is_file():
+        fail("snapshot", f"{snapshot} is missing; recompile", str(snapshot))
+    if sidecar["fileSha256"] != file_sha256(snapshot):
+        fail("snapshot", f"{snapshot} differs from its sidecar's fileSha256; recompile", str(snapshot))
+    return sidecar
+
+
+# ---------------------------------------------------------------- build
+
+
+def default_snapshot(world: str | Path) -> Path:
+    return Path(world) / DEFAULT_DIR / SNAPSHOT
+
+
+def scenario_snapshot(world: str | Path, name: str) -> Path:
+    return Path(world) / SCENARIOS / name / SNAPSHOT
+
+
+def _replace(snapshot: Path, fresh: Path) -> None:
+    snapshot.parent.mkdir(parents=True, exist_ok=True)
+    snapshot.unlink(missing_ok=True)
+    sidecar_path(snapshot).unlink(missing_ok=True)
+    shutil.move(str(fresh), str(snapshot))
+
+
+def build_default(world: str | Path, contract: Contract, *, force: bool = False) -> dict:
+    """``db/state.sqlite`` for the tree's data files; a snapshot already at this key is
+    kept. A pinned tree's default is the hosted snapshot, which the data files cannot
+    reproduce: it is kept when present and refused when absent."""
+    world = Path(world)
+    snapshot = default_snapshot(world)
+    contract_digest = contract_hash(contract)
+    digest = expected_default_digest(world)
+    kept = current(snapshot, contract_digest, digest)
+    if pin_path(world).is_file():
+        if kept is None:
+            fail(
+                "snapshot_pinned",
+                f"{DEFAULT_DIR}/{PIN} pins the hosted snapshot {digest}, which is not in the tree: "
+                f"fetch the world with its snapshot (load_world_to_sandbox, gateway worlds pull) "
+                f"or remove the pin to rebuild from the data files",
+                f"/{DEFAULT_DIR}/{SNAPSHOT}",
+            )
+        return kept
+    if not force and kept is not None:
+        return kept
+    fresh = snapshot.with_name(f".{SNAPSHOT}.building")
+    fresh.parent.mkdir(parents=True, exist_ok=True)
+    fresh.unlink(missing_ok=True)
+    try:
+        with SQLiteStore.create(fresh, contract) as store:
+            for path in _data_files(world):
+                store.apply_batch(read_json(path))
+            counts = store.counts()
+    except BaseException:
+        fresh.unlink(missing_ok=True)
+        raise
+    _replace(snapshot, fresh)
+    return write_sidecar(snapshot, contract_digest, digest, counts)
+
+
+def build_scenario(world: str | Path, name: str, contract: Contract, default: dict, *, force: bool = False) -> dict:
+    """``scenarios/<name>/state.sqlite``: the default snapshot with the overlays applied."""
+    world = Path(world)
+    scenario = scenario_dirs(world).get(name)
+    if scenario is None:
+        fail("scenario", f"No scenario {name}", f"/{SCENARIOS}/{name}")
+    snapshot = scenario_snapshot(world, name)
+    contract_digest = contract_hash(contract)
+    digest = scenario_batch_digest(scenario, default["batchDigest"])
+    if not force:
+        kept = current(snapshot, contract_digest, digest)
+        if kept is not None:
+            return kept
+    from .data import seed_batch
+
+    fresh = snapshot.with_name(f".{SNAPSHOT}.building")
+    fresh.unlink(missing_ok=True)
+    shutil.copyfile(default_snapshot(world), fresh)
+    try:
+        with SQLiteStore.open(fresh, contract, audit=False) as store:
+            overlay = scenario_overlay(scenario)
+            if overlay:
+                store.apply_batch(seed_batch(store, contract, overlay, "replace", batch_id=f"scenario:{name}"))
+            counts = store.counts()
+    except BaseException:
+        fresh.unlink(missing_ok=True)
+        raise
+    _replace(snapshot, fresh)
+    return write_sidecar(snapshot, contract_digest, digest, counts)
+
+
+def build_all(world: str | Path, contract: Contract, *, force: bool = False) -> dict:
+    """The default snapshot and one per scenario; ``{"default": sidecar, "scenarios": {name: sidecar}}``."""
+    world = Path(world)
+    default = build_default(world, contract, force=force)
+    scenarios = {name: build_scenario(world, name, contract, default, force=force) for name in scenario_dirs(world)}
+    return {"default": default, "scenarios": scenarios}
+
+
+def discard_all(world: str | Path) -> None:
+    """Remove every snapshot in the tree: nothing may claim data that no longer applies."""
+    world = Path(world)
+    targets = [default_snapshot(world)]
+    scenarios = world / SCENARIOS
+    if scenarios.is_dir():
+        targets.extend(path / SNAPSHOT for path in scenarios.iterdir() if path.is_dir())
+    for snapshot in targets:
+        snapshot.unlink(missing_ok=True)
+        sidecar_path(snapshot).unlink(missing_ok=True)
+
+
+def check_all(world: str | Path, contract: Contract) -> dict:
+    """Every snapshot present in the tree, verified against its sidecar; absent ones are not required."""
+    world = Path(world)
+    found: dict[str, Any] = {"default": None, "scenarios": {}}
+    snapshot = default_snapshot(world)
+    if snapshot.is_file() or sidecar_path(snapshot).is_file():
+        found["default"] = verify(snapshot, contract)
+        # A snapshot built from earlier data than the tree holds is reported, not
+        # refused: authoring edits journals and opens the world, and a served world
+        # replays the journals past a stale snapshot (WorldState opens only a
+        # current one). `stale` is in every check report so it never passes unseen.
+        found["default"]["stale"] = (
+            pinned_default(world) is None and found["default"]["batchDigest"] != default_batch_digest(world)
+        )
+    for name in scenario_dirs(world):
+        snapshot = scenario_snapshot(world, name)
+        if snapshot.is_file() or sidecar_path(snapshot).is_file():
+            found["scenarios"][name] = verify(snapshot, contract)
+    return found
+
+
+# ---------------------------------------------------------------- open
+
+
+def usable(snapshot: Path, contract: Contract) -> dict | None:
+    """The sidecar when the snapshot exists and was built for this contract: what a
+    served world opens. A snapshot for another contract is ignored, not an error;
+    the legacy path (replay the JSON) still serves the world."""
+    if not snapshot.is_file():
+        return None
+    sidecar = read_sidecar(snapshot)
+    if sidecar is None or sidecar["contractHash"] != contract_hash(contract):
+        return None
+    return sidecar
+
+
+def current_default(world: str | Path, contract: Contract) -> dict | None:
+    """The default snapshot's sidecar when it is exactly the tree's data through this
+    contract (what ``compile`` would skip); None when absent or stale."""
+    world = Path(world)
+    snapshot = default_snapshot(world)
+    if not snapshot.is_file() or not (world / "data/initial.json").is_file():
+        return None
+    return current(snapshot, contract_hash(contract), expected_default_digest(world))
+
+
+def open_copy(snapshot: Path, contract: Contract, destination: Path) -> SQLiteStore:
+    """A session's own copy of the snapshot, opened without the row audit: the file
+    was audited when it was written and verified by digest since."""
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copyfile(snapshot, destination)
+    return SQLiteStore.open(destination, contract, audit=False)
+
+
+# ---------------------------------------------------------------- hosted pod
+
+
+def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest: str) -> dict:
+    """Download the pod's snapshot to ``db/state.sqlite``: gunzip when the URL ends in
+    ``.gz`` or the response says so, verify the sha256 of the file bytes, write the
+    sidecar. A mismatch is a refusal naming the digest; nothing usable is left behind."""
+    world = Path(world)
+    snapshot = default_snapshot(world)
+    snapshot.parent.mkdir(parents=True, exist_ok=True)
+    partial = snapshot.with_name(f".{SNAPSHOT}.download")
+    hasher = hashlib.sha256()
+    try:
+        with urllib.request.urlopen(urllib.request.Request(url), timeout=300) as response:
+            compressed = url.split("?", 1)[0].endswith(".gz") or "gzip" in (
+                response.headers.get("Content-Encoding") or ""
+            )
+            source = gzip.GzipFile(fileobj=response) if compressed else response
+            with open(partial, "wb") as out:
+                for block in iter(lambda: source.read(_BLOCK), b""):
+                    hasher.update(block)
+                    out.write(block)
+    except (OSError, EOFError, ValueError) as error:
+        partial.unlink(missing_ok=True)
+        raise ContractError("snapshot_download", f"Snapshot {digest} could not be downloaded: {error}") from None
+    if hasher.hexdigest() != sha256.lower():
+        partial.unlink(missing_ok=True)
+        fail(
+            "snapshot_mismatch",
+            f"Snapshot {digest} downloaded with sha256 {hasher.hexdigest()}, expected {sha256}",
+        )
+    try:
+        with SQLiteStore.open(partial, contract, audit=False) as store:
+            counts = store.counts()
+    except ContractError:
+        partial.unlink(missing_ok=True)
+        raise
+    _replace(snapshot, partial)
+    sidecar = write_sidecar(snapshot, contract_hash(contract), digest, counts)
+    # The hosted snapshot IS this tree's data now: pin it, so the served world opens
+    # it as current instead of replaying journals the snapshot supersedes.
+    pin_path(world).write_text(
+        json.dumps({"batchDigest": digest, "contractHash": contract_hash(contract), "entityCounts": counts}, indent=1)
+        + "\n"
+    )
+    return sidecar
+
+
+__all__ = [
+    "EMPTY_DIGEST",
+    "ENV_DIGEST",
+    "ENV_SHA256",
+    "ENV_URL",
+    "SIDECAR",
+    "SNAPSHOT",
+    "batch_digest",
+    "build_all",
+    "build_default",
+    "build_scenario",
+    "check_all",
+    "contract_hash",
+    "current_default",
+    "default_batch_digest",
+    "default_snapshot",
+    "discard_all",
+    "expected_default_digest",
+    "fetch",
+    "file_sha256",
+    "open_copy",
+    "pinned_default",
+    "read_sidecar",
+    "scenario_dirs",
+    "scenario_overlay",
+    "scenario_snapshot",
+    "usable",
+    "verify",
+    "write_sidecar",
+]