gatewaysdk 0.3.2__py3-none-any.whl

gatewaysdk/environment/schema/store.py

@@ -0,0 +1,904 @@
+"""Schema-aware SQLite state and atomic, provider-neutral mutation batches."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+import uuid
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Iterator
+
+from .compiler import Contract, payload_value, quote
+from .validation import (
+    ContractError,
+    canonical,
+    digest,
+    encoded,
+    fail,
+    members,
+    nonempty,
+    parse_json,
+    validate,
+)
+
+PRIVATE_DDL = """
+CREATE TABLE _gateway_contract (contract TEXT NOT NULL);
+CREATE TABLE _gateway_batches (id TEXT PRIMARY KEY, digest TEXT NOT NULL, metadata TEXT NOT NULL);
+CREATE TABLE _gateway_lineage (batch TEXT NOT NULL REFERENCES _gateway_batches(id),
+    ordinal INTEGER NOT NULL, entity TEXT NOT NULL, key TEXT NOT NULL, source TEXT NOT NULL,
+    PRIMARY KEY(batch, ordinal));
+CREATE TABLE benchmark_runtime_context (context_key TEXT PRIMARY KEY, scenario_id TEXT NOT NULL);
+CREATE TABLE benchmark_runtime_parameters (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+CREATE TABLE benchmark_conversation_events (scenario_id TEXT NOT NULL, user_turn_index INTEGER NOT NULL,
+    event_type TEXT NOT NULL, actor TEXT NOT NULL, name TEXT, payload_json TEXT NOT NULL);
+
+"""
+# The host owns these tables. Schema authors cannot declare them. Tools may
+# read parameters; context/events are available only to trusted verifier reads.
+RUNTIME_TABLES = {
+    "benchmark_runtime_context",
+    "benchmark_runtime_parameters",
+    "benchmark_conversation_events",
+}
+READ_ACTIONS = {
+    sqlite3.SQLITE_SELECT,
+    sqlite3.SQLITE_READ,
+    sqlite3.SQLITE_FUNCTION,
+    sqlite3.SQLITE_RECURSIVE,
+}
+WRITE_ACTIONS = {sqlite3.SQLITE_INSERT, sqlite3.SQLITE_UPDATE, sqlite3.SQLITE_DELETE}
+# Objects an ingest session may create for its own staging; they never persist.
+TEMP_ACTIONS = {
+    sqlite3.SQLITE_CREATE_TEMP_TABLE,
+    sqlite3.SQLITE_CREATE_TEMP_INDEX,
+    sqlite3.SQLITE_CREATE_TEMP_VIEW,
+    sqlite3.SQLITE_DROP_TEMP_TABLE,
+    sqlite3.SQLITE_DROP_TEMP_INDEX,
+    sqlite3.SQLITE_DROP_TEMP_VIEW,
+}
+# Bookkeeping and host tables no SQL session may read; parameters stay readable.
+PRIVATE_TABLES = {
+    "_gateway_contract",
+    "_gateway_batches",
+    "_gateway_lineage",
+    "benchmark_runtime_context",
+    "benchmark_conversation_events",
+}
+UNSAFE_FUNCTIONS = {"load_extension", "writefile", "readfile"}
+
+
+def _source(source: Any, location: str) -> None:
+    members(source, {"record", "evidence"}, {"record"}, location)
+    nonempty(source["record"], f"{location}/record")
+    evidence = source.get("evidence", [])
+    if not isinstance(evidence, list):
+        fail(
+            "provenance",
+            "evidence must be an array of stable reference/digest pairs",
+            location,
+        )
+    for index, entry in enumerate(evidence):
+        members(
+            entry,
+            {"reference", "digest"},
+            {"reference", "digest"},
+            f"{location}/evidence/{index}",
+        )
+        nonempty(entry["reference"], location)
+        # Digest syntax is explicit; providers can translate a verified base64
+        # SHA-256 to hex without retaining staging credentials or locations.
+        value = entry["digest"]
+        if not isinstance(value, str) or len(value) != 64 or any(c not in "0123456789abcdef" for c in value):
+            fail("provenance", "Evidence digest must be lowercase SHA-256 hex", location)
+
+
+class ReadConnection:
+    """Compatibility conn= surface: execute SELECT only; no raw writes/commit."""
+
+    def __init__(self, store: "SQLiteStore"):
+        self._store = store
+
+    def execute(self, sql: str, params: Any = ()) -> "ReadCursor":
+        return ReadCursor(self._store.query(sql, tuple(params)))
+
+    def cursor(self):
+        fail(
+            "raw_write",
+            "Use conn.execute for reads and gateway_env.store for mutations",
+        )
+
+    def executescript(self, *args):
+        fail("raw_write", "Raw SQL scripts are unavailable in schema-backed tools")
+
+    def commit(self):
+        fail("raw_write", "The world owns the tool transaction")
+
+    rollback = commit
+    close = commit
+
+
+class IngestConnection:
+    """Raw SQL inside one schema-gated ingest session.
+
+    Domain tables and TEMP objects are writable; the store owns the transaction
+    and commits only when every persisted row satisfies the contract.
+    """
+
+    def __init__(self, store: "SQLiteStore"):
+        self._store = store
+        self._open = True
+        self.result: dict | None = None
+
+    def execute(self, sql: str, params: Any = ()) -> sqlite3.Cursor:
+        return self._run(lambda conn: conn.execute(sql, params))
+
+    def executemany(self, sql: str, rows: Any) -> sqlite3.Cursor:
+        return self._run(lambda conn: conn.executemany(sql, rows))
+
+    def script(self, sql: str) -> None:
+        """Run a multi-statement script one statement at a time.
+
+        ``sqlite3.executescript`` would commit the session's transaction first,
+        so it is deliberately not exposed.
+        """
+        from .._world import split_sql_statements
+
+        for statement in split_sql_statements(sql):
+            self.execute(statement)
+
+    def _run(self, operation):
+        if not self._open:
+            fail("transaction", "The ingest session has ended")
+        store = self._store
+        store._key_error = None
+        try:
+            return operation(store._conn)
+        except sqlite3.Error as error:
+            context = {"sqlite": str(error)}
+            if store._key_error is not None:
+                context["key"] = store._key_error
+            raise ContractError(
+                "ingest_sql",
+                "Statement rejected; only domain tables and TEMP objects are writable",
+                **context,
+            ) from error
+
+    def commit(self):
+        fail("raw_write", "The store commits the ingest session when it ends")
+
+    rollback = commit
+    close = commit
+
+
+class ReadCursor:
+    def __init__(self, rows: list[sqlite3.Row]):
+        self._rows = iter(rows)
+        self.rowcount = -1
+
+    def fetchone(self):
+        return next(self._rows, None)
+
+    def fetchall(self):
+        return list(self._rows)
+
+    def __iter__(self):
+        return self._rows
+
+
+class SQLiteStore:
+    """Use create/open as context managers; every public mutation is atomic.
+
+    Payloads are JSON values, keys are dicts of exactly the primary-key fields.
+    Updates shallow-merge provided properties and validate the complete result.
+    """
+
+    def __init__(self, path: Path, contract: Contract, *, create: bool = False, audit: bool = True):
+        self.path = Path(path)
+        self.contract = contract
+        self._lock = threading.RLock()
+        self._depth = 0
+        self._internal = 0
+        self._touched: set[tuple[str, str]] = set()
+        self._read_only = False
+        self._private_reads = False
+        self._ingesting = False
+        self._key_error: dict | None = None
+        self._closed = False
+        if create and self.path.exists():
+            fail(
+                "state_exists",
+                "Refusing to replace existing state; reset/rebuild explicitly",
+            )
+        if not create and not self.path.is_file():
+            fail("state_missing", "Cannot open missing state")
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(
+            self.path,
+            isolation_level=None,
+            check_same_thread=False,
+            cached_statements=0,
+        )
+        self._conn.row_factory = sqlite3.Row
+        try:
+            self._conn.execute("PRAGMA foreign_keys = ON")
+            if self._conn.execute("PRAGMA foreign_keys").fetchone()[0] != 1:
+                fail("sqlite", "Foreign-key enforcement is required")
+            if create:
+                self._conn.executescript(contract.artifacts["db/schema.sql"].decode() + PRIVATE_DDL)
+                self._conn.execute(
+                    "INSERT INTO _gateway_contract VALUES (?)",
+                    (canonical(contract.lock),),
+                )
+            self._check_contract()
+            self._conn.create_function("gateway_key", 2, self._sql_key, deterministic=True)
+            self._conn.set_authorizer(self._authorize)
+            if audit:
+                self.audit()
+        except BaseException:
+            self._conn.close()
+            if create:
+                self.path.unlink(missing_ok=True)
+            raise
+
+    @classmethod
+    def create(cls, path: str | Path, contract: Contract) -> "SQLiteStore":
+        return cls(Path(path), contract, create=True)
+
+    @classmethod
+    def open(cls, path: str | Path, contract: Contract, *, audit: bool = True) -> "SQLiteStore":
+        """Open existing state. ``audit=False`` skips the full row audit: for a snapshot
+        the store itself wrote and whose bytes a sidecar digest has just verified,
+        every row was audited when it was written, so opening is then a matter of
+        milliseconds however many rows the file holds. The contract check always runs."""
+        return cls(Path(path), contract, audit=audit)
+
+    def _check_contract(self) -> None:
+        try:
+            rows = self._conn.execute("SELECT contract FROM _gateway_contract").fetchall()
+            if len(rows) != 1 or rows[0][0] != canonical(self.contract.lock):
+                fail(
+                    "state_contract",
+                    "State belongs to a different schema/package lock; no automatic migration",
+                )
+            # Compare actual DDL with fresh generated DDL, not merely a stored hash.
+            expected = sqlite3.connect(":memory:")
+            try:
+                expected.executescript(self.contract.artifacts["db/schema.sql"].decode() + PRIVATE_DDL)
+                expected_sql = dict(expected.execute("SELECT name, sql FROM sqlite_master WHERE sql IS NOT NULL"))
+            finally:
+                expected.close()
+            actual_sql = dict(self._conn.execute("SELECT name, sql FROM sqlite_master WHERE sql IS NOT NULL"))
+            if (
+                any(actual_sql.get(k) != v for k, v in expected_sql.items())
+                or set(actual_sql) - set(expected_sql) - RUNTIME_TABLES
+            ):
+                fail(
+                    "state_contract",
+                    "State DDL differs from the compiled storage contract",
+                )
+        except sqlite3.Error as error:
+            raise ContractError("state_contract", "State has no compatible schema contract") from error
+
+    def _authorize(self, action, first, second, database, trigger):
+        if self._read_only:
+            if action not in READ_ACTIONS or (
+                action == sqlite3.SQLITE_READ
+                and first not in self.contract.entities
+                and first != "benchmark_runtime_parameters"
+                and not (self._private_reads and first in RUNTIME_TABLES)
+            ):
+                return sqlite3.SQLITE_DENY
+            if action == sqlite3.SQLITE_FUNCTION and (second or "").lower() in UNSAFE_FUNCTIONS:
+                return sqlite3.SQLITE_DENY
+            return sqlite3.SQLITE_OK
+        if self._internal:
+            return sqlite3.SQLITE_OK
+        if self._ingesting:
+            return self._authorize_ingest(action, first, second, database)
+        if action in READ_ACTIONS:
+            return sqlite3.SQLITE_OK
+        if action in WRITE_ACTIONS and first in RUNTIME_TABLES:
+            return sqlite3.SQLITE_OK
+        return sqlite3.SQLITE_DENY
+
+    def _authorize_ingest(self, action, first, second, database):
+        """Domain tables and TEMP staging are writable; everything else is not.
+
+        Transaction control, PRAGMA, ATTACH, persistent DDL and the private
+        bookkeeping tables stay with the store, so a session cannot commit
+        early, disable enforcement, or forge provenance.
+        """
+        if action in READ_ACTIONS:
+            if action == sqlite3.SQLITE_READ and first in PRIVATE_TABLES:
+                return sqlite3.SQLITE_DENY
+            if action == sqlite3.SQLITE_FUNCTION and (second or "").lower() in UNSAFE_FUNCTIONS:
+                return sqlite3.SQLITE_DENY
+            return sqlite3.SQLITE_OK
+        if action in WRITE_ACTIONS:
+            if first in self.contract.entities or database == "temp":
+                return sqlite3.SQLITE_OK
+            return sqlite3.SQLITE_DENY
+        if action in TEMP_ACTIONS or action == sqlite3.SQLITE_SAVEPOINT:
+            return sqlite3.SQLITE_OK
+        return sqlite3.SQLITE_DENY
+
+    def _sql_key(self, entity: Any, payload: Any) -> str | None:
+        """``gateway_key(entity, payload)``: the encoded ``_key`` for a payload.
+
+        Returns NULL on an invalid entity or payload so the NOT NULL constraint
+        rejects the row; the reason is attached to the resulting error.
+        """
+        try:
+            value = payload
+            if isinstance(payload, (str, bytes)):
+                value = parse_json(payload)
+            if not isinstance(value, dict):
+                fail("key", "gateway_key needs a JSON object payload", entity=entity)
+            return self._key(entity, value)
+        except (ContractError, ValueError, TypeError) as error:
+            detail = getattr(error, "detail", None)
+            self._key_error = detail or {"code": "key", "message": str(error)}
+            return None
+
+    @contextmanager
+    def _trusted(self):
+        self._internal += 1
+        try:
+            yield
+        finally:
+            self._internal -= 1
+
+    @contextmanager
+    def transaction(self) -> Iterator["SQLiteStore"]:
+        with self._lock:
+            if self._ingesting:
+                fail(
+                    "transaction",
+                    "Use SQL inside an ingest session; store mutations belong outside",
+                )
+            outer = self._depth == 0
+            savepoint = f"gateway_batch_{self._depth}"
+            with self._trusted():
+                self._conn.execute("BEGIN IMMEDIATE" if outer else f"SAVEPOINT {savepoint}")
+            if outer:
+                self._touched = set()
+            self._depth += 1
+            try:
+                yield self
+                if outer:
+                    self._audit_touched()
+                with self._trusted():
+                    self._conn.execute("COMMIT" if outer else f"RELEASE {savepoint}")
+            except BaseException:
+                with self._trusted():
+                    self._conn.execute("ROLLBACK" if outer else f"ROLLBACK TO {savepoint}")
+                    if not outer:
+                        self._conn.execute(f"RELEASE {savepoint}")
+                raise
+            finally:
+                self._depth -= 1
+
+    def _entity(self, name: str) -> dict:
+        if not isinstance(name, str) or name not in self.contract.entities:
+            fail("entity", "Unknown entity type", entity=name)
+        return self.contract.entities[name]
+
+    def _key(self, name: str, value: dict, *, exact: bool = False) -> str:
+        entity = self._entity(name)
+        fields = entity["primary_key"]
+        if not isinstance(value, dict) or not set(fields) <= set(value) or (exact and set(value) != set(fields)):
+            fail("key", "Supply exactly the declared primary-key fields", entity=name)
+        for field in fields:
+            validate(
+                entity["schema"]["properties"][field],
+                value[field],
+                location=f"{entity['location']}/schema/properties/{field}",
+            )
+        # JSON Schema integers include 1.0; identity normalizes mathematically
+        # integral keys only, without changing the actual payload value.
+        parts = [int(value[f]) if type(value[f]) is float else value[f] for f in fields]
+        return canonical(parts)
+
+    def get(self, entity: str, key: dict) -> dict | None:
+        with self._lock:
+            encoded_key = self._key(entity, key, exact=True)
+            row = self._conn.execute(f"SELECT payload FROM {quote(entity)} WHERE _key = ?", (encoded_key,)).fetchone()
+            return json.loads(row[0]) if row else None
+
+    def list(self, entity: str) -> list[dict]:
+        with self._lock:
+            self._entity(entity)
+            return [
+                json.loads(row[0]) for row in self._conn.execute(f"SELECT payload FROM {quote(entity)} ORDER BY _key")
+            ]
+
+    def query(self, sql: str, params: tuple = (), *, private: bool = False) -> list[sqlite3.Row]:
+        """Read-only SQL over domain tables, enforced by SQLite's authorizer."""
+        with self._lock:
+            self._read_only = True
+            self._private_reads = private
+            try:
+                return self._conn.execute(sql, params).fetchall()
+            except sqlite3.Error as error:
+                raise ContractError("read_only", "Query failed or attempted a forbidden operation") from error
+            finally:
+                self._read_only = False
+                self._private_reads = False
+
+    def _mutate(self, mutation: dict) -> str:
+        name = mutation["entity"]
+        entity = self._entity(name)
+        operation = mutation["op"]
+        if operation == "insert":
+            value = mutation["value"]
+            validate(entity["schema"], value, location=f"{entity['location']}/schema")
+            key = self._key(name, value)
+        else:
+            key = self._key(name, mutation["key"], exact=True)
+            current = self.get(name, mutation["key"])
+            if current is None:
+                fail(
+                    "not_found",
+                    "No entity exists at this key",
+                    entity=name,
+                    key=mutation["key"],
+                )
+            if operation == "update":
+                if not isinstance(mutation["value"], dict):
+                    fail("type", "An update must be an object")
+                value = {**current, **mutation["value"]}
+                validate(entity["schema"], value, location=f"{entity['location']}/schema")
+                if self._key(name, value) != key:
+                    fail(
+                        "key_immutable",
+                        "Updates cannot change primary keys",
+                        entity=name,
+                    )
+        self._touched.add((name, key))
+        try:
+            with self._trusted():
+                if operation == "insert":
+                    self._conn.execute(
+                        f"INSERT INTO {quote(name)} (_key, payload) VALUES (?, ?)",
+                        (key, canonical(value)),
+                    )
+                elif operation == "update":
+                    self._conn.execute(
+                        f"UPDATE {quote(name)} SET payload = ? WHERE _key = ?",
+                        (canonical(value), key),
+                    )
+                else:
+                    self._conn.execute(f"DELETE FROM {quote(name)} WHERE _key = ?", (key,))
+        except sqlite3.IntegrityError as error:
+            raise ContractError(
+                "unique",
+                "Primary or unique key conflicts with existing state",
+                entity=name,
+                key=json.loads(key),
+                schema_path=f"{entity['location']}/unique",
+            ) from error
+        return key
+
+    def apply_batch(self, batch: dict) -> dict:
+        """Atomic rejection; exact retries no-op, conflicting ID reuse rejects."""
+        members(
+            batch,
+            {"id", "target", "mapping", "coverage", "mutations"},
+            {"id", "target", "mapping", "coverage", "mutations"},
+            "/batch",
+        )
+        nonempty(batch["id"], "/batch/id")
+        if batch["target"] != self.contract.target:
+            fail(
+                "batch_target",
+                "Batch targets a different domain identity/version/hash",
+                "/batch/target",
+            )
+        members(batch["mapping"], {"id", "version"}, {"id", "version"}, "/batch/mapping")
+        for key, value in batch["mapping"].items():
+            nonempty(value, f"/batch/mapping/{key}")
+        if batch["coverage"] not in ("partial", "complete"):
+            fail(
+                "coverage",
+                "coverage must be partial or complete (neither implies deletion)",
+                "/batch/coverage",
+            )
+        if not isinstance(batch["mutations"], list):
+            fail("batch", "mutations must be an array", "/batch/mutations")
+        batch_hash = digest(encoded(batch))
+        with self.transaction():
+            prior = self._conn.execute("SELECT digest FROM _gateway_batches WHERE id = ?", (batch["id"],)).fetchone()
+            if prior:
+                if prior[0] != batch_hash:
+                    fail(
+                        "idempotency_conflict",
+                        "Batch ID was already applied with different content",
+                        "/batch/id",
+                        batch=batch["id"],
+                    )
+                return {"status": "already_applied", "batch": batch["id"], "applied": 0}
+            with self._trusted():
+                self._conn.execute(
+                    "INSERT INTO _gateway_batches VALUES (?, ?, ?)",
+                    (
+                        batch["id"],
+                        batch_hash,
+                        canonical({k: v for k, v in batch.items() if k != "mutations"}),
+                    ),
+                )
+            for index, mutation in enumerate(batch["mutations"]):
+                location = f"/batch/mutations/{index}"
+                try:
+                    members(
+                        mutation,
+                        {"op", "entity", "key", "value", "source"},
+                        {"op", "entity", "source"},
+                        location,
+                    )
+                    op = mutation["op"]
+                    if not isinstance(op, str):
+                        fail(
+                            "operation",
+                            "Operation must be insert, update or delete",
+                            location,
+                        )
+                    required = {
+                        "insert": {"value"},
+                        "update": {"key", "value"},
+                        "delete": {"key"},
+                    }.get(op)
+                    if required is None:
+                        fail(
+                            "operation",
+                            "Only insert, update and delete are supported",
+                            location,
+                        )
+                    members(
+                        mutation,
+                        {"op", "entity", "source"} | required,
+                        {"op", "entity", "source"} | required,
+                        location,
+                    )
+                    _source(mutation["source"], f"{location}/source")
+                    key = self._mutate(mutation)
+                    with self._trusted():
+                        self._conn.execute(
+                            "INSERT INTO _gateway_lineage VALUES (?, ?, ?, ?, ?)",
+                            (
+                                batch["id"],
+                                index,
+                                mutation["entity"],
+                                key,
+                                canonical(mutation["source"]),
+                            ),
+                        )
+                except ContractError as error:
+                    # Only attach source after it has passed metadata validation.
+                    context = {"batch": batch["id"], "index": index}
+                    if isinstance(mutation, dict):
+                        context["entity"] = mutation.get("entity")
+                        if "key" in mutation:
+                            context["key"] = mutation["key"]
+                        try:
+                            _source(mutation.get("source"), location)
+                            context["source"] = mutation["source"]
+                        except ContractError:
+                            pass
+                    raise error.contextualize(**context)
+            # Nested batches inside a tool defer references to the tool boundary.
+        return {
+            "status": "applied",
+            "batch": batch["id"],
+            "applied": len(batch["mutations"]),
+        }
+
+    def _single(self, op: str, entity: str, *, source: dict | None = None, **values):
+        self.apply_batch(
+            {
+                "id": uuid.uuid4().hex,
+                "target": self.contract.target,
+                "mapping": {"id": "gateway-store", "version": "1"},
+                "coverage": "partial",
+                "mutations": [
+                    {
+                        "op": op,
+                        "entity": entity,
+                        **values,
+                        "source": (source if source is not None else {"record": f"store.{op}"}),
+                    }
+                ],
+            }
+        )
+
+    def insert(self, entity: str, value: dict, *, source: dict | None = None) -> dict:
+        self._single("insert", entity, value=value, source=source)
+        return json.loads(canonical(value))
+
+    def update(self, entity: str, key: dict, patch: dict, *, source: dict | None = None) -> dict:
+        with self._lock:
+            self._single("update", entity, key=key, value=patch, source=source)
+            return self.get(entity, key)
+
+    def delete(self, entity: str, key: dict, *, source: dict | None = None) -> None:
+        self._single("delete", entity, key=key, source=source)
+
+    @contextmanager
+    def ingest(
+        self,
+        *,
+        source: dict,
+        mapping: dict,
+        batch_id: str | None = None,
+        limit: int = 50,
+    ) -> Iterator[IngestConnection]:
+        """Schema-gated raw SQL: write freely, commit only if every row conforms.
+
+        The session yields a connection that may INSERT/UPDATE/DELETE domain
+        tables and create TEMP staging objects. ``gateway_key(entity, payload)``
+        computes ``_key`` in SQL. On exit the store validates all persisted
+        state; any violation (up to ``limit`` are reported together) rolls the
+        whole session back. Committed changes are recorded as one batch with a
+        lineage row per touched key, so raw ingestion keeps provenance.
+        """
+        _source(source, "/ingest/source")
+        members(mapping, {"id", "version"}, {"id", "version"}, "/ingest/mapping")
+        for key, value in mapping.items():
+            nonempty(value, f"/ingest/mapping/{key}")
+        with self._lock:
+            if self._depth or self._ingesting:
+                fail(
+                    "transaction",
+                    "Ingest needs no active tool, batch or ingest transaction",
+                )
+            batch = batch_id if batch_id is not None else uuid.uuid4().hex
+            nonempty(batch, "/ingest/id")
+            if self._conn.execute("SELECT 1 FROM _gateway_batches WHERE id = ?", (batch,)).fetchone():
+                fail(
+                    "idempotency_conflict",
+                    "Ingest ID was already applied; raw sessions are not replayable",
+                    "/ingest/id",
+                    batch=batch,
+                )
+            before = self._state()
+            connection = IngestConnection(self)
+            with self._trusted():
+                self._conn.execute("BEGIN IMMEDIATE")
+            self._depth = 1
+            self._ingesting = True
+            try:
+                yield connection
+                violations = self._violations(limit)
+                if violations:
+                    fail(
+                        "ingest_rejected",
+                        "Persisted rows violate the contract; nothing was committed",
+                        batch=batch,
+                        violations=[error.detail for error in violations],
+                        truncated=len(violations) >= limit,
+                    )
+                changes = self._changes(before, self._state())
+                metadata = {
+                    "id": batch,
+                    "kind": "ingest",
+                    "target": self.contract.target,
+                    "mapping": mapping,
+                    "coverage": "partial",
+                }
+                with self._trusted():
+                    if changes:
+                        self._conn.execute(
+                            "INSERT INTO _gateway_batches VALUES (?, ?, ?)",
+                            (batch, digest(encoded(changes)), canonical(metadata)),
+                        )
+                        self._conn.executemany(
+                            "INSERT INTO _gateway_lineage VALUES (?, ?, ?, ?, ?)",
+                            [
+                                (batch, index, entity, key, canonical(source))
+                                for index, (entity, key, _) in enumerate(changes)
+                            ],
+                        )
+                    self._conn.execute("COMMIT")
+            except BaseException:
+                with self._trusted():
+                    self._conn.execute("ROLLBACK")
+                raise
+            finally:
+                self._ingesting = False
+                self._depth = 0
+                connection._open = False
+                self._drop_temp_objects()
+            counts = {op: 0 for op in ("insert", "update", "delete")}
+            for _, _, op in changes:
+                counts[op] += 1
+            connection.result = {
+                "status": "applied" if changes else "unchanged",
+                "batch": batch,
+                "inserted": counts["insert"],
+                "updated": counts["update"],
+                "deleted": counts["delete"],
+            }
+
+    def _drop_temp_objects(self) -> None:
+        """TEMP staging belongs to the session, not the connection."""
+        with self._trusted():
+            objects = self._conn.execute(
+                "SELECT type, name FROM sqlite_temp_master " "WHERE type IN ('view', 'table') ORDER BY type DESC"
+            ).fetchall()
+            for kind, name in objects:
+                self._conn.execute(f"DROP {kind} IF EXISTS temp.{quote(name)}")
+
+    def _state(self) -> dict[str, dict[str, str]]:
+        """Persisted ``_key -> payload`` per entity, for change attribution."""
+        with self._trusted():
+            return {
+                name: dict(self._conn.execute(f"SELECT _key, payload FROM {quote(name)}"))
+                for name in self.contract.entities
+            }
+
+    @staticmethod
+    def _changes(before: dict, after: dict) -> list[list[str]]:
+        changes = []
+        for name in sorted(before):
+            old, new = before[name], after[name]
+            for key in sorted(set(old) | set(new)):
+                if old.get(key) == new.get(key):
+                    continue
+                if key not in old:
+                    op = "insert"
+                elif key not in new:
+                    op = "delete"
+                else:
+                    op = "update"
+                changes.append([name, key, op])
+        return changes
+
+    def lineage(self, entity: str, key: dict) -> list[dict]:
+        with self._lock:
+            encoded_key = self._key(entity, key, exact=True)
+            rows = self._conn.execute(
+                "SELECT b.metadata, l.ordinal, l.source "
+                "FROM _gateway_lineage l "
+                "JOIN _gateway_batches b ON b.id=l.batch "
+                "WHERE l.entity=? AND l.key=? "
+                "ORDER BY b.rowid, l.ordinal",
+                (entity, encoded_key),
+            )
+            return [
+                {
+                    "batch": json.loads(row[0]),
+                    "index": row[1],
+                    "source": json.loads(row[2]),
+                }
+                for row in rows
+            ]
+
+    def _row_violation(self, name: str, entity: dict, row) -> ContractError | None:
+        """The contract violation for one persisted row, with lineage context."""
+        try:
+            value = parse_json(row[1], f"{entity['location']}/schema")
+        except ContractError as error:
+            return error.contextualize(entity=name)
+        except (ValueError, TypeError):
+            return ContractError("state_payload", "Stored entity payload is not JSON", entity=name)
+        try:
+            validate(entity["schema"], value, location=f"{entity['location']}/schema")
+            if self._key(name, value) != row[0]:
+                fail("key", "Stored key differs from payload identity", entity=name)
+            for rel in entity["relationships"]:
+                keys = [payload_value(value, field) for field in rel["fields"]]
+                if all(k is None for k in keys):
+                    continue
+                if any(k is None for k in keys):
+                    fail(
+                        "foreign_key",
+                        "Composite references must be wholly present or wholly absent/null",
+                        f"{entity['location']}/relationships",
+                    )
+                target = self.contract.entities[rel["target"]]
+                target_key = dict(zip(target["primary_key"], keys))
+                if self.get(rel["target"], target_key) is None:
+                    fail(
+                        "foreign_key",
+                        "Relationship target does not exist; incomplete evidence requires an explicit mapping decision",
+                        f"{entity['location']}/relationships",
+                    )
+        except ContractError as error:
+            try:
+                diagnostic_key = json.loads(row[0])
+            except (ValueError, TypeError):
+                diagnostic_key = None
+            context = {"entity": name, "key": diagnostic_key}
+            with self._trusted():
+                lineage = self._conn.execute(
+                    "SELECT batch, ordinal, source FROM _gateway_lineage "
+                    "WHERE entity=? AND key=? ORDER BY rowid DESC LIMIT 1",
+                    (name, row[0]),
+                ).fetchone()
+            if lineage:
+                context.update(batch=lineage[0], index=lineage[1], source=json.loads(lineage[2]))
+            return error.contextualize(**context)
+        return None
+
+    def _violations(self, limit: int | None = None) -> list[ContractError]:
+        """Contract violations in persisted state, at most ``limit`` of them."""
+        found: list[ContractError] = []
+        with self._trusted():
+            rows = [
+                (name, entity, row)
+                for name, entity in self.contract.entities.items()
+                for row in self._conn.execute(f"SELECT _key, payload FROM {quote(name)} ORDER BY _key").fetchall()
+            ]
+        for name, entity, row in rows:
+            error = self._row_violation(name, entity, row)
+            if error is not None:
+                found.append(error)
+                if limit is not None and len(found) >= limit:
+                    return found
+        with self._trusted():
+            if self._conn.execute("PRAGMA foreign_key_check").fetchone():
+                found.append(ContractError("foreign_key", "Persisted SQLite relationship check failed"))
+        return found
+
+    def audit(self) -> None:
+        """Validate real persisted payloads, key identity and strict references."""
+        with self._lock:
+            violations = self._violations(limit=1)
+            if violations:
+                raise violations[0]
+
+    def _audit_touched(self) -> None:
+        """The commit-time audit of one transaction: the rows it wrote, re-read from
+        disk and checked like a full audit, plus SQLite's own relationship check over
+        the whole state (a delete that orphans an untouched row shows up there).
+        Rows the transaction never touched cannot have changed, so a transaction's
+        cost follows its size, not the size of the world."""
+        with self._trusted():
+            for name, key in sorted(self._touched):
+                entity = self.contract.entities[name]
+                row = self._conn.execute(f"SELECT _key, payload FROM {quote(name)} WHERE _key = ?", (key,)).fetchone()
+                if row is None:
+                    continue
+                error = self._row_violation(name, entity, row)
+                if error is not None:
+                    raise error
+            if self._conn.execute("PRAGMA foreign_key_check").fetchone():
+                raise ContractError("foreign_key", "Persisted SQLite relationship check failed")
+
+    def counts(self) -> dict[str, int]:
+        """Rows per entity, counted by SQLite: no payload is decoded."""
+        with self._lock, self._trusted():
+            return {
+                name: self._conn.execute(f"SELECT COUNT(*) FROM {quote(name)}").fetchone()[0]
+                for name in self.contract.entities
+            }
+
+    def snapshot(self, destination: str | Path) -> Path:
+        with self._lock:
+            if self._depth:
+                fail("transaction", "Cannot snapshot uncommitted state")
+            self.audit()
+            dest = Path(destination)
+            if dest.exists():
+                fail("state_exists", "Snapshot destination already exists")
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            target = sqlite3.connect(dest)
+            try:
+                self._conn.backup(target)
+            finally:
+                target.close()
+            return dest
+
+    def close(self) -> None:
+        with self._lock:
+            if self._depth:
+                fail("transaction", "Cannot close an active tool/batch transaction")
+            self._conn.close()
+            self._closed = True
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()