agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/harness/world/snapshot.py

@@ -0,0 +1,288 @@
+"""Freezing a world, and starting every scenario from the same frozen copy.
+
+The database is built once and snapshotted; that snapshot is the base state. A scenario restores
+its own copy and layers on whatever it additionally needs, so scenarios cannot inherit each
+other's leftovers and a run is repeatable a week later.
+
+Which is why the overlay exists: a scenario that needs a customer with three open orders adds
+those rows to a restored copy rather than editing the snapshot. The base world stays the shared
+starting point instead of drifting toward whichever scenario was written last.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import sqlite3
+from pathlib import Path
+from typing import Any, Mapping
+
+from .runtime import GeneratedWorld
+
+DATABASE = "world.sqlite"
+HANDLERS = "handlers"
+MANIFEST = "manifest.json"
+STATE = "state.json"
+
+
+def saved(path: str | Path | None) -> bool:
+    """Whether a world has been written here.
+
+    One function, because this question gets asked from six places: the build stage, the
+    conversation, the session listing, the CLI and the UI. Asked as "is there a world.sqlite"
+    each of those was really asking "is this a SQLite world", so an agent whose state lives in
+    services and files saved a world that scored 1.00 and was then invisible to all of them.
+    """
+    return bool(path) and (Path(path) / MANIFEST).exists()
+
+
+WORLD_MODULE = "world.py"
+
+_MODULE = '''"""Generated world for {agent}. Do not edit by hand; regenerate instead.
+
+{notes}
+"""
+
+from pathlib import Path
+
+from fi.alk.harness.world.runtime import GeneratedWorld
+
+_HERE = Path(__file__).parent
+
+TOOLS = {tools}
+
+
+class World(GeneratedWorld):
+    name = {agent!r}
+    tools = TOOLS
+    handlers = {{
+        name: (_HERE / "handlers" / f"{{name}}.py").read_text(encoding="utf-8")
+        for name in {handler_names}
+    }}
+
+
+def load(database=None):
+    """This world, restored from the snapshot beside this file.
+
+    Through `restore` rather than by opening a database directly, because not every world has
+    one: an agent whose state lives in services and files keeps its records in the snapshot, and
+    naming a SQLite file would hand back an empty world instead of this one.
+    """
+    from fi.alk.harness.world.snapshot import restore
+
+    return restore(_HERE, into=database) if database else restore(_HERE)
+'''
+
+
+def save(
+    world: GeneratedWorld,
+    path: str | Path,
+    *,
+    notes: str = "",
+    sequences: list[dict[str, Any]] | None = None,
+    world_checks: Mapping[str, str] | None = None,
+) -> Path:
+    """Write the world out: the snapshot, the handlers, the module, and a manifest."""
+    root = Path(path)
+    (root / HANDLERS).mkdir(parents=True, exist_ok=True)
+
+    # Through the store, so a world whose records live somewhere other than a SQLite file, or
+    # nowhere at all, freezes by its own means rather than by one assumed here.
+    world.store.save_to(root)
+
+    for name, source in world.handlers.items():
+        (root / HANDLERS / f"{name}.py").write_text(source, encoding="utf-8")
+
+    (root / WORLD_MODULE).write_text(
+        _MODULE.format(
+            agent=world.name,
+            notes=notes or "Generated from the agent's contract.",
+            tools=json.dumps(world.tools, indent=4),
+            handler_names=json.dumps(sorted(world.handlers)),
+        ),
+        encoding="utf-8",
+    )
+
+    # The agent's own in-memory state, where its tools keep what they act on there rather
+    # than in the database. Frozen as JSON so restoring is the exact reverse, and so a
+    # person can read what the world starts from.
+    if world.state_object is not None:
+        # Round-tripped rather than only written. Every scenario restores from this file, so state
+        # that does not survive the trip would come back subtly different and every check after
+        # it would be grading something else. Better to fail here than to be wrong quietly.
+        frozen = json.dumps(world.state_object, indent=2, default=str)
+        if json.loads(frozen) != world.state_object:
+            raise ValueError(
+                "the agent's state does not survive being frozen as JSON, so restoring it would "
+                "not give back what was saved. Every scenario starts from that restore, so this "
+                "world cannot be trusted. What is in the state that is not plain JSON?"
+            )
+        (root / STATE).write_text(frozen, encoding="utf-8")
+
+    state = world.state()
+    (root / MANIFEST).write_text(
+        json.dumps(
+            {
+                "agent": world.name,
+                # Which store this world used, so restoring it opens the same one rather
+                # than assuming a database that may never have existed.
+                "store": getattr(world.store, "key", "sqlite"),
+                "tools": sorted(world.handlers),
+                # Written because restore reads it. Without it a restored world publishes no
+                # tool descriptions at all, and every later stage has to reconstruct them.
+                "tool_specs": list(world.tools),
+                "tables": {name: len(rows) for name, rows in state.items()},
+                # Kept because they are judgement about this agent, not something a schema
+                # implies. A world picked up again can be re-verified without redeclaring them.
+                "sequences": list(sequences or []),
+                # The world's own checks are judgement about this agent, so a world picked
+                # up again keeps them rather than having them rewritten from scratch.
+                "world_checks": dict(world_checks or {}),
+                # Where the agent's own code lives. Kept because a restored world has to
+                # be able to import the tools it was bound to, and a scenario run happens
+                # long after the build stage that found the path.
+                "source_root": world.source_root,
+                # A run refuses legacy/demo worlds whose handlers were authored by the harness.
+                # New worlds can only acquire handlers through adopt_tool, and a source root is
+                # required for that import to work.
+                "tool_implementation": "source" if world.source_root else "synthetic",
+                "runtime_tools": sorted(getattr(world, "runtime_tools", set())),
+                "external_runtime": bool(getattr(world, "external_runtime", False)),
+                # How this agent says no in a returned value. Without it a restored world
+                # cannot tell a refusal from a success, so every run records "Error: no such
+                # order" as if the call worked, and a check asking whether the agent was
+                # refused is answered wrongly rather than reported as unanswerable.
+                "refusal_signature": world.refusal_signature,
+                "notes": notes,
+            },
+            indent=2,
+            ensure_ascii=False,
+        ),
+        encoding="utf-8",
+    )
+    return root
+
+
+def restore(path: str | Path, *, into: str | Path | None = None) -> GeneratedWorld:
+    """A fresh, independent copy of the frozen world.
+
+    In memory by default, because a scenario should not be able to write back into the snapshot
+    every later scenario depends on.
+    """
+    root = Path(path)
+    source = root / DATABASE
+    if not (root / MANIFEST).exists():
+        raise FileNotFoundError(f"no world snapshot at {root}")
+
+    manifest = read_manifest(root)
+    provisioned_http = False
+    if (root / "environment.json").exists():
+        from ..provision import ProvisionedEnvironment
+
+        environment = ProvisionedEnvironment.load(root)
+        provisioned_http = bool(
+            environment
+            and any(
+                value.startswith(("http://", "https://"))
+                for value in environment.overrides.values()
+            )
+        )
+    if provisioned_http:
+        if into is not None:
+            raise ValueError(
+                "a source-provisioned world restores into its submitted service, not a database "
+                "file path"
+            )
+        from ..understand import load as load_contract
+        from .provisioned import open_provisioned_world
+
+        contract = load_contract(root)
+        if contract is None:
+            raise FileNotFoundError(f"no contract beside source environment at {root}")
+        world = open_provisioned_world(
+            root,
+            contract,
+            source_root=str(manifest.get("source_root") or ""),
+        )
+        world.store.load_from(root)
+        world.tools = manifest.get("tool_specs", [])
+        world.refusal_signature = str(manifest.get("refusal_signature") or "")
+        return world
+    handlers = {
+        name: (root / HANDLERS / f"{name}.py").read_text(encoding="utf-8")
+        for name in manifest.get("tools", [])
+        if (root / HANDLERS / f"{name}.py").exists()
+    }
+
+    named = str(manifest.get("store") or "sqlite")
+    if into is None:
+        world = GeneratedWorld(":memory:", kind=named)
+        # Only where there is one. A world whose records the agent's own code keeps has no
+        # database file, and demanding one would make it unrestorable.
+        if source.exists() and getattr(world.store, "connection", None) is not None:
+            origin = sqlite3.connect(source)
+            with world.connection:
+                origin.backup(world.connection)
+            origin.close()
+        else:
+            # A store that keeps its records somewhere other than a SQLite file loads them its own
+            # way. Without this the world comes back with an empty store, and everything a check
+            # reads is whatever happened to land in the agent's state instead.
+            world.store.load_from(root)
+    else:
+        target = Path(into)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(source, target)
+        world = GeneratedWorld(target, kind=named)
+
+    world.name = manifest.get("agent", "generated")
+    world.handlers = handlers
+    world.runtime_tools = set(manifest.get("runtime_tools") or [])
+    world.external_runtime = bool(manifest.get("external_runtime", False))
+    world.tools = manifest.get("tool_specs", [])
+    world.refusal_signature = str(manifest.get("refusal_signature") or "")
+    # A world whose handlers bind to the agent's own code cannot run them unless that code
+    # is importable again, and the frozen state is what those tools act on.
+    reached = str(manifest.get("source_root") or "")
+    if reached:
+        world.reach(reached)
+    frozen_state = root / STATE
+    if frozen_state.exists():
+        world.state_object = json.loads(frozen_state.read_text(encoding="utf-8"))
+    return world
+
+
+def read_manifest(path: str | Path) -> dict[str, Any]:
+    return json.loads((Path(path) / MANIFEST).read_text(encoding="utf-8"))
+
+
+def require_source_implementation(path: str | Path) -> None:
+    """Refuse worlds that do not prove their tools came from the submitted source."""
+    manifest = read_manifest(path)
+    if manifest.get("tool_implementation") != "source":
+        raise RuntimeError(
+            "This environment has no source-implementation provenance. It was created by an "
+            "older/synthetic harness path and may contain reimplemented handlers. Rebuild it "
+            "from the agent repository; it cannot be used for a test run."
+        )
+
+
+def apply_overlay(world: GeneratedWorld, overlay: Mapping[str, Any] | None) -> int:
+    """Layer one scenario's own rows onto a restored world.
+
+    ``{"table": [{"column": value}, ...]}``. The only sanctioned way a scenario adds data, so the
+    base world stays the shared starting point rather than drifting per scenario.
+    """
+    written = 0
+    for table, rows in (overlay or {}).items():
+        for row in rows or []:
+            if not isinstance(row, Mapping) or not row:
+                continue
+            columns = ", ".join(row)
+            marks = ", ".join("?" for _ in row)
+            world.connection.execute(
+                f"INSERT INTO {table} ({columns}) VALUES ({marks})", list(row.values())
+            )
+            written += 1
+    world.connection.commit()
+    return written

fi/alk/harness/world/stores/__init__.py

@@ -0,0 +1,305 @@
+"""The stores the harness can stand up for an agent, and what every one of them owes a world.
+
+A store is the thing underneath an agent's tools: whatever really holds the records its queries
+run against. It is never asked to execute a tool. It is asked to exist, to hold data, to say what
+it holds, to let a scenario change a little of it, and to go back to how it was.
+
+Which engine gets stood up is read off the agent, never chosen for it. Postgres and ClickHouse
+disagree about dialect, types and what a transaction even means, so testing one against the other
+grades an agent on queries it never runs. An engine the harness cannot stand up is an answer, not
+a reason to substitute something that merely resembles it.
+
+What a store owes falls into four groups, and most stores care about three:
+
+    lifecycle      start, stop, dsn        stand it up and say where it is
+    contents       apply, execute, query   statements, in whatever this engine speaks
+    records        collections, holds, records, add, amend, remove
+    going back     freeze, restore         between scenarios
+                   save_to, load_from      to and from disk, for the base world
+
+The records group is what keeps a scenario from ever naming a store. `world.put`, `world.change`
+and `world.drop` land here, so the same scenario runs against SQLite, against Postgres in a
+container, or against a structure the agent's own code holds, without a line of it changing.
+`state()` comes free from that group, and `Records` provides it.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Mapping, Protocol, Sequence, runtime_checkable
+
+
+class StoreError(RuntimeError):
+    """The store could not be stood up, or could not answer.
+
+    Distinct from anything the agent did. A store that will not start is our problem and should
+    stop the run loudly, because every result after it would be measured against something that
+    is not there.
+    """
+
+
+@dataclass
+class Snapshot:
+    """Everything a store held at one moment, and what it takes to put it back.
+
+    ``rows`` is kept in the shape ``state()`` reports, so a check written against a world's state
+    reads a snapshot without knowing which engine produced it.
+
+    ``counters`` is whatever an engine hands out that is not itself a record: a Postgres sequence,
+    a MySQL auto-increment, anything that keeps counting after the rows are gone. Restoring rows
+    without restoring these gives the next scenario ids that continue from the last one, and a
+    check naming a specific id then fails for a reason that has nothing to do with the agent.
+    Engines that hand out nothing of the sort leave it empty, which is not a gap.
+    """
+
+    rows: dict[str, list[dict[str, Any]]] = field(default_factory=dict)
+    counters: dict[str, int] = field(default_factory=dict)
+
+    def counts(self) -> dict[str, int]:
+        return {name: len(rows) for name, rows in self.rows.items()}
+
+
+@runtime_checkable
+class Store(Protocol):
+    """A running store the world's records live in."""
+
+    # What this engine is. ``key`` is the same thing under the name a saved manifest already
+    # uses, so a world written before this split still reopens.
+    engine: str
+    key: str
+
+    def start(self) -> None: ...
+    def stop(self) -> None: ...
+    def dsn(self) -> str: ...
+
+    # Statements the harness wrote, in whatever this store speaks.
+    def apply(self, script: str) -> None: ...
+    def execute(self, statement: str, params: Sequence[Any] = ()) -> int: ...
+    def query(
+        self, statement: str, params: Sequence[Any] = ()
+    ) -> list[dict[str, Any]]: ...
+
+    # What a scenario and its checks need without writing a statement themselves.
+    def collections(self) -> list[str]: ...
+    def holds(self, collection: str) -> bool: ...
+    def records(self, collection: str) -> list[dict[str, Any]]: ...
+    def state(self) -> dict[str, list[dict[str, Any]]]: ...
+    def table(self, name: str) -> list[dict[str, Any]]: ...
+    def add(self, collection: str, record: Mapping[str, Any]) -> int | dict[str, Any]: ...
+    def amend(
+        self, collection: str, key: str, changes: Mapping[str, Any], *, by: str = ""
+    ) -> int: ...
+    def remove(self, collection: str, key: str = "", *, by: str = "") -> int: ...
+
+    # Between scenarios, in memory.
+    def freeze(self) -> Snapshot: ...
+    def restore(self, snapshot: Snapshot) -> None: ...
+
+    # To and from disk, so the base world outlives the process that built it.
+    def save_to(self, path: str | Path) -> None: ...
+    def load_from(self, path: str | Path) -> None: ...
+
+    def close(self) -> None: ...
+
+
+class Records:
+    """``state`` from the record methods, for any store that has them.
+
+    Kept in one place because the two would otherwise drift, and they are the pair the gates
+    compare: the bite gate empties a store and reads ``state``, while a scenario changes it
+    through ``add`` and ``amend``. If those disagree about what a collection contains, a check
+    passes against something no scenario can produce.
+    """
+
+    def state(self) -> dict[str, list[dict[str, Any]]]:
+        return {name: self.records(name) for name in self.collections()}  # type: ignore[attr-defined]
+
+
+class Held:
+    """The record methods, and disk, for a store that already answers ``state``.
+
+    The mirror of ``Records``, for stores built the other way round: a container store reads
+    everything it holds in one go, and the per-collection questions follow from that. Saving to
+    disk is the snapshot as JSON, which works for any engine because a snapshot is already the
+    engine-independent shape.
+
+    ``add``, ``amend`` and ``remove`` are not derivable and are left to the engine. A store
+    without them refuses loudly rather than silently doing nothing, because the alternative is a
+    scenario whose setup appears to run and changes nothing, and a run then graded against a
+    world that was never set up.
+    """
+
+    engine: str = ""
+
+    @property
+    def key(self) -> str:
+        return self.engine
+
+    def collections(self) -> list[str]:
+        return sorted(self.state())  # type: ignore[attr-defined]
+
+    def holds(self, collection: str) -> bool:
+        return collection in self.state()  # type: ignore[attr-defined]
+
+    def records(self, collection: str) -> list[dict[str, Any]]:
+        return self.state().get(collection, [])  # type: ignore[attr-defined]
+
+    def execute(self, statement: str, params: Sequence[Any] = ()) -> int:
+        self.apply(statement)  # type: ignore[attr-defined]
+        return 0
+
+    def query(self, statement: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]:
+        raise StoreError(
+            f"{self.engine} does not read back arbitrary statements. Read what it holds with "
+            "records() or state()."
+        )
+
+    def table(self, name: str) -> list[dict[str, Any]]:
+        raise StoreError(
+            f"{self.engine} does not read one table at a time. Read what it holds with "
+            "records() or state()."
+        )
+
+    def add(self, collection: str, record: Mapping[str, Any]) -> int | dict[str, Any]:
+        raise StoreError(_UNWRITABLE.format(engine=self.engine, verb="add to"))
+
+    def amend(
+        self, collection: str, key: str, changes: Mapping[str, Any], *, by: str = ""
+    ) -> int:
+        raise StoreError(_UNWRITABLE.format(engine=self.engine, verb="change"))
+
+    def remove(self, collection: str, key: str = "", *, by: str = "") -> int:
+        raise StoreError(_UNWRITABLE.format(engine=self.engine, verb="remove from"))
+
+    def clear(self) -> None:
+        """Empty it, by restoring a snapshot that holds nothing."""
+        self.restore(Snapshot())  # type: ignore[attr-defined]
+
+    def save_to(self, path: str | Path) -> None:
+        import json
+
+        root = Path(path)
+        root.mkdir(parents=True, exist_ok=True)
+        frozen = self.freeze()  # type: ignore[attr-defined]
+        (root / SAVED).write_text(
+            json.dumps(
+                {
+                    # The schema as the scripts that made it, because the rows alone cannot
+                    # come back: a fresh engine has no tables to put them in.
+                    "schema": list(getattr(self, "applied", [])),
+                    "rows": frozen.rows,
+                    "counters": frozen.counters,
+                },
+                indent=2,
+                default=str,
+            ),
+            encoding="utf-8",
+        )
+
+    def load_from(self, path: str | Path) -> None:
+        import json
+
+        held = Path(path) / SAVED
+        if not held.exists():
+            raise StoreError(f"no saved store at {held}")
+        kept = json.loads(held.read_text(encoding="utf-8"))
+        for script in kept.get("schema") or []:
+            self.apply(script)  # type: ignore[attr-defined]
+        self.restore(
+            Snapshot(rows=kept.get("rows") or {}, counters=kept.get("counters") or {})
+        )  # type: ignore[attr-defined]
+
+    def close(self) -> None:
+        self.stop()  # type: ignore[attr-defined]
+
+
+# What a saved container store is written as. Not the engine's own dump format: a snapshot is
+# already engine-independent, and a dump would tie the saved world to the version that wrote it.
+SAVED = "store.json"
+
+_UNWRITABLE = (
+    "{engine} has no way to {verb} a collection one record at a time, so a scenario cannot set "
+    "up on it. Give the store add, amend and remove in this engine's own language."
+)
+
+_REGISTRY: dict[str, Callable[..., Store]] = {}
+
+# Names people and manifests actually write, pointing at the engine they mean. Kept explicit
+# rather than normalised in code, because guessing which engine an unrecognised word meant is
+# how an agent ends up graded against the wrong one.
+_ALIASES = {
+    "": "in_process",
+    "none": "in_process",
+    "memory": "in_process",
+    "in-memory": "in_process",
+    "inprocess": "in_process",
+}
+
+
+def register_store(engine: str, factory: Callable[..., Store]) -> None:
+    """Teach the harness an engine. A class and this line.
+
+    The cost of this line is what decides whether "whatever the agent uses" is real or an
+    aspiration, which is why the shared work lives in ``ContainerStore`` and an engine
+    contributes only what genuinely differs.
+    """
+    _REGISTRY[engine] = factory
+
+
+def supported() -> tuple[str, ...]:
+    return tuple(sorted(_REGISTRY))
+
+
+def resolve(engine: str = "", **options: Any) -> Store:
+    """The store for an engine, or a refusal naming what there is.
+
+    Deliberately not a fallback. An agent on an engine nobody has taught the harness to run is a
+    gap worth reporting, and quietly handing it a different store would produce a green suite
+    about queries the agent never executes.
+    """
+    named = (engine or "").strip().lower()
+    named = _ALIASES.get(named, named)
+    if named not in _REGISTRY:
+        raise StoreError(
+            f"no store for engine {named!r}; the harness can stand up "
+            f"{', '.join(supported()) or 'nothing yet'}. Adding one is a class with the record "
+            "methods and a call to register_store, or write_store_ops for an engine in a container."
+        )
+    return _REGISTRY[named](**options)
+
+
+# The name the rest of the harness has always called this by.
+open_store = resolve
+
+
+from .inprocess import InProcessStore  # noqa: E402
+from .sqlite import SqliteStore  # noqa: E402
+
+register_store(SqliteStore.engine, SqliteStore)
+register_store(InProcessStore.engine, InProcessStore)
+
+from .container import ContainerStore, docker, strays  # noqa: E402
+from .postgres import PostgresStore  # noqa: E402
+
+# Postgres is registered as the worked example, not as the supported list. An engine the harness
+# has never seen is meant to be written at build time against ``ContainerStore`` and proved by
+# the gates, rather than waiting for someone to ship a class for it.
+register_store(PostgresStore.engine, PostgresStore)
+
+__all__ = [
+    "ContainerStore",
+    "InProcessStore",
+    "PostgresStore",
+    "Records",
+    "Snapshot",
+    "SqliteStore",
+    "Store",
+    "StoreError",
+    "docker",
+    "open_store",
+    "register_store",
+    "resolve",
+    "strays",
+    "supported",
+]