agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/harness/world/kinds.py

@@ -0,0 +1,196 @@
+"""What a kind of world has to be able to do, so the checks do not care which kind it is.
+
+A world backed by a database and a world backed by a page are different in every detail and the
+same in what matters: something either exists in them or does not, an action either takes effect
+or is refused, and what an action leaves behind is either carried or lost. Those are the things
+worth checking, and none of them mention a table.
+
+So the checks are written against this, and a kind supplies the four answers only it can give:
+what exists, what the mutable state is, how to freeze it, and how to put it back. Adding a kind
+is a class and a registration; nothing in the gate changes.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Mapping, Protocol, runtime_checkable
+
+from .runtime import GeneratedWorld
+
+
+def _rows(collection: Any) -> list[Any]:
+    """One collection's members, whichever shape it is kept in.
+
+    A table gives a list of row mappings. A collection the agent's own code owns is as often a
+    mapping keyed by identifier, and iterating that yields keys rather than records, which is how
+    a check written for one shape silently reads the other.
+    """
+    if isinstance(collection, dict):
+        return list(collection.values())
+    if isinstance(collection, (list, tuple)):
+        return list(collection)
+    return [collection]
+
+
+def _identifiers(state: Mapping[str, Any]) -> set[str]:
+    found: set[str] = set()
+    for name, collection in state.items():
+        if isinstance(collection, dict):
+            # The keys of a mapping are identifiers in their own right, and usually the ones a
+            # tool is called with.
+            found.update(str(key) for key in collection if isinstance(key, str) and key)
+        for row in _rows(collection):
+            if isinstance(row, Mapping):
+                found.update(
+                    value for value in row.values() if isinstance(value, str) and value
+                )
+            elif isinstance(row, str) and row:
+                found.add(row)
+    return found
+
+
+def _sizes(state: Mapping[str, Any]) -> dict[str, int]:
+    return {name: len(_rows(collection)) for name, collection in state.items()}
+
+
+@runtime_checkable
+class WorldKind(Protocol):
+    """The per-kind half of a world. The shared half is ``GeneratedWorld``."""
+
+    key: str
+    label: str
+
+    def values_present(self, world: GeneratedWorld) -> set[str]:
+        """Every identifier this world contains.
+
+        Answers whether the catalogue is complete: a contract that permits a value the world has
+        never heard of produces a tool that refuses forever, which is indistinguishable from a
+        tool being correctly strict.
+        """
+
+    def mutable_state(self, world: GeneratedWorld) -> dict[str, int]:
+        """Named parts of the world that an action can change, and how much is in each.
+
+        Used for two things: noticing that a saved world still holds whatever the builder was
+        experimenting with, and noticing that a sequence of actions left nothing behind.
+        """
+
+    def describe(self, world: GeneratedWorld) -> str:
+        """A short human-readable account of what is in the world."""
+
+
+class SqliteWorld:
+    """A world whose state is rows in tables. Tool APIs, and anything with a data store."""
+
+    key = "sqlite"
+    label = "a database behind the agent's tools"
+
+    def values_present(self, world: GeneratedWorld) -> set[str]:
+        return _identifiers(world.state())
+
+    def mutable_state(self, world: GeneratedWorld) -> dict[str, int]:
+        return _sizes(world.state())
+
+    def describe(self, world: GeneratedWorld) -> str:
+        counts = self.mutable_state(world)
+        return ", ".join(f"{name}: {count}" for name, count in sorted(counts.items()))
+
+
+class BrowserWorld:
+    """A world whose state is pages and the actions that change them.
+
+    ALK already carries a browser environment fed DOM snapshots and action fixtures, and it
+    already refuses a click matching no fixture. So this is the same move as the database kind:
+    generate instances of a shape that exists, rather than invent a mechanism.
+
+    What exists here is the set of things an agent can reach, which is selectors and URLs rather
+    than ids; what changes is which snapshot is current and what the actions have mutated.
+    """
+
+    key = "browser"
+    label = "pages and the actions that change them"
+
+    def values_present(self, world: GeneratedWorld) -> set[str]:
+        present: set[str] = set()
+        for collection in world.state().values():
+            for row in _rows(collection):
+                if not isinstance(row, Mapping):
+                    continue
+                for column in ("url", "selector", "id", "name", "action"):
+                    value = row.get(column)
+                    if isinstance(value, str) and value:
+                        present.add(value)
+        return present
+
+    def mutable_state(self, world: GeneratedWorld) -> dict[str, int]:
+        return _sizes(world.state())
+
+    def describe(self, world: GeneratedWorld) -> str:
+        counts = self.mutable_state(world)
+        return ", ".join(f"{name}: {count}" for name, count in sorted(counts.items()))
+
+
+class InProcessWorld:
+    """A world whose state the agent's own code holds, rather than a store we stood up.
+
+    This is what an adopted world usually is: the agent's tools were written to act on a structure
+    they build themselves, so the world holds that structure and does not interpret it.
+    """
+
+    key = "in_process"
+    label = "state the agent's own code keeps"
+
+    def values_present(self, world: GeneratedWorld) -> set[str]:
+        return _identifiers(world.state())
+
+    def mutable_state(self, world: GeneratedWorld) -> dict[str, int]:
+        return _sizes(world.state())
+
+    def describe(self, world: GeneratedWorld) -> str:
+        counts = self.mutable_state(world)
+        return ", ".join(f"{name}: {count}" for name, count in sorted(counts.items()))
+
+
+_REGISTRY: dict[str, Callable[[], WorldKind]] = {
+    SqliteWorld.key: SqliteWorld,
+    BrowserWorld.key: BrowserWorld,
+    InProcessWorld.key: InProcessWorld,
+}
+
+
+def register_kind(key: str, factory: Callable[[], WorldKind]) -> None:
+    """Add a kind of world. Computer use, a filesystem, a queue: a class and this line."""
+    _REGISTRY[key] = factory
+
+
+def resolve(key: str) -> WorldKind:
+    if key not in _REGISTRY:
+        raise NotImplementedError(
+            f"no world kind {key!r}; registered kinds are {', '.join(sorted(_REGISTRY))}"
+        )
+    return _REGISTRY[key]()
+
+
+def supported() -> tuple[str, ...]:
+    return tuple(sorted(_REGISTRY))
+
+
+def for_contract(contract: Any) -> WorldKind:
+    """The kind of world an agent needs, from what the contract says it is.
+
+    Chosen rather than guessed at build time: an agent reachable by voice and by browser is one
+    agent with two runtimes, and which world to build is a decision about what is being tested.
+    """
+    # What the store is, when the contract knows. That is the honest source: how a person reaches
+    # the agent says nothing about what its tools read and write, and an agent whose state lives
+    # in its own process is not a database however it is spoken to.
+    store = getattr(contract, "data_store", None)
+    named = str(getattr(store, "kind", "") or "").lower()
+    if named in _REGISTRY:
+        return resolve(named)
+    if named in ("in_process", "memory", "in-memory", "none", ""):
+        if named:
+            return resolve("in_process")
+    modality = str(getattr(contract, "modality", "") or "").lower()
+    if modality in ("browser", "computer_use", "cua"):
+        return resolve("browser")
+    return resolve("sqlite")

fi/alk/harness/world/mutate.py

@@ -0,0 +1,186 @@
+"""Breaking a world on purpose, to find out whether its checks would notice.
+
+The checks that verify an environment are written by whoever built it. That is the right way
+round: what makes a world usable is a judgement about this agent, and no fixed set of probes
+written in advance can make it for every agent. But it leaves nothing independent confirming the
+checks work, and a check that cannot fail reports a healthy world forever.
+
+So the checks are put to a test they cannot talk their way out of. The world is damaged in ways
+that are obviously wrong, and the checks have to go red. One that stays green through every
+damaged world is not verifying anything, whatever it claims to inspect.
+
+The damage is deliberately generic, because a mutation that needed to understand the agent would
+need the same judgement the checks needed, and nothing would be gained:
+
+- **emptied**: every collection loses its contents. A world with no data at all.
+- **silenced**: every tool answers with nothing. Calls succeed and change nothing.
+
+Any check worth keeping fails against at least one of those. Most fail against both.
+"""
+
+from __future__ import annotations
+
+import copy
+from typing import Any, Callable
+
+from .runtime import GeneratedWorld
+
+EMPTIED = "emptied"
+# Not a kind of damage: how the report says the damage itself did not happen.
+UNDAMAGED = "could not be damaged"
+SILENCED = "silenced"
+
+# What a silenced tool answers. Deliberately a plain empty string: it is the shape a handler
+# returns when it has done nothing, which is exactly the failure being simulated.
+_MUTE = "def handle(args, db):\n    return ''\n"
+
+
+def _empty(world: GeneratedWorld) -> None:
+    """Take the contents out of the world, leaving its shape intact.
+
+    The store empties itself where it can, because only it knows what its engine needs: a
+    relational one has to suspend foreign keys, or deleting a referenced table fails and most of
+    the data stays. Dropping collection by collection through the world's own vocabulary is the
+    fallback, for a store that has no opinion.
+    """
+    emptied = getattr(world.store, "clear", None)
+    if callable(emptied):
+        emptied()
+    else:
+        for name in list(world.state()):
+            try:
+                world.drop(name)
+            except Exception:
+                # One collection that will not empty is not a reason to abandon the mutation. What
+                # survives is reported by `left`, so the gate can tell a check that failed to
+                # notice from a mutation that never happened.
+                continue
+    # The agent's own state, where its code keeps what its tools act on. A world can have both.
+    held = world.state_object
+    if isinstance(held, dict):
+        for name, group in held.items():
+            if isinstance(group, dict):
+                group.clear()
+            elif isinstance(group, list):
+                group.clear()
+            else:
+                held[name] = None
+
+
+def left(world: GeneratedWorld) -> dict[str, int]:
+    """What is still in the world after it was supposed to be empty.
+
+    The gate accuses a check of verifying nothing when it stays green through damage. That
+    accusation is only fair if the damage actually happened: a store that quietly refused to
+    empty leaves every check reading real data and looking vacuous, and the person then rewrites
+    a check that was right all along.
+    """
+    return {name: len(rows) for name, rows in world.state().items() if rows}
+
+
+def _silence(world: GeneratedWorld) -> None:
+    """Leave every tool answering with nothing, so no call has any effect."""
+    for name in list(world.handlers):
+        world.handlers[name] = _MUTE
+
+
+def damage() -> dict[str, Callable[[GeneratedWorld], None]]:
+    """Every way a world is broken on purpose, by name."""
+    return {EMPTIED: _empty, SILENCED: _silence}
+
+
+def unnoticed(
+    world_root: Any,
+    checks: list[tuple[str, str]],
+    *,
+    run: Callable[[str, GeneratedWorld], Any],
+    restore: Callable[[Any], GeneratedWorld],
+) -> dict[str, list[str]]:
+    """Which checks fail to notice each kind of damage.
+
+    Every mutation runs against its own restored copy, so one cannot inherit another's damage and
+    a check is never blamed for a world some earlier mutation had already emptied.
+
+    Returns damage name to the checks that stayed green through it. A check appearing under every
+    kind of damage is one that cannot fail.
+    """
+    survived: dict[str, list[str]] = {}
+    for name, apply in damage().items():
+        broken = restore(world_root)
+        try:
+            apply(broken)
+            if name == EMPTIED:
+                remaining = left(broken)
+                if remaining:
+                    # The mutation did not land, so nothing can be concluded from it. Saying so is
+                    # the point: reporting these checks as blind would have somebody rewrite a
+                    # check that was reading the world correctly the whole time.
+                    survived[name] = []
+                    survived.setdefault(UNDAMAGED, []).append(
+                        f"the world would not empty: {remaining}"
+                    )
+                    continue
+            still_green = []
+            for check_name, source in checks:
+                outcome = run(source, broken)
+                # A check that raises has not verified anything either, but that is a broken
+                # check rather than a blind one, and it is reported separately by the caller.
+                if getattr(outcome, "held", False):
+                    still_green.append(check_name)
+            survived[name] = still_green
+        finally:
+            broken.close()
+    return survived
+
+
+def unnoticed_in_place(
+    world: GeneratedWorld,
+    checks: list[tuple[str, str]],
+    *,
+    run: Callable[[str, GeneratedWorld], Any],
+) -> dict[str, list[str]]:
+    """Mutation-test checks against a live service-backed world, restoring it each time.
+
+    A source-provisioned database is owned by the repository's Compose project. Serialising it
+    to a temporary generic world loses that ownership information and makes restore try to boot
+    a second database from generated DDL. The live store already supplies exact checkpoint and
+    restore operations, so use those and put both data and forwarded handlers back after every
+    mutation.
+    """
+    baseline = world.checkpoint()
+    handlers = copy.deepcopy(world.handlers)
+    survived: dict[str, list[str]] = {}
+    for name, apply in damage().items():
+        try:
+            apply(world)
+            if name == EMPTIED:
+                remaining = left(world)
+                if remaining:
+                    survived[name] = []
+                    survived.setdefault(UNDAMAGED, []).append(
+                        f"the world would not empty: {remaining}"
+                    )
+                    continue
+            survived[name] = [
+                check_name
+                for check_name, source in checks
+                if getattr(run(source, world), "held", False)
+            ]
+        finally:
+            world.revert(baseline)
+            world.handlers = copy.deepcopy(handlers)
+    return survived
+
+
+def blind(survived: dict[str, list[str]]) -> list[str]:
+    """Checks that stayed green through every kind of damage."""
+    if not survived:
+        return []
+    kinds = [names for kind, names in survived.items() if kind != UNDAMAGED]
+    if not kinds:
+        return []
+    return (
+        sorted(set(kinds[0]).intersection(*kinds[1:]))
+        if len(kinds) > 1
+        else sorted(kinds[0])
+    )