agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/extensions.py

@@ -0,0 +1,163 @@
+"""Unit 4 (BBG U4 / ARCH §2e) — the four 13D-4 registries + extension_admission.
+
+Explicit in-process registration (AD-J: no entry-points, no import-time
+discovery — local-first, gate-scannable). One uniform record shape across four
+points; ONE choke point (``extension_admission``) enforces 13D-D5 (extension ≠
+exemption) in gated contexts. The facade pushes world-kind/role registrations
+DOWN into ``fi.simulate.simulation.contract`` setters (Appendix C-1); the engine
+never imports up.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, Mapping, Optional
+
+from .live._contract import EVIDENCE_CLASSES, RELEASE_ADMISSIBLE_EVIDENCE_CLASSES
+
+EXTENSION_POINTS = ("environment", "loss", "optimizer", "generator")
+
+# point -> name -> record
+_REGISTRY: Dict[str, Dict[str, dict]] = {point: {} for point in EXTENSION_POINTS}
+
+
+class ExtensionError(ValueError):
+    """Raised when a registration record is malformed."""
+
+
+def _validate_record(point: str, record: Mapping[str, Any]) -> dict:
+    if point not in EXTENSION_POINTS:
+        raise ExtensionError(f"point {point!r} not in {EXTENSION_POINTS}")
+    name = record.get("name")
+    if not name or "." not in str(name):
+        raise ExtensionError(
+            "extension record requires a namespaced name 'vendor.name'"
+        )
+    if str(name) in _REGISTRY[point]:
+        raise ExtensionError(f"extension name collision: {name!r} already registered for {point}")
+    caps = record.get("evidence_class_capability") or []
+    for cap in caps:
+        if cap not in EVIDENCE_CLASSES:
+            raise ExtensionError(
+                f"evidence_class_capability {cap!r} not in {EVIDENCE_CLASSES}"
+            )
+    if point == "optimizer" and not record.get("declared_budgets"):
+        raise ExtensionError(
+            "optimizer extensions REQUIRE declared_budgets (refused otherwise)"
+        )
+    stored = dict(record)
+    stored["point"] = point
+    stored.setdefault("provides", None)
+    stored.setdefault("conformance_manifest", None)
+    stored.setdefault("evidence_class_capability", list(caps))
+    stored.setdefault("version", "0.0.0")
+    stored["gated_contexts_runnable"] = False  # until a green conformance run lands
+    return stored
+
+
+def register_extension(point: str, record: Mapping[str, Any]) -> dict:
+    stored = _validate_record(point, record)
+    # world-kind/role registrations push down into the engine contract (C-1).
+    if point == "environment":
+        token = stored.get("kind_token")
+        if token:
+            if not stored.get("spec_validator") or not stored.get("rung_ladder"):
+                raise ExtensionError(
+                    "a custom world.kind extension MUST declare spec_validator + rung_ladder (R4)"
+                )
+            from fi.simulate.simulation import contract as _contract
+            _contract.register_world_kind(str(token), stored)
+    _REGISTRY[point][str(stored["name"])] = stored
+    return stored
+
+
+def register_environment(record: Mapping[str, Any]) -> dict:
+    """Register a descriptive environment **metadata record** (studio extension).
+
+    Canon correspondence (assessment §8 Gap B): the runtime sibling
+    ``fi.simulate.registry.register_environment`` registers a **runnable** plugin
+    factory in ``environment_registry``. This one records metadata (and, for a
+    ``world.kind`` extension carrying a ``kind_token``, writes the contract's
+    extension side-table via ``contract.register_world_kind`` — the frozen canon
+    constants never mutate). A record is not a factory — the two are deliberately
+    unwired.
+    """
+    return register_extension("environment", record)
+
+
+def register_objective(record: Mapping[str, Any]) -> dict:
+    return register_extension("loss", record)
+
+
+def register_optimizer(record: Mapping[str, Any]) -> dict:
+    return register_extension("optimizer", record)
+
+
+def register_generator(record: Mapping[str, Any]) -> dict:
+    return register_extension("generator", record)
+
+
+def register_role(record: Mapping[str, Any]) -> dict:
+    """Register a namespaced cast role (pushes into the engine contract)."""
+    stored = _validate_record("environment", {**record, "name": record["name"]})
+    token = stored.get("kind_token") or stored.get("role")
+    from fi.simulate.simulation import contract as _contract
+    if token:
+        _contract.register_cast_role(str(token), stored)
+    _REGISTRY["environment"][str(stored["name"])] = stored
+    return stored
+
+
+def resolve(point: str, token: str) -> Optional[dict]:
+    """Built-ins first — callers check the canon BEFORE consulting this."""
+    return _REGISTRY.get(point, {}).get(str(token))
+
+
+def registered(point: str) -> tuple[str, ...]:
+    return tuple(sorted(_REGISTRY.get(point, {})))
+
+
+def extension_admission(record: Mapping[str, Any], context: Mapping[str, Any]) -> dict:
+    """THE one choke point. In gated contexts (release-check/promotion/training)
+    a registered extension that cannot produce admissible evidence does not run.
+    Returns ``{admitted: bool}`` or the structured refusal finding."""
+    gated = bool(context.get("gated"))
+    if not gated:
+        return {"admitted": True, "reason": "non_gated_passthrough"}
+
+    point = record.get("point")
+    caps = set(record.get("evidence_class_capability") or [])
+    admissible_caps = caps & set(RELEASE_ADMISSIBLE_EVIDENCE_CLASSES)
+    conformance_green = bool(record.get("conformance_green") or record.get("gated_contexts_runnable"))
+
+    def refuse(reason: str) -> dict:
+        return {
+            "admitted": False,
+            "type": "extension_evidence_inadmissible",
+            "level": "error",
+            "name": record.get("name"),
+            "point": point,
+            "reason": reason,
+            "remediation": (
+                "run the extension's conformance_manifest green with evidence in "
+                f"{RELEASE_ADMISSIBLE_EVIDENCE_CLASSES} before any gated use"
+            ),
+        }
+
+    # (i) conformance manifest ran green with release-admissible evidence
+    if not conformance_green or not admissible_caps:
+        return refuse("no green conformance run with release-admissible evidence")
+    # (iii) optimizer extensions without declared_budgets never run
+    if point == "optimizer" and not record.get("declared_budgets"):
+        return refuse("optimizer extension has no declared_budgets")
+    # (iv) environment/world-kind extensions claiming an executable kind need a
+    #      green rung-1 fixture run recorded.
+    if point == "environment" and record.get("kind_token"):
+        if not record.get("rung1_fixture_green"):
+            return refuse("world-kind extension lacks a green rung-1 fixture run")
+    return {"admitted": True, "reason": "gated_admitted"}
+
+
+def _reset_extensions() -> None:  # test-only
+    for point in EXTENSION_POINTS:
+        _REGISTRY[point].clear()
+    from fi.simulate.simulation import contract as _contract
+    _contract._reset_contract_extensions()

fi/alk/harness/ARCHITECTURE.md

@@ -0,0 +1,231 @@
+# ALK harness execution architecture
+
+Implementation and test evidence are tracked in
+[`IMPLEMENTATION_AND_VALIDATION_STATUS.md`](IMPLEMENTATION_AND_VALIDATION_STATUS.md). Repository
+packaging behavior and the conformance matrix are documented in
+[`ENVIRONMENT_CONFORMANCE.md`](ENVIRONMENT_CONFORMANCE.md).
+
+## Ownership
+
+ALK owns all execution behavior. The Future AGI platform is a control and data plane only.
+
+| Concern | ALK package | Platform | Hosted sandbox fleet |
+|---|---:|---:|---:|
+| Understand repository and agent | yes | no | runs ALK |
+| Build databases, tools, mocks and seed data | yes | no | runs ALK |
+| Generate/validate scenarios and personas | yes | no | runs ALK |
+| Connect to and simulate the agent | yes | no | runs ALK |
+| Grade evidence and create artifacts | yes | no | runs ALK |
+| Repository/secret authorization | consumes references | yes | resolves job-scoped values |
+| Job UI, chat, cancellation and history | no | yes | reports status |
+| Event/result/artifact storage | emits data | yes | forwards data |
+
+No harness stage imports Temporal, Django, platform models, or platform worker code.
+
+## One engine, two deployments
+
+```text
+Local CLI                              Hosted product
+─────────                              ──────────────
+agent-learn harness auto               platform creates HarnessJob
+        │                                        │
+        ▼                                        ▼
+HarnessExecutor                         isolated ALK sandbox
+        │                               + HarnessExecutor
+        └──────────── same pipeline ─────────────┘
+                         │
+                         ▼
+ understand → environment → bundle → data → scenarios → connect → simulate → grade
+```
+
+`HarnessJob` is the immutable input boundary. Local jobs use a local repository path. Hosted
+jobs use a GitHub installation/repository reference, archive, image, or remote endpoint and can
+never contain a local path. Agent credentials are `SecretRef` values; resolved secrets are
+rejected from serialized jobs.
+
+## Environment boundary
+
+The environment is infrastructure owned by the harness, not a connection to customer
+production. ALK may adopt schemas, migrations, fixtures and mock services from the submitted
+repository. It then fills missing test data and dependencies itself.
+
+Every successful build is sealed as an `EnvironmentBundle`:
+
+- versioned schema;
+- SHA-256 content address;
+- exact source/generator provenance;
+- runtime document and service list;
+- named capabilities and readiness probes;
+- per-file hashes and sizes;
+- no symlinks or resolved secrets;
+- immutable verification before execution.
+
+This removes repository-path assumptions from the runtime. Local Compose and a future hosted
+Kubernetes/Firecracker provider implement the same `RuntimeProvider` interface and consume the
+same manifest.
+
+## Provisioning policy
+
+The current local provider follows this order:
+
+1. Detect the submitted Compose definition and declared default infrastructure services.
+2. Give the run a unique Compose project and free host ports.
+3. Exclude opt-in agent/worker services from infrastructure startup.
+4. Build and wait for declared health checks once.
+5. Derive only the endpoint configuration the agent already reads.
+6. Reuse a healthy build only when its complete source fingerprint matches.
+7. If no Compose file exists but a Dockerfile does, generate only supported infrastructure
+   declared by the contract (currently Postgres, ClickHouse and Redis) and compose it around the
+   submitted runtime. Never generate agent tools or proprietary service behavior.
+8. Reset between scenarios from a verified snapshot or isolated lifecycle reset.
+9. Remove the exact project and its test volumes during cleanup.
+
+Unknown database engines do not silently fall back to SQLite or Postgres. A store adapter can be
+generated against the engine's native driver, but it must pass generic freeze/restore/counter
+drift and mutation gates before scenarios may use it.
+
+## Evidence and grading invariants
+
+- Setup calls are never credited to the agent.
+- A missing agent call cannot satisfy a call-dependent check.
+- Checks must fail against an empty or deliberately damaged world.
+- Dependent checks cannot pass when their prerequisite action never happened.
+- Tool refusal, agent failure, simulator failure, connectivity failure, environment failure,
+  infrastructure failure and grading failure remain distinct.
+- Transcripts, semantic calls, resulting state, state diffs and recordings are retained according
+  to artifact policy.
+- Agent behavior failures are valid RL results; harness/infrastructure failures are not scored as
+  agent failures.
+
+## Repository-backed chat execution
+
+Chat and voice share the same autonomous lifecycle. Voice has a standard realtime rendezvous;
+chat runtimes instead declare the conversational ingress they already implement in the grounded
+agent contract. Today a repository runtime can expose either:
+
+- HTTP using ALK's turn envelope; or
+- HTTP using an OpenAI Chat Completions-compatible envelope; or
+- a JSON turn exchange over WebSocket.
+
+For every scenario ALK binds the restored world, starts the submitted Compose/Dockerfile/generated
+runtime with the environment's private endpoint overrides, exposes the declared container port on
+an ephemeral loopback port locally (or only the private project network in a hosted runner), waits
+for readiness, drives the conversation, records tool effects and removes the runtime. A default
+Compose API is identified by its declared ingress port and excluded from infrastructure startup;
+ambiguous services fail admission rather than being guessed.
+
+If the submitted agent returns model-facing tool calls, ALK executes them against the same world
+and continues the turn with tool results. If the agent executes tools itself through the injected
+environment endpoints, the world records those calls at the service boundary. Either way, setup
+activity remains separate from agent evidence.
+
+An agent with no external HTTP/WebSocket ingress is not silently reconstructed. Callable/CLI-only
+repository runtimes need a sandbox-side process adapter in a later extension; until then admission
+reports the missing interface explicitly. This preserves the invariant that ALK runs submitted
+agent behavior rather than inventing it.
+
+## Data and scenario quality
+
+Scenario validation rejects predictable/demo fixtures such as `123456`, recycled identities,
+and reused payment/booking placeholders. A suite must vary identities, communication styles,
+locations, account/payment states, instructions and expected paths. Submitted seed data is
+preserved where useful and expanded with synthetic records when it is too sparse to exercise the
+contract.
+
+The simulator is constrained by literal scenario facts, tracks facts already stated, answers the
+agent's current question, detects rephrased loops, and only retries infrastructure failures.
+Deterministic agent weaknesses remain deterministic failures.
+
+## Delivery and recovery
+
+All progress uses ALK's canonical, versioned event envelope. `EventOutbox` writes events and
+fsyncs them before attempting upload. Platform delivery is batchable and idempotent by event ID;
+partial acknowledgements leave the remainder pending. A local run therefore completes offline
+and can sync later. Hosted execution uses the same protocol.
+
+The existing Future AGI result sink remains responsible for platform run rows, transcripts,
+evaluations and recording upload. Platform views render stored data; they do not reconstruct or
+run harness stages.
+
+## Scaling and isolation
+
+One job maps to one ephemeral hosted sandbox and one resource envelope. The scheduler may place
+those sandboxes on Kubernetes pods or micro-VMs, but that decision is outside ALK. Required
+production controls are:
+
+- dedicated execution cluster/account, never ordinary platform workers;
+- per-job filesystem, network namespace and service identity;
+- deny-by-default egress with explicit provider/GitHub/platform destinations;
+- CPU, memory, disk, duration and concurrency quotas from `RuntimeRequirements`;
+- short-lived repository and provider credentials;
+- no privileged containers or host Docker socket inside untrusted sandboxes;
+- artifact size/retention enforcement;
+- cancellation, orphan reconciliation and guaranteed cleanup;
+- cache only content-addressed dependency/image layers, never mutable customer workspaces.
+
+## Extension points
+
+- `SourceAcquirer`: GitHub, archive, image or other source materialization.
+- `RuntimeProvider`: local Compose today; isolated hosted provider next.
+- ALK endpoint adapters: callable/local, HTTP, WebSocket, LiveKit, Vapi and Retell today; MCP and
+  process/container connectors are the next adapters and must use the same registry.
+- Store registry: Postgres, SQLite and in-process today; generated native adapters for new
+  engines after conformance proofs.
+- `EventTransport` and result sinks: local filesystem, Future AGI platform or customer-owned
+  telemetry.
+
+Adding an environment engine, agent connector, source type or scheduler should be one adapter;
+it must not add a branch to scenario generation or grading.
+
+## Agent/tool ownership boundary
+
+ALK provisions the environment around submitted agent code; it never supplies missing agent
+behavior. The submitted repository remains authoritative for prompts, tool schemas, tool
+implementations, orchestration and business rules. Environment adaptation is limited to:
+
+- starting declared infrastructure such as databases, queues, object stores and media services;
+- injecting non-secret endpoints through configuration seams the submitted code already reads;
+- resolving referenced credentials at runtime;
+- seeding and resetting test-owned dependency state; and
+- capturing calls, tool evidence and generated artifacts without changing their meaning.
+
+A customer-specific API or missing tool implementation is not infrastructure. If its
+implementation is absent, admission fails with an unsupported/missing dependency result. The
+harness must not generate a substitute, proxy invented behavior, or grade against its own
+replacement. A code-execution service follows the same rule: ALK may provide an isolated runtime
+when the agent already declares that dependency, but the customer's tool decides what code to
+execute and how its outputs are used.
+
+## Admission, credentials and source trust
+
+Repository admission is a read-only static preflight. It does not import or execute submitted
+code. The scanner ignores dependencies, tests, generated output, symlinks and oversized files;
+recognizes Python, JavaScript, env-template and Compose declarations; and emits only names,
+purposes and statuses. It models provider alternatives explicitly, so one Gemini API key or one
+complete Vertex credential route satisfies model authentication without asking for every option.
+
+Public GitHub source is cloned anonymously. Private source carries a GitHub App installation
+reference; the sandbox's credential broker resolves the short-lived token only for clone and
+injects it through Git configuration environment variables, never process arguments or the job.
+The resolved commit is verified when a commit SHA is supplied. Source fingerprinting hashes
+symlink metadata without following links outside the repository, and the supervisor rejects a
+source tree that changes during execution.
+
+Non-secret connector configuration and secret references have separate contracts. Inline secret-
+like configuration keys are rejected by the platform. The worker builds a fresh allowlisted
+environment, resolves only the job's `SecretRef` entries, and does not inherit the supervisor's
+model, cloud, repository or customer credentials.
+
+## Retry and ingestion invariants
+
+The supervisor retries only structured failures marked retryable in the infrastructure,
+connectivity or platform-sync domains. Each failed worker attempt is archived separately before
+a clean attempt starts. Agent behavior and grading outcomes are never retried to manufacture a
+pass. GitHub clone uses the same bounded exponential-backoff policy.
+
+Before terminal success, the artifact directory is sealed with a content-addressed manifest. The
+seal requires a terminal result and non-empty transcript per scenario, validates referenced
+recordings, rejects secret material and unsafe links, enforces the byte budget, and records every
+retained file's SHA-256, media type and size. Platform result payloads and recording uploads carry
+digests. Ingestion locks the call row, accepts identical retries idempotently, rejects conflicting
+evidence, and stores combined, stereo, customer and assistant recordings as distinct artifacts.

fi/alk/harness/DESIGN.md

@@ -0,0 +1,246 @@
+# The harness: what it builds and how it proves it
+
+The reference for the rebuild. Written after the corrections in `_scenario-generation/context/`
+7.2, 8.1, 9.1, 10.1 and 12, and it supersedes anything in the code that contradicts it.
+
+---
+
+## The model, in one paragraph
+
+The **environment step** builds everything that is common to every test of one agent: the world
+its tools act on, the prompt that drives a simulated user if it has one, and the catalogue of
+sub-goals it can be checked against. Every **scenario** is then only a change on that base: what
+it alters after reset, the instruction substituted into the simulator's prompt, and which
+sub-goals must hold. Nothing about a scenario is a template with slots; the harness writes each
+one, and proves it works before keeping it.
+
+```
+environment step  ─────────────────────────────►  base: world + simulator prompt + sub-goals
+                                                          │
+scenario 1 ──► reset → setup → run → check ───────────────┤
+scenario 2 ──► reset → setup → run → check ───────────────┤
+scenario N ──► reset → setup → run → check ───────────────┘
+```
+
+---
+
+## 1. The environment step
+
+> *"You have to first understand what the f\*\*\* this agent is, from that you will create
+> databases, you'll create a snapshot of the databases first."* — Nikhil, 12
+
+It produces four things. All of them are written by the harness. None are hardcoded here.
+
+### 1.1 The world
+
+Whatever **this** agent needs, and nothing more. For the drive-thru agent that is a database.
+For a browser agent it is a site. For something else it is a filesystem, a queue, a service — the
+harness decides from the contract what has to exist.
+
+It **subclasses ALK's `EnvironmentAdapter`**, so the runners that already exist can drive it:
+`reset` publishes the tools and the starting state, `handle_tool_call` executes one call, and the
+state afterwards is what gets graded. Nothing the harness writes should re-implement a runner.
+
+It is **frozen once** as a snapshot. Every scenario restores from that snapshot, so a run is
+repeatable and no scenario can inherit another's leftovers.
+
+### 1.2 The simulator prompt — only where the agent is conversational
+
+> *"For voice and chat there is a simulator, and the input is an instruction to that simulator
+> rather than an input to the agent under test. Where there is no actor, variability comes from
+> how the environment is designed."* — 10.1 §4
+
+The harness writes one prompt for the simulated user of **this** agent, with variables left open.
+Each scenario supplies the values. The prompt is an artifact of the environment step because it
+is the same for every scenario; only the substituted instruction differs.
+
+There is no persona field and no persona library.
+
+> *"Drop the gimmicky persona characters. Variability comes from real conditions instead: a new
+> versus an existing user, whether a payment method is on file, addresses."* — 8.1
+
+For a browser or coding agent there is no simulator at all; the instruction goes to the agent
+directly.
+
+### 1.3 The sub-goal catalogue
+
+> *"Defining the sub-goals is our call. The important property is that they are common across
+> scenarios so the results roll up: if a payment step appears in 50 scenarios, the analytics
+> should show where payment fails and how often."* — 10.1 §8
+
+Defined **once**, here, as a named list. Scenarios reference them; they do not invent their own
+wording. That is what makes `order-confirmation fails in 7 of 12 scenarios` a sentence anyone can
+say. Each entry carries its own check (see §3).
+
+### 1.4 The gate
+
+The environment is not accepted because it looks right. It is exercised: every tool called with a
+valid call, a nonexistent id, and a missing argument; sequences where state has to carry across
+calls. **A refusal is the environment working; a crash is a defect.** It cannot be saved dirty
+(rows left over from building) or unverified.
+
+---
+
+## 2. A scenario is a change on that base, and it owns a folder
+
+```
+name         identifier, and the name of its folder
+use_case     which branch of the agent's real use cases this belongs to
+setup.py     def setup(world), what changes after reset. Code, because what a
+             scenario changes is not necessarily the database alone
+ready.py     def ready(world), whether the world holds what this scenario presumes
+instruction  the task. For a conversational agent this is substituted into the
+             simulator prompt; for a browser or coding agent it goes to the agent
+solution     the reference trajectory: what a correct agent would do
+checks/*.py  one file per deterministic sub-goal, each runnable on its own
+```
+
+The file is the artifact. Code lives in files rather than as strings inside JSON, and every check
+carries a `__main__` block so a person can run it by hand against what a run left behind and get
+the same answer the harness got.
+
+Gone from the old shape: `persona`, `opening`, `goal`, and free-text `must` / `must_not` as the
+primary grading. Scenarios are organised **use case → branch**, not by adversarial flavour.
+
+> *"A login flow is not one row with happy/edge inside it; it is many rows: login-with-Google,
+> login-with-Microsoft, forgot-password, sign-up-with-email."* — Nikhil, 7.2
+
+---
+
+## 3. Checks: deterministic by default, judge as the fallback
+
+> *"When you have `==` or a python script, then I'll call that deterministic."*
+> *"Most likely we can make things deterministic."*
+> *"Deterministic, if possible. And LLM also, obviously."* — Nikhil, 10
+
+| | |
+|---|---|
+| **Deterministic** — an assert, an equality, **a python script** | The default |
+| **Non-deterministic** — an LLM judging whether a sub-goal was met | Only where nothing observable settles it |
+
+The trap, in his words: *"you are judging by LLM [so it is non-deterministic], but if you want an
+exact output to be 50, then that is deterministic."* An exact fact checked by a judge is **still
+non-deterministic**. What matters is who decides, not how precise the fact is.
+
+A check is code the harness writes, and it has two observable things to work from:
+
+1. **the world afterwards** — rows, files, whatever this environment is
+2. **the recorded tool calls** — that the call happened, *and with the right arguments*
+
+That second one answers the question left open in 7.2: a booking made for 10 PM when 11 PM was
+asked for is a failure, and it is deterministic to detect.
+
+The judge is left only with what leaves no trace: whether a refusal was explained, whether a price
+was invented, tone.
+
+> Warning from the previous run: *"Judge checkpoints, about a third of all checkpoints, are
+> returned as skipped and not graded."* Leaning on the judge does not merely weaken a result — it
+> silently produces holes.
+
+---
+
+## 4. Three gates on every scenario, before it is kept
+
+Terminal-bench's oracle run, which is the reason its tasks are known to be solvable.
+
+### Gate 1. Ready
+
+```
+reset → apply setup → run ready                           ⇒ must HOLD
+```
+
+The world must hold what the scenario presumes. A scenario about the last five items is only a
+test of the agent if there really are five; otherwise the agent fails for a precondition we got
+wrong, and the report reads as a finding about the agent. A missing precondition is ours, and this
+is where it is caught.
+
+### Gate 2. Solvable
+
+```
+reset → apply setup → run the solution → run the checks   ⇒ must PASS
+```
+
+If the checks fail with the reference solution, either the scenario is impossible or the check is
+wrong. Both have already happened here: a scenario asserted a value the agent was never permitted
+to send, and another demanded confirmation of an item that could not be ordered. This catches
+them at write time, with no model involved.
+
+### Gate 3. Not vacuous
+
+```
+reset → apply setup → run NOTHING → run the checks        ⇒ must FAIL
+```
+
+A check that passes without the agent doing anything grades nothing while reporting a result.
+This is the failure that makes a suite quietly green.
+
+Vacuity is judged on *all* checks passing, because one check surviving an empty run ("no
+unavailable item was ordered") is legitimate. A single check that survives is still named, because
+sub-goals are shared: a check that cannot fail without calls would roll up as a pass for an agent
+that did nothing at all.
+
+Neither gate asks a model anything. The environment decides.
+
+**Three things fall out of the solution for free:** it is the expected trajectory; comparing the
+agent's trajectory against it gives efficiency (Nikhil's point about the agent that succeeds on
+the 21st call after 20 failures); and a scenario that cannot be run is caught before a call is
+ever placed.
+
+---
+
+## 5. Running it
+
+> *"Use an existing harness. Just for Claude agents. Use any existing harness that is there."*
+> — Nikhil, 12
+
+The simulation runs through **ALK's own path**, not a loop written here. The world is passed in
+as the environment; the agent under test is the real agent, in its real runtime. For the voice
+case that means the Vapi assistant we already have, over LiveKit, with the tool webhook answered
+by **our world** rather than by canned mocks.
+
+That last part is the whole point of the environment. The previous run's known issues were:
+
+- *"Mocked tools always succeed, including removing an item that was never added."*
+- *"Mock responses do not vary by argument, so read-after-write flows are wrong."*
+- *"World state does not change unless a scenario sets `state_updates`, which is often empty."*
+
+A world that really holds rows and can really refuse removes all three.
+
+---
+
+## 6. What changes per kind of agent, and what does not
+
+| | Voice / chat | Browser | Coding |
+|---|---|---|---|
+| World | database, KB | a site | a filesystem, a repo |
+| Simulated user | yes — prompt written by the harness | none | none |
+| Instruction goes to | the simulator | the agent | the agent |
+| Solution | tool calls | actions | commands |
+| Check | code over world + calls | code over the page + actions | code over the tree |
+
+**What never changes:** the environment is built once and frozen; a scenario is a change on it; a
+solution proves it is solvable; a scenario whose world is not ready is rejected before it can be
+blamed on the agent; a check that cannot fail is rejected; deterministic first.
+
+---
+
+## 7. Order of work
+
+1. **Environment step** — the world, the simulator prompt, the sub-goal catalogue, all written by
+   the harness rather than by a fixed schema here.
+2. **Scenario shape**: a folder per scenario: setup / ready / instruction / solution / sub-goal
+   references / a file per check.
+3. **The three gates**: ready, solvable, and not vacuous.
+4. **Run through ALK** — the world serving the tool calls of the real agent.
+
+---
+
+## 8. Instructions, not code
+
+> *"This is a flow, this is not a harness. You will give your harness the instructions that you
+> are supposed to do all this and then the harness will do all that. It's not a code that your
+> harness follows."* — Nikhil, 12
+
+Every stage's method lives in a `SKILL.md`, editable without touching code. What stays in code is
+only what must be exact: executing a call, restoring a snapshot, running a check, and refusing
+something that does not hold up. **The harness decides what to do. Code decides what is true.**