agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/harness/run/__init__.py

@@ -0,0 +1,296 @@
+"""Stage four: run the scenarios against the world and say what happened.
+
+Every scenario gets its own world. It is restored from the frozen snapshot, the scenario's own
+setup is run against it, and it is thrown away afterwards. Nothing a scenario does can reach the
+next one, which is what makes a result mean something on its own and makes the whole suite
+repeatable a week later.
+
+The shape is the same regardless of what is being tested: restore, converse, grade against the
+state that is left behind. Where the agent actually runs is a target, so the same scenarios grade
+a hosted agent without any of this changing.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Callable, Sequence
+
+from ..contract import AgentContract
+from ..catalogue import load_catalogue
+from ..simulator import load_simulator_prompt
+from ..scenario import Scenario
+from ..folder import apply_setup, check_ready
+from ..world.snapshot import restore
+from .conversation import FINISHED, Exchange, Transcript, converse
+from .grade import (
+    Checkpoint,
+    Result,
+    as_json,
+    checkpoints,
+    grade_sub_goals,
+    judge,
+    judge_suite_evals,
+    summarise,
+    ungraded_sub_goals,
+)
+from .targets import (
+    LocalAgent,
+    RepositoryChatTarget,
+    Target,
+    register_target,
+    resolve,
+    supported,
+)
+
+
+def _cases(report: Any) -> list[Any]:
+    """The per-scenario results, whichever report this is.
+
+    The runner hands back a ``SimulationReport``, whose cases are ``test_cases`` and whose
+    messages live a level down; the plugins hand back the older ``TestReport``, whose cases are
+    ``results``. Reading only one of them finds nothing in the other and reports a run in which
+    nobody said anything, which is indistinguishable from an agent that ignored the person.
+    """
+    legacy = getattr(report, "to_legacy", None)
+    if callable(legacy):
+        try:
+            report = legacy()
+        except Exception:  # noqa: BLE001 - a report that will not convert is still readable
+            pass
+    return list(
+        getattr(report, "results", None) or getattr(report, "test_cases", None) or []
+    )
+
+
+def from_alk(report: Any, world, spent: float) -> Transcript:
+    """What ALK's report says happened, in the shape the grading already reads.
+
+    Read from ``messages``, the normalised trajectory, and not from ``transcript`` — which is a
+    string, so iterating it yields characters, matches nothing, and produces a run with no turns
+    at all. The judge is then handed an empty conversation and fails every claim about what was
+    said, which arrives looking like an agent that never spoke.
+    """
+    exchanges: list[Exchange] = []
+    for case in _cases(report):
+        for message in getattr(case, "messages", None) or []:
+            if not isinstance(message, dict):
+                continue
+            role = str(message.get("role") or "")
+            text = message.get("content") or ""
+            # Tool turns are in here too, and they are already recorded as calls. Putting them
+            # in the conversation as well would have the judge read a tool result as something
+            # the agent said.
+            if role in ("tool", "function") or not str(text).strip():
+                continue
+            exchanges.append(
+                Exchange("agent" if role == "assistant" else "customer", str(text))
+            )
+        if not exchanges and isinstance(getattr(case, "transcript", None), str):
+            # A plugin that only fills the text form still has to be readable.
+            spoken = case.transcript.strip()
+            if spoken:
+                exchanges.append(Exchange("customer", spoken))
+    return Transcript(
+        exchanges=exchanges,
+        calls=list(world.calls),
+        ended=FINISHED,
+        spent_usd=spent,
+    )
+
+
+def audio_from(report: Any) -> str:
+    """Where ALK left this run's audio, if it left any.
+
+    Artifacts are how a modality hands back what it produced, so the recording is asked of the
+    report rather than guessed at from a directory. A run with none says so.
+    """
+    for case in _cases(report):
+        for artifact in getattr(case, "artifacts", None) or []:
+            kind = str(getattr(artifact, "type", "") or "")
+            mime = str(getattr(artifact, "mime_type", "") or "")
+            if "audio" in kind.lower() or mime.startswith("audio/"):
+                found = getattr(artifact, "path", None) or getattr(
+                    artifact, "uri", None
+                )
+                if found:
+                    return str(found)
+    return ""
+
+
+RUNS = "runs.json"
+REPORT = "report.txt"
+
+__all__ = [
+    "Checkpoint",
+    "Exchange",
+    "LocalAgent",
+    "RepositoryChatTarget",
+    "Result",
+    "Target",
+    "Transcript",
+    "converse",
+    "register_target",
+    "run_scenario",
+    "run_suite",
+    "supported",
+    "summarise",
+]
+
+
+async def run_scenario(
+    scenario: Scenario,
+    contract: AgentContract,
+    world_root: Path,
+    *,
+    target: str = "local",
+    model: str | None = None,
+    through_alk: bool = False,
+    on_exchange: Callable[[Exchange], Any] | None = None,
+) -> Result:
+    """Run one scenario in its own copy of the world and grade what it left behind."""
+    catalogue = load_catalogue(world_root)
+    world = restore(world_root)
+    try:
+        # reset() is how an environment is started in ALK: it clears the call log and
+        # publishes the tools and the starting state. Going through it keeps a generated world
+        # drivable by anything that already drives an environment.
+        world.reset()
+        applied = apply_setup(scenario, world)
+        if not applied.ok:
+            raise RuntimeError(f"the scenario's setup did not run: {applied.said}")
+        ready = check_ready(scenario, world)
+        if not ready.ok:
+            raise RuntimeError(
+                f"the world is not ready for this scenario: {ready.said}. Running it would "
+                "test us rather than the agent."
+            )
+        # The setup's calls are not the agent's.
+        world.calls = []
+        if through_alk:
+            # ALK owns the simulation and drives the world through EnvironmentAdapter; the
+            # harness only grades what it is left with. Nothing here is modality-specific,
+            # which is the point: the browser and voice runners take the same adapter.
+            from .alk import simulate
+
+            report, spent = await simulate(
+                scenario,
+                contract,
+                world,
+                model=model,
+                simulator_prompt=load_simulator_prompt(world_root),
+            )
+            transcript = from_alk(report, world, spent)
+            for exchange in transcript.exchanges:
+                if on_exchange:
+                    on_exchange(exchange)
+        else:
+            agent = resolve(target)(contract, world, model=model)
+            transcript = await converse(
+                agent,
+                scenario,
+                contract,
+                world_root=world_root,
+                model=model,
+                on_exchange=on_exchange,
+            )
+        # Settled by code first. The judge is only handed the sub-goals whose catalogue entry
+        # says nothing observable decides them.
+        settled = grade_sub_goals(world, scenario, catalogue, transcript.calls)
+        ending = ", ".join(
+            f"{name}: {len(rows)} rows"
+            for name, rows in sorted(world.observe().state.items())
+        )
+        judgements, judged_cost = await judge(
+            scenario, transcript, contract, catalogue, model=model, ending=ending
+        )
+        judgements += judge_suite_evals(
+            catalogue.suite_evals, scenario, transcript, contract, ending=ending
+        )
+        return Result(
+            scenario=scenario.name,
+            tests=scenario.tests,
+            problems=[
+                f"{name} is not in this catalogue, so nothing graded it"
+                for name in ungraded_sub_goals(scenario, catalogue)
+            ],
+            state_failures=[
+                f"{one.name}: {one.said}" for one in settled if not one.held
+            ],
+            conduct=judgements,
+            checkpoints=checkpoints(settled, judgements),
+            crashes=[f"{call.name}: {call.error}" for call in transcript.crashed()],
+            ended=transcript.ended,
+            turns=len(transcript.exchanges),
+            calls=len(transcript.calls),
+            spent_usd=transcript.spent_usd + judged_cost,
+            transcript=transcript.spoken(),
+            exchanges=[
+                {"speaker": turn.speaker, "text": turn.text}
+                for turn in transcript.exchanges
+            ],
+            actions=transcript.actions(),
+        )
+    finally:
+        world.close()
+
+
+async def run_suite(
+    scenarios: Sequence[Scenario],
+    contract: AgentContract,
+    world_root: Path,
+    *,
+    target: str = "local",
+    model: str | None = None,
+    through_alk: bool = False,
+    out: Path | None = None,
+    on_result: Callable[[Result], Any] | None = None,
+    on_exchange: Callable[[Exchange], Any] | None = None,
+) -> list[Result]:
+    """Run every scenario and write the results out. One failing scenario never stops the rest."""
+    destination = Path(out or world_root)
+    results: list[Result] = []
+    for scenario in scenarios:
+        try:
+            result = await run_scenario(
+                scenario,
+                contract,
+                world_root,
+                target=target,
+                model=model,
+                through_alk=through_alk,
+                on_exchange=on_exchange,
+            )
+        except Exception as failed:
+            # A scenario that could not be run is recorded as unrunnable rather than as a
+            # failure of the agent, and the rest of the suite still runs.
+            result = Result(
+                scenario=scenario.name,
+                tests=scenario.tests,
+                crashes=[f"could not run: {type(failed).__name__}: {failed}"],
+                ended="not-run",
+            )
+        results.append(result)
+        if on_result:
+            on_result(result)
+
+    destination.mkdir(parents=True, exist_ok=True)
+    # Records for scenarios this suite did not run are kept, not clobbered. A live call and a
+    # local run write to the same file, and re-running two scenarios must not erase the third.
+    ran = {result.scenario for result in results}
+    kept = [
+        record
+        for record in load_results(destination)
+        if isinstance(record, dict) and record.get("scenario") not in ran
+    ]
+    merged = kept + json.loads(as_json(results))
+    (destination / RUNS).write_text(
+        json.dumps(merged, indent=2, ensure_ascii=False), encoding="utf-8"
+    )
+    (destination / REPORT).write_text(summarise(results), encoding="utf-8")
+    return results
+
+
+def load_results(destination: Path) -> list[dict[str, Any]]:
+    path = Path(destination) / RUNS
+    return json.loads(path.read_text(encoding="utf-8")) if path.exists() else []

fi/alk/harness/run/alk.py

@@ -0,0 +1,184 @@
+"""Running a generated world through ALK's own simulation, rather than beside it.
+
+The whole reason a generated world subclasses ``EnvironmentAdapter`` is so the runners that
+already exist can drive it. ``ChatEnvironment`` takes ``environment=<adapter>`` and owns the
+synthetic user, the turn loop, the transcript and the report; the browser and voice paths take
+the same adapter. A second loop written here would work for exactly one modality and would have
+to be rewritten for the next one, which is the thing this design exists to avoid.
+
+So the split is:
+
+- **ALK** drives the simulation: who the customer is, when they speak, when it ends.
+- **The world** answers every tool call, through ``handle_tool_call``.
+- **The harness** grades afterwards, from the state the world is left in plus the transcript.
+
+What is written here is only the two adapters between the shapes: a scenario becomes an ALK
+``Persona``, and the agent under test becomes an ``AgentWrapper``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fi.simulate import Persona, Scenario as AlkScenario
+from fi.simulate.agent.wrapper import AgentInput, AgentResponse, AgentWrapper
+from fi.simulate.environments.chat import ChatEnvironment
+
+from ..contract import AgentContract
+from ..scenario import Scenario
+from ..world.runtime import GeneratedWorld
+from .targets import LocalAgent
+
+
+def as_persona(scenario: Scenario, simulator_prompt: str = "") -> Persona:
+    """One of our scenarios, in the shape ALK's simulation consumes.
+
+    ``situation`` is the simulator prompt the harness wrote for this agent with the scenario's
+    values filled in. ALK wraps it in its own voice-execution rules, so what goes here is only
+    what changes per scenario, not a second set of instructions about how to behave on a call.
+
+    The structured persona is preserved so LiveKit can vary the caller's identity, speech style,
+    and scenario metadata instead of flattening every test into the same generic customer.
+    """
+    from ..simulator import fill
+
+    # The LiveKit voice simulator treats ``situation`` as private context rather than a
+    # line to recite. Preserve the harness-authored caller rules here: they explicitly keep
+    # the simulator in the customer role and stop it from volunteering held-back details.
+    # Fall back to the plain scenario instruction for older sessions without a prompt.
+    filled = fill(simulator_prompt, scenario.slots())[0] if simulator_prompt else ""
+    persona = (
+        scenario.persona.model_dump(exclude_none=True)
+        if scenario.persona is not None
+        else {"name": "customer"}
+    )
+    return Persona(
+        persona=persona,
+        situation=filled or scenario.instruction,
+        # Said in the person's own terms, because it is read aloud with the situation.
+        # ``scenario.tests`` describes what the suite is checking — "agent correctly counts
+        # customers filtered by country" — and a person who opens by announcing what the agent
+        # is being graded on has told it the answer.
+        outcome="get what you came for, or accept that you cannot",
+    )
+
+
+def as_alk_scenario(
+    scenarios: list[Scenario], name: str = "harness", simulator_prompt: str = ""
+) -> AlkScenario:
+    return AlkScenario(
+        name=name,
+        description="generated by the harness",
+        dataset=[as_persona(one, simulator_prompt) for one in scenarios],
+    )
+
+
+def _spoken(input: AgentInput) -> str:
+    """What the customer just said, as text.
+
+    ALK passes a message as a mapping, not a string, and hands the whole history alongside it.
+    Passing the mapping straight to a session that expects text fails inside the SDK with a
+    redacted TypeError, which says nothing about where it came from.
+    """
+    latest = input.new_message
+    if isinstance(latest, dict):
+        content = latest.get("content")
+        if isinstance(content, list):
+            content = " ".join(
+                part.get("text", "") for part in content if isinstance(part, dict)
+            )
+        if content:
+            return str(content)
+    if isinstance(latest, str) and latest:
+        return latest
+    for message in reversed(input.messages or []):
+        if isinstance(message, dict) and message.get("role") != "assistant":
+            content = message.get("content")
+            if content:
+                return str(content)
+    return "(the customer said nothing)"
+
+
+class ContractAgent(AgentWrapper):
+    """The agent under test, in the shape ALK drives agents by.
+
+    It holds the same session ``LocalAgent`` uses, so the agent being graded is identical either
+    way; what changes is who runs the conversation around it. The tool calls it made are reported
+    back to ALK so they appear in the transcript, having already gone through the world.
+    """
+
+    def __init__(
+        self,
+        contract: AgentContract,
+        world: GeneratedWorld,
+        *,
+        model: str | None = None,
+    ) -> None:
+        self.agent = LocalAgent(contract, world, model=model)
+        self.world = world
+        self._open = False
+
+    async def call(self, input: AgentInput) -> AgentResponse:
+        if not self._open:
+            await self.agent.open()
+            self._open = True
+
+        before = len(self.world.calls)
+        said = await self.agent.say(_spoken(input))
+        made = self.world.calls[before:]
+
+        return AgentResponse(
+            content=said,
+            tool_calls=[
+                {"name": call.name, "arguments": call.arguments} for call in made
+            ],
+            tool_responses=[
+                {
+                    "name": call.name,
+                    "content": call.error if not call.ok else str(call.result),
+                    "success": call.ok,
+                }
+                for call in made
+            ],
+        )
+
+    async def aclose(self) -> None:
+        if self._open:
+            await self.agent.close()
+            self._open = False
+
+    @property
+    def spent_usd(self) -> float:
+        return self.agent.spent_usd
+
+
+async def simulate(
+    scenario: Scenario,
+    contract: AgentContract,
+    world: GeneratedWorld,
+    *,
+    model: str | None = None,
+    simulator_prompt: str = "",
+) -> tuple[Any, float]:
+    """Run one scenario through ALK's chat simulation against this world.
+
+    ``auto_execute_tools`` is off because the agent's tools are bound to the world already and
+    have run by the time it answers. Turning it on would execute every call a second time, which
+    for a world that really writes rows means every order placed twice.
+    """
+    agent = ContractAgent(contract, world, model=model)
+    try:
+        report = await ChatEnvironment().run(
+            scenario=as_alk_scenario(
+                [scenario], name=scenario.name, simulator_prompt=simulator_prompt
+            ),
+            agent_callback=agent,
+            environment=world,
+            auto_execute_tools=False,
+            max_turns=max(2, scenario.max_turns),
+            min_turns=2,
+            modality=contract.modality or "text",
+        )
+    finally:
+        await agent.aclose()
+    return report, agent.spent_usd

fi/alk/harness/run/call.py

@@ -0,0 +1,162 @@
+"""One scenario, against the real hosted agent, end to end.
+
+Everything the harness built is wired together here and then ALK's own voice case places the
+call. The harness does not reimplement any of that: it supplies the world the agent's tools act
+on, the caller's instruction, and the grading afterwards.
+
+    world + setup ──► webhook ──► public url ──► assistant's own tools repointed
+                                                          │
+                        ALK's voice case places the call ──┘
+                                                          │
+                          the world afterwards + the calls ──► sub-goal checks
+
+Run it:
+
+    set -a; . ./.env.acceptance; set +a
+    uv run python -m harness.run.call --name drive_thru --scenario orders_a_big_mac
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+from ..config import artifact_dir
+from ..scenario_tools import load_scenarios
+from .live import grade, wire
+
+CASE = os.environ.get("HARNESS_VOICE_CASE", "2.1.2")
+
+
+# The default runner is part of the harness package and uses ALK's public
+# SimulationSpec/SimulationRunner API. It is overridable only for provider
+# acceptance work; normal customer runs need no second checkout or script.
+VOICE_RUNNER = Path(__file__).with_name("sdk_voice.py")
+
+
+LIVE_EVENT = "HARNESS_EXCHANGE "
+
+
+def place_the_call(case: str, dry_run: bool = False, on_exchange=None) -> int:
+    """Hand over to ALK's voice case, which owns everything about placing a call."""
+    named = os.environ.get("HARNESS_VOICE_RUNNER", "").strip()
+    runner = Path(named) if named else VOICE_RUNNER
+    if not runner.exists():
+        raise RuntimeError(
+            f"no voice runner at {runner}. It ships in the harness package; set "
+            "HARNESS_VOICE_RUNNER if it lives somewhere else."
+        )
+    command = [sys.executable, str(runner), case] + (["--dry-run"] if dry_run else [])
+    if on_exchange is None:
+        return subprocess.call(command)
+    process = subprocess.Popen(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1,
+    )
+    assert process.stdout is not None
+    for line in process.stdout:
+        if line.startswith(LIVE_EVENT):
+            try:
+                on_exchange(json.loads(line[len(LIVE_EVENT) :]))
+            except (json.JSONDecodeError, TypeError):
+                pass
+        else:
+            print(line, end="", flush=True)
+    return process.wait()
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="agent-harness-call", description=__doc__)
+    parser.add_argument("--name", required=True, help="which agent")
+    parser.add_argument("--scenario", required=True, help="which scenario, by name")
+    parser.add_argument("--case", default=CASE, help="ALK voice case id")
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="wire everything up but do not place the call",
+    )
+    args = parser.parse_args(argv)
+
+    root = artifact_dir(args.name)
+    written = load_scenarios(root)
+    scenario = next((one for one in written if one.name == args.scenario), None)
+    if scenario is None:
+        print(
+            f"no scenario called {args.scenario!r}. There is: "
+            + ", ".join(one.name for one in written),
+            file=sys.stderr,
+        )
+        return 1
+
+    world, instruction, webhook, tunnel, url, moved = wire(scenario, root)
+    try:
+        print(f"agent:     {args.name}")
+        print(f"scenario:  {scenario.name}")
+        print(f"webhook:   {url}/tool")
+        print(f"repointed: {', '.join(moved)}")
+        print(f"sub-goals: {', '.join(scenario.sub_goals)}\n")
+
+        # The caller's instruction reaches the voice case through the environment, so nothing
+        # about how a simulated caller behaves is decided twice.
+        os.environ["HARNESS_INSTRUCTION"] = instruction
+        os.environ["HARNESS_SCENARIO"] = scenario.name
+        # The caller prompt is a template the harness fills, never prose it composes, so
+        # the generated template travels to the call with everything else.
+        # The caller is never handed the grader's pass question. `tests` is written about
+        # the agent in the third person, so as an objective it reads as a rubric rather
+        # than a motive. What this person wants is already in the instruction.
+        os.environ.pop("HARNESS_OUTCOME", None)
+        os.environ["HARNESS_PERSONA"] = json.dumps(
+            scenario.persona.model_dump(exclude_none=True)
+            if scenario.persona is not None
+            else {"name": "customer"}
+        )
+        os.environ["HARNESS_INITIAL_MESSAGE"] = (
+            scenario.persona.initial_message if scenario.persona is not None else ""
+        )
+        os.environ["HARNESS_FIXTURE"] = json.dumps(
+            scenario.fixture, ensure_ascii=False, default=str
+        )
+        os.environ["HARNESS_SCRIPTED_CALLER"] = json.dumps(
+            scenario.persona.scripted_caller
+            if scenario.persona is not None
+            and scenario.persona.scripted_caller is not None
+            else {}
+        )
+
+        code = place_the_call(args.case, dry_run=args.dry_run)
+        if args.dry_run:
+            print("\ndry run: nothing was called, and the world is untouched.")
+            return code
+
+        result = grade(scenario, world, root)
+        print()
+        print(result.line())
+        for one in result.settled:
+            print(one.line())
+        for name in result.judged:
+            print(f"  [?] {name} — judged, not graded here")
+        print("\nwhat the agent actually did:")
+        for call in result.calls or ["(no tool calls reached the world)"]:
+            print(f"  {call}")
+        return 0 if result.settled and result.met == len(result.settled) else 2
+    finally:
+        webhook.stop()
+        if tunnel is not None:
+            tunnel.terminate()
+        if (root / "environment.json").exists():
+            from ..provision import stop_runtime
+
+            stop_runtime(root)
+        world.close()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())