agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/harness/run/live.py

@@ -0,0 +1,297 @@
+"""One scenario, against the real hosted agent, in the environment the harness built.
+
+The harness wires the whole thing rather than leaving it to be assembled by hand:
+
+1. restore the world and apply the scenario's setup
+2. stand the webhook up and bind that world to it
+3. expose it publicly, because a hosted agent has to reach it
+4. point the assistant's **own** tools at that address — nothing about the agent is redefined
+5. run ALK's voice case with the scenario's instruction driving the simulated caller
+6. grade from the world afterwards and the calls the webhook recorded
+
+Steps 1, 2, 4 and 6 are the whole difference from what existed before: the agent's tool calls now
+land in a database that can refuse, instead of in canned responses that always succeed.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import time
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from ..simulator_voice import fixture_caller_phone
+from ..catalogue import load_catalogue
+from ..checks import Outcome, run_check
+from ..folder import apply_setup, check_ready
+from ..scenario import Scenario
+from ..simulator import fill, load_simulator_prompt
+from ..world.runtime import GeneratedWorld
+from ..world.snapshot import restore
+from .voice import WorldWebhook, repoint_assistant
+
+
+@dataclass
+class LiveRun:
+    """What a live call left behind."""
+
+    scenario: str
+    settled: list[Outcome] = field(default_factory=list)
+    judged: list[str] = field(default_factory=list)
+    calls: list[str] = field(default_factory=list)
+    ended: str = ""
+    problems: list[str] = field(default_factory=list)
+
+    @property
+    def met(self) -> int:
+        return sum(1 for one in self.settled if one.held)
+
+    def line(self) -> str:
+        mark = "PASS" if self.settled and self.met == len(self.settled) else "FAIL"
+        if self.problems:
+            mark = "VOID"
+        return f"{mark}  {self.scenario}  {self.met}/{len(self.settled)} sub-goals settled by code"
+
+
+def scoped_agent_name(base: str, scenario: str) -> str:
+    """A dispatch name owned by one worker lifetime, never a stale registration."""
+    clean_base = re.sub(r"[^a-zA-Z0-9_-]+", "-", base).strip("-") or "agent"
+    clean_case = re.sub(r"[^a-zA-Z0-9_-]+", "-", scenario).strip("-") or "case"
+    return f"{clean_base[:28]}-{clean_case[:18]}-{uuid.uuid4().hex[:8]}"
+
+
+def public_url(
+    port: int, *, wait: float = 30.0, tries: int = 3
+) -> tuple[str, subprocess.Popen | None]:
+    """A publicly reachable address for the webhook, and the process holding it open.
+
+    A hosted agent runs on somebody else's infrastructure, so a loopback address is unreachable
+    to it. ``cloudflared`` is what the previous runs used; anything giving a public URL works, and
+    ``HARNESS_WEBHOOK_URL`` skips this entirely when a tunnel is already running.
+
+    Retried, because a free tunnel is the least reliable thing in the whole path and it fails
+    before anything interesting has happened. One slow handshake should not read as a scenario
+    the agent failed, and on a suite of forty it would not fail once.
+    """
+    named = os.environ.get("HARNESS_WEBHOOK_URL", "").strip()
+    if named:
+        return named, None
+    if not shutil.which("cloudflared"):
+        raise RuntimeError(
+            "no way to expose the webhook publicly. Either install cloudflared "
+            "(brew install cloudflared) or set HARNESS_WEBHOOK_URL to a tunnel you already have."
+        )
+    for attempt in range(max(1, tries)):
+        found, process = _tunnel(port, wait)
+        if found:
+            return found, process
+        if process is not None:
+            process.terminate()
+        if attempt + 1 < tries:
+            time.sleep(2.0)
+    raise RuntimeError(
+        f"cloudflared did not report a public URL in {tries} attempts. The tunnel is the "
+        "flakiest part of this path; set HARNESS_WEBHOOK_URL to one you control to skip it."
+    )
+
+
+def _tunnel(port: int, wait: float) -> tuple[str, subprocess.Popen | None]:
+    """One attempt at a tunnel: the URL if it came up, and the process either way."""
+    process = subprocess.Popen(
+        ["cloudflared", "tunnel", "--url", f"http://127.0.0.1:{port}"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+    )
+    deadline = time.time() + wait
+    while time.time() < deadline:
+        line = process.stdout.readline() if process.stdout else ""
+        if not line and process.poll() is not None:
+            return "", process
+        if "trycloudflare.com" in line:
+            for word in line.split():
+                if word.startswith("https://") and "trycloudflare.com" in word:
+                    return word.strip(), process
+    return "", process
+
+
+def prepare(scenario: Scenario, world_root: Path) -> tuple[GeneratedWorld, str]:
+    """The world this scenario runs in, and what the simulated caller is told.
+
+    The instruction is the scenario's values filled into the simulator prompt the environment
+    step wrote. Nothing about how a caller behaves is decided here; that belongs to the prompt.
+    """
+    world = restore(world_root)
+    world.reset()
+    applied = apply_setup(scenario, world)
+    if not applied.ok:
+        raise RuntimeError(f"the scenario's setup did not run: {applied.said}")
+    ready = check_ready(scenario, world)
+    if not ready.ok:
+        raise RuntimeError(
+            f"the world is not ready for this scenario: {ready.said}. Running it would test us "
+            "rather than the agent."
+        )
+    # The setup's own calls are not the agent's.
+    world.calls = []
+
+    written = load_simulator_prompt(world_root)
+    if not written:
+        return world, scenario.instruction
+    filled, missing = fill(written, scenario.slots())
+    if missing:
+        raise RuntimeError(
+            f"the simulator prompt asks for {', '.join(missing)}, which {scenario.name} does "
+            "not supply. An unfilled slot reaches the caller verbatim."
+        )
+    return world, filled
+
+
+def grade(scenario: Scenario, world: GeneratedWorld, world_root: Path) -> LiveRun:
+    """The same sub-goal checks every other run uses, against what the call left behind."""
+    catalogue = load_catalogue(world_root)
+    run = LiveRun(scenario=scenario.name)
+    for name in scenario.sub_goals:
+        sub_goal = catalogue.named(name)
+        if sub_goal is None:
+            run.problems.append(f"{name} is not in the catalogue")
+        elif sub_goal.deterministic():
+            run.settled.append(run_check(sub_goal.check, world, world.calls, name=name))
+        else:
+            run.judged.append(name)
+    run.calls = [
+        f"{call.name}({call.arguments}) -> "
+        + ("refused: " + call.error if call.refused else "ok" if call.ok else "crashed")
+        for call in world.calls
+    ]
+    return run
+
+
+def instruction_for(scenario: Scenario, world_root: Path) -> str:
+    """What the simulated person is told, from the prompt the environment step wrote."""
+    written = load_simulator_prompt(world_root)
+    if not written:
+        return scenario.instruction
+    filled, missing = fill(written, scenario.slots())
+    if missing:
+        raise RuntimeError(
+            f"the simulator prompt asks for {', '.join(missing)}, which {scenario.name} does "
+            "not supply. An unfilled slot reaches the caller verbatim."
+        )
+    return filled
+
+
+def fixture_phone(scenario: Scenario) -> str:
+    """The caller identity the submitted voice runtime must see for this scenario."""
+    return fixture_caller_phone(scenario.fixture)
+
+
+def wire(
+    scenario: Scenario,
+    world_root: Path,
+    *,
+    assistant_id: str = "",
+    api_key: str = "",
+    world: GeneratedWorld | None = None,
+    trace_path: str | Path | None = None,
+):
+    """Everything up to placing the call: world, webhook, tunnel, assistant.
+
+    Returns the bound world, the caller's instruction, the webhook and the tunnel, so whoever
+    places the call decides how — ALK's voice case, a phone leg, or a web call.
+
+    ``world`` is taken when the caller has already prepared one. The suite runner sets a
+    scenario's world up once and grades what that same world is left holding, so preparing a
+    second one here would answer the agent's calls in a world nobody afterwards looks at.
+    """
+    # How the agent is reached decides what has to be arranged here. A hosted assistant lives
+    # somewhere we do not control, so its tools have to be repointed at a URL it can reach from
+    # outside. An agent we run ourselves already reads where its tools are from its own
+    # environment and shares a network with us, so there is nothing to repoint and nothing to
+    # expose -- and doing either would fail for want of credentials we have no reason to hold.
+    reachable = os.environ.get("HARNESS_WEBHOOK_URL", "").strip()
+    source_environment = (Path(world_root) / "environment.json").exists()
+    ours = bool(reachable) or source_environment
+
+    if not ours:
+        assistant_id = assistant_id or os.environ.get("VAPI_ASSISTANT_ID", "")
+        api_key = api_key or os.environ.get("VAPI_API_KEY", "")
+        if not assistant_id or not api_key:
+            raise RuntimeError(
+                "VAPI_ASSISTANT_ID and VAPI_API_KEY have to be set, or HARNESS_WEBHOOK_URL "
+                "given for an agent that already knows where to find its tools."
+            )
+
+    if world is None:
+        world, instruction = prepare(scenario, world_root)
+    else:
+        instruction = instruction_for(scenario, world_root)
+    webhook = WorldWebhook().start()
+    webhook.bind(world)
+    try:
+        if source_environment:
+            from ..provision import (
+                connect_runner_network,
+                infer_livekit_agent_name,
+                start_runtime,
+            )
+
+            # The submitted worker runs in Docker while the webhook runs in this harness
+            # process. The host-gateway name is injected by start_runtime and keeps the source
+            # network private; only this one URL is substituted.
+            private_host = connect_runner_network(world_root)
+            url = os.environ.get("HARNESS_RUNTIME_WEBHOOK_URL", "").strip() or (
+                f"http://{private_host}:{webhook.port}"
+                if private_host
+                else f"http://host.docker.internal:{webhook.port}"
+            )
+            # Reusing a registered LiveKit agent name across rapid container restarts lets a new
+            # room dispatch to the just-removed worker during server-side deregistration grace.
+            # Give every worker lifetime its own name and point this call at that exact worker.
+            base_agent_name = infer_livekit_agent_name(world_root) or os.environ.get(
+                "LIVEKIT_TARGET_AGENT_NAME", "harness-agent"
+            )
+            agent_name = scoped_agent_name(base_agent_name, scenario.name)
+            runtime_overrides = {
+                "TOOLS_API_URL": url,
+                "LIVEKIT_AGENT_NAME": agent_name,
+            }
+            # The submitted agent picks its own model, and a tier that cannot emit a valid
+            # function call fails every scenario the moment it reaches for a tool. Allow an
+            # operator to pin it for a run without editing the submitted repository.
+            agent_model = os.environ.get("ALK_SUBMITTED_AGENT_MODEL", "").strip()
+            if agent_model:
+                runtime_overrides["AGENT_LLM_MODEL"] = agent_model
+            os.environ["LIVEKIT_TARGET_AGENT_NAME"] = agent_name
+            caller_phone = fixture_phone(scenario)
+            if caller_phone:
+                runtime_overrides["DEMO_CALLER_ANI"] = caller_phone
+            start_runtime(
+                world_root,
+                overrides=runtime_overrides,
+                trace_path=trace_path,
+            )
+            # Runtime-only projects have no dependency container (and therefore no Compose
+            # network) until the worker starts. The first call above reserves the alias; this
+            # second idempotent call joins a hosted runner to the newly created network.
+            connect_runner_network(world_root)
+            tunnel, moved = None, []
+        elif ours:
+            # The agent was started pointing here, so this is where its tools already go.
+            url, tunnel, moved = reachable, None, []
+        else:
+            url, tunnel = public_url(webhook.port)
+            moved = repoint_assistant(assistant_id, api_key, url)
+    except Exception:
+        webhook.stop()
+        if source_environment:
+            from ..provision import stop_runtime
+
+            stop_runtime(world_root)
+        world.close()
+        raise
+    return world, instruction, webhook, tunnel, url, moved

fi/alk/harness/run/models.py

@@ -0,0 +1,56 @@
+"""Which model plays which part.
+
+Three different jobs, and one setting for all of them was wrong for every one. The agent under
+test and the person talking to it run on every turn of every scenario; the judge runs once per
+scenario and is where a wrong answer costs the most; the harness itself writes contracts, worlds
+and checks and is a different job again.
+
+The harness's own model is deliberately not here. It is set by ``ALK_HARNESS_MODEL`` and belongs
+to the conversation you have with the harness, not to the simulation it runs.
+
+**On Gemini.** The obvious thing to want is Flash for the agent and the simulated user: they are
+the two roles that run constantly, and Vertex is already configured. It does not work yet, and
+the reason is worth writing down rather than rediscovering. The reconstructed agent runs on the
+Claude Agent SDK, which is pointed at Vertex by ``CLAUDE_CODE_USE_VERTEX`` and speaks to
+Anthropic models only. Handed a Gemini name it produced a session that said nothing at all: no
+turns, no calls, every check red, and a result that read as an agent ignoring the person.
+
+Running the agent on Gemini means giving the spec one of ALK's own endpoint adapters as the
+target — ``system_prompt`` resolves an LLM target from a prompt, which is exactly what the
+reconstruction is — instead of the harness's own. That also moves tool execution to ALK, which
+is a real change and not a configuration one. Until then these stay on what can actually be
+driven, and the guard in ``targets.py`` refuses the rest loudly.
+"""
+
+from __future__ import annotations
+
+import os
+
+# What the reconstructed agent and the simulated user run on today. Both roles run constantly, so
+# this is the setting worth revisiting first once the target can be handed to ALK.
+AGENT = "claude-sonnet-4-6"
+USER = "claude-sonnet-4-6"
+# One model for every role. A judged sub-goal was kept on a stronger model, but a run that mixes
+# tiers is slower and harder to reason about, and the checks that decide a pass are code rather
+# than judgement wherever they can be. Override with ALK_JUDGE_MODEL when a run needs it.
+JUDGE = "claude-sonnet-4-6"
+
+
+def for_roles(override: str | None = None) -> dict[str, str]:
+    """The model each part runs on.
+
+    ``override`` names one model for every role, which is what a caller comparing two models end
+    to end is asking for: same suite, same world, one thing changed.
+    """
+    if override:
+        return {"agent": override, "user": override, "judge": override}
+    return {
+        "agent": os.environ.get("ALK_AGENT_MODEL", AGENT),
+        "user": os.environ.get("ALK_USER_MODEL", USER),
+        # The judge rides the harness backend, so left on its own default it names a model the
+        # configured backend may not be able to drive. Following the harness model keeps the
+        # pairing valid with one setting; ALK_JUDGE_MODEL still wins when a run needs it.
+        "judge": os.environ.get("ALK_JUDGE_MODEL")
+        or os.environ.get("ALK_HARNESS_MODEL")
+        or JUDGE,
+    }

fi/alk/harness/run/platform_evals.py

@@ -0,0 +1,227 @@
+"""Judged sub-goals as evals on the platform, created once and invoked per run.
+
+A sub-goal that nothing observable can settle is a sentence: "the agent explained why it could
+not change the price, and did not invent a reason". That sentence is already the whole input a
+custom eval wants, so rather than asking a model here and keeping the answer in a run folder, the
+sentence becomes a named eval on the platform, created once when the world is built and invoked
+after every run.
+
+What that buys, beyond tidiness: the eval is versioned and reusable, it shows up in the product
+rather than only in our artifacts, and the same judgement can be applied to production traffic
+later without being rewritten. The harness wrote it; it is theirs to keep.
+
+Deterministic checks stay as code. They are better as code, and nothing here should tempt anyone
+to send a question a database can answer to a language model.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import time
+from typing import Any
+
+# What the conversation is called inside an eval's instructions. Deliberately plain: the platform
+# extracts variables from the instructions themselves, and the reserved roots (row, span, trace,
+# session, call) would be swallowed.
+CONVERSATION = "conversation"
+
+# Both are needed. Without them the harness falls back to judging here, rather than failing a run
+# over a credential, because a suite that cannot run without a platform account is a worse tool.
+KEYS = ("FI_API_KEY", "FI_SECRET_KEY")
+
+# The judge. A typo here does not raise: an unknown model silently falls back to this same value,
+# so the only protection against sending the wrong one is sending the right one.
+MODEL = "turing_large"
+
+# Names the platform accepts, and which are stable for the same sub-goal on the same agent, so
+# that running a suite twice reuses one eval rather than making a second.
+_ALLOWED = re.compile(r"[^a-z0-9_-]+")
+
+
+def configured() -> bool:
+    return all(os.environ.get(name) for name in KEYS)
+
+
+def eval_name(agent: str, sub_goal: str) -> str:
+    """A stable name for one agent's sub-goal.
+
+    Includes the agent, because two agents can reasonably have a sub-goal called the same thing
+    and mean different questions by it. Uniqueness on the platform is per organisation, so a
+    bare `refused_clearly` would collide across every agent anybody tests.
+    """
+    return _ALLOWED.sub("-", f"{agent}-{sub_goal}".lower()).strip("-")[:64]
+
+
+def suite_eval_name(agent: str, eval_name: str) -> str:
+    """A stable, non-colliding platform name for a suite-wide evaluation."""
+    return _ALLOWED.sub("-", f"{agent}-suite-{eval_name}".lower()).strip("-")[:64]
+
+
+def judge_builtin(name: str, inputs: dict[str, str]) -> dict[str, Any]:
+    """Run a built-in eval by identifier, with its documented inputs."""
+    from fi.evals import Evaluator
+
+    answered = Evaluator().evaluate(
+        eval_templates=name, inputs=inputs, model_name="turing_flash"
+    )
+    first = (getattr(answered, "eval_results", None) or [None])[0]
+    output = getattr(first, "output", None)
+    reason = getattr(first, "reason", "") or ""
+    if first is None or (output is None and reason):
+        raise RuntimeError(f"{name} did not run: {reason or 'no result'}")
+    return {
+        "output": output,
+        "why": reason,
+        "model": getattr(first, "model", None) or "turing_flash",
+    }
+
+
+def instructions_for(claim: str, agent: str, rules: list[str] | None = None) -> str:
+    """The eval's own prompt: what to decide, and what to decide it from.
+
+    One variable, carrying the whole record of the run rather than only what was said. Speech
+    alone cannot settle most of these: "the answer is correct" is decidable against what the
+    database actually returned and not against the sentence quoting it, and an eval given only
+    the transcript correctly reports that it has no way to tell. What the agent did is part of
+    the conversation in the sense that matters here.
+
+    Everything else is fixed when the eval is created, because it is a fact about the agent
+    rather than about one run.
+    """
+    known = (
+        "\n\nThe agent under test is bound by these rules:\n  - "
+        + "\n  - ".join(rules[:10])
+        if rules
+        else ""
+    )
+    return (
+        f"You are judging one run of {agent}.\n\n"
+        f"Decide strictly: {claim}\n\n"
+        "You are given a JSON record of the run: what the person was asked to do, every tool "
+        "call the agent made with what came back, what was said, and the state of the world "
+        "afterwards.\n\n"
+        "The tool calls are the truth about what happened. An agent that says it did something "
+        "no call performed has not done it, however convincing it sounds, and an answer is "
+        "correct when it matches what the calls returned. A refused call did not happen: judge "
+        "what the agent ended up doing, not what it tried on the way. Something merely not "
+        "contradicted does not hold. Where the claim is that something must not have happened, "
+        "it holds when the thing did not happen. Declining something holds only if the agent "
+        "both declined it and gave a true reason; refusing while inventing a reason does not "
+        "hold."
+        f"{known}\n\n"
+        "The run:\n"
+        f"{{{{{CONVERSATION}}}}}"
+    )
+
+
+def ensure(name: str, claim: str, agent: str, rules: list[str] | None = None) -> bool:
+    """Create this eval if the platform does not already have it. True when it is there.
+
+    Creation is checked against a list that is scoped to the workspace while uniqueness is
+    scoped to the organisation, so an eval made in a sibling workspace is invisible here and
+    creating it raises. That is not an error worth failing a run over: the eval exists, which is
+    all this needs to be true.
+    """
+    from fi.evals import EvalTemplateManager
+
+    manager = EvalTemplateManager()
+    wanted = instructions_for(claim, agent, rules)
+    found = manager.list_templates(search=name)
+    existing = next(
+        (one for one in getattr(found, "items", []) or [] if one.name == name), None
+    )
+    if existing is not None:
+        # Same eval, kept at the same name and id, rather than a second one beside it. Its
+        # instructions are the harness's, so when those change the eval on the platform is
+        # behind: an old one silently judging new runs is the failure mode worth avoiding, and
+        # a new name every time would litter the account with near-duplicates.
+        if (getattr(existing, "instructions", "") or "") != wanted:
+            manager.update_template(existing.id, instructions=wanted, model=MODEL)
+        return True
+    try:
+        manager.create_template(
+            name=name,
+            instructions=wanted,
+            eval_type="llm",
+            model=MODEL,
+            output_type="pass_fail",
+            pass_threshold=0.5,
+            # A draft cannot be run, and nothing later says why.
+            is_draft=False,
+            tags=["harness", "sub-goal"],
+        )
+    except Exception as refused:  # noqa: BLE001 - the one failure that means success
+        if "already exists" not in str(refused).lower():
+            raise
+    return True
+
+
+def judge(name: str, record: dict[str, Any], *, tries: int = 5) -> dict[str, Any]:
+    """Run one eval over one run, and give back what it decided.
+
+    The record is JSON-encoded rather than pasted. Rendering is sandboxed Jinja, and agents write
+    code blocks: a run containing braces would otherwise be read as template syntax and either
+    explode or quietly render as something else.
+    """
+    from fi.evals import Evaluator
+
+    payload = json.dumps(record, ensure_ascii=False, indent=2, default=str)
+    for attempt in range(max(1, tries)):
+        try:
+            # The model is named again here. The template carries one, but the run does not
+            # inherit it: without model_name the request arrives as "Model 'None'" and is
+            # refused, which is a 400 rather than anything about the conversation.
+            answered = Evaluator().evaluate(
+                eval_templates=name,
+                inputs={CONVERSATION: payload},
+                model_name=MODEL,
+            )
+            break
+        except Exception as failed:  # noqa: BLE001 - retried only when told to wait
+            after = _retry_after(failed)
+            if after is None and attempt + 1 >= tries:
+                raise
+            # Rate limiting is organisation-wide, so a suite running scenarios at once is
+            # exactly the shape that trips it, and the client does not back off on its own.
+            time.sleep(after if after is not None else 2.0**attempt)
+    else:  # pragma: no cover - the loop either breaks or raises
+        raise RuntimeError(f"{name} did not answer")
+
+    first = (getattr(answered, "eval_results", None) or [None])[0]
+    output = getattr(first, "output", None)
+    reason = getattr(first, "reason", "") or ""
+    # An eval that did not run is not an eval that failed. The SDK reports a rejected request by
+    # handing back a result whose output is empty and whose reason is the error, and reading that
+    # as "the claim does not hold" would fail an agent for an expired key or a bad payload. It
+    # raises instead, and the caller falls back to judging locally.
+    if first is None or (output is None and reason):
+        raise RuntimeError(f"{name} did not run: {reason or 'no result'}")
+    return {
+        "held": _passed(output),
+        "why": reason,
+        "output": output,
+        "eval": name,
+        # Present but null on a result, so a plain getattr default never fires.
+        "model": getattr(first, "model", None) or MODEL,
+    }
+
+
+def _retry_after(failed: Exception) -> float | None:
+    """How long the platform asked us to wait, when that is what it said."""
+    response = getattr(failed, "response", None)
+    headers = getattr(response, "headers", None) or {}
+    try:
+        return float(headers.get("Retry-After"))
+    except (TypeError, ValueError):
+        return None
+
+
+def _passed(output: Any) -> bool:
+    """Whether a verdict is a pass, given it can arrive as a word or a number."""
+    if isinstance(output, bool):
+        return output
+    if isinstance(output, (int, float)):
+        return float(output) >= 0.5
+    return str(output).strip().lower() in ("pass", "passed", "true", "yes")