agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/harness/run/voice.py

@@ -0,0 +1,340 @@
+"""Serving a real voice agent's tool calls from a generated world.
+
+A hosted voice agent executes its tools by calling a webhook. So the whole integration is one
+thing: stand up that webhook, and answer it from the world instead of from canned responses.
+
+That single swap is what the environment was built for. The previous run's known issues were all
+the same defect wearing different clothes:
+
+- *"Mocked tools always succeed, including removing an item that was never added."*
+- *"Mock responses do not vary by argument, so read-after-write flows are wrong."*
+- *"World state does not change unless a scenario sets state_updates, which is often empty."*
+
+A world that really holds rows and can really refuse answers all three, because the reply the
+agent hears is produced by running the call rather than by looking it up.
+
+Nothing here decides pass or fail. Grading reads the world afterwards and the calls this server
+recorded, through the same sub-goal checks every other run uses.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import Any, Mapping
+
+from ..world.runtime import GeneratedWorld
+
+logger = logging.getLogger(__name__)
+
+VAPI_API = os.environ.get("VAPI_API_BASE_URL", "https://api.vapi.ai").rstrip("/")
+
+# Vapi's edge rejects the default urllib User-Agent with a 403 that says nothing about why, while
+# the identical request from curl succeeds. Sending one is the whole fix.
+_AGENT = "alk-harness/0.1"
+
+
+class WorldWebhook:
+    """The webhook a hosted agent calls, answered by a generated world.
+
+    One world at a time. ``bind`` swaps which world is live between scenarios, so the assistant
+    stays configured while every scenario still starts from its own restored copy.
+    """
+
+    def __init__(self, host: str | None = None, port: int | None = None) -> None:
+        # A container must bind 0.0.0.0 on a fixed port or nothing outside can
+        # be configured to call it; a laptop keeps loopback and an ephemeral port.
+        host = (
+            host
+            if host is not None
+            else os.environ.get("HARNESS_WEBHOOK_HOST", "127.0.0.1")
+        )
+        port = (
+            port
+            if port is not None
+            else int(os.environ.get("HARNESS_WEBHOOK_PORT", "0"))
+        )
+        self._world: GeneratedWorld | None = None
+        self._lock = threading.Lock()
+        try:
+            server = HTTPServer((host, port), _handler_for(self))
+        except OSError:
+            # A leftover server from an earlier run must not block this one; any free port works
+            # because the public URL is discovered after binding.
+            logger.warning("port %s busy, binding an ephemeral port instead", port)
+            server = HTTPServer((host, 0), _handler_for(self))
+        self._server = server
+        self.port = server.server_address[1]
+        self._thread = threading.Thread(target=server.serve_forever, daemon=True)
+
+    def start(self) -> "WorldWebhook":
+        self._thread.start()
+        logger.info("world webhook listening on port %s", self.port)
+        return self
+
+    def stop(self) -> None:
+        self._server.shutdown()
+        self._server.server_close()
+
+    def bind(self, world: GeneratedWorld) -> None:
+        """Make one world live. Its own call log is what grading reads afterwards."""
+        with self._lock:
+            self._world = world
+            world.reset()
+
+    @property
+    def calls(self) -> list[Any]:
+        with self._lock:
+            return list(self._world.calls) if self._world else []
+
+    def respond(
+        self,
+        name: str,
+        arguments: Mapping[str, Any],
+        *,
+        session_id: str = "harness",
+    ) -> str:
+        """Answer one tool call by running it.
+
+        A refusal is returned as the answer, not as an error: the agent has to hear "that item is
+        unavailable" and cope with it, which is the whole reason the world can say no. What it
+        must never hear is an acknowledgement for something that did not happen.
+        """
+        with self._lock:
+            world = self._world
+        if world is None:
+            return "the environment is not ready"
+
+        # Caller hydration is an adapter concern, not a contract tool. The local
+        # LiveKit worker normally gets this from its demo API before exposing any
+        # conversational tools. Resolve the same safe profile from the generated
+        # world without adding a call that scenario grading would mistake for an
+        # agent action.
+        if name == "lookup_rider_by_phone":
+            phone = str(arguments.get("phone") or "")
+            state = world.observe().state
+            user = next(
+                (
+                    row
+                    for row in state.get("users", [])
+                    if str(row.get("phone")) == phone
+                ),
+                None,
+            )
+
+            def active_booking_ref() -> Any:
+                if user is None:
+                    return None
+                active = [
+                    row
+                    for row in state.get("bookings", [])
+                    if row.get("rider_id") == user.get("rider_id")
+                    and str(row.get("status") or "").lower()
+                    not in {"cancelled", "canceled", "completed"}
+                ]
+                active.sort(
+                    key=lambda row: str(row.get("created_at") or ""), reverse=True
+                )
+                return active[0].get("booking_ref") if active else None
+
+            forward = getattr(world, "forward", None)
+            if callable(forward):
+                hydrated = forward(
+                    name,
+                    arguments,
+                    record=False,
+                    session_id=session_id,
+                )
+                if hydrated.ok:
+                    result = hydrated.result
+                    if isinstance(result, dict):
+                        result = {**result, "booking_ref": active_booking_ref()}
+                    return (
+                        result
+                        if isinstance(result, str)
+                        else json.dumps(result, default=str)
+                    )
+                return hydrated.error
+            if user is None:
+                return json.dumps({"rider_id": None, "phone": phone})
+            market = next(
+                (
+                    row
+                    for row in state.get("market_config", [])
+                    if row.get("market") == user.get("default_market")
+                ),
+                {},
+            )
+            # A caller asking about or cancelling an existing ride does not know an internal
+            # booking reference. Real agent backends hydrate the active trip alongside ANI
+            # identity; expose the same seeded relationship from the generated world so the
+            # shipped agent can invoke its own reference-based tools without fixture leakage in
+            # the conversation.
+            return json.dumps(
+                {
+                    **user,
+                    "cash_supported_in_market": bool(market.get("cash_supported")),
+                    "accessibility_needs": [],
+                    "booking_ref": active_booking_ref(),
+                },
+                default=str,
+            )
+
+        # ``world.call`` distinguishes real dependency endpoints from actions executed inside
+        # the submitted worker and mirrored here only for evidence. Bypassing it through
+        # ``forward`` made a valid local action look like a missing HTTP endpoint and caused the
+        # agent to retry or abort after it had already updated its own state.
+        done = world.handle_tool_call({"name": name, "arguments": dict(arguments)})
+        if done is None:
+            return f"there is no tool called {name}"
+        return done.content or ("done" if done.success else "that could not be done")
+
+
+def _handler_for(owner: "WorldWebhook"):
+    class Handler(BaseHTTPRequestHandler):
+        def log_message(self, *args: Any) -> None:  # silence per-request stderr noise
+            return
+
+        def do_POST(self) -> None:  # noqa: N802 - required name
+            length = int(self.headers.get("Content-Length") or 0)
+            raw = self.rfile.read(length) if length else b"{}"
+            try:
+                payload = json.loads(raw or b"{}")
+            except json.JSONDecodeError:
+                payload = {}
+
+            calls = tool_calls(payload)
+            session_id = self.headers.get("x-session-id") or "harness"
+            if not calls:
+                # An agent whose tools are its own HTTP API asks differently: the tool is the
+                # path and the body is the arguments, with the answer expected back plainly.
+                # Serving both shapes is what lets a world stand in for such an API without the
+                # agent being changed to suit us.
+                name = self.path.strip("/").split("?")[0]
+                if name:
+                    answer = owner.respond(
+                        name,
+                        payload if isinstance(payload, dict) else {},
+                        session_id=session_id,
+                    )
+                    plain = json.dumps(_as_body(answer)).encode()
+                    self.send_response(200)
+                    self.send_header("Content-Type", "application/json")
+                    self.send_header("Content-Length", str(len(plain)))
+                    self.end_headers()
+                    self.wfile.write(plain)
+                    return
+
+            results = [
+                {
+                    "toolCallId": call_id,
+                    "result": owner.respond(name, arguments, session_id=session_id),
+                }
+                for call_id, name, arguments in calls
+            ]
+            body = json.dumps({"results": results}).encode()
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+
+    return Handler
+
+
+def _as_body(answer: str) -> Any:
+    """A tool's answer as a JSON body, keeping structure when the handler produced any.
+
+    Handlers return text because that is what a spoken agent hears. An HTTP tool API expects an
+    object, so a JSON answer is passed through as itself and anything else is wrapped, rather
+    than a caller having to parse a string out of a string.
+    """
+    try:
+        parsed = json.loads(answer)
+    except (json.JSONDecodeError, TypeError):
+        return {"result": answer}
+    return parsed if isinstance(parsed, (dict, list)) else {"result": parsed}
+
+
+def tool_calls(payload: Mapping[str, Any]) -> list[tuple[str, str, dict[str, Any]]]:
+    """Pull (id, name, arguments) out of a provider's tool-call webhook body."""
+    message = payload.get("message") or payload
+    raw = message.get("toolCalls") or message.get("toolCallList") or []
+    found: list[tuple[str, str, dict[str, Any]]] = []
+    for entry in raw if isinstance(raw, list) else []:
+        if not isinstance(entry, Mapping):
+            continue
+        function = entry.get("function") or {}
+        name = str(function.get("name") or entry.get("name") or "")
+        arguments = function.get("arguments") or entry.get("arguments") or {}
+        if isinstance(arguments, str):
+            try:
+                arguments = json.loads(arguments)
+            except json.JSONDecodeError:
+                arguments = {"_raw": arguments}
+        if name:
+            found.append((str(entry.get("id") or ""), name, dict(arguments)))
+    return found
+
+
+def pointed_at(tools: list[dict[str, Any]], webhook_url: str) -> list[dict[str, Any]]:
+    """The agent's own tools, with only where they are answered changed.
+
+    The assistant under test already has its tools — the names, the arguments, the enums are the
+    agent's, defined by whoever built it. Redefining them here would mean testing an agent we
+    wrote rather than theirs, and any drift between the two would show up as a finding about
+    them. So nothing is rebuilt: the one thing that changes is the address the call goes to.
+    """
+    repointed: list[dict[str, Any]] = []
+    for tool in tools:
+        moved = json.loads(json.dumps(tool))
+        moved.setdefault("server", {})["url"] = f"{webhook_url.rstrip('/')}/tool"
+        repointed.append(moved)
+    return repointed
+
+
+def fetch_assistant(assistant_id: str, api_key: str) -> dict[str, Any]:
+    """The assistant as it stands, so its own tools can be read rather than guessed."""
+    import urllib.request
+
+    request = urllib.request.Request(
+        f"{VAPI_API}/assistant/{assistant_id}",
+        headers={"Authorization": f"Bearer {api_key}", "User-Agent": _AGENT},
+    )
+    with urllib.request.urlopen(request, timeout=20) as answer:
+        return json.loads(answer.read())
+
+
+def repoint_assistant(assistant_id: str, api_key: str, webhook_url: str) -> list[str]:
+    """Send the assistant's existing tool calls to our webhook. Returns the tools moved."""
+    import urllib.request
+
+    assistant = fetch_assistant(assistant_id, api_key)
+    tools = (assistant.get("model") or {}).get("tools") or []
+    if not tools:
+        raise RuntimeError(
+            f"assistant {assistant_id} has no tools, so there is nothing for the environment "
+            "to answer. It is the agent's own tools that get repointed, not ones we add."
+        )
+    model = json.loads(json.dumps(assistant.get("model") or {}))
+    model["tools"] = pointed_at(tools, webhook_url)
+
+    body = json.dumps({"model": model}).encode()
+    request = urllib.request.Request(
+        f"{VAPI_API}/assistant/{assistant_id}",
+        data=body,
+        method="PATCH",
+        headers={
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+            "User-Agent": _AGENT,
+        },
+    )
+    with urllib.request.urlopen(request, timeout=20) as answer:
+        answer.read()
+    return [
+        str((one.get("function") or {}).get("name") or "") for one in model["tools"]
+    ]

fi/alk/harness/runtime.py

@@ -0,0 +1,172 @@
+"""Runtime-provider boundary for ALK-owned test environments.
+
+Providers decide *where* a sealed environment runs.  They do not decide how an agent is
+understood, how scenarios are written, or how results are graded.  The local provider below
+adapts the proven repository/Compose provisioner; the hosted sandbox implements the same port.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from enum import Enum
+from pathlib import Path
+from typing import Any, Protocol
+
+from pydantic import BaseModel, Field, JsonValue
+
+from .bundle import EnvironmentBundle
+
+
+class RuntimeState(str, Enum):
+    PREPARING = "preparing"
+    READY = "ready"
+    UNHEALTHY = "unhealthy"
+    STOPPED = "stopped"
+
+
+class RuntimeEndpoint(BaseModel):
+    capability: str
+    protocol: str
+    address: str
+    configuration_name: str | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class EnvironmentRuntime(BaseModel):
+    runtime_id: str
+    provider: str
+    bundle_digest: str
+    state: RuntimeState
+    endpoints: dict[str, RuntimeEndpoint] = Field(default_factory=dict)
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class RuntimeProvider(Protocol):
+    """Execution-location port implemented locally and by the hosted sandbox fleet."""
+
+    name: str
+
+    async def provision(
+        self,
+        bundle: EnvironmentBundle,
+        *,
+        source: Path,
+        work_directory: Path,
+        contract: Any | None = None,
+    ) -> EnvironmentRuntime: ...
+
+    async def reset(
+        self, runtime: EnvironmentRuntime, *, work_directory: Path
+    ) -> None: ...
+
+    async def healthy(
+        self, runtime: EnvironmentRuntime, *, work_directory: Path
+    ) -> bool: ...
+
+    async def close(
+        self, runtime: EnvironmentRuntime, *, work_directory: Path
+    ) -> None: ...
+
+
+class LocalComposeRuntimeProvider:
+    """Run one repository environment as an isolated Docker Compose project.
+
+    The adapter delegates lifecycle mechanics to ``harness.provision``, which gives each run a
+    unique project, allocates ports, waits for declared health checks, fingerprints source reuse,
+    and removes volumes during cleanup.  Only endpoint names and addresses cross this boundary;
+    resolved credentials remain process-local.
+    """
+
+    name = "local-compose"
+
+    async def provision(
+        self,
+        bundle: EnvironmentBundle,
+        *,
+        source: Path,
+        work_directory: Path,
+        contract: Any | None = None,
+    ) -> EnvironmentRuntime:
+        from .provision import provision
+
+        environment = await asyncio.to_thread(
+            provision, source, work_directory, contract
+        )
+        endpoints: dict[str, RuntimeEndpoint] = {}
+        overrides = dict(environment.overrides)
+        for capability, definition in bundle.capabilities.items():
+            address = ""
+            if definition.configuration_name:
+                address = overrides.get(definition.configuration_name, "")
+            if not address and len(overrides) == 1:
+                address = next(iter(overrides.values()))
+            discovered = next(
+                (
+                    endpoint
+                    for endpoint in environment.service_endpoints
+                    if endpoint["service"] == definition.service
+                    and endpoint["container_port"] == definition.container_port
+                ),
+                None,
+            )
+            if not address and discovered:
+                address = str(discovered["external_address"])
+            if address:
+                endpoints[capability] = RuntimeEndpoint(
+                    capability=capability,
+                    protocol=definition.protocol.value,
+                    address=address,
+                    configuration_name=definition.configuration_name,
+                    metadata=(
+                        {
+                            "service": str(discovered["service"]),
+                            "kind": str(discovered["kind"]),
+                        }
+                        if discovered
+                        else {}
+                    ),
+                )
+        return EnvironmentRuntime(
+            runtime_id=environment.project,
+            provider=self.name,
+            bundle_digest=bundle.digest,
+            state=RuntimeState.READY,
+            endpoints=endpoints,
+            metadata={
+                "services": environment.services,
+                "provision_seconds": environment.provision_seconds,
+                "managed": environment.managed,
+            },
+        )
+
+    async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None:
+        from .provision import reset
+
+        environment = await asyncio.to_thread(reset, work_directory)
+        runtime.state = (
+            RuntimeState.READY if environment.running else RuntimeState.UNHEALTHY
+        )
+
+    async def healthy(
+        self, runtime: EnvironmentRuntime, *, work_directory: Path
+    ) -> bool:
+        from .provision import healthy
+
+        is_healthy = await asyncio.to_thread(healthy, work_directory)
+        runtime.state = RuntimeState.READY if is_healthy else RuntimeState.UNHEALTHY
+        return is_healthy
+
+    async def close(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None:
+        from .provision import stop
+
+        await asyncio.to_thread(stop, work_directory)
+        runtime.state = RuntimeState.STOPPED
+
+
+__all__ = [
+    "EnvironmentRuntime",
+    "LocalComposeRuntimeProvider",
+    "RuntimeEndpoint",
+    "RuntimeProvider",
+    "RuntimeState",
+]