agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/harness/reception.py

@@ -0,0 +1,147 @@
+"""Stage zero: working out which agent you mean.
+
+Everything the harness does is about one agent, so something has to establish which one. That
+used to be two flags on a command line, which is the wrong place for it: the whole point is that
+you say what you want and it happens, and "here is my agent, set up a test environment for it"
+is a sentence, not an invocation.
+
+So this is a stage like any other. It can look around the filesystem to find what you are
+pointing at, it asks if what you said is ambiguous, and it finishes by naming the agent and where
+it lives. Everything after it, including where artifacts are written, follows from that.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from .backends import SessionSpec, tool, tool_server
+
+from .config import (
+    artifact_dir,
+    chosen_model,
+)
+from .session import Stage
+from .sources import AgentSource, clone_github_repository, resolve, supported
+from .tools import schema
+
+RECEPTION_SERVER = "agent"
+TOOL_NAMES = ("point_at_agent",)
+
+_INSTRUCTIONS = """
+You are the front desk of a harness that builds test environments for agents.
+
+Somebody has arrived with an agent they want tested. Your only job is to work out which agent,
+and where it lives, and then call point_at_agent. Nothing else happens until you do.
+
+Usually they will just tell you: a path, a repository, a folder, or a public GitHub URL. Take it.
+For an explicit local path, call point_at_agent immediately: that tool validates the path. Do not
+scan it first. Use Read, Glob or Grep only when the person gave an ambiguous parent directory or
+no path and you genuinely have to locate the agent. For a GitHub URL, call point_at_agent with kind "github" and the URL as
+path. The harness clones it into this session; do not ask them to clone it themselves. A name is a
+label for their artifacts, so lower case and no spaces.
+
+The agent is usually somewhere else on disk, not inside the harness. A path they give you is
+relative to where you are looking from, which is a workspace holding many repositories, so try
+it as given before deciding it does not exist.
+
+If the path really is not there, say so and say what you did find near it. If they gestured
+vaguely at a directory holding several agents, look, and ask which one with AskUserQuestion.
+
+Do not read the agent properly and do not start working anything out about it. That is the next
+stage's job and it has its own instructions. Point at the agent, say in one line what you are
+about to do, and stop.
+"""
+
+
+def point_at(
+    name: str,
+    path: str,
+    kind: str,
+    found: dict[str, AgentSource],
+    source_dir: Path | None = None,
+) -> dict[str, Any]:
+    """Establish which agent this is, or say why it cannot be.
+
+    A plain function rather than only a tool body, so what counts as a reachable agent can be
+    exercised without standing up a session.
+    """
+    name, path, kind = name.strip(), path.strip(), (kind.strip() or "repo")
+    if not name:
+        return _err("no name: the artifacts have to be filed under something")
+    if kind not in supported():
+        return _err(f"no such kind {kind!r}; there is {', '.join(supported())}")
+    if kind == "repo" and not Path(path).expanduser().exists():
+        return _err(
+            f"there is nothing at {path!r}. Look again with Glob, and if you cannot find it, "
+            "ask where the agent actually lives."
+        )
+    try:
+        if kind == "github":
+            root = clone_github_repository(
+                path, source_dir or artifact_dir(name) / "source"
+            )
+            found["source"] = resolve(kind, name=name, root=root, url=path)
+        else:
+            found["source"] = resolve(kind, name=name, root=Path(path).expanduser())
+    except Exception as failed:
+        return _err(f"could not reach that agent: {failed}")
+    return {
+        "content": [{"type": "text", "text": f"Pointed at {name} ({kind}) at {path}."}]
+    }
+
+
+def open_stage(
+    *,
+    cwd: str | Path | None = None,
+    source_dir: str | Path | None = None,
+    ask: Callable[..., Any] | None = None,
+    max_turns: int = 20,
+) -> tuple[Stage, dict[str, AgentSource]]:
+    """A stage that establishes which agent this conversation is about."""
+    found: dict[str, AgentSource] = {}
+
+    @tool(
+        "point_at_agent",
+        "Name the agent this conversation is about and say where it is. `kind` is how it is "
+        f"supplied, one of: {', '.join(supported())}. For a repository, `path` is its directory; "
+        "for github, it is the public HTTPS repository URL and the harness clones it. Call this "
+        "once you know what you are pointing at.",
+        schema({"name": str, "path": str, "kind": str}, ["name", "path"]),
+    )
+    async def point_at_agent(args: dict[str, Any]) -> dict[str, Any]:
+        return point_at(
+            str(args.get("name") or ""),
+            str(args.get("path") or ""),
+            str(args.get("kind") or "repo"),
+            found,
+            Path(source_dir) if source_dir else None,
+        )
+
+    server = tool_server(
+        name=RECEPTION_SERVER, version="0.1.0", tools=[point_at_agent]
+    )
+    spec = SessionSpec(
+        system_prompt=_INSTRUCTIONS.strip(),
+        servers={RECEPTION_SERVER: server},
+        builtins=("Read", "Glob", "Grep", "AskUserQuestion"),
+        cwd=str(cwd or Path.cwd()),
+        max_turns=max_turns,
+        model=chosen_model(),
+        ask=ask,
+        thinking=True,
+    )
+    return Stage(spec, name="reception"), found
+
+
+def opening() -> str:
+    return (
+        "Somebody has just opened the harness and has not said anything yet. Greet them in one "
+        "short line and ask which agent they want tested and where it lives. Do not list your "
+        "capabilities."
+    )
+
+
+def _err(text: str) -> dict[str, Any]:
+    return {"content": [{"type": "text", "text": text}], "is_error": True}

fi/alk/harness/retell_chat_call_runner.py

@@ -0,0 +1,373 @@
+"""Native Retell API-chat execution for hosted RL Environment runs."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from typing import Any, Mapping
+
+from fi.simulate.agent.wrapper import (
+    AgentInput,
+    AgentResponse,
+    AgentWrapper,
+    SimulationEvent,
+)
+
+from .call_runner import ArtifactUploader, CallRunnerContext, RETELL_API_KEY_ALIAS
+from .chat_call_runner import (
+    _HostedChatTarget,
+    _conversation_scenario,
+    _drive_conversation,
+    _duration_ms,
+    _scenario_document,
+    _tool_world,
+)
+from .contract import AgentContract
+from .hosted_scheduler import CallAborted, CallOutcome, Scenario, World
+from .outbound import ArtifactKind, format_rfc3339_millis
+from .process_runtime import EnvironmentRuntime
+
+
+class RetellChatError(RuntimeError):
+    """A bounded, credential-safe Retell Chat API error."""
+
+
+class RetellChatEnded(Exception):
+    """Retell reports that the remote chat reached a normal terminal state."""
+
+
+def _arguments(value: Any) -> dict[str, Any]:
+    if isinstance(value, Mapping):
+        return dict(value)
+    if isinstance(value, str):
+        try:
+            parsed = json.loads(value)
+        except ValueError:
+            return {"_raw": value}
+        return dict(parsed) if isinstance(parsed, Mapping) else {"_raw": value}
+    return {}
+
+
+def _completion_messages(
+    messages: Any,
+) -> tuple[str, list[dict[str, Any]], list[dict[str, Any]]]:
+    if not isinstance(messages, list):
+        raise RetellChatError("retell_chat_completion_messages_invalid")
+    answer = ""
+    calls: list[dict[str, Any]] = []
+    responses: list[dict[str, Any]] = []
+    for item in messages:
+        if not isinstance(item, Mapping):
+            continue
+        role = str(item.get("role") or "")
+        if role == "agent" and str(item.get("content") or "").strip():
+            answer = str(item["content"]).strip()
+        elif role == "tool_call_invocation":
+            calls.append(
+                {
+                    "id": str(item.get("tool_call_id") or item.get("message_id") or ""),
+                    "name": str(item.get("name") or ""),
+                    "arguments": _arguments(item.get("arguments")),
+                }
+            )
+        elif role == "tool_call_result":
+            responses.append(
+                {
+                    "tool_call_id": str(item.get("tool_call_id") or ""),
+                    "content": item.get("content"),
+                    "success": not bool(item.get("error")),
+                    "error": item.get("error"),
+                }
+            )
+    return answer, calls, responses
+
+
+class RetellChatWrapper(AgentWrapper):
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        agent_id: str,
+        agent_version: int | None = None,
+        dynamic_variables: Mapping[str, Any] | None = None,
+        api_base_url: str = "https://api.retellai.com",
+    ) -> None:
+        if not api_key:
+            raise RetellChatError("retell_chat_api_key_missing")
+        if not agent_id:
+            raise RetellChatError("retell_chat_agent_id_missing")
+        self._api_key = api_key
+        self._agent_id = agent_id
+        self._agent_version = agent_version
+        self._dynamic_variables = {
+            str(key): str(value) for key, value in dict(dynamic_variables or {}).items()
+        }
+        self._base = api_base_url.rstrip("/")
+        self._chat_id: str | None = None
+
+    def _request(
+        self, method: str, path: str, payload: Mapping[str, Any] | None = None
+    ) -> dict[str, Any]:
+        body = None if payload is None else json.dumps(payload).encode("utf-8")
+        request = urllib.request.Request(
+            self._base + path,
+            data=body,
+            method=method,
+            headers={
+                "Authorization": f"Bearer {self._api_key}",
+                "Content-Type": "application/json",
+                "Accept": "application/json",
+                "User-Agent": "FutureAGI-ALK/1.0",
+            },
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=120) as response:  # noqa: S310
+                raw = response.read()
+        except urllib.error.HTTPError as exc:
+            detail = " ".join(exc.read(512).decode("utf-8", errors="replace").split())
+            if exc.code == 400 and "chat already ended" in detail.lower():
+                raise RetellChatEnded from exc
+            raise RetellChatError(
+                f"retell_chat_api_error: {method} returned HTTP {exc.code}"
+                + (f": {detail}" if detail else "")
+            ) from exc
+        except (urllib.error.URLError, TimeoutError) as exc:
+            raise RetellChatError(
+                f"retell_chat_api_unavailable: {type(exc).__name__}"
+            ) from exc
+        if not raw:
+            return {}
+        try:
+            value = json.loads(raw)
+        except ValueError as exc:
+            raise RetellChatError("retell_chat_api_response_invalid_json") from exc
+        if not isinstance(value, dict):
+            raise RetellChatError("retell_chat_api_response_must_be_object")
+        return value
+
+    def _is_ended(self) -> bool:
+        if self._chat_id is None:
+            return False
+        result = self._request("GET", f"/get-chat/{self._chat_id}")
+        return str(result.get("chat_status") or "").strip().lower() == "ended"
+
+    def _ended_response(
+        self,
+        *,
+        answer: str = "",
+        calls: list[dict[str, Any]] | None = None,
+        responses: list[dict[str, Any]] | None = None,
+    ) -> AgentResponse:
+        return AgentResponse(
+            content=answer,
+            tool_calls=calls or [],
+            tool_responses=responses or [],
+            events=[
+                SimulationEvent(
+                    type="external_agent",
+                    name="retell_chat_ended",
+                    payload={"provider": "retell", "chat_id": self._chat_id},
+                )
+            ],
+            metadata={
+                "external_agent": {"success": True},
+                "retell_chat_id": self._chat_id,
+                "conversation_ended": True,
+            },
+        )
+
+    def _complete(self, content: str) -> AgentResponse:
+        if self._chat_id is None:
+            create: dict[str, Any] = {
+                "agent_id": self._agent_id,
+                "metadata": {"source": "futureagi-rl-environment"},
+            }
+            if self._agent_version is not None:
+                create["agent_version"] = self._agent_version
+            if self._dynamic_variables:
+                create["retell_llm_dynamic_variables"] = self._dynamic_variables
+            result = self._request("POST", "/create-chat", create)
+            self._chat_id = str(result.get("chat_id") or "").strip()
+            if not self._chat_id:
+                raise RetellChatError("retell_chat_create_missing_chat_id")
+        try:
+            result = self._request(
+                "POST",
+                "/create-chat-completion",
+                {"chat_id": self._chat_id, "content": content},
+            )
+        except RetellChatEnded:
+            return self._ended_response()
+        answer, calls, responses = _completion_messages(result.get("messages"))
+        explicitly_ended = (
+            bool(result.get("call_ended"))
+            or str(result.get("chat_status") or "").strip().lower() == "ended"
+        )
+        if not answer:
+            if explicitly_ended or self._is_ended():
+                return self._ended_response(calls=calls, responses=responses)
+            raise RetellChatError("retell_chat_completion_missing_agent_message")
+        if explicitly_ended:
+            return self._ended_response(answer=answer, calls=calls, responses=responses)
+        event = SimulationEvent(
+            type="external_agent",
+            name="retell_chat_completion",
+            payload={
+                "provider": "retell",
+                "chat_id": self._chat_id,
+                "tool_call_count": len(calls),
+            },
+        )
+        return AgentResponse(
+            content=answer,
+            tool_calls=calls,
+            tool_responses=responses,
+            events=[event],
+            metadata={
+                "external_agent": {"success": True},
+                "retell_chat_id": self._chat_id,
+            },
+        )
+
+    async def call(self, input: AgentInput) -> AgentResponse:
+        content = str((input.new_message or {}).get("content") or "").strip()
+        if not content:
+            raise RetellChatError("retell_chat_user_message_missing")
+        return await asyncio.to_thread(self._complete, content)
+
+    async def aclose(self) -> None:
+        if self._chat_id is None:
+            return
+        chat_id, self._chat_id = self._chat_id, None
+        try:
+            await asyncio.to_thread(self._request, "PATCH", f"/end-chat/{chat_id}")
+        except (RetellChatError, RetellChatEnded):
+            # Closing is idempotent. A chat may have ended as part of its final
+            # completion, in which case Retell rejects the redundant PATCH with
+            # "Chat already ended". Cleanup must never replace a valid call
+            # outcome with that harmless terminal response.
+            return
+
+
+class RetellChatCallRunner:
+    """Drive multi-turn scenarios directly through Retell's native Chat API."""
+
+    def __init__(self, adapter: ArtifactUploader, context: CallRunnerContext) -> None:
+        self._adapter = adapter
+        self._context = context
+        path = context.bundle_dir / "contract.json"
+        self._contract = (
+            AgentContract.model_validate_json(path.read_text())
+            if path.is_file()
+            else None
+        )
+
+    async def run(
+        self,
+        scenario: Scenario,
+        runtime: EnvironmentRuntime,
+        *,
+        world: World | None = None,
+    ) -> CallOutcome:
+        del world
+        if self._contract is None:
+            raise CallAborted(
+                "chat_contract_unavailable: bundle/contract.json is absent"
+            )
+        agent_id = str(
+            runtime.metadata.get("provider_target_id")
+            or self._context.job.agent.config.get("agent_id")
+            or ""
+        ).strip()
+        api_key = self._context.target_provider_secret_values.get(
+            RETELL_API_KEY_ALIAS, ""
+        )
+        config = self._context.job.agent.config
+        raw_dynamic = config.get("dynamic_variables")
+        dynamic_variables = raw_dynamic if isinstance(raw_dynamic, Mapping) else {}
+        version_value = config.get("agent_version")
+        agent_version = int(version_value) if version_value not in (None, "") else None
+        wrapper = RetellChatWrapper(
+            api_key=api_key,
+            agent_id=agent_id,
+            agent_version=agent_version,
+            dynamic_variables=dynamic_variables,
+            api_base_url=str(
+                config.get("provider_api_base_url") or "https://api.retellai.com"
+            ),
+        )
+        document = _scenario_document(self._context.bundle_dir, scenario.scenario_key)
+        conversation_scenario = _conversation_scenario(document)
+        target_world = _tool_world(
+            self._context.bundle_dir,
+            self._contract,
+            runtime,
+            self._context.source_directory,
+        )
+        target = _HostedChatTarget(
+            wrapper=wrapper,
+            contract=self._contract,
+            world=target_world,
+            scenario_key=scenario.scenario_key,
+            scenario_id=scenario.scenario_id,
+        )
+        started = datetime.now(timezone.utc)
+        try:
+            transcript = await _drive_conversation(
+                target, conversation_scenario, self._contract, self._context.bundle_dir
+            )
+        except CallAborted:
+            raise
+        except Exception as exc:
+            raise CallAborted(
+                f"retell_chat_target_failed: {type(exc).__name__}: {exc}"
+            ) from exc
+        finally:
+            await wrapper.aclose()
+        ended = datetime.now(timezone.utc)
+        transcript_id = await self._adapter.upload_artifact(
+            transcript.artifact(),
+            kind=ArtifactKind.TRANSCRIPT,
+            scenario_key=scenario.scenario_key,
+        )
+        calls = tuple(transcript.calls)
+        if calls:
+            trace = "\n".join(
+                json.dumps(
+                    {
+                        "name": call.name,
+                        "arguments": call.arguments,
+                        "result": call.result,
+                        "ok": call.ok,
+                        "error": call.error,
+                        "refused": call.refused,
+                        "at": call.at,
+                    },
+                    sort_keys=True,
+                    default=str,
+                )
+                for call in calls
+            ).encode()
+            await self._adapter.upload_artifact(
+                trace, kind=ArtifactKind.TOOL_TRACE, scenario_key=scenario.scenario_key
+            )
+        return CallOutcome(
+            calls=calls,
+            turns=len(transcript.exchanges),
+            started_at=format_rfc3339_millis(started),
+            ended_at=format_rfc3339_millis(ended),
+            duration_ms=_duration_ms(started, ended),
+            transcript_artifact=transcript_id,
+            messages=tuple(transcript.canonical_messages()),
+        )
+
+
+__all__ = [
+    "RetellChatCallRunner",
+    "RetellChatEnded",
+    "RetellChatError",
+    "RetellChatWrapper",
+]