agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/harness/backends/base.py

@@ -0,0 +1,241 @@
+"""What a harness backend is, said without naming any vendor.
+
+A stage of this harness is a conversation: a system prompt, a set of tools the model may call,
+a turn budget, and a loop that feeds tool results back until the model stops. Today that loop is
+Claude Code's; tomorrow it may be Gemini's, Bedrock's, or a harness of our own. The stages do
+not care, so nothing they say may mention a vendor.
+
+This module is that neutrality, in four pieces:
+
+- ``ToolSpec`` / ``ToolServer``: a tool as the harness defines one, with the async handler that
+  executes it. Backends adapt these to whatever their loop natively speaks.
+- ``SessionSpec``: everything a stage asks of a session. This is the real contract the ten
+  construction sites were already expressing through a vendor options class.
+- The reply vocabulary (``SessionOpened``, ``ModelReply``, ``ToolReturned``, ``StageDone``):
+  what a running session emits, which ``Stage`` renders into events. A backend translates its
+  provider's stream into these and nothing else leaks through.
+- ``HarnessBackend`` / ``HarnessSession``: the two protocols a new backend implements. A backend
+  with its own loop supplies its own session type; the harness never sees inside it.
+
+Nothing here imports a provider SDK, so a deployment that uses one backend does not need the
+other's dependencies installed.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, AsyncIterator, Awaitable, Callable, Protocol, runtime_checkable
+
+ToolHandler = Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]
+
+
+def qualified(server: str, tool_name: str) -> str:
+    """The fully qualified name a session grants and a model calls.
+
+    The ``mcp__{server}__{tool}`` convention comes from the first backend, but the skills and
+    gates all speak it, so every backend keeps it. Renaming per backend would mean rewriting
+    every prompt that names a tool.
+    """
+    return f"mcp__{server}__{tool_name}"
+
+
+@dataclass
+class ToolSpec:
+    """One tool: its contract for the model, and the code that executes it.
+
+    ``input_schema`` is either a JSON Schema dict or the shorthand ``{"arg": str}`` mapping the
+    tool decorator accepts. ``handler`` receives the arguments dict and returns
+    ``{"content": [{"type": "text", "text": ...}], "is_error"?: bool}``, the shape every
+    existing tool already returns.
+    """
+
+    name: str
+    description: str
+    input_schema: Any
+    handler: ToolHandler
+
+
+@dataclass
+class ToolServer:
+    """A named group of tools granted to a session together."""
+
+    name: str
+    version: str = "0.1.0"
+    tools: list[ToolSpec] = field(default_factory=list)
+
+
+def tool(
+    name: str, description: str, input_schema: Any
+) -> Callable[[ToolHandler], ToolSpec]:
+    """Declare a tool. Same signature the stages have always used, no vendor behind it."""
+
+    def decorator(handler: ToolHandler) -> ToolSpec:
+        return ToolSpec(
+            name=name,
+            description=description,
+            input_schema=input_schema,
+            handler=handler,
+        )
+
+    return decorator
+
+
+def tool_server(
+    name: str, version: str = "0.1.0", tools: list[ToolSpec] | None = None
+) -> ToolServer:
+    """Group tools under a server name, as the stages have always done."""
+    return ToolServer(name=name, version=version, tools=list(tools or []))
+
+
+# Host tools a backend may be asked to supply itself. Claude Code ships these; a backend without
+# a host CLI implements them from files.py. Anything else asked for as a builtin is refused at
+# session build time rather than silently dropped.
+FILE_TOOLS = ("Read", "Glob", "Grep")
+ASK_TOOL = "AskUserQuestion"
+KNOWN_BUILTINS = (*FILE_TOOLS, ASK_TOOL)
+
+
+@dataclass
+class SessionSpec:
+    """Everything a stage asks of a session, with no vendor vocabulary in it.
+
+    ``builtins`` are host tools by bare name (``Read``, ``Glob``, ``Grep``,
+    ``AskUserQuestion``); ``servers`` are the harness's own tools. ``ask`` is the operator
+    callback consulted when the model asks a question; None means the run is unattended.
+    ``gated`` selects the deny-by-default permission regime every tool-bearing stage runs
+    under; the one stage that runs bare (the simulated customer, which has no tools) turns it
+    off to keep its behaviour byte-identical.
+    ``thinking`` opts into the harness's thinking policy (config.thinking_config); stages that
+    never set one keep their backend's default.
+    """
+
+    system_prompt: str
+    servers: dict[str, ToolServer] = field(default_factory=dict)
+    builtins: tuple[str, ...] = ()
+    cwd: str | None = None
+    max_turns: int = 40
+    model: str = ""
+    ask: Any = None
+    gated: bool = True
+    thinking: bool = False
+    # A ready-made permission callable that replaces the backend's own gate wholesale. One
+    # stage (understand, interactive) passes its gate in fully built; backends without a
+    # permission callback concept ignore it, which is safe because their gating is structural.
+    permission_override: Any = None
+    # How long this stage may go without emitting anything before it is treated as hung. Zero
+    # takes the harness default. A stage whose work happens inside one long tool call needs its
+    # own bound: it is working the whole time and has nothing to say while it does, so the
+    # default reads honest work as a hang and kills it.
+    idle_timeout_seconds: float = 0.0
+
+    def granted(self) -> list[str]:
+        """Every tool name this session may call, qualified the way the model calls it."""
+        names = [*self.builtins]
+        for server_name, server in self.servers.items():
+            names.extend(qualified(server_name, spec.name) for spec in server.tools)
+        return names
+
+    def grant(self, server_name: str, server: ToolServer) -> None:
+        """Add a tool server before the session opens."""
+        self.servers[server_name] = server
+
+
+# -- what a running session emits --------------------------------------------------------------
+
+
+@dataclass
+class SessionOpened:
+    """The session exists and has an identity, if the backend assigns one."""
+
+    session_id: str | None = None
+
+
+@dataclass
+class Say:
+    """The model said something."""
+
+    text: str
+
+
+@dataclass
+class Call:
+    """The model called a tool."""
+
+    id: str
+    name: str
+    arguments: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class ModelReply:
+    """One assistant message: text and tool calls, in the order they were produced."""
+
+    parts: list[Any] = field(default_factory=list)
+    model: str = ""
+
+
+@dataclass
+class ToolReturned:
+    """What a tool call produced, flattened to text."""
+
+    id: str
+    text: str
+    is_error: bool = False
+
+
+@dataclass
+class StageDone:
+    """The exchange is over. The raw facts; Stage turns them into words."""
+
+    outcome: str = "success"
+    turns: int = 0
+    cost_usd: float | None = None
+    # The units behind the price, so a bill can be checked rather than trusted.
+    tokens_in: int = 0
+    tokens_out: int = 0
+    # Input the provider served from its own cache. Included in tokens_in, and billed well below
+    # fresh input, so a ledger that does not carry it overstates a rerun. Carried rather than
+    # discounted here: inventing a cache rate would be a guess presented as a price.
+    tokens_cached: int = 0
+    session_id: str | None = None
+    models: set[str] = field(default_factory=set)
+    is_error: bool = False
+    api_error_status: Any = None
+    errors: list[Any] = field(default_factory=list)
+
+
+# -- what a backend implements -----------------------------------------------------------------
+
+
+@runtime_checkable
+class HarnessSession(Protocol):
+    """One open session. Backends with their own loop implement this around it."""
+
+    async def start(self) -> None: ...
+
+    async def stop(self) -> None: ...
+
+    async def send(self, message: str) -> None: ...
+
+    def replies(self) -> AsyncIterator[Any]:
+        """Everything the session emits for the message just sent, ending with StageDone."""
+        ...
+
+
+@runtime_checkable
+class HarnessBackend(Protocol):
+    """A way of running stages. Selected by name through the registry."""
+
+    name: str
+    default_model: str
+
+    def create(self, spec: SessionSpec) -> HarnessSession: ...
+
+    def can_drive(self, model: str) -> bool:
+        """Whether this backend can actually run the named model.
+
+        A backend handed a model it cannot reach must refuse loudly here. Left unchecked it
+        produces a session that answers nothing, which downstream reads as an agent that ignored
+        the person rather than as a configuration mistake.
+        """
+        ...

fi/alk/harness/backends/claude.py

@@ -0,0 +1,211 @@
+"""The Claude Code backend: the loop this harness grew up on.
+
+Adapts the neutral ``SessionSpec`` to ``ClaudeAgentOptions`` exactly the way the stages built
+them before the seam existed: same gate hooks, same permission callback, same provider env,
+same disallowed list. With ``ALK_HARNESS`` unset this backend runs, so nothing here may drift
+from what the stages did on their own.
+
+Claude Code supplies Read/Glob/Grep and AskUserQuestion itself, so builtins are granted by
+name rather than implemented here.
+"""
+
+from __future__ import annotations
+
+from typing import Any, AsyncIterator
+
+from claude_agent_sdk import (
+    AssistantMessage,
+    ClaudeAgentOptions,
+    ClaudeSDKClient,
+    ResultMessage,
+    SdkMcpTool,
+    SystemMessage,
+    TextBlock,
+    ToolResultBlock,
+    ToolUseBlock,
+    create_sdk_mcp_server,
+)
+
+from .base import (
+    Call,
+    ModelReply,
+    Say,
+    SessionOpened,
+    SessionSpec,
+    StageDone,
+    ToolReturned,
+    ToolServer,
+    qualified,
+)
+
+DEFAULT_MODEL = "claude-sonnet-4-6"
+
+
+def _sdk_server(server: ToolServer) -> Any:
+    """A ToolServer as the in-process MCP server the SDK routes calls to."""
+    return create_sdk_mcp_server(
+        name=server.name,
+        version=server.version,
+        tools=[
+            SdkMcpTool(
+                name=spec.name,
+                description=spec.description,
+                input_schema=spec.input_schema,
+                handler=spec.handler,
+            )
+            for spec in server.tools
+        ],
+    )
+
+
+def _flattened(content: Any) -> str:
+    """A tool result's content as one string, however the SDK packaged it."""
+    if isinstance(content, list):
+        return "\n".join(
+            part.get("text", "") for part in content if isinstance(part, dict)
+        )
+    return content if isinstance(content, str) else str(content)
+
+
+class ClaudeSession:
+    """One Claude Code session, translated to the neutral reply vocabulary."""
+
+    def __init__(self, options: ClaudeAgentOptions) -> None:
+        self._options = options
+        self._client: ClaudeSDKClient | None = None
+
+    async def start(self) -> None:
+        self._client = ClaudeSDKClient(options=self._options)
+        await self._client.connect()
+
+    async def stop(self) -> None:
+        if self._client is not None:
+            await self._client.disconnect()
+            self._client = None
+
+    async def send(self, message: str) -> None:
+        if self._client is None:
+            raise RuntimeError("session is not open")
+        await self._client.query(message)
+
+    async def replies(self) -> AsyncIterator[Any]:
+        if self._client is None:
+            raise RuntimeError("session is not open")
+        async for received in self._client.receive_response():
+            for reply in self._translate(received):
+                yield reply
+
+    def _translate(self, received: Any) -> list[Any]:
+        if isinstance(received, SystemMessage):
+            data = received.data if isinstance(received.data, dict) else {}
+            return [SessionOpened(session_id=data.get("session_id"))]
+        if isinstance(received, AssistantMessage):
+            parts: list[Any] = []
+            for block in received.content:
+                if isinstance(block, TextBlock):
+                    parts.append(Say(text=block.text))
+                elif isinstance(block, ToolUseBlock):
+                    parts.append(
+                        Call(id=block.id, name=block.name, arguments=block.input)
+                    )
+            return [ModelReply(parts=parts, model=getattr(received, "model", "") or "")]
+        if isinstance(received, ResultMessage):
+            # subtype alone is not the outcome. A call that failed upstream still arrives with
+            # subtype "success", so the error facts ride along and Stage decides what failed.
+            return [
+                StageDone(
+                    outcome=received.subtype,
+                    turns=received.num_turns,
+                    cost_usd=received.total_cost_usd,
+                    **_tokens(getattr(received, "model_usage", None)),
+                    session_id=received.session_id,
+                    models=set(getattr(received, "model_usage", None) or {}),
+                    is_error=bool(getattr(received, "is_error", False)),
+                    api_error_status=getattr(received, "api_error_status", None),
+                    errors=list(getattr(received, "errors", None) or []),
+                )
+            ]
+        blocks = getattr(received, "content", None)
+        if isinstance(blocks, list):
+            returned = []
+            for block in blocks:
+                if isinstance(block, ToolResultBlock):
+                    returned.append(
+                        ToolReturned(
+                            id=block.tool_use_id,
+                            text=_flattened(block.content),
+                            is_error=bool(getattr(block, "is_error", False)),
+                        )
+                    )
+            return returned
+        return []
+
+
+def _tokens(model_usage: Any) -> dict[str, int]:
+    """Input and output tokens across every model a stage used, for the ledger to audit against.
+
+    Read defensively: this is the SDK's shape, not ours, and a stage must not fail over accounting.
+    """
+    read = 0
+    written = 0
+    for usage in (model_usage or {}).values():
+        if isinstance(usage, dict):
+            read += int(usage.get("inputTokens") or usage.get("input_tokens") or 0)
+            written += int(usage.get("outputTokens") or usage.get("output_tokens") or 0)
+        else:
+            read += int(getattr(usage, "input_tokens", 0) or 0)
+            written += int(getattr(usage, "output_tokens", 0) or 0)
+    return {"tokens_in": read, "tokens_out": written}
+
+
+class ClaudeBackend:
+    name = "claude"
+    default_model = DEFAULT_MODEL
+
+    def can_drive(self, model: str) -> bool:
+        return "claude" in (model or "").lower()
+
+    def create(self, spec: SessionSpec) -> ClaudeSession:
+        from ..config import (
+            UNWANTED,
+            gate_hooks,
+            permission_gate,
+            provider_env,
+            thinking_config,
+        )
+
+        allowed = [
+            *spec.builtins,
+            *(
+                qualified(server_name, tool_spec.name)
+                for server_name, server in spec.servers.items()
+                for tool_spec in server.tools
+            ),
+        ]
+        options = ClaudeAgentOptions(
+            system_prompt=spec.system_prompt,
+            allowed_tools=allowed,
+            mcp_servers={
+                server_name: _sdk_server(server)
+                for server_name, server in spec.servers.items()
+            },
+            setting_sources=[],
+            max_turns=spec.max_turns,
+            model=spec.model,
+            env=provider_env(spec.model),
+        )
+        if spec.cwd is not None:
+            options.cwd = spec.cwd
+        if spec.gated:
+            # Not acceptEdits: that auto-approves Edit and Write before the permission callback
+            # is consulted, so a stage could rewrite an artifact by hand and skip the tool whose
+            # whole job is to validate that change.
+            options.permission_mode = "default"
+            options.disallowed_tools = list(UNWANTED)
+            options.hooks = gate_hooks(allowed)
+            options.can_use_tool = spec.permission_override or permission_gate(
+                spec.ask, allowed
+            )
+        if spec.thinking:
+            options.thinking = thinking_config()
+        return ClaudeSession(options)

fi/alk/harness/backends/files.py

@@ -0,0 +1,182 @@
+"""Read-only file tools for backends that have no host CLI behind them.
+
+Claude Code brings its own Read, Glob and Grep. Any other loop that is granted those names gets
+these: same names, same core arguments, same read-only stance. They exist so a stage's skill
+text ("read the repository, never write to it") means the same thing on every backend.
+
+Write access is not implemented on purpose. The harness's own artifacts go through its tools,
+which validate them; the agent under test is somebody's real repository. A backend that wants
+write tools is asking to skip the gates, and the answer is no.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import re
+from pathlib import Path
+
+from .base import ToolSpec
+
+MAX_READ_LINES = 2000
+MAX_LINE_CHARS = 2000
+MAX_MATCHES = 200
+_SKIP_DIRS = {".git", ".venv", "node_modules", "__pycache__", ".pytest_cache"}
+
+
+def _ok(text: str) -> dict:
+    return {"content": [{"type": "text", "text": text}]}
+
+
+def _error(text: str) -> dict:
+    return {"content": [{"type": "text", "text": text}], "is_error": True}
+
+
+def _resolved(raw: str, cwd: Path) -> Path:
+    path = Path(raw).expanduser()
+    return path if path.is_absolute() else cwd / path
+
+
+def _walk(root: Path):
+    """Every file under root, skipping the trees nobody means when they say 'the repo'."""
+    stack = [root]
+    while stack:
+        folder = stack.pop()
+        try:
+            entries = sorted(folder.iterdir())
+        except OSError:
+            continue
+        for entry in entries:
+            if entry.is_dir():
+                if entry.name not in _SKIP_DIRS:
+                    stack.append(entry)
+            else:
+                yield entry
+
+
+def file_tools(cwd: str | None) -> list[ToolSpec]:
+    """Read, Glob and Grep rooted at the session's working directory."""
+    base = Path(cwd) if cwd else Path.cwd()
+
+    async def read(args: dict) -> dict:
+        raw = str(args.get("file_path") or args.get("path") or "")
+        if not raw:
+            return _error("file_path is required")
+        path = _resolved(raw, base)
+        if not path.is_file():
+            return _error(f"{path} is not a file that exists")
+        try:
+            lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+        except OSError as exc:
+            return _error(f"could not read {path}: {exc}")
+        offset = max(int(args.get("offset") or 1), 1)
+        limit = min(int(args.get("limit") or MAX_READ_LINES), MAX_READ_LINES)
+        window = lines[offset - 1 : offset - 1 + limit]
+        numbered = "\n".join(
+            f"{offset + index}\t{line[:MAX_LINE_CHARS]}"
+            for index, line in enumerate(window)
+        )
+        remaining = len(lines) - (offset - 1 + len(window))
+        if remaining > 0:
+            numbered += f"\n... ({remaining} more lines; call again with offset={offset + len(window)})"
+        return _ok(numbered or "(empty file)")
+
+    async def glob(args: dict) -> dict:
+        pattern = str(args.get("pattern") or "")
+        if not pattern:
+            return _error("pattern is required")
+        root = _resolved(str(args.get("path") or "."), base)
+        if not root.is_dir():
+            return _error(f"{root} is not a directory that exists")
+        matches = []
+        for entry in _walk(root):
+            relative = str(entry.relative_to(root))
+            if fnmatch.fnmatch(relative, pattern) or fnmatch.fnmatch(
+                entry.name, pattern
+            ):
+                matches.append(str(entry))
+                if len(matches) >= MAX_MATCHES:
+                    break
+        return _ok("\n".join(matches) or f"no files match {pattern!r} under {root}")
+
+    async def grep(args: dict) -> dict:
+        pattern = str(args.get("pattern") or "")
+        if not pattern:
+            return _error("pattern is required")
+        try:
+            expression = re.compile(pattern)
+        except re.error as exc:
+            return _error(f"invalid regular expression: {exc}")
+        root = _resolved(str(args.get("path") or "."), base)
+        wanted = str(args.get("glob") or "")
+        hits: list[str] = []
+        targets = [root] if root.is_file() else list(_walk(root)) if root.is_dir() else []
+        if not targets:
+            return _error(f"{root} does not exist")
+        for entry in targets:
+            if wanted and not fnmatch.fnmatch(entry.name, wanted):
+                continue
+            try:
+                text = entry.read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                continue
+            for number, line in enumerate(text.splitlines(), 1):
+                if expression.search(line):
+                    hits.append(f"{entry}:{number}:{line[:400]}")
+                    if len(hits) >= MAX_MATCHES:
+                        break
+            if len(hits) >= MAX_MATCHES:
+                break
+        return _ok("\n".join(hits) or f"no lines match {pattern!r}")
+
+    return [
+        ToolSpec(
+            name="Read",
+            description=(
+                "Read a file. Arguments: file_path (absolute or relative to the working "
+                "directory), optional offset (1-based first line) and limit (line count)."
+            ),
+            input_schema={
+                "type": "object",
+                "properties": {
+                    "file_path": {"type": "string"},
+                    "offset": {"type": "integer"},
+                    "limit": {"type": "integer"},
+                },
+                "required": ["file_path"],
+            },
+            handler=read,
+        ),
+        ToolSpec(
+            name="Glob",
+            description=(
+                "Find files by name pattern, e.g. **/*.py. Arguments: pattern, optional path "
+                "to search under."
+            ),
+            input_schema={
+                "type": "object",
+                "properties": {
+                    "pattern": {"type": "string"},
+                    "path": {"type": "string"},
+                },
+                "required": ["pattern"],
+            },
+            handler=glob,
+        ),
+        ToolSpec(
+            name="Grep",
+            description=(
+                "Search file contents with a regular expression. Arguments: pattern, optional "
+                "path (file or directory) and glob to filter file names."
+            ),
+            input_schema={
+                "type": "object",
+                "properties": {
+                    "pattern": {"type": "string"},
+                    "path": {"type": "string"},
+                    "glob": {"type": "string"},
+                },
+                "required": ["pattern"],
+            },
+            handler=grep,
+        ),
+    ]