quickstarted 0.2.0__py3-none-any.whl

quickstarted/__init__.py

@@ -0,0 +1,24 @@
+"""quickstarted: test whether an AI agent can complete your quickstart.
+
+Journeys (YAML) state a goal, a docs entrypoint, and a machine-checkable
+success assertion. The harness runs an agent in a sandbox whose only docs
+access is a recorded, allowlisted fetch tool, then scores the run with the
+assertion script. See README.md.
+"""
+
+from ._version import __version__
+from .journey import Budgets, Journey, JourneyError, load_journey
+from .run import RunResult, ScoreResult, run_journey
+from .trace import Trace
+
+__all__ = [
+    "Budgets",
+    "Journey",
+    "JourneyError",
+    "RunResult",
+    "ScoreResult",
+    "Trace",
+    "__version__",
+    "load_journey",
+    "run_journey",
+]

quickstarted/_version.py

@@ -0,0 +1,7 @@
+"""Single source of the version, importable without pulling in the package.
+
+Kept separate so low-level modules can stamp the version (User-Agent, results
+schema) without a circular import through `quickstarted/__init__.py`.
+"""
+
+__version__ = "0.2.0"

quickstarted/agents/__init__.py

@@ -0,0 +1,4 @@
+from .base import Agent, AgentOutcome, Toolbelt
+from .replay import ReplayAgent
+
+__all__ = ["Agent", "AgentOutcome", "ReplayAgent", "Toolbelt"]

quickstarted/agents/base.py

@@ -0,0 +1,145 @@
+"""Agent adapter interface and the harness-owned toolbelt.
+
+Every agent, whatever the model or vendor, acts on the sandbox only through
+the Toolbelt. That is a deliberate design choice: because docs access flows
+through `fetch`, the harness records every page the agent reads (failure
+attribution) and enforces the journey's host allowlist. Adapters never talk
+to the filesystem or network directly.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Protocol
+
+from ..docs import DocsClient
+from ..exec.base import Executor, truncate
+from ..journey import Journey
+from ..trace import Trace
+from ..transport import html_to_text
+
+_FETCH_LIMIT = 60_000
+
+
+class Toolbelt:
+    def __init__(
+        self,
+        journey: Journey,
+        executor: Executor,
+        trace: Trace,
+        docs: DocsClient | None = None,
+        http_get: Callable[[str], tuple[str, str]] | None = None,
+    ):
+        self.journey = journey
+        self.executor = executor
+        self.trace = trace
+        self.docs = docs or DocsClient()
+        self._http_get = http_get
+
+    def bash(self, command: str) -> str:
+        budgets = self.journey.budgets
+        self.trace.add("tool_call", tool="bash", command=command)
+        result = self.executor.run(
+            command,
+            timeout=budgets.max_command_seconds,
+            max_output_chars=budgets.max_output_chars,
+        )
+        self.trace.add(
+            "tool_result",
+            tool="bash",
+            exit_code=result.exit_code,
+            duration=round(result.duration, 3),
+            timed_out=result.timed_out,
+            output=result.output,
+        )
+        return f"exit code: {result.exit_code}\n{result.output}"
+
+    def fetch(self, url: str) -> str:
+        self.trace.add("tool_call", tool="fetch", url=url)
+        if not self.journey.host_allowed(url):
+            allowed = ", ".join(self.journey.docs_allow)
+            message = (
+                f"BLOCKED: {url} is outside this journey's documentation allowlist "
+                f"({allowed}). Only the target project's docs may be read."
+            )
+            self.trace.add("fetch_blocked", url=url)
+            return message
+
+        if self._http_get is not None:  # legacy injection point, used by tests
+            try:
+                content_type, body = self._http_get(url)
+            except Exception as exc:
+                self.trace.add("fetch_error", url=url, error=str(exc))
+                return f"FETCH ERROR for {url}: {exc}"
+            if "html" in content_type.lower():
+                body = html_to_text(body)
+            body = truncate(body, _FETCH_LIMIT)
+            self.trace.add("docs_fetch", url=url, chars=len(body))
+            return body
+
+        try:
+            result = self.docs.get(url)
+        except Exception as exc:
+            self.trace.add("fetch_error", url=url, error=str(exc))
+            return f"FETCH ERROR for {url}: {exc}"
+
+        if result.blocked_reason == "affordance_withheld":
+            # Ablation condition: the file exists, the agent may not have it.
+            self.trace.add("affordance_withheld", url=url)
+            return (
+                f"NOT AVAILABLE: {url} could not be retrieved. Use the regular "
+                "documentation pages."
+            )
+        if result.blocked_reason:
+            self.trace.add("fetch_blocked", url=url, reason=result.blocked_reason)
+            return f"BLOCKED: {url} ({result.blocked_reason})"
+
+        body = result.text
+        original = len(body)
+        body = truncate(body, _FETCH_LIMIT)
+        # A page too large to read is itself an agent-experience defect, so
+        # record it rather than silently trimming.
+        self.trace.add(
+            "docs_fetch",
+            url=url,
+            chars=len(body),
+            original_chars=original,
+            truncated=original > _FETCH_LIMIT,
+            from_cache=result.from_cache,
+            content_hash=result.content_hash,
+        )
+        if result.changed:
+            self.trace.add("docs_changed", url=url, content_hash=result.content_hash)
+        return body
+
+
+@dataclass(frozen=True)
+class AgentOutcome:
+    stop_reason: str  # completed | command_failed | max_turns | timeout | refusal | error
+    turns: int
+    detail: str = ""
+    #: Uncached prompt tokens. Vendors disagree here: Anthropic reports
+    #: `input_tokens` already excluding cache traffic, while OpenAI and Google
+    #: report a prompt total that *includes* it. Adapters normalise to the
+    #: Anthropic meaning, so that the four counters sum to the run's real cost
+    #: and no token is billed twice in a cross-vendor comparison.
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cache_write_tokens: int = 0
+    cache_read_tokens: int = 0
+
+    @property
+    def total_tokens(self) -> int:
+        return (
+            self.input_tokens
+            + self.output_tokens
+            + self.cache_write_tokens
+            + self.cache_read_tokens
+        )
+
+
+class Agent(Protocol):
+    name: str
+
+    def run(self, journey: Journey, toolbelt: Toolbelt, deadline: float) -> AgentOutcome:
+        ...

quickstarted/agents/claude.py

@@ -0,0 +1,242 @@
+"""Claude agent adapter: a manual tool-use loop on the plain anthropic SDK.
+
+The loop is deliberately hand-rolled rather than delegated to a heavyweight
+agent framework: the harness must own the tools (sandboxed bash + allowlisted
+docs fetch) so that every action and every page read lands in the trace.
+
+Requires the `anthropic` package (pip install "quickstarted[claude]") and an API
+key. The key is read from QUICKSTARTED_ANTHROPIC_API_KEY first so it can live in
+your environment without other Anthropic tooling (Claude Code, for one) picking
+it up and billing against it; ANTHROPIC_API_KEY is honoured as a fallback for
+CI, where that name is the convention.
+"""
+
+from __future__ import annotations
+
+import os
+import random
+import time
+from typing import Any
+
+from ..journey import Journey
+from .base import AgentOutcome, Toolbelt
+from .prompt import READ_DOCS_DESCRIPTION, SYSTEM
+from .prompt import kickoff as build_kickoff
+
+DEFAULT_MODEL = "claude-opus-5"
+
+#: Transient upstream conditions. A benchmark sweep that dies on one 529 is
+#: not a benchmark, and a run that reports these as a documentation failure is
+#: worse than one that dies.
+RETRYABLE_STATUS = (408, 409, 429, 500, 502, 503, 529)
+MAX_ATTEMPTS = 5
+KEY_ENV = "QUICKSTARTED_ANTHROPIC_API_KEY"
+FALLBACK_KEY_ENV = "ANTHROPIC_API_KEY"
+
+
+class _Usage:
+    """Running token totals for one journey.
+
+    The API reports cached prompt tokens outside `input_tokens`, so a run with
+    prompt caching on looks nearly free if you only add up that field. All four
+    counters are kept so the reported cost is the real one.
+    """
+
+    def __init__(self):
+        self.input_tokens = 0
+        self.output_tokens = 0
+        self.cache_write_tokens = 0
+        self.cache_read_tokens = 0
+
+    @property
+    def total(self) -> int:
+        return (
+            self.input_tokens
+            + self.output_tokens
+            + self.cache_write_tokens
+            + self.cache_read_tokens
+        )
+
+    def add(self, usage) -> dict:
+        turn = {
+            "input_tokens": usage.input_tokens or 0,
+            "output_tokens": usage.output_tokens or 0,
+            "cache_write_tokens": getattr(usage, "cache_creation_input_tokens", 0) or 0,
+            "cache_read_tokens": getattr(usage, "cache_read_input_tokens", 0) or 0,
+        }
+        for field, value in turn.items():
+            setattr(self, field, getattr(self, field) + value)
+        return turn
+
+
+def resolve_api_key():
+    """The quickstarted-specific name wins, so the key can live in a shell
+    without other Anthropic tooling on the machine spending it."""
+    return os.environ.get(KEY_ENV) or os.environ.get(FALLBACK_KEY_ENV) or None
+
+_READ_DOCS_TOOL = {
+    "name": "read_docs",
+    "description": READ_DOCS_DESCRIPTION,
+    "input_schema": {
+        "type": "object",
+        "properties": {
+            "url": {"type": "string", "description": "Absolute http(s) URL of the docs page"}
+        },
+        "required": ["url"],
+    },
+}
+
+
+class ClaudeAgent:
+    name = "claude"
+
+    def __init__(self, model: str = DEFAULT_MODEL, max_tokens: int = 16000):
+        self.model = model
+        self.max_tokens = max_tokens
+        self.name = f"claude:{model}"
+        #: The exact model the API served, which is what a benchmark must cite:
+        #: an alias like "claude-opus-5" can resolve to different builds over
+        #: time, and a pass-rate trend across a silent change is meaningless.
+        self.model_reported = ""
+
+    def run(self, journey: Journey, toolbelt: Toolbelt, deadline: float) -> AgentOutcome:
+        try:
+            import anthropic
+        except ImportError:
+            return AgentOutcome(
+                stop_reason="error",
+                turns=0,
+                detail='anthropic package not installed; pip install "quickstarted[claude]"',
+            )
+
+        api_key = resolve_api_key()
+        try:
+            # max_retries=0: the harness owns retry policy so that every wait
+            # and every transient failure lands in the trace instead of being
+            # silently absorbed by the SDK.
+            client = (
+                anthropic.Anthropic(api_key=api_key, max_retries=0)
+                if api_key
+                else anthropic.Anthropic(max_retries=0)
+            )
+        except TypeError as exc:
+            return AgentOutcome(
+                stop_reason="error",
+                turns=0,
+                detail=f"no Anthropic credentials: set {KEY_ENV} ({exc})",
+            )
+        tools = [
+            {"type": "bash_20250124", "name": "bash"},
+            _READ_DOCS_TOOL,
+        ]
+        kickoff = build_kickoff(journey)
+        messages: list[dict[str, Any]] = [{"role": "user", "content": kickoff}]
+        used = _Usage()
+
+        def outcome(stop_reason: str, turns: int, detail: str = "") -> AgentOutcome:
+            return AgentOutcome(
+                stop_reason,
+                turns,
+                detail,
+                used.input_tokens,
+                used.output_tokens,
+                used.cache_write_tokens,
+                used.cache_read_tokens,
+            )
+
+        def call_with_retries(turn: int):
+            """Returns (response, error_detail). Retries transient upstream faults."""
+            last = ""
+            for attempt in range(1, MAX_ATTEMPTS + 1):
+                try:
+                    return (
+                        # The SDK's overloads require literal-typed tool
+                        # params; ours are built at runtime. Behaviour is
+                        # covered by the live runs, not by these types.
+                        client.messages.create(  # type: ignore[call-overload]
+                            model=self.model,
+                            max_tokens=self.max_tokens,
+                            thinking={"type": "adaptive"},
+                            cache_control={"type": "ephemeral"},
+                            system=SYSTEM,
+                            tools=tools,
+                            messages=messages,
+                        ),
+                        "",
+                    )
+                except TypeError as exc:
+                    # The SDK resolves credentials lazily, at request time.
+                    return None, f"no Anthropic credentials: set {KEY_ENV} ({exc})"
+                except anthropic.APIConnectionError as exc:
+                    last = f"connection error: {exc}"
+                except anthropic.APIStatusError as exc:
+                    last = f"API error {exc.status_code}: {exc.message}"
+                    if exc.status_code not in RETRYABLE_STATUS:
+                        return None, last
+                if attempt == MAX_ATTEMPTS:
+                    break
+                # Exponential backoff with jitter, never past the deadline.
+                delay = min(2.0 ** (attempt - 1), 30.0) * (0.5 + random.random())
+                if time.monotonic() + delay > deadline:
+                    return None, last
+                toolbelt.trace.add(
+                    "api_retry", turn=turn, attempt=attempt,
+                    sleep=round(delay, 2), error=last,
+                )
+                time.sleep(delay)
+            return None, last
+
+        for turn in range(1, journey.budgets.max_turns + 1):
+            if time.monotonic() > deadline:
+                return outcome("timeout", turn - 1)
+            cap = journey.budgets.max_tokens
+            if cap and used.total >= cap:
+                return outcome("token_budget", turn - 1, f"token budget {cap} exhausted")
+
+            response, error = call_with_retries(turn)
+            if response is None:
+                return outcome("error", turn - 1, error)
+
+            self.model_reported = getattr(response, "model", "") or self.model
+            turn_usage = used.add(response.usage)
+            toolbelt.trace.add(
+                "agent_turn", turn=turn, stop_reason=response.stop_reason,
+                model=self.model_reported, **turn_usage
+            )
+
+            if response.stop_reason == "refusal":
+                return outcome("refusal", turn, "model refused")
+
+            messages.append({"role": "assistant", "content": response.content})
+
+            if response.stop_reason == "pause_turn":
+                continue
+
+            tool_uses = [b for b in response.content if b.type == "tool_use"]
+            if not tool_uses:
+                final_text = " ".join(
+                    b.text for b in response.content if b.type == "text"
+                ).strip()
+                toolbelt.trace.add("agent_final", text=final_text[:2000])
+                return outcome("completed", turn, final_text[:500])
+
+            results = []
+            for tu in tool_uses:
+                if tu.name == "bash":
+                    if tu.input.get("restart"):
+                        out = (
+                            "Shell session reset. (Each command already runs "
+                            "in a fresh shell.)"
+                        )
+                    else:
+                        out = toolbelt.bash(tu.input.get("command", ""))
+                elif tu.name == "read_docs":
+                    out = toolbelt.fetch(tu.input.get("url", ""))
+                else:
+                    out = f"Unknown tool: {tu.name}"
+                results.append(
+                    {"type": "tool_result", "tool_use_id": tu.id, "content": out}
+                )
+            messages.append({"role": "user", "content": results})
+
+        return outcome("max_turns", journey.budgets.max_turns)

quickstarted/agents/gemini_agent.py

@@ -0,0 +1,138 @@
+"""Gemini adapter, via the google-genai SDK.
+
+As with the OpenAI adapter there is no default model, and credentials are read
+from QUICKSTARTED_GEMINI_API_KEY before GOOGLE_API_KEY.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+
+from ..journey import Journey
+from .base import AgentOutcome, Toolbelt
+from .prompt import BASH_DESCRIPTION, READ_DOCS_DESCRIPTION, SYSTEM, kickoff
+
+KEY_ENV = "QUICKSTARTED_GEMINI_API_KEY"
+FALLBACK_KEY_ENV = "GOOGLE_API_KEY"
+
+FUNCTIONS = [
+    {
+        "name": "bash",
+        "description": BASH_DESCRIPTION,
+        "parameters": {
+            "type": "OBJECT",
+            "properties": {"command": {"type": "STRING"}},
+            "required": ["command"],
+        },
+    },
+    {
+        "name": "read_docs",
+        "description": READ_DOCS_DESCRIPTION,
+        "parameters": {
+            "type": "OBJECT",
+            "properties": {"url": {"type": "STRING"}},
+            "required": ["url"],
+        },
+    },
+]
+
+
+def resolve_api_key():
+    return os.environ.get(KEY_ENV) or os.environ.get(FALLBACK_KEY_ENV) or None
+
+
+class GeminiAgent:
+    def __init__(self, model: str = ""):
+        self.model = model
+        self.name = f"gemini:{model}" if model else "gemini"
+        self.model_reported = ""
+
+    def run(self, journey: Journey, toolbelt: Toolbelt, deadline: float) -> AgentOutcome:
+        if not self.model:
+            return AgentOutcome(
+                "error", 0, "gemini adapter requires --model (no default is assumed)"
+            )
+        try:
+            from google import genai
+            from google.genai import types
+        except ImportError:
+            return AgentOutcome(
+                "error", 0,
+                'google-genai not installed; pip install "quickstarted[gemini]"',
+            )
+        api_key = resolve_api_key()
+        if not api_key:
+            return AgentOutcome("error", 0, f"no Gemini credentials: set {KEY_ENV}")
+
+        client = genai.Client(api_key=api_key)
+        config = types.GenerateContentConfig(
+            system_instruction=SYSTEM,
+            tools=[
+                types.Tool(function_declarations=FUNCTIONS)  # type: ignore[arg-type]
+            ],
+        )
+        contents = [
+            types.Content(role="user", parts=[types.Part(text=kickoff(journey))])
+        ]
+        input_tokens = output_tokens = cached = 0
+
+        def outcome(reason: str, turns: int, detail: str = "") -> AgentOutcome:
+            return AgentOutcome(
+                reason, turns, detail, input_tokens, output_tokens, 0, cached
+            )
+
+        for turn in range(1, journey.budgets.max_turns + 1):
+            if time.monotonic() > deadline:
+                return outcome("timeout", turn - 1)
+            try:
+                response = client.models.generate_content(
+                    model=self.model, contents=contents, config=config
+                )
+            except Exception as exc:
+                return outcome("error", turn - 1, f"API error: {exc}")
+
+            usage = getattr(response, "usage_metadata", None)
+            if usage:
+                prompt = getattr(usage, "prompt_token_count", 0) or 0
+                hit = getattr(usage, "cached_content_token_count", 0) or 0
+                # As with OpenAI, the prompt count includes cached tokens.
+                input_tokens += max(prompt - hit, 0)
+                cached += hit
+                output_tokens += getattr(usage, "candidates_token_count", 0) or 0
+            self.model_reported = getattr(response, "model_version", "") or self.model
+            toolbelt.trace.add("agent_turn", turn=turn, model=self.model_reported)
+
+            candidates = response.candidates or []
+            candidate = candidates[0] if candidates else None
+            if candidate is None or not candidate.content:
+                return outcome("error", turn, "empty response from model")
+            contents.append(candidate.content)
+
+            calls = [
+                p.function_call
+                for p in (candidate.content.parts or [])
+                if p.function_call
+            ]
+            if not calls:
+                final = (getattr(response, "text", "") or "").strip()
+                toolbelt.trace.add("agent_final", text=final[:2000])
+                return outcome("completed", turn, final[:500])
+
+            replies = []
+            for call in calls:
+                arguments = dict(call.args or {})
+                if call.name == "bash":
+                    result = toolbelt.bash(arguments.get("command", ""))
+                elif call.name == "read_docs":
+                    result = toolbelt.fetch(arguments.get("url", ""))
+                else:
+                    result = f"Unknown tool: {call.name}"
+                replies.append(
+                    types.Part.from_function_response(
+                        name=call.name or "unknown", response={"result": result}
+                    )
+                )
+            contents.append(types.Content(role="user", parts=replies))
+
+        return outcome("max_turns", journey.budgets.max_turns)