quickstarted 0.2.0__py3-none-any.whl

quickstarted/agents/openai_agent.py

@@ -0,0 +1,167 @@
+"""OpenAI adapter: the same tool loop, the same harness-owned tools.
+
+There is deliberately no default model. A benchmark that silently picks a
+model for you produces numbers nobody can reproduce, and vendor model names
+change faster than a pinned default would survive. Pass `--model`.
+
+Credentials come from QUICKSTARTED_OPENAI_API_KEY first, falling back to
+OPENAI_API_KEY, so a key can live in a shell without other tooling spending
+it.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from typing import Any
+
+from ..journey import Journey
+from .base import AgentOutcome, Toolbelt
+from .prompt import BASH_DESCRIPTION, READ_DOCS_DESCRIPTION, SYSTEM, kickoff
+
+KEY_ENV = "QUICKSTARTED_OPENAI_API_KEY"
+FALLBACK_KEY_ENV = "OPENAI_API_KEY"
+
+TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "bash",
+            "description": BASH_DESCRIPTION,
+            "parameters": {
+                "type": "object",
+                "properties": {"command": {"type": "string"}},
+                "required": ["command"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "read_docs",
+            "description": READ_DOCS_DESCRIPTION,
+            "parameters": {
+                "type": "object",
+                "properties": {"url": {"type": "string"}},
+                "required": ["url"],
+            },
+        },
+    },
+]
+
+
+def resolve_api_key():
+    return os.environ.get(KEY_ENV) or os.environ.get(FALLBACK_KEY_ENV) or None
+
+
+class OpenAIAgent:
+    def __init__(self, model: str = "", max_tokens: int = 16000):
+        self.model = model
+        self.max_tokens = max_tokens
+        self.name = f"openai:{model}" if model else "openai"
+        self.model_reported = ""
+
+    def run(self, journey: Journey, toolbelt: Toolbelt, deadline: float) -> AgentOutcome:
+        if not self.model:
+            return AgentOutcome(
+                "error", 0, "openai adapter requires --model (no default is assumed)"
+            )
+        try:
+            import openai
+        except ImportError:
+            return AgentOutcome(
+                "error", 0,
+                'openai package not installed; pip install "quickstarted[openai]"',
+            )
+        api_key = resolve_api_key()
+        if not api_key:
+            return AgentOutcome("error", 0, f"no OpenAI credentials: set {KEY_ENV}")
+        client = openai.OpenAI(api_key=api_key, max_retries=0)
+
+        messages: list[dict[str, Any]] = [
+            {"role": "system", "content": SYSTEM},
+            {"role": "user", "content": kickoff(journey)},
+        ]
+        input_tokens = output_tokens = cached = 0
+
+        def outcome(reason: str, turns: int, detail: str = "") -> AgentOutcome:
+            return AgentOutcome(
+                reason, turns, detail, input_tokens, output_tokens, 0, cached
+            )
+
+        for turn in range(1, journey.budgets.max_turns + 1):
+            if time.monotonic() > deadline:
+                return outcome("timeout", turn - 1)
+            try:
+                response = client.chat.completions.create(
+                    model=self.model,
+                    # The SDK types these as TypedDicts; ours are built at
+                    # runtime and shared across adapters. Behaviour is covered
+                    # by live runs, not by these annotations.
+                    messages=messages,  # type: ignore[arg-type]
+                    tools=TOOLS,  # type: ignore[arg-type]
+                    max_completion_tokens=self.max_tokens,
+                )
+            except Exception as exc:
+                return outcome("error", turn - 1, f"API error: {exc}")
+
+            usage = getattr(response, "usage", None)
+            if usage:
+                prompt = getattr(usage, "prompt_tokens", 0) or 0
+                details = getattr(usage, "prompt_tokens_details", None)
+                hit = (getattr(details, "cached_tokens", 0) or 0) if details else 0
+                # prompt_tokens includes the cached ones here, unlike Anthropic.
+                # Subtract so the counters do not double count the same tokens.
+                input_tokens += max(prompt - hit, 0)
+                cached += hit
+                output_tokens += getattr(usage, "completion_tokens", 0) or 0
+            self.model_reported = getattr(response, "model", "") or self.model
+
+            choice = response.choices[0]
+            message = choice.message
+            toolbelt.trace.add(
+                "agent_turn", turn=turn, stop_reason=choice.finish_reason,
+                model=self.model_reported,
+            )
+            calls = list(getattr(message, "tool_calls", None) or [])
+            messages.append(
+                {
+                    "role": "assistant",
+                    "content": message.content or "",
+                    "tool_calls": [
+                        {
+                            "id": call.id,
+                            "type": "function",
+                            "function": {
+                                "name": call.function.name,
+                                "arguments": call.function.arguments,
+                            },
+                        }
+                        for call in calls
+                    ],
+                }
+                if calls
+                else {"role": "assistant", "content": message.content or ""}
+            )
+            if not calls:
+                final = (message.content or "").strip()
+                toolbelt.trace.add("agent_final", text=final[:2000])
+                return outcome("completed", turn, final[:500])
+
+            for call in calls:
+                try:
+                    arguments = json.loads(call.function.arguments or "{}")
+                except ValueError:
+                    arguments = {}
+                if call.function.name == "bash":
+                    result = toolbelt.bash(arguments.get("command", ""))
+                elif call.function.name == "read_docs":
+                    result = toolbelt.fetch(arguments.get("url", ""))
+                else:
+                    result = f"Unknown tool: {call.function.name}"
+                messages.append(
+                    {"role": "tool", "tool_call_id": call.id, "content": result}
+                )
+
+        return outcome("max_turns", journey.budgets.max_turns)

quickstarted/agents/prompt.py

@@ -0,0 +1,61 @@
+"""The prompt every adapter shares.
+
+Kept in one place deliberately. If each vendor adapter phrased the task its
+own way, a cross-model pass-rate comparison would be measuring the prompts as
+much as the documentation, and the benchmark would mean nothing.
+"""
+
+from __future__ import annotations
+
+from ..journey import Journey
+
+SYSTEM = """You are a developer trying out an unfamiliar software project by \
+following its documentation, inside a fresh throwaway workspace (your current \
+directory). Your goal is stated in the first user message.
+
+Rules of the exercise:
+- Your ONLY source of information about the project is its documentation, \
+read via the read_docs tool. Do not rely on prior knowledge of this specific \
+project; the point is to test whether the docs alone get a newcomer to the \
+goal. General programming knowledge is fine.
+- Start by reading the documentation entrypoint, then follow links from it \
+with read_docs when you need more.
+- Use the bash tool to run commands in the workspace. Each command runs in a \
+fresh shell in the workspace directory; state persists only on disk (use \
+files, virtualenvs, etc.).
+- The shell cannot reach documentation websites. That is deliberate: read \
+documentation with read_docs, which records what you read.
+- When you believe the goal is achieved, verify it yourself with a command, \
+then stop and summarize what you did in one short paragraph. Do not keep \
+polishing after the goal is met.
+- If you are genuinely stuck because the documentation is missing or wrong, \
+stop and say exactly where the docs failed you: which page, which step.
+"""
+
+READ_DOCS_DESCRIPTION = (
+    "Read a documentation page. Call this whenever you need information about "
+    "the target project: before your first command, and again whenever the "
+    "docs you have already read do not answer the question at hand. Only "
+    "documentation hosts on this journey's allowlist are reachable; other "
+    "URLs return BLOCKED."
+)
+
+BASH_DESCRIPTION = (
+    "Run a shell command in the workspace. Each call runs in a fresh shell "
+    "whose working directory is the workspace; state persists only on disk."
+)
+
+
+def kickoff(journey: Journey) -> str:
+    text = (
+        f"Goal: {journey.goal}\n\n"
+        f"Documentation entrypoint: {journey.docs_entrypoint}\n"
+        f"Allowed documentation hosts: {', '.join(journey.docs_allow)}"
+    )
+    if journey.setup:
+        # Without this the agent cannot tell a prepared workspace from an empty
+        # one, and may rebuild state that setup already created.
+        text += "\n\nThe workspace was already prepared by running:\n" + "\n".join(
+            f"  $ {cmd}" for cmd in journey.setup
+        )
+    return text

quickstarted/agents/registry.py

@@ -0,0 +1,31 @@
+"""Agent adapter registry.
+
+Adapters exist for more than one vendor for a reason that is not marketing:
+a benchmark run against a single model measures that model's habits as much
+as the documentation. "Agents cannot follow your quickstart" is only a
+defensible sentence when several of them could not.
+"""
+
+from __future__ import annotations
+
+AGENTS = ("replay", "claude", "openai", "gemini")
+
+
+def build_agent(name: str, model: str = ""):
+    if name == "replay":
+        from .replay import ReplayAgent
+
+        return ReplayAgent()
+    if name == "claude":
+        from .claude import DEFAULT_MODEL, ClaudeAgent
+
+        return ClaudeAgent(model=model or DEFAULT_MODEL)
+    if name == "openai":
+        from .openai_agent import OpenAIAgent
+
+        return OpenAIAgent(model=model)
+    if name == "gemini":
+        from .gemini_agent import GeminiAgent
+
+        return GeminiAgent(model=model)
+    raise SystemExit(f"unknown agent {name!r} (choose from: {', '.join(AGENTS)})")

quickstarted/agents/replay.py

@@ -0,0 +1,39 @@
+"""Replay agent: runs the journey's documented commands literally, no LLM.
+
+This is the free CI mode. If the commands your docs tell users to type do not
+work verbatim, no agent (and no human) stands a chance; replay catches that on
+every docs change without spending a token. It stops at the first failing
+command, which is exactly the "docs break at step N" signal.
+"""
+
+from __future__ import annotations
+
+import time
+
+from ..journey import Journey
+from .base import AgentOutcome, Toolbelt
+
+
+class ReplayAgent:
+    name = "replay"
+
+    def run(self, journey: Journey, toolbelt: Toolbelt, deadline: float) -> AgentOutcome:
+        if not journey.replay:
+            return AgentOutcome(
+                stop_reason="error",
+                turns=0,
+                detail="journey has no 'replay' commands; replay mode needs them",
+            )
+        toolbelt.fetch(journey.docs_entrypoint)
+        for i, command in enumerate(journey.replay, start=1):
+            if time.monotonic() > deadline:
+                return AgentOutcome(stop_reason="timeout", turns=i - 1)
+            result = toolbelt.bash(command)
+            first_line = result.splitlines()[0] if result else ""
+            if not first_line.startswith("exit code: 0"):
+                return AgentOutcome(
+                    stop_reason="command_failed",
+                    turns=i,
+                    detail=f"replay step {i} failed: {command}",
+                )
+        return AgentOutcome(stop_reason="completed", turns=len(journey.replay))

quickstarted/cli.py

@@ -0,0 +1,250 @@
+"""quickstarted CLI.
+
+    quickstarted validate journeys/*.yaml
+    quickstarted run journeys/foo.yaml --agent replay
+    quickstarted run journeys/*.yaml --agent claude --repeat 5 --out results/
+    quickstarted doctor
+
+Exit code 0 when every journey passed every evidential attempt, 1 otherwise,
+so CI can gate on it. Runs that produced no evidence (rate limits, budget
+exhaustion, harness bugs) do not silently turn into failures; use
+`--strict-inconclusive` if you would rather they did.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from .agents.registry import AGENTS, build_agent
+from .docs import AFFORDANCE_POLICIES, DocsClient
+from .exec import available_backends, resolve_backend
+from .journey import JourneyError, load_journey
+from .pricing import PriceBook
+from .report import console_summary, markdown_report, markdown_suite_report, suite_summary
+from .results import write_json, write_junit
+from .run import EVIDENTIAL
+from .suite import run_suite
+
+
+def cmd_validate(args) -> int:
+    failures = 0
+    for path in args.journeys:
+        try:
+            journey = load_journey(path)
+        except JourneyError as exc:
+            print(f"INVALID  {exc}")
+            failures += 1
+        else:
+            modes = "replay+agent" if journey.replay else "agent-only"
+            print(f"ok       {path} ({journey.name}, {modes})")
+            for host in journey.network_conflicts:
+                print(
+                    f"warning  {host} is declared a docs host, so the shell "
+                    f"cannot reach it; installs that need it will fail. "
+                    f"Add it under network.allow if that is intended."
+                )
+            for host in journey.attribution_gaps:
+                print(
+                    f"note     {host} is both a docs host and network-allowed, "
+                    f"so pages the shell reads there are not recorded."
+                )
+    return 1 if failures else 0
+
+
+def cmd_doctor(args) -> int:
+    """What this machine can actually enforce, before you trust its numbers."""
+    backends = available_backends()
+    chosen = resolve_backend("auto")
+    print("quickstarted doctor")
+    print(f"  backends available: {', '.join(backends)}")
+    print(f"  auto would choose:  {chosen}")
+    if chosen == "local":
+        print()
+        print("  WARNING: no enforced backend on this machine.")
+        print("  Commands would run as you, on your filesystem, with your network,")
+        print("  and an agent could read docs pages without the trace recording it.")
+        print("  Fine for journeys you wrote; do not benchmark other people's")
+        print("  projects this way. Install Docker, or run on macOS.")
+    prices = PriceBook.load(args.prices)
+    print(f"  price book loaded:  {'yes' if prices else 'no (token counts only)'}")
+    from .agents.claude import KEY_ENV, resolve_api_key
+
+    print(f"  {KEY_ENV}: {'set' if resolve_api_key() else 'not set'}")
+    return 0
+
+
+def cmd_run(args) -> int:
+    journeys = []
+    invalid = False
+    for path in args.journeys:
+        try:
+            journeys.append(load_journey(path))
+        except JourneyError as exc:
+            print(f"INVALID  {exc}")
+            invalid = True
+    if not journeys:
+        return 1
+
+    backend = resolve_backend(args.backend)
+    if backend == "local" and not args.allow_unenforced:
+        print(
+            "REFUSING: no enforced execution backend is available, so the docs "
+            "allowlist and the page-read record cannot be guaranteed.\n"
+            "Run `quickstarted doctor` for details, or pass --allow-unenforced if "
+            "you wrote the journeys and the project under test yourself.",
+            file=sys.stderr,
+        )
+        return 1
+
+    prices = PriceBook.load(args.prices)
+    out_root = Path(args.out) if args.out else None
+
+    def emit(result) -> None:
+        print(console_summary(result))
+        if args.keep_sandbox:
+            print(f"  sandbox kept at: {result.sandbox_path}")
+        if out_root:
+            out_dir = out_root / result.journey.name
+            if result.attempt > 1:
+                out_dir = out_dir / f"attempt-{result.attempt}"
+            out_dir.mkdir(parents=True, exist_ok=True)
+            result.trace.write_jsonl(out_dir / "trace.jsonl")
+            (out_dir / "report.md").write_text(
+                markdown_report(result), encoding="utf-8"
+            )
+
+    docs = DocsClient(
+        cache_dir=args.cache_dir or None,
+        rate_limit_seconds=args.rate_limit,
+        respect_robots=not args.ignore_robots,
+        affordances=args.affordances,
+        refresh=args.refresh,
+        offline=args.offline,
+    )
+
+    suite = run_suite(
+        journeys,
+        agent_factory=lambda: build_agent(args.agent, args.model),
+        repeat=args.repeat,
+        workers=args.workers,
+        backend=args.backend,
+        keep_sandbox=args.keep_sandbox,
+        image=args.image or None,
+        docs=docs,
+        probe_affordances=args.probe_affordances,
+        on_result=emit,
+    )
+
+    print(suite_summary(suite, prices))
+
+    if out_root:
+        out_root.mkdir(parents=True, exist_ok=True)
+        write_json(suite, out_root / "results.json", prices)
+        (out_root / "suite.md").write_text(
+            markdown_suite_report(suite, prices), encoding="utf-8"
+        )
+        if args.junit:
+            write_junit(suite, args.junit)
+        print(f"  results written to {out_root}/")
+    elif args.junit:
+        write_junit(suite, args.junit)
+
+    if invalid:
+        return 1
+    inconclusive = [r for r in suite.runs if r.classification not in EVIDENTIAL]
+    if inconclusive and args.strict_inconclusive:
+        return 1
+    return 0 if suite.all_passed else 1
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="quickstarted",
+        description=(
+            "Test whether an AI agent can complete your quickstart from your docs alone."
+        ),
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_validate = sub.add_parser("validate", help="validate journey files")
+    p_validate.add_argument("journeys", nargs="+")
+    p_validate.set_defaults(func=cmd_validate)
+
+    p_doctor = sub.add_parser("doctor", help="report what this machine can enforce")
+    p_doctor.add_argument("--prices", default="", help="path to a price book JSON file")
+    p_doctor.set_defaults(func=cmd_doctor)
+
+    p_run = sub.add_parser("run", help="run journeys")
+    p_run.add_argument("journeys", nargs="+")
+    p_run.add_argument(
+        "--agent", default="replay",
+        help=f"one of: {', '.join(sorted(AGENTS))} (default: replay)",
+    )
+    p_run.add_argument("--model", default="", help="model override for LLM agents")
+    p_run.add_argument("--out", default="", help="directory for traces and reports")
+    p_run.add_argument("--junit", default="", help="write JUnit XML to this path")
+    p_run.add_argument("--prices", default="", help="path to a price book JSON file")
+    p_run.add_argument(
+        "--repeat", type=int, default=1,
+        help="attempts per journey; >1 turns a verdict into a pass rate",
+    )
+    p_run.add_argument(
+        "--workers", type=int, default=1, help="run this many attempts in parallel"
+    )
+    p_run.add_argument(
+        "--backend", default="auto",
+        help="execution backend: auto (default), docker, seatbelt, local",
+    )
+    p_run.add_argument(
+        "--image", default="", help="container image for the docker backend"
+    )
+    p_run.add_argument(
+        "--allow-unenforced", action="store_true",
+        help="permit the local backend, whose network policy is advisory only",
+    )
+    p_run.add_argument(
+        "--strict-inconclusive", action="store_true",
+        help="exit non-zero when any run produced no evidence",
+    )
+    p_run.add_argument(
+        "--keep-sandbox", action="store_true",
+        help="keep the sandbox directory for inspection",
+    )
+    p_run.add_argument(
+        "--affordances", default="all", choices=list(AFFORDANCE_POLICIES),
+        help=(
+            "which machine-facing files the agent may read. 'none' withholds "
+            "llms.txt and .md variants: run both and compare pass rates to "
+            "measure whether the affordance helps"
+        ),
+    )
+    p_run.add_argument(
+        "--probe-affordances", action="store_true",
+        help="record which machine-facing files exist (context, never scored)",
+    )
+    p_run.add_argument("--cache-dir", default="", help="content-addressed docs cache")
+    p_run.add_argument(
+        "--refresh", action="store_true",
+        help="re-fetch cached pages and flag any whose content changed",
+    )
+    p_run.add_argument(
+        "--offline", action="store_true", help="use the cache only; never fetch"
+    )
+    p_run.add_argument(
+        "--rate-limit", type=float, default=1.0,
+        help="minimum seconds between requests to the same host (default: 1.0)",
+    )
+    p_run.add_argument(
+        "--ignore-robots", action="store_true",
+        help="fetch documentation even where robots.txt disallows it",
+    )
+    p_run.set_defaults(func=cmd_run)
+
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())