quickstarted 0.2.0__py3-none-any.whl

quickstarted/run.py

@@ -0,0 +1,240 @@
+"""Run orchestration: setup, agent, deterministic scoring, classification.
+
+Pass/fail is decided by the journey's success script exit code, run by the
+harness after the agent stops. The agent's own opinion of whether it finished
+is recorded but never trusted for scoring.
+
+A verdict alone is not enough to publish, though. "FAIL" has to distinguish a
+documentation gap from a network flake, an exhausted budget, or a bug in this
+harness, because a benchmark that reports infrastructure noise as a docs
+failure is worse than no benchmark. That is what `Classification` is for.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+
+from .agents.base import Agent, AgentOutcome, Toolbelt
+from .docs import DocsClient, affordance_summary
+from .exec import ExecutorError, make_executor, needs_host_proxy, resolve_backend
+from .journey import Journey
+from .net.proxy import EgressProxy
+from .trace import Trace
+
+#: Only PASS and DOCS_GAP are statements about the documentation. Everything
+#: else says the run did not produce evidence, and belongs outside the
+#: numerator and the denominator of a pass rate.
+PASSED = "passed"
+DOCS_GAP = "docs_gap"
+BUDGET_EXHAUSTED = "budget_exhausted"
+INFRA_ERROR = "infra_error"
+HARNESS_ERROR = "harness_error"
+AGENT_REFUSAL = "agent_refusal"
+
+EVIDENTIAL = (PASSED, DOCS_GAP)
+
+_INFRA_MARKERS = (
+    "connection error",
+    "api error 429",
+    "api error 500",
+    "api error 502",
+    "api error 503",
+    "api error 529",
+    "overloaded",
+    "rate limit",
+)
+
+
+@dataclass(frozen=True)
+class ScoreResult:
+    passed: bool
+    exit_code: int
+    output: str
+
+
+@dataclass
+class RunResult:
+    journey: Journey
+    agent_name: str
+    outcome: AgentOutcome
+    score: ScoreResult | None
+    trace: Trace
+    duration: float
+    sandbox_path: str
+    backend: str = "local"
+    enforced: bool = False
+    classification: str = HARNESS_ERROR
+    attempt: int = 1
+    model_reported: str = ""
+    affordance_policy: str = "all"
+
+    @property
+    def passed(self) -> bool:
+        return bool(self.score and self.score.passed)
+
+    @property
+    def evidential(self) -> bool:
+        """True when this run says something about the documentation."""
+        return self.classification in EVIDENTIAL
+
+    @property
+    def suspect_page(self) -> str | None:
+        if self.passed:
+            return None
+        return self.trace.last_fetch_before_failure()
+
+    @property
+    def bypass_attempts(self) -> int:
+        return len(
+            [
+                e
+                for e in self.trace.of_type("egress_blocked")
+                if e.data.get("reason") == "docs_host_requires_read_docs"
+            ]
+        )
+
+
+def classify(outcome: AgentOutcome, score: ScoreResult | None, trace: Trace) -> str:
+    if score and score.passed:
+        return PASSED
+    reason = outcome.stop_reason
+    detail = (outcome.detail or "").lower()
+    if reason == "refusal":
+        return AGENT_REFUSAL
+    if reason == "error":
+        if any(marker in detail for marker in _INFRA_MARKERS):
+            return INFRA_ERROR
+        return HARNESS_ERROR
+    if reason in ("max_turns", "timeout", "token_budget"):
+        return BUDGET_EXHAUSTED
+    # A run whose shell could not reach a package registry has not tested the
+    # documentation; it has tested the network.
+    if trace.of_type("egress_error"):
+        return INFRA_ERROR
+    blocked = trace.of_type("egress_blocked")
+    # Documented commands that died because our own policy refused their
+    # traffic say the journey's network allowlist is wrong, not the docs.
+    if blocked and reason == "command_failed":
+        return HARNESS_ERROR
+    if any(e.data.get("reason") == "not_allowlisted" for e in blocked):
+        return HARNESS_ERROR
+    return DOCS_GAP
+
+
+def run_journey(
+    journey: Journey,
+    agent: Agent,
+    keep_sandbox: bool = False,
+    http_get=None,
+    backend: str = "auto",
+    image: str | None = None,
+    attempt: int = 1,
+    docs: DocsClient | None = None,
+    probe_affordances: bool = False,
+) -> RunResult:
+    backend = resolve_backend(backend)
+    docs = docs or DocsClient()
+    trace = Trace()
+    proxy: EgressProxy | None = None
+    start = time.monotonic()
+
+    if needs_host_proxy(backend):
+        proxy = EgressProxy(
+            network_allow=journey.network_allow,
+            docs_hosts=journey.docs_allow,
+            explicit_allow=journey.network_explicit,
+            trace=trace,
+        )
+        proxy.start()
+
+    try:
+        executor = make_executor(
+            backend,
+            keep=keep_sandbox,
+            proxy_url=proxy.url if proxy else None,
+            network_allow=journey.network_allow,
+            docs_hosts=journey.docs_allow,
+            image=image,
+            trace=trace,
+        )
+    except ExecutorError as exc:
+        if proxy:
+            proxy.stop()
+        trace.add("run_end", stop_reason="error", passed=False)
+        return RunResult(
+            journey, agent.name, AgentOutcome("error", 0, str(exc)), None, trace,
+            time.monotonic() - start, "", backend, False, HARNESS_ERROR, attempt,
+        )
+
+    trace.add(
+        "run_start",
+        journey=journey.name,
+        agent=agent.name,
+        backend=backend,
+        enforced=executor.enforced,
+        attempt=attempt,
+        affordance_policy=docs.affordances,
+    )
+
+    if probe_affordances:
+        # Recorded as context for whoever reads a failure, and as the variable
+        # for an ablation. Never part of the score.
+        trace.add(
+            "affordances", entrypoint=journey.docs_entrypoint,
+            found=affordance_summary(docs.probe(journey.docs_entrypoint)),
+        )
+
+    def finish(outcome: AgentOutcome, score: ScoreResult | None) -> RunResult:
+        classification = classify(outcome, score, trace)
+        trace.add(
+            "run_end",
+            stop_reason=outcome.stop_reason,
+            passed=bool(score and score.passed),
+            classification=classification,
+        )
+        return RunResult(
+            journey, agent.name, outcome, score, trace, time.monotonic() - start,
+            str(executor.root), backend, executor.enforced, classification, attempt,
+            getattr(agent, "model_reported", ""), docs.affordances,
+        )
+
+    try:
+        for command in journey.setup:
+            result = executor.run(
+                command,
+                timeout=journey.budgets.max_command_seconds,
+                max_output_chars=journey.budgets.max_output_chars,
+            )
+            trace.add(
+                "setup", command=command, exit_code=result.exit_code,
+                output=result.output,
+            )
+            if result.exit_code != 0:
+                return finish(
+                    AgentOutcome("error", 0, f"setup command failed: {command}"), None
+                )
+
+        toolbelt = Toolbelt(journey, executor, trace, docs=docs, http_get=http_get)
+        deadline = time.monotonic() + journey.budgets.max_seconds
+        outcome = agent.run(journey, toolbelt, deadline)
+
+        check = executor.run(
+            journey.success_script,
+            timeout=journey.budgets.max_command_seconds,
+            max_output_chars=journey.budgets.max_output_chars,
+        )
+        score = ScoreResult(
+            passed=check.exit_code == 0,
+            exit_code=check.exit_code,
+            output=check.output,
+        )
+        trace.add(
+            "success_check", exit_code=check.exit_code, passed=score.passed,
+            output=check.output,
+        )
+        return finish(outcome, score)
+    finally:
+        executor.cleanup()
+        if proxy:
+            proxy.stop()

quickstarted/sandbox.py

@@ -0,0 +1,15 @@
+"""Backwards-compatible aliases for the pre-executor API.
+
+`Sandbox` was the only execution backend in v0. It is now `LocalExecutor`, one
+of three, and the least safe: see `quickstarted.exec`.
+"""
+
+from __future__ import annotations
+
+from .exec.base import CommandResult, truncate
+from .exec.local import LocalExecutor
+
+#: Deprecated alias. Prefer `quickstarted.exec.make_executor`.
+Sandbox = LocalExecutor
+
+__all__ = ["CommandResult", "LocalExecutor", "Sandbox", "truncate"]

quickstarted/suite.py

@@ -0,0 +1,203 @@
+"""Running many journeys many times, and reporting rates rather than verdicts.
+
+A single agent run is a sample, not a measurement. The same docs and the same
+model can pass at 10:00 and fail at 10:05, so a benchmark that publishes one
+run per project is publishing noise with a confident face on it.
+
+Two rules follow, and both are enforced here rather than left to the reader:
+
+* A pass *rate* is computed over evidential runs only. Runs that died on a
+  429, exhausted their budget, or hit a harness bug are excluded from the
+  numerator and the denominator alike, and reported separately, because they
+  are not evidence about the documentation either way.
+* Nothing is aggregated across models. Pass rates for different models are
+  different measurements that happen to share a journey.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Sequence
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from typing import Callable
+
+from .agents.base import Agent
+from .exec import resolve_backend
+from .journey import Journey
+from .pricing import PriceBook
+from .run import EVIDENTIAL, PASSED, RunResult, run_journey
+
+
+@dataclass
+class JourneyStats:
+    journey: str
+    agent: str
+    runs: list[RunResult] = field(default_factory=list)
+
+    @property
+    def attempts(self) -> int:
+        return len(self.runs)
+
+    @property
+    def evidential(self) -> list[RunResult]:
+        return [r for r in self.runs if r.classification in EVIDENTIAL]
+
+    @property
+    def passes(self) -> int:
+        return len([r for r in self.runs if r.classification == PASSED])
+
+    @property
+    def pass_rate(self) -> float | None:
+        """None when no run produced evidence: an honest 'we do not know'."""
+        usable = self.evidential
+        if not usable:
+            return None
+        return self.passes / len(usable)
+
+    @property
+    def discarded(self) -> dict[str, int]:
+        counts: dict[str, int] = {}
+        for run in self.runs:
+            if run.classification not in EVIDENTIAL:
+                counts[run.classification] = counts.get(run.classification, 0) + 1
+        return counts
+
+    @property
+    def models_seen(self) -> list[str]:
+        return sorted({r.model_reported for r in self.runs if r.model_reported})
+
+    def tokens(self) -> dict[str, int]:
+        totals = {"input": 0, "output": 0, "cache_write": 0, "cache_read": 0}
+        for run in self.runs:
+            totals["input"] += run.outcome.input_tokens
+            totals["output"] += run.outcome.output_tokens
+            totals["cache_write"] += run.outcome.cache_write_tokens
+            totals["cache_read"] += run.outcome.cache_read_tokens
+        return totals
+
+    def cost(self, prices: PriceBook) -> float | None:
+        if not prices:
+            return None
+        total = 0.0
+        seen = False
+        for run in self.runs:
+            model = run.model_reported or run.agent_name
+            estimate = prices.estimate(model, run.outcome)
+            if estimate is not None:
+                total += estimate
+                seen = True
+        return total if seen else None
+
+    @property
+    def suspect_pages(self) -> dict[str, int]:
+        """Which docs page failing runs were last on, most common first."""
+        counts: dict[str, int] = {}
+        for run in self.runs:
+            if run.classification == PASSED:
+                continue
+            page = run.suspect_page
+            if page:
+                counts[page] = counts.get(page, 0) + 1
+        return dict(sorted(counts.items(), key=lambda kv: -kv[1]))
+
+
+@dataclass
+class SuiteResult:
+    stats: list[JourneyStats] = field(default_factory=list)
+    duration: float = 0.0
+    repeat: int = 1
+    backend: str = ""
+
+    @property
+    def runs(self) -> list[RunResult]:
+        return [r for s in self.stats for r in s.runs]
+
+    @property
+    def all_passed(self) -> bool:
+        """CI gate: every journey passed every attempt that produced evidence."""
+        for stat in self.stats:
+            if stat.pass_rate is None or stat.pass_rate < 1.0:
+                return False
+        return bool(self.stats)
+
+    def tokens(self) -> dict[str, int]:
+        totals = {"input": 0, "output": 0, "cache_write": 0, "cache_read": 0}
+        for stat in self.stats:
+            for key, value in stat.tokens().items():
+                totals[key] += value
+        return totals
+
+    def cost(self, prices: PriceBook) -> float | None:
+        if not prices:
+            return None
+        estimates = [s.cost(prices) for s in self.stats]
+        real = [e for e in estimates if e is not None]
+        return sum(real) if real else None
+
+
+def run_suite(
+    journeys: Sequence[Journey],
+    agent_factory: Callable[[], Agent],
+    repeat: int = 1,
+    workers: int = 1,
+    backend: str = "auto",
+    keep_sandbox: bool = False,
+    http_get=None,
+    image: str | None = None,
+    docs=None,
+    probe_affordances: bool = False,
+    on_result: Callable[[RunResult], None] | None = None,
+) -> SuiteResult:
+    start = time.monotonic()
+    jobs = [
+        (journey, attempt)
+        for journey in journeys
+        for attempt in range(1, repeat + 1)
+    ]
+
+    def execute(job) -> RunResult:
+        journey, attempt = job
+        return run_journey(
+            journey,
+            agent_factory(),
+            keep_sandbox=keep_sandbox,
+            http_get=http_get,
+            backend=backend,
+            image=image,
+            attempt=attempt,
+            docs=docs,
+            probe_affordances=probe_affordances and attempt == 1,
+        )
+
+    results: list[RunResult] = []
+    if workers > 1 and len(jobs) > 1:
+        with ThreadPoolExecutor(max_workers=workers) as pool:
+            for result in pool.map(execute, jobs):
+                results.append(result)
+                if on_result:
+                    on_result(result)
+    else:
+        for job in jobs:
+            result = execute(job)
+            results.append(result)
+            if on_result:
+                on_result(result)
+
+    by_journey: dict[str, JourneyStats] = {}
+    order: list[str] = []
+    for result in results:
+        key = f"{result.journey.name} {result.agent_name}"
+        if key not in by_journey:
+            by_journey[key] = JourneyStats(result.journey.name, result.agent_name)
+            order.append(key)
+        by_journey[key].runs.append(result)
+
+    return SuiteResult(
+        stats=[by_journey[k] for k in order],
+        duration=time.monotonic() - start,
+        repeat=repeat,
+        # The resolved backend, not the request: "auto" in a published result
+        # tells a reader nothing about what was enforced.
+        backend=resolve_backend(backend),
+    )

quickstarted/trace.py

@@ -0,0 +1,53 @@
+"""Run traces: an append-only event log, written as JSONL.
+
+The trace is the product's raw material: it is what lets a failure be
+attributed to the docs page the agent was reading when things went wrong.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Event:
+    ts: float
+    type: str
+    data: dict[str, Any]
+
+
+@dataclass
+class Trace:
+    events: list[Event] = field(default_factory=list)
+    # The egress proxy appends from its own threads while the agent loop
+    # appends from the main one.
+    _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
+
+    def add(self, type: str, **data: Any) -> Event:
+        event = Event(ts=time.time(), type=type, data=data)
+        with self._lock:
+            self.events.append(event)
+        return event
+
+    def of_type(self, *types: str) -> list[Event]:
+        with self._lock:
+            return [e for e in self.events if e.type in types]
+
+    def fetched_urls(self) -> list[str]:
+        return [e.data["url"] for e in self.events if e.type == "docs_fetch"]
+
+    def last_fetch_before_failure(self) -> str | None:
+        urls = self.fetched_urls()
+        return urls[-1] if urls else None
+
+    def write_jsonl(self, path: str | Path) -> None:
+        path = Path(path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with path.open("w", encoding="utf-8") as fh:
+            for e in self.events:
+                fh.write(json.dumps({"ts": e.ts, "type": e.type, **e.data}) + "\n")

quickstarted/transport.py

@@ -0,0 +1,88 @@
+"""Low-level HTTP for documentation reads, and HTML to text.
+
+Separated from the docs client so that caching, rate limiting, robots, and
+affordance policy can be tested without a network, and so tests have one
+obvious place to intercept.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import gzip
+import io
+import re
+import urllib.request
+from dataclasses import dataclass
+from html.parser import HTMLParser
+from typing import ClassVar
+
+from ._version import __version__ as _version
+
+USER_AGENT = (
+    f"quickstarted/{_version} (+https://github.com/snehankekre/quickstarted; "
+    "documentation agent-readiness testing)"
+)
+MAX_BODY_BYTES = 4_000_000
+
+
+@dataclass(frozen=True)
+class HttpResponse:
+    status: int
+    content_type: str
+    text: str
+    url: str = ""
+
+
+class _TextExtractor(HTMLParser):
+    _SKIP: ClassVar[frozenset] = frozenset({"script", "style", "noscript", "svg"})
+
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self._chunks: list[str] = []
+        self._skip_depth = 0
+
+    def handle_starttag(self, tag, attrs):
+        if tag in self._SKIP:
+            self._skip_depth += 1
+
+    def handle_endtag(self, tag):
+        if tag in self._SKIP and self._skip_depth:
+            self._skip_depth -= 1
+
+    def handle_data(self, data):
+        if not self._skip_depth:
+            self._chunks.append(data)
+
+    def text(self) -> str:
+        raw = "".join(self._chunks)
+        return re.sub(r"\n{3,}", "\n\n", re.sub(r"[ \t]+", " ", raw)).strip()
+
+
+def html_to_text(html: str) -> str:
+    parser = _TextExtractor()
+    try:
+        parser.feed(html)
+        return parser.text()
+    except Exception:
+        return html
+
+
+def http_get(url: str, timeout: int = 30, method: str = "GET") -> HttpResponse:
+    """Fetch a URL. Raises the urllib.error family on transport failure."""
+    request = urllib.request.Request(
+        url,
+        headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"},
+        method=method,
+    )
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        content_type = response.headers.get("Content-Type", "")
+        body = response.read(MAX_BODY_BYTES) if method != "HEAD" else b""
+        if response.headers.get("Content-Encoding") == "gzip" and body:
+            with contextlib.suppress(OSError):
+                body = gzip.GzipFile(fileobj=io.BytesIO(body)).read()
+        return HttpResponse(
+            status=getattr(response, "status", 200),
+            content_type=content_type,
+            text=body.decode("utf-8", errors="replace"),
+            url=response.geturl(),
+        )