quickstarted 0.2.0__py3-none-any.whl

quickstarted/pricing.py

@@ -0,0 +1,93 @@
+"""Optional cost estimation.
+
+There are deliberately no built-in prices. Vendor rates change, a stale table
+baked into a benchmarking tool would quietly misreport what a run costs, and a
+wrong number here is worse than no number. Supply rates yourself:
+
+    QUICKSTARTED_PRICES=/path/to/prices.json
+
+    {
+      "claude-opus-5": {
+        "input": 5.0, "output": 25.0, "cache_write": 6.25, "cache_read": 0.5
+      }
+    }
+
+Units are US dollars per million tokens, matching how vendors publish them.
+Token counts are always reported; dollars appear only when you provide rates.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+PRICES_ENV = "QUICKSTARTED_PRICES"
+_PER_MILLION = 1_000_000.0
+
+
+@dataclass(frozen=True)
+class ModelPrice:
+    input: float = 0.0
+    output: float = 0.0
+    cache_write: float = 0.0
+    cache_read: float = 0.0
+
+    def cost(
+        self,
+        input_tokens: int,
+        output_tokens: int,
+        cache_write_tokens: int = 0,
+        cache_read_tokens: int = 0,
+    ) -> float:
+        return (
+            input_tokens * self.input
+            + output_tokens * self.output
+            + cache_write_tokens * self.cache_write
+            + cache_read_tokens * self.cache_read
+        ) / _PER_MILLION
+
+
+class PriceBook:
+    def __init__(self, prices: dict[str, ModelPrice] | None = None):
+        self.prices = prices or {}
+
+    def __bool__(self) -> bool:
+        return bool(self.prices)
+
+    def for_model(self, model: str) -> ModelPrice | None:
+        if model in self.prices:
+            return self.prices[model]
+        # Adapters report names like "claude:claude-opus-5"; match the tail.
+        tail = model.rsplit(":", 1)[-1]
+        return self.prices.get(tail)
+
+    def estimate(self, model: str, outcome) -> float | None:
+        price = self.for_model(model)
+        if price is None:
+            return None
+        return price.cost(
+            outcome.input_tokens,
+            outcome.output_tokens,
+            outcome.cache_write_tokens,
+            outcome.cache_read_tokens,
+        )
+
+    @classmethod
+    def load(cls, path: str | None = None) -> PriceBook:
+        path = path or os.environ.get(PRICES_ENV)
+        if not path:
+            return cls()
+        data = json.loads(Path(path).read_text(encoding="utf-8"))
+        prices = {}
+        for model, fields in data.items():
+            if not isinstance(fields, dict):
+                raise ValueError(f"price entry for {model!r} must be a mapping")
+            prices[model] = ModelPrice(
+                input=float(fields.get("input", 0.0)),
+                output=float(fields.get("output", 0.0)),
+                cache_write=float(fields.get("cache_write", 0.0)),
+                cache_read=float(fields.get("cache_read", 0.0)),
+            )
+        return cls(prices)

quickstarted/report.py

@@ -0,0 +1,206 @@
+"""Human-readable reporting: console summaries and markdown.
+
+Reports name the classification, not just PASS/FAIL, because "we could not
+reach PyPI" and "your quickstart is missing a step" require different people
+to do different things.
+"""
+
+from __future__ import annotations
+
+from .pricing import PriceBook
+from .run import DOCS_GAP, PASSED, RunResult
+from .suite import JourneyStats, SuiteResult
+
+_LABELS = {
+    PASSED: "PASS",
+    DOCS_GAP: "FAIL",
+    "budget_exhausted": "INCONCLUSIVE",
+    "infra_error": "INCONCLUSIVE",
+    "harness_error": "INCONCLUSIVE",
+    "agent_refusal": "INCONCLUSIVE",
+}
+
+
+def label(run: RunResult) -> str:
+    return _LABELS.get(run.classification, "INCONCLUSIVE")
+
+
+def _token_line(outcome) -> str:
+    """Tokens for one run, cache traffic included so the number reflects cost."""
+    parts = []
+    if outcome.input_tokens or outcome.output_tokens:
+        parts.append(f"{outcome.input_tokens} in / {outcome.output_tokens} out")
+    if outcome.cache_write_tokens or outcome.cache_read_tokens:
+        parts.append(
+            f"cache {outcome.cache_write_tokens} written / "
+            f"{outcome.cache_read_tokens} read"
+        )
+    return ", ".join(parts)
+
+
+def console_summary(result: RunResult) -> str:
+    lines = [
+        f"[{label(result)}] {result.journey.name} ({result.agent_name})",
+        f"  classification: {result.classification}",
+        f"  stop reason: {result.outcome.stop_reason}"
+        + (f" ({result.outcome.detail})" if result.outcome.detail else ""),
+        f"  turns: {result.outcome.turns}, duration: {result.duration:.1f}s",
+        f"  backend: {result.backend}"
+        + ("" if result.enforced else "  [UNENFORCED: policy is advisory here]"),
+    ]
+    tokens = _token_line(result.outcome)
+    if tokens:
+        lines.append(f"  tokens: {tokens}")
+    fetched = result.trace.fetched_urls()
+    if fetched:
+        lines.append(f"  docs pages read: {len(fetched)}")
+    if result.bypass_attempts:
+        lines.append(
+            f"  docs fetched outside read_docs: {result.bypass_attempts} blocked attempt(s)"
+        )
+    if not result.passed:
+        if result.score is not None:
+            check_line = (result.score.output or "").strip().splitlines()
+            lines.append(
+                f"  success check exit code: {result.score.exit_code}"
+                + (f" ({check_line[-1]})" if check_line else "")
+            )
+        if result.classification == DOCS_GAP and result.suspect_page:
+            lines.append(f"  last docs page read before failure: {result.suspect_page}")
+    return "\n".join(lines)
+
+
+def suite_summary(suite: SuiteResult, prices: PriceBook | None = None) -> str:
+    prices = prices or PriceBook()
+    lines = ["", "=" * 62, f"Suite: {len(suite.stats)} journey(s), repeat={suite.repeat}"]
+    for stat in suite.stats:
+        rate = stat.pass_rate
+        shown = "no evidence" if rate is None else f"{rate:.0%}"
+        lines.append(
+            f"  {stat.journey} ({stat.agent}): pass rate {shown} "
+            f"[{stat.passes}/{len(stat.evidential)} evidential of {stat.attempts} run(s)]"
+        )
+        if stat.discarded:
+            detail = ", ".join(f"{k}={v}" for k, v in sorted(stat.discarded.items()))
+            lines.append(f"      discarded: {detail}")
+        if len(stat.models_seen) > 1:
+            lines.append(
+                f"      WARNING: more than one model served this journey "
+                f"({', '.join(stat.models_seen)}); do not compare these runs"
+            )
+        for page, count in list(stat.suspect_pages.items())[:3]:
+            lines.append(f"      failed after: {page} ({count}x)")
+    totals = suite.tokens()
+    lines.append(
+        f"  tokens: {totals['input']} in / {totals['output']} out, "
+        f"cache {totals['cache_write']} written / {totals['cache_read']} read"
+    )
+    cost = suite.cost(prices)
+    if cost is not None:
+        lines.append(f"  estimated cost: ${cost:.4f}")
+    lines.append(f"  wall clock: {suite.duration:.1f}s")
+    lines.append("=" * 62)
+    return "\n".join(lines)
+
+
+def markdown_report(result: RunResult) -> str:
+    out = [
+        f"# quickstarted: {result.journey.name}",
+        "",
+        f"**Result: {label(result)}** (`{result.classification}`, agent: "
+        f"`{result.agent_name}`)",
+        "",
+        f"- Goal: {result.journey.goal}",
+        f"- Stop reason: {result.outcome.stop_reason}"
+        + (f" ({result.outcome.detail})" if result.outcome.detail else ""),
+        f"- Turns: {result.outcome.turns}",
+        f"- Duration: {result.duration:.1f}s",
+        f"- Backend: `{result.backend}` "
+        + ("(enforced)" if result.enforced else "(**unenforced**)"),
+    ]
+    if result.model_reported:
+        out.append(f"- Model served: `{result.model_reported}`")
+    tokens = _token_line(result.outcome)
+    if tokens:
+        out.append(f"- Tokens: {tokens}")
+    out.append("")
+    if not result.evidential:
+        out += [
+            "## This run is not evidence about the documentation",
+            "",
+            f"It was classified `{result.classification}`, which means the run "
+            "failed for a reason unrelated to whether the docs are usable. It "
+            "is excluded from pass rates rather than counted as a failure.",
+            "",
+        ]
+    fetched = result.trace.fetched_urls()
+    if fetched:
+        out += ["## Docs pages the agent read", ""]
+        out += [f"1. {url}" for url in fetched]
+        out.append("")
+    if result.bypass_attempts:
+        out += [
+            "## Blocked shell access to documentation",
+            "",
+            f"The agent tried {result.bypass_attempts} time(s) to fetch docs "
+            "through the shell instead of the recorded tool. The proxy refused, "
+            "so the page list above is complete.",
+            "",
+        ]
+    if result.score is not None:
+        out += [
+            "## Success check",
+            "",
+            f"Exit code: {result.score.exit_code}",
+            "",
+            "```",
+            (result.score.output or "").strip(),
+            "```",
+            "",
+        ]
+    if result.classification == DOCS_GAP and result.suspect_page:
+        out += [
+            "## Where to look first",
+            "",
+            f"The last documentation page the agent read before failing was "
+            f"<{result.suspect_page}>. That page (or a gap right after it) is "
+            f"the first suspect.",
+            "",
+        ]
+    return "\n".join(out)
+
+
+def markdown_suite_report(suite: SuiteResult, prices: PriceBook | None = None) -> str:
+    prices = prices or PriceBook()
+    out = [
+        "# quickstarted suite",
+        "",
+        f"- Journeys: {len(suite.stats)}",
+        f"- Attempts per journey: {suite.repeat}",
+        f"- Backend: `{suite.backend}`",
+        f"- Duration: {suite.duration:.1f}s",
+        "",
+        "| Journey | Agent | Pass rate | Evidential | Discarded |",
+        "| --- | --- | --- | --- | --- |",
+    ]
+    for stat in suite.stats:
+        rate = stat.pass_rate
+        shown = "n/a" if rate is None else f"{rate:.0%}"
+        discarded = (
+            ", ".join(f"{k}={v}" for k, v in sorted(stat.discarded.items())) or "none"
+        )
+        out.append(
+            f"| {stat.journey} | `{stat.agent}` | {shown} | "
+            f"{stat.passes}/{len(stat.evidential)} | {discarded} |"
+        )
+    out.append("")
+    cost = suite.cost(prices)
+    if cost is not None:
+        out.append(f"Estimated cost: ${cost:.4f}")
+        out.append("")
+    return "\n".join(out)
+
+
+def stats_line(stat: JourneyStats) -> str:
+    rate = stat.pass_rate
+    return f"{stat.journey}: {'n/a' if rate is None else f'{rate:.0%}'}"

quickstarted/results.py

@@ -0,0 +1,166 @@
+"""Machine-readable results: a versioned JSON document and JUnit XML.
+
+`SCHEMA_VERSION` is a promise. The hosted product, the benchmark report, and
+anyone's CI dashboard all parse this document, so fields get added but not
+repurposed, and the version goes up when that stops being true.
+"""
+
+from __future__ import annotations
+
+import json
+import platform
+import socket
+import sys
+import time
+from pathlib import Path
+from xml.etree import ElementTree as ET
+
+from ._version import __version__
+from .pricing import PriceBook
+from .run import PASSED, RunResult
+from .suite import SuiteResult
+
+SCHEMA_VERSION = "1.0"
+
+
+def _run_document(run: RunResult) -> dict:
+    return {
+        "journey": run.journey.name,
+        "attempt": run.attempt,
+        "agent": run.agent_name,
+        "model_reported": run.model_reported,
+        "classification": run.classification,
+        "passed": run.passed,
+        "evidential": run.evidential,
+        "stop_reason": run.outcome.stop_reason,
+        "turns": run.outcome.turns,
+        "duration_seconds": round(run.duration, 2),
+        "backend": run.backend,
+        "enforced": run.enforced,
+        "docs_pages_read": run.trace.fetched_urls(),
+        "suspect_page": run.suspect_page,
+        "docs_bypass_attempts": run.bypass_attempts,
+        "success_check": (
+            None
+            if run.score is None
+            else {"exit_code": run.score.exit_code, "output": run.score.output[-2000:]}
+        ),
+        "tokens": {
+            "input": run.outcome.input_tokens,
+            "output": run.outcome.output_tokens,
+            "cache_write": run.outcome.cache_write_tokens,
+            "cache_read": run.outcome.cache_read_tokens,
+        },
+    }
+
+
+def suite_document(suite: SuiteResult, prices: PriceBook | None = None) -> dict:
+    prices = prices or PriceBook()
+    journeys = []
+    for stat in suite.stats:
+        journeys.append(
+            {
+                "journey": stat.journey,
+                "agent": stat.agent,
+                "attempts": stat.attempts,
+                "passes": stat.passes,
+                "evidential_runs": len(stat.evidential),
+                "pass_rate": stat.pass_rate,
+                "discarded": stat.discarded,
+                "models_reported": stat.models_seen,
+                "suspect_pages": stat.suspect_pages,
+                "tokens": stat.tokens(),
+                "estimated_cost_usd": stat.cost(prices),
+                "runs": [_run_document(r) for r in stat.runs],
+            }
+        )
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "quickstarted_version": __version__,
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "environment": {
+            "python": sys.version.split()[0],
+            "platform": platform.platform(),
+            "hostname": socket.gethostname(),
+            "backend": suite.backend,
+        },
+        "repeat": suite.repeat,
+        "duration_seconds": round(suite.duration, 2),
+        "totals": {
+            "runs": len(suite.runs),
+            "tokens": suite.tokens(),
+            "estimated_cost_usd": suite.cost(prices),
+        },
+        "journeys": journeys,
+    }
+
+
+def write_json(suite: SuiteResult, path: str | Path, prices=None) -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps(suite_document(suite, prices), indent=2) + "\n", encoding="utf-8"
+    )
+
+
+def junit_xml(suite: SuiteResult) -> str:
+    """JUnit XML so existing CI reporters can render a run with no new tooling."""
+    suites = ET.Element("testsuites", name="quickstarted")
+    total_failures = total_errors = total_tests = 0
+    for stat in suite.stats:
+        element = ET.SubElement(
+            suites,
+            "testsuite",
+            name=stat.journey,
+            tests=str(stat.attempts),
+        )
+        failures = errors = 0
+        for run in stat.runs:
+            case = ET.SubElement(
+                element,
+                "testcase",
+                classname=f"quickstarted.{stat.journey}",
+                name=f"{stat.agent} attempt {run.attempt}",
+                time=f"{run.duration:.2f}",
+            )
+            if run.classification == PASSED:
+                continue
+            if run.evidential:
+                failures += 1
+                node = ET.SubElement(
+                    case, "failure", type=run.classification,
+                    message=(
+                        "success check exit "
+                        f"{run.score.exit_code if run.score else 'n/a'}"
+                    ),
+                )
+                detail = [f"stop reason: {run.outcome.stop_reason}"]
+                if run.suspect_page:
+                    detail.append(f"last docs page read: {run.suspect_page}")
+                if run.score:
+                    detail.append(run.score.output[-1000:])
+                node.text = "\n".join(detail)
+            else:
+                # Not a docs failure: infrastructure, budget, or our own bug.
+                errors += 1
+                node = ET.SubElement(
+                    case, "error", type=run.classification,
+                    message=run.outcome.detail or run.classification,
+                )
+                node.text = run.outcome.detail
+        element.set("failures", str(failures))
+        element.set("errors", str(errors))
+        total_failures += failures
+        total_errors += errors
+        total_tests += stat.attempts
+    suites.set("tests", str(total_tests))
+    suites.set("failures", str(total_failures))
+    suites.set("errors", str(total_errors))
+    suites.set("time", f"{suite.duration:.2f}")
+    return ET.tostring(suites, encoding="unicode")
+
+
+def write_junit(suite: SuiteResult, path: str | Path) -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(junit_xml(suite) + "\n", encoding="utf-8")