spantruth 0.3.0__py3-none-any.whl

spantruth/__init__.py

@@ -0,0 +1,3 @@
+"""spantruth: check LLM telemetry values against the provider wire."""
+
+__version__ = "0.3.0"

spantruth/calls.py

@@ -0,0 +1,181 @@
+"""The app side of each scenario: how a normal application calls the provider SDK.
+
+Runs inside a subject's virtual environment. Every function returns what the app saw, so the
+runner can compare an instrumented run with an uninstrumented one.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from .model import Scenario
+
+PROMPT = [{"role": "user", "content": "What is the capital of France?"}]
+OPENAI_TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": name,
+            "parameters": {"type": "object", "properties": {key: {"type": "string"}}},
+        },
+    }
+    for name, key in (("get_weather", "city"), ("get_time", "zone"))
+]
+ANTHROPIC_TOOLS = [
+    {"name": name, "input_schema": {"type": "object", "properties": {key: {"type": "string"}}}}
+    for name, key in (("get_weather", "city"), ("get_time", "zone"))
+]
+
+
+class Seen:
+    """What the application observed. Compared between the baseline and instrumented runs."""
+
+    def __init__(self) -> None:
+        self.text = ""
+        self.events = 0
+        self.response_id: str | None = None
+        self.usage: dict | None = None
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "text": self.text,
+            "events": self.events,
+            "response_id": self.response_id,
+            "usage": self.usage,
+        }
+
+
+def _openai_kwargs(sc: Scenario) -> dict:
+    kw: dict = {"model": sc.request_model, "messages": PROMPT}
+    if sc.tools:
+        kw["tools"] = OPENAI_TOOLS
+    if sc.stream:
+        kw["stream"] = True
+    if sc.stream and sc.stream_usage:
+        kw["stream_options"] = {"include_usage": True}
+    return kw
+
+
+def _anthropic_kwargs(sc: Scenario) -> dict:
+    kw: dict = {"model": sc.request_model, "max_tokens": 1024, "messages": PROMPT}
+    if sc.tools:
+        kw["tools"] = ANTHROPIC_TOOLS
+    if sc.stream:
+        kw["stream"] = True
+    return kw
+
+
+def _openai_result(r: Any, seen: Seen) -> None:
+    msg = r.choices[0].message
+    seen.text = (msg.content or "") + "".join(t.function.name for t in msg.tool_calls or ())
+    seen.response_id, seen.usage = r.id, r.usage.model_dump()
+
+
+def _openai_chunk(c: Any, seen: Seen) -> None:
+    seen.events += 1
+    seen.response_id = c.id
+    if c.choices and c.choices[0].delta.content:
+        seen.text += c.choices[0].delta.content
+    if c.usage:
+        seen.usage = c.usage.model_dump()
+
+
+def _responses_event(e: Any, seen: Seen) -> None:
+    seen.events += 1
+    if e.type == "response.created":
+        seen.response_id = e.response.id
+    elif e.type == "response.output_text.delta":
+        seen.text += e.delta
+    elif e.type == "response.completed" and e.response.usage:
+        seen.usage = e.response.usage.model_dump()
+
+
+def _openai_create(client: Any, sc: Scenario):
+    """The call for the scenario's API, and how the app reads a stream from it."""
+    if sc.api == "responses":
+        kw: dict = {"model": sc.request_model, "input": PROMPT, "stream": sc.stream}
+        return client.responses.create(**kw), None, _responses_event
+    return client.chat.completions.create(**_openai_kwargs(sc)), _openai_result, _openai_chunk
+
+
+def _anthropic_result(r: Any, seen: Seen) -> None:
+    seen.text = "".join(b.text if b.type == "text" else b.name for b in r.content)
+    seen.response_id, seen.usage = r.id, r.usage.model_dump()
+
+
+def _anthropic_event(e: Any, seen: Seen) -> None:
+    seen.events += 1
+    if e.type == "message_start":
+        seen.response_id = e.message.id
+    elif e.type == "content_block_delta" and e.delta.type == "text_delta":
+        seen.text += e.delta.text
+
+
+def run(sc: Scenario, base_url: str, seen: Seen) -> None:
+    """Run one scenario the way an application would. Exceptions propagate to the caller."""
+    if sc.asynchronous:
+        asyncio.run(_run_async(sc, base_url, seen))
+        return
+    if sc.provider == "openai":
+        import openai
+
+        client = openai.OpenAI(base_url=base_url, api_key="wt-test", max_retries=0, timeout=10)
+        result, on_result, on_event = _openai_create(client, sc)
+    else:
+        import anthropic
+
+        client = anthropic.Anthropic(
+            base_url=base_url, api_key="wt-test", max_retries=0, timeout=10
+        )
+        result = client.messages.create(**_anthropic_kwargs(sc))
+        on_result, on_event = _anthropic_result, _anthropic_event
+    if not sc.stream:
+        on_result(result, seen)
+        return
+    if sc.with_block:
+        with result as stream:
+            _read(stream, sc, seen, on_event)
+        return
+    _read(result, sc, seen, on_event)
+    result.close()
+
+
+def _read(stream: Any, sc: Scenario, seen: Seen, on_event) -> None:
+    for event in stream:
+        on_event(event, seen)
+        if sc.read_events is not None and seen.events >= sc.read_events:
+            break
+
+
+async def _run_async(sc: Scenario, base_url: str, seen: Seen) -> None:
+    if sc.provider == "openai":
+        import openai
+
+        client = openai.AsyncOpenAI(base_url=base_url, api_key="wt-test", max_retries=0, timeout=10)
+        pending, on_result, on_event = _openai_create(client, sc)
+        result = await pending
+    else:
+        import anthropic
+
+        client = anthropic.AsyncAnthropic(
+            base_url=base_url, api_key="wt-test", max_retries=0, timeout=10
+        )
+        result = await client.messages.create(**_anthropic_kwargs(sc))
+        on_result, on_event = _anthropic_result, _anthropic_event
+    if not sc.stream:
+        on_result(result, seen)
+        return
+    if sc.with_block:
+        async with result as stream:
+            await _aread(stream, sc, seen, on_event)
+        return
+    await _aread(result, sc, seen, on_event)
+    await result.close()
+
+
+async def _aread(stream: Any, sc: Scenario, seen: Seen, on_event) -> None:
+    async for event in stream:
+        on_event(event, seen)
+        if sc.read_events is not None and seen.events >= sc.read_events:
+            break

spantruth/checks.py

@@ -0,0 +1,210 @@
+"""Grade one cell: what the app saw, and what the telemetry says, against the wire truth.
+
+Pure functions over plain records, so every verdict is testable without a subprocess.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+from typing import Any
+
+from .conventions import LLM_MARKERS, defined_for, lookup, openinference_only, same_finish
+from .model import Scenario, Truth
+from .receiver import SpanRecord
+from .rules import RULES
+
+# Ordered by severity. The cell takes the most severe verdict of its checks.
+SEVERITY = ("n/a", "pass", "missing", "wrong", "lost", "broken", "inconclusive")
+# A failing check is any check that did not pass and applies, `inconclusive` included.
+FAILING = SEVERITY[2:]
+TOKENS = (
+    "input_tokens",
+    "output_tokens",
+    "cache_read_tokens",
+    "cache_write_tokens",
+    "reasoning_tokens",
+)
+VALUES = ("request_model", "response_model", "response_id", "finish_reasons", *TOKENS)
+
+
+@dataclass(frozen=True)
+class Check:
+    id: str
+    verdict: str
+    expected: Any = None
+    got: Any = None
+    where: str | None = None
+    detail: str = ""
+
+    def as_dict(self) -> dict[str, Any]:
+        rule, url = RULES.get(self.id, ("", ""))
+        return asdict(self) | {"rule": rule, "rule_url": url}
+
+
+def cell_verdict(checks: list[Check]) -> str:
+    return max((c.verdict for c in checks), key=SEVERITY.index, default="n/a")
+
+
+def _app(base: dict, obs: dict) -> Check:
+    if obs.get("hang"):
+        return Check("app.unchanged", "broken", detail="the app hung until the watchdog fired")
+    diffs = []
+    if obs.get("exception") != base.get("exception"):
+        diffs.append(f"exception {base.get('exception')} became {obs.get('exception')}")
+    for key in ("text", "events", "response_id", "usage"):
+        if obs["seen"].get(key) != base["seen"].get(key):
+            diffs.append(f"{key} {base['seen'].get(key)!r} became {obs['seen'].get(key)!r}")
+    return Check("app.unchanged", "broken" if diffs else "pass", detail="; ".join(diffs))
+
+
+def _formula(fact: str, got: int, t: Truth) -> str:
+    """Name the arithmetic a wrong token count most likely came from."""
+    read, write = t.cache_read_tokens or 0, t.cache_write_tokens or 0
+    if fact == "input_tokens" and t.input_tokens is not None:
+        total = t.input_tokens
+        guesses = {
+            total - read - write: "cache reads and writes left out",
+            total - write: "cache writes left out",
+            total - read: "cache reads left out",
+            total + read + write: "cache tokens counted twice",
+            2 * total: "counted twice",
+        }
+        if got in guesses and read + write:
+            return guesses[got]
+    for other in TOKENS:
+        if other != fact and getattr(t, other) == got:
+            return f"equals {other.replace('_', ' ')}"
+    return ""
+
+
+def _json_list(value: Any) -> list | None:
+    """The list inside a JSON string, for a library that encodes an array attribute as text."""
+    if not isinstance(value, str):
+        return None
+    try:
+        parsed = json.loads(value)
+    except ValueError:
+        return None
+    return parsed if isinstance(parsed, list) else None
+
+
+def _value(fact: str, expected: Any, found: list[tuple[SpanRecord, str, Any]], t: Truth) -> Check:
+    if expected is None:
+        for _span, name, v in found:
+            if fact not in TOKENS or not v:
+                continue
+            if t.no_usage:
+                return Check(fact, "wrong", None, v, name, "estimated: the stream carried no usage")
+            if v in t.unread_counts:
+                return Check(
+                    fact,
+                    "wrong",
+                    None,
+                    v,
+                    name,
+                    "only the unread part of the stream carries this count",
+                )
+        return Check(fact, "n/a")
+    if not found:
+        return Check(fact, "missing", expected)
+    for _span, name, v in found:
+        ok = (
+            same_finish(expected, v)
+            if fact == "finish_reasons"
+            else (isinstance(v, (int, float)) and v == expected)
+            if fact in TOKENS
+            else v == expected
+        )
+        if not ok and fact == "response_model" and name == "llm.model_name":
+            ok = v == t.request_model  # this name does not separate request and response model
+        if not ok:
+            detail = _formula(fact, v, t) if fact in TOKENS and isinstance(v, int) else ""
+            if fact == "finish_reasons" and same_finish(expected, _json_list(v)):
+                detail = (
+                    f"the right values, recorded as the JSON string {v} instead of a string array"
+                )
+            return Check(fact, "wrong", expected, v, name, detail)
+    return Check(fact, "pass", expected, found[0][2], found[0][1])
+
+
+def grade(
+    sc: Scenario, t: Truth, base: dict | None, obs: dict | None, spans: list[SpanRecord]
+) -> list[Check]:
+    if obs is None or base is None:
+        return [Check("harness", "inconclusive", detail="no observation was written")]
+    if "instrument_error" in obs:
+        # The subject's setup raised. That is a failing check of the subject, not a gap in the run.
+        return [Check("subject.instrument", "inconclusive", detail=obs["instrument_error"])]
+    checks = [_app(base, obs)]
+    if obs.get("hang"):
+        return checks
+    parent = obs["parent"]
+    llm = [s for s in spans if s.span_id != parent["span_id"] and LLM_MARKERS & s.attributes.keys()]
+    open_spans = [
+        r["name"]
+        for sid, r in obs.get("probe", {}).items()
+        if r["end_ns"] is None and sid != parent["span_id"]
+    ]
+    if open_spans:
+        checks.append(Check("span.ended", "lost", detail=f"never ended: {', '.join(open_spans)}"))
+    if not llm:
+        if not open_spans:
+            checks.append(Check("span.recorded", "missing", detail="no LLM span arrived"))
+        return checks
+    orphans = [s.name for s in llm if s.trace_id != parent["trace_id"]]
+    checks.append(
+        Check(
+            "span.parented",
+            "wrong" if orphans else "pass",
+            detail=f"outside the app's trace: {', '.join(orphans)}" if orphans else "",
+        )
+    )
+    if t.error is not None:
+        errored = [s for s in llm if s.status == "ERROR"]
+        checks.append(
+            Check(
+                "error.status",
+                "pass" if errored else "wrong",
+                "ERROR",
+                sorted({s.status for s in llm}),
+            )
+        )
+        typed = [s.attributes["error.type"] for s in llm if s.attributes.get("error.type")]
+        if typed:
+            checks.append(Check("error.type", "pass", got=typed[0]))
+        elif all(openinference_only(s.attributes) for s in llm):
+            checks.append(
+                Check(
+                    "error.type",
+                    "n/a",
+                    detail="OpenInference records errors as span status and events",
+                )
+            )
+        else:
+            checks.append(Check("error.type", "missing"))
+        return checks
+    for fact in VALUES:
+        if not any(defined_for(s.attributes, fact) for s in llm):
+            checks.append(
+                Check(fact, "n/a", detail="not defined by the conventions this library follows")
+            )
+            continue
+        found = [(s, *hit) for s in llm if (hit := lookup(s.attributes, fact))]
+        checks.append(_value(fact, getattr(t, fact), found, t))
+    reporting = [
+        s.name
+        for s in llm
+        if lookup(s.attributes, "input_tokens") or lookup(s.attributes, "output_tokens")
+    ]
+    if len(reporting) > 1:
+        checks.append(
+            Check(
+                "tokens.single-report",
+                "wrong",
+                1,
+                len(reporting),
+                detail=f"usage on {len(reporting)} spans: {', '.join(reporting)}",
+            )
+        )
+    return checks

spantruth/cli.py

@@ -0,0 +1,214 @@
+"""spantruth command line: list, run, report."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import traceback
+from pathlib import Path
+
+from . import __version__, expectations, paths, report, runner, scenarios, subjects
+
+# GitHub caps a step summary at 1 MiB. Stay below it, and say where the rest is.
+SUMMARY_LIMIT = 1_000_000
+
+
+def _list(_: argparse.Namespace) -> int:
+    print("Scenarios:")
+    for sid, sc in scenarios.SCENARIOS.items():
+        print(f"  {sid:38} {sc.title}")
+    print(f"\nSubjects, from {subjects.SUBJECTS}:")
+    for name in subjects.all_names():
+        s = subjects.load(name)
+        print(f"  {name:38} {s.title} ({', '.join(s.providers)})")
+    return 0
+
+
+def _usage(message: object) -> int:
+    print(f"spantruth: {message}", file=sys.stderr)
+    return 2
+
+
+def exit_code(doc: dict, differences: list[expectations.Difference]) -> int:
+    """3 when the harness could not run something, else 1 on a difference from the expectations."""
+    if runner.incomplete(doc):
+        return 3
+    return 1 if differences else 0
+
+
+def _append_summary(path: Path, text: str) -> None:
+    if len(text.encode()) > SUMMARY_LIMIT:
+        text = text.encode()[:SUMMARY_LIMIT].decode(errors="ignore")
+        text += "\n\n... cut at 1,000,000 bytes. The results file has everything.\n"
+    with path.open("a") as f:
+        f.write(text + "\n")
+
+
+def _run(args: argparse.Namespace) -> int:
+    try:
+        selected = [subjects.load(ref) for ref in args.subject or subjects.all_names()]
+        overrides = tuple(subjects.resolve_override(spec) for spec in args.overrides or ())
+        ids = args.scenario or list(scenarios.SCENARIOS)
+        for sid in ids:
+            scenarios.get(sid)
+    except subjects.SubjectError as exc:
+        return _usage(exc)
+    except KeyError as exc:
+        return _usage(exc.args[0])
+    if (args.expect or args.write_expect) and len(selected) != 1:
+        return _usage("--expect and --write-expect take exactly one --subject")
+    if (args.expect or args.write_expect) and not any(
+        scenarios.get(sid).provider in selected[0].providers for sid in ids
+    ):
+        # Without one cell, a comparison would pass on nothing.
+        return _usage(
+            f"no selected scenario applies to {selected[0].name}, which covers "
+            f"{', '.join(selected[0].providers)}"
+        )
+    try:
+        expected = expectations.load(Path(args.expect)) if args.expect else None
+    except expectations.ExpectationError as exc:
+        return _usage(exc)
+    names = [s.name for s in selected]
+    if twice := sorted({n for n in names if names.count(n) > 1}):
+        return _usage(f"two subjects named {', '.join(twice)}: rename one of the directories")
+    installed = {
+        subjects.package_name(r) for s in selected for r in subjects.requirements(s, False)
+    }
+    for spec in overrides:
+        if (name := subjects.package_name(spec)) not in installed:
+            return _usage(f"--with {spec}: no selected subject installs {name}")
+    ids = scenarios.with_dependencies(ids)
+    doc = runner.run(
+        selected, ids, Path(args.cache), latest=args.latest, jobs=args.jobs, overrides=overrides
+    )
+    out = Path(args.out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(doc, indent=1, sort_keys=True) + "\n")
+    text = report.markdown(doc)
+    gaps = runner.incomplete(doc)
+    differences: list[expectations.Difference] = []
+    if args.expect or args.write_expect:
+        result = doc["subjects"][selected[0].name]
+        found = expectations.failing(result)
+    if expected is not None and gaps:
+        text += "\n\nThe run is incomplete, so it was not compared with the expectation file."
+    elif expected is not None:
+        differences = expectations.compare(expected, found, ran=set(result["cells"]))
+        skipped = sorted(set(expected) - set(result["cells"]))
+        text += "\n\n" + expectations.describe(differences, skipped)
+    print(text)
+    print(f"\nwrote {out}")
+    if args.write_expect and gaps:
+        # A gap in the run is not a known failure of the library, so nothing is recorded.
+        print("spantruth: the run is incomplete, so --write-expect wrote nothing", file=sys.stderr)
+    elif args.write_expect:
+        header = (
+            f"Known failures of {selected[0].name}, written by spantruth {__version__}.\n"
+            "A run with --expect fails when the failing checks differ from this file."
+        )
+        written = Path(args.write_expect)
+        written.parent.mkdir(parents=True, exist_ok=True)
+        written.write_text(expectations.dumps(found, header))
+        print(f"wrote {written}")
+    if args.summary:
+        _append_summary(Path(args.summary), text)
+    for gap in gaps:
+        print(f"spantruth: incomplete run: {gap}", file=sys.stderr)
+    return exit_code(doc, differences)
+
+
+def _report(args: argparse.Namespace) -> int:
+    doc = json.loads(Path(args.results).read_text())
+    if args.html:
+        Path(args.html).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.html).write_text(report.html(doc))
+        print(f"wrote {args.html}")
+    if args.badges:
+        out = Path(args.badges)
+        out.mkdir(parents=True, exist_ok=True)
+        for name, badge in report.badges(doc).items():
+            (out / f"{name}.json").write_text(json.dumps(badge) + "\n")
+        print(f"wrote badges to {out}")
+    if args.markdown or not (args.html or args.badges):
+        print(report.markdown(doc))
+    return 0
+
+
+def parser() -> argparse.ArgumentParser:
+    cache = paths.cache_dir()
+    parser = argparse.ArgumentParser(
+        prog="spantruth",
+        description="Check what LLM instrumentation reports against what the provider sent.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+    sub.add_parser("list", help="list scenarios and subjects").set_defaults(func=_list)
+    run = sub.add_parser("run", help="run subjects against scenarios")
+    run.add_argument(
+        "--subject",
+        action="append",
+        help="a built-in subject name, or a subject directory: a path with a slash, such as "
+        "./spantruth; repeatable",
+    )
+    run.add_argument("--scenario", action="append", help="scenario id, repeatable")
+    run.add_argument("--latest", action="store_true", help="unpin the subjects' own packages")
+    run.add_argument(
+        "--with",
+        dest="overrides",
+        action="append",
+        metavar="SPEC",
+        help="replace a package with this requirement: a version, a pull request git ref, or a "
+        "local checkout (a path with a slash); repeatable",
+    )
+    run.add_argument(
+        "--out",
+        default=str(cache / "results" / "latest.json"),
+        help="where to write the results document (default: %(default)s)",
+    )
+    run.add_argument(
+        "--cache",
+        default=str(cache / "venvs"),
+        help="where to keep the subject environments (default: %(default)s)",
+    )
+    run.add_argument("--jobs", type=int, default=4)
+    run.add_argument(
+        "--expect",
+        metavar="FILE",
+        help="compare the failing checks of the one selected subject with this file, and exit 1 "
+        "when they differ",
+    )
+    run.add_argument(
+        "--write-expect",
+        metavar="FILE",
+        help="write the failing checks of the one selected subject to this file",
+    )
+    run.add_argument(
+        "--summary",
+        metavar="FILE",
+        help="append the Markdown report to this file, such as $GITHUB_STEP_SUMMARY",
+    )
+    run.set_defaults(func=_run)
+    rep = sub.add_parser("report", help="render a results document")
+    rep.add_argument("results")
+    rep.add_argument("--html", help="write a static HTML report to this path")
+    rep.add_argument(
+        "--badges", help="write a shields.io endpoint JSON per library to this directory"
+    )
+    rep.add_argument("--markdown", action="store_true", help="print the Markdown matrix")
+    rep.set_defaults(func=_report)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parser().parse_args(argv)
+    try:
+        return args.func(args)
+    except Exception as exc:  # unexpected means the harness failed, never a verdict (exit 3)
+        traceback.print_exc()
+        print(f"spantruth: could not finish: {type(exc).__name__}: {exc}", file=sys.stderr)
+        return 3
+
+
+if __name__ == "__main__":
+    sys.exit(main())