tokenaudit 0.1.0__py3-none-any.whl

tokenaudit/__init__.py

@@ -0,0 +1,3 @@
+"""tokenaudit: root-cause diagnosis of wasted LLM tokens from GenAI traces."""
+
+__version__ = "0.1.0"

tokenaudit/__main__.py

@@ -0,0 +1,152 @@
+"""Command-line entry point: ``python -m tokenaudit ingest <paths>``.
+
+Prints the normalized ingest result as JSON (default) or a human summary
+(``--summary``). A dedicated console script with a fuller CLI arrives with
+the report phase; this stays argparse-only so the package keeps zero
+runtime dependencies.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections import Counter
+from dataclasses import asdict
+from importlib.resources import files
+from pathlib import Path
+
+from tokenaudit import __version__
+from tokenaudit.detect import SessionDiagnosis, diagnose
+from tokenaudit.ingest import UnknownTraceFormatError, ingest_path
+from tokenaudit.model.records import IngestResult
+from tokenaudit.report import render_html
+from tokenaudit.rulebook import load_rulebook
+
+
+def _trace_files(path: Path) -> list[Path]:
+    if path.is_file():
+        return [path]
+    if path.is_dir():
+        found = sorted(p for p in path.rglob("*") if p.name.endswith((".trace.json", ".jsonl")))
+        if not found:
+            raise UnknownTraceFormatError(f"{path}: no trace files found")
+        return found
+    raise UnknownTraceFormatError(f"{path}: no such file or directory")
+
+
+def _summarize(result: IngestResult) -> str:
+    lines = [
+        f"{result.source}  [{result.container}]  "
+        f"spans={result.spans_total} skipped={result.spans_skipped} "
+        f"deduped={len(result.deduped_span_ids)}"
+    ]
+    for session in result.sessions:
+        requests = session.requests
+        known_input = [r.usage.input_total for r in requests if r.usage.input_total is not None]
+        known_read = [r.usage.cache_read for r in requests if r.usage.cache_read is not None]
+        total_input = sum(known_input)
+        total_read = sum(known_read)
+        hit = f"{total_read / total_input * 100:.1f}%" if total_input and known_read else "n/a"
+        flags = Counter(flag for r in requests for flag in r.quality)
+        unpriceable = sum(1 for r in requests if r.unpriceable)
+        lines.append(
+            f"  session {session.session_id} ({session.grouping}): "
+            f"{len(requests)} requests, input={total_input}, "
+            f"cache_read={total_read} ({hit}), unpriceable={unpriceable}"
+        )
+        if flags:
+            rendered = ", ".join(f"{name} x{count}" for name, count in sorted(flags.items()))
+            lines.append(f"    flags: {rendered}")
+    return "\n".join(lines)
+
+
+def _render_diagnosis(diagnosis: SessionDiagnosis) -> str:
+    lines = [
+        f"session {diagnosis.session_id}  "
+        f"({diagnosis.requests_examined} requests examined, "
+        f"{diagnosis.requests_skipped_unpriceable} unpriceable skipped)"
+    ]
+    if not diagnosis.findings:
+        lines.append("  no findings")
+    for finding in diagnosis.findings:
+        lines.append(
+            f"  [{finding.severity.value.upper()}] {finding.detector_id}: {finding.summary}"
+        )
+        if finding.impact is not None:
+            impact = finding.impact
+            tokens = (
+                f"~{impact.tokens_low:,} tokens"
+                if impact.tokens_low == impact.tokens_high
+                else f"~{impact.tokens_low:,}-{impact.tokens_high:,} tokens"
+            )
+            lines.append(f"      waste: {tokens} full-price equivalent ({impact.assumption})")
+        lines.append(f"      fix: {finding.fix.summary}")
+    clean = [d for d in diagnosis.checked if d not in {f.detector_id for f in diagnosis.findings}]
+    if clean:
+        lines.append(f"  checked, clean: {', '.join(clean)}")
+    for detector_id, reason in diagnosis.abstained.items():
+        lines.append(f"  not checked: {detector_id} ({reason})")
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="tokenaudit")
+    parser.add_argument("--version", action="version", version=f"tokenaudit {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    ingest = subparsers.add_parser("ingest", help="normalize trace files")
+    ingest.add_argument("paths", nargs="+", type=Path)
+    ingest.add_argument("--summary", action="store_true", help="human summary instead of JSON")
+    diag = subparsers.add_parser("diagnose", help="diagnose token waste in trace files")
+    diag.add_argument("paths", nargs="*", type=Path)
+    diag.add_argument("--json", action="store_true", help="machine-readable output")
+    diag.add_argument("--html", type=Path, help="also write a self-contained HTML report")
+    diag.add_argument(
+        "--sample", action="store_true", help="diagnose the bundled sample trace (a demo)"
+    )
+    subparsers.add_parser("rulebook", help="render the provider caching rulebook as markdown")
+    args = parser.parse_args(argv)
+
+    if args.command == "rulebook":
+        print(load_rulebook().render_markdown())
+        return 0
+
+    paths = list(args.paths)
+    if args.command == "diagnose" and args.sample:
+        paths.append(Path(str(files("tokenaudit.examples").joinpath("sample.trace.json"))))
+    if not paths:
+        parser.error("no trace paths given (or use: tokenaudit diagnose --sample)")
+    results = []
+    try:
+        for path in paths:
+            for trace_file in _trace_files(path):
+                results.append(ingest_path(trace_file))
+    except UnknownTraceFormatError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    if args.command == "diagnose":
+        per_result = [(result, diagnose(result)) for result in results]
+        if args.html:
+            args.html.write_text(render_html(per_result, load_rulebook()), encoding="utf-8")
+            print(f"wrote {args.html}", file=sys.stderr)
+        if args.json:
+            payload = [asdict(d) for _, diagnoses in per_result for d in diagnoses]
+            print(json.dumps(payload, indent=2, default=str))
+        else:
+            for result, diagnoses in per_result:
+                print(result.source)
+                for diagnosis in diagnoses:
+                    print(_render_diagnosis(diagnosis))
+        return 0
+
+    if args.summary:
+        for result in results:
+            print(_summarize(result))
+    else:
+        print(json.dumps([asdict(result) for result in results], indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tokenaudit/ci/__init__.py

@@ -0,0 +1 @@
+"""CI cost-regression gate."""

tokenaudit/counterfactual/__init__.py

@@ -0,0 +1 @@
+"""Counterfactual pricing: what the bill would have been under a fix."""

tokenaudit/detect/__init__.py

@@ -0,0 +1,15 @@
+"""The diagnosis engine: deterministic waste detectors."""
+
+from tokenaudit.detect.engine import DETECTORS, diagnose, diagnose_session
+from tokenaudit.detect.findings import Finding, Fix, Impact, SessionDiagnosis, Severity
+
+__all__ = [
+    "DETECTORS",
+    "Finding",
+    "Fix",
+    "Impact",
+    "SessionDiagnosis",
+    "Severity",
+    "diagnose",
+    "diagnose_session",
+]

tokenaudit/detect/base.py

@@ -0,0 +1,96 @@
+"""Detector contract and shared helpers.
+
+A detector is a pure function of ``(Session, Rulebook)``. It must do one of
+three things, and the distinction is part of the product:
+
+- **abstain** with a reason when it cannot examine the session (no content
+  captured, model unknown to the rulebook, too few requests) — reported as
+  "not checked", never as "clean";
+- **run and find nothing** — reported as checked-and-clean;
+- **run and emit findings**, each carrying evidence a stranger could verify
+  against the raw trace.
+
+Token estimates from block lengths use the same ~4-chars-per-token stand-in
+as the corpus simulator, and every impact that relies on it says so in its
+assumption string.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from itertools import pairwise
+from typing import ClassVar
+
+from tokenaudit.detect.findings import Finding
+from tokenaudit.model.records import BlockDigest, Request, Session
+from tokenaudit.rulebook import Rulebook
+
+CHARS_PER_TOKEN = 4
+TOKEN_ESTIMATE_NOTE = "block token counts estimated from lengths at ~4 chars/token"
+
+
+class Abstain(Exception):
+    """Raised by a detector that cannot examine this session; carries the reason."""
+
+    def __init__(self, reason: str) -> None:
+        super().__init__(reason)
+        self.reason = reason
+
+
+class Detector(ABC):
+    detector_id: ClassVar[str]
+
+    @abstractmethod
+    def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
+        """Return findings for this session, or raise :class:`Abstain`."""
+
+
+def est_tokens(chars: int) -> int:
+    return max(1, chars // CHARS_PER_TOKEN)
+
+
+def blocks_tokens(blocks: tuple[BlockDigest, ...]) -> int:
+    return sum(est_tokens(b.chars) for b in blocks)
+
+
+def last_marked_index(blocks: tuple[BlockDigest, ...]) -> int | None:
+    marked = [i for i, b in enumerate(blocks) if b.cache_marker]
+    return marked[-1] if marked else None
+
+
+@dataclass(frozen=True)
+class Pair:
+    """Two consecutive same-model requests that both captured content."""
+
+    first: Request
+    second: Request
+
+    @property
+    def first_divergence(self) -> int | None:
+        a = self.first.blocks or ()
+        b = self.second.blocks or ()
+        for i in range(min(len(a), len(b))):
+            if a[i].sha256 != b[i].sha256:
+                return i
+        return None
+
+
+def consecutive_pairs(session: Session) -> list[Pair]:
+    """Adjacent same-model request pairs with content on both sides."""
+    pairs: list[Pair] = []
+    requests = session.requests
+    for first, second in pairwise(requests):
+        if first.blocks and second.blocks and first.model == second.model:
+            pairs.append(Pair(first, second))
+    return pairs
+
+
+def require_content(session: Session, minimum_requests: int = 1) -> list[Request]:
+    with_content = [r for r in session.requests if r.blocks]
+    if len(with_content) < minimum_requests:
+        raise Abstain(
+            f"needs at least {minimum_requests} request(s) with captured message content; "
+            "usage numbers alone can prove waste but not its cause"
+        )
+    return with_content

tokenaudit/detect/corpus_record.json

@@ -0,0 +1,56 @@
+{
+  "breakpoint_limit": {
+    "false_alarms": 0,
+    "negatives_checked": 14,
+    "positives_found": 1,
+    "positives_total": 1
+  },
+  "context_bloat": {
+    "false_alarms": 0,
+    "negatives_checked": 12,
+    "positives_found": 1,
+    "positives_total": 1
+  },
+  "lookback_overflow": {
+    "false_alarms": 0,
+    "negatives_checked": 14,
+    "positives_found": 1,
+    "positives_total": 1
+  },
+  "model_switch": {
+    "false_alarms": 0,
+    "negatives_checked": 14,
+    "positives_found": 1,
+    "positives_total": 1
+  },
+  "prefix_drift": {
+    "false_alarms": 0,
+    "negatives_checked": 12,
+    "positives_found": 3,
+    "positives_total": 4
+  },
+  "retry_storm": {
+    "false_alarms": 0,
+    "negatives_checked": 14,
+    "positives_found": 1,
+    "positives_total": 1
+  },
+  "sub_minimum": {
+    "false_alarms": 0,
+    "negatives_checked": 14,
+    "positives_found": 1,
+    "positives_total": 1
+  },
+  "tool_schema_churn": {
+    "false_alarms": 0,
+    "negatives_checked": 14,
+    "positives_found": 1,
+    "positives_total": 1
+  },
+  "ttl_expiry": {
+    "false_alarms": 0,
+    "negatives_checked": 12,
+    "positives_found": 1,
+    "positives_total": 1
+  }
+}

tokenaudit/detect/engine.py

@@ -0,0 +1,87 @@
+"""The engine: run every registered detector over ingested sessions.
+
+Contract enforced here rather than trusted to each detector:
+
+- requests the quality gate marked ``unpriceable`` are removed from the
+  session view before any detector sees them, and the count is reported —
+  wrong dollar figures are worse than no figures;
+- an abstention (with its reason) and a clean pass are kept apart in the
+  result, because "not checked" and "checked, nothing found" support very
+  different conclusions;
+- findings come back ordered by severity, then by impact upper bound.
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+
+from tokenaudit.detect.base import Abstain, Detector
+from tokenaudit.detect.findings import Finding, SessionDiagnosis, Severity
+from tokenaudit.detect.rules.breakpoint_limit import BreakpointLimitDetector
+from tokenaudit.detect.rules.context_bloat import ContextBloatDetector
+from tokenaudit.detect.rules.lookback_overflow import LookbackOverflowDetector
+from tokenaudit.detect.rules.model_switch import ModelSwitchDetector
+from tokenaudit.detect.rules.prefix_drift import PrefixDriftDetector
+from tokenaudit.detect.rules.retry_storm import RetryStormDetector
+from tokenaudit.detect.rules.sub_minimum import SubMinimumDetector
+from tokenaudit.detect.rules.tool_schema_churn import ToolSchemaChurnDetector
+from tokenaudit.detect.rules.ttl_expiry import TtlExpiryDetector
+from tokenaudit.model.records import IngestResult, Session
+from tokenaudit.rulebook import Rulebook, load_rulebook
+
+DETECTORS: tuple[Detector, ...] = (
+    PrefixDriftDetector(),
+    SubMinimumDetector(),
+    ModelSwitchDetector(),
+    TtlExpiryDetector(),
+    BreakpointLimitDetector(),
+    LookbackOverflowDetector(),
+    ToolSchemaChurnDetector(),
+    ContextBloatDetector(),
+    RetryStormDetector(),
+)
+
+_SEVERITY_ORDER = {Severity.HIGH: 0, Severity.MEDIUM: 1, Severity.LOW: 2}
+
+
+def _rank(finding: Finding) -> tuple[int, int]:
+    impact = finding.impact.tokens_high if finding.impact else 0
+    return (_SEVERITY_ORDER[finding.severity], -impact)
+
+
+def diagnose_session(
+    session: Session,
+    rulebook: Rulebook,
+    detectors: tuple[Detector, ...] = DETECTORS,
+) -> SessionDiagnosis:
+    priceable = tuple(r for r in session.requests if not r.unpriceable)
+    skipped = len(session.requests) - len(priceable)
+    view = replace(session, requests=priceable)
+
+    findings: list[Finding] = []
+    checked: list[str] = []
+    abstained: dict[str, str] = {}
+    for detector in detectors:
+        try:
+            found = detector.check(view, rulebook)
+        except Abstain as abstention:
+            abstained[detector.detector_id] = abstention.reason
+        else:
+            checked.append(detector.detector_id)
+            findings.extend(found)
+
+    source = session.requests[0].source if session.requests else ""
+    return SessionDiagnosis(
+        session_id=session.session_id,
+        source=source,
+        findings=tuple(sorted(findings, key=_rank)),
+        checked=tuple(checked),
+        abstained=abstained,
+        requests_examined=len(priceable),
+        requests_skipped_unpriceable=skipped,
+    )
+
+
+def diagnose(result: IngestResult, rulebook: Rulebook | None = None) -> list[SessionDiagnosis]:
+    book = rulebook or load_rulebook()
+    return [diagnose_session(session, book) for session in result.sessions]

tokenaudit/detect/findings.py

@@ -0,0 +1,86 @@
+"""The diagnosis result schema — frozen early, because everything downstream
+(report rendering, the eval harness, the CI gate) builds on this shape.
+
+Design commitments:
+
+- A finding is a **claim with evidence**: every field a stranger would need
+  to verify it by hand against the raw trace is present, or the finding
+  should not exist.
+- ``Impact`` is a **bounded range with a stated assumption**, denominated in
+  full-price-equivalent input tokens; dollars appear only when the rulebook
+  holds a verified price for the model. No point estimates, no unstated
+  extrapolation, and the rulebook version that produced the numbers rides
+  along.
+- ``confidence`` is **measured, never asserted**: it stays ``None`` until
+  the evaluation harness computes per-detector precision against the
+  labeled corpus and attaches it at report time.
+- A detector that could not examine a session **abstains with a reason** —
+  a different fact from "checked and found nothing", and the report must
+  keep the two apart.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any
+
+
+class Severity(StrEnum):
+    HIGH = "high"
+    MEDIUM = "medium"
+    LOW = "low"
+
+
+@dataclass(frozen=True)
+class Impact:
+    """Waste attributed to one finding, as a bounded range.
+
+    ``tokens_*`` are full-price-equivalent input tokens: the amount of input
+    that was (or will keep being) billed at the full rate but would have
+    been served at the cached rate under the fix. One finding at a time —
+    impacts of different findings on the same session are not additive.
+    """
+
+    tokens_low: int
+    tokens_high: int
+    usd_low: float | None
+    usd_high: float | None
+    assumption: str
+    rulebook_version: str
+
+
+@dataclass(frozen=True)
+class Fix:
+    summary: str
+    hint: str | None = None
+    doc_url: str | None = None
+
+
+@dataclass(frozen=True)
+class Finding:
+    """One diagnosed cause of waste in one session."""
+
+    detector_id: str
+    severity: Severity
+    session_id: str
+    source: str
+    span_ids: tuple[str, ...]
+    summary: str
+    evidence: dict[str, Any]
+    impact: Impact | None  # None when the quality gate blocked pricing
+    fix: Fix
+    confidence: float | None = None  # measured precision, attached by the eval harness
+
+
+@dataclass(frozen=True)
+class SessionDiagnosis:
+    """Everything the engine concluded about one session."""
+
+    session_id: str
+    source: str
+    findings: tuple[Finding, ...]
+    checked: tuple[str, ...]  # detectors that ran (clean if not in findings)
+    abstained: dict[str, str] = field(default_factory=dict)  # detector_id -> reason
+    requests_examined: int = 0
+    requests_skipped_unpriceable: int = 0

tokenaudit/detect/rules/__init__.py

@@ -0,0 +1 @@
+"""Detector rules, one module per failure mode."""

tokenaudit/detect/rules/breakpoint_limit.py

@@ -0,0 +1,102 @@
+"""Breakpoint limit: more cache markers than the provider honors.
+
+Anthropic honors at most 4 explicit cache breakpoints per request; content
+marked beyond the honored set is silently re-billed every call. Modeled as
+silent truncation, matching the corpus simulator — a note rides the
+evidence because live-API behavior for over-marking should be confirmed
+before this claim is made in a published case study.
+"""
+
+from __future__ import annotations
+
+from tokenaudit.detect.base import (
+    TOKEN_ESTIMATE_NOTE,
+    Abstain,
+    Detector,
+    blocks_tokens,
+    require_content,
+)
+from tokenaudit.detect.findings import Finding, Fix, Impact, Severity
+from tokenaudit.model.records import Session
+from tokenaudit.rulebook import Rulebook
+
+
+class BreakpointLimitDetector(Detector):
+    detector_id = "breakpoint_limit"
+
+    def check(self, session: Session, rulebook: Rulebook) -> list[Finding]:
+        requests = require_content(session, minimum_requests=1)
+        over: list[tuple[int, int, int]] = []  # (request idx, marked count, unhonored tokens)
+        max_allowed: int | None = None
+        for i, request in enumerate(requests):
+            blocks = request.blocks or ()
+            marked_indices = [j for j, b in enumerate(blocks) if b.cache_marker]
+            if not marked_indices:
+                continue
+            rules = rulebook.lookup(request.model, request.provider)
+            if rules is None:
+                raise Abstain(f"model not in the rulebook: {request.model}")
+            if rules.max_explicit_breakpoints is None:
+                continue  # provider has no explicit-marker concept
+            max_allowed = rules.max_explicit_breakpoints
+            if len(marked_indices) > max_allowed:
+                honored_end = marked_indices[max_allowed - 1]
+                last_marked = marked_indices[-1]
+                unhonored = blocks_tokens(blocks[honored_end + 1 : last_marked + 1])
+                over.append((i, len(marked_indices), unhonored))
+        if not over or max_allowed is None:
+            return []
+
+        repeat_calls = max(len(over) - 1, 1)
+        unhonored_tokens = max(tokens for _, _, tokens in over)
+        # Stable-tail check: the low bound needs the unhonored region to be
+        # identical across calls, otherwise it wouldn't have cached anyway.
+        tails = {tuple(b.sha256 for b in (requests[i].blocks or ()))[-2:] for i, _, _ in over}
+        tokens_high = round(0.9 * unhonored_tokens * repeat_calls)
+        tokens_low = tokens_high if len(tails) == 1 else 0
+
+        first_idx, marked_count, _ = over[0]
+        rules = rulebook.lookup(requests[first_idx].model, requests[first_idx].provider)
+        finding = Finding(
+            detector_id=self.detector_id,
+            severity=Severity.MEDIUM,
+            session_id=session.session_id,
+            source=requests[first_idx].source,
+            span_ids=tuple(requests[i].span_id for i, _, _ in over),
+            summary=(
+                f"{marked_count} blocks carry cache markers but the provider honors at most "
+                f"{max_allowed}; ~{unhonored_tokens} tokens of marked content past the limit "
+                "are re-billed on every call"
+            ),
+            evidence={
+                "marked_blocks": marked_count,
+                "max_explicit_breakpoints": max_allowed,
+                "unhonored_est_tokens": unhonored_tokens,
+                "calls_affected": len(over),
+                "note": (
+                    "over-marking modeled as silent truncation of the extra markers; "
+                    "confirm live-API behavior before publishing this claim"
+                ),
+            },
+            impact=Impact(
+                tokens_low=tokens_low,
+                tokens_high=tokens_high,
+                usd_low=None,
+                usd_high=None,
+                assumption=(
+                    "the region past the last honored marker would have been cached and read "
+                    f"on repeat calls (0.9x delta); {TOKEN_ESTIMATE_NOTE}; low bound requires "
+                    "a byte-stable tail"
+                ),
+                rulebook_version=rulebook.version,
+            ),
+            fix=Fix(
+                summary=(
+                    "Consolidate cached content behind at most the honored number of markers "
+                    "— merge adjacent stable blocks so one marker covers them."
+                ),
+                hint=None,
+                doc_url=rules.source_url if rules else None,
+            ),
+        )
+        return [finding]