cache-autopsy 0.1.0__py3-none-any.whl

cache_autopsy/__init__.py

@@ -0,0 +1,21 @@
+"""cache-autopsy: find out why an LLM prompt cache missed, offline.
+
+Public API::
+
+    diff_requests(a, b, provider=None) -> Divergence
+    analyze_records(records, pricing=None, session_key=None) -> Report
+    lint_request(req, provider=None) -> list[Finding]
+"""
+from .cost import Pricing
+from .diff import diff_requests
+from .lint import lint_request
+from .model import (CacheAutopsyError, Divergence, Finding, InputFormatError, PairVerdict, Report,
+                    UnknownProviderError, Usage)
+from .verdict import analyze_records
+
+__version__ = "0.1.0"
+__all__ = [
+    "diff_requests", "analyze_records", "lint_request", "Pricing",
+    "Divergence", "PairVerdict", "Report", "Finding", "Usage",
+    "CacheAutopsyError", "InputFormatError", "UnknownProviderError", "__version__",
+]

cache_autopsy/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

cache_autopsy/cli.py

@@ -0,0 +1,143 @@
+"""Command line entry point. Exit codes: 0 ok, 1 gate failed, 2 input error."""
+from __future__ import annotations
+
+import argparse
+import sys
+
+from . import __version__
+from .cost import Pricing
+from .diff import diff_requests
+from .lint import has_problems, lint_request
+from .model import CacheAutopsyError
+from .records import read_jsonl, read_request
+from .report import analysis_json, analysis_md, analysis_text, divergence_text, findings_text
+from .rules import CAUSES, causes_markdown
+from .verdict import DEFAULT_MIN_TOKENS, analyze_records
+
+PROVIDER_CHOICES = ["auto", "anthropic", "openai_chat", "openai_responses"]
+
+
+def _parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="cache-autopsy",
+                                description="Find out why an LLM prompt cache missed (offline, no network).")
+    p.add_argument("--version", action="version", version=f"cache-autopsy {__version__}")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    def common(sp, fmt=True):
+        sp.add_argument("--provider", choices=PROVIDER_CHOICES, default="auto", help="force the provider (default: detect)")
+        sp.add_argument("--context", type=int, default=40, help="characters of context to show (default 40)")
+        sp.add_argument("--redact", action="store_true", help="do not print any request text")
+        if fmt:
+            sp.add_argument("--format", choices=["text", "json", "md"], default="text")
+
+    d = sub.add_parser("diff", help="first divergence between two request bodies")
+    d.add_argument("a"), d.add_argument("b")
+    common(d)
+
+    a = sub.add_parser("analyze", help="hit rate, missed cost and causes for a JSONL call log")
+    a.add_argument("log")
+    common(a)
+    a.add_argument("--pricing", help="JSON file overriding the bundled prices")
+    a.add_argument("--session-key", help="dotted path to group calls, e.g. request.metadata.user_id")
+    a.add_argument("--min-tokens", type=int, default=DEFAULT_MIN_TOKENS, help="provider minimum cacheable tokens")
+    a.add_argument("--fail-under", type=float, help="exit 1 if the overall hit rate is below this (0-1)")
+    a.add_argument("--fail-on", help="comma list of verdicts or cause codes that make the run fail")
+
+    l = sub.add_parser("lint", help="risky patterns in one request body")
+    l.add_argument("request")
+    common(l, fmt=False)
+    l.add_argument("--min-tokens", type=int, default=DEFAULT_MIN_TOKENS)
+
+    c = sub.add_parser("causes", help="list cause codes and how to fix them")
+    c.add_argument("--format", choices=["text", "md"], default="text")
+
+    sub.add_parser("demo", help="run the bundled practice (mock) data, no keys needed")
+    return p
+
+
+def _provider(args) -> str | None:
+    return None if args.provider == "auto" else args.provider
+
+
+VERDICTS = {"hit", "expected_hit", "explained_miss", "unexplained_miss", "unexplained"}
+
+
+def _check_gate_options(args) -> set[str]:
+    """Validate --fail-under / --fail-on before any work so a typo cannot silently disable the gate."""
+    if args.fail_under is not None and not 0 <= args.fail_under <= 1:
+        raise CacheAutopsyError(f"--fail-under must be a ratio between 0 and 1 (got {args.fail_under}). "
+                                "Write 0.6 for 60%.")
+    wanted = {w.strip() for w in (args.fail_on or "").split(",") if w.strip()}
+    unknown = sorted(wanted - set(CAUSES) - VERDICTS)
+    if unknown:
+        raise CacheAutopsyError(f"unknown --fail-on value(s): {', '.join(unknown)}. "
+                                "Run `cache-autopsy causes` for cause codes; verdicts are "
+                                + ", ".join(sorted(VERDICTS)) + ".")
+    return wanted
+
+
+def _run(args) -> int:
+    if args.cmd == "diff":
+        div = diff_requests(read_request(args.a), read_request(args.b), _provider(args), args.context)
+        if args.format == "json":
+            import json
+            print(json.dumps(div.to_dict(), indent=2, ensure_ascii=False))
+        else:
+            print(divergence_text(div, args.redact))
+        return 0
+
+    if args.cmd == "analyze":
+        wanted = _check_gate_options(args)
+        pricing = Pricing.from_file(args.pricing) if args.pricing else Pricing.default()
+        report = analyze_records(read_jsonl(args.log), pricing, args.session_key, _provider(args),
+                                 args.min_tokens, args.context)
+        if report.summary["records"] == 0:
+            print(f"warning: no records in {args.log}.", file=sys.stderr)
+        render = {"text": analysis_text, "json": analysis_json, "md": analysis_md}[args.format]
+        print(render(report, args.redact))
+        failed = False
+        rate = report.summary["hit_rate"]
+        if args.fail_under is not None:
+            if rate is None:
+                print("cannot apply --fail-under: the log has no usage data.", file=sys.stderr)
+                return 2
+            if rate < args.fail_under:
+                print(f"FAIL: hit rate {rate:.2f} < {args.fail_under:.2f}", file=sys.stderr)
+                failed = True
+        if args.fail_on:
+            hits = [p for p in report.pairs if p.verdict in wanted or wanted & set(p.causes) or p.reason in wanted]
+            if hits:
+                print(f"FAIL: {len(hits)} comparison(s) match --fail-on {args.fail_on}", file=sys.stderr)
+                failed = True
+        return 1 if failed else 0
+
+    if args.cmd == "lint":
+        findings = lint_request(read_request(args.request), _provider(args), args.min_tokens)
+        print(findings_text(findings))
+        return 1 if has_problems(findings) else 0
+
+    if args.cmd == "causes":
+        if args.format == "md":
+            print(causes_markdown(), end="")
+        else:
+            for code, (layer, meaning, fix, _) in CAUSES.items():
+                print(f"{code} [{layer}]\n    {meaning}\n    fix: {fix}")
+        return 0
+
+    from .demo import run_demo
+    return run_demo()
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+    try:
+        return _run(args)
+    except CacheAutopsyError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    except BrokenPipeError:  # e.g. `| head`
+        return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cache_autopsy/cost.py

@@ -0,0 +1,96 @@
+"""Pricing lookup and the missed-cost arithmetic."""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+from .model import InputFormatError, Usage
+
+
+@dataclass(frozen=True)
+class Rates:
+    """USD per 1M tokens for one model."""
+
+    input: float
+    write_5m: float
+    write_1h: float
+    read: float
+
+
+class Pricing:
+    """Per-model unit prices. Unknown models return None, and callers then report tokens only."""
+
+    def __init__(self, table: dict[str, Any]):
+        self.table = table
+
+    @classmethod
+    def default(cls) -> "Pricing":
+        text = resources.files("cache_autopsy").joinpath("pricing.json").read_text(encoding="utf-8")
+        return cls(json.loads(text))
+
+    @classmethod
+    def from_file(cls, path: str | Path) -> "Pricing":
+        """Default table with the entries of ``path`` laid over it."""
+        try:
+            override = json.loads(Path(path).read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            raise InputFormatError(f"pricing file not found: {path}. Pass an existing JSON file.") from None
+        except UnicodeDecodeError:
+            raise InputFormatError(f"pricing file {path} is not valid UTF-8. Pass a UTF-8 JSON file.") from None
+        except OSError as exc:
+            raise InputFormatError(f"cannot read pricing file {path}: {exc.strerror or exc}.") from None
+        except json.JSONDecodeError as exc:
+            raise InputFormatError(f"pricing file {path} is not valid JSON: {exc}") from None
+        if not isinstance(override, dict):
+            raise InputFormatError(f"pricing file {path} must contain a JSON object like the bundled pricing.json.")
+        base = cls.default().table
+        merged = {**base, "_meta": {**base.get("_meta", {}), **override.get("_meta", {})}}
+        for family in ("anthropic", "openai"):
+            merged[family] = {**base.get(family, {}), **override.get(family, {})}
+        return cls(merged)
+
+    def rates(self, provider: str, model: str | None) -> Rates | None:
+        family = "anthropic" if provider == "anthropic" else "openai"
+        models = self.table.get(family, {})
+        matches = [k for k in models if model and model.startswith(k)]
+        if not matches:
+            return None
+        e = models[max(matches, key=len)]
+        try:
+            if family == "anthropic":
+                return Rates(e["input"], e["cache_write_5m"], e["cache_write_1h"], e["cache_read"])
+            mult = self.table.get("_meta", {}).get("openai_cache_write_multiplier", 1.25)
+            return Rates(e["input"], e["input"] * mult, e["input"] * mult, e["cached_input"])
+        except KeyError as exc:
+            raise InputFormatError(f"pricing entry for {model!r} lacks field {exc}.") from None
+
+
+def hit_rate(usages: list[Usage]) -> float | None:
+    """Cache reads divided by all input tokens (each provider's own total, see providers/)."""
+    total = sum(u.total_input for u in usages)
+    return sum(u.cache_read for u in usages) / total if total else None
+
+
+def cacheable_tokens(provider: str, usage: Usage) -> int:
+    """Tokens the previous call left in the cache (what the next call could have read)."""
+    if provider == "anthropic":
+        return usage.cache_read + usage.cache_write
+    if usage.cache_write:
+        return usage.cache_read + usage.cache_write
+    # OpenAI caches from 1024 tokens upward in 128-token steps (docs, checked 2026-11-09)
+    return usage.total_input // 128 * 128 if usage.total_input >= 1024 else 0
+
+
+def missed_cost(rates: Rates, missed: int, current: Usage, one_hour: bool) -> float:
+    """What the missed tokens cost beyond what a cache read would have cost.
+
+    Missed tokens that the response reports as cache writes were billed at the write rate;
+    the remainder was billed as ordinary input.
+    """
+    write_rate = rates.write_1h if one_hour else rates.write_5m
+    written = min(missed, current.cache_write)
+    extra = written * (write_rate - rates.read) + (missed - written) * (rates.input - rates.read)
+    return extra / 1_000_000

cache_autopsy/demo.py

@@ -0,0 +1,34 @@
+"""`cache-autopsy demo`: run the analyzer on the bundled practice (mock) logs."""
+from __future__ import annotations
+
+from importlib import resources
+
+from .cost import Pricing
+from .records import read_jsonl
+from .report import pct, usd
+from .verdict import analyze_records
+
+
+def samples_dir():
+    return resources.files("cache_autopsy").joinpath("samples")
+
+
+def run_demo() -> int:
+    print("NOTE: practice mock data (연습용 목(mock) 데이터). These are NOT real API logs; "
+          "usage numbers are invented to follow the providers' rules.")
+    print("The analyzer really renders, diffs and classifies the fake requests below.\n")
+    pricing = Pricing.default()
+    reports = {}
+    for label in ("broken", "fixed"):
+        path = samples_dir().joinpath(f"anthropic_{label}.jsonl")
+        with resources.as_file(path) as real:
+            reports[label] = analyze_records(read_jsonl(real), pricing)
+    print(f"{'session':<8} {'calls':>5} {'hit rate':>9} {'missed tokens':>14} {'missed cost':>12}")
+    for label, rep in reports.items():
+        s = rep.summary
+        print(f"{label:<8} {s['records']:>5} {pct(s['hit_rate']):>9} {s['missed_tokens']:>14,} {usd(s['missed_cost_usd']):>12}")
+    print("\nWhy 'broken' misses (cause: calls, missed tokens):")
+    for code, row in sorted(reports["broken"].summary["by_cause"].items(), key=lambda kv: -kv[1]["missed_tokens"]):
+        print(f"  {code:<26} {row['pairs']} call(s)  {row['missed_tokens']:,} tokens  {usd(row['missed_cost_usd'])}")
+    print("\nTry: cache-autopsy analyze <your-log>.jsonl   |   cache-autopsy causes")
+    return 0

cache_autopsy/diff.py

@@ -0,0 +1,138 @@
+"""First-divergence search and cause classification for two consecutive requests."""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .model import Divergence, InputFormatError, ProviderSpec
+from .providers import detect
+from .render import byte_offset, dumps, first_diff, locate, render_sections
+from .rules import VOLATILE
+
+# Canonical section order across providers; each provider only fills the sections it has.
+SECTION_ORDER = ("model", "tools", "system", "instructions", "messages", "input")
+HISTORY_SECTIONS = ("messages", "input")  # last sections: appending to them keeps the prefix intact
+
+
+def _tool_name(tool: Any) -> str:
+    if isinstance(tool, dict):
+        return str(tool.get("name") or (tool.get("function") or {}).get("name") or dumps(tool))
+    return dumps(tool)
+
+
+def _canonical(value: Any) -> str:
+    return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+
+
+def _volatile_codes(text_a: str, text_b: str) -> list[str]:
+    """Volatile-value kinds whose matches differ between the two texts."""
+    found: list[str] = []
+    for code, pattern in VOLATILE:
+        if code not in found and set(pattern.findall(text_a)) != set(pattern.findall(text_b)):
+            found.append(code)
+    return found
+
+
+def _is_append(name: str, va: Any, vb: Any) -> bool:
+    if name not in HISTORY_SECTIONS:
+        return False
+    if isinstance(va, list) and isinstance(vb, list):
+        return len(va) <= len(vb) and all(dumps(x) == dumps(y) for x, y in zip(va, vb))
+    if isinstance(va, str) and isinstance(vb, str):
+        return vb.startswith(va)
+    return False
+
+
+def _classify(name: str, va: Any, vb: Any, ta: str, tb: str, window_a: str, window_b: str) -> list[str]:
+    if va is None or vb is None:
+        return ["section_added_or_removed"]
+    if name == "tools" and isinstance(va, list) and isinstance(vb, list):
+        names_a, names_b = [_tool_name(t) for t in va], [_tool_name(t) for t in vb]
+        if sorted(names_a) == sorted(names_b) and names_a != names_b and \
+                _canonical(sorted(va, key=_tool_name)) == _canonical(sorted(vb, key=_tool_name)):
+            return ["tool_order_changed"]
+        if _canonical(va) == _canonical(vb):
+            return ["json_key_order_changed"]
+        return ["tool_definition_changed", *_volatile_codes(ta, tb)]
+    if name in ("system", "instructions"):
+        return _volatile_codes(ta, tb) or ["system_edited"]
+    if name in HISTORY_SECTIONS:
+        return ["history_edited", *_volatile_codes(window_a, window_b)]
+    return []
+
+
+def _structural(spec: ProviderSpec, a: dict, b: dict, ctx: int) -> tuple[Divergence | None, str | None]:
+    """Earliest section that differs. Returns (divergence, section name)."""
+    sa, sb = spec.sections(a), spec.sections(b)
+    da, db = dict(sa), dict(sb)
+    ra, _, _ = render_sections(sa)
+    rb, spans_b, bounds_b = render_sections(sb)
+    offset = 0  # equal sections have equal length, so one running offset serves both renderings
+    for name in SECTION_ORDER:
+        va, vb = da.get(name), db.get(name)
+        if name not in da and name not in db:
+            continue
+        ta = dumps(va) if name in da else ""
+        tb = dumps(vb) if name in db else ""
+        if name in da and name in db and ta == tb:
+            offset += len(ta)
+            continue
+        inner = first_diff(ta, tb) or 0
+        at = offset + inner
+        # a new request that is shorter than the old one diverges where it ends: report the section
+        path = locate(spans_b, at, name) if at < offset + len(tb) else name
+        append = _is_append(name, va, vb)
+        if append and isinstance(va, list) and len(vb) > len(va):
+            # appended items: point at the first new member instead of the closing bracket
+            target = f"{name}[{len(va)}]"
+            start = next((sp[0] for sp in spans_b if sp[2] == target), None)
+            if start is not None:
+                at, path = start, target
+        window_a, window_b = ra[max(0, at - ctx): at + 2 * ctx], rb[max(0, at - ctx): at + 2 * ctx]
+        causes = ["append_only_ok"] if append else _classify(name, va, vb, ta, tb, window_a, window_b)
+        div = Divergence(
+            byte_offset=byte_offset(rb, at),
+            json_path=path,
+            reason=spec.section_reason.get(name, f"{name}_changed"),
+            causes=causes,
+            context_before=rb[max(0, at - ctx): at],
+            context_after=rb[at: at + ctx],
+            context_prev_after=ra[at: at + ctx],
+            prefix_intact=append,
+        )
+        return div, name
+    return None, None
+
+
+def _setting_divergence(spec: ProviderSpec, a: dict, b: dict, ctx: int) -> Divergence | None:
+    for (reason, label, path, va), (_, _, _, vb) in zip(spec.settings(a), spec.settings(b)):
+        if dumps(va) != dumps(vb):
+            return Divergence(
+                byte_offset=None, json_path=path, reason=reason, causes=[f"setting_changed:{label}"],
+                context_before="", context_after=dumps(vb)[:ctx], context_prev_after=dumps(va)[:ctx])
+    return None
+
+
+def diff_requests(a: dict, b: dict, provider: str | None = None, context: int = 40) -> Divergence:
+    """Compare two request bodies and report the first point where the cacheable prefix differs.
+
+    ``a`` is the earlier request, ``b`` the later one. Offsets refer to a local rendering of the
+    prefix in the provider's documented order, not to the provider's private bytes.
+    """
+    for label, req in (("first", a), ("second", b)):
+        if not isinstance(req, dict):
+            raise InputFormatError(f"the {label} request must be a JSON object, got {type(req).__name__}")
+    spec = detect(b, hint=provider)
+    struct, section = _structural(spec, a, b, context)
+    setting = _setting_divergence(spec, a, b, context)
+    if struct and section == "model":
+        return struct
+    if setting and spec.settings_first:
+        return setting
+    if struct and not struct.prefix_intact:
+        return struct
+    if setting:
+        return setting
+    if struct:
+        return struct
+    return Divergence(byte_offset=None, json_path=None, reason="unchanged", causes=["identical"], prefix_intact=True)

cache_autopsy/lint.py

@@ -0,0 +1,92 @@
+"""Single-request checks: patterns that will break caching on the next call."""
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from .model import Finding
+from .providers import detect
+from .render import estimate_tokens, render_sections
+from .rules import CAUSES, find_volatile
+
+MAX_BREAKPOINTS = 4
+
+
+def _strings(node: Any, path: str):
+    """Yield (json path, string) for every string value in ``node``."""
+    if isinstance(node, str):
+        yield path, node
+    elif isinstance(node, dict):
+        for key, child in node.items():
+            if key != "cache_control":
+                yield from _strings(child, f"{path}.{key}" if path else key)
+    elif isinstance(node, list):
+        for i, child in enumerate(node):
+            yield from _strings(child, f"{path}[{i}]")
+
+
+_ORDER = {"tools": 0, "system": 1, "messages": 2}
+
+
+def _position(path: str) -> tuple[int, int]:
+    """Sort key of a JSON path inside the prefix: (section order, top-level index)."""
+    m = re.match(r"(\w+)(?:\[(\d+)\])?", path)
+    return (_ORDER.get(m.group(1), 3), int(m.group(2) or 0)) if m else (3, 0)
+
+
+def lint_request(req: dict, provider: str | None = None, min_tokens: int = 1024) -> list[Finding]:
+    """Return findings for one request body. Warnings and errors mean "fix before shipping"."""
+    spec = detect(req, hint=provider)
+    findings: list[Finding] = []
+    prefix_names = [n for n in ("tools", "system", "instructions") if req.get(n) is not None]
+    volatile: list[tuple[str, list[str]]] = []  # (path, codes) inside the stable part of the prefix
+    for name in prefix_names:
+        for path, text in _strings(req[name], name):
+            codes = find_volatile(text)
+            if codes:
+                volatile.append((path, codes))
+    # An OpenAI chat prompt keeps its system text inside messages: scan leading system/developer ones.
+    for i, msg in enumerate(req.get("messages") or []):
+        if isinstance(msg, dict) and msg.get("role") in ("system", "developer"):
+            for path, text in _strings(msg.get("content"), f"messages[{i}].content"):
+                codes = find_volatile(text)
+                if codes:
+                    volatile.append((path, codes))
+    for path, codes in volatile:
+        for code in codes:
+            findings.append(Finding(code, "warning", path,
+                                    f"a value that changes between calls sits in the cached prefix ({path}).",
+                                    CAUSES[code][2]))
+
+    markers = spec.breakpoints(req)
+    if spec.name == "anthropic":
+        if not markers:
+            findings.append(Finding("no_cache_marker", "info", "", "no cache_control marker found; nothing will be cached.",
+                                    CAUSES["no_cache_marker"][2]))
+        explicit = [m for m in markers if m != "cache_control"]
+        if len(markers) > MAX_BREAKPOINTS:
+            findings.append(Finding("too_many_breakpoints", "error", markers[MAX_BREAKPOINTS],
+                                    f"{len(markers)} cache markers found; the limit is {MAX_BREAKPOINTS}.",
+                                    CAUSES["too_many_breakpoints"][2]))
+        for m in explicit:
+            # a marker caches everything before it: any changing value at or before it breaks the key
+            culprit = next((path for path, _ in volatile if _position(path) <= _position(m)), None)
+            if culprit:
+                findings.append(Finding("breakpoint_on_volatile_block", "warning", m,
+                                        f"{m} is cached, but the prefix up to it contains a changing value "
+                                        f"({culprit}), so the cache key never repeats.",
+                                        CAUSES["breakpoint_on_volatile_block"][2]))
+
+    text, _, _ = render_sections(spec.sections(req))
+    if markers or spec.name != "anthropic":
+        est = estimate_tokens(text)
+        if est < min_tokens:
+            findings.append(Finding("below_min_tokens", "info", "",
+                                    f"estimated ~{est} tokens (characters/4) is below {min_tokens}; "
+                                    "providers skip caching under their per-model minimum.",
+                                    CAUSES["below_min_tokens"][2]))
+    return findings
+
+
+def has_problems(findings: list[Finding]) -> bool:
+    return any(f.severity in ("warning", "error") for f in findings)