cache-doctor 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cache_doctor/__init__.py +4 -0
- cache_doctor/analyze.py +169 -0
- cache_doctor/cli.py +105 -0
- cache_doctor/report.py +69 -0
- cache_doctor/suggest.py +124 -0
- cache_doctor-0.1.0.dist-info/METADATA +140 -0
- cache_doctor-0.1.0.dist-info/RECORD +11 -0
- cache_doctor-0.1.0.dist-info/WHEEL +5 -0
- cache_doctor-0.1.0.dist-info/entry_points.txt +2 -0
- cache_doctor-0.1.0.dist-info/licenses/LICENSE +21 -0
- cache_doctor-0.1.0.dist-info/top_level.txt +1 -0
cache_doctor/__init__.py
ADDED
cache_doctor/analyze.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Parse semantic-cache event logs and compute health metrics.
|
|
2
|
+
|
|
3
|
+
Log format: JSONL, one event per line. Fields (all except ``prompt``
|
|
4
|
+
optional)::
|
|
5
|
+
|
|
6
|
+
{
|
|
7
|
+
"ts": "2026-10-11T01:00:00Z", # ISO-8601 timestamp
|
|
8
|
+
"prompt": "What is the capital of France?",
|
|
9
|
+
"cache_key": "9f3a...",
|
|
10
|
+
"hit": true, # served from cache?
|
|
11
|
+
"similarity": 0.93, # similarity score of the hit, if any
|
|
12
|
+
"threshold": 0.90, # configured similarity threshold
|
|
13
|
+
"ttl_seconds": 3600,
|
|
14
|
+
"cache_age_seconds": 1200, # age of the entry served (hits only)
|
|
15
|
+
"miss_reason": "no_key", # "no_key" | "expired" | "low_similarity"
|
|
16
|
+
"flag_wrong": false # ground truth: the hit was a false positive
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
Everything is stdlib-only.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import re
|
|
26
|
+
from collections import Counter, defaultdict
|
|
27
|
+
from datetime import datetime
|
|
28
|
+
from typing import Any
|
|
29
|
+
|
|
30
|
+
MISS_REASONS = ("no_key", "expired", "low_similarity")
|
|
31
|
+
BORDERLINE_MARGIN = 0.05 # hits within +margin of threshold are "borderline"
|
|
32
|
+
STALE_FRACTION = 0.9 # hits served past 90% of TTL are "stale"
|
|
33
|
+
|
|
34
|
+
_WS_RE = re.compile(r"\s+")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def norm_prompt(prompt: str) -> str:
|
|
38
|
+
"""Canonical prompt form used for key-fragmentation detection."""
|
|
39
|
+
return _WS_RE.sub(" ", (prompt or "").strip()).casefold()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _parse_ts(value: Any) -> datetime | None:
|
|
43
|
+
if not value:
|
|
44
|
+
return None
|
|
45
|
+
try:
|
|
46
|
+
return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
47
|
+
except ValueError:
|
|
48
|
+
return None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def parse_log(path: str) -> list[dict]:
|
|
52
|
+
"""Read a JSONL cache log. Raises ValueError on malformed lines."""
|
|
53
|
+
events: list[dict] = []
|
|
54
|
+
with open(path, "r", encoding="utf-8") as fh:
|
|
55
|
+
for lineno, line in enumerate(fh, start=1):
|
|
56
|
+
line = line.strip()
|
|
57
|
+
if not line:
|
|
58
|
+
continue
|
|
59
|
+
try:
|
|
60
|
+
obj = json.loads(line)
|
|
61
|
+
except json.JSONDecodeError as exc:
|
|
62
|
+
raise ValueError(f"line {lineno}: invalid JSON ({exc.msg})") from exc
|
|
63
|
+
if not isinstance(obj, dict):
|
|
64
|
+
raise ValueError(f"line {lineno}: expected a JSON object")
|
|
65
|
+
events.append(obj)
|
|
66
|
+
return events
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _bucket_hour(ts: datetime | None) -> str:
|
|
70
|
+
return ts.strftime("%Y-%m-%dT%H:00") if ts else "unknown"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def compute_metrics(events: list[dict]) -> dict:
|
|
74
|
+
"""Compute the full health-metrics dict for a list of cache events."""
|
|
75
|
+
total = len(events)
|
|
76
|
+
hits = [e for e in events if e.get("hit")]
|
|
77
|
+
misses = [e for e in events if not e.get("hit")]
|
|
78
|
+
hit_count = len(hits)
|
|
79
|
+
miss_count = len(misses)
|
|
80
|
+
|
|
81
|
+
miss_reasons: Counter = Counter()
|
|
82
|
+
for e in misses:
|
|
83
|
+
reason = e.get("miss_reason") or "no_key"
|
|
84
|
+
miss_reasons[reason if reason in MISS_REASONS else "no_key"] += 1
|
|
85
|
+
|
|
86
|
+
thresholds = [e["threshold"] for e in events
|
|
87
|
+
if isinstance(e.get("threshold"), (int, float))]
|
|
88
|
+
threshold = (sorted(thresholds)[len(thresholds) // 2]
|
|
89
|
+
if thresholds else 0.9)
|
|
90
|
+
|
|
91
|
+
hit_sims = [e["similarity"] for e in hits
|
|
92
|
+
if isinstance(e.get("similarity"), (int, float))]
|
|
93
|
+
miss_sims = [e["similarity"] for e in misses
|
|
94
|
+
if isinstance(e.get("similarity"), (int, float))]
|
|
95
|
+
|
|
96
|
+
# Borderline hits: barely above the threshold -> false-positive risk.
|
|
97
|
+
borderline_hits = [e for e in hits
|
|
98
|
+
if isinstance(e.get("similarity"), (int, float))
|
|
99
|
+
and e["similarity"] < threshold + BORDERLINE_MARGIN]
|
|
100
|
+
|
|
101
|
+
# Ground-truth confirmed pollution: paraphrase served the wrong response.
|
|
102
|
+
wrong_hits = [e for e in hits if e.get("flag_wrong")]
|
|
103
|
+
|
|
104
|
+
# Key fragmentation: the same normalized prompt stored under many keys.
|
|
105
|
+
by_prompt: dict[str, set] = defaultdict(set)
|
|
106
|
+
for e in events:
|
|
107
|
+
p = e.get("prompt")
|
|
108
|
+
k = e.get("cache_key")
|
|
109
|
+
if p and k:
|
|
110
|
+
by_prompt[norm_prompt(p)].add(str(k))
|
|
111
|
+
frag_groups = {p: sorted(keys) for p, keys in by_prompt.items()
|
|
112
|
+
if len(keys) > 1}
|
|
113
|
+
frag_requests = sum(1 for e in events
|
|
114
|
+
if e.get("prompt") and e.get("cache_key")
|
|
115
|
+
and norm_prompt(e["prompt"]) in frag_groups)
|
|
116
|
+
|
|
117
|
+
# Stale serving: hits from entries that were nearly expired.
|
|
118
|
+
stale_hits = [e for e in hits
|
|
119
|
+
if isinstance(e.get("cache_age_seconds"), (int, float))
|
|
120
|
+
and isinstance(e.get("ttl_seconds"), (int, float))
|
|
121
|
+
and e["ttl_seconds"] > 0
|
|
122
|
+
and e["cache_age_seconds"] >= STALE_FRACTION * e["ttl_seconds"]]
|
|
123
|
+
|
|
124
|
+
# Rolling hit rate per hour (chronological).
|
|
125
|
+
buckets: dict[str, list[int]] = defaultdict(lambda: [0, 0])
|
|
126
|
+
for e in events:
|
|
127
|
+
b = _bucket_hour(_parse_ts(e.get("ts")))
|
|
128
|
+
buckets[b][0] += 1
|
|
129
|
+
buckets[b][1] += 1 if e.get("hit") else 0
|
|
130
|
+
rolling = {b: round(h / t, 4) for b, (t, h) in sorted(buckets.items())}
|
|
131
|
+
|
|
132
|
+
# Near misses: misses whose best similarity almost reached the threshold.
|
|
133
|
+
near_misses = [e for e in misses
|
|
134
|
+
if isinstance(e.get("similarity"), (int, float))
|
|
135
|
+
and threshold - 0.05 <= e["similarity"] < threshold]
|
|
136
|
+
|
|
137
|
+
expired_misses = miss_reasons.get("expired", 0)
|
|
138
|
+
|
|
139
|
+
def _pctl(values: list[float], q: float) -> float | None:
|
|
140
|
+
if not values:
|
|
141
|
+
return None
|
|
142
|
+
s = sorted(values)
|
|
143
|
+
return round(s[min(len(s) - 1, int(q * len(s)))], 4)
|
|
144
|
+
|
|
145
|
+
return {
|
|
146
|
+
"total": total,
|
|
147
|
+
"hits": hit_count,
|
|
148
|
+
"misses": miss_count,
|
|
149
|
+
"hit_rate": round(hit_count / total, 4) if total else 0.0,
|
|
150
|
+
"miss_reasons": dict(miss_reasons),
|
|
151
|
+
"expired_misses": expired_misses,
|
|
152
|
+
"expired_miss_share": round(expired_misses / miss_count, 4) if miss_count else 0.0,
|
|
153
|
+
"threshold": round(threshold, 4),
|
|
154
|
+
"borderline_hits": len(borderline_hits),
|
|
155
|
+
"borderline_share": round(len(borderline_hits) / hit_count, 4) if hit_count else 0.0,
|
|
156
|
+
"wrong_hits": len(wrong_hits),
|
|
157
|
+
"near_misses": len(near_misses),
|
|
158
|
+
"near_miss_share": round(len(near_misses) / miss_count, 4) if miss_count else 0.0,
|
|
159
|
+
"frag_groups": len(frag_groups),
|
|
160
|
+
"frag_requests": frag_requests,
|
|
161
|
+
"frag_share": round(frag_requests / total, 4) if total else 0.0,
|
|
162
|
+
"stale_hits": len(stale_hits),
|
|
163
|
+
"stale_share": round(len(stale_hits) / hit_count, 4) if hit_count else 0.0,
|
|
164
|
+
"avg_sim_hit": round(sum(hit_sims) / len(hit_sims), 4) if hit_sims else None,
|
|
165
|
+
"p10_sim_hit": _pctl(hit_sims, 0.10),
|
|
166
|
+
"avg_sim_miss": round(sum(miss_sims) / len(miss_sims), 4) if miss_sims else None,
|
|
167
|
+
"rolling_hourly": rolling,
|
|
168
|
+
"events_analyzed": total,
|
|
169
|
+
}
|
cache_doctor/cli.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""CLI entry point for cache-doctor."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from . import __version__
|
|
10
|
+
from .analyze import compute_metrics, parse_log
|
|
11
|
+
from .report import render_metrics, render_suggestions
|
|
12
|
+
from .suggest import suggest
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _cmd_analyze(args: argparse.Namespace) -> int:
|
|
16
|
+
try:
|
|
17
|
+
events = parse_log(args.log)
|
|
18
|
+
except (OSError, ValueError) as exc:
|
|
19
|
+
print(f"cache-doctor: error: {exc}", file=sys.stderr)
|
|
20
|
+
return 2
|
|
21
|
+
metrics = compute_metrics(events)
|
|
22
|
+
if args.json:
|
|
23
|
+
print(json.dumps(metrics, indent=2))
|
|
24
|
+
else:
|
|
25
|
+
print(render_metrics(metrics))
|
|
26
|
+
print()
|
|
27
|
+
print(render_suggestions(suggest(metrics)))
|
|
28
|
+
if args.state:
|
|
29
|
+
try:
|
|
30
|
+
with open(args.state, "w", encoding="utf-8") as fh:
|
|
31
|
+
json.dump(metrics, fh, indent=2)
|
|
32
|
+
except OSError as exc:
|
|
33
|
+
print(f"cache-doctor: warning: cannot write state: {exc}",
|
|
34
|
+
file=sys.stderr)
|
|
35
|
+
return 0
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _load_metrics_for_suggest(args: argparse.Namespace) -> dict | None:
|
|
39
|
+
if args.log:
|
|
40
|
+
try:
|
|
41
|
+
return compute_metrics(parse_log(args.log))
|
|
42
|
+
except (OSError, ValueError) as exc:
|
|
43
|
+
print(f"cache-doctor: error: {exc}", file=sys.stderr)
|
|
44
|
+
return None
|
|
45
|
+
if args.state:
|
|
46
|
+
try:
|
|
47
|
+
with open(args.state, encoding="utf-8") as fh:
|
|
48
|
+
data = json.load(fh)
|
|
49
|
+
if not isinstance(data, dict):
|
|
50
|
+
raise ValueError("state file must contain a JSON object")
|
|
51
|
+
return data
|
|
52
|
+
except (OSError, ValueError) as exc:
|
|
53
|
+
print(f"cache-doctor: error: {exc}", file=sys.stderr)
|
|
54
|
+
return None
|
|
55
|
+
print("cache-doctor: error: suggest needs --log or --state",
|
|
56
|
+
file=sys.stderr)
|
|
57
|
+
return None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _cmd_suggest(args: argparse.Namespace) -> int:
|
|
61
|
+
metrics = _load_metrics_for_suggest(args)
|
|
62
|
+
if metrics is None:
|
|
63
|
+
return 2
|
|
64
|
+
out = suggest(metrics)
|
|
65
|
+
if args.json:
|
|
66
|
+
print(json.dumps(out, indent=2))
|
|
67
|
+
else:
|
|
68
|
+
print(render_suggestions(out))
|
|
69
|
+
return 0
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
73
|
+
p = argparse.ArgumentParser(
|
|
74
|
+
prog="cache-doctor",
|
|
75
|
+
description="Semantic-cache health diagnostics: hit rate, expiry, "
|
|
76
|
+
"and cache-pollution analysis with optimization advice.")
|
|
77
|
+
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
78
|
+
|
|
79
|
+
sub = p.add_subparsers(dest="command", required=True)
|
|
80
|
+
|
|
81
|
+
a = sub.add_parser("analyze", help="Analyze a cache event log (JSONL).")
|
|
82
|
+
a.add_argument("log", help="Path to the JSONL cache log.")
|
|
83
|
+
a.add_argument("--json", action="store_true",
|
|
84
|
+
help="Emit metrics as JSON instead of a text report.")
|
|
85
|
+
a.add_argument("--state", default=None, metavar="PATH",
|
|
86
|
+
help="Also save metrics JSON to PATH for later `suggest`.")
|
|
87
|
+
a.set_defaults(func=_cmd_analyze)
|
|
88
|
+
|
|
89
|
+
s = sub.add_parser("suggest", help="Show optimization suggestions.")
|
|
90
|
+
s.add_argument("--log", default=None, help="Path to the JSONL cache log.")
|
|
91
|
+
s.add_argument("--state", default=None,
|
|
92
|
+
help="Path to a metrics JSON saved by `analyze --state`.")
|
|
93
|
+
s.add_argument("--json", action="store_true",
|
|
94
|
+
help="Emit suggestions as JSON.")
|
|
95
|
+
s.set_defaults(func=_cmd_suggest)
|
|
96
|
+
return p
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def main(argv: list[str] | None = None) -> int:
|
|
100
|
+
args = build_parser().parse_args(argv)
|
|
101
|
+
return args.func(args)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
if __name__ == "__main__":
|
|
105
|
+
raise SystemExit(main())
|
cache_doctor/report.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Human-readable report rendering for cache-doctor."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
_BAR_WIDTH = 30
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _bar(frac: float) -> str:
|
|
9
|
+
filled = int(round(max(0.0, min(1.0, frac)) * _BAR_WIDTH))
|
|
10
|
+
return "#" * filled + "-" * (_BAR_WIDTH - filled)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _pct(frac: float | None) -> str:
|
|
14
|
+
return f"{frac * 100:6.1f}%" if frac is not None else " n/a"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def render_metrics(m: dict) -> str:
|
|
18
|
+
lines = [
|
|
19
|
+
"cache-doctor health report",
|
|
20
|
+
"=" * 44,
|
|
21
|
+
f"events analyzed : {m['total']:>10}",
|
|
22
|
+
f"cache hits : {m['hits']:>10} {_pct(m['hit_rate'])} [{_bar(m['hit_rate'])}]",
|
|
23
|
+
f"cache misses : {m['misses']:>10}",
|
|
24
|
+
f"similarity thr. : {m['threshold']:>10}",
|
|
25
|
+
"",
|
|
26
|
+
"miss reasons:",
|
|
27
|
+
]
|
|
28
|
+
for reason, count in sorted(m.get("miss_reasons", {}).items()):
|
|
29
|
+
share = count / m["misses"] if m["misses"] else 0.0
|
|
30
|
+
lines.append(f" {reason:<15} {count:>6} {_pct(share)}")
|
|
31
|
+
lines += [
|
|
32
|
+
"",
|
|
33
|
+
"pollution signals:",
|
|
34
|
+
f" borderline hits : {m['borderline_hits']:>6} ({_pct(m['borderline_share'])} of hits)",
|
|
35
|
+
f" wrong hits : {m['wrong_hits']:>6} <- flag_wrong=true",
|
|
36
|
+
f" near misses : {m['near_misses']:>6} ({_pct(m['near_miss_share'])} of misses)",
|
|
37
|
+
"",
|
|
38
|
+
"policy signals:",
|
|
39
|
+
f" expired misses : {m['expired_misses']:>6} ({_pct(m['expired_miss_share'])} of misses)",
|
|
40
|
+
f" stale hits : {m['stale_hits']:>6} ({_pct(m['stale_share'])} of hits)",
|
|
41
|
+
f" frag. groups : {m['frag_groups']:>6} ({m['frag_requests']} requests)",
|
|
42
|
+
"",
|
|
43
|
+
"similarity stats:",
|
|
44
|
+
f" avg hit sim : {m['avg_sim_hit'] if m['avg_sim_hit'] is not None else 'n/a':>10}",
|
|
45
|
+
f" p10 hit sim : {m['p10_sim_hit'] if m['p10_sim_hit'] is not None else 'n/a':>10}",
|
|
46
|
+
f" avg miss sim : {m['avg_sim_miss'] if m['avg_sim_miss'] is not None else 'n/a':>10}",
|
|
47
|
+
]
|
|
48
|
+
rolling = m.get("rolling_hourly") or {}
|
|
49
|
+
if rolling:
|
|
50
|
+
lines += ["", "rolling hourly hit rate:"]
|
|
51
|
+
for hour, rate in rolling.items():
|
|
52
|
+
lines.append(f" {hour} {_pct(rate)} [{_bar(rate)}]")
|
|
53
|
+
return "\n".join(lines)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
_SEV_MARK = {"critical": "!!", "high": "**", "medium": "*", "low": "-", "info": "ok"}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def render_suggestions(suggestions: list[dict]) -> str:
|
|
60
|
+
lines = ["optimization suggestions", "=" * 44]
|
|
61
|
+
for i, s in enumerate(suggestions, 1):
|
|
62
|
+
mark = _SEV_MARK.get(s["severity"], "?")
|
|
63
|
+
lines += [
|
|
64
|
+
f"{i}. [{mark}] {s['severity'].upper():8} {s['title']}",
|
|
65
|
+
f" {s['detail']}",
|
|
66
|
+
f" -> {s['action']}",
|
|
67
|
+
"",
|
|
68
|
+
]
|
|
69
|
+
return "\n".join(lines).rstrip()
|
cache_doctor/suggest.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Rule-based optimization suggestions from cache health metrics."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3, "info": 4}
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _mk(severity: str, code: str, title: str, detail: str, action: str) -> dict:
|
|
9
|
+
return {"severity": severity, "code": code, "title": title,
|
|
10
|
+
"detail": detail, "action": action}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def suggest(metrics: dict) -> list[dict]:
|
|
14
|
+
"""Return prioritized suggestions, most severe first."""
|
|
15
|
+
out: list[dict] = []
|
|
16
|
+
total = metrics.get("total", 0) or 0
|
|
17
|
+
hits = metrics.get("hits", 0) or 0
|
|
18
|
+
misses = metrics.get("misses", 0) or 0
|
|
19
|
+
hit_rate = metrics.get("hit_rate", 0.0) or 0.0
|
|
20
|
+
threshold = metrics.get("threshold", 0.9)
|
|
21
|
+
|
|
22
|
+
if total == 0:
|
|
23
|
+
return [_mk("info", "NO_DATA", "No cache events to analyze",
|
|
24
|
+
"The log contained zero events.",
|
|
25
|
+
"Point cache-doctor at a JSONL log with at least a few "
|
|
26
|
+
"dozen requests for meaningful diagnostics.")]
|
|
27
|
+
|
|
28
|
+
# 1. Confirmed pollution: paraphrases served the wrong cached response.
|
|
29
|
+
wrong = metrics.get("wrong_hits", 0) or 0
|
|
30
|
+
if wrong:
|
|
31
|
+
out.append(_mk(
|
|
32
|
+
"critical", "POLLUTION_CONFIRMED",
|
|
33
|
+
f"{wrong} cache hit(s) served the WRONG response",
|
|
34
|
+
f"{wrong} hit(s) were flagged (flag_wrong=true): a lightly "
|
|
35
|
+
f"rewritten prompt matched a cached response it should not have. "
|
|
36
|
+
f"This is cache pollution - silent, wrong answers served fast.",
|
|
37
|
+
f"Raise the similarity threshold above {threshold}, require "
|
|
38
|
+
"stricter embedding agreement (e.g. rerank top-3 candidates), "
|
|
39
|
+
"and audit prompts with similarity below "
|
|
40
|
+
f"{round(threshold + 0.05, 2)}."))
|
|
41
|
+
|
|
42
|
+
# 2. Borderline hits: high false-positive risk.
|
|
43
|
+
bshare = metrics.get("borderline_share", 0.0) or 0.0
|
|
44
|
+
if bshare >= 0.10 and hits:
|
|
45
|
+
out.append(_mk(
|
|
46
|
+
"high", "POLLUTION_RISK",
|
|
47
|
+
f"{round(bshare * 100, 1)}% of hits are borderline",
|
|
48
|
+
f"{metrics.get('borderline_hits', 0)} hits landed within "
|
|
49
|
+
f"+0.05 of the threshold ({threshold}). Slight prompt rewrites "
|
|
50
|
+
"slipping through are the classic pollution vector.",
|
|
51
|
+
"Bump the threshold by 0.02-0.05 and re-run analyze to confirm "
|
|
52
|
+
"the hit rate does not collapse; consider embedding-model "
|
|
53
|
+
"upgrade if legitimate paraphrases then stop matching."))
|
|
54
|
+
|
|
55
|
+
# 3. Low hit rate overall.
|
|
56
|
+
if hit_rate < 0.35:
|
|
57
|
+
out.append(_mk(
|
|
58
|
+
"high", "LOW_HIT_RATE",
|
|
59
|
+
f"Hit rate {round(hit_rate * 100, 1)}% is low",
|
|
60
|
+
f"Only {hits} of {total} requests were served from cache. The "
|
|
61
|
+
"cache is paying storage and lookup cost for little benefit.",
|
|
62
|
+
"Check prompt normalization (whitespace/case), widen the "
|
|
63
|
+
"similarity threshold cautiously, and confirm the cache is "
|
|
64
|
+
"populated on miss (write-through)."))
|
|
65
|
+
|
|
66
|
+
# 4. Near misses: threshold may be too strict.
|
|
67
|
+
nshare = metrics.get("near_miss_share", 0.0) or 0.0
|
|
68
|
+
if nshare >= 0.15 and hit_rate < 0.6:
|
|
69
|
+
out.append(_mk(
|
|
70
|
+
"medium", "THRESHOLD_STRICT",
|
|
71
|
+
f"{round(nshare * 100, 1)}% of misses almost matched",
|
|
72
|
+
f"{metrics.get('near_misses', 0)} misses scored within 0.05 "
|
|
73
|
+
f"below the threshold ({threshold}) - likely legitimate "
|
|
74
|
+
"paraphrases being rejected.",
|
|
75
|
+
"Lower the threshold by 0.02-0.03 and watch wrong_hits; pair "
|
|
76
|
+
"with a reranker so looser matching stays safe."))
|
|
77
|
+
|
|
78
|
+
# 5. Expiry waste.
|
|
79
|
+
eshare = metrics.get("expired_miss_share", 0.0) or 0.0
|
|
80
|
+
if eshare >= 0.15 and misses:
|
|
81
|
+
out.append(_mk(
|
|
82
|
+
"medium", "EXPIRY_WASTE",
|
|
83
|
+
f"{round(eshare * 100, 1)}% of misses are expired entries",
|
|
84
|
+
f"{metrics.get('expired_misses', 0)} requests missed only "
|
|
85
|
+
"because the entry had expired - the content was cacheable "
|
|
86
|
+
"and would have been a hit.",
|
|
87
|
+
"Increase TTL (or switch to sliding TTL) for stable content; "
|
|
88
|
+
"split TTLs by content volatility if some entries go stale."))
|
|
89
|
+
|
|
90
|
+
# 6. Key fragmentation: same prompt, many keys.
|
|
91
|
+
fgroups = metrics.get("frag_groups", 0) or 0
|
|
92
|
+
if fgroups:
|
|
93
|
+
out.append(_mk(
|
|
94
|
+
"medium", "KEY_FRAGMENTATION",
|
|
95
|
+
f"{fgroups} prompt group(s) stored under multiple keys",
|
|
96
|
+
f"{metrics.get('frag_requests', 0)} requests hit distinct cache "
|
|
97
|
+
"keys for prompts that are identical after normalization - "
|
|
98
|
+
"duplicate storage and diluted hit rates.",
|
|
99
|
+
"Normalize prompts before keying: trim, collapse whitespace, "
|
|
100
|
+
"casefold, and strip volatile prefixes (timestamps, session "
|
|
101
|
+
"ids) from the cache key."))
|
|
102
|
+
|
|
103
|
+
# 7. Stale serving.
|
|
104
|
+
sshare = metrics.get("stale_share", 0.0) or 0.0
|
|
105
|
+
if sshare >= 0.10 and hits:
|
|
106
|
+
out.append(_mk(
|
|
107
|
+
"low", "STALE_SERVING",
|
|
108
|
+
f"{round(sshare * 100, 1)}% of hits served near-expiry entries",
|
|
109
|
+
f"{metrics.get('stale_hits', 0)} hits were served from entries "
|
|
110
|
+
"past 90% of their TTL - fresh on the wire, stale in spirit.",
|
|
111
|
+
"Add background refresh (stale-while-revalidate) or shorten "
|
|
112
|
+
"TTL for fast-changing content."))
|
|
113
|
+
|
|
114
|
+
# 8. All clear.
|
|
115
|
+
if not out:
|
|
116
|
+
out.append(_mk(
|
|
117
|
+
"info", "HEALTHY",
|
|
118
|
+
f"Cache looks healthy ({round(hit_rate * 100, 1)}% hit rate)",
|
|
119
|
+
f"{hits} hits / {total} requests, no pollution signals, no "
|
|
120
|
+
"fragmentation, expiry under control.",
|
|
121
|
+
"Keep monitoring; re-run after traffic mix changes or "
|
|
122
|
+
"embedding-model upgrades."))
|
|
123
|
+
|
|
124
|
+
return sorted(out, key=lambda s: SEVERITY_ORDER.get(s["severity"], 9))
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cache-doctor
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Semantic-cache health diagnostics: hit rate, expiry, and cache-pollution analysis for LLM response caches
|
|
5
|
+
Author: hao li
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/hahahahahahahahah6/cache-doctor
|
|
8
|
+
Project-URL: Repository, https://github.com/hahahahahahahahah6/cache-doctor
|
|
9
|
+
Project-URL: Issues, https://github.com/hahahahahahahahah6/cache-doctor/issues
|
|
10
|
+
Keywords: llm,semantic-cache,cache,diagnostics,prompt-caching
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
16
|
+
Requires-Python: >=3.9
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# cache-doctor
|
|
22
|
+
|
|
23
|
+
The checkup your semantic cache never had.
|
|
24
|
+
|
|
25
|
+
LLM gateways all ship semantic caches now — but every tool on the market only
|
|
26
|
+
*operates* the cache (store, lookup, evict). Nobody diagnoses it. **cache-doctor
|
|
27
|
+
is the first CLI that performs a health check on a semantic cache**: it reads
|
|
28
|
+
your cache event log and reports hit rate, expiry waste, key fragmentation, and
|
|
29
|
+
— the silent killer — **cache pollution**, where a lightly rewritten prompt
|
|
30
|
+
matches a cached response it should never have matched.
|
|
31
|
+
|
|
32
|
+
## Differentiation
|
|
33
|
+
|
|
34
|
+
| | Cache products (Redis, GPTCache, LangChain caches, …) | cache-doctor |
|
|
35
|
+
|---|---|---|
|
|
36
|
+
| Job | Store and serve cached LLM responses | **Diagnose** the cache |
|
|
37
|
+
| Hit rate | You compute it yourself | Measured, with rolling hourly trend |
|
|
38
|
+
| Expiry | Configured once, forgotten | Expired-miss share + stale-serving detection |
|
|
39
|
+
| Pollution | Invisible | Borderline-hit risk + ground-truth `flag_wrong` detection |
|
|
40
|
+
| Key design | Whatever you happened to key on | Fragmentation detection (same prompt, many keys) |
|
|
41
|
+
| Advice | None | Prioritized, severity-ranked suggestions |
|
|
42
|
+
|
|
43
|
+
Think of it as the relationship between a database and a database health
|
|
44
|
+
monitor: the cache is the patient, cache-doctor is the checkup instrument.
|
|
45
|
+
|
|
46
|
+
## Install
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install cache-doctor
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Zero dependencies, stdlib only. Python ≥ 3.9. Works fully offline — your
|
|
53
|
+
prompts never leave the machine.
|
|
54
|
+
|
|
55
|
+
## Usage
|
|
56
|
+
|
|
57
|
+
Point it at a JSONL cache log — one event per line:
|
|
58
|
+
|
|
59
|
+
```jsonc
|
|
60
|
+
{"ts": "2026-10-11T01:00:01Z", "prompt": "What is the capital of France?",
|
|
61
|
+
"cache_key": "k001", "hit": false, "miss_reason": "no_key",
|
|
62
|
+
"threshold": 0.9, "ttl_seconds": 3600}
|
|
63
|
+
{"ts": "2026-10-11T01:00:22Z", "prompt": "Capital of France?",
|
|
64
|
+
"cache_key": "k001", "hit": true, "similarity": 0.91,
|
|
65
|
+
"threshold": 0.9, "ttl_seconds": 3600, "cache_age_seconds": 60}
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
# Full health report + suggestions
|
|
70
|
+
cache-doctor analyze cache-log.jsonl
|
|
71
|
+
|
|
72
|
+
# Machine-readable metrics
|
|
73
|
+
cache-doctor analyze cache-log.jsonl --json
|
|
74
|
+
|
|
75
|
+
# Suggestions only (from a log, or from saved --state)
|
|
76
|
+
cache-doctor suggest --log cache-log.jsonl
|
|
77
|
+
cache-doctor analyze cache-log.jsonl --state state.json
|
|
78
|
+
cache-doctor suggest --state state.json
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
### What it reports
|
|
82
|
+
|
|
83
|
+
- **Hit rate** — overall plus a rolling hourly trend, so you can see *when* it degraded.
|
|
84
|
+
- **Miss reasons** — `no_key` / `expired` / `low_similarity`, with expired-miss share.
|
|
85
|
+
- **Pollution signals** — borderline hits (similarity within +0.05 of threshold),
|
|
86
|
+
ground-truth wrong hits (`flag_wrong: true`), and near misses.
|
|
87
|
+
- **Key fragmentation** — the same normalized prompt stored under multiple keys
|
|
88
|
+
(a prompt-normalization bug wasting storage and diluting hits).
|
|
89
|
+
- **Stale serving** — hits served from entries past 90% of their TTL.
|
|
90
|
+
- **Suggestions** — severity-ranked fixes (`POLLUTION_CONFIRMED`,
|
|
91
|
+
`THRESHOLD_STRICT`, `EXPIRY_WASTE`, `KEY_FRAGMENTATION`, `STALE_SERVING`, …).
|
|
92
|
+
|
|
93
|
+
### Log schema
|
|
94
|
+
|
|
95
|
+
| Field | Required | Meaning |
|
|
96
|
+
|---|---|---|
|
|
97
|
+
| `prompt` | yes | Full prompt text |
|
|
98
|
+
| `cache_key` | no | The key used for lookup |
|
|
99
|
+
| `hit` | yes | Was it served from cache? |
|
|
100
|
+
| `similarity` | no | Similarity score of the hit / best candidate |
|
|
101
|
+
| `threshold` | no | Configured similarity threshold (default 0.9) |
|
|
102
|
+
| `ttl_seconds` | no | TTL of the entry |
|
|
103
|
+
| `cache_age_seconds` | no | Age of the served entry (hits) |
|
|
104
|
+
| `miss_reason` | no | `no_key`, `expired`, `low_similarity` |
|
|
105
|
+
| `flag_wrong` | no | Ground truth: the hit was a false positive |
|
|
106
|
+
| `ts` | no | ISO-8601 timestamp (enables hourly trend) |
|
|
107
|
+
|
|
108
|
+
## Example
|
|
109
|
+
|
|
110
|
+
```
|
|
111
|
+
$ cache-doctor analyze examples/sample-log.jsonl
|
|
112
|
+
|
|
113
|
+
cache-doctor health report
|
|
114
|
+
============================================
|
|
115
|
+
events analyzed : 16
|
|
116
|
+
cache hits : 6 37.5% [###########-------------------]
|
|
117
|
+
cache misses : 10
|
|
118
|
+
similarity thr. : 0.9
|
|
119
|
+
|
|
120
|
+
miss reasons:
|
|
121
|
+
expired 2 20.0%
|
|
122
|
+
low_similarity 3 30.0%
|
|
123
|
+
no_key 5 50.0%
|
|
124
|
+
|
|
125
|
+
pollution signals:
|
|
126
|
+
borderline hits : 3 ( 50.0% of hits)
|
|
127
|
+
wrong hits : 1 <- flag_wrong=true
|
|
128
|
+
near misses : 3 ( 30.0% of misses)
|
|
129
|
+
...
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## Development
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
python3 -m unittest discover -s tests -p 'test_*.py'
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## License
|
|
139
|
+
|
|
140
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
cache_doctor/__init__.py,sha256=nFoCtaHeN9OtewmYekg_pGzq8nJkOxpElJgsEx3YDuM,104
|
|
2
|
+
cache_doctor/analyze.py,sha256=EIiGiZ4jQcEaHHluXtdEOWkgZ83reqFz_uASFibsHr0,6738
|
|
3
|
+
cache_doctor/cli.py,sha256=9bnvUZMKNO7rRKUczQ_xO5hjaG7zasqApNIcnflpF8g,3560
|
|
4
|
+
cache_doctor/report.py,sha256=_Vf1ebKPW4Gsu-0dwQntUfIGFzbhazC4GYsUEJ1rGdo,2697
|
|
5
|
+
cache_doctor/suggest.py,sha256=5wuP-t1AAuakuQZq2GG_t1R_EpJ3hRKdB_EzGTd6aZA,5836
|
|
6
|
+
cache_doctor-0.1.0.dist-info/licenses/LICENSE,sha256=OYhQDg7nxWoFtnF6J8NkFUA8NMw5TKv9a1WTXHY-fu4,1063
|
|
7
|
+
cache_doctor-0.1.0.dist-info/METADATA,sha256=cnVDadiVWKnOUN87rsFLXtf3HGeedHnB66NIG7UjfAM,5118
|
|
8
|
+
cache_doctor-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
9
|
+
cache_doctor-0.1.0.dist-info/entry_points.txt,sha256=en0OlKcpxkpYjG3lvDGwQBY9icUYdXKOA-qLgAulMcg,55
|
|
10
|
+
cache_doctor-0.1.0.dist-info/top_level.txt,sha256=6zORJVyxLyh-gZyaWd5xLV3eP5j9t25hNxlPF2TSqPQ,13
|
|
11
|
+
cache_doctor-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 hao li
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cache_doctor
|