llmtap-proxy 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
llmtap/__init__.py ADDED
@@ -0,0 +1,2 @@
1
+ """llmtap — the flight recorder & firewall for LLM apps."""
2
+ __version__ = "0.2.0"
llmtap/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from llmtap.cli import main
2
+
3
+ raise SystemExit(main())
llmtap/budget.py ADDED
@@ -0,0 +1,166 @@
1
+ """Spend caps (daily / per-session) and a runaway-agent loop breaker.
2
+
3
+ Loop threshold semantics: ``loop_threshold = N`` allows N identical requests (same key) within
4
+ ``loop_window_s`` seconds and blocks the (N+1)th and every further one until older attempts age out
5
+ of the window. Every call to :meth:`Budget.check` records an attempt (blocked ones included), so a
6
+ stuck agent that keeps hammering stays blocked until it pauses for a full window.
7
+ ``loop_threshold = 0`` disables the detector.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import threading
12
+ import time
13
+ from collections import OrderedDict, deque
14
+ from dataclasses import dataclass
15
+ from datetime import datetime
16
+ from typing import Callable
17
+
18
+ from .config import Config
19
+
20
+ SPENT_CACHE_TTL_S = 1.0
21
+ DEFAULT_MAX_KEYS = 10_000
22
+ MAX_HITS_PER_KEY = 10_000 # bounds per-key memory under extreme hammering
23
+
24
+
25
+ @dataclass
26
+ class Verdict:
27
+ """Outcome of a budget check. ``kind`` is "budget", "loop" or "" when allowed."""
28
+ allowed: bool
29
+ reason: str = ""
30
+ kind: str = ""
31
+
32
+
33
+ def _usd(x: float) -> str:
34
+ """Format dollars: 2 decimals normally, 4 for small non-zero amounts."""
35
+ if x != 0 and abs(x) < 1:
36
+ return f"${x:.4f}" if abs(x) < 0.01 else f"${x:.2f}"
37
+ return f"${x:,.2f}"
38
+
39
+
40
+ def _fmt_window(seconds: float) -> str:
41
+ s = int(seconds) if float(seconds).is_integer() else seconds
42
+ return f"{s}s"
43
+
44
+
45
+ def local_midnight(now: float) -> float:
46
+ """Epoch timestamp of the most recent local midnight at or before ``now``."""
47
+ dt = datetime.fromtimestamp(now)
48
+ return dt.replace(hour=0, minute=0, second=0, microsecond=0).timestamp()
49
+
50
+
51
+ class Budget:
52
+ """Thread-safe spend caps + identical-request loop detector."""
53
+
54
+ def __init__(
55
+ self,
56
+ cfg: Config,
57
+ spent_since: Callable[[float], float],
58
+ clock: Callable[[], float] = time.time,
59
+ max_keys: int = DEFAULT_MAX_KEYS,
60
+ spent_cache_ttl_s: float = SPENT_CACHE_TTL_S,
61
+ ) -> None:
62
+ self.cfg = cfg
63
+ self._spent_since = spent_since
64
+ self._clock = clock
65
+ self._max_keys = max(1, int(max_keys))
66
+ self._ttl = spent_cache_ttl_s
67
+ self._lock = threading.Lock()
68
+ self._session = 0.0
69
+ self._seen: OrderedDict[str, deque[float]] = OrderedDict()
70
+ # cached daily spend: (midnight it was computed for, fetched_at, value)
71
+ self._daily_cache: tuple[float, float, float] | None = None
72
+
73
+ # ------------------------------------------------------------------ spend
74
+ def add_spend(self, usd: float) -> None:
75
+ """Add upstream cost to the session accumulator (and the cached daily figure)."""
76
+ if not usd or usd < 0:
77
+ return
78
+ with self._lock:
79
+ self._session += usd
80
+ if self._daily_cache is not None:
81
+ m, at, v = self._daily_cache
82
+ self._daily_cache = (m, at, v + usd)
83
+
84
+ def session_spent(self) -> float:
85
+ with self._lock:
86
+ return self._session
87
+
88
+ def daily_spent(self) -> float:
89
+ """Spend since local midnight (cached for ~1s)."""
90
+ now = self._clock()
91
+ midnight = local_midnight(now)
92
+ with self._lock:
93
+ c = self._daily_cache
94
+ if c is not None and c[0] == midnight and 0 <= now - c[1] < self._ttl:
95
+ return c[2]
96
+ try:
97
+ value = float(self._spent_since(midnight) or 0.0)
98
+ except Exception: # fail open on storage errors: never block traffic because the DB hiccuped
99
+ with self._lock:
100
+ c = self._daily_cache
101
+ return c[2] if c is not None and c[0] == midnight else 0.0
102
+ with self._lock:
103
+ self._daily_cache = (midnight, now, value)
104
+ return value
105
+
106
+ # ------------------------------------------------------------------ check
107
+ def check(self, key: str) -> Verdict:
108
+ """Decide whether a request (identified by its cache ``key``) may go upstream."""
109
+ cfg = self.cfg
110
+ count = self._record_attempt(key)
111
+
112
+ daily_cap = float(cfg.budget_daily_usd or 0)
113
+ if daily_cap > 0:
114
+ spent = self.daily_spent()
115
+ if spent >= daily_cap:
116
+ return Verdict(False, (
117
+ f"llmtap: daily budget of {_usd(daily_cap)} reached ({_usd(spent)} spent today). "
118
+ "Raise it with --budget-daily or wait until midnight."), "budget")
119
+
120
+ session_cap = float(cfg.budget_session_usd or 0)
121
+ if session_cap > 0:
122
+ spent = self.session_spent()
123
+ if spent >= session_cap:
124
+ return Verdict(False, (
125
+ f"llmtap: session budget of {_usd(session_cap)} reached ({_usd(spent)} spent this session). "
126
+ "Raise it with --budget-session or restart llmtap to start a new session."), "budget")
127
+
128
+ threshold = int(cfg.loop_threshold or 0)
129
+ if threshold > 0 and count > threshold:
130
+ return Verdict(False, (
131
+ f"llmtap: runaway loop detected — this exact request was sent {count} times in "
132
+ f"{_fmt_window(cfg.loop_window_s)}. Your agent may be stuck. "
133
+ "(Disable with --loop-threshold 0)"), "loop")
134
+ return Verdict(True)
135
+
136
+ def _record_attempt(self, key: str) -> int:
137
+ """Record an attempt for ``key``; return attempts within the window including this one."""
138
+ if int(self.cfg.loop_threshold or 0) <= 0:
139
+ return 0
140
+ now = self._clock()
141
+ cutoff = now - float(self.cfg.loop_window_s or 0)
142
+ with self._lock:
143
+ dq = self._seen.get(key)
144
+ if dq is None:
145
+ dq = deque(maxlen=MAX_HITS_PER_KEY)
146
+ self._seen[key] = dq
147
+ else:
148
+ self._seen.move_to_end(key)
149
+ while dq and dq[0] <= cutoff:
150
+ dq.popleft()
151
+ dq.append(now)
152
+ while len(self._seen) > self._max_keys:
153
+ self._seen.popitem(last=False)
154
+ return len(dq)
155
+
156
+ def tracked_keys(self) -> int:
157
+ """Number of request keys currently tracked by the loop detector."""
158
+ with self._lock:
159
+ return len(self._seen)
160
+
161
+ def reset(self) -> None:
162
+ """Forget session spend, loop history and cached daily spend."""
163
+ with self._lock:
164
+ self._session = 0.0
165
+ self._seen.clear()
166
+ self._daily_cache = None
llmtap/cache.py ADDED
@@ -0,0 +1,129 @@
1
+ """Response cache: identical requests are served from SQLite instead of hitting the upstream."""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+ import json
6
+ import sqlite3
7
+ import threading
8
+ import time
9
+ from pathlib import Path
10
+
11
+ from .models import CachedResponse
12
+
13
+ VOLATILE_KEYS: frozenset[str] = frozenset({"user", "metadata", "request_id"})
14
+
15
+
16
+ def _canonical_body(req_body: bytes) -> bytes:
17
+ """Canonical JSON (sorted keys, compact) minus volatile top-level keys; raw bytes if not JSON."""
18
+ try:
19
+ obj = json.loads(req_body.decode("utf-8")) if req_body else None
20
+ except (UnicodeDecodeError, ValueError):
21
+ return bytes(req_body)
22
+ if obj is None and not req_body:
23
+ return b""
24
+ if isinstance(obj, dict):
25
+ obj = {k: v for k, v in obj.items() if k not in VOLATILE_KEYS}
26
+ return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
27
+
28
+
29
+ def cache_key(provider: str, path: str, req_body: bytes) -> str:
30
+ """sha256 hex over provider, path (no query) and the canonicalized request body."""
31
+ bare_path = path.split("?", 1)[0]
32
+ h = hashlib.sha256()
33
+ h.update(provider.encode("utf-8"))
34
+ h.update(b"\n")
35
+ h.update(bare_path.encode("utf-8"))
36
+ h.update(b"\n")
37
+ h.update(_canonical_body(req_body))
38
+ return h.hexdigest()
39
+
40
+
41
+ class ResponseCache:
42
+ """Thread-safe SQLite-backed cache of 2xx responses (table ``cache``)."""
43
+
44
+ def __init__(self, path: str, ttl_s: int = 0) -> None:
45
+ self.path = path
46
+ self.ttl_s = int(ttl_s or 0)
47
+ self._lock = threading.Lock()
48
+ self._hits = 0
49
+ if path != ":memory:":
50
+ Path(path).expanduser().parent.mkdir(parents=True, exist_ok=True)
51
+ self._conn = sqlite3.connect(path, check_same_thread=False, timeout=30.0)
52
+ with self._lock:
53
+ if path != ":memory:":
54
+ try:
55
+ self._conn.execute("PRAGMA journal_mode=WAL")
56
+ except sqlite3.DatabaseError:
57
+ pass
58
+ self._conn.execute(
59
+ "CREATE TABLE IF NOT EXISTS cache ("
60
+ " key TEXT PRIMARY KEY,"
61
+ " status INTEGER NOT NULL,"
62
+ " headers TEXT NOT NULL,"
63
+ " body TEXT NOT NULL,"
64
+ " streamed INTEGER NOT NULL,"
65
+ " cost_usd REAL NOT NULL DEFAULT 0,"
66
+ " created_at REAL NOT NULL)"
67
+ )
68
+ self._conn.commit()
69
+
70
+ def _expired(self, created_at: float) -> bool:
71
+ return self.ttl_s > 0 and (time.time() - created_at) > self.ttl_s
72
+
73
+ def get(self, key: str) -> CachedResponse | None:
74
+ """Return the cached response for ``key`` or None (missing or older than ttl)."""
75
+ with self._lock:
76
+ row = self._conn.execute(
77
+ "SELECT status, headers, body, streamed, cost_usd, created_at FROM cache WHERE key = ?", (key,)
78
+ ).fetchone()
79
+ if row is None:
80
+ return None
81
+ status, headers, body, streamed, cost, created_at = row
82
+ if self._expired(float(created_at)):
83
+ self._conn.execute("DELETE FROM cache WHERE key = ?", (key,))
84
+ self._conn.commit()
85
+ return None
86
+ try:
87
+ hdrs = json.loads(headers) or {}
88
+ except ValueError:
89
+ hdrs = {}
90
+ self._hits += 1
91
+ return CachedResponse(
92
+ status=int(status), headers=dict(hdrs), body=body, streamed=bool(streamed), cost_usd=float(cost or 0.0)
93
+ )
94
+
95
+ def put(self, key: str, resp: CachedResponse) -> None:
96
+ """Store (or replace) a response. Callers should only pass 2xx responses."""
97
+ with self._lock:
98
+ self._conn.execute(
99
+ "INSERT OR REPLACE INTO cache (key, status, headers, body, streamed, cost_usd, created_at)"
100
+ " VALUES (?, ?, ?, ?, ?, ?, ?)",
101
+ (
102
+ key,
103
+ int(resp.status),
104
+ json.dumps(dict(resp.headers or {}), sort_keys=True),
105
+ resp.body,
106
+ 1 if resp.streamed else 0,
107
+ float(resp.cost_usd or 0.0),
108
+ time.time(),
109
+ ),
110
+ )
111
+ self._conn.commit()
112
+
113
+ def stats(self) -> dict:
114
+ """{"entries": rows in table, "hits": hits served by this process}."""
115
+ with self._lock:
116
+ (n,) = self._conn.execute("SELECT COUNT(*) FROM cache").fetchone()
117
+ return {"entries": int(n), "hits": self._hits}
118
+
119
+ def clear(self) -> None:
120
+ """Delete every cached entry and reset the hit counter."""
121
+ with self._lock:
122
+ self._conn.execute("DELETE FROM cache")
123
+ self._conn.commit()
124
+ self._hits = 0
125
+
126
+ def close(self) -> None:
127
+ """Close the underlying connection."""
128
+ with self._lock:
129
+ self._conn.close()
llmtap/cassette.py ADDED
@@ -0,0 +1,172 @@
1
+ """Cassettes: VCR-style JSONL recordings of LLM exchanges for deterministic, keyless replay."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import re
6
+ import threading
7
+ import warnings
8
+ from datetime import datetime, timezone
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from .models import CachedResponse
13
+
14
+ MODES: frozenset[str] = frozenset({"record", "replay", "auto"})
15
+
16
+ # sk-ant-..., sk-proj-..., sk-... (not inside a longer word, with a reasonably long tail)
17
+ _KEY_RE = re.compile(r"(?<![A-Za-z0-9_\-])sk-(?:ant-[A-Za-z0-9]*-?|proj-)?[A-Za-z0-9_\-]{16,}")
18
+ _SENSITIVE_HEADER_PARTS = ("key", "auth", "token", "cookie", "secret", "password")
19
+
20
+
21
+ class CassetteMiss(Exception):
22
+ """Raised (by the proxy) when replay mode has no recording for a request."""
23
+
24
+
25
+ def _redact_str(s: str) -> str:
26
+ def sub(m: re.Match[str]) -> str:
27
+ return "[REDACTED:anthropic_key]" if m.group(0).startswith("sk-ant-") else "[REDACTED:api_key]"
28
+
29
+ return _KEY_RE.sub(sub, s)
30
+
31
+
32
+ def _redact(obj: Any) -> Any:
33
+ if isinstance(obj, str):
34
+ return _redact_str(obj)
35
+ if isinstance(obj, dict):
36
+ return {k: _redact(v) for k, v in obj.items()}
37
+ if isinstance(obj, list):
38
+ return [_redact(v) for v in obj]
39
+ return obj
40
+
41
+
42
+ def redact_request_for_cassette(req: dict | str) -> dict | str:
43
+ """Return a copy of ``req`` with API-key-looking strings (sk-..., sk-ant-...) replaced."""
44
+ return _redact(req)
45
+
46
+
47
+ def strip_sensitive_headers(headers: dict[str, str] | None) -> dict[str, str]:
48
+ """Drop set-cookie and any header whose name mentions key/auth/token/cookie/secret/password."""
49
+ out: dict[str, str] = {}
50
+ for k, v in (headers or {}).items():
51
+ lk = str(k).lower()
52
+ if any(p in lk for p in _SENSITIVE_HEADER_PARTS):
53
+ continue
54
+ out[k] = v
55
+ return out
56
+
57
+
58
+ class Cassette:
59
+ """A JSONL cassette file. Thread-safe; ordered replay of duplicate keys."""
60
+
61
+ def __init__(self, path: str, mode: str = "auto") -> None:
62
+ if mode not in MODES:
63
+ raise ValueError(f"invalid cassette mode {mode!r}; expected one of record, replay, auto")
64
+ self.path = path
65
+ self.mode = mode
66
+ self._lock = threading.Lock()
67
+ self._entries: dict[str, list[CachedResponse]] = {}
68
+ self._cursor: dict[str, int] = {}
69
+ self._count = 0
70
+ p = Path(path)
71
+ # record mode truncates lazily on the first write, so merely starting the proxy
72
+ # in record mode never wipes an existing cassette.
73
+ self._truncate_pending = mode == "record"
74
+ if mode != "record" and p.is_file():
75
+ self._load(p)
76
+ if mode == "auto":
77
+ self._ensure_trailing_newline(p)
78
+
79
+ @staticmethod
80
+ def _ensure_trailing_newline(p: Path) -> None:
81
+ """So appended lines never glue onto a truncated final line."""
82
+ with p.open("rb+") as f:
83
+ f.seek(0, 2)
84
+ if f.tell() == 0:
85
+ return
86
+ f.seek(-1, 2)
87
+ if f.read(1) != b"\n":
88
+ f.write(b"\n")
89
+
90
+ def _load(self, p: Path) -> None:
91
+ with p.open("r", encoding="utf-8", errors="replace") as f:
92
+ for lineno, line in enumerate(f, 1):
93
+ line = line.strip()
94
+ if not line:
95
+ continue
96
+ try:
97
+ d = json.loads(line)
98
+ key = d["key"]
99
+ r = d["response"]
100
+ resp = CachedResponse(
101
+ status=int(r["status"]),
102
+ headers=dict(r.get("headers") or {}),
103
+ body=r.get("body", ""),
104
+ streamed=bool(r.get("streamed", False)),
105
+ cost_usd=float(r.get("cost_usd", 0.0) or 0.0),
106
+ )
107
+ if not isinstance(key, str) or not isinstance(resp.body, str):
108
+ raise TypeError("bad field types")
109
+ except (ValueError, KeyError, TypeError, AttributeError) as e:
110
+ warnings.warn(f"llmtap cassette {self.path}: skipping corrupted line {lineno}: {e}", stacklevel=3)
111
+ continue
112
+ self._entries.setdefault(key, []).append(resp)
113
+ self._count += 1
114
+
115
+ def lookup(self, key: str) -> CachedResponse | None:
116
+ """Next recorded response for ``key`` (in order; last one repeats). Always None in record mode."""
117
+ if self.mode == "record":
118
+ return None
119
+ with self._lock:
120
+ recs = self._entries.get(key)
121
+ if not recs:
122
+ return None
123
+ i = self._cursor.get(key, 0)
124
+ self._cursor[key] = i + 1
125
+ return recs[min(i, len(recs) - 1)]
126
+
127
+ def should_record(self) -> bool:
128
+ """True in record and auto modes."""
129
+ return self.mode in ("record", "auto")
130
+
131
+ def record(self, key: str, request: dict | str, resp: CachedResponse) -> None:
132
+ """Append one JSON line (sorted keys) and flush. No-op in replay mode."""
133
+ if self.mode == "replay":
134
+ return
135
+ clean = CachedResponse(
136
+ status=int(resp.status),
137
+ headers=strip_sensitive_headers(resp.headers),
138
+ body=resp.body,
139
+ streamed=bool(resp.streamed),
140
+ cost_usd=float(resp.cost_usd or 0.0),
141
+ )
142
+ line = json.dumps(
143
+ {
144
+ "key": key,
145
+ "request": redact_request_for_cassette(request),
146
+ "response": {
147
+ "status": clean.status,
148
+ "headers": clean.headers,
149
+ "body": clean.body,
150
+ "streamed": clean.streamed,
151
+ "cost_usd": clean.cost_usd,
152
+ },
153
+ "recorded_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
154
+ },
155
+ sort_keys=True,
156
+ separators=(",", ":"),
157
+ ensure_ascii=False,
158
+ )
159
+ with self._lock:
160
+ p = Path(self.path)
161
+ p.parent.mkdir(parents=True, exist_ok=True)
162
+ open_mode = "w" if self._truncate_pending else "a"
163
+ self._truncate_pending = False
164
+ with p.open(open_mode, encoding="utf-8") as f:
165
+ f.write(line + "\n")
166
+ f.flush()
167
+ self._entries.setdefault(key, []).append(clean)
168
+ self._count += 1
169
+
170
+ def __len__(self) -> int:
171
+ with self._lock:
172
+ return self._count