llm-tally 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_tally/__init__.py +72 -0
- llm_tally/__main__.py +5 -0
- llm_tally/cache.py +611 -0
- llm_tally/cli.py +84 -0
- llm_tally/core.py +2135 -0
- llm_tally/dashboard.py +98 -0
- llm_tally/pricing.py +369 -0
- llm_tally/py.typed +0 -0
- llm_tally/retry.py +510 -0
- llm_tally/templates/dashboard.css +322 -0
- llm_tally/templates/dashboard.html +177 -0
- llm_tally/templates/dashboard.js +376 -0
- llm_tally/tracker.py +874 -0
- llm_tally-0.2.0.dist-info/METADATA +295 -0
- llm_tally-0.2.0.dist-info/RECORD +18 -0
- llm_tally-0.2.0.dist-info/WHEEL +4 -0
- llm_tally-0.2.0.dist-info/entry_points.txt +2 -0
- llm_tally-0.2.0.dist-info/licenses/LICENSE +22 -0
llm_tally/__init__.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""
|
|
2
|
+
`llm-tally`: Zero-dependency LLM helper for Python.
|
|
3
|
+
|
|
4
|
+
Features:
|
|
5
|
+
1. Self-evicting local in-memory cache (`cached_llm_call`, `LLMTally.cached_call`, `@tally.cache`)
|
|
6
|
+
that stores `(input + settings) -> output`, vanishes after `ttl` seconds on fetch,
|
|
7
|
+
and kills itself after `max_requests` fetches.
|
|
8
|
+
2. Automatic retry on HTTP 500/5xx, 429 rate limits, or network issues (`LLMTally.retry_call`,
|
|
9
|
+
`@tally.track`) with default delays of [0.5s, 1.0s, 1.5s] (honoring `Retry-After`), raising
|
|
10
|
+
`RetryExhaustedError` with all attempt errors if all attempts fail.
|
|
11
|
+
3. Per-feature & per-config cost analysis across OpenAI, Anthropic, Google Gemini, and
|
|
12
|
+
custom models, persisted to JSONL (and/or custom `on_record` sinks) with an interactive
|
|
13
|
+
static HTML dashboard. Tag calls with `metadata=` or `tally.context(user_id=...)`.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from .cache import CacheConfig, CacheEntry, InMemoryCache, build_cache_key, cached_llm_call
|
|
17
|
+
from .core import AILogHelper, LLMTally, get_default_tally
|
|
18
|
+
from .dashboard import render_dashboard_html, write_dashboard_file
|
|
19
|
+
from .pricing import DEFAULT_PRICING, ModelPricing, PricingRegistry
|
|
20
|
+
from .retry import (
|
|
21
|
+
DEFAULT_RETRY_DELAYS,
|
|
22
|
+
AttemptFailure,
|
|
23
|
+
HTTPStatus5xxError,
|
|
24
|
+
RetryConfig,
|
|
25
|
+
RetryExhaustedError,
|
|
26
|
+
execute_with_retry_async,
|
|
27
|
+
execute_with_retry_sync,
|
|
28
|
+
extract_retry_after,
|
|
29
|
+
is_retryable_exception,
|
|
30
|
+
)
|
|
31
|
+
from .tracker import (
|
|
32
|
+
CallRecord,
|
|
33
|
+
JSONLLogger,
|
|
34
|
+
TokenUsage,
|
|
35
|
+
analyze_logs,
|
|
36
|
+
extract_token_usage,
|
|
37
|
+
filter_records_since,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
__version__ = "0.2.0"
|
|
41
|
+
|
|
42
|
+
__all__ = [
|
|
43
|
+
"__version__",
|
|
44
|
+
"LLMTally",
|
|
45
|
+
"AILogHelper",
|
|
46
|
+
"get_default_tally",
|
|
47
|
+
"CacheConfig",
|
|
48
|
+
"CacheEntry",
|
|
49
|
+
"InMemoryCache",
|
|
50
|
+
"build_cache_key",
|
|
51
|
+
"cached_llm_call",
|
|
52
|
+
"DEFAULT_RETRY_DELAYS",
|
|
53
|
+
"RetryConfig",
|
|
54
|
+
"AttemptFailure",
|
|
55
|
+
"HTTPStatus5xxError",
|
|
56
|
+
"RetryExhaustedError",
|
|
57
|
+
"execute_with_retry_sync",
|
|
58
|
+
"execute_with_retry_async",
|
|
59
|
+
"is_retryable_exception",
|
|
60
|
+
"extract_retry_after",
|
|
61
|
+
"DEFAULT_PRICING",
|
|
62
|
+
"ModelPricing",
|
|
63
|
+
"PricingRegistry",
|
|
64
|
+
"TokenUsage",
|
|
65
|
+
"CallRecord",
|
|
66
|
+
"JSONLLogger",
|
|
67
|
+
"extract_token_usage",
|
|
68
|
+
"analyze_logs",
|
|
69
|
+
"filter_records_since",
|
|
70
|
+
"render_dashboard_html",
|
|
71
|
+
"write_dashboard_file",
|
|
72
|
+
]
|
llm_tally/__main__.py
ADDED
llm_tally/cache.py
ADDED
|
@@ -0,0 +1,611 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Feature 1: Self-Evicting In-Memory Cache for LLM Inputs, Settings, and Outputs.
|
|
3
|
+
|
|
4
|
+
How this cache works:
|
|
5
|
+
1. It stores a mapping of `(function_identity + input_args + input_kwargs + config_settings) -> output`
|
|
6
|
+
in local memory alongside the input settings snapshot.
|
|
7
|
+
2. If the user calls the function with the exact same input and settings, a deep copy of the
|
|
8
|
+
cached output is returned immediately without re-invoking the underlying LLM function.
|
|
9
|
+
3. Lifetime (TTL) eviction: Each entry has a configurable `ttl` (lifetime in seconds).
|
|
10
|
+
Once that lifetime passes, the entry vanishes from memory and executes the function
|
|
11
|
+
to generate and store a fresh response.
|
|
12
|
+
4. Request-count (`max_requests`) self-destruction: Each entry can be configured with
|
|
13
|
+
`max_requests=X`. After serving `X` cached requests for that input, the entry kills
|
|
14
|
+
(vanishes) itself from memory so the subsequent request generates a fresh response.
|
|
15
|
+
5. Bounded memory (`max_size`): Automatically purges expired entries and evicts the oldest
|
|
16
|
+
entries when `max_size` (default 1,000) is reached, preventing memory leaks in long-running
|
|
17
|
+
processes.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import collections.abc
|
|
23
|
+
import copy
|
|
24
|
+
import dataclasses
|
|
25
|
+
import hashlib
|
|
26
|
+
import inspect
|
|
27
|
+
import io
|
|
28
|
+
import json
|
|
29
|
+
import re
|
|
30
|
+
import threading
|
|
31
|
+
import time
|
|
32
|
+
import uuid
|
|
33
|
+
import weakref
|
|
34
|
+
from dataclasses import dataclass, field
|
|
35
|
+
from types import SimpleNamespace
|
|
36
|
+
from typing import Any, Callable, Dict, Optional, Set, Tuple, TypeVar
|
|
37
|
+
|
|
38
|
+
T = TypeVar("T")
|
|
39
|
+
|
|
40
|
+
# Matches default Python memory address suffixes like " at 0x104a8b2c0"
|
|
41
|
+
_MEM_ADDR_RE = re.compile(r" at 0x[0-9a-fA-F]+")
|
|
42
|
+
|
|
43
|
+
# Factory-created closures share a qualified name but can produce different answers.
|
|
44
|
+
# Keep a stable token per live closure without retaining the function or hashing its
|
|
45
|
+
# mutable captured state (such as a call counter) on every lookup.
|
|
46
|
+
_CLOSURE_TOKENS: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary()
|
|
47
|
+
_CLOSURE_TOKEN_LOCK = threading.Lock()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class CacheConfig:
|
|
52
|
+
"""Configuration settings for the self-evicting in-memory cache."""
|
|
53
|
+
|
|
54
|
+
enabled: bool = True
|
|
55
|
+
ttl: Optional[float] = 300.0 # Lifetime in seconds (None = no time expiration)
|
|
56
|
+
max_requests: Optional[int] = None # Kill entry after X cached requests (None = unlimited)
|
|
57
|
+
max_size: Optional[int] = 1000 # Maximum entries in memory (None = unlimited)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class CacheEntry:
|
|
62
|
+
"""Represents a single stored input+settings -> output entry in memory."""
|
|
63
|
+
|
|
64
|
+
key: str
|
|
65
|
+
input_settings: Dict[str, Any]
|
|
66
|
+
output: Any
|
|
67
|
+
created_at: float = field(default_factory=time.monotonic)
|
|
68
|
+
ttl: Optional[float] = 300.0
|
|
69
|
+
max_requests: Optional[int] = None
|
|
70
|
+
hits: int = 0
|
|
71
|
+
|
|
72
|
+
def is_expired_by_ttl(self, now: Optional[float] = None) -> bool:
|
|
73
|
+
"""Return True if the entry's lifetime (TTL) has elapsed."""
|
|
74
|
+
if self.ttl is None:
|
|
75
|
+
return False
|
|
76
|
+
current = time.monotonic() if now is None else now
|
|
77
|
+
return (current - self.created_at) >= self.ttl
|
|
78
|
+
|
|
79
|
+
def is_expired_by_requests(self) -> bool:
|
|
80
|
+
"""Return True if the entry has reached or exceeded its max_requests limit."""
|
|
81
|
+
if self.max_requests is None:
|
|
82
|
+
return False
|
|
83
|
+
return self.hits >= self.max_requests
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _canonical_json(obj: Any) -> str:
|
|
87
|
+
return json.dumps(obj, sort_keys=True, separators=(",", ":"))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _safe_clone(value: Any, _memo: Optional[Dict[int, Any]] = None) -> Any:
|
|
91
|
+
"""
|
|
92
|
+
Return a deep copy of `value` so callers mutating a result cannot corrupt cached state.
|
|
93
|
+
If `value` (such as an SDK response object) holds an uncopyable attribute like a
|
|
94
|
+
`threading.Lock` or socket, clones the object shell and all copyable attributes/containers
|
|
95
|
+
recursively while preserving the uncopyable attribute reference.
|
|
96
|
+
"""
|
|
97
|
+
if value is None or isinstance(value, (bool, int, float, str, bytes, frozenset, type)):
|
|
98
|
+
return value
|
|
99
|
+
|
|
100
|
+
if _memo is None:
|
|
101
|
+
_memo = {}
|
|
102
|
+
|
|
103
|
+
obj_id = id(value)
|
|
104
|
+
if obj_id in _memo:
|
|
105
|
+
return _memo[obj_id]
|
|
106
|
+
|
|
107
|
+
try:
|
|
108
|
+
return copy.deepcopy(value, _memo)
|
|
109
|
+
except Exception:
|
|
110
|
+
pass
|
|
111
|
+
|
|
112
|
+
# Fallback recursive clone when `copy.deepcopy` fails due to an embedded lock/socket
|
|
113
|
+
if isinstance(value, dict):
|
|
114
|
+
cloned_dict: Dict[Any, Any] = {}
|
|
115
|
+
_memo[obj_id] = cloned_dict
|
|
116
|
+
for k, v in value.items():
|
|
117
|
+
cloned_dict[_safe_clone(k, _memo)] = _safe_clone(v, _memo)
|
|
118
|
+
return cloned_dict
|
|
119
|
+
|
|
120
|
+
if isinstance(value, list):
|
|
121
|
+
cloned_list: list[Any] = []
|
|
122
|
+
_memo[obj_id] = cloned_list
|
|
123
|
+
cloned_list.extend(_safe_clone(v, _memo) for v in value)
|
|
124
|
+
return cloned_list
|
|
125
|
+
|
|
126
|
+
if isinstance(value, tuple):
|
|
127
|
+
cloned_tuple = tuple(_safe_clone(v, _memo) for v in value)
|
|
128
|
+
_memo[obj_id] = cloned_tuple
|
|
129
|
+
return cloned_tuple
|
|
130
|
+
|
|
131
|
+
if isinstance(value, set):
|
|
132
|
+
cloned_set = {_safe_clone(v, _memo) for v in value}
|
|
133
|
+
_memo[obj_id] = cloned_set
|
|
134
|
+
return cloned_set
|
|
135
|
+
|
|
136
|
+
# Custom object or SDK response instance holding an uncopyable field (e.g. a Lock)
|
|
137
|
+
try:
|
|
138
|
+
cloned_obj = copy.copy(value)
|
|
139
|
+
_memo[obj_id] = cloned_obj
|
|
140
|
+
if hasattr(value, "__dict__"):
|
|
141
|
+
for attr_k, attr_v in list(vars(value).items()):
|
|
142
|
+
try:
|
|
143
|
+
setattr(cloned_obj, attr_k, _safe_clone(attr_v, _memo))
|
|
144
|
+
except Exception:
|
|
145
|
+
pass
|
|
146
|
+
cls = type(value)
|
|
147
|
+
for base in cls.__mro__:
|
|
148
|
+
slots = getattr(base, "__slots__", ())
|
|
149
|
+
if isinstance(slots, str):
|
|
150
|
+
slots = (slots,)
|
|
151
|
+
for slot_name in slots:
|
|
152
|
+
if slot_name in ("__dict__", "__weakref__"):
|
|
153
|
+
continue
|
|
154
|
+
if hasattr(value, slot_name):
|
|
155
|
+
try:
|
|
156
|
+
setattr(
|
|
157
|
+
cloned_obj,
|
|
158
|
+
slot_name,
|
|
159
|
+
_safe_clone(getattr(value, slot_name), _memo),
|
|
160
|
+
)
|
|
161
|
+
except Exception:
|
|
162
|
+
pass
|
|
163
|
+
return cloned_obj
|
|
164
|
+
except Exception:
|
|
165
|
+
return value
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def is_stream_or_generator(value: Any) -> bool:
|
|
169
|
+
"""Return True if `value` is a single-use generator, iterator, or async iterator/stream."""
|
|
170
|
+
if inspect.isgenerator(value) or inspect.isasyncgen(value):
|
|
171
|
+
return True
|
|
172
|
+
if isinstance(value, (collections.abc.Iterator, collections.abc.AsyncIterator)):
|
|
173
|
+
return True
|
|
174
|
+
return False
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def is_llm_stream(value: Any) -> bool:
|
|
178
|
+
"""
|
|
179
|
+
Return True if `value` is a generator, an SDK stream iterator, or an SDK stream context
|
|
180
|
+
manager such as Anthropic's `MessageStreamManager` / OpenAI's `ChatCompletionStreamManager`
|
|
181
|
+
(not a file handle or built-in collection iterator like `map`, `zip`, `filter`, `enumerate`).
|
|
182
|
+
"""
|
|
183
|
+
if isinstance(value, io.IOBase):
|
|
184
|
+
return False
|
|
185
|
+
if type(value).__name__.endswith("StreamManager") and (
|
|
186
|
+
hasattr(value, "__enter__") or hasattr(value, "__aenter__")
|
|
187
|
+
):
|
|
188
|
+
return True
|
|
189
|
+
if isinstance(value, (map, filter, zip, enumerate)):
|
|
190
|
+
return False
|
|
191
|
+
if type(value).__name__ in (
|
|
192
|
+
"list_iterator",
|
|
193
|
+
"tuple_iterator",
|
|
194
|
+
"dict_keyiterator",
|
|
195
|
+
"dict_valueiterator",
|
|
196
|
+
"dict_itemiterator",
|
|
197
|
+
"set_iterator",
|
|
198
|
+
"str_iterator",
|
|
199
|
+
"bytes_iterator",
|
|
200
|
+
"range_iterator",
|
|
201
|
+
"callable_iterator",
|
|
202
|
+
):
|
|
203
|
+
return False
|
|
204
|
+
return is_stream_or_generator(value)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _extract_object_state(value: Any) -> Dict[str, Any]:
|
|
208
|
+
"""Extract all instance attributes from `__dict__` and `__slots__`."""
|
|
209
|
+
state: Dict[str, Any] = {}
|
|
210
|
+
if hasattr(value, "__dict__"):
|
|
211
|
+
state.update(vars(value))
|
|
212
|
+
for base in type(value).__mro__:
|
|
213
|
+
slots = getattr(base, "__slots__", ())
|
|
214
|
+
if isinstance(slots, str):
|
|
215
|
+
slots = (slots,)
|
|
216
|
+
for slot in slots:
|
|
217
|
+
if slot in ("__dict__", "__weakref__"):
|
|
218
|
+
continue
|
|
219
|
+
if hasattr(value, slot):
|
|
220
|
+
try:
|
|
221
|
+
state[slot] = getattr(value, slot)
|
|
222
|
+
except Exception:
|
|
223
|
+
pass
|
|
224
|
+
return state
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _normalize_for_key(value: Any, _seen: Optional[Set[int]] = None) -> Any:
|
|
228
|
+
"""
|
|
229
|
+
Recursively convert arbitrary Python objects into a type-tagged, cycle-safe,
|
|
230
|
+
deterministic JSON-serializable structure.
|
|
231
|
+
|
|
232
|
+
- Distinguishes `{1: "a"}` from `{"1": "a"}` and `(1, 2)` from `[1, 2]` and `True` from `1`.
|
|
233
|
+
- Handles mixed-type sets (`{1, "a"}`) and sets of objects without raising `TypeError`.
|
|
234
|
+
- Detects cyclic references via `_seen` to prevent `RecursionError`.
|
|
235
|
+
- Walks attributes (`__dict__` and `__slots__`) of custom input classes (e.g. `Prompt("...")`)
|
|
236
|
+
so distinct object instances never collide.
|
|
237
|
+
- Hashes binary buffer / array objects (like `numpy.ndarray`) by their full bytes/shape/dtype
|
|
238
|
+
so truncated `...` reprs never collide.
|
|
239
|
+
"""
|
|
240
|
+
if value is None:
|
|
241
|
+
return ["none"]
|
|
242
|
+
if isinstance(value, bool):
|
|
243
|
+
return ["bool", value]
|
|
244
|
+
if isinstance(value, int):
|
|
245
|
+
return ["int", value]
|
|
246
|
+
if isinstance(value, float):
|
|
247
|
+
return ["float", repr(value)]
|
|
248
|
+
if isinstance(value, str):
|
|
249
|
+
return ["str", value]
|
|
250
|
+
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
251
|
+
return ["bytes", bytes(value).hex()]
|
|
252
|
+
|
|
253
|
+
if _seen is None:
|
|
254
|
+
_seen = set()
|
|
255
|
+
|
|
256
|
+
obj_id = id(value)
|
|
257
|
+
if obj_id in _seen:
|
|
258
|
+
return ["cycle", f"{type(value).__module__}.{type(value).__qualname__}"]
|
|
259
|
+
|
|
260
|
+
_seen.add(obj_id)
|
|
261
|
+
try:
|
|
262
|
+
if isinstance(value, tuple):
|
|
263
|
+
return ["tuple", [_normalize_for_key(v, _seen) for v in value]]
|
|
264
|
+
if isinstance(value, list):
|
|
265
|
+
return ["list", [_normalize_for_key(v, _seen) for v in value]]
|
|
266
|
+
if isinstance(value, (set, frozenset)):
|
|
267
|
+
tag = "frozenset" if isinstance(value, frozenset) else "set"
|
|
268
|
+
normalized_items = [_normalize_for_key(v, _seen) for v in value]
|
|
269
|
+
normalized_items.sort(key=_canonical_json)
|
|
270
|
+
return [tag, normalized_items]
|
|
271
|
+
if isinstance(value, dict):
|
|
272
|
+
pairs = [
|
|
273
|
+
[_normalize_for_key(k, _seen), _normalize_for_key(v, _seen)]
|
|
274
|
+
for k, v in value.items()
|
|
275
|
+
]
|
|
276
|
+
pairs.sort(key=lambda pair: _canonical_json(pair[0]))
|
|
277
|
+
return ["dict", pairs]
|
|
278
|
+
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
279
|
+
fields_data = [
|
|
280
|
+
[f.name, _normalize_for_key(getattr(value, f.name), _seen)]
|
|
281
|
+
for f in dataclasses.fields(value)
|
|
282
|
+
]
|
|
283
|
+
return [
|
|
284
|
+
"dataclass",
|
|
285
|
+
f"{type(value).__module__}.{type(value).__qualname__}",
|
|
286
|
+
fields_data,
|
|
287
|
+
]
|
|
288
|
+
if isinstance(value, SimpleNamespace):
|
|
289
|
+
ns_pairs = [
|
|
290
|
+
[str(k), _normalize_for_key(v, _seen)]
|
|
291
|
+
for k, v in sorted(vars(value).items(), key=lambda item: str(item[0]))
|
|
292
|
+
]
|
|
293
|
+
return ["namespace", ns_pairs]
|
|
294
|
+
|
|
295
|
+
cls = type(value)
|
|
296
|
+
type_name = f"{cls.__module__}.{cls.__qualname__}"
|
|
297
|
+
|
|
298
|
+
# Array / tensor objects (e.g. numpy.ndarray) whose repr truncates with "..."
|
|
299
|
+
if hasattr(value, "tobytes") and hasattr(value, "shape") and hasattr(value, "dtype"):
|
|
300
|
+
try:
|
|
301
|
+
raw_bytes = value.tobytes()
|
|
302
|
+
digest = hashlib.sha256(raw_bytes).hexdigest()
|
|
303
|
+
return ["ndarray", type_name, str(value.dtype), list(value.shape), digest]
|
|
304
|
+
except Exception:
|
|
305
|
+
pass
|
|
306
|
+
|
|
307
|
+
# Extract attributes from `__dict__` and `__slots__` (covers custom classes like `Prompt("...")`)
|
|
308
|
+
state = _extract_object_state(value)
|
|
309
|
+
if state:
|
|
310
|
+
state_pairs = [
|
|
311
|
+
[str(k), _normalize_for_key(v, _seen)]
|
|
312
|
+
for k, v in sorted(state.items(), key=lambda item: str(item[0]))
|
|
313
|
+
]
|
|
314
|
+
return ["object_state", type_name, state_pairs]
|
|
315
|
+
|
|
316
|
+
# Built-in / C types without `__dict__` (e.g., datetime, Decimal, UUID, Path, Regex)
|
|
317
|
+
try:
|
|
318
|
+
raw_repr = repr(value)
|
|
319
|
+
cleaned_repr = _MEM_ADDR_RE.sub("", raw_repr)
|
|
320
|
+
return ["repr", type_name, cleaned_repr]
|
|
321
|
+
except Exception:
|
|
322
|
+
return ["instance", type_name]
|
|
323
|
+
finally:
|
|
324
|
+
_seen.discard(obj_id)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def build_cache_key(
|
|
328
|
+
func_name: str,
|
|
329
|
+
args: Tuple[Any, ...],
|
|
330
|
+
kwargs: Dict[str, Any],
|
|
331
|
+
settings: Optional[Dict[str, Any]] = None,
|
|
332
|
+
func: Optional[Callable[..., Any]] = None,
|
|
333
|
+
) -> Tuple[str, Dict[str, Any]]:
|
|
334
|
+
"""
|
|
335
|
+
Build a deterministic SHA-256 cache key and structured snapshot from the
|
|
336
|
+
function identifier, positional/keyword inputs (including `self` on methods),
|
|
337
|
+
and configuration settings.
|
|
338
|
+
"""
|
|
339
|
+
snapshot = {
|
|
340
|
+
"function": func_name,
|
|
341
|
+
"args": _normalize_for_key(args),
|
|
342
|
+
"kwargs": _normalize_for_key(kwargs),
|
|
343
|
+
"settings": _normalize_for_key(settings or {}),
|
|
344
|
+
}
|
|
345
|
+
target = getattr(func, "__func__", func)
|
|
346
|
+
if inspect.isfunction(target) and target.__closure__:
|
|
347
|
+
with _CLOSURE_TOKEN_LOCK:
|
|
348
|
+
token = _CLOSURE_TOKENS.get(target)
|
|
349
|
+
if token is None:
|
|
350
|
+
token = uuid.uuid4().hex
|
|
351
|
+
_CLOSURE_TOKENS[target] = token
|
|
352
|
+
snapshot["closure_identity"] = token
|
|
353
|
+
serialized = _canonical_json(snapshot)
|
|
354
|
+
digest = hashlib.sha256(serialized.encode("utf-8")).hexdigest()
|
|
355
|
+
return digest, snapshot
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
class InMemoryCache:
|
|
359
|
+
"""
|
|
360
|
+
Thread-safe local in-memory cache that stores `(input + settings) -> output`.
|
|
361
|
+
|
|
362
|
+
Eviction rules:
|
|
363
|
+
- **TTL (Lifetime)**: If `ttl` seconds have elapsed since the entry was created,
|
|
364
|
+
it vanishes from memory and signals a cache miss so a new response is generated.
|
|
365
|
+
- **Max Requests (`max_requests`)**: After the entry is fetched `max_requests` times,
|
|
366
|
+
it kills itself from memory so the next fetch generates a fresh response.
|
|
367
|
+
- **Max Size (`max_size`)**: Purges expired entries and evicts oldest entries when
|
|
368
|
+
the cache reaches `max_size` (default 1,000) so memory cannot grow unbounded.
|
|
369
|
+
"""
|
|
370
|
+
|
|
371
|
+
def __init__(
|
|
372
|
+
self,
|
|
373
|
+
default_ttl: Optional[float] = 300.0,
|
|
374
|
+
default_max_requests: Optional[int] = None,
|
|
375
|
+
max_size: Optional[int] = 1000,
|
|
376
|
+
) -> None:
|
|
377
|
+
self.default_ttl = default_ttl
|
|
378
|
+
self.default_max_requests = default_max_requests
|
|
379
|
+
self.max_size = max_size
|
|
380
|
+
self._store: Dict[str, CacheEntry] = {}
|
|
381
|
+
self._lock = threading.RLock()
|
|
382
|
+
self._total_hits: int = 0
|
|
383
|
+
self._total_misses: int = 0
|
|
384
|
+
self._total_evictions: int = 0
|
|
385
|
+
|
|
386
|
+
def _purge_expired_locked(self, now: Optional[float] = None) -> int:
|
|
387
|
+
"""Remove all TTL-expired or request-exhausted entries while holding `self._lock`."""
|
|
388
|
+
current = time.monotonic() if now is None else now
|
|
389
|
+
expired_keys = [
|
|
390
|
+
k
|
|
391
|
+
for k, entry in self._store.items()
|
|
392
|
+
if entry.is_expired_by_ttl(current) or entry.is_expired_by_requests()
|
|
393
|
+
]
|
|
394
|
+
for k in expired_keys:
|
|
395
|
+
del self._store[k]
|
|
396
|
+
self._total_evictions += 1
|
|
397
|
+
return len(expired_keys)
|
|
398
|
+
|
|
399
|
+
def fetch(self, key: str) -> Tuple[bool, Any, str]:
|
|
400
|
+
"""
|
|
401
|
+
Attempt to fetch a cached output by `key`.
|
|
402
|
+
|
|
403
|
+
Returns:
|
|
404
|
+
Tuple of `(hit: bool, output: Any, reason: str)` where `reason` is one of:
|
|
405
|
+
- `"hit"`: Valid cache hit; deep-copied output returned.
|
|
406
|
+
- `"hit_and_vanished"`: Valid cache hit on the X-th (`max_requests`) fetch;
|
|
407
|
+
output is returned and the entry immediately kills itself from memory.
|
|
408
|
+
- `"expired_ttl"`: Entry lifetime expired on fetch; entry vanished itself.
|
|
409
|
+
- `"expired_max_requests"`: Entry request limit reached; entry vanished itself.
|
|
410
|
+
- `"miss"`: No entry found for `key`.
|
|
411
|
+
"""
|
|
412
|
+
with self._lock:
|
|
413
|
+
entry = self._store.get(key)
|
|
414
|
+
if entry is None:
|
|
415
|
+
self._total_misses += 1
|
|
416
|
+
return False, None, "miss"
|
|
417
|
+
|
|
418
|
+
now = time.monotonic()
|
|
419
|
+
if entry.is_expired_by_ttl(now):
|
|
420
|
+
# Lifetime elapsed: vanish itself upon fetch
|
|
421
|
+
del self._store[key]
|
|
422
|
+
self._total_evictions += 1
|
|
423
|
+
self._total_misses += 1
|
|
424
|
+
return False, None, "expired_ttl"
|
|
425
|
+
|
|
426
|
+
if entry.is_expired_by_requests():
|
|
427
|
+
# Request limit already reached: vanish itself upon fetch
|
|
428
|
+
del self._store[key]
|
|
429
|
+
self._total_evictions += 1
|
|
430
|
+
self._total_misses += 1
|
|
431
|
+
return False, None, "expired_max_requests"
|
|
432
|
+
|
|
433
|
+
# Valid cache hit
|
|
434
|
+
entry.hits += 1
|
|
435
|
+
self._total_hits += 1
|
|
436
|
+
output_copy = _safe_clone(entry.output)
|
|
437
|
+
|
|
438
|
+
# Refresh LRU position on hit
|
|
439
|
+
self._store.pop(key, None)
|
|
440
|
+
|
|
441
|
+
# If this fetch reached `max_requests`, kill itself immediately after serving
|
|
442
|
+
if entry.max_requests is not None and entry.hits >= entry.max_requests:
|
|
443
|
+
self._total_evictions += 1
|
|
444
|
+
return True, output_copy, "hit_and_vanished"
|
|
445
|
+
|
|
446
|
+
self._store[key] = entry
|
|
447
|
+
return True, output_copy, "hit"
|
|
448
|
+
|
|
449
|
+
def store(
|
|
450
|
+
self,
|
|
451
|
+
key: str,
|
|
452
|
+
input_settings: Dict[str, Any],
|
|
453
|
+
output: Any,
|
|
454
|
+
ttl: Optional[float] = None,
|
|
455
|
+
max_requests: Optional[int] = None,
|
|
456
|
+
use_defaults: bool = True,
|
|
457
|
+
) -> Optional[CacheEntry]:
|
|
458
|
+
"""Store an input+settings snapshot and its corresponding output in memory."""
|
|
459
|
+
# Never cache single-use generators/iterators/streams
|
|
460
|
+
if is_stream_or_generator(output):
|
|
461
|
+
return None
|
|
462
|
+
|
|
463
|
+
effective_ttl = self.default_ttl if (ttl is None and use_defaults) else ttl
|
|
464
|
+
effective_max_req = (
|
|
465
|
+
self.default_max_requests if (max_requests is None and use_defaults) else max_requests
|
|
466
|
+
)
|
|
467
|
+
|
|
468
|
+
# If ttl <= 0 or max_requests <= 0, do not retain the entry
|
|
469
|
+
if (effective_ttl is not None and effective_ttl <= 0) or (
|
|
470
|
+
effective_max_req is not None and effective_max_req <= 0
|
|
471
|
+
):
|
|
472
|
+
with self._lock:
|
|
473
|
+
self._store.pop(key, None)
|
|
474
|
+
return None
|
|
475
|
+
|
|
476
|
+
with self._lock:
|
|
477
|
+
self._purge_expired_locked()
|
|
478
|
+
|
|
479
|
+
if self.max_size is not None and self.max_size <= 0:
|
|
480
|
+
self._store.clear()
|
|
481
|
+
return None
|
|
482
|
+
|
|
483
|
+
# Re-inserting an existing key removes the old position first
|
|
484
|
+
if key in self._store:
|
|
485
|
+
del self._store[key]
|
|
486
|
+
|
|
487
|
+
# Evict oldest entries if max_size is reached
|
|
488
|
+
if self.max_size is not None:
|
|
489
|
+
while len(self._store) >= self.max_size:
|
|
490
|
+
oldest_key = next(iter(self._store))
|
|
491
|
+
del self._store[oldest_key]
|
|
492
|
+
self._total_evictions += 1
|
|
493
|
+
|
|
494
|
+
entry = CacheEntry(
|
|
495
|
+
key=key,
|
|
496
|
+
input_settings=input_settings,
|
|
497
|
+
output=_safe_clone(output),
|
|
498
|
+
created_at=time.monotonic(),
|
|
499
|
+
ttl=effective_ttl,
|
|
500
|
+
max_requests=effective_max_req,
|
|
501
|
+
hits=0,
|
|
502
|
+
)
|
|
503
|
+
self._store[key] = entry
|
|
504
|
+
return entry
|
|
505
|
+
|
|
506
|
+
def get_entry(self, key: str) -> Optional[CacheEntry]:
|
|
507
|
+
"""Inspect a raw cache entry without incrementing its hit counter."""
|
|
508
|
+
with self._lock:
|
|
509
|
+
entry = self._store.get(key)
|
|
510
|
+
if entry is not None and (entry.is_expired_by_ttl() or entry.is_expired_by_requests()):
|
|
511
|
+
del self._store[key]
|
|
512
|
+
self._total_evictions += 1
|
|
513
|
+
return None
|
|
514
|
+
return entry
|
|
515
|
+
|
|
516
|
+
def invalidate(self, key: str) -> bool:
|
|
517
|
+
"""Manually remove a specific key from the cache."""
|
|
518
|
+
with self._lock:
|
|
519
|
+
if key in self._store:
|
|
520
|
+
del self._store[key]
|
|
521
|
+
self._total_evictions += 1
|
|
522
|
+
return True
|
|
523
|
+
return False
|
|
524
|
+
|
|
525
|
+
def clear(self) -> None:
|
|
526
|
+
"""Clear all entries from the in-memory cache."""
|
|
527
|
+
with self._lock:
|
|
528
|
+
self._store.clear()
|
|
529
|
+
|
|
530
|
+
@property
|
|
531
|
+
def size(self) -> int:
|
|
532
|
+
"""Return current number of active (non-expired) entries in memory."""
|
|
533
|
+
with self._lock:
|
|
534
|
+
self._purge_expired_locked()
|
|
535
|
+
return len(self._store)
|
|
536
|
+
|
|
537
|
+
def stats(self) -> Dict[str, int]:
|
|
538
|
+
"""Return cache telemetry statistics."""
|
|
539
|
+
with self._lock:
|
|
540
|
+
self._purge_expired_locked()
|
|
541
|
+
return {
|
|
542
|
+
"active_entries": len(self._store),
|
|
543
|
+
"total_hits": self._total_hits,
|
|
544
|
+
"total_misses": self._total_misses,
|
|
545
|
+
"total_evictions": self._total_evictions,
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def cached_llm_call(
|
|
550
|
+
func: Callable[..., T],
|
|
551
|
+
*args: Any,
|
|
552
|
+
cache: Optional[InMemoryCache] = None,
|
|
553
|
+
ttl: Optional[float] = 300.0,
|
|
554
|
+
max_requests: Optional[int] = None,
|
|
555
|
+
settings: Optional[Dict[str, Any]] = None,
|
|
556
|
+
**kwargs: Any,
|
|
557
|
+
) -> T:
|
|
558
|
+
"""
|
|
559
|
+
Execute `func(*args, **kwargs)` with local in-memory caching of Input Settings + Output.
|
|
560
|
+
|
|
561
|
+
IMPORTANT — How this function works:
|
|
562
|
+
- Saves the input arguments (`args`, `kwargs`) and configuration `settings` alongside the
|
|
563
|
+
generated `output` in an in-memory cache.
|
|
564
|
+
- When called again with the exact same input and settings, returns a copy of the stored
|
|
565
|
+
output without calling `func`.
|
|
566
|
+
- **Lifetime (`ttl`)**: After `ttl` seconds have elapsed, the cached entry vanishes itself
|
|
567
|
+
from memory and `func` is called to generate and cache a fresh response.
|
|
568
|
+
- **Self-Destruct after X Requests (`max_requests`)**: If `max_requests` is set to `X`,
|
|
569
|
+
the cached entry will vanish itself after being fetched `X` times, forcing the next
|
|
570
|
+
call to generate a fresh response.
|
|
571
|
+
"""
|
|
572
|
+
active_cache = cache or _DEFAULT_GLOBAL_CACHE
|
|
573
|
+
func_name = f"{getattr(func, '__module__', '__main__')}.{getattr(func, '__qualname__', getattr(func, '__name__', 'callable'))}"
|
|
574
|
+
key, snapshot = build_cache_key(func_name, args, kwargs, settings=settings, func=func)
|
|
575
|
+
|
|
576
|
+
if inspect.iscoroutinefunction(func):
|
|
577
|
+
|
|
578
|
+
async def _async_runner() -> Any:
|
|
579
|
+
hit, cached_output, _ = active_cache.fetch(key)
|
|
580
|
+
if hit:
|
|
581
|
+
return cached_output
|
|
582
|
+
result = await func(*args, **kwargs)
|
|
583
|
+
active_cache.store(
|
|
584
|
+
key=key,
|
|
585
|
+
input_settings=snapshot,
|
|
586
|
+
output=result,
|
|
587
|
+
ttl=ttl,
|
|
588
|
+
max_requests=max_requests,
|
|
589
|
+
use_defaults=False,
|
|
590
|
+
)
|
|
591
|
+
return result
|
|
592
|
+
|
|
593
|
+
return _async_runner() # type: ignore[return-value]
|
|
594
|
+
|
|
595
|
+
hit, cached_output, _ = active_cache.fetch(key)
|
|
596
|
+
if hit:
|
|
597
|
+
return cached_output
|
|
598
|
+
|
|
599
|
+
result = func(*args, **kwargs)
|
|
600
|
+
active_cache.store(
|
|
601
|
+
key=key,
|
|
602
|
+
input_settings=snapshot,
|
|
603
|
+
output=result,
|
|
604
|
+
ttl=ttl,
|
|
605
|
+
max_requests=max_requests,
|
|
606
|
+
use_defaults=False,
|
|
607
|
+
)
|
|
608
|
+
return result
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
_DEFAULT_GLOBAL_CACHE = InMemoryCache()
|