llm-tally 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
llm_tally/__init__.py ADDED
@@ -0,0 +1,72 @@
1
+ """
2
+ `llm-tally`: Zero-dependency LLM helper for Python.
3
+
4
+ Features:
5
+ 1. Self-evicting local in-memory cache (`cached_llm_call`, `LLMTally.cached_call`, `@tally.cache`)
6
+ that stores `(input + settings) -> output`, vanishes after `ttl` seconds on fetch,
7
+ and kills itself after `max_requests` fetches.
8
+ 2. Automatic retry on HTTP 500/5xx, 429 rate limits, or network issues (`LLMTally.retry_call`,
9
+ `@tally.track`) with default delays of [0.5s, 1.0s, 1.5s] (honoring `Retry-After`), raising
10
+ `RetryExhaustedError` with all attempt errors if all attempts fail.
11
+ 3. Per-feature & per-config cost analysis across OpenAI, Anthropic, Google Gemini, and
12
+ custom models, persisted to JSONL (and/or custom `on_record` sinks) with an interactive
13
+ static HTML dashboard. Tag calls with `metadata=` or `tally.context(user_id=...)`.
14
+ """
15
+
16
+ from .cache import CacheConfig, CacheEntry, InMemoryCache, build_cache_key, cached_llm_call
17
+ from .core import AILogHelper, LLMTally, get_default_tally
18
+ from .dashboard import render_dashboard_html, write_dashboard_file
19
+ from .pricing import DEFAULT_PRICING, ModelPricing, PricingRegistry
20
+ from .retry import (
21
+ DEFAULT_RETRY_DELAYS,
22
+ AttemptFailure,
23
+ HTTPStatus5xxError,
24
+ RetryConfig,
25
+ RetryExhaustedError,
26
+ execute_with_retry_async,
27
+ execute_with_retry_sync,
28
+ extract_retry_after,
29
+ is_retryable_exception,
30
+ )
31
+ from .tracker import (
32
+ CallRecord,
33
+ JSONLLogger,
34
+ TokenUsage,
35
+ analyze_logs,
36
+ extract_token_usage,
37
+ filter_records_since,
38
+ )
39
+
40
+ __version__ = "0.2.0"
41
+
42
+ __all__ = [
43
+ "__version__",
44
+ "LLMTally",
45
+ "AILogHelper",
46
+ "get_default_tally",
47
+ "CacheConfig",
48
+ "CacheEntry",
49
+ "InMemoryCache",
50
+ "build_cache_key",
51
+ "cached_llm_call",
52
+ "DEFAULT_RETRY_DELAYS",
53
+ "RetryConfig",
54
+ "AttemptFailure",
55
+ "HTTPStatus5xxError",
56
+ "RetryExhaustedError",
57
+ "execute_with_retry_sync",
58
+ "execute_with_retry_async",
59
+ "is_retryable_exception",
60
+ "extract_retry_after",
61
+ "DEFAULT_PRICING",
62
+ "ModelPricing",
63
+ "PricingRegistry",
64
+ "TokenUsage",
65
+ "CallRecord",
66
+ "JSONLLogger",
67
+ "extract_token_usage",
68
+ "analyze_logs",
69
+ "filter_records_since",
70
+ "render_dashboard_html",
71
+ "write_dashboard_file",
72
+ ]
llm_tally/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
llm_tally/cache.py ADDED
@@ -0,0 +1,611 @@
1
+ """
2
+ Feature 1: Self-Evicting In-Memory Cache for LLM Inputs, Settings, and Outputs.
3
+
4
+ How this cache works:
5
+ 1. It stores a mapping of `(function_identity + input_args + input_kwargs + config_settings) -> output`
6
+ in local memory alongside the input settings snapshot.
7
+ 2. If the user calls the function with the exact same input and settings, a deep copy of the
8
+ cached output is returned immediately without re-invoking the underlying LLM function.
9
+ 3. Lifetime (TTL) eviction: Each entry has a configurable `ttl` (lifetime in seconds).
10
+ Once that lifetime passes, the entry vanishes from memory and executes the function
11
+ to generate and store a fresh response.
12
+ 4. Request-count (`max_requests`) self-destruction: Each entry can be configured with
13
+ `max_requests=X`. After serving `X` cached requests for that input, the entry kills
14
+ (vanishes) itself from memory so the subsequent request generates a fresh response.
15
+ 5. Bounded memory (`max_size`): Automatically purges expired entries and evicts the oldest
16
+ entries when `max_size` (default 1,000) is reached, preventing memory leaks in long-running
17
+ processes.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import collections.abc
23
+ import copy
24
+ import dataclasses
25
+ import hashlib
26
+ import inspect
27
+ import io
28
+ import json
29
+ import re
30
+ import threading
31
+ import time
32
+ import uuid
33
+ import weakref
34
+ from dataclasses import dataclass, field
35
+ from types import SimpleNamespace
36
+ from typing import Any, Callable, Dict, Optional, Set, Tuple, TypeVar
37
+
38
+ T = TypeVar("T")
39
+
40
+ # Matches default Python memory address suffixes like " at 0x104a8b2c0"
41
+ _MEM_ADDR_RE = re.compile(r" at 0x[0-9a-fA-F]+")
42
+
43
+ # Factory-created closures share a qualified name but can produce different answers.
44
+ # Keep a stable token per live closure without retaining the function or hashing its
45
+ # mutable captured state (such as a call counter) on every lookup.
46
+ _CLOSURE_TOKENS: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary()
47
+ _CLOSURE_TOKEN_LOCK = threading.Lock()
48
+
49
+
50
+ @dataclass
51
+ class CacheConfig:
52
+ """Configuration settings for the self-evicting in-memory cache."""
53
+
54
+ enabled: bool = True
55
+ ttl: Optional[float] = 300.0 # Lifetime in seconds (None = no time expiration)
56
+ max_requests: Optional[int] = None # Kill entry after X cached requests (None = unlimited)
57
+ max_size: Optional[int] = 1000 # Maximum entries in memory (None = unlimited)
58
+
59
+
60
+ @dataclass
61
+ class CacheEntry:
62
+ """Represents a single stored input+settings -> output entry in memory."""
63
+
64
+ key: str
65
+ input_settings: Dict[str, Any]
66
+ output: Any
67
+ created_at: float = field(default_factory=time.monotonic)
68
+ ttl: Optional[float] = 300.0
69
+ max_requests: Optional[int] = None
70
+ hits: int = 0
71
+
72
+ def is_expired_by_ttl(self, now: Optional[float] = None) -> bool:
73
+ """Return True if the entry's lifetime (TTL) has elapsed."""
74
+ if self.ttl is None:
75
+ return False
76
+ current = time.monotonic() if now is None else now
77
+ return (current - self.created_at) >= self.ttl
78
+
79
+ def is_expired_by_requests(self) -> bool:
80
+ """Return True if the entry has reached or exceeded its max_requests limit."""
81
+ if self.max_requests is None:
82
+ return False
83
+ return self.hits >= self.max_requests
84
+
85
+
86
+ def _canonical_json(obj: Any) -> str:
87
+ return json.dumps(obj, sort_keys=True, separators=(",", ":"))
88
+
89
+
90
+ def _safe_clone(value: Any, _memo: Optional[Dict[int, Any]] = None) -> Any:
91
+ """
92
+ Return a deep copy of `value` so callers mutating a result cannot corrupt cached state.
93
+ If `value` (such as an SDK response object) holds an uncopyable attribute like a
94
+ `threading.Lock` or socket, clones the object shell and all copyable attributes/containers
95
+ recursively while preserving the uncopyable attribute reference.
96
+ """
97
+ if value is None or isinstance(value, (bool, int, float, str, bytes, frozenset, type)):
98
+ return value
99
+
100
+ if _memo is None:
101
+ _memo = {}
102
+
103
+ obj_id = id(value)
104
+ if obj_id in _memo:
105
+ return _memo[obj_id]
106
+
107
+ try:
108
+ return copy.deepcopy(value, _memo)
109
+ except Exception:
110
+ pass
111
+
112
+ # Fallback recursive clone when `copy.deepcopy` fails due to an embedded lock/socket
113
+ if isinstance(value, dict):
114
+ cloned_dict: Dict[Any, Any] = {}
115
+ _memo[obj_id] = cloned_dict
116
+ for k, v in value.items():
117
+ cloned_dict[_safe_clone(k, _memo)] = _safe_clone(v, _memo)
118
+ return cloned_dict
119
+
120
+ if isinstance(value, list):
121
+ cloned_list: list[Any] = []
122
+ _memo[obj_id] = cloned_list
123
+ cloned_list.extend(_safe_clone(v, _memo) for v in value)
124
+ return cloned_list
125
+
126
+ if isinstance(value, tuple):
127
+ cloned_tuple = tuple(_safe_clone(v, _memo) for v in value)
128
+ _memo[obj_id] = cloned_tuple
129
+ return cloned_tuple
130
+
131
+ if isinstance(value, set):
132
+ cloned_set = {_safe_clone(v, _memo) for v in value}
133
+ _memo[obj_id] = cloned_set
134
+ return cloned_set
135
+
136
+ # Custom object or SDK response instance holding an uncopyable field (e.g. a Lock)
137
+ try:
138
+ cloned_obj = copy.copy(value)
139
+ _memo[obj_id] = cloned_obj
140
+ if hasattr(value, "__dict__"):
141
+ for attr_k, attr_v in list(vars(value).items()):
142
+ try:
143
+ setattr(cloned_obj, attr_k, _safe_clone(attr_v, _memo))
144
+ except Exception:
145
+ pass
146
+ cls = type(value)
147
+ for base in cls.__mro__:
148
+ slots = getattr(base, "__slots__", ())
149
+ if isinstance(slots, str):
150
+ slots = (slots,)
151
+ for slot_name in slots:
152
+ if slot_name in ("__dict__", "__weakref__"):
153
+ continue
154
+ if hasattr(value, slot_name):
155
+ try:
156
+ setattr(
157
+ cloned_obj,
158
+ slot_name,
159
+ _safe_clone(getattr(value, slot_name), _memo),
160
+ )
161
+ except Exception:
162
+ pass
163
+ return cloned_obj
164
+ except Exception:
165
+ return value
166
+
167
+
168
+ def is_stream_or_generator(value: Any) -> bool:
169
+ """Return True if `value` is a single-use generator, iterator, or async iterator/stream."""
170
+ if inspect.isgenerator(value) or inspect.isasyncgen(value):
171
+ return True
172
+ if isinstance(value, (collections.abc.Iterator, collections.abc.AsyncIterator)):
173
+ return True
174
+ return False
175
+
176
+
177
+ def is_llm_stream(value: Any) -> bool:
178
+ """
179
+ Return True if `value` is a generator, an SDK stream iterator, or an SDK stream context
180
+ manager such as Anthropic's `MessageStreamManager` / OpenAI's `ChatCompletionStreamManager`
181
+ (not a file handle or built-in collection iterator like `map`, `zip`, `filter`, `enumerate`).
182
+ """
183
+ if isinstance(value, io.IOBase):
184
+ return False
185
+ if type(value).__name__.endswith("StreamManager") and (
186
+ hasattr(value, "__enter__") or hasattr(value, "__aenter__")
187
+ ):
188
+ return True
189
+ if isinstance(value, (map, filter, zip, enumerate)):
190
+ return False
191
+ if type(value).__name__ in (
192
+ "list_iterator",
193
+ "tuple_iterator",
194
+ "dict_keyiterator",
195
+ "dict_valueiterator",
196
+ "dict_itemiterator",
197
+ "set_iterator",
198
+ "str_iterator",
199
+ "bytes_iterator",
200
+ "range_iterator",
201
+ "callable_iterator",
202
+ ):
203
+ return False
204
+ return is_stream_or_generator(value)
205
+
206
+
207
+ def _extract_object_state(value: Any) -> Dict[str, Any]:
208
+ """Extract all instance attributes from `__dict__` and `__slots__`."""
209
+ state: Dict[str, Any] = {}
210
+ if hasattr(value, "__dict__"):
211
+ state.update(vars(value))
212
+ for base in type(value).__mro__:
213
+ slots = getattr(base, "__slots__", ())
214
+ if isinstance(slots, str):
215
+ slots = (slots,)
216
+ for slot in slots:
217
+ if slot in ("__dict__", "__weakref__"):
218
+ continue
219
+ if hasattr(value, slot):
220
+ try:
221
+ state[slot] = getattr(value, slot)
222
+ except Exception:
223
+ pass
224
+ return state
225
+
226
+
227
+ def _normalize_for_key(value: Any, _seen: Optional[Set[int]] = None) -> Any:
228
+ """
229
+ Recursively convert arbitrary Python objects into a type-tagged, cycle-safe,
230
+ deterministic JSON-serializable structure.
231
+
232
+ - Distinguishes `{1: "a"}` from `{"1": "a"}` and `(1, 2)` from `[1, 2]` and `True` from `1`.
233
+ - Handles mixed-type sets (`{1, "a"}`) and sets of objects without raising `TypeError`.
234
+ - Detects cyclic references via `_seen` to prevent `RecursionError`.
235
+ - Walks attributes (`__dict__` and `__slots__`) of custom input classes (e.g. `Prompt("...")`)
236
+ so distinct object instances never collide.
237
+ - Hashes binary buffer / array objects (like `numpy.ndarray`) by their full bytes/shape/dtype
238
+ so truncated `...` reprs never collide.
239
+ """
240
+ if value is None:
241
+ return ["none"]
242
+ if isinstance(value, bool):
243
+ return ["bool", value]
244
+ if isinstance(value, int):
245
+ return ["int", value]
246
+ if isinstance(value, float):
247
+ return ["float", repr(value)]
248
+ if isinstance(value, str):
249
+ return ["str", value]
250
+ if isinstance(value, (bytes, bytearray, memoryview)):
251
+ return ["bytes", bytes(value).hex()]
252
+
253
+ if _seen is None:
254
+ _seen = set()
255
+
256
+ obj_id = id(value)
257
+ if obj_id in _seen:
258
+ return ["cycle", f"{type(value).__module__}.{type(value).__qualname__}"]
259
+
260
+ _seen.add(obj_id)
261
+ try:
262
+ if isinstance(value, tuple):
263
+ return ["tuple", [_normalize_for_key(v, _seen) for v in value]]
264
+ if isinstance(value, list):
265
+ return ["list", [_normalize_for_key(v, _seen) for v in value]]
266
+ if isinstance(value, (set, frozenset)):
267
+ tag = "frozenset" if isinstance(value, frozenset) else "set"
268
+ normalized_items = [_normalize_for_key(v, _seen) for v in value]
269
+ normalized_items.sort(key=_canonical_json)
270
+ return [tag, normalized_items]
271
+ if isinstance(value, dict):
272
+ pairs = [
273
+ [_normalize_for_key(k, _seen), _normalize_for_key(v, _seen)]
274
+ for k, v in value.items()
275
+ ]
276
+ pairs.sort(key=lambda pair: _canonical_json(pair[0]))
277
+ return ["dict", pairs]
278
+ if dataclasses.is_dataclass(value) and not isinstance(value, type):
279
+ fields_data = [
280
+ [f.name, _normalize_for_key(getattr(value, f.name), _seen)]
281
+ for f in dataclasses.fields(value)
282
+ ]
283
+ return [
284
+ "dataclass",
285
+ f"{type(value).__module__}.{type(value).__qualname__}",
286
+ fields_data,
287
+ ]
288
+ if isinstance(value, SimpleNamespace):
289
+ ns_pairs = [
290
+ [str(k), _normalize_for_key(v, _seen)]
291
+ for k, v in sorted(vars(value).items(), key=lambda item: str(item[0]))
292
+ ]
293
+ return ["namespace", ns_pairs]
294
+
295
+ cls = type(value)
296
+ type_name = f"{cls.__module__}.{cls.__qualname__}"
297
+
298
+ # Array / tensor objects (e.g. numpy.ndarray) whose repr truncates with "..."
299
+ if hasattr(value, "tobytes") and hasattr(value, "shape") and hasattr(value, "dtype"):
300
+ try:
301
+ raw_bytes = value.tobytes()
302
+ digest = hashlib.sha256(raw_bytes).hexdigest()
303
+ return ["ndarray", type_name, str(value.dtype), list(value.shape), digest]
304
+ except Exception:
305
+ pass
306
+
307
+ # Extract attributes from `__dict__` and `__slots__` (covers custom classes like `Prompt("...")`)
308
+ state = _extract_object_state(value)
309
+ if state:
310
+ state_pairs = [
311
+ [str(k), _normalize_for_key(v, _seen)]
312
+ for k, v in sorted(state.items(), key=lambda item: str(item[0]))
313
+ ]
314
+ return ["object_state", type_name, state_pairs]
315
+
316
+ # Built-in / C types without `__dict__` (e.g., datetime, Decimal, UUID, Path, Regex)
317
+ try:
318
+ raw_repr = repr(value)
319
+ cleaned_repr = _MEM_ADDR_RE.sub("", raw_repr)
320
+ return ["repr", type_name, cleaned_repr]
321
+ except Exception:
322
+ return ["instance", type_name]
323
+ finally:
324
+ _seen.discard(obj_id)
325
+
326
+
327
+ def build_cache_key(
328
+ func_name: str,
329
+ args: Tuple[Any, ...],
330
+ kwargs: Dict[str, Any],
331
+ settings: Optional[Dict[str, Any]] = None,
332
+ func: Optional[Callable[..., Any]] = None,
333
+ ) -> Tuple[str, Dict[str, Any]]:
334
+ """
335
+ Build a deterministic SHA-256 cache key and structured snapshot from the
336
+ function identifier, positional/keyword inputs (including `self` on methods),
337
+ and configuration settings.
338
+ """
339
+ snapshot = {
340
+ "function": func_name,
341
+ "args": _normalize_for_key(args),
342
+ "kwargs": _normalize_for_key(kwargs),
343
+ "settings": _normalize_for_key(settings or {}),
344
+ }
345
+ target = getattr(func, "__func__", func)
346
+ if inspect.isfunction(target) and target.__closure__:
347
+ with _CLOSURE_TOKEN_LOCK:
348
+ token = _CLOSURE_TOKENS.get(target)
349
+ if token is None:
350
+ token = uuid.uuid4().hex
351
+ _CLOSURE_TOKENS[target] = token
352
+ snapshot["closure_identity"] = token
353
+ serialized = _canonical_json(snapshot)
354
+ digest = hashlib.sha256(serialized.encode("utf-8")).hexdigest()
355
+ return digest, snapshot
356
+
357
+
358
+ class InMemoryCache:
359
+ """
360
+ Thread-safe local in-memory cache that stores `(input + settings) -> output`.
361
+
362
+ Eviction rules:
363
+ - **TTL (Lifetime)**: If `ttl` seconds have elapsed since the entry was created,
364
+ it vanishes from memory and signals a cache miss so a new response is generated.
365
+ - **Max Requests (`max_requests`)**: After the entry is fetched `max_requests` times,
366
+ it kills itself from memory so the next fetch generates a fresh response.
367
+ - **Max Size (`max_size`)**: Purges expired entries and evicts oldest entries when
368
+ the cache reaches `max_size` (default 1,000) so memory cannot grow unbounded.
369
+ """
370
+
371
+ def __init__(
372
+ self,
373
+ default_ttl: Optional[float] = 300.0,
374
+ default_max_requests: Optional[int] = None,
375
+ max_size: Optional[int] = 1000,
376
+ ) -> None:
377
+ self.default_ttl = default_ttl
378
+ self.default_max_requests = default_max_requests
379
+ self.max_size = max_size
380
+ self._store: Dict[str, CacheEntry] = {}
381
+ self._lock = threading.RLock()
382
+ self._total_hits: int = 0
383
+ self._total_misses: int = 0
384
+ self._total_evictions: int = 0
385
+
386
+ def _purge_expired_locked(self, now: Optional[float] = None) -> int:
387
+ """Remove all TTL-expired or request-exhausted entries while holding `self._lock`."""
388
+ current = time.monotonic() if now is None else now
389
+ expired_keys = [
390
+ k
391
+ for k, entry in self._store.items()
392
+ if entry.is_expired_by_ttl(current) or entry.is_expired_by_requests()
393
+ ]
394
+ for k in expired_keys:
395
+ del self._store[k]
396
+ self._total_evictions += 1
397
+ return len(expired_keys)
398
+
399
+ def fetch(self, key: str) -> Tuple[bool, Any, str]:
400
+ """
401
+ Attempt to fetch a cached output by `key`.
402
+
403
+ Returns:
404
+ Tuple of `(hit: bool, output: Any, reason: str)` where `reason` is one of:
405
+ - `"hit"`: Valid cache hit; deep-copied output returned.
406
+ - `"hit_and_vanished"`: Valid cache hit on the X-th (`max_requests`) fetch;
407
+ output is returned and the entry immediately kills itself from memory.
408
+ - `"expired_ttl"`: Entry lifetime expired on fetch; entry vanished itself.
409
+ - `"expired_max_requests"`: Entry request limit reached; entry vanished itself.
410
+ - `"miss"`: No entry found for `key`.
411
+ """
412
+ with self._lock:
413
+ entry = self._store.get(key)
414
+ if entry is None:
415
+ self._total_misses += 1
416
+ return False, None, "miss"
417
+
418
+ now = time.monotonic()
419
+ if entry.is_expired_by_ttl(now):
420
+ # Lifetime elapsed: vanish itself upon fetch
421
+ del self._store[key]
422
+ self._total_evictions += 1
423
+ self._total_misses += 1
424
+ return False, None, "expired_ttl"
425
+
426
+ if entry.is_expired_by_requests():
427
+ # Request limit already reached: vanish itself upon fetch
428
+ del self._store[key]
429
+ self._total_evictions += 1
430
+ self._total_misses += 1
431
+ return False, None, "expired_max_requests"
432
+
433
+ # Valid cache hit
434
+ entry.hits += 1
435
+ self._total_hits += 1
436
+ output_copy = _safe_clone(entry.output)
437
+
438
+ # Refresh LRU position on hit
439
+ self._store.pop(key, None)
440
+
441
+ # If this fetch reached `max_requests`, kill itself immediately after serving
442
+ if entry.max_requests is not None and entry.hits >= entry.max_requests:
443
+ self._total_evictions += 1
444
+ return True, output_copy, "hit_and_vanished"
445
+
446
+ self._store[key] = entry
447
+ return True, output_copy, "hit"
448
+
449
+ def store(
450
+ self,
451
+ key: str,
452
+ input_settings: Dict[str, Any],
453
+ output: Any,
454
+ ttl: Optional[float] = None,
455
+ max_requests: Optional[int] = None,
456
+ use_defaults: bool = True,
457
+ ) -> Optional[CacheEntry]:
458
+ """Store an input+settings snapshot and its corresponding output in memory."""
459
+ # Never cache single-use generators/iterators/streams
460
+ if is_stream_or_generator(output):
461
+ return None
462
+
463
+ effective_ttl = self.default_ttl if (ttl is None and use_defaults) else ttl
464
+ effective_max_req = (
465
+ self.default_max_requests if (max_requests is None and use_defaults) else max_requests
466
+ )
467
+
468
+ # If ttl <= 0 or max_requests <= 0, do not retain the entry
469
+ if (effective_ttl is not None and effective_ttl <= 0) or (
470
+ effective_max_req is not None and effective_max_req <= 0
471
+ ):
472
+ with self._lock:
473
+ self._store.pop(key, None)
474
+ return None
475
+
476
+ with self._lock:
477
+ self._purge_expired_locked()
478
+
479
+ if self.max_size is not None and self.max_size <= 0:
480
+ self._store.clear()
481
+ return None
482
+
483
+ # Re-inserting an existing key removes the old position first
484
+ if key in self._store:
485
+ del self._store[key]
486
+
487
+ # Evict oldest entries if max_size is reached
488
+ if self.max_size is not None:
489
+ while len(self._store) >= self.max_size:
490
+ oldest_key = next(iter(self._store))
491
+ del self._store[oldest_key]
492
+ self._total_evictions += 1
493
+
494
+ entry = CacheEntry(
495
+ key=key,
496
+ input_settings=input_settings,
497
+ output=_safe_clone(output),
498
+ created_at=time.monotonic(),
499
+ ttl=effective_ttl,
500
+ max_requests=effective_max_req,
501
+ hits=0,
502
+ )
503
+ self._store[key] = entry
504
+ return entry
505
+
506
+ def get_entry(self, key: str) -> Optional[CacheEntry]:
507
+ """Inspect a raw cache entry without incrementing its hit counter."""
508
+ with self._lock:
509
+ entry = self._store.get(key)
510
+ if entry is not None and (entry.is_expired_by_ttl() or entry.is_expired_by_requests()):
511
+ del self._store[key]
512
+ self._total_evictions += 1
513
+ return None
514
+ return entry
515
+
516
+ def invalidate(self, key: str) -> bool:
517
+ """Manually remove a specific key from the cache."""
518
+ with self._lock:
519
+ if key in self._store:
520
+ del self._store[key]
521
+ self._total_evictions += 1
522
+ return True
523
+ return False
524
+
525
+ def clear(self) -> None:
526
+ """Clear all entries from the in-memory cache."""
527
+ with self._lock:
528
+ self._store.clear()
529
+
530
+ @property
531
+ def size(self) -> int:
532
+ """Return current number of active (non-expired) entries in memory."""
533
+ with self._lock:
534
+ self._purge_expired_locked()
535
+ return len(self._store)
536
+
537
+ def stats(self) -> Dict[str, int]:
538
+ """Return cache telemetry statistics."""
539
+ with self._lock:
540
+ self._purge_expired_locked()
541
+ return {
542
+ "active_entries": len(self._store),
543
+ "total_hits": self._total_hits,
544
+ "total_misses": self._total_misses,
545
+ "total_evictions": self._total_evictions,
546
+ }
547
+
548
+
549
+ def cached_llm_call(
550
+ func: Callable[..., T],
551
+ *args: Any,
552
+ cache: Optional[InMemoryCache] = None,
553
+ ttl: Optional[float] = 300.0,
554
+ max_requests: Optional[int] = None,
555
+ settings: Optional[Dict[str, Any]] = None,
556
+ **kwargs: Any,
557
+ ) -> T:
558
+ """
559
+ Execute `func(*args, **kwargs)` with local in-memory caching of Input Settings + Output.
560
+
561
+ IMPORTANT — How this function works:
562
+ - Saves the input arguments (`args`, `kwargs`) and configuration `settings` alongside the
563
+ generated `output` in an in-memory cache.
564
+ - When called again with the exact same input and settings, returns a copy of the stored
565
+ output without calling `func`.
566
+ - **Lifetime (`ttl`)**: After `ttl` seconds have elapsed, the cached entry vanishes itself
567
+ from memory and `func` is called to generate and cache a fresh response.
568
+ - **Self-Destruct after X Requests (`max_requests`)**: If `max_requests` is set to `X`,
569
+ the cached entry will vanish itself after being fetched `X` times, forcing the next
570
+ call to generate a fresh response.
571
+ """
572
+ active_cache = cache or _DEFAULT_GLOBAL_CACHE
573
+ func_name = f"{getattr(func, '__module__', '__main__')}.{getattr(func, '__qualname__', getattr(func, '__name__', 'callable'))}"
574
+ key, snapshot = build_cache_key(func_name, args, kwargs, settings=settings, func=func)
575
+
576
+ if inspect.iscoroutinefunction(func):
577
+
578
+ async def _async_runner() -> Any:
579
+ hit, cached_output, _ = active_cache.fetch(key)
580
+ if hit:
581
+ return cached_output
582
+ result = await func(*args, **kwargs)
583
+ active_cache.store(
584
+ key=key,
585
+ input_settings=snapshot,
586
+ output=result,
587
+ ttl=ttl,
588
+ max_requests=max_requests,
589
+ use_defaults=False,
590
+ )
591
+ return result
592
+
593
+ return _async_runner() # type: ignore[return-value]
594
+
595
+ hit, cached_output, _ = active_cache.fetch(key)
596
+ if hit:
597
+ return cached_output
598
+
599
+ result = func(*args, **kwargs)
600
+ active_cache.store(
601
+ key=key,
602
+ input_settings=snapshot,
603
+ output=result,
604
+ ttl=ttl,
605
+ max_requests=max_requests,
606
+ use_defaults=False,
607
+ )
608
+ return result
609
+
610
+
611
+ _DEFAULT_GLOBAL_CACHE = InMemoryCache()