advalcache 0.1.0__py3-none-any.whl

advalcache/_engine.py

@@ -0,0 +1,522 @@
+"""advalcache/_engine.py — Thread-safe AdVal in-memory cache engine.
+
+This is the core of the SDK. It is intentionally free of any research-repo
+dependencies (models.py, sim/, generator/, etc.) and uses only Python stdlib.
+
+Two engines are provided:
+  - AdValEngine: synchronous, uses threading.RLock
+  - AsyncAdValEngine: asynchronous, uses asyncio.Lock
+"""
+
+from __future__ import annotations
+
+import asyncio
+import collections
+import logging
+import time
+import threading
+from dataclasses import dataclass, field
+from typing import Any, Callable, Deque, Dict, List, Optional, Tuple
+
+from advalcache._scoring import (
+    compute_score,
+    estimate_size_bytes,
+    get_freshness,
+    touch_hotness,
+)
+from advalcache._eviction import (
+    classify_eviction_reason,
+    pick_victim,
+    sample_victim_pool,
+)
+from advalcache._refresh import BackgroundRefreshPool
+
+logger = logging.getLogger("advalcache.engine")
+
+# Type alias for the refresh callback
+RefreshCallback = Callable[[], Tuple[Any, float, int]]
+
+
+# ---------------------------------------------------------------------------
+# Cache Entry
+# ---------------------------------------------------------------------------
+
+@dataclass
+class CacheEntry:
+    """A single resident entry in the AdVal cache."""
+
+    key: str
+    data: Any
+    cls: str
+    size_bytes: int
+    regen_ms: float
+    regen_usd: float
+    change_interval_s: float
+    last_sync_ts: float
+    created_ts: float = field(default_factory=time.time)
+    last_access_ts: float = field(default_factory=time.time)
+    hotness: float = 1.0
+    hit_count: int = 0
+
+    def freshness(self, now_ts: Optional[float] = None) -> float:
+        """P(still valid) at now_ts. Complexity: O(1)."""
+        now = now_ts if now_ts is not None else time.time()
+        return get_freshness(self.last_sync_ts, now, self.change_interval_s)
+
+    def score(
+        self,
+        now_ts: Optional[float] = None,
+        p99_ms: float = 2_000.0,
+        p99_usd: float = 0.05,
+    ) -> float:
+        """AdVal value score at now_ts. Complexity: O(1)."""
+        now = now_ts if now_ts is not None else time.time()
+        return compute_score(
+            self.hotness, self.regen_ms, self.regen_usd,
+            self.last_sync_ts, now, self.change_interval_s,
+            self.size_bytes, p99_ms, p99_usd,
+        )
+
+    def touch(self, now_ts: float, decay_lambda: float) -> None:
+        """LRFU update on access. Complexity: O(1)."""
+        self.hotness = touch_hotness(self.hotness, self.last_access_ts, now_ts, decay_lambda)
+        self.last_access_ts = now_ts
+        self.hit_count += 1
+
+    def to_dict(self, now_ts: Optional[float] = None) -> Dict[str, Any]:
+        """Serializable snapshot of this entry. Complexity: O(1)."""
+        now = now_ts if now_ts is not None else time.time()
+        return {
+            "key": self.key,
+            "cls": self.cls,
+            "size_bytes": self.size_bytes,
+            "regen_ms": round(self.regen_ms, 2),
+            "regen_usd": round(self.regen_usd, 4),
+            "change_interval_s": self.change_interval_s,
+            "age_s": round(now - self.last_sync_ts, 1),
+            "freshness": round(self.freshness(now), 3),
+            "hotness": round(self.hotness, 3),
+            "score": round(self.score(now), 4),
+            "hit_count": self.hit_count,
+        }
+
+
+# ---------------------------------------------------------------------------
+# Synchronous Engine
+# ---------------------------------------------------------------------------
+
+class AdValEngine:
+    """Thread-safe in-memory AdVal cache engine (sync).
+
+    This class is the implementation detail behind the public ``Cache`` API.
+    It is not intended to be used directly by end-users.
+    """
+
+    def __init__(
+        self,
+        capacity_mb: float,
+        decay_lambda: float,
+        refresh_threshold: float,
+        refresh_min_score: float,
+        max_refresh_workers: int,
+        eviction_pool_fraction: float,
+        p99_ms: float,
+        p99_usd: float,
+        max_events: int = 200,
+    ) -> None:
+        self._capacity_bytes = int(capacity_mb * 1024 * 1024)
+        self._decay_lambda = decay_lambda
+        self._refresh_threshold = refresh_threshold
+        self._refresh_min_score = refresh_min_score
+        self._pool_fraction = eviction_pool_fraction
+        self._p99_ms = p99_ms
+        self._p99_usd = p99_usd
+
+        self._lock = threading.RLock()
+        self._entries: Dict[str, CacheEntry] = {}
+        self._keys: List[str] = []
+        self._resident_bytes: int = 0
+
+        self._refresh_pool = BackgroundRefreshPool(max_workers=max_refresh_workers)
+        self._recent_events: Deque[Dict[str, Any]] = collections.deque(maxlen=max_events)
+
+        # Metrics
+        self.total_requests: int = 0
+        self.hits: int = 0
+        self.misses: int = 0
+        self.fresh_hits: int = 0
+        self.stale_hits: int = 0
+        self.proactive_refreshes: int = 0
+        self.total_usd_saved: float = 0.0
+        self.ram_latency_sum_ms: float = 0.0
+        self.origin_latency_sum_ms: float = 0.0
+        self.evictions: Dict[str, int] = {
+            "CAPACITY": 0, "STALE": 0, "COLD": 0, "BULKY": 0, "CHEAP": 0,
+        }
+        self.declines: int = 0
+
+    # ------------------------------------------------------------------
+    # Properties
+    # ------------------------------------------------------------------
+
+    @property
+    def resident_bytes(self) -> int:
+        with self._lock:
+            return self._resident_bytes
+
+    @property
+    def resident_mb(self) -> float:
+        with self._lock:
+            return round(self._resident_bytes / (1024 * 1024), 3)
+
+    @property
+    def capacity_mb(self) -> float:
+        return round(self._capacity_bytes / (1024 * 1024), 2)
+
+    @property
+    def count(self) -> int:
+        with self._lock:
+            return len(self._entries)
+
+    # ------------------------------------------------------------------
+    # Core get / put / invalidate
+    # ------------------------------------------------------------------
+
+    def get(
+        self,
+        key: str,
+        refresh_callback: Optional[RefreshCallback] = None,
+        now_ts: Optional[float] = None,
+    ) -> Optional[Tuple[Any, bool, float, float]]:
+        """Look up a key. Returns (data, is_fresh, freshness, score) or None on miss.
+
+        Complexity: O(1) average — dict lookup, O(1) touch/score, and a
+        non-blocking background-refresh submit when stale but high-value.
+        """
+        now = now_ts if now_ts is not None else time.time()
+        with self._lock:
+            self.total_requests += 1
+            if key not in self._entries:
+                self.misses += 1
+                self._log("MISS", key, "")
+                return None
+
+            entry = self._entries[key]
+            entry.touch(now, self._decay_lambda)
+            fresh = entry.freshness(now)
+            sc = entry.score(now, self._p99_ms, self._p99_usd)
+            is_fresh = fresh >= self._refresh_threshold
+
+            self.hits += 1
+            self.total_usd_saved += entry.regen_usd
+            if is_fresh:
+                self.fresh_hits += 1
+            else:
+                self.stale_hits += 1
+
+            # Proactive background refresh for stale but high-value entries
+            if (
+                not is_fresh
+                and sc >= self._refresh_min_score
+                and refresh_callback is not None
+                and not self._refresh_pool.is_refreshing(key)
+            ):
+                self._refresh_pool.submit(
+                    key, refresh_callback,
+                    on_done=self._on_refresh_done,
+                    on_error=self._on_refresh_error,
+                )
+
+            self._log("HIT" if is_fresh else "HIT_STALE", key,
+                      f"score={sc:.3f} fresh={fresh:.2f}")
+            return entry.data, is_fresh, fresh, sc
+
+    def put(
+        self,
+        key: str,
+        data: Any,
+        cls: str = "generic",
+        size_bytes: Optional[int] = None,
+        regen_ms: float = 100.0,
+        regen_usd: float = 0.0,
+        change_interval_s: float = 0.0,
+        now_ts: Optional[float] = None,
+    ) -> bool:
+        """Admit or update an entry. Returns True if admitted, False if declined.
+
+        Complexity: O(1) average for admission control + dict store; O(e)
+        worst case when evictions are needed, where e = number of entries
+        evicted (each eviction is O(n) due to list.remove on the key list,
+        but the sampled pool itself is O(1)).
+        """
+        now = now_ts if now_ts is not None else time.time()
+        sz = size_bytes if size_bytes is not None else estimate_size_bytes(data)
+
+        with self._lock:
+            # Admission control: refuse highly volatile, low-value objects
+            if change_interval_s > 0 and change_interval_s < 1.0 and regen_usd < 0.0005:
+                self.declines += 1
+                self._log("DECLINE", key, f"change_interval={change_interval_s:.2f}s regen_usd={regen_usd}")
+                return False
+
+            # Update existing entry in-place
+            if key in self._entries:
+                old = self._entries[key]
+                self._resident_bytes -= old.size_bytes
+                old.data = data
+                old.size_bytes = sz
+                old.regen_ms = regen_ms
+                old.regen_usd = regen_usd
+                old.last_sync_ts = now
+                self._resident_bytes += sz
+                self._make_room(0, now)
+                self._log("UPDATE", key, f"size={sz}B")
+                return True
+
+            # Evict until there is room for the new entry
+            self._make_room(sz, now)
+
+            entry = CacheEntry(
+                key=key, data=data, cls=cls, size_bytes=sz,
+                regen_ms=regen_ms, regen_usd=regen_usd,
+                change_interval_s=change_interval_s, last_sync_ts=now,
+            )
+            self._entries[key] = entry
+            self._keys.append(key)
+            self._resident_bytes += sz
+            self._log("ADMIT", key, f"size={sz}B cls={cls}")
+            return True
+
+    def invalidate(self, key: str) -> bool:
+        """Remove a single key. Returns True if it was resident.
+
+        Complexity: O(n) — list.remove(key) on the resident key list.
+        """
+        with self._lock:
+            if key in self._entries:
+                self._evict_key(key, "INVALIDATE")
+                return True
+            return False
+
+    def invalidate_prefix(self, prefix: str) -> int:
+        """Remove all keys that start with prefix. Returns count removed.
+
+        Complexity: O(n * m) — n resident keys scanned, each startswith
+        check is O(m) in the prefix length; plus O(k * n) for the k
+        matched removals (list.remove per key).
+        """
+        with self._lock:
+            targets = [k for k in list(self._keys) if k.startswith(prefix)]
+            for k in targets:
+                self._evict_key(k, "INVALIDATE")
+            return len(targets)
+
+    def clear(self) -> None:
+        """Flush all entries and reset all metrics.
+
+        Complexity: O(n) — clearing the entries dict and key list.
+        """
+        with self._lock:
+            self._entries.clear()
+            self._keys.clear()
+            self._resident_bytes = 0
+            self.total_requests = self.hits = self.misses = 0
+            self.fresh_hits = self.stale_hits = self.proactive_refreshes = 0
+            self.total_usd_saved = 0.0
+            self.ram_latency_sum_ms = self.origin_latency_sum_ms = 0.0
+            for k in self.evictions:
+                self.evictions[k] = 0
+            self.declines = 0
+            self._recent_events.clear()
+
+    # ------------------------------------------------------------------
+    # Latency telemetry helpers
+    # ------------------------------------------------------------------
+
+    def record_ram_latency(self, ms: float) -> None:
+        with self._lock:
+            self.ram_latency_sum_ms += ms
+
+    def record_origin_latency(self, ms: float) -> None:
+        with self._lock:
+            self.origin_latency_sum_ms += ms
+
+    # ------------------------------------------------------------------
+    # Stats snapshot
+    # ------------------------------------------------------------------
+
+    def snapshot(self) -> Dict[str, Any]:
+        """Return a dict of current telemetry values (for CacheStats).
+
+        Complexity: O(n log n) — sorts all resident entries by score to
+        produce the top-25 list (heap-free full sort; n = resident count).
+        """
+        with self._lock:
+            now = time.time()
+            hit_rate = self.hits / self.total_requests if self.total_requests else 0.0
+            fresh_rate = self.fresh_hits / self.hits if self.hits else 0.0
+            avg_ram = self.ram_latency_sum_ms / self.hits if self.hits else 0.15
+            avg_origin = self.origin_latency_sum_ms / self.misses if self.misses else 0.0
+            top_keys = [
+                e.to_dict(now)
+                for e in sorted(
+                    self._entries.values(),
+                    key=lambda e: e.score(now, self._p99_ms, self._p99_usd),
+                    reverse=True,
+                )[:25]
+            ]
+            return {
+                "total_requests": self.total_requests,
+                "hits": self.hits,
+                "misses": self.misses,
+                "fresh_hits": self.fresh_hits,
+                "stale_hits": self.stale_hits,
+                "declines": self.declines,
+                "proactive_refreshes": self.proactive_refreshes,
+                "hit_rate_pct": round(hit_rate * 100, 2),
+                "freshness_pct": round(fresh_rate * 100, 2),
+                "usd_saved": round(self.total_usd_saved, 4),
+                "resident_mb": self.resident_mb,
+                "capacity_mb": self.capacity_mb,
+                "resident_count": len(self._entries),
+                "avg_ram_latency_ms": round(avg_ram, 3),
+                "avg_origin_latency_ms": round(avg_origin, 1),
+                "speedup_factor": round(avg_origin / max(0.001, avg_ram), 1) if avg_origin else 0.0,
+                "evictions": dict(self.evictions),
+                "recent_events": list(self._recent_events)[-20:],
+                "top_resident_keys": top_keys,
+            }
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def shutdown(self, wait: bool = True) -> None:
+        self._refresh_pool.shutdown(wait=wait)
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _make_room(self, needed_bytes: int, now_ts: float) -> None:
+        """Evict lowest-score entries until needed_bytes fits.
+
+        Complexity: O(e * (k + n)) where e = evictions performed,
+        k = victim pool size (bounded, <= _POOL_MAX) and the n term is
+        the list.remove(key) per eviction. With a bounded pool this is
+        O(e * n) worst case; typically e is small so it behaves ~O(n).
+        """
+        while (self._resident_bytes + needed_bytes > self._capacity_bytes) and self._entries:
+            candidates = sample_victim_pool(self._keys, self._pool_fraction)
+            victim_key = pick_victim(
+                candidates,
+                score_fn=lambda k: self._entries[k].score(now_ts, self._p99_ms, self._p99_usd),
+            )
+            if victim_key is None:
+                break
+            victim = self._entries[victim_key]
+            reason = classify_eviction_reason(
+                victim.freshness(now_ts), victim.hotness,
+                victim.size_bytes, victim.regen_ms, victim.regen_usd,
+            )
+            self.evictions[reason] = self.evictions.get(reason, 0) + 1
+            self._evict_key(victim_key, reason)
+
+    def _evict_key(self, key: str, reason: str) -> None:
+        """Remove one entry and update bookkeeping.
+
+        Complexity: O(n) — dominated by list.remove(key) on the key list.
+        """
+        if key in self._entries:
+            entry = self._entries.pop(key)
+            self._resident_bytes -= entry.size_bytes
+            try:
+                self._keys.remove(key)
+            except ValueError:
+                pass
+            self._log("EVICT", key, f"reason={reason}")
+
+    def _on_refresh_done(self, key: str, data: Any, latency_ms: float, size_bytes: int) -> None:
+        now = time.time()
+        with self._lock:
+            if key in self._entries:
+                entry = self._entries[key]
+                self._resident_bytes -= entry.size_bytes
+                entry.data = data
+                entry.size_bytes = size_bytes
+                entry.regen_ms = latency_ms
+                entry.last_sync_ts = now
+                self._resident_bytes += size_bytes
+                self.proactive_refreshes += 1
+                self._log("REFRESH", key, f"latency={latency_ms:.1f}ms")
+
+    def _on_refresh_error(self, key: str, exc: Exception) -> None:
+        self._log("REFRESH_ERR", key, str(exc))
+
+    def _log(self, op: str, key: str, details: str) -> None:
+        self._recent_events.append({
+            "time": time.strftime("%H:%M:%S"),
+            "op": op,
+            "key": key,
+            "details": details,
+        })
+
+
+# ---------------------------------------------------------------------------
+# Asynchronous Engine
+# ---------------------------------------------------------------------------
+
+class AsyncAdValEngine:
+    """Async in-memory AdVal cache engine using asyncio.Lock.
+
+    Wraps a synchronous engine but executes all mutations under an
+    asyncio.Lock so it is safe to use inside an async event loop without
+    blocking the thread.
+    """
+
+    def __init__(self, **kwargs: Any) -> None:
+        self._sync = AdValEngine(**kwargs)
+        self._lock = asyncio.Lock()
+
+    async def get(
+        self,
+        key: str,
+        refresh_callback: Optional[Callable[[], Any]] = None,
+        now_ts: Optional[float] = None,
+    ) -> Optional[Tuple[Any, bool, float, float]]:
+        async with self._lock:
+            return self._sync.get(key, refresh_callback, now_ts)
+
+    async def put(self, key: str, data: Any, **kwargs: Any) -> bool:
+        async with self._lock:
+            return self._sync.put(key, data, **kwargs)
+
+    async def invalidate(self, key: str) -> bool:
+        async with self._lock:
+            return self._sync.invalidate(key)
+
+    async def invalidate_prefix(self, prefix: str) -> int:
+        async with self._lock:
+            return self._sync.invalidate_prefix(prefix)
+
+    async def clear(self) -> None:
+        async with self._lock:
+            self._sync.clear()
+
+    def snapshot(self) -> Dict[str, Any]:
+        return self._sync.snapshot()
+
+    @property
+    def count(self) -> int:
+        return self._sync.count
+
+    @property
+    def resident_mb(self) -> float:
+        return self._sync.resident_mb
+
+    @property
+    def capacity_mb(self) -> float:
+        return self._sync.capacity_mb
+
+    def shutdown(self, wait: bool = True) -> None:
+        self._sync.shutdown(wait)

advalcache/_eviction.py

@@ -0,0 +1,93 @@
+"""advalcache/_eviction.py — Sampled victim pool eviction.
+
+Separated from the engine so the eviction strategy can be extended or
+replaced without touching core state management.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Callable, Dict, List, Optional
+
+# ---------------------------------------------------------------------------
+# Eviction reason labels
+# ---------------------------------------------------------------------------
+
+REASON_STALE = "STALE"
+REASON_COLD = "COLD"
+REASON_BULKY = "BULKY"
+REASON_CHEAP = "CHEAP"
+REASON_CAPACITY = "CAPACITY"
+
+#: Minimum pool fraction (10% of resident keys).
+_POOL_FRACTION = 0.10
+#: Minimum victim pool size.
+_POOL_MIN = 4
+#: Maximum victim pool size (keeps eviction O(log n) in practice).
+_POOL_MAX = 64
+
+
+def sample_victim_pool(
+    keys: List[str],
+    pool_fraction: float = _POOL_FRACTION,
+    pool_min: int = _POOL_MIN,
+    pool_max: int = _POOL_MAX,
+) -> List[str]:
+    """Return a random sample of resident keys to consider for eviction.
+
+    Size = clamp(len(keys) * pool_fraction, pool_min, pool_max).
+    Sampling is the standard practical answer when scores move continuously
+    with wall-clock time — a heap's entries go stale the moment they are pushed.
+
+    Complexity: O(k) where k = min(pool_max, n * pool_fraction) — k is a
+    bounded constant (<= _POOL_MAX), so effectively O(1) w.r.t. cache size n.
+    """
+    n = len(keys)
+    if n == 0:
+        return []
+    k = max(pool_min, min(pool_max, int(n * pool_fraction)))
+    return random.sample(keys, min(k, n))
+
+
+def pick_victim(
+    candidates: List[str],
+    score_fn: Callable[[str], float],
+) -> Optional[str]:
+    """Return the candidate key with the lowest AdVal score.
+
+    Complexity: O(k) score evaluations over k <= _POOL_MAX candidates,
+    i.e. effectively O(1) w.r.t. total cache size n.
+    """
+    if not candidates:
+        return None
+    return min(candidates, key=score_fn)
+
+
+def classify_eviction_reason(
+    freshness: float,
+    hotness: float,
+    size_bytes: int,
+    regen_ms: float,
+    regen_usd: float,
+    *,
+    stale_threshold: float = 0.20,
+    cold_threshold: float = 1.2,
+    bulky_threshold_bytes: int = 200 * 1024,
+) -> str:
+    """Classify why an entry is the weakest in its victim pool.
+
+    Priority: STALE > COLD > BULKY > CHEAP > CAPACITY.
+    The label tells the caller (and dashboard) what characteristic made
+    this entry the most disposable.
+
+    Complexity: O(1) — fixed chain of constant-time threshold checks.
+    """
+    if freshness < stale_threshold:
+        return REASON_STALE
+    if hotness < cold_threshold:
+        return REASON_COLD
+    if size_bytes > bulky_threshold_bytes:
+        return REASON_BULKY
+    if regen_usd == 0.0 and regen_ms < 50.0:
+        return REASON_CHEAP
+    return REASON_CAPACITY

advalcache/_refresh.py

@@ -0,0 +1,91 @@
+"""advalcache/_refresh.py — Background refresh worker pool.
+
+Manages a thread pool for proactive, non-blocking refresh of stale but
+high-value entries. Prevents double-refresh via an in-flight key set.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import logging
+import time
+import threading
+from typing import Any, Callable, Optional, Set, Tuple
+
+logger = logging.getLogger("advalcache.refresh")
+
+
+class BackgroundRefreshPool:
+    """Thread pool that runs refresh callbacks for high-value stale entries.
+
+    Usage:
+        pool = BackgroundRefreshPool(max_workers=4)
+        pool.submit("mykey", fetch_fn, on_done_callback)
+        pool.shutdown()
+    """
+
+    def __init__(self, max_workers: int = 4) -> None:
+        self._executor = concurrent.futures.ThreadPoolExecutor(
+            max_workers=max_workers,
+            thread_name_prefix="adval-refresh",
+        )
+        self._lock = threading.Lock()
+        self._in_flight: Set[str] = set()
+
+    def is_refreshing(self, key: str) -> bool:
+        """True if a refresh is already running for this key.
+
+        Complexity: O(1) — set membership check.
+        """
+        with self._lock:
+            return key in self._in_flight
+
+    def submit(
+        self,
+        key: str,
+        fetch_fn: Callable[[], Tuple[Any, float, int]],
+        on_done: Callable[[str, Any, float, int], None],
+        on_error: Optional[Callable[[str, Exception], None]] = None,
+    ) -> bool:
+        """Schedule a background refresh for key.
+
+        Returns False (no-op) if a refresh is already in-flight for key.
+        ``fetch_fn`` must return (data, latency_ms, size_bytes).
+        ``on_done`` is called with (key, data, latency_ms, size_bytes) on success.
+        ``on_error`` is called with (key, exc) on failure.
+
+        Complexity: O(1) — set check/add plus an executor submit;
+        ``fetch_fn`` itself runs on a worker thread in O(cost of fetch).
+        """
+        with self._lock:
+            if key in self._in_flight:
+                return False
+            self._in_flight.add(key)
+
+        self._executor.submit(self._run, key, fetch_fn, on_done, on_error)
+        return True
+
+    def _run(
+        self,
+        key: str,
+        fetch_fn: Callable[[], Tuple[Any, float, int]],
+        on_done: Callable[[str, Any, float, int], None],
+        on_error: Optional[Callable[[str, Exception], None]],
+    ) -> None:
+        try:
+            data, latency_ms, size_bytes = fetch_fn()
+            on_done(key, data, latency_ms, size_bytes)
+        except Exception as exc:
+            logger.warning("advalcache: background refresh failed for %r: %s", key, exc)
+            if on_error is not None:
+                try:
+                    on_error(key, exc)
+                except Exception:
+                    pass
+        finally:
+            with self._lock:
+                self._in_flight.discard(key)
+
+    def shutdown(self, wait: bool = True) -> None:
+        """Gracefully shut down the thread pool."""
+        self._executor.shutdown(wait=wait)