sourcecode 2.3.0__py3-none-any.whl → 2.5.0__py3-none-any.whl

sourcecode/context_cache.py

@@ -0,0 +1,637 @@
+"""
+AI Context Cache — structured-knowledge cache for ASK.
+
+This is **not** a prompt cache. It never stores prompts, generated text, or LLM
+responses. It stores the *structured context* that ASK's Knowledge Layer
+produces — entities, evidence, observations, summaries and graph metrics — so
+that expensive commands (``explain``, ``impact``, ``onboard``, ``prepare-context``,
+``archetype`` …) can reuse the same derived knowledge across runs.
+
+The component is completely independent of any AI provider. The cached objects
+are consumed identically by GPT, Claude, Gemini, a local model, or any future
+tool. The LLM never participates in producing or reading this cache.
+
+Design
+------
+    Context Request → Cache Key → Lookup → Hit / Miss → Store → Return Context
+
+Cache key
+---------
+The key depends **only on the state of the knowledge**, never on any prompt text::
+
+    repo_id + worktree_signature + KL_SCHEMA_VERSION + PRODUCER_VERSION
+            + command + target + relevant_options
+
+``worktree_signature`` is the committed HEAD SHA, folded with a hash of the
+working-tree diff when the tree is dirty — so an uncommitted edit deterministically
+invalidates the entry instead of serving stale context. If *any* component
+changes, the key changes and the old context is no longer served.
+
+Invalidation
+------------
+Deterministic and state-based, never TTL-based for correctness:
+  * new commit           → different ``worktree_signature`` → miss
+  * working-tree edit    → different ``worktree_signature`` → miss
+  * Knowledge Layer bump → different ``KL_SCHEMA_VERSION``  → miss
+  * producer bump        → different ``PRODUCER_VERSION``   → miss
+  * relevant option flip → different ``relevant_options``   → miss
+Stored entries also carry their own ``kl_version`` / ``signature`` and are
+re-validated on read as a defensive second check. TTL and a max-entry cap exist
+only as *cleanup*, never as a consistency guarantee.
+
+Persistence
+-----------
+    <base>/context-cache/<repo_id>/
+        ctx-<key16>.json.gz    ← one gzip'd CachedContext envelope per key
+        stats.json             ← advisory counters (hits/misses/…)
+
+Env vars
+--------
+  SOURCECODE_CONTEXT_CACHE_DIR         Override base dir (default: ~/.sourcecode)
+  SOURCECODE_CONTEXT_CACHE_DISABLE     "1" to disable (always miss, never store)
+  SOURCECODE_CONTEXT_CACHE_TTL_DAYS    Cleanup age (default 30; 0 = never)
+  SOURCECODE_CONTEXT_CACHE_MAX_ENTRIES Max entries per repo (default 256; 0 = unlimited)
+"""
+from __future__ import annotations
+
+import gzip
+import hashlib
+import json
+import os
+import subprocess
+import time
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+from sourcecode import __version__ as _PKG_VERSION
+
+# ---------------------------------------------------------------------------
+# Versioning — the stable half of the cache key
+# ---------------------------------------------------------------------------
+
+#: Bump when the *shape or meaning* of the Knowledge Layer / context objects
+#: changes in a way that makes older cached context invalid. Independent of the
+#: package version — a patch release must NOT bump this unless the context
+#: format actually changed.
+KL_SCHEMA_VERSION: str = "1"
+
+#: Envelope schema version. Bump to invalidate *every* stored context file.
+ENVELOPE_VERSION: str = "1"
+
+#: Producer identity — the exact code that built the context. Package version
+#: bumps naturally participate in the key so a new release never reads context
+#: built by an older one.
+PRODUCER_VERSION: str = _PKG_VERSION
+
+_DEFAULT_TTL_DAYS: int = 30
+_DEFAULT_MAX_ENTRIES: int = 256
+
+_UNIT_SEP = "\x1f"  # key-material field separator (never appears in inputs)
+
+
+# ---------------------------------------------------------------------------
+# Cached object
+# ---------------------------------------------------------------------------
+
+@dataclass
+class CachedContext:
+    """One unit of structured, inspectable context produced by ASK.
+
+    Deliberately *not* an opaque blob: every field is a plain JSON-serialisable
+    structure so the cache can be inspected, diffed and reasoned about. No
+    prompts, no generated text, no LLM output ever lands here.
+    """
+
+    metadata: dict[str, Any] = field(default_factory=dict)
+    entities: list[Any] = field(default_factory=list)
+    evidence: list[Any] = field(default_factory=list)
+    observations: list[Any] = field(default_factory=list)
+    summaries: dict[str, Any] = field(default_factory=dict)
+    graph_metrics: dict[str, Any] = field(default_factory=dict)
+    #: The reusable, command-agnostic knowledge artifact itself (e.g. the raw IR
+    #: dict from which a full CIR is rebuilt cheaply). This is the expensive thing
+    #: worth caching; the fields above are inspectable metadata over it.
+    payload: dict[str, Any] = field(default_factory=dict)
+    timestamps: dict[str, Any] = field(default_factory=dict)
+    producer_version: str = PRODUCER_VERSION
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "metadata": self.metadata,
+            "entities": self.entities,
+            "evidence": self.evidence,
+            "observations": self.observations,
+            "summaries": self.summaries,
+            "graph_metrics": self.graph_metrics,
+            "payload": self.payload,
+            "timestamps": self.timestamps,
+            "producer_version": self.producer_version,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "CachedContext":
+        return cls(
+            metadata=d.get("metadata", {}) or {},
+            entities=d.get("entities", []) or [],
+            evidence=d.get("evidence", []) or [],
+            observations=d.get("observations", []) or [],
+            summaries=d.get("summaries", {}) or {},
+            graph_metrics=d.get("graph_metrics", {}) or {},
+            payload=d.get("payload", {}) or {},
+            timestamps=d.get("timestamps", {}) or {},
+            producer_version=d.get("producer_version", "") or "",
+        )
+
+
+# ---------------------------------------------------------------------------
+# Location / identity helpers
+# ---------------------------------------------------------------------------
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _repo_id(repo_root: Path) -> str:
+    """Stable 16-char id from the canonical repo path (mirrors cache.repo_id)."""
+    return hashlib.sha256(str(repo_root.resolve()).encode()).hexdigest()[:16]
+
+
+def _base_dir() -> Path:
+    env = os.environ.get("SOURCECODE_CONTEXT_CACHE_DIR", "")
+    return Path(env) if env else Path.home() / ".sourcecode"
+
+
+def _cache_dir(repo_root: Path) -> Path:
+    return _base_dir() / "context-cache" / _repo_id(repo_root)
+
+
+def _git_head(repo_root: Path) -> str:
+    try:
+        r = subprocess.run(
+            ["git", "-C", str(repo_root), "rev-parse", "HEAD"],
+            capture_output=True, text=True, timeout=3,
+        )
+        if r.returncode == 0:
+            return r.stdout.strip()
+    except Exception:
+        pass
+    return ""
+
+
+def _worktree_signature(repo_root: Path) -> str:
+    """Deterministic fingerprint of the *exact* tree state.
+
+    committed HEAD, plus — only when the tree is dirty — a hash of the porcelain
+    status and the diff against HEAD. Two identical tree states yield the same
+    signature (correct reuse); any tracked edit changes it (correct invalidation).
+    Returns ``""`` when the path is not a git repo (caller disables caching).
+    """
+    head = _git_head(repo_root)
+    if not head:
+        return ""
+    try:
+        status = subprocess.run(
+            ["git", "-C", str(repo_root), "status", "--porcelain"],
+            capture_output=True, text=True, timeout=5,
+        )
+        porcelain = status.stdout if status.returncode == 0 else ""
+    except Exception:
+        porcelain = ""
+    if not porcelain.strip():
+        return head  # clean tree — HEAD fully describes it
+    try:
+        diff = subprocess.run(
+            ["git", "-C", str(repo_root), "diff", "HEAD"],
+            capture_output=True, text=True, timeout=10,
+        )
+        diff_txt = diff.stdout if diff.returncode == 0 else ""
+    except Exception:
+        diff_txt = ""
+    dirty = hashlib.sha256((porcelain + "\x00" + diff_txt).encode("utf-8", "replace")).hexdigest()[:16]
+    return f"{head}+dirty:{dirty}"
+
+
+def _options_fingerprint(options: Optional[dict[str, Any]]) -> str:
+    """Stable hash of the knowledge-relevant options (order-independent)."""
+    if not options:
+        return "-"
+    try:
+        canon = json.dumps(options, sort_keys=True, ensure_ascii=False, default=str)
+    except Exception:
+        canon = repr(sorted((str(k), str(v)) for k, v in options.items()))
+    return hashlib.sha256(canon.encode("utf-8")).hexdigest()[:12]
+
+
+# ---------------------------------------------------------------------------
+# Cache
+# ---------------------------------------------------------------------------
+
+class ContextCache:
+    """Provider-agnostic structured-context cache for one repository.
+
+    Construct via :meth:`for_repo`. All operations are best-effort: a broken or
+    unwritable cache degrades to a permanent miss, never to an error that would
+    break the command. Knows nothing about any AI model.
+    """
+
+    def __init__(
+        self,
+        cache_dir: Path,
+        signature: str,
+        *,
+        enabled: bool = True,
+    ) -> None:
+        self._dir = cache_dir
+        self._signature = signature
+        # No signature (non-git tree) means we cannot invalidate deterministically,
+        # so caching is disabled rather than risk serving stale context.
+        self.enabled = bool(enabled and signature)
+
+    @classmethod
+    def for_repo(cls, repo_root: Path, *, enabled: Optional[bool] = None) -> "ContextCache":
+        if enabled is None:
+            enabled = os.environ.get("SOURCECODE_CONTEXT_CACHE_DISABLE", "") not in ("1", "true", "yes")
+        root = repo_root.resolve()
+        return cls(_cache_dir(root), _worktree_signature(root), enabled=enabled)
+
+    # ── key ────────────────────────────────────────────────────────────────
+
+    def knowledge_key(
+        self,
+        scope: str,
+        *,
+        options: Optional[dict[str, Any]] = None,
+    ) -> str:
+        """Key for a **reusable, command-agnostic** knowledge artifact.
+
+        Depends only on the knowledge state and the artifact *scope* (e.g.
+        ``"java-cir"``) — deliberately NOT on any command or target, so every
+        command that needs the same knowledge shares one cache entry. This is
+        the level the cache operates at: cache the knowledge once, let commands
+        derive their answers from it for free.
+        """
+        return self.make_key(scope, options=options)
+
+    def make_key(
+        self,
+        scope: str,
+        *,
+        target: Optional[str] = None,
+        options: Optional[dict[str, Any]] = None,
+    ) -> str:
+        """Deterministic cache key. Depends only on knowledge state, never on prompt text.
+
+        ``scope`` names what is being cached. ``target`` is an optional finer
+        discriminator; for command-agnostic knowledge (the normal case) leave it
+        unset and prefer :meth:`knowledge_key`.
+        """
+        material = _UNIT_SEP.join([
+            self._signature,
+            KL_SCHEMA_VERSION,
+            PRODUCER_VERSION,
+            ENVELOPE_VERSION,
+            scope,
+            target or "",
+            _options_fingerprint(options),
+        ])
+        return hashlib.sha256(material.encode("utf-8")).hexdigest()[:16]
+
+    def _path(self, key: str) -> Path:
+        return self._dir / f"ctx-{key}.json.gz"
+
+    # ── read ─────────────────────────────────────────────────────────────
+
+    def get(self, key: str) -> Optional[CachedContext]:
+        """Return the cached context for *key*, or ``None`` on miss.
+
+        Records hit/miss and lookup latency. A stored entry whose embedded
+        signature/version no longer matches the current knowledge state is
+        treated as an invalidation (removed, counted, reported as a miss).
+        """
+        if not self.enabled:
+            return None
+        start = time.perf_counter()
+        path = self._path(key)
+        try:
+            if not path.exists():
+                self._record(miss=True, lookup_ms=(time.perf_counter() - start) * 1000)
+                return None
+            envelope = json.loads(gzip.decompress(path.read_bytes()).decode("utf-8"))
+        except Exception:
+            # Corrupted file — evict and count as a miss.
+            _safe_unlink(path)
+            self._record(miss=True, lookup_ms=(time.perf_counter() - start) * 1000)
+            return None
+
+        lookup_ms = (time.perf_counter() - start) * 1000
+        if not isinstance(envelope, dict) or envelope.get("ev") != ENVELOPE_VERSION:
+            _safe_unlink(path)
+            self._record(invalidation=True, lookup_ms=lookup_ms)
+            return None
+        # Defensive re-validation against live knowledge state.
+        if envelope.get("sig") != self._signature or envelope.get("kl") != KL_SCHEMA_VERSION:
+            _safe_unlink(path)
+            self._record(invalidation=True, lookup_ms=lookup_ms)
+            return None
+
+        data = envelope.get("context")
+        if not isinstance(data, dict):
+            _safe_unlink(path)
+            self._record(invalidation=True, lookup_ms=lookup_ms)
+            return None
+
+        self._record(hit=True, lookup_ms=lookup_ms)
+        return CachedContext.from_dict(data)
+
+    # ── write ─────────────────────────────────────────────────────────────
+
+    def put(
+        self,
+        key: str,
+        context: CachedContext,
+        *,
+        build_time_ms: Optional[float] = None,
+    ) -> None:
+        """Store *context* under *key*. Best-effort; failures are swallowed."""
+        if not self.enabled:
+            return
+        ts = dict(context.timestamps)
+        ts.setdefault("created_at", _now_iso())
+        if build_time_ms is not None:
+            ts["build_time_ms"] = round(build_time_ms, 3)
+        context.timestamps = ts
+
+        envelope = {
+            "ev": ENVELOPE_VERSION,
+            "kl": KL_SCHEMA_VERSION,
+            "sig": self._signature,
+            "key": key,
+            "ts": _now_iso(),
+            "context": context.to_dict(),
+        }
+        try:
+            self._dir.mkdir(parents=True, exist_ok=True)
+            payload = gzip.compress(json.dumps(envelope, ensure_ascii=False).encode("utf-8"), 6)
+            _atomic_write(self._path(key), payload)
+        except Exception:
+            return
+        self._record(store=True, build_ms=build_time_ms)
+        self._gc()
+
+    # ── stats ─────────────────────────────────────────────────────────────
+
+    def stats(self) -> dict[str, Any]:
+        """Return advisory metrics for this repo's context cache."""
+        s = _read_stats(self._dir)
+        files = list(self._dir.glob("ctx-*.json.gz")) if self._dir.exists() else []
+        bytes_stored = sum(f.stat().st_size for f in files if f.exists())
+        hits = s.get("hits", 0)
+        misses = s.get("misses", 0)
+        invalidations = s.get("invalidations", 0)
+        lookups = hits + misses + invalidations
+        lookup_total = s.get("lookup_time_total_ms", 0.0)
+        build_count = s.get("build_count", 0)
+        build_total = s.get("build_time_total_ms", 0.0)
+        return {
+            "cache_dir": str(self._dir),
+            "enabled": self.enabled,
+            "contexts": len(files),
+            "hits": hits,
+            "misses": misses,
+            "invalidations": invalidations,
+            "stores": s.get("stores", 0),
+            "hit_ratio": round(hits / lookups, 4) if lookups else 0.0,
+            "avg_lookup_ms": round(lookup_total / lookups, 4) if lookups else 0.0,
+            "avg_build_ms": round(build_total / build_count, 4) if build_count else 0.0,
+            "bytes_stored": bytes_stored,
+            "kl_schema_version": KL_SCHEMA_VERSION,
+            "producer_version": PRODUCER_VERSION,
+        }
+
+    def clear(self) -> int:
+        """Delete all context files and reset stats. Returns files removed."""
+        removed = 0
+        if self._dir.exists():
+            for f in self._dir.glob("ctx-*.json.gz"):
+                _safe_unlink(f)
+                removed += 1
+            _safe_unlink(self._dir / "stats.json")
+        return removed
+
+    # ── internals ─────────────────────────────────────────────────────────
+
+    def _record(
+        self,
+        *,
+        hit: bool = False,
+        miss: bool = False,
+        invalidation: bool = False,
+        store: bool = False,
+        lookup_ms: Optional[float] = None,
+        build_ms: Optional[float] = None,
+    ) -> None:
+        try:
+            s = _read_stats(self._dir)
+            if hit:
+                s["hits"] = s.get("hits", 0) + 1
+            if miss:
+                s["misses"] = s.get("misses", 0) + 1
+            if invalidation:
+                s["invalidations"] = s.get("invalidations", 0) + 1
+            if store:
+                s["stores"] = s.get("stores", 0) + 1
+            if lookup_ms is not None:
+                s["lookup_time_total_ms"] = s.get("lookup_time_total_ms", 0.0) + lookup_ms
+            if build_ms is not None:
+                s["build_count"] = s.get("build_count", 0) + 1
+                s["build_time_total_ms"] = s.get("build_time_total_ms", 0.0) + build_ms
+            _write_stats(self._dir, s)
+        except Exception:
+            pass  # stats are advisory — never fatal
+
+    def _gc(self) -> None:
+        """Cleanup pass: drop entries past TTL, then cap total entry count.
+
+        Cleanup only — correctness never depends on it (stale-key entries are
+        already unreachable because the key embeds the knowledge signature).
+        """
+        try:
+            ttl_days = int(os.environ.get("SOURCECODE_CONTEXT_CACHE_TTL_DAYS", _DEFAULT_TTL_DAYS))
+            max_entries = int(os.environ.get("SOURCECODE_CONTEXT_CACHE_MAX_ENTRIES", _DEFAULT_MAX_ENTRIES))
+            files = list(self._dir.glob("ctx-*.json.gz"))
+
+            if ttl_days > 0:
+                cutoff = time.time() - ttl_days * 86400
+                for f in list(files):
+                    try:
+                        if f.stat().st_mtime < cutoff:
+                            _safe_unlink(f)
+                            files.remove(f)
+                    except Exception:
+                        pass
+
+            if max_entries > 0 and len(files) > max_entries:
+                files.sort(key=lambda p: p.stat().st_mtime, reverse=True)  # newest first
+                for f in files[max_entries:]:
+                    _safe_unlink(f)
+        except Exception:
+            pass
+
+
+# ---------------------------------------------------------------------------
+# Stats file (advisory, plain JSON)
+# ---------------------------------------------------------------------------
+
+def _stats_path(cache_dir: Path) -> Path:
+    return cache_dir / "stats.json"
+
+
+def _read_stats(cache_dir: Path) -> dict[str, Any]:
+    p = _stats_path(cache_dir)
+    if not p.exists():
+        return {}
+    try:
+        d = json.loads(p.read_text(encoding="utf-8"))
+        return d if isinstance(d, dict) else {}
+    except Exception:
+        _safe_unlink(p)  # corrupted — reset
+        return {}
+
+
+def _write_stats(cache_dir: Path, stats: dict[str, Any]) -> None:
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    _atomic_write(_stats_path(cache_dir), json.dumps(stats, ensure_ascii=False).encode("utf-8"))
+
+
+# ---------------------------------------------------------------------------
+# I/O utilities
+# ---------------------------------------------------------------------------
+
+def _atomic_write(dest: Path, data: bytes) -> None:
+    """Write *data* to *dest* atomically via a unique temp file + os.replace."""
+    tmp = dest.parent / f"{dest.name}.{os.getpid()}.{uuid.uuid4().hex[:8]}.tmp"
+    try:
+        tmp.write_bytes(data)
+        os.replace(tmp, dest)
+    finally:
+        _safe_unlink(tmp)
+
+
+def _safe_unlink(path: Path) -> None:
+    try:
+        path.unlink(missing_ok=True)
+    except Exception:
+        pass
+
+
+# ---------------------------------------------------------------------------
+# Knowledge-level API — the reusable-context surface commands build on
+# ---------------------------------------------------------------------------
+
+#: Scope name for the Java/Spring Canonical IR — the expensive parse artifact
+#: shared by explain / impact / onboard / review-pr / modernize / archetype …
+SCOPE_JAVA_CIR: str = "java-cir"
+
+
+@dataclass
+class KnowledgeLookup:
+    """Outcome of a knowledge fetch, for command-side observability."""
+    hit: bool
+    enabled: bool
+    lookup_ms: float = 0.0
+    build_ms: float = 0.0
+
+    def render(self) -> str:
+        if not self.enabled:
+            return "Context cache: disabled"
+        if self.hit:
+            return f"Context cache: HIT  (lookup {self.lookup_ms:.1f}ms)"
+        return f"Context cache: MISS  (build {self.build_ms:.0f}ms)"
+
+
+def get_or_build_cir(
+    repo_root: Path,
+    root: Path,
+    file_list: list[str],
+    *,
+    since: Optional[str] = None,
+) -> tuple[Any, KnowledgeLookup]:
+    """Return ``(CanonicalRepositoryIR, KnowledgeLookup)``, caching the CIR.
+
+    This is the knowledge-level entry point. It caches the **reusable** raw IR
+    (the expensive Java parse) keyed only by knowledge state — not by command —
+    so every command that needs a CIR shares one cache entry. On a hit the full
+    CIR is rebuilt cheaply from the cached raw IR via ``ir_dict_to_canonical``,
+    skipping the parse entirely. Best-effort: any cache fault falls back to a
+    fresh build.
+    """
+    from sourcecode.canonical_ir import (
+        build_canonical_ir,
+        ir_dict_to_canonical,
+        validate_canonical_ir,
+    )
+
+    cache = ContextCache.for_repo(repo_root)
+    if not cache.enabled:
+        cir = build_canonical_ir(file_list, root, since=since)
+        return cir, KnowledgeLookup(hit=False, enabled=False)
+
+    key = cache.knowledge_key(SCOPE_JAVA_CIR, options={"since": since} if since else None)
+
+    t0 = time.perf_counter()
+    cached = cache.get(key)
+    lookup_ms = (time.perf_counter() - t0) * 1000
+    if cached is not None:
+        raw = cached.payload.get("raw_ir")
+        if isinstance(raw, dict):
+            try:
+                cir = ir_dict_to_canonical(raw, file_paths=file_list)
+                if not validate_canonical_ir(cir):
+                    return cir, KnowledgeLookup(hit=True, enabled=True, lookup_ms=lookup_ms)
+            except Exception:
+                pass  # corrupt/incompatible payload — fall through to rebuild
+
+    t1 = time.perf_counter()
+    cir = build_canonical_ir(file_list, root, since=since)
+    build_ms = (time.perf_counter() - t1) * 1000
+    try:
+        ctx = CachedContext(
+            metadata={
+                "scope": SCOPE_JAVA_CIR,
+                "cir_hash": cir.cir_hash,
+                "schema_version": cir.schema_version,
+                "file_count": len(cir.files),
+                "symbol_count": len(cir.symbols),
+            },
+            entities=list(cir.symbols[:200]),  # inspectable sample, not exhaustive
+            graph_metrics={
+                "symbol_count": len(cir.symbols),
+                "edge_count": len(cir.call_graph),
+                "endpoint_count": len(cir.endpoints),
+            },
+            payload={"raw_ir": cir._raw_ir},
+        )
+        cache.put(key, ctx, build_time_ms=build_ms)
+    except Exception:
+        pass
+    return cir, KnowledgeLookup(hit=False, enabled=True, build_ms=build_ms)
+
+
+# ---------------------------------------------------------------------------
+# Module-level observability helper
+# ---------------------------------------------------------------------------
+
+def stats_for_repo(repo_root: Path) -> dict[str, Any]:
+    """Convenience: stats for a repo without a live signature (read-only view)."""
+    root = repo_root.resolve()
+    cache = ContextCache(_cache_dir(root), _worktree_signature(root), enabled=True)
+    return cache.stats()
+
+
+def clear_for_repo(repo_root: Path) -> int:
+    root = repo_root.resolve()
+    cache = ContextCache(_cache_dir(root), "read-only", enabled=True)
+    return cache.clear()