codna 0.1.0__py3-none-any.whl

codna/memory.py

@@ -0,0 +1,885 @@
+"""Codna's local, in-process code memory — a thin consumer seam over Telys.
+
+Telys is an OPTIONAL dependency: the public SDK distribution is ``telys`` and the on-device runtime
+distribution is ``telys-runtime`` (import package ``memengine``). Install it with
+``pip install 'codna[memory]'``. Codna resolves the native kernel from explicit ``$TELYS_KERNEL`` /
+``$AME_KERNEL``, a packaged ``codna/_telys_runtime`` artifact, ``$CODNA_TELYS_INSTALL_ROOT``,
+``build/local-telys`` in a source checkout, or a verified ``telys runtime install``. Core codna never
+imports it — every telys import here is lazy, so ``from codna.memory import CodeMemory`` works without
+the extra and only fails (with a friendly message) when you actually use it.
+
+OEM license handling is explicit and secret-safe: Codna reads ``CODNA_TELYS_LICENSE_JWT`` or
+``CODNA_TELYS_LICENSE_PATH`` when configured, verifies it offline through Telys, and stores only metadata.
+Set ``CODNA_TELYS_LICENSE_REQUIRED=1`` in production channels that must fail closed without the license.
+
+The embedder is resolved in :func:`_embedder`: when an engine URL + API key are configured, recall uses
+the **API-key-gated engine** ``/v1/embeddings`` for REAL semantic vectors (the model auto-tracks the LLM
+used to fix the code — ``ALGENTA_AGENT_PROVIDER`` — overridable via ``CODNA_MEMORY_EMBED_MODEL``). With no
+engine/key (or ``CODNA_MEMORY_EMBED=local``) it falls back to the in-process ``AlgentaMultigramEmbedder``
+stand-in (384-d unigram+bigram+trigram fusion, lexical, zero network — the MC-validated upgrade over the
+old bigram default). Switching embedders changes the ``space_id`` → re-index required.
+
+Precision rerank (ON by default): the top-k pool is reordered by a lexical-heavy BLEND of the lexical
+recall score and an on-device WordLlama semantic score (``alpha*lexical + (1-alpha)*wordllama``, alpha=0.7
+— the best-measured config). Tune/disable via ``CODNA_MEMORY_RERANK`` (``blend`` default | ``wordllama`` |
+``lexical``/``off``) and ``CODNA_MEMORY_RERANK_ALPHA``. WordLlama ships in the ``[memory]`` extra; if it's
+somehow absent the blend falls back to lexical (never hard-fails).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import time
+
+from .codeunits import FILTER_COLUMNS, UnsupportedLanguageError, extract_repo, resolve_languages
+
+COLLECTION = "repo_symbols"
+DEFAULT_DB_DIRNAME = ".codna-memory"
+META_FILENAME = "codna_memory.json"
+HASH_FILENAME = "content_hashes.json"   # local id->content-hash manifest for incremental re-index (Merkle-style)
+_LICENSE_ENV_NAMES = ("CODNA_TELYS_LICENSE_JWT", "TELYS_LICENSE_JWT")
+_LICENSE_PATH_ENV_NAMES = ("CODNA_TELYS_LICENSE_PATH", "TELYS_LICENSE_PATH")
+_LICENSE_REQUIRED_ENV = "CODNA_TELYS_LICENSE_REQUIRED"
+_KERNEL_ENV_NAMES = ("TELYS_KERNEL", "AME_KERNEL")
+_CODNA_KERNEL_ENV_NAME = "CODNA_TELYS_KERNEL"
+_CODNA_INSTALL_ROOT_ENV_NAME = "CODNA_TELYS_INSTALL_ROOT"
+_PACKAGE_RUNTIME_DIRNAME = "_telys_runtime"
+_PACKAGE_RUNTIME_MANIFEST = "manifest.json"
+_PACKAGE_RUNTIME_SCHEMA_VERSION = 1
+_LAST_KERNEL_RESOLUTION: dict | None = None
+
+
+class CodeMemoryError(Exception):
+    """A codna-memory error surfaced to the user (missing extra / kernel / usage).
+
+    Deliberately NOT named ``MemoryError`` — that is a Python builtin and shadowing it would make
+    ``except MemoryError`` ambiguous. The CLI maps this to ``_die`` and the MCP tool to an error string.
+    """
+
+
+def _kernel_filename() -> str:
+    if sys.platform == "darwin":
+        return "libame_kernel.dylib"
+    if sys.platform.startswith("win"):
+        return "libame_kernel.dll"
+    return "libame_kernel.so"
+
+
+def _source_checkout_root() -> Path:
+    return Path(__file__).resolve().parents[2]
+
+
+def _runtime_platform_tag() -> str:
+    machine = platform.machine().lower() or "unknown"
+    if sys.platform == "darwin":
+        return f"macos-{machine}"
+    if sys.platform.startswith("linux"):
+        return f"linux-{machine}"
+    if sys.platform.startswith("win"):
+        return f"windows-{machine}"
+    return f"{sys.platform}-{machine}"
+
+
+def _sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _package_runtime_dir() -> Path:
+    return Path(__file__).resolve().parent / _PACKAGE_RUNTIME_DIRNAME
+
+
+def _load_package_runtime_manifest(path: Path) -> dict:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise CodeMemoryError(f"Codna packaged Telys runtime manifest is missing: {path}") from exc
+    except json.JSONDecodeError as exc:
+        raise CodeMemoryError(f"Codna packaged Telys runtime manifest is invalid JSON: {path}") from exc
+    if not isinstance(payload, dict):
+        raise CodeMemoryError(f"Codna packaged Telys runtime manifest must be a JSON object: {path}")
+    return payload
+
+
+def _validate_package_runtime_manifest(runtime_dir: Path, manifest: dict) -> dict:
+    expected_artifact = _kernel_filename()
+    if manifest.get("schema_version") != _PACKAGE_RUNTIME_SCHEMA_VERSION:
+        raise CodeMemoryError("Codna packaged Telys runtime manifest has unsupported schema_version")
+    if manifest.get("artifact") != expected_artifact:
+        raise CodeMemoryError(
+            f"Codna packaged Telys runtime artifact must be {expected_artifact}; "
+            f"manifest has {manifest.get('artifact')!r}"
+        )
+    if manifest.get("platform") != _runtime_platform_tag():
+        raise CodeMemoryError(
+            f"Codna packaged Telys runtime platform mismatch: {manifest.get('platform')!r}"
+        )
+    for key in ("sha256", "size_bytes", "source_basename"):
+        if key not in manifest:
+            raise CodeMemoryError(f"Codna packaged Telys runtime manifest is missing {key}")
+    if "source_path" in manifest:
+        raise CodeMemoryError("Codna packaged Telys runtime manifest must not store source_path")
+    artifact = runtime_dir / expected_artifact
+    if not artifact.is_file():
+        raise CodeMemoryError(f"Codna packaged Telys runtime kernel is missing: {artifact}")
+    if artifact.stat().st_size != manifest["size_bytes"]:
+        raise CodeMemoryError("Codna packaged Telys runtime kernel size does not match manifest")
+    if _sha256_file(artifact) != manifest["sha256"]:
+        raise CodeMemoryError("Codna packaged Telys runtime kernel sha256 does not match manifest")
+    return {
+        "manifest_path": str((runtime_dir / _PACKAGE_RUNTIME_MANIFEST).resolve()),
+        "sha256": manifest["sha256"],
+        "size_bytes": manifest["size_bytes"],
+        "platform": manifest["platform"],
+    }
+
+
+def _package_runtime_candidate() -> tuple[Path, dict] | None:
+    runtime_dir = _package_runtime_dir()
+    manifest_path = runtime_dir / _PACKAGE_RUNTIME_MANIFEST
+    kernels = sorted(runtime_dir.glob("libame_kernel.*")) if runtime_dir.is_dir() else []
+    if not manifest_path.exists() and not kernels:
+        return None
+    expected = _kernel_filename()
+    unexpected = [path.name for path in kernels if path.name != expected]
+    if unexpected:
+        raise CodeMemoryError(
+            "Codna packaged Telys runtime contains unsupported kernel artifact(s): "
+            + ", ".join(unexpected)
+        )
+    manifest = _load_package_runtime_manifest(manifest_path)
+    metadata = _validate_package_runtime_manifest(runtime_dir, manifest)
+    return runtime_dir / expected, metadata
+
+
+def _candidate_codna_install_roots() -> list[tuple[str, Path]]:
+    roots: list[tuple[str, Path]] = []
+    raw = os.environ.get(_CODNA_INSTALL_ROOT_ENV_NAME)
+    if raw and raw.strip():
+        roots.append((f"env:{_CODNA_INSTALL_ROOT_ENV_NAME}", Path(raw).expanduser()))
+    roots.append(("codna:source-build", _source_checkout_root() / "build" / "local-telys"))
+    return roots
+
+
+def _candidate_kernel_paths() -> list[tuple[str, Path]]:
+    filename = _kernel_filename()
+    candidates: list[tuple[str, Path]] = []
+    raw_kernel = os.environ.get(_CODNA_KERNEL_ENV_NAME)
+    if raw_kernel and raw_kernel.strip():
+        candidates.append((f"env:{_CODNA_KERNEL_ENV_NAME}", Path(raw_kernel).expanduser()))
+    packaged = _package_runtime_candidate()
+    if packaged is not None:
+        candidates.append(("codna:package-runtime", packaged[0]))
+    for source, root in _candidate_codna_install_roots():
+        candidates.append((source, root / "kernel" / filename))
+    try:
+        from telys.paths import AME_KERNEL_STEM, installed_lib_path
+
+        installed = installed_lib_path(AME_KERNEL_STEM)
+        if installed:
+            candidates.append(("telys:installed-runtime", Path(installed)))
+    except Exception:  # noqa: BLE001 - kernel diagnostics must not mask the real runtime import error.
+        pass
+    return candidates
+
+
+def _explicit_kernel_env() -> tuple[str, str] | None:
+    for name in _KERNEL_ENV_NAMES:
+        if name in os.environ:
+            return name, os.environ.get(name, "")
+    return None
+
+
+def _resolve_telys_kernel(*, configure_env: bool) -> dict:
+    explicit = _explicit_kernel_env()
+    if explicit is not None:
+        name, raw = explicit
+        if not raw.strip():
+            raise CodeMemoryError(f"{name} is set but empty")
+        path = Path(raw).expanduser().resolve()
+        if not path.is_file():
+            raise CodeMemoryError(f"Telys kernel path from {name} does not exist or is not a file: {path}")
+        return {"found": True, "source": f"env:{name}", "path": str(path)}
+
+    checked: list[str] = []
+    for source, raw_path in _candidate_kernel_paths():
+        path = raw_path.expanduser().resolve()
+        checked.append(f"{source}:{path}")
+        if path.is_file():
+            if configure_env:
+                os.environ["TELYS_KERNEL"] = str(path)
+            result = {"found": True, "source": source, "path": str(path)}
+            if source == "codna:package-runtime":
+                _, metadata = _package_runtime_candidate() or (path, {})
+                result.update(metadata)
+            return result
+    return {"found": False, "source": "missing", "path": None, "checked": checked}
+
+
+def _configure_telys_kernel_env() -> dict:
+    global _LAST_KERNEL_RESOLUTION
+    _LAST_KERNEL_RESOLUTION = _resolve_telys_kernel(configure_env=True)
+    return dict(_LAST_KERNEL_RESOLUTION)
+
+
+def _current_kernel_resolution() -> dict:
+    if _LAST_KERNEL_RESOLUTION is not None:
+        return dict(_LAST_KERNEL_RESOLUTION)
+    return _resolve_telys_kernel(configure_env=False)
+
+
+def _kernel_hint() -> str:
+    resolution = _current_kernel_resolution()
+    checked = resolution.get("checked") or []
+    checked_lines = "\n".join(f"  - {item}" for item in checked[:6])
+    if checked_lines:
+        checked_lines = "\nCodna checked:\n" + checked_lines
+    return (
+        "the Telys kernel could not be loaded. Run `telys runtime install --file <bundle> "
+        "--license <license.jwt>`, set TELYS_KERNEL=/abs/path/to/libame_kernel, or set "
+        "CODNA_TELYS_INSTALL_ROOT to a Codna-managed install containing kernel/"
+        f"{_kernel_filename()}.{checked_lines}"
+    )
+
+
+def _require_telys():
+    """Lazily import the Telys surface. Raises CodeMemoryError with an install hint if absent."""
+    try:
+        from telys import Telys, scope_key
+        _configure_telys_kernel_env()
+        # Multigram fusion ([unigram|bigram|trigram], per-block L2): MC-validated to beat the plain bigram
+        # by ~+12-17 pts recall@10 on NL->code and to tie a 16 MB on-device semantic model on recall, at
+        # zero model cost. The local lexical default (no engine key) uses it.
+        from telys.embedding import AlgentaMultigramEmbedder as _LocalEmbedder
+    except ImportError as exc:
+        raise CodeMemoryError(
+            "code memory needs the optional Telys engine — install it with:  pip install 'codna[memory]'"
+        ) from exc
+    return Telys, scope_key, _LocalEmbedder
+
+
+def _truthy_env(name: str) -> bool:
+    return (os.environ.get(name) or "").strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _configured_license_token() -> tuple[str | None, str | None]:
+    for name in _LICENSE_ENV_NAMES:
+        raw = os.environ.get(name)
+        if raw is None:
+            continue
+        token = raw.strip()
+        if not token:
+            raise CodeMemoryError(f"{name} is set but empty")
+        return token, f"env:{name}"
+
+    for name in _LICENSE_PATH_ENV_NAMES:
+        raw = os.environ.get(name)
+        if raw is None:
+            continue
+        if not raw.strip():
+            raise CodeMemoryError(f"{name} is set but empty")
+        path = os.path.abspath(os.path.expanduser(raw.strip()))
+        try:
+            with open(path, encoding="utf-8") as fh:
+                token = fh.read().strip()
+        except OSError as exc:
+            raise CodeMemoryError(f"Telys license path from {name} is unreadable: {path}") from exc
+        if not token:
+            raise CodeMemoryError(f"Telys license path from {name} is empty: {path}")
+        return token, f"path:{name}"
+
+    return None, None
+
+
+def _load_telys_license_metadata() -> dict:
+    """Verify the configured Telys license offline and return safe metadata only.
+
+    This deliberately does not return the JWT. Callers can surface the metadata in status/diagnostics without
+    risking token disclosure.
+    """
+    required = _truthy_env(_LICENSE_REQUIRED_ENV)
+    token, source = _configured_license_token()
+    if token is None:
+        if required:
+            raise CodeMemoryError(
+                "Telys license is required but not configured; set CODNA_TELYS_LICENSE_JWT "
+                "or CODNA_TELYS_LICENSE_PATH"
+            )
+        return {"configured": False, "required": False}
+    try:
+        from telys import verify as license_verify
+    except ImportError as exc:
+        raise CodeMemoryError(
+            "Telys license is configured but the Telys verifier is unavailable; install codna[memory]"
+        ) from exc
+    try:
+        claims = license_verify.verify_license(token, now=int(time.time()))
+    except Exception as exc:  # noqa: BLE001
+        raise CodeMemoryError(f"Telys license from {source} is invalid: {type(exc).__name__}") from exc
+
+    product = (claims.get("products") or {}).get("telys") or {}
+    if not product:
+        raise CodeMemoryError(f"Telys license from {source} does not include the telys product")
+    return {
+        "configured": True,
+        "required": required,
+        "source": source,
+        "license_id": claims.get("license_id"),
+        "customer": claims.get("customer"),
+        "org_id": claims.get("org_id"),
+        "deployment_class": claims.get("deployment_class"),
+        "tier": product.get("tier"),
+        "features": list(product.get("features") or []),
+    }
+
+
+# Map the fix LLM provider → its registered embedding-model id. Chat models (claude/gpt) can't embed, so
+# we use the SAME provider's embedding model. Anthropic ships no embeddings API → use OpenAI's (whose key
+# is already configured for the gpt-5.4 fix path). Override per run with CODNA_MEMORY_EMBED_MODEL.
+_PROVIDER_EMBED_MODEL = {
+    "anthropic": "codna.embed.openai",
+    "openai-native": "codna.embed.openai", "openai": "codna.embed.openai",
+    "gemini": "codna.embed.gemini", "google_genai": "codna.embed.gemini",
+}
+_EMBED_DIM = {"codna.embed.openai": 1024, "codna.embed.gemini": 1536}
+
+
+def _engine_embed_config():
+    """Resolve (engine_url, api_key, model_id, dimension) for the API-key-gated remote code embedder,
+    or ``None`` to fall back to the local stand-in. The model auto-tracks the fix provider
+    (``ALGENTA_AGENT_PROVIDER``); override with ``CODNA_MEMORY_EMBED_MODEL`` / ``_DIM``. Force the
+    in-process stand-in with ``CODNA_MEMORY_EMBED=local``; require the remote with ``=remote``."""
+    if (os.environ.get("CODNA_MEMORY_EMBED") or "").lower() == "local":
+        return None
+    url = os.environ.get("CODNA_ENGINE_URL") or os.environ.get("ALGENTA_ENGINE_URL")
+    key = (os.environ.get("CODNA_API_KEY") or os.environ.get("ALGENTA_API_KEY")
+           or os.environ.get("ALGENTA_ENGINE_API_KEY"))
+    if not (url and key):
+        return None
+    model = os.environ.get("CODNA_MEMORY_EMBED_MODEL")
+    if not model:
+        prov = (os.environ.get("ALGENTA_AGENT_PROVIDER") or "anthropic").lower()
+        model = _PROVIDER_EMBED_MODEL.get(prov, "codna.embed.openai")
+    dim = int(os.environ.get("CODNA_MEMORY_EMBED_DIM") or _EMBED_DIM.get(model, 1024))
+    return url.rstrip("/"), key, model, dim
+
+
+def _remote_embedder(url: str, key: str, model: str, dim: int):
+    """A ``CallableEmbedder`` backed by the engine's keyed ``/v1/embeddings`` — REAL semantic vectors
+    (the "get it from Telys/Algenta with the API key" path). L2-normalized so inner-product == cosine."""
+    import httpx
+    import numpy as np
+    from telys.embedding import CallableEmbedder, EmbeddingProfile
+
+    def _embed(texts):
+        out = []
+        for i in range(0, len(texts), 128):  # bounded request bodies
+            chunk = list(texts[i:i + 128])
+            body = {"input": chunk, "model": model, "dimensions": dim}
+            # Resilient to TRANSIENT blips (connect/timeout, 429, 5xx): a momentary embedding-endpoint
+            # hiccup must NOT fail an otherwise-good recall/fix. Retry with backoff; only a persistent
+            # failure (or a real 4xx) propagates — so `--memory on` still fails loud when memory is truly
+            # broken, but survives a one-off network/engine stutter (the websockets soak case).
+            for attempt in range(4):
+                try:
+                    r = httpx.post(f"{url}/v1/embeddings",
+                                   headers={"Authorization": f"Bearer {key}"}, json=body, timeout=120)
+                except httpx.TransportError:
+                    if attempt == 3:
+                        raise
+                    time.sleep(0.5 * (2 ** attempt))
+                    continue
+                if r.status_code in (429, 500, 502, 503, 504) and attempt < 3:
+                    time.sleep(0.5 * (2 ** attempt))
+                    continue
+                r.raise_for_status()
+                out.extend(item["embedding"] for item in sorted(r.json()["data"], key=lambda d: d["index"]))
+                break
+        v = np.asarray(out, dtype="float32")
+        n = np.linalg.norm(v, axis=1, keepdims=True)
+        return v / np.where(n == 0, 1.0, n)
+    return CallableEmbedder(
+        fn=_embed,
+        profile=EmbeddingProfile(provider="algenta-engine", model_id=model, model_version="1",
+                                 dimension=dim, normalization="l2", distance="ip", tokenizer_hash=model))
+
+
+def _embedder():
+    """Resolve the code embedder. Prefer the API-key-gated engine embedder (real semantic recall,
+    auto-tracking the fix provider); fall back to the in-process multigram fusion stand-in (lexical,
+    on-device, no key) — the MC-validated upgrade over the old bigram default."""
+    cfg = _engine_embed_config()
+    if cfg is not None:
+        try:
+            return _remote_embedder(*cfg)
+        except Exception as exc:  # noqa: BLE001 — telys import / httpx / engine down
+            if (os.environ.get("CODNA_MEMORY_EMBED") or "").lower() == "remote":
+                raise CodeMemoryError(f"remote embedder required (CODNA_MEMORY_EMBED=remote) "
+                                      f"but unavailable: {exc}") from exc
+            # else: silently fall through to the local stand-in below
+    _, _, LocalEmbedder = _require_telys()
+    try:
+        return LocalEmbedder()    # AlgentaMultigramEmbedder: unigram+bigram+trigram fusion, zero network
+    except OSError as exc:   # FileNotFoundError (kernel missing) / CDLL load failure
+        raise CodeMemoryError(_kernel_hint()) from exc
+
+
+def rerank(query: str, candidates: list[tuple[str, float]]) -> list[tuple[str, float]]:
+    """Consumer-side rerank seam. Day-1: identity (keep Telys' partition-aware order + scores).
+    A real cross-encoder / Algenta reranker plugs in here later — no network on day 1."""
+    return candidates
+
+
+# Query-aware soft rerank (lossless, recall-time policy). Tests stay FULLY indexed; when the issue is NOT
+# test-oriented we softly DOWN-WEIGHT test symbols so source files surface. Monte-Carlo-validated over 900
+# sampled queries: recall@10 0.638→0.741 (non-overlapping 95% CIs), MRR 0.324→0.414, no repo regressed —
+# and test-oriented queries (failing test / assertion / fixture / flaky / junit …) are UNTOUCHED so bugs in
+# test files stay findable. This is NOT exclusion: blind exclusion makes test-file bugs unfixable. Tune or
+# disable with CODNA_MEMORY_TEST_DOWNWEIGHT (1.0 = off).
+_TEST_QUERY_RE = re.compile(
+    r"\b(test|tests|assert|assertion|fixture|flaky|junit|pytest|unittest|mock|conftest|failing)\b", re.I)
+
+
+def _query_is_test_oriented(query: str) -> bool:
+    return bool(_TEST_QUERY_RE.search(query or ""))
+
+
+def _is_test_symbol(symbol_type: str | None, path: str | None) -> bool:
+    if symbol_type == "test":
+        return True
+    p = (path or "").replace("\\", "/").lower()
+    b = p.rsplit("/", 1)[-1]
+    return ("/tests/" in p or "/test/" in p or b.startswith("test_")
+            or b.endswith("_test.py") or b == "conftest.py")
+
+
+def _soft_test_downweight(query: str, ids_scores: list, meta_by_id: dict) -> list:
+    """Re-sort (id, score) by a soft test penalty when the query isn't test-oriented; identity otherwise.
+    Operates over the full candidate set so a buried source symbol can surface into final-k."""
+    try:
+        factor = float(os.environ.get("CODNA_MEMORY_TEST_DOWNWEIGHT", "0.85"))
+    except ValueError:
+        factor = 0.85
+    if factor == 1.0 or _query_is_test_oriented(query):
+        return list(ids_scores)
+
+    def _adj(id_, sc):
+        m = meta_by_id.get(id_) or {}
+        return sc * factor if _is_test_symbol(m.get("symbol_type"), m.get("path")) else sc
+
+    return sorted(ids_scores, key=lambda t: -_adj(t[0], t[1]))
+
+
+# Optional on-device semantic PRECISION reranker. The lexical multigram fusion does the cheap, fast first-pass
+# recall; an ultra-light on-device model (WordLlama, 16 MB, CPU) re-scores only the top-k survivors to lift
+# ranking precision (MRR). A benchmark put WordLlama's MRR at 0.48 vs fusion's 0.37 at equal recall@10. OFF by
+# default; lossless (only reorders); enable with CODNA_MEMORY_RERANK=wordllama (CODNA_MEMORY_RERANK_K caps the
+# pool, default 40). WordLlama is an OPTIONAL dependency — imported lazily, with a clear hint if missing.
+_WL = []
+
+
+def _wordllama():
+    if not _WL:
+        try:
+            from wordllama import WordLlama
+        except ImportError as exc:
+            raise CodeMemoryError(
+                "semantic reranking (CODNA_MEMORY_RERANK=wordllama) needs the optional WordLlama model — "
+                "install it with:  pip install wordllama") from exc
+        _WL.append(WordLlama.load())
+    return _WL[0]
+
+
+def _wl_cosine(wl, query: str, texts: list) -> "list":
+    """Cosine of each candidate text vs the query under WordLlama (L2-normalized embeddings)."""
+    import numpy as np
+    m = np.asarray(wl.embed([query] + list(texts)), np.float32)
+    n = np.linalg.norm(m, axis=1, keepdims=True)
+    n[n == 0] = 1.0
+    m = m / n
+    return list(m[1:] @ m[0])
+
+
+def _semantic_rerank(query: str, cands: list, mem) -> list:
+    """DEFAULT precision rerank of the top-k pool: a lexical-heavy BLEND of the lexical recall score and an
+    on-device WordLlama semantic score — alpha*z(lexical) + (1-alpha)*z(wordllama), alpha=0.7. That blend was
+    the best-measured config (recall@10 0.775 vs 0.765 lexical-only / 0.665 WordLlama-only, 480-query CI study).
+
+    Modes via CODNA_MEMORY_RERANK: 'blend' (DEFAULT) | 'wordllama' (pure semantic) | 'lexical'/'off' (no rerank).
+    alpha via CODNA_MEMORY_RERANK_ALPHA (default 0.7). Lossless: only the top-`CODNA_MEMORY_RERANK_K` reorder.
+    GRACEFUL: if WordLlama isn't installed the default blend silently falls back to lexical (never hard-fails);
+    only an EXPLICIT CODNA_MEMORY_RERANK=wordllama surfaces the missing-dependency error."""
+    mode = (os.environ.get("CODNA_MEMORY_RERANK") or "blend").lower()
+    if mode in ("off", "none", "lexical") or not cands:
+        return cands
+    try:
+        k = int(os.environ.get("CODNA_MEMORY_RERANK_K", "40"))
+    except ValueError:
+        k = 40
+    head, tail = cands[:k], cands[k:]
+    id2text = mem._unit_texts()
+    texts = [id2text.get(i, "") for i, _ in head]
+    if not any(texts):                       # no recoverable text (e.g. repo moved) → leave order untouched
+        return cands
+    try:
+        wl = _wordllama()
+    except CodeMemoryError:
+        if mode in ("wordllama", "wl"):      # explicitly requested -> surface the missing-dependency error
+            raise
+        return cands                         # default blend: WordLlama absent -> fall back to lexical
+    wls = _wl_cosine(wl, query, texts)
+    if mode in ("wordllama", "wl"):          # pure semantic order
+        order = sorted(range(len(head)), key=lambda j: -float(wls[j]))
+        return [(head[j][0], float(wls[j])) for j in order] + tail
+    import numpy as np                        # blend (default): alpha*z(lexical) + (1-alpha)*z(wordllama)
+    try:
+        alpha = min(max(float(os.environ.get("CODNA_MEMORY_RERANK_ALPHA", "0.7")), 0.0), 1.0)
+    except ValueError:
+        alpha = 0.7
+
+    def _z(x):
+        sd = x.std()
+        return (x - x.mean()) / sd if sd > 1e-9 else x * 0.0
+
+    lex = np.array([s for _, s in head], dtype=float)
+    blend = alpha * _z(lex) + (1.0 - alpha) * _z(np.array(wls, dtype=float))
+    order = list(np.argsort(-blend))
+    return [(head[j][0], float(blend[j])) for j in order] + tail
+
+
+def repo_id_for(repo_path: str) -> str:
+    """Stable repo identity for partitioning: ``github.com/owner/repo`` if there's an origin remote,
+    else the absolute path (so memory works on any local checkout, not only GitHub repos)."""
+    try:
+        out = subprocess.run(["git", "-C", repo_path, "remote", "get-url", "origin"],
+                             capture_output=True, text=True, timeout=5)
+        if out.returncode == 0:
+            m = re.search(r"github\.com[:/]([^/]+/[^/.]+)", out.stdout)
+            if m:
+                return f"github.com/{m.group(1)}"
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return os.path.abspath(os.path.expanduser(repo_path))
+
+
+def _git(repo_path: str, *args: str) -> str | None:
+    try:
+        out = subprocess.run(["git", "-C", repo_path, *args],
+                             capture_output=True, text=True, timeout=5)
+        return out.stdout.strip() if out.returncode == 0 else None
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+def _provenance(unit_id: str) -> dict:
+    """Recover path + qualname from a stable unit id (``repo_id:relpath:qualname``)."""
+    parts = unit_id.split(":")
+    if len(parts) >= 3:
+        return {"path": parts[-2], "qualname": parts[-1]}
+    return {"path": None, "qualname": parts[-1] if parts else unit_id}
+
+
+def _metadata_matches(metadata: dict, filters: dict[str, str]) -> bool:
+    return all(str(metadata.get(key) or "") == str(value) for key, value in filters.items())
+
+
+class CodeMemory:
+    """In-process semantic code memory for one repository, backed by a Telys collection."""
+
+    def __init__(self, repo_path: str, db_path: str | None = None, *,
+                 service: str | None = None, branch: str | None = None, tuner=None) -> None:
+        self.repo_path = os.path.abspath(os.path.expanduser(repo_path))
+        self.repo_id = repo_id_for(self.repo_path)
+        self.db_path = os.path.abspath(db_path) if db_path else os.path.join(self.repo_path, DEFAULT_DB_DIRNAME)
+        self.service = service
+        self.branch = branch
+        self._tuner = tuner
+        self._Telys, self._scope_key, _ = _require_telys()
+        self.license = _load_telys_license_metadata()
+        self.kernel = _current_kernel_resolution()
+        self._embedder = _embedder()                       # constructs (and validates) the kernel up-front
+        self._eng = self._Telys(self.db_path, embedding_providers={self._embedder.profile.model_id: self._embedder})
+        self._col = None
+
+    # ── collection lifecycle ──────────────────────────────────────────────────────────────────────
+    def _collection(self):
+        if self._col is None:
+            if COLLECTION in self._eng.collections():
+                self._col = self._eng.open_collection(COLLECTION, embedder=self._embedder)
+            else:
+                self._col = self._eng.create_collection(
+                    COLLECTION, dim=self._embedder.profile.dimension, partition_by="scope_key",
+                    embedder=self._embedder, filter_columns=FILTER_COLUMNS, tuner=self._tuner)
+        return self._col
+
+    def is_empty(self) -> bool:
+        if COLLECTION not in self._eng.collections():
+            return True
+        return (self._collection().stats().get("external_ids") or 0) == 0
+
+    def _scope_for(self, unit) -> str:
+        return self._scope_key(self.repo_id, self.service or unit.service or "", unit.language)
+
+    def _unit_texts(self, languages=None) -> dict:
+        """{unit_id: source_text} for this repo, cached. Used ONLY by the optional WordLlama reranker to get
+        candidate text at query time, so the index stores no extra text column.
+
+        Normally the cache is populated by ``index()`` (reusing its extraction, all indexed languages). If
+        recall runs on a fresh instance without a prior ``index()``, fall back to extracting the languages
+        last indexed (persisted intent) or all registered languages — never silently Python-only when the
+        repo is multi-language."""
+        cache = getattr(self, "_unit_text_cache", None)
+        if cache is None:
+            langs = languages or getattr(self, "_indexed_languages", None)
+            units, _ = extract_repo(self.repo_path, self.repo_id, languages=langs)
+            cache = {u.id: u.text for u in units}
+            self._unit_text_cache = cache
+        return cache
+
+    # ── index / recall / status / compact / reset ─────────────────────────────────────────────────
+    def index(self, *, languages: tuple[str, ...] | None = None, refresh: bool = True) -> dict:
+        """Extract the repo into structural units and upsert them per scope; delete vanished symbols.
+
+        The vanished-diff reads each scope's live ids back from Telys (`col.ids(where=scope)`) — Codna
+        keeps no side id map. The prune is SCOPE-LOCAL, so re-indexing one repo/scope never touches
+        another's rows in a shared collection.
+        """
+        try:
+            resolved_languages = resolve_languages(languages)
+            units, stats = extract_repo(self.repo_path, self.repo_id, languages=resolved_languages)
+        except UnsupportedLanguageError as exc:
+            raise CodeMemoryError(str(exc)) from exc
+        col = self._collection()
+        # Reuse this full extraction as the WordLlama reranker's candidate-text cache: multi-language
+        # (every indexed language, not just Python) AND no re-extraction at query time. Without this the
+        # reranker would only see Python text and score every non-Python candidate as an empty string.
+        self._indexed_languages = resolved_languages
+        self._unit_text_cache = {u.id: u.text for u in units}
+        commit = _git(self.repo_path, "rev-parse", "HEAD") or ""
+        branch = self.branch or _git(self.repo_path, "rev-parse", "--abbrev-ref", "HEAD") or ""
+
+        # Incremental re-index: skip embedding symbols whose content is unchanged since the last index.
+        # The per-symbol hash is salted by the embedder's space_id, so a model change invalidates every entry
+        # (a fresh embed). Manifest is local (no engine round-trip); delete it to force a full re-embed.
+        salt = self._embedder.profile.space_id()
+        def _uhash(u):
+            service_key = self.service or u.service or ""
+            return hashlib.sha256((salt + "\x1f" + service_key + "\x1f" + u.text).encode("utf-8")).hexdigest()[:16]
+        prev = self._load_hashes() if refresh else {}
+        new_hashes: dict[str, str] = {}
+
+        by_scope: dict[str, list] = {}
+        new_ids: dict[str, set] = {}
+        for u in units:
+            scope = self._scope_for(u)
+            by_scope.setdefault(scope, []).append(u)
+            new_ids.setdefault(scope, set()).add(u.id)
+
+        removed = 0
+        if refresh:
+            # The scopes this index owns ((repo_id, service) × the scopes written + the requested
+            # languages, so an emptied scope is still pruned). Read each scope's current live ids from
+            # Telys and delete the ones this index no longer produces.
+            owned = set(new_ids) | {self._scope_key(self.repo_id, self.service or "", lang)
+                                    for lang in resolved_languages}
+            for scope in owned:
+                vanished = sorted(set(col.ids(where={"scope_key": scope})) - new_ids.get(scope, set()))
+                if vanished:
+                    col.delete(vanished)
+                    removed += len(vanished)
+
+        written = unchanged = 0
+        for scope, group in by_scope.items():
+            changed = []
+            for u in group:
+                h = _uhash(u)
+                new_hashes[u.id] = h
+                if prev.get(u.id) == h:        # unchanged content (same space) -> skip the embed/upsert
+                    unchanged += 1
+                else:
+                    changed.append(u)
+            if changed:
+                metadata = []
+                for u in changed:
+                    item = u.metadata(self.repo_id, scope, commit_sha=commit, branch=branch)
+                    item["service"] = self.service or u.service or ""
+                    metadata.append(item)
+                col.upsert_texts([u.text for u in changed], [u.id for u in changed],
+                                 metadata)
+                written += len(changed)
+
+        col.compact()
+        col.save()
+        self._save_hashes(new_hashes)
+        self._write_meta(commit=commit, branch=branch)
+        return {"indexed": written, "unchanged": unchanged, "removed": removed, "scopes": len(by_scope),
+                "files": stats.files, "skipped": stats.skipped, "languages": list(resolved_languages)}
+
+    def _hash_path(self) -> str:
+        return os.path.join(self.db_path, HASH_FILENAME)
+
+    def _load_hashes(self) -> dict:
+        try:
+            with open(self._hash_path()) as f:
+                return json.load(f)
+        except (OSError, ValueError):
+            return {}
+
+    def _save_hashes(self, m: dict) -> None:
+        os.makedirs(self.db_path, exist_ok=True)
+        tmp = self._hash_path() + ".tmp"
+        with open(tmp, "w") as f:
+            json.dump(m, f)
+        os.replace(tmp, self._hash_path())
+
+    def current_commit(self) -> str:
+        return _git(self.repo_path, "rev-parse", "HEAD") or ""
+
+    def is_stale(self) -> bool:
+        """True when the repo HEAD moved since the last index, so recall would serve OLD symbols.
+        Conservative: only reports stale when BOTH commits are known and differ (a non-git repo or a
+        missing meta never forces a re-index)."""
+        indexed = (self._read_meta().get("commit_sha") or "")
+        cur = self.current_commit()
+        return bool(cur) and bool(indexed) and cur != indexed
+
+    def embedder_changed(self) -> bool:
+        """True when the active embedder's vector SPACE differs from the indexed one (different embedding
+        model / dim / normalization — e.g. the fix provider switched and auto-selected a new embed model).
+        Vectors across spaces are incompatible, so the index must be rebuilt, not reopened."""
+        prev = self._read_meta().get("space_id")
+        return bool(prev) and prev != self._embedder.profile.space_id()
+
+    def ensure_fresh(self, *, languages: tuple[str, ...] | None = None) -> dict | None:
+        """Keep recall honest before use. Rebuilds the index when the EMBEDDER changed (incompatible
+        vector space → must re-embed everything), else refreshes when the repo HEAD moved (add new,
+        re-embed changed, prune vanished). No-op (returns ``None``) when both are unchanged. This is what
+        makes memory robust across LLM/embedder switches — switching the fix LLM with a pinned embedder is
+        a no-op; switching the embedder transparently re-indexes instead of erroring on a space mismatch."""
+        if self.embedder_changed():
+            self.reset()                       # drop the incompatible-space index, then rebuild fresh
+            return self.index(languages=languages, refresh=True)
+        if self.is_stale():
+            return self.index(languages=languages, refresh=True)
+        return None
+
+    def recall(self, query: str, *, service: str | None = None, language: str | None = None,
+               path: str | None = None, top_k: int = 40, final_k: int = 8,
+               target_recall: float = 0.98) -> dict:
+        """Partition-aware retrieve -> consumer rerank -> final-k symbols with provenance + explain."""
+        col = self._collection()
+        active_service = service if service is not None else self.service
+        post_filters: dict[str, str] = {}
+        search_where: dict[str, str] | None = None
+        if language is not None:
+            search_where = {"scope_key": self._scope_key(self.repo_id, active_service or "", language)}
+            if path is not None:
+                post_filters["path"] = path
+        elif path is not None:
+            search_where = {"path": path}
+            if active_service is not None:
+                post_filters["service"] = active_service
+        elif active_service is not None:
+            search_where = {"service": active_service}
+        res = col.search_text(query, top_k=top_k, where=search_where, explain=True,
+                              target_recall=target_recall, with_metadata=True)
+        md = res.get("metadata") or [{}] * len(res["ids"])   # provenance returned WITH the hit (Telys)
+        meta_by_id = dict(zip(res["ids"], md))
+        # Lossless query-aware soft rerank over the FULL candidate set, then the (future) rerank seam,
+        # then final-k — so a source symbol buried under tests can surface without dropping any coverage.
+        raw_cands = [
+            (unit_id, score)
+            for unit_id, score in zip(res["ids"], res["scores"])
+            if _metadata_matches(meta_by_id.get(unit_id) or {}, post_filters)
+        ]
+        cands = _soft_test_downweight(query, raw_cands, meta_by_id)
+        cands = _semantic_rerank(query, cands, self)    # DEFAULT: lexical+WordLlama blend (alpha 0.7); falls back
+        ranked = rerank(query, cands)[:final_k]          # to lexical if WordLlama absent. CODNA_MEMORY_RERANK to tune.
+        symbols = []
+        for i, s in ranked:
+            m = meta_by_id.get(i) or {}
+            symbols.append({"id": i, "score": s, "symbol_type": m.get("symbol_type"),
+                            "path": m.get("path") or _provenance(i)["path"]})
+        return {"symbols": symbols, "explain": res.get("explain", {}),
+                "candidate_count": len(raw_cands), "final_k": len(symbols)}
+
+    def status(self) -> dict:
+        meta = self._read_meta()
+        if COLLECTION not in self._eng.collections():
+            return {"engine": "telys", "collection": COLLECTION, "indexed": False,
+                    "db_path": self.db_path, "repo_id": self.repo_id, "license": self.license,
+                    "kernel": self.kernel}
+        s = self._collection().stats()
+        return {
+            "engine": "telys",
+            "collection": COLLECTION,
+            "indexed": True,
+            "partition_key": s.get("key_name", "scope_key"),
+            "partitions": s.get("partitions"),
+            "documents": s.get("external_ids"),
+            "delta_rows": s.get("delta_rows"),
+            "embedding_space": s.get("embedding_space"),
+            "last_indexed_commit": meta.get("commit_sha"),
+            "last_indexed_at": meta.get("indexed_at"),
+            "fallback_rate": None,                         # no query log yet (Telys exports no metrics)
+            "db_path": self.db_path,
+            "repo_id": self.repo_id,
+            "license": self.license,
+            "kernel": self.kernel,
+        }
+
+    def compact(self) -> dict:
+        col = self._collection()
+        col.compact()
+        col.save()
+        return {"compacted": True, "documents": col.stats().get("external_ids")}
+
+    def reset(self) -> dict:
+        """Drop the local memory for this repo (collection + meta + incremental hash manifest). Safe no-op
+        if nothing is indexed.
+
+        The hash manifest MUST be cleared too: it drives the incremental re-index (#56), so a stale manifest
+        after dropping the collection makes the next index() treat every symbol as "unchanged" and skip
+        embedding it — leaving an empty collection (recall returns nothing). reset() must be a full reset."""
+        self._col = None
+        existed = os.path.isdir(os.path.join(self.db_path, COLLECTION))
+        shutil.rmtree(os.path.join(self.db_path, COLLECTION), ignore_errors=True)
+        for fn in (META_FILENAME, HASH_FILENAME, HASH_FILENAME + ".tmp"):
+            p = os.path.join(self.db_path, fn)
+            if os.path.exists(p):
+                os.remove(p)
+        return {"reset": True, "existed": existed, "db_path": self.db_path}
+
+    # ── side metadata (commit/branch/symbol-types) — Codna-side, separate from telys' collection.json ─
+    def _meta_path(self) -> str:
+        return os.path.join(self.db_path, META_FILENAME)
+
+    def _read_meta(self) -> dict:
+        try:
+            with open(self._meta_path()) as f:
+                return json.load(f)
+        except (OSError, ValueError):
+            return {}
+
+    def _write_meta(self, *, commit: str, branch: str) -> None:
+        os.makedirs(self.db_path, exist_ok=True)
+        # Just the index-run record for `status`. NO per-id map: provenance now comes from
+        # search(with_metadata=True) and the vanished-diff from col.ids(where=scope) (Telys 0.1.0-alpha).
+        payload = {
+            "commit_sha": commit,
+            "branch": branch,
+            "indexed_at": int(time.time()),
+            "space_id": self._embedder.profile.space_id(),
+        }
+        with open(self._meta_path(), "w") as f:
+            json.dump(payload, f)