data-olympus 0.3.0__py3-none-any.whl

data_olympus/embeddings.py

@@ -0,0 +1,317 @@
+"""Optional local-embedding hybrid ranking (issue #42).
+
+Real paraphrase / synonymy handling via LOCAL embeddings, blended with BM25. No
+external API is ever called at query time; a small ONNX MiniLM-class model runs
+in-process. The feature is OFF by default and, when off, this module is the ONLY
+place that would touch the embedding libraries, and it imports them LAZILY
+(inside :func:`_load_text_embedding_cls`), so with the ``embeddings`` extra NOT
+installed and ``KB_EMBEDDINGS_MODE`` unset the default product is byte-for-byte
+unchanged and never imports ``fastembed`` / ``onnxruntime``.
+
+Design (mirrors ``trigram.py`` / ``cooccurrence.py``):
+
+- ``EMBEDDINGS_SCHEMA``: the ``doc_vectors`` table DDL, appended to the index
+  schema so it is created (and, via the tmp-DB build + ``os.replace``, swapped)
+  atomically with the primary FTS table. The table is created unconditionally
+  (an empty table costs nothing) but only POPULATED when the feature is enabled,
+  so a default build needs no embedding dependency.
+- ``embeddings_enabled`` / ``embeddings_config``: env config. ``KB_EMBEDDINGS_MODE``
+  gates the whole feature (default off); ``KB_EMBEDDINGS_MODEL`` and
+  ``KB_EMBEDDINGS_WEIGHT`` tune the model and blend weight.
+- ``build_embedder``: construct the local embedder, raising a LOUD, actionable
+  :class:`EmbeddingsUnavailableError` when the dep/model is missing rather than
+  silently falling back (which would hide misconfiguration).
+- ``serialize_vector`` / ``deserialize_vector`` / ``cosine``: pure-Python vector
+  storage + similarity, so the query-time blend needs no numpy.
+- ``make_hybrid_reranker``: blend NORMALISED bm25 with cosine over candidate
+  hits' stored vectors and re-sort. Composed as an INNER reranker under the
+  id/tag short-circuit in ``server.build_app`` so an exact id/tag still wins.
+
+Ranking discipline: the hybrid reranker only RE-ORDERS the existing candidate
+pool; it never drops a hit (a doc with no stored vector keeps its bm25
+contribution and a neutral cosine of 0). Because it is composed as ``inner``
+under the id/tag short-circuit, an exact-id or exact-tag query is unaffected.
+"""
+from __future__ import annotations
+
+import array
+import math
+import os
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from data_olympus.index import SearchHit
+
+
+# --- config ------------------------------------------------------------------
+
+# A small, local, ONNX MiniLM-class model. bge-small-en-v1.5 is 384-dim, emits
+# L2-normalised vectors, and ships as a quantised ONNX file (~33 MB) that
+# fastembed fetches once and caches; it needs no torch. A deployment can point
+# ``KB_EMBEDDINGS_MODEL`` at any fastembed-supported model.
+DEFAULT_MODEL_NAME = "BAAI/bge-small-en-v1.5"
+
+# Blend weight in [0, 1]: the fraction of the final score contributed by cosine
+# similarity; ``1 - weight`` is contributed by normalised bm25. 0.0 == pure
+# lexical (bm25) ordering, 1.0 == pure semantic (cosine). The default leans on
+# bm25 (the tuned lexical signal) while letting cosine rescue paraphrase misses.
+DEFAULT_WEIGHT = 0.35
+
+
+@dataclass(frozen=True, slots=True)
+class EmbeddingsConfig:
+    """Resolved embeddings configuration (model + blend weight)."""
+
+    model_name: str
+    weight: float
+
+
+class EmbeddingsUnavailableError(RuntimeError):
+    """Raised when embeddings are ENABLED but the dep/model cannot be loaded.
+
+    Carries an actionable message so a misconfigured deployment fails loudly at
+    startup instead of silently shipping default (lexical-only) ranking.
+    """
+
+
+def embeddings_enabled() -> bool:
+    """Whether local-embedding hybrid ranking is active. Default OFF.
+
+    ``KB_EMBEDDINGS_MODE=on`` (case-insensitive) enables it; any other value
+    (including unset) leaves it off so the zero-dependency lexical product is
+    unchanged and the embedding libraries are never imported.
+    """
+    return os.getenv("KB_EMBEDDINGS_MODE", "off").strip().lower() == "on"
+
+
+def embeddings_config() -> EmbeddingsConfig:
+    """Resolve the embeddings config from env, applying defaults.
+
+    ``KB_EMBEDDINGS_WEIGHT`` must parse to a float in [0, 1]; a malformed or
+    out-of-range value raises ``ValueError`` rather than silently clamping, so a
+    misconfigured blend fails loudly (matching ``KB_STATUS_WEIGHTS`` semantics).
+    """
+    model_name = os.getenv("KB_EMBEDDINGS_MODEL", "").strip() or DEFAULT_MODEL_NAME
+    raw_weight = os.getenv("KB_EMBEDDINGS_WEIGHT", "").strip()
+    if not raw_weight:
+        weight = DEFAULT_WEIGHT
+    else:
+        try:
+            weight = float(raw_weight)
+        except ValueError as e:
+            raise ValueError(
+                f"KB_EMBEDDINGS_WEIGHT must be a float in [0, 1]; got {raw_weight!r}"
+            ) from e
+        if not 0.0 <= weight <= 1.0:
+            raise ValueError(
+                f"KB_EMBEDDINGS_WEIGHT must be in [0, 1]; got {weight}"
+            )
+    return EmbeddingsConfig(model_name=model_name, weight=weight)
+
+
+# --- vector storage + similarity (pure Python, no numpy) ---------------------
+
+
+def serialize_vector(vec: list[float]) -> bytes:
+    """Pack a float vector into a compact little-endian float32 blob.
+
+    float32 halves the stored size versus float64 at a precision loss well below
+    the ranking signal. ``array`` keeps this dependency-free; the blob round-trips
+    through :func:`deserialize_vector`.
+    """
+    return array.array("f", vec).tobytes()
+
+
+def deserialize_vector(blob: bytes) -> list[float]:
+    """Unpack a float32 blob written by :func:`serialize_vector`."""
+    a = array.array("f")
+    a.frombytes(blob)
+    return list(a)
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+    """Cosine similarity of two equal-length vectors, in [-1, 1].
+
+    A zero-norm vector has no direction, so its similarity is defined as 0 rather
+    than dividing by zero. A length mismatch is a programming error and raises.
+    """
+    if len(a) != len(b):
+        raise ValueError(f"vector length mismatch: {len(a)} != {len(b)}")
+    dot = 0.0
+    na = 0.0
+    nb = 0.0
+    for x, y in zip(a, b, strict=True):
+        dot += x * y
+        na += x * x
+        nb += y * y
+    if na == 0.0 or nb == 0.0:
+        return 0.0
+    return dot / (math.sqrt(na) * math.sqrt(nb))
+
+
+# --- embedder (lazy fastembed import) ----------------------------------------
+
+
+def _load_text_embedding_cls() -> Any:
+    """Import and return ``fastembed.TextEmbedding``.
+
+    Isolated so it can be monkeypatched in tests and so the import stays LAZY:
+    it runs only from :func:`build_embedder`, which is only reached when the
+    feature is enabled. With the feature off this line is never executed and
+    ``fastembed`` is never imported.
+    """
+    from fastembed import TextEmbedding
+
+    return TextEmbedding
+
+
+class Embedder:
+    """Thin wrapper over a local fastembed model.
+
+    Holds the loaded model and exposes ``embed_one`` (query) and ``embed_many``
+    (build) returning plain ``list[float]`` so the rest of the codebase stays
+    numpy-free. Construction is via :func:`build_embedder`.
+    """
+
+    def __init__(self, model: Any, *, model_name: str) -> None:
+        self._model = model
+        self.model_name = model_name
+
+    def embed_one(self, text: str) -> list[float]:
+        """Embed a single text into a plain float list."""
+        for vec in self._model.embed([text]):
+            return [float(x) for x in vec]
+        return []
+
+    def embed_many(self, texts: list[str]) -> list[list[float]]:
+        """Embed a batch, preserving input order."""
+        return [[float(x) for x in vec] for vec in self._model.embed(texts)]
+
+
+def build_embedder(config: EmbeddingsConfig) -> Embedder:
+    """Load the local embedding model, or fail loudly.
+
+    Raises :class:`EmbeddingsUnavailableError` with an actionable message when
+    the ``embeddings`` extra is not installed or the model cannot be loaded
+    (e.g. it could not be fetched to the local cache). The caller (startup path)
+    surfaces this so a misconfigured deployment fails visibly rather than
+    silently reverting to lexical-only ranking.
+    """
+    try:
+        text_embedding_cls = _load_text_embedding_cls()
+    except ImportError as e:
+        raise EmbeddingsUnavailableError(
+            "KB_EMBEDDINGS_MODE is on but the embeddings dependency is not "
+            "installed. Install the extra: `uv sync --extra embeddings` (or "
+            "`pip install 'data-olympus[embeddings]'`), or set KB_EMBEDDINGS_MODE=off "
+            f"to run lexical-only. Underlying import error: {e}"
+        ) from e
+    try:
+        model = text_embedding_cls(model_name=config.model_name)
+    except Exception as e:  # noqa: BLE001 - any load failure must fail loudly
+        raise EmbeddingsUnavailableError(
+            f"KB_EMBEDDINGS_MODE is on but the embedding model "
+            f"{config.model_name!r} could not be loaded (it is fetched once and "
+            f"cached; a first-run needs the model available). Set a cached "
+            f"KB_EMBEDDINGS_MODEL or KB_EMBEDDINGS_MODE=off. Underlying error: {e}"
+        ) from e
+    return Embedder(model, model_name=config.model_name)
+
+
+# --- SQLite persistence (built inside Index.build, read at query time) -------
+
+# One row per doc: the id (FK to docs.id) and its float32 vector blob. Created
+# unconditionally in the schema (empty when the feature is off) so a default
+# build needs no embedding dep; populated only when embeddings are enabled.
+EMBEDDINGS_SCHEMA = """
+CREATE TABLE IF NOT EXISTS doc_vectors (
+    id TEXT PRIMARY KEY,
+    vector BLOB NOT NULL
+);
+"""
+
+
+# --- hybrid reranker ---------------------------------------------------------
+
+
+def _normalise_bm25(hits: list[SearchHit]) -> dict[str, float]:
+    """Map each hit id to a bm25 GOODNESS in [0, 1] (1 == best in this pool).
+
+    search() orders bm25 ASCENDING (lower is better), so we invert: the minimum
+    (best) score maps to 1.0 and the maximum (worst) to 0.0. When all scores are
+    equal (single hit, or a synthetic neutral pool) every hit gets 1.0, so the
+    blend is decided purely by cosine, and there is no divide-by-zero.
+    """
+    if not hits:
+        return {}
+    scores = [h.score for h in hits]
+    lo = min(scores)
+    hi = max(scores)
+    spread = hi - lo
+    if spread == 0.0:
+        return {h.id: 1.0 for h in hits}
+    # goodness = 1 when score == lo (best), 0 when score == hi (worst).
+    return {h.id: (hi - h.score) / spread for h in hits}
+
+
+def make_hybrid_reranker(
+    *,
+    embed_query: Callable[[str], list[float] | None],
+    get_vector: Callable[[str], list[float] | None],
+    weight: float,
+) -> Callable[[str, list[SearchHit]], list[SearchHit]]:
+    """Build a reranker that blends normalised bm25 with query-doc cosine.
+
+    ``embed_query(query)`` returns the query vector (or ``None`` if it cannot be
+    embedded); ``get_vector(id)`` returns a stored doc vector (or ``None`` when
+    the doc has no vector). ``weight`` in [0, 1] is the cosine fraction of the
+    blended score (``1 - weight`` is bm25).
+
+    The blended score is::
+
+        blended = (1 - weight) * bm25_goodness + weight * cosine_component
+
+    where ``bm25_goodness`` is in [0, 1] (higher = better within this pool) and
+    ``cosine_component`` maps cosine [-1, 1] to [0, 1]. Hits are re-sorted by
+    blended score DESCENDING (higher = better) and the result is re-scored onto
+    the search()-native ascending-bm25 convention (negated blended score) so a
+    later stage that sorts ascending keeps the same order.
+
+    Discipline: no hit is dropped. A doc with no stored vector, or a query that
+    cannot be embedded, contributes a neutral cosine (the reranker degrades to
+    bm25 ordering for that case). The re-sort is stable for equal blended scores.
+    """
+
+    def reranker(query: str, hits: list[SearchHit]) -> list[SearchHit]:
+        if not hits:
+            return hits
+        from dataclasses import replace
+
+        qvec = embed_query(query)
+        bm25_goodness = _normalise_bm25(hits)
+
+        def blended(h: SearchHit) -> float:
+            good = bm25_goodness.get(h.id, 0.0)
+            cos_component = 0.0
+            if qvec is not None:
+                dvec = get_vector(h.id)
+                if dvec is not None and len(dvec) == len(qvec):
+                    # Map cosine [-1, 1] -> [0, 1].
+                    cos_component = (cosine(qvec, dvec) + 1.0) / 2.0
+            return (1.0 - weight) * good + weight * cos_component
+
+        scored = [(blended(h), i, h) for i, h in enumerate(hits)]
+        # Sort by rank_class FIRST (primaries before backfill), then blended DESC;
+        # ties keep incoming order via the enumerate index. The rank-class outer
+        # key means the cosine blend can re-order WITHIN a class but never lift a
+        # backfill (expansion/trigram) hit above a primary (finding (d)): the
+        # blend re-normalises scores, so a score-only floor would not hold.
+        scored.sort(key=lambda t: (t[2].rank_class, -t[0], t[1]))
+        # Re-score onto the ascending-bm25 convention (lower == better) so a
+        # downstream ascending sort agrees with this order: negate blended.
+        return [replace(h, score=-b) for b, _i, h in scored]
+
+    return reranker

data_olympus/enforce_policy.py

@@ -0,0 +1,297 @@
+"""Policy core for enforcement: the heuristic intent classifier and the
+in-memory consultation ledger. Pure and dependency-free so it is unit-testable
+without a FastMCP server."""
+from __future__ import annotations
+
+import contextlib
+import fnmatch
+import json
+import logging
+import os
+import re
+import tempfile
+import threading
+from dataclasses import dataclass, field
+
+# Keyword signals that a user prompt is a governed code/architectural decision.
+GOVERNED_KEYWORDS: tuple[str, ...] = (
+    "library", "dependency", "dependencies", "framework", "package",
+    "pattern", "migration", "migrate", "refactor", "architecture",
+    "api design", "endpoint", "schema", "auth", "authorization",
+    "authentication", "rls", "secret", "convention", "standard",
+)
+
+# Path globs that mark a pending file action as a governed decision. Deliberately
+# narrow (dependency manifests, migrations, schema, container/build config) so
+# that ordinary source edits flow without a consult; the prompt-level classifier
+# carries the broad net.
+GOVERNED_PATH_GLOBS: tuple[str, ...] = (
+    "pyproject.toml", "*/pyproject.toml",
+    "package.json", "*/package.json",
+    "*/requirements*.txt", "requirements*.txt",
+    "*/go.mod", "go.mod", "*/Cargo.toml", "Cargo.toml", "*/pom.xml",
+    "*/migrations/*", "*/migration/*",
+    "*/schema/*", "*/schema.sql", "*.sql",
+    "Dockerfile", "*/Dockerfile", "*/docker-compose*.yml", "docker-compose*.yml",
+)
+
+# Command fragments (matched against action_diff) that indicate a governed
+# dependency/install action, so Bash/shell tool actions can be classified.
+GOVERNED_COMMAND_PATTERNS: tuple[str, ...] = (
+    "pip install", "pip3 install", "uv add", "uv pip install", "poetry add",
+    "npm install", "npm i ", "yarn add", "pnpm add",
+    "apt install", "apt-get install", "brew install",
+    "go get", "cargo add", "gem install", "bundle add",
+)
+
+
+@dataclass(frozen=True)
+class ClassifyResult:
+    """Outcome of a classification: governed or not, plus the matched signals."""
+
+    is_governed_decision: bool
+    signals: list[str] = field(default_factory=list)
+
+
+class IntentClassifier:
+    """Heuristic classifier. Pluggable: a future LLM-backed classifier can
+    implement the same ``classify`` signature without touching callers."""
+
+    def __init__(
+        self,
+        *,
+        keywords: tuple[str, ...] = GOVERNED_KEYWORDS,
+        path_globs: tuple[str, ...] = GOVERNED_PATH_GLOBS,
+        command_patterns: tuple[str, ...] = GOVERNED_COMMAND_PATTERNS,
+    ) -> None:
+        self._keywords = tuple(k.lower() for k in keywords)
+        self._path_globs = tuple(path_globs)
+        self._command_patterns = tuple(p.lower() for p in command_patterns)
+        # Pre-compile a word-boundary regex per keyword so "authored" does not
+        # match "auth". \b around each keyword; keywords with spaces still work.
+        self._keyword_res = tuple(
+            (kw, re.compile(rf"\b{re.escape(kw)}\b")) for kw in self._keywords
+        )
+
+    def classify(
+        self,
+        *,
+        intent: str = "",
+        action_path: str | None = None,
+        action_diff: str = "",
+    ) -> ClassifyResult:
+        signals: list[str] = []
+        text = f"{intent} {action_diff}".lower()
+        for kw, rx in self._keyword_res:
+            if rx.search(text):
+                signals.append(f"keyword:{kw}")
+        diff_lower = action_diff.lower()
+        for pat in self._command_patterns:
+            if pat in diff_lower:
+                signals.append(f"command:{pat.strip()}")
+        if action_path:
+            p = action_path.replace("\\", "/")
+            base = p.rsplit("/", 1)[-1]
+            for glob in self._path_globs:
+                if fnmatch.fnmatch(p, glob) or fnmatch.fnmatch(base, glob):
+                    signals.append(f"path:{glob}")
+                    break
+        return ClassifyResult(is_governed_decision=bool(signals), signals=signals)
+
+
+# Consultation trigger provenance. "explicit" is a deliberate agent call to
+# kb_consult (the MCP tool default, and any old client that sends no trigger).
+# "prompt_hook" is an installer-driven UserPromptSubmit/BeforeAgent auto-consult:
+# recorded for audit/compliance but NOT sufficient to clear the gate, because it
+# fires on every user turn and would otherwise keep the ledger perpetually fresh.
+EXPLICIT_TRIGGER = "explicit"
+PROMPT_HOOK_TRIGGER = "prompt_hook"
+
+
+@dataclass
+class LedgerEntry:
+    """A recorded consultation for a (session, workspace) pair.
+
+    ``consulted_at`` is the last touch of ANY trigger (used for retention/TTL of
+    the row itself and audit). ``explicit_at`` is the last EXPLICIT consult, or
+    None if only prompt-hook auto-consults have been recorded. The gate clears on
+    ``explicit_at`` freshness so a prompt-hook auto-consult (which fires every
+    turn) can never satisfy the gate, while an explicit consult is not silently
+    downgraded by a later prompt-hook consult on the same (session, workspace)."""
+
+    consulted_at: float
+    rule_ids: list[str]
+    explicit_at: float | None = None
+
+
+log = logging.getLogger("data_olympus")
+
+
+class ConsultationLedger:
+    """Records which (session, workspace) pairs consulted and when.
+
+    With no ``path`` it is purely in-memory (the slice-1 behavior). With a
+    ``path`` it loads an existing JSON file on construction and rewrites it on
+    every ``record`` so consultations survive a server restart. A corrupt or
+    unreadable file degrades to empty with a logged warning and never crashes."""
+
+    def __init__(
+        self,
+        path: str | None = None,
+        *,
+        retention_sec: float = 3600.0,
+        max_entries: int = 50_000,
+    ) -> None:
+        self._path = path
+        self._entries: dict[tuple[str, str], LedgerEntry] = {}
+        # An entry is a TTL freshness cache: once it is older than the consult
+        # TTL it can never be fresh again, so we evict it on the next record().
+        # ``retention_sec`` should be >= the ttl_sec passed to is_fresh (the
+        # server threads config.consult_ttl_sec in) so a still-fresh entry is
+        # never dropped. ``max_entries`` is a hard belt-and-suspenders cap that
+        # bounds memory/disk even under a flood of unique sessions inside one
+        # retention window. Together they stop _entries growing without bound and
+        # keep the per-record file rewrite O(active window) instead of O(all
+        # sessions ever seen).
+        self._retention_sec = retention_sec
+        self._max_entries = max_entries
+        # record() mutates _entries and rewrites the whole file; once consult
+        # handlers are offloaded to the anyio threadpool these can run
+        # concurrently. Without this lock, _save()'s iteration over _entries can
+        # race a concurrent insert (RuntimeError / dropped entries). Public
+        # methods take the lock; _save()/_evict() do not (called only while held).
+        self._lock = threading.Lock()
+        if path:
+            self._load()
+            # A ledger persisted by the previous unbounded implementation can be
+            # arbitrarily large; cap it on load so an oversized /state/ledger.json
+            # is not held in memory until the first record() prunes it. TTL
+            # eviction still runs on the first record() (it needs a caller "now").
+            self._enforce_cap()
+
+    def _load(self) -> None:
+        if not self._path or not os.path.exists(self._path):
+            return
+        try:
+            with open(self._path, encoding="utf-8") as f:
+                rows = json.load(f)
+            for row in rows:
+                key = (row["session_id"], row["workspace"])
+                # Back-compat: a ledger persisted before the trigger split has no
+                # explicit_at. Treat those legacy rows as explicit (they were all
+                # gate-clearing under the old policy), so a server upgrade does not
+                # spuriously re-block a session that already consulted.
+                consulted_at = float(row["consulted_at"])
+                explicit_at = row.get("explicit_at", consulted_at)
+                self._entries[key] = LedgerEntry(
+                    consulted_at=consulted_at,
+                    rule_ids=list(row.get("rule_ids", [])),
+                    explicit_at=None if explicit_at is None else float(explicit_at),
+                )
+        except Exception as exc:  # noqa: BLE001 - corrupt file -> empty, never crash
+            log.warning("consultation ledger at %s unreadable, starting empty: %s",
+                        self._path, exc)
+            self._entries = {}
+
+    def _save(self) -> None:
+        if not self._path:
+            return
+        rows = [
+            {"session_id": s, "workspace": w,
+             "consulted_at": e.consulted_at, "rule_ids": e.rule_ids,
+             "explicit_at": e.explicit_at}
+            for (s, w), e in self._entries.items()
+        ]
+        d = os.path.dirname(self._path) or "."
+        os.makedirs(d, exist_ok=True)
+        # Atomic write: serialize to a temp file in the same directory, then
+        # os.replace() over the target so a crash/full-disk mid-write cannot
+        # truncate or corrupt the existing ledger.
+        fd, tmp = tempfile.mkstemp(dir=d, prefix=".ledger-", suffix=".tmp")
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as f:
+                json.dump(rows, f)
+            os.replace(tmp, self._path)
+        except Exception:
+            with contextlib.suppress(OSError):
+                os.unlink(tmp)
+            raise
+
+    def _evict(self, now: float) -> None:
+        """Drop entries that can no longer be fresh, then enforce the hard cap.
+
+        Caller must hold ``self._lock``. Reassigns ``self._entries`` to a pruned
+        dict; O(n) but n is exactly what this bounds.
+        """
+        cutoff = now - self._retention_sec
+        self._entries = {
+            key: e for key, e in self._entries.items() if e.consulted_at >= cutoff
+        }
+        self._enforce_cap()
+
+    def _enforce_cap(self) -> None:
+        """Bound _entries to the newest ``max_entries`` by consulted_at. Clock-free
+        so it can run on load (before any caller "now" is available). Caller holds
+        the lock, except the single-threaded construction-time call."""
+        if len(self._entries) > self._max_entries:
+            newest = sorted(
+                self._entries.items(), key=lambda kv: kv[1].consulted_at, reverse=True
+            )[: self._max_entries]
+            self._entries = dict(newest)
+
+    def record(
+        self,
+        *,
+        session_id: str,
+        workspace: str,
+        rule_ids: list[str],
+        now: float,
+        trigger: str = EXPLICIT_TRIGGER,
+    ) -> None:
+        """Record a consultation. ``trigger`` is EXPLICIT_TRIGGER (a deliberate
+        agent consult that clears the gate) or PROMPT_HOOK_TRIGGER (an installer
+        auto-consult that is audited but never clears the gate).
+
+        A prompt-hook record refreshes ``consulted_at`` (row liveness/audit) but
+        carries forward any existing ``explicit_at`` so it cannot downgrade a
+        still-fresh explicit consult into a non-clearing one."""
+        with self._lock:
+            prior = self._entries.get((session_id, workspace))
+            if trigger == EXPLICIT_TRIGGER:
+                explicit_at: float | None = now
+            else:
+                explicit_at = prior.explicit_at if prior is not None else None
+            self._entries[(session_id, workspace)] = LedgerEntry(
+                consulted_at=now, rule_ids=list(rule_ids), explicit_at=explicit_at
+            )
+            self._evict(now)
+            self._save()
+
+    def is_fresh(
+        self,
+        *,
+        session_id: str,
+        workspace: str,
+        now: float,
+        ttl_sec: float,
+        require_explicit: bool = True,
+    ) -> bool:
+        """True when a fresh consult is on record for (session, workspace).
+
+        With ``require_explicit`` (the gate's default) only an explicit consult
+        within ``ttl_sec`` counts: a prompt-hook auto-consult never clears the
+        gate. With ``require_explicit=False`` any consult (explicit or prompt
+        hook) within the TTL counts (used where the mere fact of a recent consult,
+        not its provenance, matters)."""
+        with self._lock:
+            entry = self._entries.get((session_id, workspace))
+        if entry is None:
+            return False
+        ts = entry.explicit_at if require_explicit else entry.consulted_at
+        if ts is None:
+            return False
+        return (now - ts) <= ttl_sec
+
+    def get(self, *, session_id: str, workspace: str) -> LedgerEntry | None:
+        with self._lock:
+            return self._entries.get((session_id, workspace))

data_olympus/format/__init__.py

@@ -0,0 +1,16 @@
+"""Public API for the data-olympus format core."""
+
+from .document import Document
+from .frontmatter import parse_frontmatter
+from .lint import discover_bundle_files, lint_bundle, lint_files
+from .validate import Finding, validate_document
+
+__all__ = [
+    "Document",
+    "Finding",
+    "parse_frontmatter",
+    "validate_document",
+    "lint_bundle",
+    "discover_bundle_files",
+    "lint_files",
+]

data_olympus/format/document.py

@@ -0,0 +1,39 @@
+"""The Document model: a parsed concept file (frontmatter + body)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .frontmatter import parse_frontmatter
+
+
+@dataclass(frozen=True)
+class Document:
+    path: Path
+    frontmatter: dict[str, Any]
+    body: str
+
+    @classmethod
+    def load(cls, path: str | Path) -> Document:
+        """Load a document from disk. Callers must not mutate `frontmatter` in place."""
+        p = Path(path)
+        fm, body = parse_frontmatter(p.read_text(encoding="utf-8"))
+        return cls(path=p, frontmatter=fm, body=body)
+
+    @property
+    def id(self) -> str | None:
+        return self.frontmatter.get("id")
+
+    @property
+    def type(self) -> str | None:
+        return self.frontmatter.get("type")
+
+    @property
+    def status(self) -> str | None:
+        return self.frontmatter.get("status")
+
+    @property
+    def tier(self) -> str | None:
+        return self.frontmatter.get("tier")