veccore 0.2.0__py3-none-any.whl

veccore/__init__.py

@@ -0,0 +1,56 @@
+"""veccore — the ports-and-adapters vector substrate for the estate (keelworks L2).
+
+Generic MECHANISMS + clean SEAMS for `embed → index → search → hydrate`. Each consumer builds its domain
+POLICY on top; nothing consumer-specific lives here, and no consumer reinvents embed/index/idempotency.
+
+Quick start (a consumer's ENTIRE veccore-side adoption is: register a collection + hydrate from its own SoR):
+
+    from veccore import (VecCore, CollectionRegistry, CollectionSpec, AuditEnvelope,
+                         MockEmbedder, SqliteNumpyStore)
+
+    store = SqliteNumpyStore(path="/tmp/mine.sqlite")
+    core = VecCore(registry=CollectionRegistry(),
+                   stores={"sqlite_numpy": store},
+                   embedders={"mock": MockEmbedder()})
+    core.register(CollectionSpec(name="notes", backend="sqlite_numpy", embedder="mock", dim=64))
+
+    audit = AuditEnvelope(who="my_tool", why="index notes for reuse search")
+    core.embed_upsert("notes", [{"id": "n1", "text": "vector search over notes", "payload": {"src": "n1"}}], audit)
+
+    hits = core.search("notes", "nearest ideas", k=5, audit=AuditEnvelope(who="my_tool", why="find related"))
+    # each hit is {id, score, payload}; hydrate the full record from your own SoR by hit.id
+"""
+from .audit import AuditEnvelope, JsonlObserver, NullObserver, ObserverPort
+from .batch_ingest import BatchIngest, BatchReport
+from .core import Estimate, IngestResult, VecCore
+from .evaluate import bakeoff, recall_at_k, reciprocal_rank, score_queries
+from .embedders import CompositeEmbedder, LocalEmbedder, MockEmbedder, OpenAIEmbedder, make_embedder
+from .errors import (DimMismatch, GateRefused, ModelMismatch, NotSupported, OversizeRefused, Unavailable,
+                     VeccoreError)
+from .ports import EmbedderPort, GatePort, Hit, Item, NullGate, Point, VectorStorePort
+from .registry import CollectionRegistry, CollectionSpec
+from .adapters import QdrantStore, SqliteNumpyStore
+
+__version__ = "0.2.0"
+
+__all__ = [
+    # orchestrator + records
+    "VecCore", "IngestResult", "Estimate",
+    "CollectionRegistry", "CollectionSpec",
+    "AuditEnvelope", "ObserverPort", "NullObserver", "JsonlObserver",
+    "Item", "Point", "Hit",
+    # ports
+    "EmbedderPort", "VectorStorePort", "GatePort", "NullGate",
+    # embedders
+    "MockEmbedder", "OpenAIEmbedder", "LocalEmbedder", "CompositeEmbedder", "make_embedder",
+    # backends
+    "SqliteNumpyStore", "QdrantStore",
+    # batch seam
+    "BatchIngest", "BatchReport",
+    # measurement seam (choose/fuse embedders on evidence)
+    "bakeoff", "score_queries", "recall_at_k", "reciprocal_rank",
+    # errors
+    "VeccoreError", "Unavailable", "NotSupported", "DimMismatch", "ModelMismatch", "OversizeRefused",
+    "GateRefused",
+    "__version__",
+]

veccore/adapters/__init__.py

@@ -0,0 +1,6 @@
+"""Backend adapters: `sqlite_numpy` (in-process) and `qdrant` (server). Future faiss/pgvector slot in
+here by implementing VectorStorePort. Backend IDs are `sqlite_numpy` / `qdrant`, never `db`."""
+from .qdrant_store import QdrantStore
+from .sqlite_numpy_store import SqliteNumpyStore
+
+__all__ = ["SqliteNumpyStore", "QdrantStore"]

veccore/adapters/qdrant_store.py

@@ -0,0 +1,245 @@
+"""QdrantStore — the server backend (backend id "qdrant", never "db").
+
+For millions of points, an on-disk Qdrant collection with an ANN index is the right substrate (7thsense's
+9.3M-point corpus). SQLite/numpy cannot; a server can. This adapter is the mature 7thsense Qdrant path
+re-expressed on veccore's ports: dim-mismatch refusal, resumable existing-ids scroll, deterministic
+point-ids, and the OPTIONAL filter push-down seam.
+
+Capabilities: implements `search(filters=…)` (metadata / ACL push-down — permission enforcement in the
+query, O(index) not O(retrieved)) and is a Batch-ingest target. Does NOT implement `matrix()` — pulling
+millions of vectors client-side is not what a server backend is for; it raises typed NotSupported at that
+seam, exactly as the design intends (the honest negotiable "no").
+
+Point ids: Qdrant ids must be int or UUID, but a consumer's id is an arbitrary string. So the point id is
+`uuid5(NAMESPACE_URL, id)` (deterministic ⇒ re-upsert updates in place), and the real id + text_sha are
+carried in the payload under the reserved keys `_vid` / `_text_sha`, stripped from the payload returned to
+the consumer.
+
+Transport errors NEVER masquerade as 'missing'. `collection_exists()` separates a genuinely-absent
+collection from a timeout/auth failure: a real absence takes the create/empty path; any transport error
+PROPAGATES, so `ensure` never re-creates an existing collection and `existing_ids`/`stored_shas` never
+report points absent (which would re-embed the whole corpus, breaking the never-re-embed guarantee).
+"""
+from __future__ import annotations
+
+import uuid
+import warnings
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+import numpy as np
+
+from ..errors import DimMismatch, ModelMismatch, NotSupported, Unavailable
+from ..ports import Hit, Point
+
+BACKEND = "qdrant"
+_VID = "_vid"            # reserved payload key: the consumer's real string id
+_TEXT_SHA = "_text_sha"  # reserved payload key: idempotency sha
+_MODEL = "_model"        # reserved payload key: the embedder identity (model guard; Qdrant has no
+#                          collection-metadata field, so the model is carried on every point and checked
+#                          via a 1-point read on ensure())
+
+
+def _point_uuid(vid: str) -> str:
+    """Deterministic Qdrant point id from the consumer's string id (uuid5 over a stable namespace) — so a
+    re-upsert of the same id updates in place rather than duplicating. Parsing an id, not a judgement."""
+    return str(uuid.uuid5(uuid.NAMESPACE_URL, vid))
+
+
+class QdrantStore:
+    backend = BACKEND
+
+    def __init__(self, url: str = "http://localhost:6333", timeout: int = 60, client=None):
+        self.url = url
+        self.timeout = timeout
+        self._client = client   # injectable for tests; else lazily constructed
+        # in-process cache of each collection's model (set by ensure(model=...)) so upsert can stamp
+        # `_model` on its points; the DURABLE record is the payload itself, re-read on a fresh process.
+        self._model_by_collection = {}
+        # count of foreign/legacy points (no reserved _vid) skipped by the last search — for a consumer
+        # that wants to observe a mixed/migration collection programmatically (also warned about).
+        self.last_search_foreign_skipped = 0
+
+    def _qc(self):
+        if self._client is None:
+            try:
+                from qdrant_client import QdrantClient
+            except ImportError as e:
+                raise Unavailable("qdrant-client is not installed — `pip install veccore[qdrant]`",
+                                  code="backend_unavailable") from e
+            # check_compatibility=False: server/client minor gap is fine for the stable REST ops used here
+            self._client = QdrantClient(url=self.url, timeout=self.timeout, check_compatibility=False)
+        return self._client
+
+    # ── core port ──────────────────────────────────────────────────────────────────────────────────────
+    def ensure(self, collection: str, dim: int, model: str = None) -> None:
+        from qdrant_client import models as qm
+        qc = self._qc()
+        # collection_exists() distinguishes "genuinely missing" from a transport/auth error: a timeout or
+        # 401 raises and PROPAGATES (fail-safe) rather than being mistaken for "missing" and triggering a
+        # create over an existing collection. Only a real absence takes the create path.
+        if not qc.collection_exists(collection):
+            qc.create_collection(
+                collection_name=collection,
+                vectors_config=qm.VectorParams(size=dim, distance=qm.Distance.COSINE, on_disk=True))
+            if model is not None:
+                self._model_by_collection[collection] = model
+            return
+        info = qc.get_collection(collection)
+        size = info.config.params.vectors.size
+        if size != dim:
+            raise DimMismatch(collection, size, dim)   # 7thsense guard, kept as a core invariant
+        if model is not None:
+            stored = self._first_model(qc, collection)   # None if the collection is empty / pre-guard
+            if stored is not None and stored != model:
+                raise ModelMismatch(collection, stored, model)
+            self._model_by_collection[collection] = model
+
+    def upsert(self, collection: str, points: Sequence[Point]) -> None:
+        from qdrant_client import models as qm
+        qc = self._qc()
+        model = self._model_by_collection.get(collection)
+        structs = []
+        for p in points:
+            payload = dict(p.payload)
+            payload[_VID] = p.id
+            payload[_TEXT_SHA] = p.text_sha
+            if model is not None:
+                payload[_MODEL] = model   # carries the model identity so ensure() can guard it later
+            structs.append(qm.PointStruct(
+                id=_point_uuid(p.id),
+                vector=np.asarray(p.vector, dtype=np.float32).ravel().tolist(),
+                payload=payload))
+        qc.upsert(collection_name=collection, wait=True, points=structs)
+
+    def search(self, collection: str, qvec: np.ndarray, k: int,
+               filters: Optional[Dict[str, Any]] = None) -> List[Hit]:
+        qc = self._qc()
+        flt = self._build_filter(filters) if filters else None
+        res = qc.query_points(
+            collection_name=collection,
+            query=np.asarray(qvec, dtype=np.float32).ravel().tolist(),
+            limit=k, query_filter=flt, with_payload=True).points
+        out = []
+        foreign = 0
+        for p in res:
+            payload = dict(p.payload or {})
+            vid = payload.pop(_VID, None)
+            payload.pop(_TEXT_SHA, None)
+            payload.pop(_MODEL, None)   # veccore-internal — never surfaced to the consumer
+            if vid is None:
+                # a legacy/foreign point (not written by veccore, so no reserved _vid). SKIP it rather than
+                # emit Hit(id=None) — a null id crashes a consumer doing int(hit.id); a mixed/migration
+                # collection must degrade safely, not poison the result set. Counted + warned, never silent.
+                foreign += 1
+                continue
+            out.append(Hit(id=vid, score=float(p.score), payload=payload))
+        self.last_search_foreign_skipped = foreign
+        if foreign:
+            warnings.warn(
+                f"veccore.QdrantStore.search: skipped {foreign} foreign point(s) with no {_VID} in "
+                f"collection {collection!r} (legacy/non-veccore points, e.g. a migration in progress) — "
+                f"not returned as null ids. Rebuild the collection through veccore to clear them.",
+                stacklevel=2)
+        return out
+
+    def delete(self, collection: str, ids: Sequence[str]) -> int:
+        """Remove points by id; return the count confirmed removed. Idempotent (absent ids → 0).
+
+        The DELETE is unconditional over the FULL requested set — Qdrant no-ops ids that are absent, so the
+        deletion always completes and can never leave a requested id half-deleted regardless of any lookup.
+        Qdrant's delete does not report a removed-count, so the returned number comes from a separate
+        pre-read (`retrieve`) of how many requested ids were present; that count is best-effort and only
+        ever CONSERVATIVE (a transient partial read under-reports, never over-reports) — the state change
+        does not depend on it, so a conservative count never corresponds to data left behind."""
+        from qdrant_client import models as qm
+        ids = list(ids)
+        if not ids:
+            return 0
+        qc = self._qc()
+        if not qc.collection_exists(collection):
+            return 0
+        puids = [_point_uuid(i) for i in ids]
+        confirmed = len(qc.retrieve(collection_name=collection, ids=puids,
+                                    with_payload=False, with_vectors=False))
+        qc.delete(collection_name=collection, wait=True,   # full requested set; absent ids are a no-op
+                  points_selector=qm.PointIdsList(points=puids))
+        return confirmed
+
+    def existing_ids(self, collection: str) -> set:
+        return set(self._scroll_field(collection, _VID).keys())
+
+    def stored_shas(self, collection: str) -> Dict[str, str]:
+        return self._scroll_field(collection, _TEXT_SHA)
+
+    def capabilities(self) -> Dict[str, bool]:
+        return {"matrix": False, "filters": True, "batch": True}
+
+    # ── optional capability seam (unimplemented here → honest typed NotSupported) ──────────────────────
+    def matrix(self, collection: str) -> Tuple[List[str], np.ndarray]:
+        raise NotSupported(
+            "matrix (materialize all vectors for X @ Xᵀ analytics)", self.backend,
+            "qdrant does not implement matrix() — pulling a whole collection client-side defeats the "
+            "server ANN index. Use the sqlite_numpy backend for in-process matrix analytics, or run the "
+            "analysis as server-side searches.")
+
+    # ── internals ────────────────────────────────────────────────────────────────────────────────────
+    def _first_model(self, qc, collection: str):
+        """The model identity carried by any one point (they all carry the same), or None if the
+        collection is empty / predates the guard. A transport error PROPAGATES (not caught) so a failed
+        read never silently skips the model guard."""
+        points, _off = qc.scroll(collection_name=collection, limit=1,
+                                 with_payload=[_MODEL], with_vectors=False)
+        for p in points:
+            return (p.payload or {}).get(_MODEL)
+        return None
+
+    def _scroll_field(self, collection: str, field: str) -> Dict[str, str]:
+        """Scroll the whole collection, payload-only, collecting {_vid: field_value}. This is what makes
+        an ingest RESUMABLE (a point already present is never re-embedded).
+
+        A genuinely-absent collection returns {} (nothing stored yet). A transport/auth error PROPAGATES
+        — it must NOT read as 'no points', because that would make an ingest treat every existing point as
+        new and re-embed the whole corpus (the never-re-embed guarantee broken, real money burned)."""
+        qc = self._qc()
+        if not qc.collection_exists(collection):
+            return {}
+        out: Dict[str, str] = {}
+        offset = None
+        while True:
+            points, offset = qc.scroll(
+                collection_name=collection, limit=4000, offset=offset,
+                with_payload=[_VID, field], with_vectors=False)
+            for p in points:
+                pl = p.payload or {}
+                vid = pl.get(_VID)
+                if vid is not None:
+                    out[vid] = pl.get(field)
+            if offset is None:
+                return out
+
+    def _build_filter(self, filters: Dict[str, Any]):
+        """Translate a generic filters dict into a Qdrant Filter — the metadata/ACL push-down MECHANISM;
+        the consumer builds its ACL POLICY on this (which principals/groups → which clauses).
+
+        Shapes:
+          {field: value}            → must match value (MatchValue)
+          {field: [v1, v2]}         → must match any of (MatchAny)
+          {"must": {...}, "should": {...}}  → explicit AND / OR groups (e.g. an ACL 'should' set)
+        A `should` group is deny-by-default OR: at least one clause must hold (7thsense's ACL parity).
+        """
+        from qdrant_client import models as qm
+
+        def conds(d: Dict[str, Any]):
+            out = []
+            for key, val in d.items():
+                if isinstance(val, (list, tuple, set)):
+                    out.append(qm.FieldCondition(key=key, match=qm.MatchAny(any=list(val))))
+                else:
+                    out.append(qm.FieldCondition(key=key, match=qm.MatchValue(value=val)))
+            return out
+
+        if "must" in filters or "should" in filters:
+            must = conds(filters.get("must", {})) or None
+            should = conds(filters.get("should", {})) or None
+            return qm.Filter(must=must, should=should)
+        return qm.Filter(must=conds(filters))

veccore/adapters/sqlite_numpy_store.py

@@ -0,0 +1,162 @@
+"""SqliteNumpyStore — the in-process backend (backend id "sqlite_numpy", never "db").
+
+For a corpus of thousands-to-low-hundred-thousands of vectors, brute-force cosine in numpy is ~ms and
+keeps the consumer self-contained / portable — no vector server (warden's design). SQLite holds the
+vectors + their text_sha + payload, keyed by (collection, id); search is one L2-normalized matrix and a
+dot product (both sides unit vectors ⇒ dot == cosine).
+
+Capabilities: implements the OPTIONAL `matrix()` seam (the whole point of an in-process store — X @ Xᵀ
+analytics: kNN, centroid, edge-gating). Does NOT implement `search(filters=…)` push-down — that is
+qdrant's ACL/metadata seam; a filters request here raises typed NotSupported (never a silent ignore,
+which for an ACL filter would leak). No Batch API (that is qdrant/openai's ingest seam).
+"""
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+import numpy as np
+
+from ..errors import DimMismatch, ModelMismatch, NotSupported, Unavailable
+from ..ports import Hit, Point
+
+BACKEND = "sqlite_numpy"
+
+
+class SqliteNumpyStore:
+    backend = BACKEND
+
+    def __init__(self, path: str = ":memory:"):
+        self.path = path if path == ":memory:" else os.path.expanduser(path)
+        if self.path != ":memory:":
+            os.makedirs(os.path.dirname(self.path) or ".", exist_ok=True)
+        try:
+            self._con = sqlite3.connect(self.path)
+        except sqlite3.Error as e:
+            raise Unavailable(f"sqlite_numpy store at {self.path} could not be opened "
+                              f"({type(e).__name__}: {str(e)[:80]})", code="store_unusable") from e
+        self._con.execute("PRAGMA journal_mode=WAL")
+        self._ensure_schema()
+        # a small matrix cache keyed by (collection, row-count) so repeated searches within a process
+        # don't reload the whole table each time; the row-count auto-invalidates after an upsert.
+        self._matrix_cache: Dict[str, Tuple[int, List[str], np.ndarray]] = {}
+
+    def _ensure_schema(self) -> None:
+        self._con.execute(
+            "CREATE TABLE IF NOT EXISTS collections(name TEXT PRIMARY KEY, dim INTEGER NOT NULL)")
+        # additive migration: a nullable `model` column for the model-identity guard. A db created before
+        # this column has it added here as NULL (an existing collection's model is 'unknown' → adopted, not
+        # refused, on its next ensure(model=...)).
+        cols = {r[1] for r in self._con.execute("PRAGMA table_info(collections)")}
+        if "model" not in cols:
+            self._con.execute("ALTER TABLE collections ADD COLUMN model TEXT")
+        self._con.execute(
+            "CREATE TABLE IF NOT EXISTS points("
+            "  collection TEXT NOT NULL, id TEXT NOT NULL, text_sha TEXT NOT NULL,"
+            "  dim INTEGER NOT NULL, vec BLOB NOT NULL, payload TEXT NOT NULL,"
+            "  PRIMARY KEY(collection, id))")
+        self._con.commit()
+
+    # ── core port ──────────────────────────────────────────────────────────────────────────────────────
+    def ensure(self, collection: str, dim: int, model: str = None) -> None:
+        row = self._con.execute(
+            "SELECT dim, model FROM collections WHERE name=?", (collection,)).fetchone()
+        if row is not None:
+            stored_dim, stored_model = row
+            if stored_dim != dim:
+                raise DimMismatch(collection, stored_dim, dim)   # 7thsense guard, kept as a core invariant
+            if model is not None:
+                if stored_model is not None and stored_model != model:
+                    raise ModelMismatch(collection, stored_model, model)
+                if stored_model is None:
+                    # unknown stored model (legacy collection) → ADOPT the requested one, don't refuse
+                    self._con.execute("UPDATE collections SET model=? WHERE name=?", (model, collection))
+                    self._con.commit()
+            return
+        self._con.execute("INSERT INTO collections(name, dim, model) VALUES(?,?,?)",
+                          (collection, dim, model))
+        self._con.commit()
+
+    def upsert(self, collection: str, points: Sequence[Point]) -> None:
+        rows = []
+        for p in points:
+            v = np.asarray(p.vector, dtype=np.float32).ravel()
+            rows.append((collection, p.id, p.text_sha, int(v.shape[0]), v.tobytes(),
+                         json.dumps(p.payload, ensure_ascii=False)))
+        with self._con:
+            self._con.executemany(
+                "INSERT OR REPLACE INTO points(collection,id,text_sha,dim,vec,payload) "
+                "VALUES(?,?,?,?,?,?)", rows)
+        self._matrix_cache.pop(collection, None)   # index changed → drop cached matrix
+
+    def search(self, collection: str, qvec: np.ndarray, k: int,
+               filters: Optional[Dict[str, Any]] = None) -> List[Hit]:
+        if filters:
+            # NEVER silently ignore a filter — for an ACL filter that would leak. Honest typed refusal.
+            raise NotSupported(
+                "search filters (metadata/ACL push-down)", self.backend,
+                "sqlite_numpy does not implement filter push-down — use the qdrant backend for ACL/"
+                "metadata-filtered search, or probe capabilities() before passing filters.")
+        ids, M, payloads = self._load(collection)
+        if not ids:
+            return []
+        q = np.asarray(qvec, dtype=np.float32).ravel()
+        qn = q / max(float(np.linalg.norm(q)), 1e-9)
+        scores = M @ qn   # rows are stored L2-normalized ⇒ dot == cosine
+        order = np.argsort(-scores)[:max(0, k)]
+        return [Hit(id=ids[i], score=float(scores[i]), payload=payloads[i]) for i in order]
+
+    def delete(self, collection: str, ids: Sequence[str]) -> int:
+        """Remove points by id; return the count removed. Idempotent (absent ids → 0). Chunks the IN-list
+        and invalidates the collection's matrix cache so a later search/matrix reflects the removal."""
+        ids = list(ids)
+        if not ids:
+            return 0
+        removed = 0
+        with self._con:
+            for i in range(0, len(ids), 500):
+                chunk = ids[i:i + 500]
+                marks = ",".join("?" * len(chunk))
+                cur = self._con.execute(
+                    f"DELETE FROM points WHERE collection=? AND id IN ({marks})", [collection, *chunk])
+                removed += cur.rowcount
+        self._matrix_cache.pop(collection, None)
+        return removed
+
+    def existing_ids(self, collection: str) -> set:
+        return {r[0] for r in self._con.execute(
+            "SELECT id FROM points WHERE collection=?", (collection,))}
+
+    def stored_shas(self, collection: str) -> Dict[str, str]:
+        return {r[0]: r[1] for r in self._con.execute(
+            "SELECT id, text_sha FROM points WHERE collection=?", (collection,))}
+
+    def capabilities(self) -> Dict[str, bool]:
+        return {"matrix": True, "filters": False, "batch": False}
+
+    # ── optional capability seam ─────────────────────────────────────────────────────────────────────
+    def matrix(self, collection: str) -> Tuple[List[str], np.ndarray]:
+        ids, M, _payloads = self._load(collection)
+        return ids, M
+
+    # ── internals ────────────────────────────────────────────────────────────────────────────────────
+    def _load(self, collection: str) -> Tuple[List[str], np.ndarray, List[Dict[str, Any]]]:
+        n = self._con.execute("SELECT COUNT(*) FROM points WHERE collection=?", (collection,)).fetchone()[0]
+        cached = self._matrix_cache.get(collection)
+        ids: List[str] = []
+        vecs: List[np.ndarray] = []
+        payloads: List[Dict[str, Any]] = []
+        for id_, dim, blob, payload in self._con.execute(
+                "SELECT id, dim, vec, payload FROM points WHERE collection=? ORDER BY id", (collection,)):
+            ids.append(id_)
+            vecs.append(np.frombuffer(blob, dtype=np.float32))
+            payloads.append(json.loads(payload))
+        M = np.vstack(vecs).astype(np.float32) if vecs else np.zeros((0, 0), dtype=np.float32)
+        if cached is None or cached[0] != n:
+            self._matrix_cache[collection] = (n, ids, M)
+        return ids, M, payloads
+
+    def close(self) -> None:
+        self._con.close()

veccore/audit.py

@@ -0,0 +1,100 @@
+"""The audit envelope — WHERE / WHO / WHAT / WHY on every veccore call (auditability + observability).
+
+The user requirement: a caller must say to WHICH store it is sending things (no hidden default store),
+and every call must record WHAT is being sent, WHO is sending it, and WHY — so the estate has a complete
+audit trail of every embed/upsert/search and an observability feed it can watch live.
+
+The envelope is REQUIRED by every core call. `who`/`why` the caller supplies (veccore refuses a blank
+`why` — an un-reasoned spend/mutation is exactly what this exists to prevent). `where`/`what` veccore
+fills from the call itself (collection + resolved backend; operation + item count + sha summary). The
+completed envelope is emitted to an INJECTED ObserverPort, so the audit sink (a JSONL file, ccwatch, an
+OTel exporter) composes WITHOUT veccore importing any of them — the same acyclic discipline as the gate.
+
+Library sinks are DURABLE by construction: NullObserver (discard) and JsonlObserver (append-only, fsync'd
+file of record). An in-memory capturing observer is test/inspection tooling and lives with the tests, not
+here — the library never makes an in-process list the sole record of an audit trail.
+"""
+from __future__ import annotations
+
+import json
+import os
+import threading
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, Optional, Protocol, runtime_checkable
+
+
+@dataclass
+class AuditEnvelope:
+    """One auditable veccore operation.
+
+    Caller-supplied (required): `who` (the consumer, e.g. "warden", "7thsense", "saga"), `why` (a human
+    intent string). veccore-filled: `where` (collection + backend), `op`, `n_items`, `n_embedded`,
+    `n_skipped`, `sha_summary`, `est_usd`, `outcome`, `error`.
+    """
+    who: str
+    why: str
+    # veccore fills these as the call executes
+    op: str = ""
+    collection: str = ""
+    backend: str = ""
+    n_items: int = 0
+    n_embedded: int = 0
+    n_skipped: int = 0
+    sha_summary: str = ""
+    est_usd: Optional[float] = None
+    outcome: str = "started"          # started | ok | refused | error
+    error: str = ""
+    extra: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        # WHY IS REQUIRED. An un-reasoned embed/upsert/search is the thing this envelope exists to make
+        # impossible — refuse a blank one loudly rather than record an empty audit line.
+        if not (self.who or "").strip():
+            raise ValueError("AuditEnvelope.who is required — name the calling consumer.")
+        if not (self.why or "").strip():
+            raise ValueError("AuditEnvelope.why is required — state why this call is being made "
+                             "(auditability: every embed/upsert/search records its intent).")
+
+    def where(self) -> str:
+        """The human-readable target: 'collection@backend'. Filled once the core resolves the backend."""
+        return f"{self.collection}@{self.backend}" if self.backend else self.collection
+
+    def to_dict(self) -> Dict[str, Any]:
+        d = asdict(self)
+        d["where"] = self.where()
+        return d
+
+
+@runtime_checkable
+class ObserverPort(Protocol):
+    """The injected audit/observability sink. `emit` is called once per completed operation with the
+    filled envelope. Implementations must never raise back into veccore (observability must not break the
+    operation) — the core guards the call, but a well-behaved observer swallows its own errors too."""
+
+    def emit(self, envelope: AuditEnvelope) -> None: ...
+
+
+class NullObserver:
+    """Default sink: discard. veccore works with no observability configured; a consumer opts in."""
+
+    def emit(self, envelope: AuditEnvelope) -> None:
+        pass
+
+
+class JsonlObserver:
+    """Durable append-only audit log — one JSON line per operation. Append-only + per-record flush + fsync
+    so a long run's audit trail is visible live and survives a crash (the durable-log standard). The file
+    is the estate's veccore audit of record; a cockpit/OTel exporter can tail it."""
+
+    def __init__(self, path: str):
+        self.path = os.path.expanduser(path)
+        os.makedirs(os.path.dirname(self.path) or ".", exist_ok=True)
+        self._lock = threading.Lock()
+
+    def emit(self, envelope: AuditEnvelope) -> None:
+        line = json.dumps(envelope.to_dict(), ensure_ascii=False, default=str)
+        with self._lock:
+            with open(self.path, "a", encoding="utf-8") as fh:
+                fh.write(line + "\n")
+                fh.flush()
+                os.fsync(fh.fileno())