embedflow 0.1.0__py3-none-any.whl

embedflow/__init__.py

@@ -0,0 +1,25 @@
+"""EmbedFlow: progressive embedding-model migration for existing indexes."""
+
+__version__ = "0.1.0"
+
+from .config import EmbedFlowConfig, load_config
+
+
+def migrate(*args, **kwargs):
+    """Start progressive migration over an existing FAISS/Qdrant index.
+
+    Imported lazily to keep the lightweight configuration package free of
+    model-serving dependencies at import time.  See ``embedflow.migration``
+    for the ``MigrationSession`` type.
+    """
+    from .migration.facade import migrate as _migrate
+    return _migrate(*args, **kwargs)
+
+
+def analyze_migration(*args, **kwargs):
+    """Run the leakage-safe no-target-index analysis programmatically."""
+    from .analysis import analyze_migration as _analyze_migration
+    return _analyze_migration(*args, **kwargs)
+
+
+__all__ = ["EmbedFlowConfig", "load_config", "migrate", "analyze_migration", "__version__"]

embedflow/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

embedflow/analysis.py

@@ -0,0 +1,192 @@
+"""Programmatic Mode-B migration analysis.
+
+The CLI is the most discoverable interface, but applications and notebooks
+often already hold a parsed EmbedFlowConfig. This module delegates to the
+same runtime and frozen T2-v1 implementation used by ``embedflow analyze``;
+it does not introduce a second scoring path.
+"""
+
+from __future__ import annotations
+
+import json
+import tempfile
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+from typing import Any
+
+from .compatibility.report import report_markdown, write_report
+from .config import EmbedFlowConfig, from_dict, load_config, save_config
+from .migration.compatibility import run_probe, save_probe
+from .registry import load_evidence, match_config
+from .runtime import load_documents, open_engine
+
+
+def _query_rows(value: str | Path | Iterable[tuple[str, str]]) -> list[tuple[str, str]]:
+    if isinstance(value, (str, Path)):
+        path = Path(value)
+        if not path.exists():
+            raise FileNotFoundError(path)
+        rows: list[tuple[str, str]] = []
+        seen: set[str] = set()
+        with path.open(encoding="utf-8") as handle:
+            for number, line in enumerate(handle, 1):
+                if not line.strip():
+                    continue
+                try:
+                    item = json.loads(line)
+                except json.JSONDecodeError as exc:
+                    raise ValueError(f"invalid query JSON at {path}:{number}") from exc
+                if not isinstance(item, Mapping):
+                    raise ValueError(f"query row {number} in {path} must be a JSON object")
+                query_id = str(item.get("id", item.get("query_id", number)))
+                text = item.get("text", item.get("query"))
+                if query_id in seen:
+                    raise ValueError(f"duplicate query ID {query_id!r} in {path}")
+                if not isinstance(text, str) or not text.strip():
+                    raise ValueError(f"query {query_id!r} has no text")
+                seen.add(query_id); rows.append((query_id, text))
+        if not rows:
+            raise ValueError(f"query file is empty: {value}")
+        return rows
+    try:
+        rows = [(str(query_id), text) for query_id, text in value]
+    except (TypeError, ValueError) as exc:
+        raise ValueError("probe_queries must be an iterable of (query_id, text) pairs") from exc
+    if len({query_id for query_id, _ in rows}) != len(rows):
+        raise ValueError("probe_queries must not contain duplicate query IDs")
+    if not rows or any(not isinstance(text, str) or not text.strip() for _, text in rows):
+        raise ValueError("probe_queries must contain non-empty query text")
+    return rows
+
+
+def _registry_values(match: Any) -> list[dict[str, Any]]:
+    """Return only the canonical measurements explicitly reused by a match."""
+    values: list[dict[str, Any]] = []
+    for row in match.records:
+        data = row.to_dict()
+        values.append({
+            "evidence_id": row.evidence_id,
+            "dataset": data.get("dataset", {}).get("name"),
+            "candidate_gap": data.get("candidate_gap", {}),
+            "containment": data.get("containment", {}),
+            "source_quality": data.get("source_quality"),
+            "native_target_quality": data.get("native_target_quality"),
+            "restricted_target_quality": data.get("restricted_target_quality", {}),
+            "observed_migration_depth": data.get("observed_migration_depth"),
+            "ci_certified_migration_depth": data.get("ci_certified_migration_depth"),
+            "epsilon": data.get("epsilon"),
+            "provenance": data.get("provenance", {}),
+        })
+    return values
+
+
+def analyze_migration(
+    config: str | Path | EmbedFlowConfig | Mapping[str, Any],
+    *,
+    probe_queries: str | Path | Iterable[tuple[str, str]] | None = None,
+    device: str | None = None,
+    demo: bool = False,
+    corpus_name: str | None = None,
+    corpus_fingerprint: str | None = None,
+    use_registry: bool = False,
+    output_dir: str | Path | None = None,
+) -> dict[str, Any]:
+    """Run leakage-safe no-target-index analysis and return a JSON report.
+
+    ``use_registry`` only succeeds for an exact contract-and-corpus match. A
+    prior or related row is displayed in the returned ``registry`` field but
+    never converted into a current-corpus compatibility decision.
+    """
+    temporary_config: Path | None = None
+    if isinstance(config, (str, Path)):
+        config_path = Path(config).expanduser().resolve()
+        cfg = load_config(config_path)
+    else:
+        cfg = config if isinstance(config, EmbedFlowConfig) else from_dict(dict(config))
+        cfg.validate()
+        # open_engine intentionally has one configuration-loading path.
+        # Serialize an in-memory config to a short-lived file so this API
+        # inherits all of its validation, model, and index checks.
+        cfg.resolve_paths(Path.cwd().resolve())
+        handle = tempfile.NamedTemporaryFile(prefix="embedflow-analysis-", suffix=".yaml", delete=False)
+        handle.close()
+        temporary_config = Path(handle.name)
+        save_config(cfg, temporary_config)
+        config_path = temporary_config
+    if probe_queries is None:
+        probe_queries = cfg.probe.queries
+    if probe_queries is None:
+        raise ValueError("probe_queries or config.probe.queries is required")
+    engine = None
+    try:
+        docs = load_documents(cfg)
+        registry = match_config(
+            cfg,
+            corpus_fingerprint=corpus_fingerprint,
+            corpus_name=corpus_name,
+            corpus_size=docs.size(),
+            records=load_evidence(),
+        )
+        if use_registry and registry.level != "EXACT REGISTRY MATCH":
+            raise ValueError("--use-registry requires an EXACT REGISTRY MATCH")
+        engine = open_engine(config_path, device=device, demo=demo, start_worker=False)
+        result = run_probe(
+            engine.source_model,
+            engine.target_model,
+            engine.source_index,
+            docs,
+            _query_rows(probe_queries),
+            kmax=cfg.probe.kmax,
+            seed=cfg.probe.seed,
+            limit=cfg.probe.limit,
+        )
+        report: dict[str, Any] = {
+            "schema_version": "0.1",
+            "source_model": cfg.source.model,
+            "target_model": cfg.target.model,
+            "corpus_documents": docs.size(),
+            "diagnostic": result["diagnostic"],
+            "recommended_initial_k": result.get("recommended_k"),
+            "observed_k_epsilon": None,
+            "epsilon": cfg.probe.epsilon,
+            "ann_status": "UNKNOWN",
+            "native_target_index_used": False,
+            "registry": registry.to_dict(),
+            "registry_reused": bool(use_registry),
+            "registry_reuse": {
+                "used": bool(use_registry),
+                "evidence_ids": [row.evidence_id for row in registry.records] if use_registry else [],
+                "reused_fields": ["candidate_gap", "containment", "native_target_quality", "source_quality", "observed_migration_depth", "ci_certified_migration_depth"] if use_registry else [],
+                "note": "Canonical values are prior measurements; the current-corpus probe remains the deployment analysis." if use_registry else "Canonical rows were displayed but not reused.",
+            },
+            "registry_reused_values": _registry_values(registry) if use_registry else [],
+            "registry_evidence": [row.to_dict() for row in registry.records],
+            "probe": result,
+            "candidate_gap_curve": [],
+            "t2_warning": result.get("warning"),
+            "recommendation": (
+                "Progressive migration is a reasonable candidate for further deployment validation."
+                if str(result.get("diagnostic", "")).upper() == "SAFE"
+                else "Further validation is required before relying on progressive migration."
+            ),
+            "limitations": [
+                "No native target index or qrels were used; this is Mode B deployment analysis.",
+                "Recommended initial K is not observed K*.",
+                "ANN fidelity is UNKNOWN until an exact/reference source comparison is supplied.",
+            ],
+        }
+        if output_dir is not None:
+            destination = Path(output_dir).expanduser().resolve()
+            destination.mkdir(parents=True, exist_ok=True)
+            save_probe(result, destination / "probe_result.json")
+            write_report(destination / "migration_report.json", report)
+            (destination / "report.md").write_text(report_markdown(report), encoding="utf-8")
+        return report
+    finally:
+        if engine is not None:
+            engine.close()
+        if temporary_config is not None:
+            temporary_config.unlink(missing_ok=True)
+
+
+__all__ = ["analyze_migration"]

embedflow/cache/__init__.py

@@ -0,0 +1,4 @@
+from .base import TargetVectorCache
+from .persistent_cache import CacheCorruptionError, SQLiteVectorCache
+
+__all__ = ["TargetVectorCache", "SQLiteVectorCache", "CacheCorruptionError"]

embedflow/cache/base.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+import numpy as np
+
+
+class TargetVectorCache(ABC):
+    @abstractmethod
+    def get(self, document_ids: list[str]) -> dict[str, np.ndarray]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def put(self, document_ids: list[str], vectors: np.ndarray) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def contains(self, document_ids: list[str]) -> set[str]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def stats(self) -> dict[str, Any]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def close(self) -> None:
+        raise NotImplementedError

embedflow/cache/persistent_cache.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+import hashlib
+import sqlite3
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from .base import TargetVectorCache
+
+
+class CacheCorruptionError(RuntimeError):
+    pass
+
+
+class SQLiteVectorCache(TargetVectorCache):
+    """Persistent target-vector cache keyed by document and model fingerprint."""
+
+    # SQLite builds differ in their maximum number of bound variables.  Keep
+    # lookups below the conservative default so prewarming large corpora works
+    # on older and embedded SQLite versions as well as on modern builds.
+    _LOOKUP_BATCH_SIZE = 900
+
+    def __init__(self, path: str | Path, model_fingerprint: str, dimension: int,
+                 dtype: str = "float32"):
+        p = Path(path)
+        if p.suffix in {".sqlite", ".sqlite3", ".db"}:
+            self.db_path = p
+            self.root = p.parent
+        else:
+            self.root = p
+            self.db_path = p / "cache.sqlite3"
+        self.root.mkdir(parents=True, exist_ok=True)
+        self.model_fingerprint = str(model_fingerprint).strip()
+        if not self.model_fingerprint:
+            raise ValueError("model_fingerprint must be non-empty")
+        try:
+            parsed_dimension = int(dimension)
+            exact = float(dimension) == parsed_dimension
+        except (TypeError, ValueError, OverflowError):
+            parsed_dimension, exact = 0, False
+        if isinstance(dimension, bool) or not exact or parsed_dimension < 1:
+            raise ValueError("cache dimension must be a positive integer")
+        self.dimension = parsed_dimension
+        try:
+            self.dtype = np.dtype(dtype)
+        except TypeError as exc:
+            raise ValueError(f"unsupported cache dtype: {dtype!r}") from exc
+        if self.dtype.kind not in {"f", "i", "u"}:
+            raise ValueError("cache dtype must be a numeric vector dtype")
+        self._lock = threading.RLock()
+        self._db = None
+        try:
+            self._db = sqlite3.connect(str(self.db_path), check_same_thread=False, timeout=30)
+            self._db.execute("PRAGMA busy_timeout=30000")
+            self._db.execute("PRAGMA journal_mode=WAL")
+            self._db.execute("PRAGMA synchronous=NORMAL")
+            self._db.execute("""CREATE TABLE IF NOT EXISTS target_vectors (
+                document_id TEXT NOT NULL,
+                model_fingerprint TEXT NOT NULL,
+                dimension INTEGER NOT NULL,
+                dtype TEXT NOT NULL,
+                vector BLOB NOT NULL,
+                checksum TEXT NOT NULL,
+                created_at REAL NOT NULL,
+                accessed_at REAL NOT NULL,
+                PRIMARY KEY(document_id, model_fingerprint)
+            )""")
+            self._db.execute("CREATE INDEX IF NOT EXISTS idx_target_vectors_model ON target_vectors(model_fingerprint)")
+            self._db.execute("""CREATE TABLE IF NOT EXISTS cache_counters (
+                model_fingerprint TEXT PRIMARY KEY,
+                hits INTEGER NOT NULL DEFAULT 0,
+                misses INTEGER NOT NULL DEFAULT 0
+            )""")
+            self._db.execute("""CREATE TABLE IF NOT EXISTS cache_access_events (
+                event_id INTEGER PRIMARY KEY AUTOINCREMENT,
+                model_fingerprint TEXT NOT NULL,
+                hit INTEGER NOT NULL,
+                accessed_at REAL NOT NULL
+            )""")
+            self._db.execute("CREATE INDEX IF NOT EXISTS idx_cache_events_model ON cache_access_events(model_fingerprint, event_id)")
+            self._db.execute("INSERT OR IGNORE INTO cache_counters(model_fingerprint) VALUES (?)", (self.model_fingerprint,))
+            self._db.commit()
+        except sqlite3.DatabaseError as exc:
+            if self._db is not None:
+                self._db.close()
+            self._db = None
+            raise CacheCorruptionError(f"invalid SQLite target-vector cache: {self.db_path}") from exc
+        self._closed = False
+
+    def _ensure_open(self) -> None:
+        if self._closed or self._db is None:
+            raise RuntimeError("target vector cache is closed")
+
+    def _decode(self, row: tuple[Any, ...]) -> np.ndarray:
+        document_id, _, dimension, dtype, blob, checksum, *_ = row
+        if int(dimension) != self.dimension or np.dtype(dtype) != self.dtype:
+            raise CacheCorruptionError(f"cache contract mismatch for document {document_id}")
+        if hashlib.sha256(blob).hexdigest() != checksum:
+            raise CacheCorruptionError(f"cache checksum mismatch for document {document_id}")
+        value = np.frombuffer(blob, dtype=self.dtype).copy()
+        if value.size != self.dimension or not np.isfinite(value).all():
+            raise CacheCorruptionError(f"invalid cached vector for document {document_id}")
+        return value.astype("float32", copy=False)
+
+    def get(self, document_ids: list[str]) -> dict[str, np.ndarray]:
+        ids = [str(x) for x in document_ids]
+        if not ids: return {}
+        with self._lock:
+            self._ensure_open()
+            rows: list[tuple[Any, ...]] = []
+            for start in range(0, len(ids), self._LOOKUP_BATCH_SIZE):
+                chunk = ids[start:start + self._LOOKUP_BATCH_SIZE]
+                placeholders = ",".join("?" for _ in chunk)
+                rows.extend(self._db.execute(
+                    f"SELECT document_id,model_fingerprint,dimension,dtype,vector,checksum,created_at,accessed_at "
+                    f"FROM target_vectors WHERE model_fingerprint=? AND document_id IN ({placeholders})",
+                    [self.model_fingerprint, *chunk]).fetchall())
+            now = time.time()
+            values = {str(row[0]): self._decode(row) for row in rows}
+            if rows:
+                self._db.executemany("UPDATE target_vectors SET accessed_at=? WHERE document_id=? AND model_fingerprint=?",
+                                     [(now, str(row[0]), self.model_fingerprint) for row in rows])
+            # Count accesses, rather than unique rows, so the hit-rate remains
+            # meaningful when a caller supplies duplicate IDs.
+            hits = sum(document_id in values for document_id in ids)
+            misses = len(ids) - hits
+            self._db.execute("UPDATE cache_counters SET hits=hits+?,misses=misses+? WHERE model_fingerprint=?",
+                             (hits, misses, self.model_fingerprint))
+            self._db.executemany("INSERT INTO cache_access_events(model_fingerprint,hit,accessed_at) VALUES (?,?,?)",
+                                 [(self.model_fingerprint, int(document_id in values), now) for document_id in ids])
+            # Keep the persistent telemetry table bounded while retaining a
+            # useful recent-hit-rate window for status and dashboards.
+            self._db.execute("""DELETE FROM cache_access_events
+                WHERE model_fingerprint=? AND event_id NOT IN
+                (SELECT event_id FROM cache_access_events WHERE model_fingerprint=? ORDER BY event_id DESC LIMIT 1000)""",
+                             (self.model_fingerprint, self.model_fingerprint))
+            self._db.commit()
+            return values
+
+    def put(self, document_ids: list[str], vectors: np.ndarray) -> None:
+        ids = [str(x) for x in document_ids]
+        values = np.asarray(vectors, dtype=self.dtype)
+        if values.ndim != 2 or values.shape != (len(ids), self.dimension):
+            raise ValueError(f"vectors must have shape ({len(ids)}, {self.dimension})")
+        if len(set(ids)) != len(ids):
+            raise ValueError("duplicate IDs in cache write")
+        if not np.isfinite(values).all():
+            raise ValueError("cannot cache non-finite vectors")
+        now = time.time(); records = []
+        for document_id, value in zip(ids, values):
+            blob = np.ascontiguousarray(value).tobytes()
+            records.append((document_id, self.model_fingerprint, self.dimension, self.dtype.name, sqlite3.Binary(blob),
+                            hashlib.sha256(blob).hexdigest(), now, now))
+        with self._lock:
+            self._ensure_open()
+            self._db.executemany("""INSERT INTO target_vectors
+                (document_id,model_fingerprint,dimension,dtype,vector,checksum,created_at,accessed_at)
+                VALUES (?,?,?,?,?,?,?,?)
+                ON CONFLICT(document_id,model_fingerprint) DO UPDATE SET
+                    dimension=excluded.dimension,dtype=excluded.dtype,vector=excluded.vector,
+                    checksum=excluded.checksum,accessed_at=excluded.accessed_at""", records)
+            self._db.commit()
+
+    def contains(self, document_ids: list[str]) -> set[str]:
+        return set(self.get(document_ids))
+
+    def stats(self) -> dict[str, Any]:
+        with self._lock:
+            self._ensure_open()
+            count = self._db.execute("SELECT COUNT(*) FROM target_vectors WHERE model_fingerprint=?", (self.model_fingerprint,)).fetchone()[0]
+            total = self._db.execute("SELECT COUNT(*) FROM target_vectors").fetchone()[0]
+            counters = self._db.execute("SELECT hits,misses FROM cache_counters WHERE model_fingerprint=?",
+                                        (self.model_fingerprint,)).fetchone() or (0, 0)
+            recent = self._db.execute("""SELECT SUM(hit),COUNT(*) FROM cache_access_events
+                WHERE model_fingerprint=? AND event_id IN
+                (SELECT event_id FROM cache_access_events WHERE model_fingerprint=? ORDER BY event_id DESC LIMIT 100)""",
+                                     (self.model_fingerprint, self.model_fingerprint)).fetchone()
+        total_hits, total_misses = int(counters[0]), int(counters[1])
+        recent_hits, recent_count = int(recent[0] or 0), int(recent[1] or 0)
+        return {"cached_target_vectors": int(count), "all_model_vectors": int(total),
+                "model_fingerprint": self.model_fingerprint, "dimension": self.dimension,
+                "cache_path": str(self.db_path), "total_cache_hits": total_hits,
+                "total_cache_misses": total_misses,
+                "recent_hit_rate": (recent_hits / recent_count) if recent_count else 0.0}
+
+    def close(self) -> None:
+        with self._lock:
+            if not self._closed:
+                self._db.close()
+                self._db = None
+                self._closed = True
+
+
+__all__ = ["SQLiteVectorCache", "CacheCorruptionError"]