embedflow 0.1.0__py3-none-any.whl

embedflow/registry/matcher.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from .schema import (
+    MATCH_EXACT,
+    MATCH_NONE,
+    MATCH_PRIOR,
+    MATCH_RELATED,
+    EvidenceRecord,
+    contract_fingerprint,
+)
+
+_ALIASES = {
+    "minilm_l6": "sentence-transformers/all-MiniLM-L6-v2",
+    "minilm-l6": "sentence-transformers/all-MiniLM-L6-v2",
+    "minilm-l6-v2": "sentence-transformers/all-MiniLM-L6-v2",
+    "sentence-transformers/all-minilm-l6-v2": "sentence-transformers/all-MiniLM-L6-v2",
+    "qwen3-0.6b": "Qwen/Qwen3-Embedding-0.6B",
+    "qwen3_0_6b": "Qwen/Qwen3-Embedding-0.6B",
+    "qwen/qwen3-embedding-0.6b": "Qwen/Qwen3-Embedding-0.6B",
+    "qwen3-4b": "Qwen/Qwen3-Embedding-4B",
+    "qwen3_4b": "Qwen/Qwen3-Embedding-4B",
+    "qwen/qwen3-embedding-4b": "Qwen/Qwen3-Embedding-4B",
+    "qwen3-8b": "Qwen/Qwen3-Embedding-8B",
+    "qwen3_8b": "Qwen/Qwen3-Embedding-8B",
+    "qwen/qwen3-embedding-8b": "Qwen/Qwen3-Embedding-8B",
+}
+
+
+def canonical_model_id(value: Any) -> str:
+    if isinstance(value, Mapping):
+        value = value.get("canonical_model_id", value.get("model", value.get("model_id", "")))
+    else:
+        value = getattr(value, "model", getattr(value, "model_id", value))
+    text = str(value or "").strip()
+    return _ALIASES.get(text.lower(), text)
+
+
+def _contract(value: Any) -> tuple[str, str | None]:
+    if isinstance(value, Mapping):
+        ident = canonical_model_id(value)
+        fp = value.get("contract_fingerprint") or value.get("fingerprint")
+        if fp is None and any(key in value for key in ("revision", "dimension", "pooling", "max_length", "normalization")):
+            candidate = dict(value)
+            candidate["canonical_model_id"] = ident
+            fp = contract_fingerprint(candidate)
+        return ident, str(fp) if fp else None
+    ident = canonical_model_id(value)
+    if hasattr(value, "contract"):
+        raw = dict(value.contract())
+        raw["canonical_model_id"] = ident
+        return ident, contract_fingerprint(raw)
+    if hasattr(value, "fingerprint"):
+        return ident, str(value.fingerprint)
+    # A bare model ID can identify a prior transition for display, but cannot
+    # establish exact contract equivalence (revision/prompt/etc. are unknown).
+    return ident, None
+
+
+def _family(value: str) -> str:
+    text = value.lower().replace("embedding", "").replace("sentence-transformers/", "")
+    if "minilm" in text:
+        return "minilm"
+    # Keep the family intentionally broad: a Qwen3 revision/size that is not
+    # in the core rows can still receive RELATED evidence, but never an
+    # automatic decision.  Contract-level matching above remains strict.
+    if re.search(r"qwen3", text):
+        return "qwen3"
+    return text
+
+
+@dataclass(frozen=True)
+class RegistryMatch:
+    """Evidence lookup result; only EXACT matches can be reused as results."""
+
+    level: str
+    records: tuple[EvidenceRecord, ...]
+    exact_source_contract: bool
+    exact_target_contract: bool
+    exact_corpus: bool
+    reason: str
+    recommended_k: tuple[int, ...] = ()
+
+    @property
+    def prior_datasets(self) -> tuple[str, ...]:
+        return tuple(dict.fromkeys(str(row.dataset.get("name")) for row in self.records))
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "level": self.level,
+            "exact_source_contract": self.exact_source_contract,
+            "exact_target_contract": self.exact_target_contract,
+            "exact_corpus": self.exact_corpus,
+            "reason": self.reason,
+            "prior_datasets": list(self.prior_datasets),
+            "recommended_k": list(self.recommended_k),
+            "evidence_ids": [row.evidence_id for row in self.records],
+        }
+
+
+def match_evidence(
+    *,
+    source_model: Any,
+    target_model: Any,
+    corpus_fingerprint: str | None = None,
+    corpus_name: str | None = None,
+    corpus_size: int | None = None,
+    records: Iterable[EvidenceRecord],
+) -> RegistryMatch:
+    source_id, source_fp = _contract(source_model)
+    target_id, target_fp = _contract(target_model)
+    # ``records`` is intentionally typed as an iterable so callers can stream
+    # custom registries.  Materialise it once: transition and family matching
+    # both need to inspect the same rows, and consuming a generator twice
+    # would otherwise produce a false ``NO REGISTRY MATCH`` result.
+    rows = tuple(records)
+    source_family, target_family = _family(source_id), _family(target_id)
+    transition = tuple(
+        row for row in rows
+        if canonical_model_id(row.source) == source_id and canonical_model_id(row.target) == target_id
+    )
+    if not transition:
+        family_rows = tuple(
+            row for row in rows
+            if _family(canonical_model_id(row.source)) == source_family
+            and _family(canonical_model_id(row.target)) == target_family
+        )
+        return RegistryMatch(
+            MATCH_RELATED if family_rows else MATCH_NONE,
+            family_rows,
+            False,
+            False,
+            False,
+            "No identical transition was found; related family evidence is not a compatibility decision." if family_rows else "No registry record matches this transition.",
+            _priority_k(family_rows),
+        )
+    exact_source = source_fp is not None and all(str(row.source.get("contract_fingerprint")) == source_fp for row in transition)
+    exact_target = target_fp is not None and all(str(row.target.get("contract_fingerprint")) == target_fp for row in transition)
+    corpus_matches = []
+    for row in transition:
+        dataset = row.dataset
+        fingerprint_match = corpus_fingerprint is not None and dataset.get("fingerprint") is not None and str(dataset.get("fingerprint")) == str(corpus_fingerprint)
+        canonical_match = (
+            corpus_name is not None and str(dataset.get("canonical_dataset_id", dataset.get("name"))) == str(corpus_name)
+            and corpus_size is not None and dataset.get("corpus_size") is not None and int(dataset["corpus_size"]) == int(corpus_size)
+            and bool(dataset.get("canonical_construction", False))
+        )
+        corpus_matches.append(bool(fingerprint_match or canonical_match))
+    exact_corpus = any(corpus_matches)
+    if exact_source and exact_target and exact_corpus:
+        selected = tuple(row for row, matched in zip(transition, corpus_matches) if matched)
+        return RegistryMatch(MATCH_EXACT, selected, True, True, True, "Source and target contracts and corpus construction match a canonical row.", _priority_k(selected))
+    if exact_source and exact_target:
+        return RegistryMatch(MATCH_PRIOR, transition, True, True, False, "Contracts match, but the supplied corpus is new or its fingerprint is unavailable.", _priority_k(transition))
+    return RegistryMatch(MATCH_RELATED, transition, exact_source, exact_target, False, "A model transition matches by canonical name, but its full contract fingerprint differs.", _priority_k(transition))
+
+
+def _priority_k(records: Iterable[EvidenceRecord]) -> tuple[int, ...]:
+    depths: set[int] = set()
+    for row in records:
+        observed = row.raw.get("observed_migration_depth")
+        if observed is not None:
+            depths.add(int(observed))
+        depths.update(int(k) for k in row.candidate_gap)
+    return tuple(sorted(depths))
+
+
+def match_config(config: Any, *, corpus_fingerprint: str | None = None, corpus_name: str | None = None, corpus_size: int | None = None, records: Iterable[EvidenceRecord] | None = None) -> RegistryMatch:
+    """Match an ``EmbedFlowConfig`` without making network/model calls."""
+    from ..config import hydrate_research_contract
+    from .loader import load_evidence
+
+    if corpus_size is None:
+        docs = getattr(config, "documents", None)
+        if docs is not None:
+            corpus_size = None
+    source = config.source if hasattr(config, "source") else config.get("source")
+    target = config.target if hasattr(config, "target") else config.get("target")
+    # ``load_config`` already hydrates the frozen contracts.  Hydrating here as
+    # well makes the Python API safe for callers who construct a config object
+    # directly, while explicit non-default contract fields remain untouched.
+    if hasattr(source, "model"):
+        source = hydrate_research_contract(source)
+    if hasattr(target, "model"):
+        target = hydrate_research_contract(target)
+    return match_evidence(
+        source_model=source,
+        target_model=target,
+        corpus_fingerprint=corpus_fingerprint,
+        corpus_name=corpus_name,
+        corpus_size=corpus_size,
+        records=list(records) if records is not None else load_evidence(),
+    )

embedflow/registry/schema.py

@@ -0,0 +1,266 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+REGISTRY_VERSION = "0.1.0"
+SCHEMA_VERSION = "1"
+MATCH_EXACT = "EXACT REGISTRY MATCH"
+MATCH_PRIOR = "PRIOR EVIDENCE AVAILABLE"
+MATCH_RELATED = "RELATED EVIDENCE ONLY"
+MATCH_NONE = "NO REGISTRY MATCH"
+
+
+class RegistryError(ValueError):
+    """Raised when packaged evidence is malformed or internally inconsistent."""
+
+
+def _finite(value: Any, label: str) -> float:
+    try:
+        result = float(value)
+    except (TypeError, ValueError) as exc:
+        raise RegistryError(f"{label} must be a finite number") from exc
+    if not math.isfinite(result):
+        raise RegistryError(f"{label} must be finite")
+    return result
+
+
+def _positive_integer(value: Any, label: str, *, allow_zero: bool = False) -> int:
+    """Parse an integer without accepting lossy values such as ``1.5``."""
+    if isinstance(value, bool):
+        raise RegistryError(f"{label} must be an integer")
+    try:
+        number = int(value)
+    except (TypeError, ValueError) as exc:
+        raise RegistryError(f"{label} must be an integer") from exc
+    try:
+        exact = float(value) == number
+    except (TypeError, ValueError, OverflowError):
+        exact = False
+    if not exact or (number < 0 if allow_zero else number < 1):
+        qualifier = "non-negative" if allow_zero else "positive"
+        raise RegistryError(f"{label} must be a {qualifier} integer")
+    return number
+
+
+def contract_fingerprint(contract: Mapping[str, Any]) -> str:
+    """Hash the behaviorally relevant embedding contract deterministically."""
+    required = {
+        # Keep the key names identical to ``ModelConfig.fingerprint``.  This
+        # lets registry matching prove that a packaged row and a runtime
+        # cache use the same semantic contract, rather than merely the same
+        # model display name.
+        "model": contract.get("canonical_model_id", contract.get("model", "")),
+        "revision": contract.get("revision"),
+        "dimension": contract.get("dimension"),
+        "max_length": contract.get("max_length"),
+        "pooling": contract.get("pooling"),
+        "padding_side": contract.get("padding_side"),
+        "truncation_side": contract.get("truncation_side"),
+        "query_instruction": contract.get("query_instruction", ""),
+        "document_instruction": contract.get("document_instruction", ""),
+        "normalization": contract.get("normalization"),
+        "dtype": contract.get("dtype", "float32"),
+    }
+    return hashlib.sha256(json.dumps(required, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()).hexdigest()
+
+
+@dataclass(frozen=True)
+class EvidenceRecord:
+    """Validated view of one canonical migration evidence row."""
+
+    raw: dict[str, Any]
+
+    @property
+    def evidence_id(self) -> str:
+        return str(self.raw["evidence_id"])
+
+    @property
+    def source(self) -> Mapping[str, Any]:
+        return self.raw["source"]
+
+    @property
+    def target(self) -> Mapping[str, Any]:
+        return self.raw["target"]
+
+    @property
+    def dataset(self) -> Mapping[str, Any]:
+        return self.raw["dataset"]
+
+    @property
+    def candidate_gap(self) -> dict[int, float]:
+        return {int(k): float(v) for k, v in self.raw.get("candidate_gap", {}).items()}
+
+    @property
+    def containment(self) -> dict[int, float]:
+        return {int(k): float(v) for k, v in self.raw.get("containment", {}).items()}
+
+    @property
+    def epsilon(self) -> float:
+        return float(self.raw.get("epsilon", 0.01))
+
+    def to_dict(self) -> dict[str, Any]:
+        return json.loads(json.dumps(self.raw, sort_keys=True))
+
+    @classmethod
+    def from_dict(cls, raw: Mapping[str, Any]) -> EvidenceRecord:
+        if not isinstance(raw, Mapping):
+            raise RegistryError("evidence row must be an object")
+        try:
+            row = json.loads(json.dumps(dict(raw), ensure_ascii=False))
+        except (TypeError, ValueError) as exc:
+            raise RegistryError("evidence row must contain JSON-serializable values") from exc
+        for key in ("registry_version", "evidence_id", "source", "target", "dataset", "metric", "provenance"):
+            if key not in row:
+                raise RegistryError(f"evidence row missing {key!r}")
+        if str(row["registry_version"]) != REGISTRY_VERSION:
+            raise RegistryError(f"{row['evidence_id']}: unsupported registry_version {row['registry_version']!r}")
+        for side in ("source", "target"):
+            contract = row[side]
+            if not isinstance(contract, dict):
+                raise RegistryError(f"{row['evidence_id']}: {side} contract must be an object")
+            if not str(contract.get("canonical_model_id", "")).strip():
+                raise RegistryError(f"{row['evidence_id']}: {side}.canonical_model_id is required")
+            stored = contract.get("contract_fingerprint")
+            if not stored:
+                raise RegistryError(f"{row['evidence_id']}: {side}.contract_fingerprint is required")
+            expected = contract_fingerprint(contract)
+            if str(stored) != expected:
+                raise RegistryError(f"{row['evidence_id']}: {side} contract fingerprint does not match contract fields")
+            if contract.get("dimension") is not None:
+                _positive_integer(contract["dimension"], f"{row['evidence_id']}: {side}.dimension")
+        dataset = row["dataset"]
+        if not isinstance(dataset, dict) or not str(dataset.get("name", "")).strip():
+            raise RegistryError(f"{row['evidence_id']}: dataset.name is required")
+        for key in ("corpus_size", "query_count"):
+            if dataset.get(key) is not None:
+                _positive_integer(dataset[key], f"{row['evidence_id']}: dataset.{key}", allow_zero=True)
+        metric = str(row["metric"])
+        if metric.lower() not in {"ndcg@10", "ndcg@k"}:
+            raise RegistryError(f"{row['evidence_id']}: unsupported metric {metric!r}")
+        epsilon = _finite(row.get("epsilon", 0.01), f"{row['evidence_id']}.epsilon")
+        if epsilon < 0:
+            raise RegistryError(f"{row['evidence_id']}: epsilon cannot be negative")
+        gaps = row.get("candidate_gap", {}) or {}
+        if not isinstance(gaps, dict):
+            raise RegistryError(f"{row['evidence_id']}: candidate_gap must be an object")
+        depths = []
+        for key, value in gaps.items():
+            try:
+                depth = _positive_integer(key, f"{row['evidence_id']}: candidate depth {key!r}")
+            except RegistryError as exc:
+                raise RegistryError(f"{row['evidence_id']}: invalid candidate depth {key!r}") from exc
+            if str(depth) != str(key):
+                raise RegistryError(f"{row['evidence_id']}: candidate depths must be positive canonical integers")
+            _finite(value, f"{row['evidence_id']}.candidate_gap[{key}]")
+            depths.append(depth)
+        if len(depths) != len(set(depths)):
+            raise RegistryError(f"{row['evidence_id']}: candidate_gap depths must be unique")
+        containment = row.get("containment", {}) or {}
+        if not isinstance(containment, dict):
+            raise RegistryError(f"{row['evidence_id']}: containment must be an object")
+        for key, value in containment.items():
+            try:
+                containment_depth = _positive_integer(key, f"{row['evidence_id']}: containment depth {key!r}")
+            except RegistryError as exc:
+                raise RegistryError(f"{row['evidence_id']}: containment has invalid K={key!r}") from exc
+            if containment_depth not in depths:
+                raise RegistryError(f"{row['evidence_id']}: containment has unknown K={key}")
+            fraction = _finite(value, f"{row['evidence_id']}.containment[{key}]")
+            if not 0.0 <= fraction <= 1.0:
+                raise RegistryError(f"{row['evidence_id']}: containment must be in [0, 1]")
+        declared_depths = row.get("candidate_depths")
+        if declared_depths is not None:
+            if not isinstance(declared_depths, list):
+                raise RegistryError(f"{row['evidence_id']}: candidate_depths must be a list")
+            parsed_declared = [_positive_integer(value, f"{row['evidence_id']}: candidate_depths item") for value in declared_depths]
+            if parsed_declared != sorted(set(parsed_declared)):
+                raise RegistryError(f"{row['evidence_id']}: candidate_depths must be sorted and unique")
+            if parsed_declared != sorted(depths):
+                raise RegistryError(f"{row['evidence_id']}: candidate_depths disagrees with candidate_gap keys")
+        observed = row.get("observed_migration_depth")
+        if observed is not None:
+            observed = _positive_integer(observed, f"{row['evidence_id']}: observed migration depth")
+            if observed not in depths:
+                raise RegistryError(f"{row['evidence_id']}: observed migration depth is not in candidate_gap")
+            expected = next((k for k in sorted(depths) if float(gaps[str(k)]) <= epsilon), None)
+            if expected != observed:
+                raise RegistryError(f"{row['evidence_id']}: observed migration depth disagrees with candidate_gap and epsilon")
+        ci = row.get("ci_certified_migration_depth")
+        if ci is not None and _positive_integer(ci, f"{row['evidence_id']}: CI-certified depth") not in depths:
+            raise RegistryError(f"{row['evidence_id']}: CI-certified depth is not in candidate_gap")
+        provenance = row["provenance"]
+        if not isinstance(provenance, dict) or not str(provenance.get("artifact", "")).strip():
+            raise RegistryError(f"{row['evidence_id']}: provenance.artifact is required")
+        digest = provenance.get("artifact_sha256")
+        if digest is not None and (not isinstance(digest, str) or len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest.lower())):
+            raise RegistryError(f"{row['evidence_id']}: provenance.artifact_sha256 must be a SHA-256 digest")
+        return cls(row)
+
+
+@dataclass(frozen=True)
+class BenchmarkProfile:
+    """Measured latency/throughput profile kept separate from compatibility evidence."""
+
+    raw: dict[str, Any]
+
+    @property
+    def profile_id(self) -> str:
+        return str(self.raw["profile_id"])
+
+    def to_dict(self) -> dict[str, Any]:
+        return json.loads(json.dumps(self.raw, sort_keys=True))
+
+    @classmethod
+    def from_dict(cls, raw: Mapping[str, Any]) -> BenchmarkProfile:
+        if not isinstance(raw, Mapping):
+            raise RegistryError("benchmark profile must be an object")
+        try:
+            row = json.loads(json.dumps(dict(raw), ensure_ascii=False))
+        except (TypeError, ValueError) as exc:
+            raise RegistryError("benchmark profile must contain JSON-serializable values") from exc
+        for key in ("registry_version", "profile_id", "kind", "provenance", "measurements"):
+            if key not in row:
+                raise RegistryError(f"benchmark profile missing {key!r}")
+        if not str(row["profile_id"]).strip():
+            raise RegistryError("benchmark profile profile_id must be non-empty")
+        if not str(row["kind"]).strip():
+            raise RegistryError(f"{row['profile_id']}: kind must be non-empty")
+        if str(row["registry_version"]) != REGISTRY_VERSION:
+            raise RegistryError(f"{row['profile_id']}: unsupported registry_version")
+        if not isinstance(row["measurements"], dict):
+            raise RegistryError(f"{row['profile_id']}: measurements must be an object")
+        if not isinstance(row["provenance"], dict) or not row["provenance"].get("artifact"):
+            raise RegistryError(f"{row['profile_id']}: provenance.artifact is required")
+        digest = row["provenance"].get("artifact_sha256")
+        if digest is not None and (not isinstance(digest, str) or len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest.lower())):
+            raise RegistryError(f"{row['profile_id']}: provenance.artifact_sha256 must be a SHA-256 digest")
+
+        def validate_measurement(value: Any, label: str) -> None:
+            if isinstance(value, Mapping):
+                for key, child in value.items():
+                    validate_measurement(child, f"{label}.{key}")
+                return
+            if value is None:
+                return
+            if isinstance(value, bool):
+                raise RegistryError(f"{row['profile_id']}: {label} must be numeric")
+            try:
+                numeric = float(value)
+            except (TypeError, ValueError) as exc:
+                raise RegistryError(f"{row['profile_id']}: {label} must be numeric") from exc
+            if not math.isfinite(numeric):
+                raise RegistryError(f"{row['profile_id']}: {label} must be finite")
+
+        validate_measurement(row["measurements"], "measurements")
+        return cls(row)
+
+
+def canonical_json_hash(value: Any) -> str:
+    """Hash a JSON value for manifest/checksum use."""
+    payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
+    return hashlib.sha256(payload).hexdigest()

embedflow/runtime.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from .cache import SQLiteVectorCache
+from .config import EmbedFlowConfig, load_config
+from .indexes import FaissIndex, NumpyIndex, QdrantIndex
+from .migration.state import DocumentStore
+from .models import load_embedding_model
+from .serving.engine import MigrationEngine
+
+
+def load_documents(cfg: EmbedFlowConfig) -> DocumentStore:
+    return DocumentStore(cfg.documents.path, cfg.documents.id_field, cfg.documents.text_field)
+
+
+def load_index(cfg: EmbedFlowConfig, documents: DocumentStore):
+    metadata = documents.documents
+    if cfg.index.backend.lower() == "faiss":
+        try:
+            return FaissIndex.load(cfg.index.path, ids_path=cfg.index.ids, metric=cfg.index.metric,
+                                   documents=metadata, nprobe=cfg.index.nprobe)
+        except (RuntimeError, ValueError) as exc:
+            # A NumPy fallback index may be opened on a machine that happens
+            # to have FAISS installed.  Try the fallback for any FAISS read
+            # error, but preserve the original exception when the file is not
+            # a valid NumPy artifact either.
+            try:
+                return NumpyIndex.load(cfg.index.path, metric=cfg.index.metric, documents=metadata)
+            except Exception as fallback_exc:
+                raise exc from fallback_exc
+    return QdrantIndex.connect(cfg.index.path if not cfg.index.url else cfg.index.url,
+                               cfg.index.collection, int(cfg.source.dimension or 0), documents=metadata,
+                               metric=cfg.index.metric, api_key_env=cfg.index.api_key_env,
+                               vector_name=cfg.index.vector_name)
+
+
+def open_engine(config_path: str | Path, device: str | None = None, demo: bool = False,
+                start_worker: bool = True) -> MigrationEngine:
+    cfg = load_config(config_path)
+    if cfg.index.metric.lower() == "cosine" and (cfg.source.normalization.lower() != "l2" or cfg.target.normalization.lower() != "l2"):
+        raise ValueError("cosine index/reranking requires l2-normalized source and target vectors")
+    documents = load_documents(cfg)
+    override_device = device
+    source_device = override_device or cfg.source.device or cfg.target.device or "cpu"
+    target_device = override_device or cfg.target.device or cfg.source.device or "cpu"
+    source_model = None
+    target_model = None
+    source_index = None
+    cache = None
+    try:
+        source_model = load_embedding_model(cfg.source, model_root=Path(config_path).parent / "models", device=source_device, demo=demo)
+        if cfg.source.dimension and int(source_model.dimension) != int(cfg.source.dimension):
+            raise ValueError(f"source model dimension {source_model.dimension} != configured {cfg.source.dimension}")
+        source_index = load_index(cfg, documents)
+        if source_index.size() <= 0:
+            raise ValueError("legacy index is empty or its configured Qdrant collection is unavailable")
+        if int(source_index.dimension) != int(source_model.dimension):
+            raise ValueError(f"source model dimension {source_model.dimension} != existing index dimension {source_index.dimension}")
+        index_ids = getattr(source_index, "ids", None)
+        if index_ids is not None:
+            missing_text = sorted(set(map(str, index_ids)) - set(documents.documents))
+            if missing_text:
+                raise ValueError(f"legacy index references {len(missing_text)} documents missing from the document store (e.g. {missing_text[:3]})")
+        stored_fingerprint = source_index.metadata().get("model_fingerprint")
+        if stored_fingerprint and stored_fingerprint != source_model.fingerprint:
+            raise ValueError("source model fingerprint does not match the existing index contract")
+        target_model = load_embedding_model(cfg.target, model_root=Path(config_path).parent / "models", device=target_device, demo=demo)
+        if cfg.target.dimension and int(target_model.dimension) != int(cfg.target.dimension):
+            raise ValueError(f"target model dimension {target_model.dimension} != configured {cfg.target.dimension}")
+        cache = SQLiteVectorCache(cfg.cache.path, target_model.fingerprint, target_model.dimension)
+        probe = {}
+        probe_path = Path(cfg.state_path).with_name("probe_result.json")
+        if probe_path.exists():
+            try:
+                loaded_probe = json.loads(probe_path.read_text())
+            except (OSError, json.JSONDecodeError) as exc:
+                raise ValueError(f"invalid probe result {probe_path}") from exc
+            if not isinstance(loaded_probe, dict):
+                raise ValueError(f"probe result {probe_path} must contain a JSON object")
+            probe = loaded_probe
+        return MigrationEngine(cfg, source_model, target_model, source_index, cache, documents,
+                               probe=probe, ann_status="UNKNOWN", start_worker=start_worker)
+    except Exception:
+        if cache is not None:
+            cache.close()
+        if target_model is not None:
+            target_model.close()
+        if source_model is not None:
+            source_model.close()
+        if source_index is not None:
+            try:
+                close_index = getattr(source_index, "close", None)
+                if callable(close_index):
+                    close_index()
+            except Exception:
+                pass
+        raise
+
+
+def build_faiss_from_documents(cfg: EmbedFlowConfig, model: Any, documents: DocumentStore,
+                               path: str | Path | None = None, nlist: int | None = None):
+    vectors = model.encode_documents(list(documents.documents.values()), batch_size=32)
+    target = path or cfg.index.path
+    try:
+        return FaissIndex.build(vectors, list(documents.documents), path=target,
+                                metric=cfg.index.metric, nlist=nlist, ids_path=cfg.index.ids, documents=documents.documents,
+                                metadata={"model_fingerprint": model.fingerprint, "model_id": model.model_id})
+    except RuntimeError as exc:
+        if "FAISS" not in str(exc): raise
+        return NumpyIndex.build(vectors, list(documents.documents), path=target, metric=cfg.index.metric, ids_path=cfg.index.ids,
+                                documents=documents.documents,
+                                metadata={"model_fingerprint": model.fingerprint, "model_id": model.model_id})

embedflow/serving/__init__.py

@@ -0,0 +1,3 @@
+from .engine import MigrationEngine
+
+__all__ = ["MigrationEngine"]