embedflow 0.1.0__py3-none-any.whl

embedflow/compatibility/report.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import csv
+import json
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from .candidate_gap import CandidateGapCurve, observed_k_epsilon
+
+
+def write_curve_csv(path: str | Path, curve: Sequence[CandidateGapCurve], *, field: str = "candidate_gap") -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    rows = [point.to_dict() for point in curve]
+    with path.open("w", newline="") as handle:
+        writer = csv.DictWriter(handle, fieldnames=list(rows[0]) if rows else ["k", field])
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def migration_report(
+    *,
+    source_model: str,
+    target_model: str,
+    corpus_size: int,
+    curve: Sequence[CandidateGapCurve],
+    diagnostic: str | None = None,
+    recommended_k: int | None = None,
+    ann_status: str = "UNKNOWN",
+    t2_warning: str = "T2-v1 is an empirical finite-tail diagnostic, not a compatibility guarantee.",
+    epsilon: float = 0.01,
+    native_target_index_used: bool = True,
+) -> dict[str, Any]:
+    observed = observed_k_epsilon(curve, epsilon=epsilon) if native_target_index_used else None
+    if str(diagnostic or "").upper() == "SAFE":
+        recommendation = "Progressive migration is a reasonable candidate for further deployment validation."
+    elif str(diagnostic or "").upper() == "EXPAND":
+        recommendation = "Expand source retrieval depth and validate manually before deployment."
+    else:
+        recommendation = "Prefer manual validation, a larger probe, or a full target backfill before relying on progressive migration."
+    return {
+        "schema_version": "0.1",
+        "source_model": source_model,
+        "target_model": target_model,
+        "corpus_documents": int(corpus_size),
+        "diagnostic": diagnostic,
+        "recommended_initial_k": recommended_k,
+        "observed_k_epsilon": observed,
+        "epsilon": float(epsilon),
+        "ann_status": ann_status,
+        "recommendation": recommendation,
+        "native_target_index_used": bool(native_target_index_used),
+        "t2_warning": t2_warning,
+        "candidate_gap_curve": [point.to_dict() for point in curve],
+        "limitations": [
+            "Candidate gap and containment are separate metrics.",
+            "A SAFE T2-v1 diagnostic is empirical and is not a compatibility guarantee.",
+            "ANN fidelity is UNKNOWN unless an exact/reference source comparison was supplied.",
+        ],
+    }
+
+
+def write_report(path: str | Path, report: Mapping[str, Any]) -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(dict(report), indent=2, ensure_ascii=False, default=float) + "\n")
+
+
+def report_markdown(report: Mapping[str, Any]) -> str:
+    curve = list(report.get("candidate_gap_curve", []))
+    corpus_value = report.get("corpus_documents", "unknown")
+    try:
+        corpus_display = f"{int(corpus_value):,}"
+    except (TypeError, ValueError):
+        corpus_display = str(corpus_value)
+    lines = [
+        "# EmbedFlow Migration Report", "", "## Models", "",
+        f"- Source: `{report.get('source_model', 'unknown')}`",
+        f"- Target: `{report.get('target_model', 'unknown')}`",
+        f"- Corpus documents: `{corpus_display}`",
+        "", "## Decision", "",
+        f"- T2-v1 diagnostic: **{report.get('diagnostic') or 'not run'}**",
+        f"- Recommended initial candidate depth: **K={report.get('recommended_initial_k') or 'n/a'}**",
+        f"- Observed K*_epsilon: **{report.get('observed_k_epsilon') or 'not computed'}** (epsilon={report.get('epsilon', 0.01)})",
+        f"- Finite-tail behavior: **{report.get('finite_tail_behavior', 'not established')}**",
+        f"- ANN health: **{report.get('ann_status', 'UNKNOWN')}**",
+        f"- Recommendation: {report.get('recommendation', 'Further validation required.')}",
+        "", "> SAFE is an empirical finite-tail diagnostic, not a compatibility guarantee.",
+        "", "## Candidate gap curve", "", "| K | native target | target within source candidates | G(K) | containment |", "|---:|---:|---:|---:|---:|",
+    ]
+    for row in curve:
+        lines.append(f"| {row.get('k')} | {float(row.get('native_target_quality', 0)):.4f} | {float(row.get('restricted_target_quality', 0)):.4f} | {float(row.get('candidate_gap', 0)):.4f} | {float(row.get('containment', 0)):.4f} |")
+    reused = report.get("registry_reused_values") or []
+    if reused:
+        lines.extend(["", "## Reused registry evidence", "", "> These are canonical prior measurements reused because the source/target contracts and corpus construction matched exactly. They are not a new-corpus probe result.", "", "| Evidence | Dataset | G(50) | Observed K* | CI-certified K* |", "|---|---|---:|---:|---:|"])
+        for item in reused:
+            gaps = item.get("candidate_gap") or {}
+            gap_50 = gaps.get("50", gaps.get(50, "unavailable"))
+            lines.append(f"| `{item.get('evidence_id', 'unknown')}` | {item.get('dataset', 'unknown')} | {float(gap_50):.5f} | {item.get('observed_migration_depth') or 'unavailable'} | {item.get('ci_certified_migration_depth') or 'unavailable'} |" if gap_50 != "unavailable" else f"| `{item.get('evidence_id', 'unknown')}` | {item.get('dataset', 'unknown')} | unavailable | {item.get('observed_migration_depth') or 'unavailable'} | {item.get('ci_certified_migration_depth') or 'unavailable'} |")
+    lines.extend(["", "## Limitations", "", *[f"- {item}" for item in report.get("limitations", [])], ""])
+    return "\n".join(lines)

embedflow/compatibility/t2.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class T2Diagnostic:
+    """A frozen finite-tail diagnostic, never a compatibility guarantee."""
+
+    diagnostic: str
+    features: dict[str, float]
+    implementation: str
+    warning: str = "T2-v1 is an empirical finite-tail diagnostic, not a compatibility guarantee."
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "diagnostic": self.diagnostic,
+            "features": self.features,
+            "implementation": self.implementation,
+            "warning": self.warning,
+        }
+
+
+def diagnose_t2(features: Mapping[str, Any], root: str | Path | None = None) -> T2Diagnostic:
+    """Run the repository's frozen T2-v1 rule without reimplementing it."""
+    from src.t2_v1 import decide, verify_t2_hash
+
+    if not isinstance(features, Mapping):
+        raise TypeError("T2-v1 features must be a mapping")
+    forbidden = ("qrel", "native", "ndcg", "candidate_gap", "g(", "target_rank", "label", "observed_k")
+    leaked = [str(key) for key in features if any(token in str(key).lower() for token in forbidden)]
+    if leaked:
+        raise ValueError(f"T2-v1 refuses label/native-target data in features: {leaked}")
+    required = {
+        "probe_residual_tail_50_mean", "deepest_p90", "late_tail_area",
+        "stability_to_500_50_mean", "last_shell_any_rate", "fraction_margin_nonpositive",
+    }
+    missing = sorted(required.difference(features))
+    if missing:
+        raise ValueError(f"T2-v1 feature row is missing required fields: {missing}")
+
+    if root is None:
+        root = Path(__file__).resolve().parents[2]
+        if not (Path(root) / "frozen").exists():
+            root = Path(__file__).resolve().parents[1]
+    root = Path(root)
+    frozen = root / "frozen" / "T2_V1_FROZEN_SPEC.md"
+    if frozen.exists():
+        verify_t2_hash(root)
+    scalar = {}
+    for key, value in features.items():
+        if isinstance(value, bool):
+            raise ValueError(f"T2-v1 feature {key!r} must be numeric, not boolean")
+        try:
+            scalar[str(key)] = float(value)
+        except (TypeError, ValueError) as exc:
+            raise ValueError(f"T2-v1 feature {key!r} must be numeric") from exc
+    import math
+    if not all(math.isfinite(value) for value in scalar.values()):
+        raise ValueError("T2-v1 features must be finite")
+    return T2Diagnostic(decide(scalar), scalar, "src.t2_v1.decide")

embedflow/config.py

@@ -0,0 +1,455 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+from collections.abc import Mapping
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+_UNSET = object()
+
+
+def _integer(value: Any, label: str, *, minimum: int | None = None) -> int:
+    """Validate integer-valued YAML fields without lossy coercion."""
+    if isinstance(value, bool):
+        raise ValueError(f"{label} must be an integer")
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError, OverflowError) as exc:
+        raise ValueError(f"{label} must be an integer") from exc
+    try:
+        exact = float(value) == parsed
+    except (TypeError, ValueError, OverflowError):
+        exact = False
+    if not exact or (minimum is not None and parsed < minimum):
+        qualifier = f" >= {minimum}" if minimum is not None else ""
+        raise ValueError(f"{label} must be an integer{qualifier}")
+    return parsed
+
+
+@dataclass(init=False)
+class ModelConfig:
+    model: str
+    revision: str | None = None
+    dimension: int | None = None
+    max_length: int = 512
+    pooling: str = "mean_tokens"
+    padding_side: str = "right"
+    truncation_side: str = "right"
+    query_instruction: str = ""
+    document_instruction: str = ""
+    normalization: str = "l2"
+    dtype: str = "float32"
+    local_path: str | None = None
+    device: str | None = None
+
+    def __init__(
+        self,
+        model: str,
+        revision: str | None | object = _UNSET,
+        dimension: int | None | object = _UNSET,
+        max_length: int | object = _UNSET,
+        pooling: str | object = _UNSET,
+        padding_side: str | object = _UNSET,
+        truncation_side: str | object = _UNSET,
+        query_instruction: str | object = _UNSET,
+        document_instruction: str | object = _UNSET,
+        normalization: str | object = _UNSET,
+        dtype: str | object = _UNSET,
+        local_path: str | None | object = _UNSET,
+        device: str | None | object = _UNSET,
+    ) -> None:
+        """Construct a model contract while remembering explicit overrides.
+
+        The research contracts have non-generic defaults (for example,
+        Qwen3-8B uses length 8192 while the product's generic default is
+        512).  A normal dataclass cannot distinguish an omitted value from a
+        caller explicitly requesting that generic default.  The sentinel
+        arguments let ``hydrate_research_contract`` fill only omitted fields,
+        preserving deliberate prompt/pooling/max-length changes for matching
+        and cache safety.
+        """
+        defaults = {
+            "revision": None,
+            "dimension": None,
+            "max_length": 512,
+            "pooling": "mean_tokens",
+            "padding_side": "right",
+            "truncation_side": "right",
+            "query_instruction": "",
+            "document_instruction": "",
+            "normalization": "l2",
+            "dtype": "float32",
+            "local_path": None,
+            "device": None,
+        }
+        values = {
+            "revision": revision,
+            "dimension": dimension,
+            "max_length": max_length,
+            "pooling": pooling,
+            "padding_side": padding_side,
+            "truncation_side": truncation_side,
+            "query_instruction": query_instruction,
+            "document_instruction": document_instruction,
+            "normalization": normalization,
+            "dtype": dtype,
+            "local_path": local_path,
+            "device": device,
+        }
+        self.model = str(model)
+        self._explicit_fields = {key for key, value in values.items() if value is not _UNSET}
+        for key, value in values.items():
+            setattr(self, key, defaults[key] if value is _UNSET else value)
+
+    def contract(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @property
+    def fingerprint(self) -> str:
+        # Paths and execution devices are deployment details, not embedding
+        # semantics.  Excluding them keeps a cache/index portable across
+        # machines while retaining model revision, prompts, pooling, padding,
+        # truncation, normalization, dtype, and dimension in the contract.
+        semantic = {key: value for key, value in self.contract().items() if key not in {"local_path", "device"}}
+        payload = json.dumps(semantic, sort_keys=True, separators=(",", ":"))
+        return hashlib.sha256(payload.encode()).hexdigest()
+
+
+@dataclass
+class IndexConfig:
+    backend: str = "faiss"
+    path: str = "./legacy.index"
+    metric: str = "cosine"
+    nprobe: int = 64
+    collection: str = "embedflow"
+    url: str | None = None
+    ids: str | None = None
+    vector_name: str | None = None
+    api_key_env: str | None = "QDRANT_API_KEY"
+
+
+@dataclass
+class DocumentsConfig:
+    path: str = "./documents.jsonl"
+    id_field: str = "id"
+    text_field: str = "text"
+
+
+@dataclass
+class MigrationConfig:
+    candidate_depth: int | str = 50
+    kmax_probe: int = 500
+    probe_queries: int = 100
+    max_sync_misses: int = 4
+    background_batch_size: int = 32
+    max_retries: int = 3
+    worker_count: int = 1
+
+
+@dataclass
+class CacheConfig:
+    path: str = "./embedflow_cache"
+
+
+@dataclass
+class EconomicsConfig:
+    gpu_price_per_hour: float | None = None
+    target_docs_per_second: float | None = None
+
+
+@dataclass
+class ProbeConfig:
+    """Settings for the leakage-safe finite-pool compatibility probe."""
+
+    queries: str | None = None
+    k_values: list[int] = field(default_factory=lambda: [10, 20, 50, 100, 200, 500])
+    kmax: int = 500
+    epsilon: float = 0.01
+    seed: int = 42
+    limit: int | None = None
+
+
+@dataclass
+class TelemetryConfig:
+    latency_log: str = "./logs/latency.jsonl"
+
+
+@dataclass
+class EmbedFlowConfig:
+    source: ModelConfig
+    target: ModelConfig
+    index: IndexConfig = field(default_factory=IndexConfig)
+    documents: DocumentsConfig = field(default_factory=DocumentsConfig)
+    migration: MigrationConfig = field(default_factory=MigrationConfig)
+    cache: CacheConfig = field(default_factory=CacheConfig)
+    economics: EconomicsConfig = field(default_factory=EconomicsConfig)
+    probe: ProbeConfig = field(default_factory=ProbeConfig)
+    telemetry: TelemetryConfig = field(default_factory=TelemetryConfig)
+    state_path: str = "./embedflow_state.json"
+    dashboard_title: str = "EmbedFlow — Progressive Embedding Migration"
+
+    def resolve_paths(self, base: Path) -> EmbedFlowConfig:
+        """Resolve relative paths against the configuration file directory."""
+        for obj, attr in ((self.index, "path"), (self.documents, "path"),
+                          (self.cache, "path"), (self, "state_path"),
+                          (self.telemetry, "latency_log")):
+            raw_value = str(getattr(obj, attr))
+            # A Qdrant URL is a connection endpoint, not a filesystem path.
+            # Leave it untouched so ``index: {backend: qdrant, path: https://…}``
+            # works even when no separate ``url`` field is supplied.
+            if "://" in raw_value:
+                continue
+            value = Path(raw_value)
+            if not value.is_absolute():
+                setattr(obj, attr, str((base / value).resolve()))
+        if self.source.local_path:
+            p = Path(self.source.local_path)
+            self.source.local_path = str((base / p).resolve()) if not p.is_absolute() else str(p)
+        if self.target.local_path:
+            p = Path(self.target.local_path)
+            self.target.local_path = str((base / p).resolve()) if not p.is_absolute() else str(p)
+        if self.probe.queries:
+            p = Path(self.probe.queries)
+            self.probe.queries = str((base / p).resolve()) if not p.is_absolute() else str(p)
+        if self.index.ids:
+            p = Path(self.index.ids)
+            self.index.ids = str((base / p).resolve()) if not p.is_absolute() else str(p)
+        return self
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def validate(self) -> None:
+        for label, model in (("source", self.source), ("target", self.target)):
+            if not model.model.strip():
+                raise ValueError(f"{label}.model is required")
+            if model.dimension is not None:
+                model.dimension = _integer(model.dimension, f"{label}.dimension", minimum=1)
+            model.max_length = _integer(model.max_length, f"{label}.max_length", minimum=1)
+            if not isinstance(model.normalization, str) or model.normalization.lower() not in {"l2", "none", "identity"}:
+                raise ValueError(f"{label}.normalization must be l2, none, or identity")
+        if self.source.fingerprint == self.target.fingerprint:
+            raise ValueError("source and target embedding contracts must differ for migration")
+        if not isinstance(self.index.backend, str) or self.index.backend.lower() not in {"faiss", "qdrant"}:
+            raise ValueError("index.backend must be faiss or qdrant")
+        if not isinstance(self.index.metric, str) or self.index.metric.lower() not in {"cosine", "dot", "inner_product"}:
+            raise ValueError("index.metric must be cosine, dot, or inner_product")
+        candidate_depth = self.migration.candidate_depth
+        if isinstance(candidate_depth, str) and candidate_depth.strip().lower() == "auto":
+            candidate_depth = "auto"
+            self.migration.candidate_depth = "auto"
+        if candidate_depth != "auto":
+            candidate_depth = _integer(candidate_depth, "migration.candidate_depth", minimum=1)
+            self.migration.candidate_depth = candidate_depth
+        self.migration.kmax_probe = _integer(self.migration.kmax_probe, "migration.kmax_probe", minimum=1)
+        self.migration.probe_queries = _integer(self.migration.probe_queries, "migration.probe_queries", minimum=1)
+        if candidate_depth != "auto" and self.migration.kmax_probe < int(candidate_depth):
+            raise ValueError("migration.kmax_probe must be >= candidate_depth")
+        self.migration.max_sync_misses = _integer(self.migration.max_sync_misses, "migration.max_sync_misses", minimum=0)
+        self.migration.background_batch_size = _integer(self.migration.background_batch_size, "migration.background_batch_size", minimum=1)
+        self.migration.max_retries = _integer(self.migration.max_retries, "migration.max_retries", minimum=1)
+        self.migration.worker_count = _integer(self.migration.worker_count, "migration.worker_count", minimum=1)
+        self.index.nprobe = _integer(self.index.nprobe, "index.nprobe", minimum=1)
+        self.probe.kmax = _integer(self.probe.kmax, "probe.kmax", minimum=10)
+        if self.probe.kmax < 10:
+            raise ValueError("probe.kmax must be at least 10")
+        if not self.probe.k_values:
+            raise ValueError("probe.k_values must contain positive integers")
+        try:
+            self.probe.k_values = [_integer(k, "probe.k_values item", minimum=1) for k in self.probe.k_values]
+        except ValueError as exc:
+            raise ValueError("probe.k_values must contain positive integers") from exc
+        if len(set(self.probe.k_values)) != len(self.probe.k_values):
+            raise ValueError("probe.k_values must not contain duplicates")
+        if any(k > int(self.probe.kmax) for k in self.probe.k_values):
+            raise ValueError("probe.k_values cannot exceed probe.kmax")
+        self.probe.epsilon = float(self.probe.epsilon)
+        if not math.isfinite(self.probe.epsilon) or self.probe.epsilon < 0:
+            raise ValueError("probe.epsilon must be finite and non-negative")
+
+
+def _model(raw: dict[str, Any], fallback: str) -> ModelConfig:
+    raw = dict(raw or {})
+    model = raw.pop("model", raw.pop("model_id", fallback))
+    # Accept the research project's names and hydrate their exact contracts.
+    return ModelConfig(model=str(model), **raw)
+
+
+def from_dict(raw: dict[str, Any]) -> EmbedFlowConfig:
+    if not isinstance(raw, Mapping):
+        raise ValueError("configuration root must be a YAML object")
+    raw = dict(raw or {})
+    source = hydrate_research_contract(_model(raw.get("source", {}), ""))
+    target = hydrate_research_contract(_model(raw.get("target", {}), ""))
+    migration_raw = dict(raw.get("migration", {}))
+    if isinstance(migration_raw.get("candidate_depth"), str) and migration_raw["candidate_depth"].strip().lower() == "auto":
+        migration_raw["candidate_depth"] = "auto"
+    cfg = EmbedFlowConfig(
+        source=source,
+        target=target,
+        index=IndexConfig(**dict(raw.get("index", {}))),
+        documents=DocumentsConfig(**dict(raw.get("documents", {}))),
+        migration=MigrationConfig(**migration_raw),
+        cache=CacheConfig(**dict(raw.get("cache", {}))),
+        economics=EconomicsConfig(**dict(raw.get("economics", {}))),
+        probe=ProbeConfig(**dict(raw.get("probe", {}))),
+        telemetry=TelemetryConfig(**dict(raw.get("telemetry", {}))),
+        state_path=str(raw.get("state_path", "./embedflow_state.json")),
+        dashboard_title=str(raw.get("dashboard_title", EmbedFlowConfig.__dataclass_fields__["dashboard_title"].default)),
+    )
+    cfg.validate()
+    return cfg
+
+
+def hydrate_research_contract(model: ModelConfig, project_root: Path | None = None) -> ModelConfig:
+    """Use the frozen research contract for one of its registered model keys."""
+    # Keep the public package self-contained.  The research checkout also has
+    # these contracts in ``config/nq_5090_1m.yaml``; the built-ins ensure that
+    # installing EmbedFlow from a clean Git clone still preserves the exact
+    # pooling, prompt, padding, and revision semantics for known checkpoints.
+    builtin = {
+        "sentence-transformers/all-MiniLM-L6-v2": {
+            "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
+            "dimension": 384, "max_length": 512, "pooling": "mean_tokens",
+            "padding_side": "right", "truncation_side": "right",
+            "query_instruction": "", "document_instruction": "", "normalization": "l2",
+        },
+        "Qwen/Qwen3-Embedding-0.6B": {
+            "revision": "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3",
+            "dimension": 1024, "max_length": 512, "pooling": "last_non_padding_token",
+            "padding_side": "left", "truncation_side": "right",
+            "query_instruction": "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: {text}",
+            "document_instruction": "", "normalization": "l2",
+        },
+        "Qwen/Qwen3-Embedding-4B": {
+            "revision": "5cf2132abc99cad020ac570b19d031efec650f2b",
+            "dimension": 2560, "max_length": 8192, "pooling": "last_non_padding_token",
+            "padding_side": "left", "truncation_side": "right",
+            "query_instruction": "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:{text}",
+            "document_instruction": "", "normalization": "l2",
+        },
+        "Qwen/Qwen3-Embedding-8B": {
+            "revision": "1d8ad4ca9b3dd8059ad90a75d4983776a23d44af",
+            "dimension": 4096, "max_length": 8192, "pooling": "last_non_padding_token",
+            "padding_side": "left", "truncation_side": "right",
+            "query_instruction": "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:{text}",
+            "document_instruction": "", "normalization": "l2",
+        },
+    }
+    if model.model in builtin:
+        for field_name, value in builtin[model.model].items():
+            # An explicit user value wins, while dataclass defaults are filled
+            # from the frozen contract.  ``revision``/dimension are especially
+            # important for cache and index safety.
+            current = getattr(model, field_name)
+            default = ModelConfig.__dataclass_fields__[field_name].default
+            explicit = field_name in getattr(model, "_explicit_fields", set())
+            if not explicit and (current is None or current == default or field_name in {"query_instruction", "document_instruction"} and not current):
+                setattr(model, field_name, value)
+    try:
+        import yaml
+        root = project_root or Path(__file__).resolve().parents[1]
+        config_path = root / "config" / "nq_5090_1m.yaml"
+        if config_path.exists():
+            raw = yaml.safe_load(config_path.read_text()) or {}
+            for key, item in (raw.get("models") or {}).items():
+                if item.get("model_id") == model.model or key == model.model:
+                    known = dict(item)
+                    known.pop("min_vram_gib", None)
+                    known.pop("dtype_preference", None)
+                    known["model"] = known.pop("model_id", model.model)
+                    # The research YAML calls Qwen pooling `last_token`; the
+                    # product encoder accepts that as last_non_padding_token.
+                    if known.get("pooling") == "last_token":
+                        known["pooling"] = "last_non_padding_token"
+                    for field_name in ModelConfig.__dataclass_fields__:
+                        if field_name in known and field_name not in getattr(model, "_explicit_fields", set()):
+                            setattr(model, field_name, known[field_name])
+                    return model
+    except Exception:
+        # Generic Hugging Face models remain usable when the research config is
+        # not present. The caller can still inspect the resulting fingerprint.
+        pass
+    return model
+
+
+def load_config(path: str | Path) -> EmbedFlowConfig:
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(path)
+    import yaml
+    try:
+        parsed = yaml.safe_load(path.read_text())
+    except yaml.YAMLError as exc:
+        raise ValueError(f"invalid YAML configuration: {path}: {exc}") from exc
+    cfg = from_dict(parsed or {})
+    _apply_environment_overrides(cfg)
+    cfg.validate()
+    return cfg.resolve_paths(path.parent.resolve())
+
+
+def _apply_environment_overrides(cfg: EmbedFlowConfig) -> None:
+    """Apply deliberately small, documented deployment overrides.
+
+    YAML remains the source of truth for experiments. These variables are
+    useful for containerized serving and secrets-free deployment manifests;
+    unset variables leave the file untouched.
+    """
+    paths: dict[str, tuple[Any, str]] = {
+        "EMBEDFLOW_SOURCE_MODEL": (cfg.source, "model"),
+        "EMBEDFLOW_TARGET_MODEL": (cfg.target, "model"),
+        "EMBEDFLOW_SOURCE_DEVICE": (cfg.source, "device"),
+        "EMBEDFLOW_TARGET_DEVICE": (cfg.target, "device"),
+        "EMBEDFLOW_INDEX_BACKEND": (cfg.index, "backend"),
+        "EMBEDFLOW_INDEX_PATH": (cfg.index, "path"),
+        "EMBEDFLOW_INDEX_URL": (cfg.index, "url"),
+        "EMBEDFLOW_INDEX_COLLECTION": (cfg.index, "collection"),
+        "EMBEDFLOW_INDEX_VECTOR_NAME": (cfg.index, "vector_name"),
+        "EMBEDFLOW_QDRANT_API_KEY_ENV": (cfg.index, "api_key_env"),
+        "EMBEDFLOW_DOCUMENTS_PATH": (cfg.documents, "path"),
+        "EMBEDFLOW_CACHE_PATH": (cfg.cache, "path"),
+        "EMBEDFLOW_STATE_PATH": (cfg, "state_path"),
+        "EMBEDFLOW_LATENCY_LOG": (cfg.telemetry, "latency_log"),
+    }
+    for variable, (target, field_name) in paths.items():
+        value = os.environ.get(variable)
+        if value is not None and value.strip():
+            setattr(target, field_name, value.strip())
+    candidate = os.environ.get("EMBEDFLOW_CANDIDATE_DEPTH")
+    if candidate:
+        if candidate.strip().lower() == "auto":
+            cfg.migration.candidate_depth = "auto"
+        else:
+            try:
+                parsed = int(candidate)
+                if float(candidate) != parsed:
+                    raise ValueError
+            except (TypeError, ValueError, OverflowError) as exc:
+                raise ValueError("EMBEDFLOW_CANDIDATE_DEPTH must be auto or an integer") from exc
+            cfg.migration.candidate_depth = parsed
+    for variable, target, field_name in (
+        ("EMBEDFLOW_MAX_SYNC_MISSES", cfg.migration, "max_sync_misses"),
+        ("EMBEDFLOW_BACKGROUND_BATCH_SIZE", cfg.migration, "background_batch_size"),
+        ("EMBEDFLOW_PROBE_KMAX", cfg.probe, "kmax"),
+        ("EMBEDFLOW_INDEX_NPROBE", cfg.index, "nprobe"),
+    ):
+        value = os.environ.get(variable)
+        if value is not None and value.strip():
+            try:
+                parsed = int(value)
+                if float(value) != parsed:
+                    raise ValueError
+            except (TypeError, ValueError, OverflowError) as exc:
+                raise ValueError(f"{variable} must be an integer") from exc
+            setattr(target, field_name, parsed)
+
+
+def save_config(cfg: EmbedFlowConfig, path: str | Path) -> None:
+    import yaml
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(yaml.safe_dump(cfg.to_dict(), sort_keys=False))

embedflow/data/__init__.py

@@ -0,0 +1 @@
+"""Packaged, small data assets for EmbedFlow."""

embedflow/data/registry/__init__.py

@@ -0,0 +1 @@
+"""Core migration evidence shipped with EmbedFlow."""

embedflow/data/registry/benchmark_profiles.jsonl

@@ -0,0 +1,3 @@
+{"registry_version":"0.1.0","profile_id":"latency_25k_qwen3_4b_to_8b_k50_nprobe64_warm","kind":"serving_latency","status":"measured_workload_specific","scope":"25K-document serving benchmark","source":{"canonical_model_id":"Qwen/Qwen3-Embedding-4B","contract_fingerprint":"0607b87cc200f80bfc498e595891fb533f42677355b18279be42d7d988896c44"},"target":{"canonical_model_id":"Qwen/Qwen3-Embedding-8B","contract_fingerprint":"11d6d071a07a73b0bb3588a49447bb8bf58cb5426af732b534d374581ac4b018"},"dataset":{"name":"Natural Questions canonical 25K prefix","corpus_size":25000,"query_count":1000,"split":"latency_25k","fingerprint":null},"configuration":{"K":50,"nprobe":64,"target_nprobe":64,"cache_state":"warm","index_type":"ivf_flat"},"measurements":{"p50_ms":38.792666,"p95_ms":42.68700545,"added_vs_native_p50_ms":19.8086975,"added_vs_native_p95_ms":21.12569415,"mean_stage_ms":{"source_query_encode":19.48547872,"source_ann":0.401102655,"target_query_encode":18.742604425,"candidate_cache_lookup":0.213127613,"target_score":0.061910947,"topk":0.06689177}},"hardware":{"gpu":"unspecified in retained CSV","peak_gpu_memory_mb":37917.7666015625},"provenance":{"artifact":"remote_scale_experiment/experiments/latency/results_25k/latency_summary.csv","artifact_sha256":"13136c9efa90a569d8a771237a3f8536934ea94111a05701872a7c0ba3674416","table":"latency_summary.csv","status":"retained_external","git_commit":null},"notes":["Measured on a 25K-document workload; the retained CSV does not identify the GPU, so this row must not be labeled an H100 benchmark.","Latency is hardware- and workload-dependent; it is not a million-document latency claim."]}
+{"registry_version":"0.1.0","profile_id":"latency_25k_native_qwen3_8b","kind":"serving_latency","status":"measured_workload_specific","scope":"25K-document serving benchmark","source":{"canonical_model_id":"native_target","contract_fingerprint":null},"target":{"canonical_model_id":"Qwen/Qwen3-Embedding-8B","contract_fingerprint":"11d6d071a07a73b0bb3588a49447bb8bf58cb5426af732b534d374581ac4b018"},"dataset":{"name":"Natural Questions canonical 25K prefix","corpus_size":25000,"query_count":1000,"split":"latency_25k","fingerprint":null},"configuration":{"cache_state":"not_applicable","index_type":"ivf_flat","target_nprobe":64},"measurements":{"p50_ms":18.9839685,"p95_ms":21.5613113},"hardware":{"gpu":"unspecified in retained CSV","peak_gpu_memory_mb":37917.7666015625},"provenance":{"artifact":"remote_scale_experiment/experiments/latency/results_25k/latency_summary.csv","artifact_sha256":"13136c9efa90a569d8a771237a3f8536934ea94111a05701872a7c0ba3674416","table":"latency_summary.csv","status":"retained_external","git_commit":null},"notes":["Native target baseline for the same 25K workload; hardware identity is not retained in this CSV."]}
+{"registry_version":"0.1.0","profile_id":"throughput_qwen3_8b_document_encoding_profile","kind":"document_encoding_throughput","status":"measured_workload_specific","scope":"retained model-profile benchmark","source":{"canonical_model_id":"Qwen/Qwen3-Embedding-8B","contract_fingerprint":"11d6d071a07a73b0bb3588a49447bb8bf58cb5426af732b534d374581ac4b018"},"target":null,"dataset":{"name":"profile fixture (dataset details not retained)","corpus_size":null,"query_count":null,"split":null,"fingerprint":null},"configuration":{"repetitions":5,"hardware":"not recorded in profile JSON"},"measurements":{"best_rows_per_second":331.19314884980446,"best_batch_size":40,"best_mean_seconds":0.1207754452014342,"all_batch_rows_per_second":{"1":27.222915766054818,"2":54.38365517199809,"4":109.39658538220068,"8":193.1738374840373,"12":247.2968027380519,"16":262.2610551452983,"20":301.8062592513787,"24":270.37271409519764,"32":317.50042263060874,"40":331.19314884980446,"50":311.4730361008796}},"hardware":{"gpu":"not recorded in profile JSON"},"provenance":{"artifact":"iclr_candidate_compatibility/remote_outputs/profiles/qwen3_8b_latency.json","artifact_sha256":"f46dc680f87cf0d0a9ea96fcd25c3ddf20a92afd84acda72f7ca72a918950a30","table":"batch_sizes","status":"retained_external","git_commit":null},"notes":["Measured workload-specific model-profile throughput. The often-used 106.98 docs/s value was not traceable to a retained measured artifact and is intentionally not included.","Do not use this profile as a universal H100 rate; supply your own measured throughput for economics projections."]}

embedflow/data/registry/checksums.sha256

@@ -0,0 +1,4 @@
+c3d851bcfe66d9cc1aabc4ec8b61512110d8802ac0bcc7af3239a9ac6cec14a9  schema_version.json
+1471e7a937d87eeba58d19a801fb8c147f6982d2c1522b5da43f6986b92800d5  migrations.jsonl
+fcbf2fa0456fc3fc2e30d1b945b05e7d7dcfa11b7187910ced8a3fa0e69f6720  research_summaries.json
+4ef2bdfc9f1803ef108400986be6a06915879d302e14a2757fc091c32b5b6e54  benchmark_profiles.jsonl