embedprobe 0.0.0__tar.gz → 0.1.0__tar.gz

embedprobe-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,22 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install package
+        run: python -m pip install -e ".[dev]"
+      - name: Run tests
+        run: pytest

embedprobe-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+context
+CLAUDE.md
+.venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+__pycache__/
+*.py[cod]
+.claude
+.agents
+

embedprobe-0.1.0/PKG-INFO

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: embedprobe
+Version: 0.1.0
+Summary: A diagnostic toolkit for evaluating and selecting multilingual embedding models on your data.
+Project-URL: Homepage, https://github.com/Sainath26/embedprobe
+Author-email: Harish Sainath S <harishsainth036@gmail.com>
+License: MIT
+Keywords: diagnostics,embeddings,evaluation,model-selection,nlp,transformers
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: matplotlib>=3.6
+Requires-Dist: numpy>=1.22
+Requires-Dist: pandas>=1.5
+Requires-Dist: rich>=13
+Requires-Dist: scikit-learn>=1.1
+Requires-Dist: scipy>=1.9
+Requires-Dist: sentence-transformers>=2.2
+Requires-Dist: typer>=0.9
+Requires-Dist: umap-learn>=0.5
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: pytest>=7; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# embedprobe
+
+**A diagnostic toolkit for evaluating and selecting multilingual embedding models on _your_ data.**
+
+Leaderboards like [MTEB](https://github.com/embeddings-benchmark/mteb) tell you _which_ embedding
+model ranks higher on average. **embedprobe tells you _why_ a model fails on your domain and
+language pair,** so you can pick a compact encoder for your task without fine-tuning every
+candidate.
+
+Given a parallel dataset (source text, target text, topic), embedprobe dissects each candidate
+model across four diagnostic levels:
+
+| Level                    | Question it answers                                         | Signals                                                                                                                     |
+| ------------------------ | ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- |
+| **0 — Signal vs. noise** | Does the model separate true pairs from random ones at all? | true-vs-random cosine distributions, SNR, Kolmogorov–Smirnov test                                                           |
+| **1 — Retrieval**        | How reliably does it retrieve the true counterpart?         | Recall@k, Precision@k, MRR, CMC curve                                                                                       |
+| **2 — Topic structure**  | Is the space organized by meaning or leaking across topics? | retrieval-based topic confusion, topic-pair average cosine, UMAP projections                                                |
+| **3 — Error taxonomy**   | _Why_ do retrievals miss?                                   | Jaccard-based categorization of misses into **lexical confusion**, **semantic confusion**, and **topic-boundary fuzziness** |
+
+The Level 3 error taxonomy is the headline: instead of a single aggregate score, each retrieval
+miss is classified by token-overlap between the retrieved and the true target, telling you whether
+a model is being fooled by surface overlap, drifting semantically, or blurring topic boundaries.
+
+## Install
+
+```bash
+pip install embedprobe
+```
+
+UMAP projection support is included for Level 2 visual diagnostics.
+
+## Quickstart
+
+```python
+import pandas as pd
+from embedprobe import probe
+
+# parallel data: one row per pair, plus a topic column
+df = pd.read_csv("pairs.csv")   # columns: en, es, topic
+
+report = probe(
+    models=["sentence-transformers/LaBSE",
+            "sentence-transformers/distiluse-base-multilingual-cased-v2"],
+    data=df,
+    src_col="en",
+    tgt_col="es",
+    seed=42,
+)
+
+report.summary()            # cross-model DataFrame of all metrics
+report.to_json("report.json")
+report.to_html("report.html")   # self-contained diagnostic dashboard
+```
+
+Or from the command line:
+
+```bash
+embedprobe run --models sentence-transformers/LaBSE --data pairs.csv \
+    --src en --tgt es --out report
+```
+
+## Status
+
+Pre-release (`0.x`). The toolkit originates from an MSc dissertation study of 21
+sentence-transformer models across EN–ES, EN–FR and EN–ZH parallel data; the packaged version is
+being hardened for a 2026 workshop submission. APIs may change until `1.0`.
+
+## Roadmap
+
+- [ ] Empirical validation of the Jaccard taxonomy thresholds
+- [ ] Selection-prediction study: do these diagnostics predict downstream task ranking?
+- [ ] MTEB adapter: shortlist from the leaderboard → diagnose on your data
+- [ ] Decoder-only LLM support (MEXA-style alignment probing)
+
+## License
+
+MIT

embedprobe-0.1.0/README.md

@@ -0,0 +1,77 @@
+# embedprobe
+
+**A diagnostic toolkit for evaluating and selecting multilingual embedding models on _your_ data.**
+
+Leaderboards like [MTEB](https://github.com/embeddings-benchmark/mteb) tell you _which_ embedding
+model ranks higher on average. **embedprobe tells you _why_ a model fails on your domain and
+language pair,** so you can pick a compact encoder for your task without fine-tuning every
+candidate.
+
+Given a parallel dataset (source text, target text, topic), embedprobe dissects each candidate
+model across four diagnostic levels:
+
+| Level                    | Question it answers                                         | Signals                                                                                                                     |
+| ------------------------ | ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- |
+| **0 — Signal vs. noise** | Does the model separate true pairs from random ones at all? | true-vs-random cosine distributions, SNR, Kolmogorov–Smirnov test                                                           |
+| **1 — Retrieval**        | How reliably does it retrieve the true counterpart?         | Recall@k, Precision@k, MRR, CMC curve                                                                                       |
+| **2 — Topic structure**  | Is the space organized by meaning or leaking across topics? | retrieval-based topic confusion, topic-pair average cosine, UMAP projections                                                |
+| **3 — Error taxonomy**   | _Why_ do retrievals miss?                                   | Jaccard-based categorization of misses into **lexical confusion**, **semantic confusion**, and **topic-boundary fuzziness** |
+
+The Level 3 error taxonomy is the headline: instead of a single aggregate score, each retrieval
+miss is classified by token-overlap between the retrieved and the true target, telling you whether
+a model is being fooled by surface overlap, drifting semantically, or blurring topic boundaries.
+
+## Install
+
+```bash
+pip install embedprobe
+```
+
+UMAP projection support is included for Level 2 visual diagnostics.
+
+## Quickstart
+
+```python
+import pandas as pd
+from embedprobe import probe
+
+# parallel data: one row per pair, plus a topic column
+df = pd.read_csv("pairs.csv")   # columns: en, es, topic
+
+report = probe(
+    models=["sentence-transformers/LaBSE",
+            "sentence-transformers/distiluse-base-multilingual-cased-v2"],
+    data=df,
+    src_col="en",
+    tgt_col="es",
+    seed=42,
+)
+
+report.summary()            # cross-model DataFrame of all metrics
+report.to_json("report.json")
+report.to_html("report.html")   # self-contained diagnostic dashboard
+```
+
+Or from the command line:
+
+```bash
+embedprobe run --models sentence-transformers/LaBSE --data pairs.csv \
+    --src en --tgt es --out report
+```
+
+## Status
+
+Pre-release (`0.x`). The toolkit originates from an MSc dissertation study of 21
+sentence-transformer models across EN–ES, EN–FR and EN–ZH parallel data; the packaged version is
+being hardened for a 2026 workshop submission. APIs may change until `1.0`.
+
+## Roadmap
+
+- [ ] Empirical validation of the Jaccard taxonomy thresholds
+- [ ] Selection-prediction study: do these diagnostics predict downstream task ranking?
+- [ ] MTEB adapter: shortlist from the leaderboard → diagnose on your data
+- [ ] Decoder-only LLM support (MEXA-style alignment probing)
+
+## License
+
+MIT

embedprobe-0.1.0/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "embedprobe"
+version = "0.1.0"
+description = "A diagnostic toolkit for evaluating and selecting multilingual embedding models on your data."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [
+    { name = "Harish Sainath S", email = "harishsainth036@gmail.com" },
+]
+keywords = ["embeddings", "nlp", "evaluation", "diagnostics", "transformers", "model-selection"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "numpy>=1.22",
+    "pandas>=1.5",
+    "scipy>=1.9",
+    "scikit-learn>=1.1",
+    "matplotlib>=3.6",
+    "sentence-transformers>=2.2",
+    "typer>=0.9",
+    "rich>=13",
+    "umap-learn>=0.5",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=7", "build"]
+
+[project.scripts]
+embedprobe = "embedprobe.cli:app"
+
+[project.urls]
+Homepage = "https://github.com/Sainath26/embedprobe"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/embedprobe"]

embedprobe-0.1.0/src/embedprobe/__init__.py

@@ -0,0 +1,19 @@
+"""embedprobe — a diagnostic toolkit for evaluating language-model embedding spaces."""
+
+__version__ = "0.1.0"
+
+
+def __getattr__(name):
+    # Lazy imports keep `import embedprobe` cheap and avoid circular imports.
+    if name == "probe":
+        from embedprobe.probe import probe
+
+        return probe
+    if name in ("ProbeReport", "ModelDiagnostics"):
+        from embedprobe import report
+
+        return getattr(report, name)
+    raise AttributeError(f"module 'embedprobe' has no attribute {name!r}")
+
+
+__all__ = ["probe", "ProbeReport", "ModelDiagnostics", "__version__"]

embedprobe-0.1.0/src/embedprobe/cli.py

@@ -0,0 +1,85 @@
+"""Command-line interface: ``embedprobe run`` and ``embedprobe compare``."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import List, Optional
+
+from rich.console import Console
+import typer
+
+app = typer.Typer(help="Diagnose multilingual embedding models on your own data.")
+console = Console()
+
+
+@app.command()
+def run(
+    models: List[str] = typer.Option(..., "--models", "-m", help="Hub model name (repeatable)."),
+    data: Path = typer.Option(..., "--data", "-d", help="CSV/Parquet with parallel pairs."),
+    src_col: str = typer.Option(..., "--src", "--src-col", help="Source-language text column."),
+    tgt_col: str = typer.Option(..., "--tgt", "--tgt-col", help="Target-language text column."),
+    topic_col: str = typer.Option("topic", help="Topic column (Level 2 is skipped if absent)."),
+    out: Path = typer.Option(Path("embedprobe_report"), "--out", "-o", help="Output basename."),
+    seed: int = typer.Option(42, help="Random seed."),
+    batch_size: int = typer.Option(32, help="Encoding batch size."),
+    device: Optional[str] = typer.Option(None, help="Torch device, e.g. cuda."),
+    max_pairs: Optional[int] = typer.Option(None, help="Subsample the dataset to this many pairs."),
+    umap: bool = typer.Option(False, "--umap", help="Compute UMAP projections (needs umap-learn)."),
+):
+    """Run the four-level diagnostic and write <out>.json and <out>.html."""
+    from embedprobe.probe import probe
+
+    model_names = _normalize_models(models)
+    report = probe(
+        models=model_names, data=data, src_col=src_col, tgt_col=tgt_col, topic_col=topic_col,
+        seed=seed, batch_size=batch_size, device=device, max_pairs=max_pairs,
+        compute_umap=umap,
+    )
+    out.parent.mkdir(parents=True, exist_ok=True)
+    json_path, html_path = out.with_suffix(".json"), out.with_suffix(".html")
+    report.to_json(json_path)
+    report.to_html(html_path)
+    console.print(report.summary().round(4).to_string())
+    console.print(f"\n[green]Wrote[/green] {json_path} and {html_path}")
+
+
+@app.command()
+def compare(
+    reports: List[Path] = typer.Argument(..., help="embedprobe JSON reports to merge."),
+    out: Optional[Path] = typer.Option(None, "--out", "-o", help="Write merged HTML here."),
+):
+    """Merge previously saved JSON reports into one comparison."""
+    from embedprobe.report import ModelDiagnostics, ProbeReport
+
+    diagnostics, run_meta = [], {}
+    for path in reports:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+        run_meta = run_meta or payload.get("run", {})
+        for entry in payload.get("models", []):
+            diagnostics.append(
+                ModelDiagnostics(
+                    model_name=entry.get("model", path.stem),
+                    meta=entry.get("meta", {}),
+                    level0=entry.get("level0"),
+                    level1=entry.get("level1"),
+                    level2=entry.get("level2"),
+                    level3=entry.get("level3"),
+                )
+            )
+    merged = ProbeReport(models=diagnostics, run_meta=run_meta)
+    console.print(merged.summary().round(4).to_string())
+    if out is not None:
+        merged.to_html(out)
+        console.print(f"\n[green]Wrote[/green] {out}")
+
+
+def _normalize_models(models: List[str]) -> List[str]:
+    normalized = []
+    for item in models:
+        normalized.extend(part.strip() for part in item.split(",") if part.strip())
+    return normalized
+
+
+if __name__ == "__main__":
+    app()

embedprobe-0.1.0/src/embedprobe/data.py

@@ -0,0 +1,67 @@
+"""Loading and validating parallel-pair datasets."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Optional, Union
+
+import pandas as pd
+
+TOPIC_COL = "topic"
+
+
+def load_pairs(
+    source: Union[str, Path, pd.DataFrame],
+    src_col: str,
+    tgt_col: str,
+    topic_col: Optional[str] = TOPIC_COL,
+    max_pairs: Optional[int] = None,
+    seed: Optional[int] = None,
+) -> pd.DataFrame:
+    """Load a parallel dataset from a DataFrame, CSV or Parquet file.
+
+    Returns a DataFrame with the source, target and (if present) topic columns,
+    with empty/missing texts dropped and the index reset. When ``max_pairs`` is
+    given, rows are sampled reproducibly with ``seed``.
+    """
+    if isinstance(source, pd.DataFrame):
+        df = source.copy()
+    else:
+        path = Path(source)
+        if not path.exists():
+            raise FileNotFoundError(f"Dataset not found: {path}")
+        if path.suffix.lower() == ".parquet":
+            df = pd.read_parquet(path)
+        else:
+            df = pd.read_csv(path)
+
+    missing = [c for c in (src_col, tgt_col) if c not in df.columns]
+    if missing:
+        raise ValueError(
+            f"Dataset is missing required column(s) {missing}; found {list(df.columns)}"
+        )
+
+    cols = [src_col, tgt_col]
+    if topic_col is not None and topic_col in df.columns:
+        cols.append(topic_col)
+    df = df[cols]
+
+    for col in (src_col, tgt_col):
+        df = df[df[col].notna() & (df[col].astype(str).str.strip() != "")]
+
+    if max_pairs is not None and len(df) > max_pairs:
+        df = df.sample(n=max_pairs, random_state=seed)
+
+    return df.reset_index(drop=True)
+
+
+def has_topics(df: pd.DataFrame, topic_col: str = TOPIC_COL) -> bool:
+    return topic_col in df.columns and df[topic_col].notna().any()
+
+
+def dataset_hash(df: pd.DataFrame) -> str:
+    """Stable SHA-256 fingerprint of the loaded evaluation pairs."""
+    normalized = df.astype("string").fillna("")
+    payload = normalized.to_csv(index=False, lineterminator="\n").encode("utf-8")
+    return hashlib.sha256(payload).hexdigest()

embedprobe-0.1.0/src/embedprobe/embeddings.py

@@ -0,0 +1,77 @@
+"""Embedding extraction and similarity computation.
+
+Heavy dependencies (sentence-transformers / torch) are imported lazily so that
+the pure-numpy diagnostics in :mod:`embedprobe.levels` stay usable without them.
+"""
+
+from __future__ import annotations
+
+from typing import List, Optional, Sequence, Union
+
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+
+
+def load_model(model: Union[str, object]):
+    """Return a SentenceTransformer, loading it from the Hub if given a name."""
+    if isinstance(model, str):
+        from sentence_transformers import SentenceTransformer
+
+        return SentenceTransformer(model)
+    return model
+
+
+def encode(
+    model,
+    texts: Sequence[str],
+    batch_size: int = 32,
+    device: Optional[str] = None,
+    show_progress: bool = False,
+) -> np.ndarray:
+    """Encode texts with a (loaded) SentenceTransformer into a 2-D array."""
+    return model.encode(
+        [str(t) for t in texts],
+        batch_size=batch_size,
+        device=device,
+        show_progress_bar=show_progress,
+        convert_to_numpy=True,
+    )
+
+
+def pairwise_cosine(emb_a: np.ndarray, emb_b: np.ndarray) -> np.ndarray:
+    """Cosine-similarity matrix between two sets of embeddings."""
+    return cosine_similarity(emb_a, emb_b)
+
+
+def model_info(model, fallback_name: str = "") -> dict:
+    """Best-effort architecture metadata for a SentenceTransformer."""
+    info = {
+        "name": getattr(model, "model_name_or_path", None) or fallback_name,
+        "hub_id": getattr(model, "model_name_or_path", None) or fallback_name,
+        "embedding_dim": None,
+        "arch": None,
+        "layers": None,
+        "vocab_size": None,
+        "params": None,
+        "license": None,
+    }
+    try:
+        info["embedding_dim"] = int(model.get_sentence_embedding_dimension())
+    except Exception:
+        pass
+    try:
+        cfg = model._first_module().auto_model.config
+        info["arch"] = getattr(cfg, "model_type", None)
+        for key in ("num_hidden_layers", "num_layers", "n_layer"):
+            if hasattr(cfg, key):
+                info["layers"] = int(getattr(cfg, key))
+                break
+        info["vocab_size"] = getattr(cfg, "vocab_size", None)
+        info["license"] = getattr(cfg, "license", None)
+        try:
+            info["params"] = int(model._first_module().auto_model.num_parameters())
+        except Exception:
+            pass
+    except Exception:
+        pass
+    return info

embedprobe-0.1.0/src/embedprobe/levels/__init__.py

@@ -0,0 +1,15 @@
+"""The four diagnostic levels.
+
+Each level is a pure function over a precomputed similarity matrix (and texts /
+topic labels where needed), returning a dict with two keys:
+
+- ``"metrics"``: flat, JSON-serializable summary numbers
+- ``"data"``: richer arrays/records used for plotting and drill-down
+"""
+
+from embedprobe.levels.level0 import signal_to_noise
+from embedprobe.levels.level1 import retrieval_metrics
+from embedprobe.levels.level2 import topic_structure
+from embedprobe.levels.level3 import error_taxonomy
+
+__all__ = ["signal_to_noise", "retrieval_metrics", "topic_structure", "error_taxonomy"]

embedprobe-0.1.0/src/embedprobe/levels/level0.py

@@ -0,0 +1,72 @@
+"""Level 0 — signal-to-noise separability.
+
+Compares cosine similarities of true (diagonal) pairs against a matched random
+sample of off-diagonal pairs. A model that has not learned cross-lingual
+alignment shows heavily overlapping distributions and a KS p-value near 1.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+import numpy as np
+from scipy.stats import ks_2samp
+
+
+def signal_to_noise(sim_matrix: np.ndarray, seed: Optional[int] = None) -> dict:
+    sim_matrix = np.asarray(sim_matrix)
+    n = sim_matrix.shape[0]
+    if sim_matrix.shape[0] != sim_matrix.shape[1]:
+        raise ValueError("Level 0 expects a square src-x-tgt similarity matrix")
+    if n < 2:
+        raise ValueError("Need at least 2 pairs for signal-to-noise analysis")
+
+    true_sims = np.diag(sim_matrix).astype(float)
+    off_diag = sim_matrix[~np.eye(n, dtype=bool)].astype(float)
+
+    rng = np.random.RandomState(seed)
+    random_sims = rng.choice(off_diag, size=n, replace=len(off_diag) < n)
+
+    ks = ks_2samp(true_sims, random_sims)
+    random_std = float(random_sims.std())
+    snr = float((true_sims.mean() - random_sims.mean()) / random_std) if random_std > 0 else float("inf")
+    overlap = _distribution_overlap(true_sims, random_sims)
+
+    metrics = {
+        "mean_true": float(true_sims.mean()),
+        "std_true": float(true_sims.std()),
+        "mean_random": float(random_sims.mean()),
+        "std_random": random_std,
+        "snr": snr,
+        "ks_statistic": float(ks.statistic),
+        "ks_p_value": float(ks.pvalue),
+        "p_value": float(ks.pvalue),
+        "overlap_fraction": overlap,
+        "verdict": _verdict(snr, float(ks.pvalue), overlap),
+        "n_pairs": int(n),
+    }
+    data = {
+        "true_sims": true_sims.tolist(),
+        "random_sims": random_sims.tolist(),
+    }
+    return {"metrics": metrics, "data": data}
+
+
+def _distribution_overlap(a: np.ndarray, b: np.ndarray, bins: int = 50) -> float:
+    lo = float(min(a.min(), b.min()))
+    hi = float(max(a.max(), b.max()))
+    if lo == hi:
+        return 1.0
+    counts_a, edges = np.histogram(a, bins=bins, range=(lo, hi), density=True)
+    counts_b, _ = np.histogram(b, bins=edges, density=True)
+    widths = np.diff(edges)
+    overlap = np.minimum(counts_a, counts_b) * widths
+    return float(np.clip(overlap.sum(), 0.0, 1.0))
+
+
+def _verdict(snr: float, p_value: float, overlap: float) -> str:
+    if p_value < 0.01 and snr >= 2.0 and overlap <= 0.25:
+        return "strong signal"
+    if p_value < 0.05 and snr > 0.5:
+        return "weak signal"
+    return "poor separation"

embedprobe-0.1.0/src/embedprobe/levels/level1.py

@@ -0,0 +1,58 @@
+"""Level 1 — retrieval performance.
+
+For each source sentence, rank all target sentences by cosine similarity and
+locate the true counterpart. Ranks are computed on the full similarity matrix,
+so Recall@k, MRR and the CMC curve are exact for any k.
+"""
+
+from __future__ import annotations
+
+from typing import Sequence
+
+import numpy as np
+
+DEFAULT_KS = (1, 5, 10)
+
+
+def true_ranks(sim_matrix: np.ndarray) -> np.ndarray:
+    """Rank (1-based) of the true target for each source row."""
+    sim_matrix = np.asarray(sim_matrix)
+    n = sim_matrix.shape[0]
+    diag = sim_matrix[np.arange(n), np.arange(n)]
+    # Rank = 1 + number of candidates scoring strictly higher than the true one.
+    return (sim_matrix > diag[:, None]).sum(axis=1) + 1
+
+
+def retrieval_metrics(
+    sim_matrix: np.ndarray,
+    ks: Sequence[int] = DEFAULT_KS,
+    cmc_max_rank: int = 10,
+) -> dict:
+    sim_matrix = np.asarray(sim_matrix)
+    if sim_matrix.shape[0] != sim_matrix.shape[1]:
+        raise ValueError("Level 1 expects a square src-x-tgt similarity matrix")
+
+    ranks = true_ranks(sim_matrix)
+    n = len(ranks)
+
+    metrics = {"mrr": float((1.0 / ranks).mean()), "n_queries": int(n)}
+    for k in ks:
+        recall = float((ranks <= k).mean())
+        metrics[f"recall@{k}"] = recall
+        metrics[f"precision@{k}"] = recall / k
+
+    cmc_ranks = list(range(1, cmc_max_rank + 1))
+    cmc_values = [float((ranks <= k).mean()) for k in cmc_ranks]
+
+    data = {
+        "ranks": ranks.tolist(),
+        "cmc_ranks": cmc_ranks,
+        "cmc_values": cmc_values,
+        "similarity_histogram": _histogram(sim_matrix),
+    }
+    return {"metrics": metrics, "data": data}
+
+
+def _histogram(sim_matrix: np.ndarray, bins: int = 20) -> dict:
+    counts, edges = np.histogram(sim_matrix.ravel(), bins=bins)
+    return {"counts": counts.tolist(), "bin_edges": edges.tolist()}