vecshift 0.1.0__py3-none-any.whl

vecshift/__init__.py

@@ -0,0 +1,15 @@
+"""VecShift: safe, observable embedding migrations for any vector store."""
+
+from vecshift.core.capabilities import Capability
+from vecshift.core.fingerprint import EmbeddingFingerprint
+from vecshift.core.record import Record, SparseVector
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Capability",
+    "EmbeddingFingerprint",
+    "Record",
+    "SparseVector",
+    "__version__",
+]

vecshift/assets/bench.css

@@ -0,0 +1,39 @@
+/* vecshift bench leaderboard, on top of report.css */
+.picks { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(220px, 100%), 1fr)); gap: 12px; margin-top: 12px; }
+.pick { padding: 16px 18px; min-width: 0; }
+.pick .tile-label { font-weight: 600; }
+.pick .who { margin-top: 6px; font-size: 17px; font-weight: 650; overflow-wrap: anywhere; }
+.pick .why { margin-top: 2px; font-size: 13px; color: var(--ink-2); }
+
+.lb { padding: 6px 0 4px; }
+.lb .table-scroll { padding: 0 18px; }
+.lb table { margin: 0; min-width: 760px; }
+.lb th { font-size: 12px; white-space: nowrap; }
+.lb td { white-space: nowrap; vertical-align: middle; }
+.lb tr:last-child td { border-bottom: 0; }
+.lb .rank { color: var(--muted); width: 1%; }
+.lb .model { white-space: normal; min-width: 180px; }
+.lb .model b { display: block; font-weight: 600; overflow-wrap: anywhere; }
+.lb .model code { font-size: 11px; color: var(--muted); overflow-wrap: anywhere; }
+.lb .best { font-weight: 700; }
+.lb .failed td { color: var(--ink-2); }
+.recall { display: flex; align-items: center; gap: 8px; justify-content: flex-end; }
+.recall .track { width: 72px; height: 6px; border-radius: 3px; background: color-mix(in oklab, var(--series) 16%, var(--surface)); overflow: hidden; }
+.recall .track span { display: block; height: 100%; background: var(--series); border-radius: 3px; }
+
+.scatter { display: grid; grid-template-columns: auto minmax(0, 1fr); gap: 0 8px; }
+.scatter .plot { height: 220px; border-left: 1px solid var(--axis); }
+.vline { position: absolute; top: 0; bottom: 0; border-left: 1px solid var(--grid); }
+.pt {
+  position: absolute; width: 24px; height: 24px; margin: 0 0 -12px -12px; border-radius: 50%;
+  display: grid; place-items: center; font-size: 11px; font-weight: 700; color: #fff;
+  background: var(--series); box-shadow: 0 0 0 2px var(--surface); cursor: default;
+}
+.pt:hover, .pt:focus-visible { filter: brightness(1.15); z-index: 2; }
+.xaxis.log { position: relative; display: block; height: 16px; }
+.xaxis.log span { position: absolute; transform: translateX(-50%); }
+.method { padding: 16px 18px; }
+.method p { margin: 0 0 8px; color: var(--ink-2); }
+.method p:last-child { margin-bottom: 0; }
+.method .err { color: var(--ink); }
+.chart-note { margin: 8px 0 0; font-size: 12px; color: var(--muted); }

vecshift/assets/eval.css

@@ -0,0 +1,91 @@
+/* vecshift eval report, on top of report.css. Old and new are two steps of one blue:
+   the lighter one is the baseline, the stronger one the new model. */
+:root { --old: #86b6ef; --new: #2a78d6; }
+@media (prefers-color-scheme: dark) {
+  :root:not([data-theme="light"]) { --old: #1c5cab; --new: #5598e7; }
+}
+:root[data-theme="dark"] { --old: #1c5cab; --new: #5598e7; }
+@media print { :root, :root[data-theme="dark"] { --old: #86b6ef; --new: #2a78d6; } }
+
+.reasons { margin: 10px 0 0; padding-left: 18px; color: var(--ink-2); }
+.reasons li { margin: 2px 0; overflow-wrap: anywhere; }
+.reasons .warn { list-style: none; margin-left: -18px; display: flex; gap: 6px; align-items: flex-start; }
+.reasons .warn .sev { width: 16px; height: 16px; margin-top: 2px; }
+
+.delta { display: inline-flex; align-items: center; gap: 4px; }
+.delta i { font-style: normal; font-size: 11px; }
+.delta.good i { color: var(--good); }
+.delta.bad i { color: var(--critical); }
+
+.legend { display: flex; flex-wrap: wrap; gap: 6px 16px; margin: -4px 0 12px; padding: 0; list-style: none; font-size: 12px; color: var(--ink-2); }
+.legend li { display: inline-flex; align-items: center; gap: 6px; }
+.key { width: 10px; height: 10px; border-radius: 50%; display: inline-block; flex: none; }
+.key.line { width: 14px; height: 2px; border-radius: 1px; }
+.key.old { background: var(--old); }
+.key.new { background: var(--new); }
+.key.ring { background: transparent; box-shadow: inset 0 0 0 2px var(--ink-2); }
+
+/* Dumbbell: one row per language pair, old and new recall@10 on one track. */
+.db { display: grid; gap: 4px; }
+.db, .db-axis { --db-cols: minmax(96px, 30%) minmax(0, 1fr) 9.5em; }
+.db-row { display: grid; grid-template-columns: var(--db-cols); gap: 12px; align-items: center; min-height: 32px; }
+.db-label { font-size: 13px; color: var(--ink); overflow-wrap: anywhere; line-height: 1.25; }
+.db-label span { display: block; font-size: 11px; color: var(--muted); }
+.db-track { position: relative; height: 24px; }
+.db-track::before { content: ""; position: absolute; left: 0; right: 0; top: 50%; border-top: 1px solid var(--grid); }
+.db-grid { position: absolute; top: 0; bottom: 0; border-left: 1px solid var(--grid); }
+.db-line { position: absolute; top: 50%; height: 2px; margin-top: -1px; background: var(--axis); border-radius: 1px; }
+.db-dot {
+  position: absolute; top: 50%; width: 12px; height: 12px; margin: -6px 0 0 -6px; border-radius: 50%;
+  box-shadow: 0 0 0 2px var(--surface); cursor: default;
+}
+.db-dot::before { content: ""; position: absolute; inset: -6px; border-radius: 50%; }
+.db-dot.old { background: var(--old); }
+.db-dot.new { background: var(--new); z-index: 1; }
+.db-dot:hover, .db-dot:focus-visible { filter: brightness(1.12); z-index: 2; outline: none; box-shadow: 0 0 0 2px var(--surface), 0 0 0 4px var(--axis); }
+.db-value { font-size: 12px; color: var(--ink-2); white-space: nowrap; font-variant-numeric: tabular-nums; text-align: right; }
+.db-axis { display: grid; grid-template-columns: var(--db-cols); gap: 12px; }
+.db-axis .ticks { position: relative; height: 16px; font-size: 11px; color: var(--muted); font-variant-numeric: tabular-nums; }
+.db-axis .ticks span { position: absolute; transform: translateX(-50%); }
+.db-axis .ticks span:first-child { transform: none; }
+.db-axis .ticks span:last-child { transform: translateX(-100%); }
+
+/* Latency vs accuracy: index recall against search p95, one point per index setting. */
+.lc { display: grid; grid-template-columns: auto minmax(0, 1fr); gap: 0 8px; }
+.lc .yaxis { height: 220px; width: 40px; }
+.lc .plot { height: 220px; border-left: 1px solid var(--axis); }
+.lc svg { position: absolute; inset: 0; width: 100%; height: 100%; overflow: visible; pointer-events: none; }
+.lc polyline { fill: none; stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; vector-effect: non-scaling-stroke; }
+.lc polyline.old { stroke: var(--old); }
+.lc polyline.new { stroke: var(--new); }
+.lc .xaxis { position: relative; display: block; height: 16px; }
+.lc .xaxis span { position: absolute; transform: translateX(-50%); white-space: nowrap; }
+.lc .xaxis span:first-child { transform: none; }
+.lc .xaxis span:last-child { transform: translateX(-100%); }
+.lp {
+  position: absolute; width: 10px; height: 10px; margin: 0 0 -5px -5px; border-radius: 50%;
+  box-shadow: 0 0 0 2px var(--surface); cursor: default;
+}
+.lp::before { content: ""; position: absolute; inset: -7px; border-radius: 50%; }
+.lp.old { background: var(--old); }
+.lp.new { background: var(--new); z-index: 1; }
+.lp.current { box-shadow: 0 0 0 2px var(--surface), 0 0 0 4px var(--ink-2); }
+.lp:hover, .lp:focus-visible { filter: brightness(1.12); z-index: 3; outline: none; }
+.lc-label {
+  position: absolute; transform: translate(8px, 50%); font-size: 11px; color: var(--ink-2);
+  white-space: nowrap; pointer-events: none;
+}
+.lc-label.flip { transform: translate(calc(-100% - 8px), 50%); }
+
+.lat { padding: 6px 0 4px; }
+.lat .table-scroll { padding: 0 18px; }
+.lat table { margin: 0; }
+.lat tr:last-child td { border-bottom: 0; }
+.lat td:first-child { color: var(--ink-2); }
+.lat .total td { font-weight: 650; color: var(--ink); }
+.lat td.num { white-space: nowrap; }
+@media (max-width: 560px) {
+  .db, .db-axis { --db-cols: minmax(72px, 28%) minmax(0, 1fr) 7.5em; }
+  .db-row, .db-axis { gap: 8px; }
+  .db-axis .ticks span:nth-child(2), .db-axis .ticks span:nth-child(4) { display: none; }
+}

vecshift/assets/report.css

@@ -0,0 +1,252 @@
+/* vecshift doctor report. Self-contained: no fonts, scripts, or images are fetched. */
+:root {
+  color-scheme: light;
+  --page: #f9f9f7;
+  --surface: #fcfcfb;
+  --ink: #0b0b0b;
+  --ink-2: #52514e;
+  --muted: #898781;
+  --grid: #e1e0d9;
+  --axis: #c3c2b7;
+  --border: rgba(11, 11, 11, 0.1);
+  --series: #2a78d6;
+  --mark-0: #a9a8a1; --mark-1: #839ab5; --mark-2: #5b8bc7; --mark-3: #2a78d6;
+  --good: #0ca30c;
+  --warning: #fab219;
+  --critical: #d03b3b;
+  --info: #52514e;
+  --on-status: #ffffff;
+  --on-warning: #0b0b0b;
+  --shadow: 0 1px 2px rgba(11, 11, 11, 0.04), 0 4px 16px rgba(11, 11, 11, 0.04);
+  --radius: 14px;
+  --mono: ui-monospace, "SF Mono", "Cascadia Code", Menlo, Consolas, monospace;
+}
+@media (prefers-color-scheme: dark) {
+  :root:not([data-theme="light"]) {
+    color-scheme: dark;
+    --page: #0d0d0d;
+    --surface: #1a1a19;
+    --ink: #ffffff;
+    --ink-2: #c3c2b7;
+    --grid: #2c2c2a;
+    --axis: #383835;
+    --border: rgba(255, 255, 255, 0.1);
+    --series: #3987e5;
+    --mark-0: #8a8983; --mark-1: #738ba5; --mark-2: #5a8ac5; --mark-3: #3987e5;
+    --info: #c3c2b7;
+    --on-status: #ffffff;
+    --shadow: none;
+  }
+}
+:root[data-theme="dark"] {
+  color-scheme: dark;
+  --page: #0d0d0d;
+  --surface: #1a1a19;
+  --ink: #ffffff;
+  --ink-2: #c3c2b7;
+  --grid: #2c2c2a;
+  --axis: #383835;
+  --border: rgba(255, 255, 255, 0.1);
+  --series: #3987e5;
+  --mark-0: #8a8983; --mark-1: #738ba5; --mark-2: #5a8ac5; --mark-3: #3987e5;
+  --info: #c3c2b7;
+  --shadow: none;
+}
+
+* { box-sizing: border-box; }
+html { -webkit-text-size-adjust: 100%; }
+body {
+  margin: 0;
+  background: var(--page);
+  color: var(--ink);
+  font: 15px/1.5 system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
+}
+code, .mono { font-family: var(--mono); font-size: 0.92em; }
+.wrap { max-width: 1080px; margin: 0 auto; padding: 32px 16px 48px; }
+@media (min-width: 720px) { .wrap { padding: 48px 32px 64px; } }
+
+/* Header */
+.brand {
+  display: flex; align-items: center; gap: 8px;
+  color: var(--ink-2); font-size: 13px; font-weight: 600; letter-spacing: 0.02em;
+}
+h1 {
+  margin: 10px 0 6px; font-size: clamp(22px, 4vw, 30px); line-height: 1.2;
+  font-weight: 650; overflow-wrap: anywhere;
+}
+.meta { margin: 0; color: var(--ink-2); font-size: 14px; }
+.meta span + span::before { content: "·"; margin: 0 8px; color: var(--muted); }
+
+section { margin-top: 28px; }
+h2 { margin: 0 0 12px; font-size: 17px; font-weight: 650; }
+.card {
+  background: var(--surface);
+  border: 1px solid var(--border);
+  border-radius: var(--radius);
+  box-shadow: var(--shadow);
+}
+
+/* Severity icons */
+.sev { width: 18px; height: 18px; flex: none; display: block; }
+.sev-lg { width: 32px; height: 32px; }
+
+/* Verdict */
+.verdict {
+  margin-top: 24px; padding: 20px 22px;
+  display: flex; gap: 16px; align-items: flex-start;
+  background: color-mix(in oklab, var(--tone) 7%, var(--surface));
+  border-color: color-mix(in oklab, var(--tone) 28%, transparent);
+}
+.verdict[data-tone="error"] { --tone: var(--critical); }
+.verdict[data-tone="warning"] { --tone: var(--warning); }
+.verdict[data-tone="ok"] { --tone: var(--good); }
+.verdict[data-tone="info"] { --tone: var(--info); }
+.verdict > div { min-width: 0; }
+.verdict h2 { margin: 2px 0 2px; font-size: 20px; }
+.verdict p { margin: 0; color: var(--ink-2); }
+.counts { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 12px; padding: 0; list-style: none; }
+.counts li {
+  display: inline-flex; align-items: center; gap: 6px;
+  padding: 3px 10px 3px 6px; border-radius: 999px;
+  background: var(--surface); border: 1px solid var(--border);
+  font-size: 13px; color: var(--ink-2);
+}
+.counts b { color: var(--ink); font-weight: 650; }
+.counts .sev { width: 16px; height: 16px; }
+
+/* KPI tiles */
+.tiles { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(150px, 100%), 1fr)); gap: 12px; }
+.tile { padding: 16px 16px 14px; display: flex; flex-direction: column; min-width: 0; }
+.tile-label { display: flex; align-items: center; gap: 6px; font-size: 13px; color: var(--ink-2); }
+.tile-label .sev { width: 15px; height: 15px; }
+.tile-value { margin-top: 6px; font-size: 28px; line-height: 1.15; font-weight: 650; letter-spacing: -0.01em; }
+.tile-caption { margin-top: 4px; font-size: 13px; color: var(--ink-2); overflow-wrap: anywhere; }
+.meter {
+  margin-top: 12px; height: 6px; border-radius: 3px; overflow: hidden;
+  background: color-mix(in oklab, var(--fill) 18%, var(--surface));
+}
+.meter > span { display: block; height: 100%; border-radius: 3px; background: var(--fill); }
+.meter[data-state="series"] { --fill: var(--series); }
+.meter[data-state="warning"] { --fill: var(--warning); }
+.meter[data-state="error"] { --fill: var(--critical); }
+.tile .meter { margin-top: auto; }
+.tile .tile-caption + .meter { margin-top: 12px; }
+
+/* Findings */
+.filters { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 12px; }
+.filters button {
+  font: inherit; font-size: 13px; color: var(--ink-2);
+  display: inline-flex; align-items: center; gap: 6px;
+  padding: 5px 12px; border-radius: 999px; cursor: pointer;
+  background: var(--surface); border: 1px solid var(--border);
+}
+.filters button:hover:not(:disabled) { color: var(--ink); border-color: var(--axis); }
+.filters button[aria-pressed="true"] {
+  color: var(--surface); background: var(--ink); border-color: var(--ink);
+}
+.filters button:disabled { opacity: 0.45; cursor: default; }
+.filters button:focus-visible, [tabindex]:focus-visible {
+  outline: 2px solid var(--series); outline-offset: 2px;
+}
+.findings { display: grid; gap: 10px; }
+.finding { padding: 16px 18px; display: grid; grid-template-columns: auto minmax(0, 1fr); gap: 4px 12px; }
+.finding[hidden] { display: none; }
+.finding .sev { margin-top: 2px; }
+.finding-head { display: flex; flex-wrap: wrap; align-items: baseline; gap: 4px 10px; }
+.finding h3 { margin: 0; font-size: 15px; font-weight: 650; }
+.finding .label { font-size: 12px; font-weight: 600; color: var(--ink-2); text-transform: uppercase; letter-spacing: 0.04em; }
+.finding p code, .finding h3 code, .fact dd code, figcaption code { overflow-wrap: anywhere; }
+.finding .fid { margin-left: auto; font-family: var(--mono); font-size: 12px; color: var(--muted); }
+.finding p { margin: 2px 0 0; color: var(--ink-2); grid-column: 2; overflow-wrap: anywhere; }
+.finding .hint { color: var(--ink); }
+.finding .hint::before { content: "→ "; color: var(--muted); }
+
+/* Charts */
+.charts { display: grid; gap: 12px; grid-template-columns: minmax(0, 1fr); }
+@media (min-width: 760px) { .charts { grid-template-columns: repeat(2, minmax(0, 1fr)); } .charts .wide { grid-column: 1 / -1; } }
+figure { margin: 0; padding: 18px 18px 14px; min-width: 0; }
+figcaption { margin-bottom: 14px; }
+figcaption b { display: block; font-size: 15px; font-weight: 650; }
+figcaption span { font-size: 13px; color: var(--ink-2); }
+.statement { margin: 4px 0 6px; font-size: 20px; font-weight: 600; overflow-wrap: anywhere; }
+.statement + .note { margin: 0; color: var(--ink-2); font-size: 13px; }
+
+.hist { display: grid; grid-template-columns: auto minmax(0, 1fr); gap: 0 8px; }
+.yaxis { position: relative; width: 36px; height: 180px; font-size: 11px; color: var(--muted); font-variant-numeric: tabular-nums; }
+.yaxis span { position: absolute; right: 0; transform: translateY(50%); }
+.plot { position: relative; height: 180px; border-bottom: 1px solid var(--axis); }
+.gridline { position: absolute; left: 0; right: 0; border-top: 1px solid var(--grid); }
+.bins { position: absolute; inset: 0; display: flex; align-items: stretch; }
+.bin { flex: 1 1 0; display: flex; align-items: flex-end; justify-content: center; padding: 0 1px; cursor: default; }
+.bin > i {
+  display: block; width: min(24px, 100%); border-radius: 4px 4px 0 0;
+  background: var(--series); transition: filter 0.12s;
+}
+.bin:hover > i, .bin:focus-visible > i { filter: brightness(1.18) saturate(1.05); }
+.refline { position: absolute; top: 0; bottom: 0; border-left: 1px solid var(--ink-2); pointer-events: none; }
+.refline span {
+  position: absolute; top: -2px; left: 6px; white-space: nowrap;
+  font-size: 11px; color: var(--ink-2); background: var(--surface); padding: 0 3px;
+}
+.refline.flip span { left: auto; right: 6px; }
+.xaxis { grid-column: 2; display: flex; justify-content: space-between; margin-top: 6px; font-size: 11px; color: var(--muted); font-variant-numeric: tabular-nums; }
+.axis-title { grid-column: 2; text-align: center; font-size: 11px; color: var(--muted); margin-top: 2px; }
+
+.bars { display: grid; gap: 10px; }
+.bar-row {
+  display: grid; grid-template-columns: minmax(80px, 38%) minmax(0, 1fr); gap: 12px; align-items: center;
+  border-radius: 6px; cursor: default;
+}
+.bar-row:hover .bar, .bar-row:focus-visible .bar { filter: brightness(1.18) saturate(1.05); }
+.bar-label { font-size: 13px; color: var(--ink-2); overflow-wrap: anywhere; line-height: 1.3; }
+.bar-cell { display: flex; align-items: center; gap: 8px; min-width: 0; }
+.bar { height: 14px; min-width: 3px; border-radius: 0 4px 4px 0; background: var(--series); flex: none; }
+.bar-value { font-size: 12px; color: var(--ink-2); white-space: nowrap; font-variant-numeric: tabular-nums; }
+
+details.table-view { margin-top: 14px; font-size: 13px; }
+details.table-view summary { cursor: pointer; color: var(--ink-2); width: fit-content; }
+details.table-view summary:hover { color: var(--ink); }
+.table-scroll { overflow-x: auto; }
+table { border-collapse: collapse; width: 100%; margin-top: 8px; font-variant-numeric: tabular-nums; }
+th, td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--grid); }
+th { color: var(--ink-2); font-weight: 600; }
+td.num, th.num { text-align: right; }
+
+/* Setup facts */
+.facts { display: grid; grid-template-columns: minmax(0, 1fr); margin: 0; padding: 4px 18px; }
+@media (min-width: 720px) { .facts { grid-template-columns: repeat(2, minmax(0, 1fr)); column-gap: 32px; } }
+.fact { display: grid; grid-template-columns: minmax(96px, 40%) minmax(0, 1fr); gap: 12px; padding: 12px 0; border-bottom: 1px solid var(--grid); }
+.fact dt { color: var(--ink-2); font-size: 13px; }
+.fact dd { margin: 0; font-size: 14px; overflow-wrap: anywhere; }
+@media (min-width: 720px) { .fact:nth-last-child(-n + 2) { border-bottom: 0; } }
+@media (max-width: 719px) { .fact:last-child { border-bottom: 0; } }
+
+footer { margin-top: 32px; color: var(--muted); font-size: 12px; }
+footer p { margin: 4px 0; }
+
+/* Tooltip */
+.tip {
+  position: fixed; z-index: 10; pointer-events: none; display: none;
+  padding: 7px 10px; border-radius: 8px; font-size: 12px; line-height: 1.35;
+  background: var(--ink); color: var(--surface); box-shadow: 0 6px 20px rgba(0, 0, 0, 0.18);
+  max-width: 260px;
+}
+.tip strong { display: block; font-size: 13px; }
+.tip span { opacity: 0.8; }
+
+.sr-only {
+  position: absolute; width: 1px; height: 1px; padding: 0; margin: -1px;
+  overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; border: 0;
+}
+
+@media print {
+  :root, :root[data-theme="dark"] {
+    color-scheme: light;
+    --page: #ffffff; --surface: #ffffff; --ink: #0b0b0b; --ink-2: #52514e;
+    --grid: #e1e0d9; --axis: #c3c2b7; --border: rgba(11, 11, 11, 0.15);
+    --series: #2a78d6; --info: #52514e; --shadow: none;
+  }
+  .filters, details.table-view summary { display: none; }
+  .card, figure { break-inside: avoid; }
+}
+@media (prefers-reduced-motion: reduce) { * { transition: none !important; } }

vecshift/assets/report.js

@@ -0,0 +1,50 @@
+// Tooltips and the severity filter. The report is fully readable without this script.
+(() => {
+  const tip = document.createElement("div");
+  tip.className = "tip";
+  tip.setAttribute("role", "tooltip");
+  document.body.append(tip);
+
+  const show = (el, x, y) => {
+    tip.replaceChildren();
+    const value = document.createElement("strong");
+    value.textContent = el.dataset.tipValue || "";
+    const label = document.createElement("span");
+    label.textContent = el.dataset.tipLabel || "";
+    tip.append(value, label);
+    tip.style.display = "block";
+    const pad = 12;
+    const w = tip.offsetWidth;
+    const h = tip.offsetHeight;
+    let left = x - w / 2;
+    left = Math.max(pad, Math.min(left, window.innerWidth - w - pad));
+    let top = y - h - 14;
+    if (top < pad) top = y + 18;
+    tip.style.left = `${left}px`;
+    tip.style.top = `${top}px`;
+  };
+  const hide = () => { tip.style.display = "none"; };
+
+  document.querySelectorAll("[data-tip-value]").forEach((el) => {
+    el.addEventListener("pointermove", (e) => show(el, e.clientX, e.clientY));
+    el.addEventListener("pointerleave", hide);
+    el.addEventListener("focus", () => {
+      const r = el.getBoundingClientRect();
+      show(el, r.left + r.width / 2, r.top);
+    });
+    el.addEventListener("blur", hide);
+  });
+  window.addEventListener("scroll", hide, { passive: true });
+
+  const buttons = document.querySelectorAll(".filters button");
+  const findings = document.querySelectorAll(".finding");
+  buttons.forEach((button) => {
+    button.addEventListener("click", () => {
+      const severity = button.dataset.filter;
+      buttons.forEach((b) => b.setAttribute("aria-pressed", String(b === button)));
+      findings.forEach((f) => {
+        f.hidden = severity !== "all" && f.dataset.severity !== severity;
+      });
+    });
+  });
+})();

vecshift/bench/__init__.py

@@ -0,0 +1,18 @@
+"""``vecshift bench``: compare embedding models on a sample of your own data."""
+
+from vecshift.bench.corpus import Benchmark, CorpusError, Document, Query
+from vecshift.bench.metrics import Scores
+from vecshift.bench.runner import BenchResult, ModelResult, PlanItem, plan, run
+
+__all__ = [
+    "BenchResult",
+    "Benchmark",
+    "CorpusError",
+    "Document",
+    "ModelResult",
+    "PlanItem",
+    "Query",
+    "Scores",
+    "plan",
+    "run",
+]

vecshift/bench/corpus.py

@@ -0,0 +1,175 @@
+"""Documents and queries for a benchmark."""
+
+from __future__ import annotations
+
+import json
+import random
+import re
+from collections import Counter
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+
+MAX_CHARS = 8000
+"""Longer documents are cut to this length, well inside common model input limits."""
+
+
+class CorpusError(ValueError):
+    """Documents or queries couldn't be loaded."""
+
+
+@dataclass(frozen=True, slots=True)
+class Document:
+    id: str
+    text: str
+
+
+@dataclass(frozen=True, slots=True)
+class Query:
+    text: str
+    relevant: frozenset[str]
+    """IDs of the documents that answer this query."""
+
+
+@dataclass(slots=True)
+class Benchmark:
+    documents: list[Document]
+    queries: list[Query]
+    query_source: str
+    """``proxy``, ``generated``, or ``labeled``."""
+    notes: list[str] = field(default_factory=list)
+
+
+def clip(text: str) -> str:
+    return text if len(text) <= MAX_CHARS else text[:MAX_CHARS]
+
+
+def _read_jsonl(path: Path) -> Iterable[tuple[int, dict[str, object]]]:
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError as exc:
+        raise CorpusError(f"Couldn't read {path}: {exc.strerror}") from exc
+    for number, line in enumerate(lines, 1):
+        if not line.strip():
+            continue
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError as exc:
+            raise CorpusError(f"{path}:{number}: not valid JSON ({exc.msg})") from exc
+        if not isinstance(row, dict):
+            raise CorpusError(f"{path}:{number}: expected a JSON object")
+        yield number, row
+
+
+def load_documents(path: Path) -> list[Document]:
+    """Read ``{"id": ..., "text": ...}`` lines. ``id`` is optional and defaults to the line."""
+    docs: list[Document] = []
+    seen: set[str] = set()
+    for number, row in _read_jsonl(path):
+        text = row.get("text")
+        if not isinstance(text, str):
+            raise CorpusError(f'{path}:{number}: each line needs a "text" string')
+        if not text.strip():
+            continue
+        doc_id = str(row.get("id", number))
+        if doc_id in seen:
+            raise CorpusError(f"{path}:{number}: duplicate id {doc_id!r}")
+        seen.add(doc_id)
+        docs.append(Document(doc_id, clip(text)))
+    if not docs:
+        raise CorpusError(f"{path} has no documents")
+    return docs
+
+
+def load_queries(path: Path) -> list[Query]:
+    """Read ``{"query": ..., "relevant": [ids]}`` lines."""
+    queries: list[Query] = []
+    for number, row in _read_jsonl(path):
+        text, relevant = row.get("query"), row.get("relevant")
+        if not isinstance(text, str) or not text.strip():
+            raise CorpusError(f'{path}:{number}: each line needs a "query" string')
+        if isinstance(relevant, (str, int)):
+            relevant = [relevant]
+        if not isinstance(relevant, list) or not relevant:
+            raise CorpusError(f'{path}:{number}: "relevant" must list at least one document id')
+        queries.append(Query(text.strip(), frozenset(str(r) for r in relevant)))
+    if not queries:
+        raise CorpusError(f"{path} has no queries")
+    return queries
+
+
+def save_queries(path: Path, queries: Sequence[Query]) -> None:
+    lines = [
+        json.dumps({"query": q.text, "relevant": sorted(q.relevant)}, ensure_ascii=False)
+        for q in queries
+    ]
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def sample(docs: Sequence[Document], n: int, keep: set[str], seed: int) -> list[Document]:
+    """Up to ``n`` documents, always including the IDs in ``keep``."""
+    kept = [d for d in docs if d.id in keep]
+    rest = [d for d in docs if d.id not in keep]
+    room = max(0, n - len(kept))
+    chosen = random.Random(seed).sample(rest, min(room, len(rest)))
+    return kept + chosen
+
+
+def _norm(sentence: str) -> str:
+    return " ".join(sentence.lower().split())
+
+
+_SENTENCE = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9\"'(])")
+_WORDS = re.compile(r"\S+")
+
+
+def proxy_queries(
+    docs: Sequence[Document], n: int, seed: int
+) -> tuple[list[Document], list[Query], list[str]]:
+    """Make queries from the documents themselves, with no LLM.
+
+    For each chosen document, one sentence becomes the query and is removed from the
+    document, so a model has to match it by meaning to the rest of the text (the
+    inverse cloze task). Single-sentence documents use their opening words instead.
+    Returns the edited documents, the queries, and notes for the report.
+    """
+    rng = random.Random(seed)
+    split = {doc.id: [s for s in _SENTENCE.split(doc.text.strip()) if s.strip()] for doc in docs}
+    # A sentence repeated across documents (boilerplate, templates) would make the true
+    # document the only one *without* the query text, so only unique sentences qualify.
+    counts = Counter(_norm(s) for sentences in split.values() for s in set(sentences))
+    candidates: dict[str, tuple[str, str]] = {}
+    for doc in docs:
+        sentences = split[doc.id]
+        usable = [
+            i
+            for i, s in enumerate(sentences)
+            if 6 <= len(_WORDS.findall(s)) <= 40 and counts[_norm(s)] == 1
+        ]
+        if len(sentences) >= 2 and usable:
+            i = rng.choice(usable)
+            rest = " ".join(s for j, s in enumerate(sentences) if j != i)
+            if len(_WORDS.findall(rest)) >= 8:
+                candidates[doc.id] = (sentences[i], rest)
+                continue
+        words = _WORDS.findall(doc.text)
+        if len(words) >= 20:
+            cut = min(12, len(words) // 3)
+            candidates[doc.id] = (" ".join(words[:cut]), " ".join(words[cut:]))
+
+    if not candidates:
+        raise CorpusError(
+            "The documents are too short to make proxy queries from. Use --generate-queries "
+            "or --queries instead."
+        )
+    chosen = set(rng.sample(sorted(candidates), min(n, len(candidates))))
+    edited = [Document(d.id, candidates[d.id][1]) if d.id in chosen else d for d in docs]
+    queries = [Query(candidates[i][0], frozenset({i})) for i in sorted(chosen)]
+    notes = [
+        "Queries are sentences taken out of the documents (a proxy, no LLM involved). "
+        "That's good for ranking models against each other; use --generate-queries or "
+        "--queries for numbers closer to real search."
+    ]
+    if len(chosen) < n:
+        notes.append(f"Only {len(chosen)} documents were long enough to make queries from.")
+    return edited, queries, notes