retrieval-eval 0.1.0

package/dist/index.js

@@ -0,0 +1,971 @@
+// src/identity.ts
+import { createHash } from "crypto";
+var SEP = "";
+function normalize(text) {
+  return text.normalize("NFC").replace(/\r\n/g, "\n").replace(/\r/g, "\n").trim();
+}
+function h128(input) {
+  return createHash("sha256").update(input, "utf8").digest("hex").slice(0, 32);
+}
+function textSha(text) {
+  return `t1:${h128(normalize(text))}`;
+}
+function chunkId(input) {
+  const canonical = [
+    input.docUri,
+    input.docRevision,
+    String(input.ordinal),
+    normalize(input.text),
+    input.chunkerFingerprint
+  ].join(SEP);
+  return `c1:${h128(canonical)}`;
+}
+
+// src/metrics.ts
+function judgmentKey(j) {
+  return j.chunk_id ?? j.doc_uri;
+}
+function dedupe(ranking) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const key of ranking) {
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(key);
+  }
+  return out;
+}
+function dcg(gains) {
+  let total = 0;
+  for (let i = 0; i < gains.length; i++) total += gains[i] / Math.log2(i + 2);
+  return total;
+}
+function queryMetrics(relevance, ranking, options = {}) {
+  const k = options.k ?? 10;
+  const threshold = options.threshold ?? 1;
+  const topK = dedupe(ranking).slice(0, k);
+  const relevantKeys = new Set(
+    [...relevance.entries()].filter(([, r]) => r >= threshold).map(([key]) => key)
+  );
+  const hits = topK.filter((key) => relevantKeys.has(key));
+  const gains = topK.map((key) => 2 ** (relevance.get(key) ?? 0) - 1);
+  const idealGains = [...relevance.values()].map((r) => 2 ** r - 1).sort((a, b) => b - a).slice(0, k);
+  const idcg = dcg(idealGains);
+  let mrr = 0;
+  for (let i = 0; i < topK.length; i++) {
+    if (relevantKeys.has(topK[i])) {
+      mrr = 1 / (i + 1);
+      break;
+    }
+  }
+  let found = 0;
+  let apSum = 0;
+  for (let i = 0; i < topK.length; i++) {
+    if (relevantKeys.has(topK[i])) {
+      found++;
+      apSum += found / (i + 1);
+    }
+  }
+  return {
+    precision: hits.length / k,
+    recall: relevantKeys.size > 0 ? hits.length / relevantKeys.size : 0,
+    ndcg: idcg > 0 ? dcg(gains) / idcg : 0,
+    mrr,
+    ap: relevantKeys.size > 0 ? apSum / relevantKeys.size : 0,
+    hit_rate: hits.length > 0 ? 1 : 0
+  };
+}
+function relevanceByQuery(judgments) {
+  const out = /* @__PURE__ */ new Map();
+  for (const j of judgments) {
+    let inner = out.get(j.query_id);
+    if (!inner) {
+      inner = /* @__PURE__ */ new Map();
+      out.set(j.query_id, inner);
+    }
+    inner.set(judgmentKey(j), j.relevance);
+  }
+  return out;
+}
+var deterministic = (value) => ({
+  value: round(value),
+  n: 1,
+  deterministic: true
+});
+function round(value) {
+  return Math.round(value * 1e12) / 1e12;
+}
+var T_95 = [
+  0,
+  12.706204736175,
+  4.302652729749,
+  3.182446305284,
+  2.776445105198,
+  2.570581835636,
+  2.446911851145,
+  2.364624251593,
+  2.306004135204,
+  2.262157162798,
+  2.228138851986,
+  2.200985160092,
+  2.178812829667,
+  2.160368656463,
+  2.144786687918,
+  2.13144954556,
+  2.119905299221,
+  2.109815577833,
+  2.100922040241,
+  2.093024054408,
+  2.085963447266,
+  2.079613844728,
+  2.073873067904,
+  2.068657610419,
+  2.063898561628,
+  2.059538552753,
+  2.055529438643,
+  2.05183051648,
+  2.048407141795,
+  2.045229642133,
+  2.042272456301,
+  2.039513446396,
+  2.03693334346,
+  2.034515297449,
+  2.032244509318,
+  2.03010792825,
+  2.02809400098,
+  2.026192463029,
+  2.024394163912,
+  2.022690920037,
+  2.021075390306,
+  2.019540970441,
+  2.018081702818,
+  2.016692199228,
+  2.015367574444,
+  2.014103388881,
+  2.012895598919,
+  2.01174051373,
+  2.010634757624,
+  2.009575237129,
+  2.008559112101,
+  2.007583770316,
+  2.006646805062,
+  2.005745995318,
+  2.004879288188,
+  2.004044783289,
+  2.003240718848,
+  2.002465459291,
+  2.001717484145,
+  2.000995378088,
+  2.000297822014,
+  1.999623584995,
+  1.998971517033,
+  1.998340542521,
+  1.997729654318,
+  1.997137908392,
+  1.996564418952,
+  1.996008354025,
+  1.99546893143,
+  1.994945415107,
+  1.994437111771,
+  1.993943367846,
+  1.993463566662,
+  1.99299712589,
+  1.992543495181,
+  1.992102154002,
+  1.991672609645,
+  1.991254395388,
+  1.990847068812,
+  1.99045021023,
+  1.990063421254,
+  1.989686323457,
+  1.989318557137,
+  1.988959780175,
+  1.988609666976,
+  1.988267907477,
+  1.987934206239,
+  1.987608281589,
+  1.987289864831,
+  1.986978699506,
+  1.986674540704,
+  1.986377154419,
+  1.986086316951,
+  1.985801814346,
+  1.985523441867,
+  1.985251003505,
+  1.984984311522,
+  1.984723186014,
+  1.984467454508,
+  1.984216951586,
+  1.983971518524,
+  1.983731002956,
+  1.983495258563,
+  1.983264144773,
+  1.983037526484,
+  1.982815273795,
+  1.982597261765,
+  1.982383370176,
+  1.982173483308,
+  1.981967489736,
+  1.981765282132,
+  1.981566757075,
+  1.981371814876,
+  1.981180359415,
+  1.980992297976,
+  1.980807541104,
+  1.980626002459,
+  1.980447598683,
+  1.980272249273,
+  1.980099876457,
+  1.979930405082
+];
+function summarize(samples) {
+  if (samples.length === 0) throw new Error("summarize needs at least one sample");
+  const n = samples.length;
+  const mean = samples.reduce((sum, value) => sum + value, 0) / n;
+  if (n < 2) return { value: round(mean), n, deterministic: false };
+  const variance = samples.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (n - 1);
+  const stdev = Math.sqrt(variance);
+  const df = Math.min(n - 1, T_95.length - 1);
+  const margin = T_95[df] * stdev / Math.sqrt(n);
+  return {
+    value: round(mean),
+    n,
+    stdev: round(stdev),
+    ci: [round(mean - margin), round(mean + margin)],
+    deterministic: false
+  };
+}
+function score(judgments, run, options = {}) {
+  const k = options.k ?? 10;
+  const threshold = options.threshold ?? 1;
+  const byQuery = relevanceByQuery(judgments);
+  const rankings = new Map(run.map((r) => [r.query_id, r.ranking]));
+  const perQuery = /* @__PURE__ */ new Map();
+  const missingQueries = [];
+  const queriesWithoutPositives = [];
+  const queriesWithDuplicates = [];
+  const scored = [];
+  for (const [queryId, relevance] of byQuery) {
+    const ranking = rankings.get(queryId);
+    if (!ranking) missingQueries.push(queryId);
+    else if (dedupe(ranking).length !== ranking.length) queriesWithDuplicates.push(queryId);
+    const metrics = queryMetrics(relevance, ranking ?? [], options);
+    perQuery.set(queryId, metrics);
+    if ([...relevance.values()].some((r) => r >= threshold)) scored.push(metrics);
+    else queriesWithoutPositives.push(queryId);
+  }
+  if (scored.length === 0) {
+    return {
+      metrics: {},
+      perQuery,
+      missingQueries,
+      queriesWithoutPositives,
+      queriesWithDuplicates
+    };
+  }
+  const mean = (pick) => scored.reduce((sum, m) => sum + pick(m), 0) / scored.length;
+  return {
+    metrics: {
+      [`precision@${k}`]: deterministic(mean((m) => m.precision)),
+      [`recall@${k}`]: deterministic(mean((m) => m.recall)),
+      [`ndcg@${k}`]: deterministic(mean((m) => m.ndcg)),
+      [`mrr@${k}`]: deterministic(mean((m) => m.mrr)),
+      [`map@${k}`]: deterministic(mean((m) => m.ap)),
+      [`hit_rate@${k}`]: deterministic(mean((m) => m.hit_rate))
+    },
+    perQuery,
+    missingQueries,
+    queriesWithoutPositives,
+    queriesWithDuplicates
+  };
+}
+function scoreByStratum(judgments, run, options = {}) {
+  const strata = /* @__PURE__ */ new Map();
+  for (const j of judgments) {
+    const name = j.stratum ?? "_unstratified";
+    const list = strata.get(name);
+    if (list) list.push(j);
+    else strata.set(name, [j]);
+  }
+  const out = {};
+  for (const [name, subset] of strata) {
+    const result = score(subset, run, options);
+    out[name] = {
+      n: result.perQuery.size - result.queriesWithoutPositives.length,
+      metrics: result.metrics
+    };
+  }
+  return out;
+}
+function worstStratum(perStratum, metric) {
+  let worst = null;
+  for (const [name, stratum] of Object.entries(perStratum)) {
+    if (stratum.n === 0) continue;
+    const measurement = stratum.metrics[metric];
+    if (!measurement) continue;
+    if (!worst || measurement.value < worst.value) {
+      worst = { name, value: measurement.value, n: stratum.n };
+    }
+  }
+  return worst;
+}
+
+// src/judgments.ts
+function parseJsonl(content, label) {
+  const out = [];
+  const lines = content.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (line === "" || line.startsWith("//")) continue;
+    try {
+      out.push({ value: JSON.parse(line), line: i + 1 });
+    } catch (error) {
+      throw new Error(`${label}:${i + 1}: invalid JSON, ${error.message}`);
+    }
+  }
+  return out;
+}
+function parseJudgments(content, label = "judgments") {
+  const rows = parseJsonl(content, label);
+  for (const { value, line } of rows) {
+    if (typeof value.query_id !== "string" || value.query_id === "")
+      throw new Error(`${label}:${line}: missing query_id`);
+    if (typeof value.doc_uri !== "string" || value.doc_uri === "")
+      throw new Error(`${label}:${line}: missing doc_uri`);
+    if (!Number.isInteger(value.relevance) || value.relevance < 0)
+      throw new Error(`${label}:${line}: relevance must be a non-negative integer`);
+  }
+  return rows.map((row) => row.value);
+}
+function parseRun(content, label = "run") {
+  const rows = parseJsonl(content, label);
+  const firstSeen = /* @__PURE__ */ new Map();
+  for (const { value, line } of rows) {
+    if (typeof value.query_id !== "string" || value.query_id === "")
+      throw new Error(`${label}:${line}: missing query_id`);
+    if (!Array.isArray(value.ranking))
+      throw new Error(`${label}:${line}: ranking must be an array`);
+    for (let i = 0; i < value.ranking.length; i++) {
+      const key = value.ranking[i];
+      if (typeof key !== "string" || key === "")
+        throw new Error(`${label}:${line}: ranking[${i}] must be a non-empty string`);
+    }
+    const previous = firstSeen.get(value.query_id);
+    if (previous !== void 0)
+      throw new Error(
+        `${label}:${line}: duplicate entry for query ${value.query_id}, already on line ${previous}`
+      );
+    firstSeen.set(value.query_id, line);
+  }
+  return rows.map((row) => row.value);
+}
+function byCodePoint(a, b) {
+  const left = [...a];
+  const right = [...b];
+  for (let i = 0; i < Math.min(left.length, right.length); i++) {
+    const difference = (left[i].codePointAt(0) ?? 0) - (right[i].codePointAt(0) ?? 0);
+    if (difference !== 0) return difference;
+  }
+  return left.length - right.length;
+}
+var FIELD_ORDER = [
+  "query_id",
+  "query",
+  "doc_uri",
+  "chunk_id",
+  "text_sha",
+  "chunk_text",
+  "relevance",
+  "corpus_fingerprint",
+  "labeled_by",
+  "labeled_at",
+  "stratum",
+  "notes"
+];
+function serializeJudgments(judgments) {
+  const lines = judgments.map((judgment) => {
+    const ordered = {};
+    for (const field of FIELD_ORDER) {
+      if (judgment[field] !== void 0) ordered[field] = judgment[field];
+    }
+    for (const [key, value] of Object.entries(judgment)) {
+      if (!FIELD_ORDER.includes(key)) ordered[key] = value;
+    }
+    return JSON.stringify(ordered);
+  });
+  return `${lines.join("\n")}
+`;
+}
+function parseCorpus(content, label = "corpus") {
+  const parsed = JSON.parse(content);
+  if (!Array.isArray(parsed.chunks)) throw new Error(`${label}: missing chunks array`);
+  return parsed;
+}
+function validate(judgments) {
+  const issues = [];
+  const queries = /* @__PURE__ */ new Set();
+  const strata = {};
+  const fingerprints = /* @__PURE__ */ new Set();
+  const seen = /* @__PURE__ */ new Set();
+  const queryText = /* @__PURE__ */ new Map();
+  let humanLabels = 0;
+  let syntheticLabels = 0;
+  let positives = 0;
+  for (const j of judgments) {
+    queries.add(j.query_id);
+    if (j.relevance >= 1) positives++;
+    if (j.corpus_fingerprint) fingerprints.add(j.corpus_fingerprint);
+    if (j.labeled_by?.startsWith("human:")) humanLabels++;
+    else if (j.labeled_by?.startsWith("synthetic:")) syntheticLabels++;
+    const stratum = j.stratum ?? "_unstratified";
+    strata[stratum] = (strata[stratum] ?? 0) + 1;
+    const key = `${j.query_id}\0${j.chunk_id ?? j.doc_uri}`;
+    if (seen.has(key)) {
+      issues.push({
+        severity: "error",
+        code: "duplicate-label",
+        message: `duplicate judgment for query ${j.query_id} and target ${j.chunk_id ?? j.doc_uri}`
+      });
+    }
+    seen.add(key);
+    if (j.query) {
+      const existing = queryText.get(j.query_id);
+      if (existing !== void 0 && existing !== j.query) {
+        issues.push({
+          severity: "error",
+          code: "inconsistent-query-text",
+          message: `query ${j.query_id} has two different query strings`
+        });
+      }
+      queryText.set(j.query_id, j.query);
+    }
+    if (j.chunk_id && !/^c1:[0-9a-f]{32}$/.test(j.chunk_id)) {
+      issues.push({
+        severity: "error",
+        code: "bad-chunk-id",
+        message: `malformed chunk_id on query ${j.query_id}: ${j.chunk_id}`
+      });
+    }
+    if (j.text_sha && !/^t1:[0-9a-f]{32}$/.test(j.text_sha)) {
+      issues.push({
+        severity: "error",
+        code: "bad-text-sha",
+        message: `malformed text_sha on query ${j.query_id}: ${j.text_sha}`
+      });
+    }
+  }
+  for (const queryId of [...queries].sort(byCodePoint)) {
+    if (!queryText.has(queryId)) {
+      issues.push({
+        severity: "warning",
+        code: "missing-query-text",
+        message: `query ${queryId} has no query text on any row`
+      });
+    }
+  }
+  if (judgments.length === 0) {
+    issues.push({ severity: "error", code: "empty", message: "no judgments" });
+  }
+  if (positives === 0 && judgments.length > 0) {
+    issues.push({
+      severity: "error",
+      code: "no-positives",
+      message: "no judgment has relevance >= 1, so recall is undefined for every query"
+    });
+  }
+  if (humanLabels === 0 && syntheticLabels > 0) {
+    issues.push({
+      severity: "warning",
+      code: "no-human-labels",
+      message: "every label is synthetic. Without human labels you cannot measure judge calibration, and synthetic labels quietly become ground truth"
+    });
+  }
+  if (fingerprints.size > 1) {
+    issues.push({
+      severity: "warning",
+      code: "mixed-fingerprints",
+      message: `judgments span ${fingerprints.size} corpus fingerprints; run 'drift' before trusting any metric`
+    });
+  }
+  for (const [name, count] of Object.entries(strata).sort(([a], [b]) => byCodePoint(a, b))) {
+    if (name !== "_unstratified" && count < 5) {
+      issues.push({
+        severity: "warning",
+        code: "thin-stratum",
+        message: `stratum '${name}' has only ${count} labels, too few to gate on`
+      });
+    }
+  }
+  const withChunkId = judgments.filter((j) => j.chunk_id).length;
+  if (withChunkId > 0 && withChunkId < judgments.length) {
+    issues.push({
+      severity: "warning",
+      code: "mixed-granularity",
+      message: `${withChunkId}/${judgments.length} labels have chunk_id; the rest are document-level`
+    });
+  }
+  if (withChunkId === 0 && judgments.length > 0) {
+    issues.push({
+      severity: "warning",
+      code: "no-chunk-ids",
+      message: "no label has a chunk_id, so drift can only work at document granularity. See spec/chunk-id.md"
+    });
+  }
+  const fingerprint = fingerprints.size === 1 ? [...fingerprints][0] : null;
+  return {
+    issues,
+    queries: queries.size,
+    labels: judgments.length,
+    humanLabels,
+    syntheticLabels,
+    fingerprint,
+    strata,
+    ok: !issues.some((i) => i.severity === "error")
+  };
+}
+
+// src/qrels.ts
+function toQrels(judgments) {
+  const lines = judgments.map((j) => `${j.query_id} 0 ${j.chunk_id ?? j.doc_uri} ${j.relevance}`);
+  return `${lines.join("\n")}
+`;
+}
+function fromQrels(content, options = {}) {
+  const out = [];
+  const lines = content.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (line === "" || line.startsWith("#")) continue;
+    const parts = line.split(/\s+/);
+    if (parts.length < 3)
+      throw new Error(`qrels:${i + 1}: expected 3 or 4 fields, got ${parts.length}`);
+    const [queryId, second, third, fourth] = parts;
+    const docId = parts.length >= 4 ? third : second;
+    const relevance = parts.length >= 4 ? fourth : third;
+    const parsed = Number.parseInt(relevance, 10);
+    if (Number.isNaN(parsed)) {
+      if (out.length === 0 && /^query[-_ ]?id$/i.test(queryId)) continue;
+      throw new Error(`qrels:${i + 1}: relevance '${relevance}' is not an integer`);
+    }
+    const judgment = {
+      query_id: queryId,
+      doc_uri: docId,
+      // Negative relevance appears in some TREC collections; clamp to the schema's minimum.
+      relevance: Math.max(0, parsed)
+    };
+    if (options.asChunkIds) judgment.chunk_id = docId;
+    if (options.corpusFingerprint) judgment.corpus_fingerprint = options.corpusFingerprint;
+    if (options.labeledBy) judgment.labeled_by = options.labeledBy;
+    out.push(judgment);
+  }
+  return out;
+}
+function toTrecRun(run, runName = "retrieval-eval") {
+  const lines = [];
+  for (const entry of run) {
+    entry.ranking.forEach((docId, index) => {
+      const score2 = (entry.ranking.length - index).toFixed(4);
+      lines.push(`${entry.query_id} Q0 ${docId} ${index + 1} ${score2} ${runName}`);
+    });
+  }
+  return `${lines.join("\n")}
+`;
+}
+function fromTrecRun(content) {
+  const byQuery = /* @__PURE__ */ new Map();
+  for (const raw of content.split("\n")) {
+    const line = raw.trim();
+    if (line === "" || line.startsWith("#")) continue;
+    const parts = line.split(/\s+/);
+    if (parts.length < 4) continue;
+    const [queryId, , docId, rank] = parts;
+    const list = byQuery.get(queryId) ?? [];
+    list.push({ docId, rank: Number.parseInt(rank, 10) });
+    byQuery.set(queryId, list);
+  }
+  return [...byQuery.entries()].map(([queryId, rows]) => ({
+    query_id: queryId,
+    ranking: rows.sort((a, b) => a.rank - b.rank).map((r) => r.docId)
+  }));
+}
+
+// src/drift.ts
+function buildIndex(corpus) {
+  const byChunkId = /* @__PURE__ */ new Map();
+  const byTextSha = /* @__PURE__ */ new Map();
+  const byDoc = /* @__PURE__ */ new Map();
+  for (const chunk of corpus.chunks) {
+    byChunkId.set(chunk.chunk_id, chunk);
+    const sha = chunk.text_sha ?? (chunk.text !== void 0 ? textSha(chunk.text) : void 0);
+    if (sha) {
+      const list = byTextSha.get(sha);
+      if (list) list.push(chunk);
+      else byTextSha.set(sha, [chunk]);
+    }
+    const docList = byDoc.get(chunk.doc_uri);
+    if (docList) docList.push(chunk);
+    else byDoc.set(chunk.doc_uri, [chunk]);
+  }
+  for (const list of byDoc.values()) {
+    list.sort((a, b) => (a.ordinal ?? 0) - (b.ordinal ?? 0));
+  }
+  return { byChunkId, byTextSha, byDoc };
+}
+function findSplit(labeledText, docChunks) {
+  const haystack = normalize(labeledText);
+  if (haystack === "") return null;
+  const covering = docChunks.filter((chunk) => {
+    if (chunk.text === void 0) return false;
+    const needle = normalize(chunk.text);
+    return needle !== "" && haystack.includes(needle);
+  });
+  return covering.length >= 2 ? covering : null;
+}
+function findMerged(labeledText, docChunks) {
+  const needle = normalize(labeledText);
+  if (needle === "") return null;
+  const containing = docChunks.filter(
+    (chunk) => chunk.text !== void 0 && normalize(chunk.text).includes(needle)
+  );
+  if (containing.length === 0) return null;
+  return containing.reduce(
+    (best, chunk) => normalize(chunk.text).length < normalize(best.text).length ? chunk : best
+  );
+}
+function drift(judgments, corpus, options = {}) {
+  const index = buildIndex(corpus);
+  const findings = [];
+  const fingerprints = /* @__PURE__ */ new Set();
+  for (const j of judgments) {
+    if (j.corpus_fingerprint) fingerprints.add(j.corpus_fingerprint);
+    const finding = {
+      query_id: j.query_id,
+      doc_uri: j.doc_uri,
+      status: "ORPHANED"
+    };
+    if (j.chunk_id !== void 0) finding.chunk_id = j.chunk_id;
+    const sha = j.text_sha ?? (j.chunk_text !== void 0 ? textSha(j.chunk_text) : void 0);
+    const docChunks = index.byDoc.get(j.doc_uri) ?? [];
+    if (j.chunk_id === void 0) {
+      finding.status = docChunks.length > 0 ? "VALID" : "ORPHANED";
+      findings.push(finding);
+      continue;
+    }
+    if (index.byChunkId.has(j.chunk_id)) {
+      finding.status = "VALID";
+      findings.push(finding);
+      continue;
+    }
+    if (sha) {
+      const matches = index.byTextSha.get(sha) ?? [];
+      const sameDoc = matches.filter((chunk) => chunk.doc_uri === j.doc_uri);
+      const target = sameDoc[0] ?? matches[0];
+      if (target) {
+        finding.status = "RE_ANCHORABLE";
+        finding.reanchor_to = target.chunk_id;
+        findings.push(finding);
+        continue;
+      }
+    }
+    if (j.chunk_text !== void 0) {
+      const split = findSplit(j.chunk_text, docChunks);
+      if (split) {
+        finding.status = "SPLIT";
+        finding.split_into = split.map((chunk) => chunk.chunk_id);
+        findings.push(finding);
+        continue;
+      }
+      const merged = findMerged(j.chunk_text, docChunks);
+      if (merged) {
+        finding.status = "MERGED";
+        finding.reanchor_to = merged.chunk_id;
+        findings.push(finding);
+        continue;
+      }
+    }
+    findings.push(finding);
+  }
+  const count = (status) => findings.filter((f) => f.status === status).length;
+  const valid = count("VALID");
+  const total = findings.length;
+  return {
+    findings,
+    summary: {
+      valid,
+      re_anchorable: count("RE_ANCHORABLE"),
+      merged: count("MERGED"),
+      split: count("SPLIT"),
+      orphaned: count("ORPHANED"),
+      invalid_ratio: total === 0 ? 0 : Math.round((total - valid) / total * 1e12) / 1e12
+    },
+    judgments_fingerprint: fingerprints.size === 1 ? [...fingerprints][0] : null,
+    corpus_fingerprint: corpus.corpus_fingerprint ?? options.corpusFingerprint ?? null
+  };
+}
+function fix(judgments, result) {
+  const byQueryAndChunk = /* @__PURE__ */ new Map();
+  for (const finding of result.findings) {
+    byQueryAndChunk.set(`${finding.query_id}\0${finding.chunk_id ?? ""}`, finding);
+  }
+  let reanchored = 0;
+  const needsReview = [];
+  const out = judgments.map((j) => {
+    const finding = byQueryAndChunk.get(`${j.query_id}\0${j.chunk_id ?? ""}`);
+    if (!finding) return j;
+    if ((finding.status === "RE_ANCHORABLE" || finding.status === "MERGED") && finding.reanchor_to) {
+      reanchored++;
+      const next = { ...j, chunk_id: finding.reanchor_to };
+      if (result.corpus_fingerprint) next.corpus_fingerprint = result.corpus_fingerprint;
+      return next;
+    }
+    if (finding.status === "SPLIT" || finding.status === "ORPHANED") needsReview.push(finding);
+    if (finding.status === "VALID" && result.corpus_fingerprint) {
+      return { ...j, corpus_fingerprint: result.corpus_fingerprint };
+    }
+    return j;
+  });
+  return { judgments: out, reanchored, needsReview };
+}
+
+// src/gate.ts
+function parseGate(expression) {
+  const parts = expression.split(":");
+  if (parts[0] === "worst-stratum") {
+    if (parts.length !== 3)
+      throw new Error(`gate '${expression}': expected worst-stratum:<metric>:<floor>`);
+    return {
+      raw: expression,
+      kind: "worst-stratum",
+      metric: parts[1],
+      threshold: parseNumber(parts[2], expression)
+    };
+  }
+  if (parts.length === 3 && parts[1] === "ci-lower") {
+    return {
+      raw: expression,
+      kind: "ci-lower",
+      metric: parts[0],
+      threshold: parseNumber(parts[2], expression)
+    };
+  }
+  if (parts.length !== 2) throw new Error(`gate '${expression}': expected <metric>:<threshold>`);
+  const value = parts[1];
+  return {
+    raw: expression,
+    kind: value.startsWith("-") || value.startsWith("+") ? "delta" : "absolute",
+    metric: parts[0],
+    threshold: parseNumber(value, expression)
+  };
+}
+var NUMBER = /^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/;
+function parseNumber(value, expression) {
+  if (!NUMBER.test(value)) throw new Error(`gate '${expression}': '${value}' is not a number`);
+  return Number.parseFloat(value);
+}
+function worseStatus(a, b) {
+  if (a === "FAIL" || b === "FAIL") return "FAIL";
+  if (a === "INDETERMINATE" || b === "INDETERMINATE") return "INDETERMINATE";
+  return "PASS";
+}
+function evaluateGates({ report, baseline, gates }) {
+  const results = [];
+  const reasons = [];
+  for (const gate of gates) {
+    const result = evaluateGate(gate, report, baseline);
+    results.push(result);
+    if (result.status === "FAIL" || result.status === "INDETERMINATE") {
+      reasons.push(describe(gate, result, report));
+    }
+  }
+  const status = results.some((r) => r.status === "FAIL") ? "FAIL" : results.some((r) => r.status === "INDETERMINATE") ? "INDETERMINATE" : "PASS";
+  return { status, results, reasons };
+}
+function evaluateGate(gate, report, baseline) {
+  if (gate.kind === "worst-stratum") {
+    if (!report.per_stratum) {
+      return { expression: gate.raw, status: "INDETERMINATE" };
+    }
+    const worst = worstStratum(report.per_stratum, gate.metric);
+    if (!worst) return { expression: gate.raw, status: "INDETERMINATE" };
+    return {
+      expression: gate.raw,
+      status: worst.value >= gate.threshold ? "PASS" : "FAIL",
+      observed: worst.value
+    };
+  }
+  const measurement = report.metrics[gate.metric];
+  if (!measurement) return { expression: gate.raw, status: "INDETERMINATE" };
+  if (gate.kind === "ci-lower") {
+    if (!measurement.ci) {
+      return { expression: gate.raw, status: "INDETERMINATE", observed: measurement.value };
+    }
+    const lower = measurement.ci[0];
+    return {
+      expression: gate.raw,
+      status: lower >= gate.threshold ? "PASS" : "FAIL",
+      observed: lower
+    };
+  }
+  if (gate.kind === "delta") {
+    const previous = baseline?.metrics[gate.metric];
+    if (!previous) {
+      return {
+        expression: gate.raw,
+        status: "INDETERMINATE",
+        observed: measurement.value,
+        baseline: null
+      };
+    }
+    const delta = measurement.value - previous.value;
+    return {
+      expression: gate.raw,
+      status: delta >= gate.threshold ? "PASS" : "FAIL",
+      observed: measurement.value,
+      baseline: previous.value
+    };
+  }
+  return {
+    expression: gate.raw,
+    status: measurement.value >= gate.threshold ? "PASS" : "FAIL",
+    observed: measurement.value
+  };
+}
+function describe(gate, result, report) {
+  if (result.status === "INDETERMINATE") {
+    const nothingScored = report.judgments.queries_scored === 0 && result.observed === void 0;
+    if (nothingScored && gate.kind !== "worst-stratum") {
+      return `${gate.raw}: no query was scored, so '${gate.metric}' was not computed`;
+    }
+    if (gate.kind === "ci-lower") {
+      return `${gate.raw}: no confidence interval on '${gate.metric}', sample it more than once`;
+    }
+    if (gate.kind === "delta") return `${gate.raw}: no baseline value for '${gate.metric}'`;
+    if (gate.kind === "worst-stratum") {
+      const strata = Object.values(report.per_stratum ?? {});
+      if (strata.length > 0 && strata.every((stratum) => stratum.n === 0)) {
+        return `${gate.raw}: no stratum was scored for '${gate.metric}'`;
+      }
+      return `${gate.raw}: no per-stratum data for '${gate.metric}'`;
+    }
+    return `${gate.raw}: metric '${gate.metric}' not present in the report`;
+  }
+  if (gate.kind === "delta") {
+    const from = result.baseline ?? 0;
+    const to = result.observed ?? 0;
+    const change = to - from;
+    const direction = change < 0 ? `fell ${(-change).toFixed(4)}` : `rose ${change.toFixed(4)}`;
+    return `${gate.metric} ${direction}, from ${from.toFixed(4)} to ${to.toFixed(4)}, outside ${gate.threshold}`;
+  }
+  if (gate.kind === "worst-stratum") {
+    return `worst stratum ${gate.metric} is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
+  }
+  if (gate.kind === "ci-lower") {
+    return `${gate.metric} lower bound is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
+  }
+  return `${gate.metric} is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
+}
+
+// src/report.ts
+var TOOL_NAME = "retrieval-eval";
+var TOOL_VERSION = "0.1.0";
+var SPEC_VERSION = "1";
+function buildReport(options) {
+  const { judgments, run, corpus, driftResult } = options;
+  const metricOptions = { k: options.k ?? 10, threshold: options.threshold ?? 1 };
+  const scored = score(judgments, run, metricOptions);
+  const validation = validate(judgments);
+  const hasStrata = judgments.some((j) => j.stratum !== void 0);
+  const corpusInfo = { fingerprint: corpus?.corpus_fingerprint ?? null };
+  if (corpus) {
+    corpusInfo.documents = new Set(corpus.chunks.map((c) => c.doc_uri)).size;
+    corpusInfo.chunks = corpus.chunks.length;
+  }
+  const reasons = [];
+  if (driftResult && driftResult.summary.invalid_ratio > 0) {
+    reasons.push(
+      `${Math.round(driftResult.summary.invalid_ratio * 100)}% of judgments no longer match the live corpus`
+    );
+  }
+  const missingAndScored = scored.missingQueries.filter(
+    (id) => !scored.queriesWithoutPositives.includes(id)
+  );
+  if (missingAndScored.length > 0) {
+    reasons.push(`${missingAndScored.length} judged queries had no run entry and scored zero`);
+  }
+  const queriesScored = validation.queries - scored.queriesWithoutPositives.length;
+  if (scored.queriesWithoutPositives.length > 0 && queriesScored > 0) {
+    reasons.push(
+      `${scored.queriesWithoutPositives.length} judged queries have no label at relevance >= ${metricOptions.threshold} and were excluded from the averages`
+    );
+  }
+  if (queriesScored === 0) {
+    reasons.push(
+      validation.queries === 0 ? "no judged queries, so no metric was computed" : `no query had a label at relevance >= ${metricOptions.threshold}, so no metric was computed`
+    );
+  }
+  if (scored.queriesWithDuplicates.length > 0) {
+    reasons.push(
+      `${scored.queriesWithDuplicates.length} queries repeated a key in their ranking; only the first occurrence of each was counted`
+    );
+  }
+  const validationErrors = validation.issues.filter((issue) => issue.severity === "error");
+  if (validationErrors.length > 0) {
+    const codes = [...new Set(validationErrors.map((issue) => issue.code))].sort(byCodePoint);
+    reasons.push(
+      `the judgment set is unsound (${codes.join(", ")}); run 'retrieval-eval validate' for detail. No metric computed from it can be trusted`
+    );
+  }
+  const report = {
+    spec_version: SPEC_VERSION,
+    tool: { name: TOOL_NAME, version: TOOL_VERSION },
+    generated_at: (options.now ?? /* @__PURE__ */ new Date()).toISOString(),
+    corpus: corpusInfo,
+    judgments: {
+      queries: validation.queries,
+      queries_scored: queriesScored,
+      labels: validation.labels,
+      fingerprint: validation.fingerprint,
+      human_labels: validation.humanLabels,
+      synthetic_labels: validation.syntheticLabels,
+      ...driftResult ? { drift: driftResult.summary } : {}
+    },
+    metrics: scored.metrics,
+    ...hasStrata ? { per_stratum: scoreByStratum(judgments, run, metricOptions) } : {},
+    verdict: {
+      status: queriesScored === 0 || validationErrors.length > 0 ? "INDETERMINATE" : "PASS",
+      reasons
+    }
+  };
+  return report;
+}
+export {
+  SPEC_VERSION,
+  TOOL_NAME,
+  TOOL_VERSION,
+  buildReport,
+  chunkId,
+  dedupe,
+  drift,
+  evaluateGates,
+  fix,
+  fromQrels,
+  fromTrecRun,
+  judgmentKey,
+  normalize,
+  parseCorpus,
+  parseGate,
+  parseJudgments,
+  parseRun,
+  queryMetrics,
+  relevanceByQuery,
+  score,
+  scoreByStratum,
+  serializeJudgments,
+  summarize,
+  textSha,
+  toQrels,
+  toTrecRun,
+  validate,
+  worseStatus,
+  worstStratum
+};