retrieval-eval 0.1.0

package/dist/cli.js

@@ -0,0 +1,1344 @@
+#!/usr/bin/env node
+
+// src/cli.ts
+import { readFileSync, realpathSync, writeFileSync } from "fs";
+import { pathToFileURL } from "url";
+import { parseArgs } from "util";
+
+// src/identity.ts
+import { createHash } from "crypto";
+function normalize(text) {
+  return text.normalize("NFC").replace(/\r\n/g, "\n").replace(/\r/g, "\n").trim();
+}
+function h128(input) {
+  return createHash("sha256").update(input, "utf8").digest("hex").slice(0, 32);
+}
+function textSha(text) {
+  return `t1:${h128(normalize(text))}`;
+}
+
+// src/drift.ts
+function buildIndex(corpus) {
+  const byChunkId = /* @__PURE__ */ new Map();
+  const byTextSha = /* @__PURE__ */ new Map();
+  const byDoc = /* @__PURE__ */ new Map();
+  for (const chunk of corpus.chunks) {
+    byChunkId.set(chunk.chunk_id, chunk);
+    const sha = chunk.text_sha ?? (chunk.text !== void 0 ? textSha(chunk.text) : void 0);
+    if (sha) {
+      const list = byTextSha.get(sha);
+      if (list) list.push(chunk);
+      else byTextSha.set(sha, [chunk]);
+    }
+    const docList = byDoc.get(chunk.doc_uri);
+    if (docList) docList.push(chunk);
+    else byDoc.set(chunk.doc_uri, [chunk]);
+  }
+  for (const list of byDoc.values()) {
+    list.sort((a, b) => (a.ordinal ?? 0) - (b.ordinal ?? 0));
+  }
+  return { byChunkId, byTextSha, byDoc };
+}
+function findSplit(labeledText, docChunks) {
+  const haystack = normalize(labeledText);
+  if (haystack === "") return null;
+  const covering = docChunks.filter((chunk) => {
+    if (chunk.text === void 0) return false;
+    const needle = normalize(chunk.text);
+    return needle !== "" && haystack.includes(needle);
+  });
+  return covering.length >= 2 ? covering : null;
+}
+function findMerged(labeledText, docChunks) {
+  const needle = normalize(labeledText);
+  if (needle === "") return null;
+  const containing = docChunks.filter(
+    (chunk) => chunk.text !== void 0 && normalize(chunk.text).includes(needle)
+  );
+  if (containing.length === 0) return null;
+  return containing.reduce(
+    (best, chunk) => normalize(chunk.text).length < normalize(best.text).length ? chunk : best
+  );
+}
+function drift(judgments, corpus, options = {}) {
+  const index = buildIndex(corpus);
+  const findings = [];
+  const fingerprints = /* @__PURE__ */ new Set();
+  for (const j of judgments) {
+    if (j.corpus_fingerprint) fingerprints.add(j.corpus_fingerprint);
+    const finding = {
+      query_id: j.query_id,
+      doc_uri: j.doc_uri,
+      status: "ORPHANED"
+    };
+    if (j.chunk_id !== void 0) finding.chunk_id = j.chunk_id;
+    const sha = j.text_sha ?? (j.chunk_text !== void 0 ? textSha(j.chunk_text) : void 0);
+    const docChunks = index.byDoc.get(j.doc_uri) ?? [];
+    if (j.chunk_id === void 0) {
+      finding.status = docChunks.length > 0 ? "VALID" : "ORPHANED";
+      findings.push(finding);
+      continue;
+    }
+    if (index.byChunkId.has(j.chunk_id)) {
+      finding.status = "VALID";
+      findings.push(finding);
+      continue;
+    }
+    if (sha) {
+      const matches = index.byTextSha.get(sha) ?? [];
+      const sameDoc = matches.filter((chunk) => chunk.doc_uri === j.doc_uri);
+      const target = sameDoc[0] ?? matches[0];
+      if (target) {
+        finding.status = "RE_ANCHORABLE";
+        finding.reanchor_to = target.chunk_id;
+        findings.push(finding);
+        continue;
+      }
+    }
+    if (j.chunk_text !== void 0) {
+      const split = findSplit(j.chunk_text, docChunks);
+      if (split) {
+        finding.status = "SPLIT";
+        finding.split_into = split.map((chunk) => chunk.chunk_id);
+        findings.push(finding);
+        continue;
+      }
+      const merged = findMerged(j.chunk_text, docChunks);
+      if (merged) {
+        finding.status = "MERGED";
+        finding.reanchor_to = merged.chunk_id;
+        findings.push(finding);
+        continue;
+      }
+    }
+    findings.push(finding);
+  }
+  const count = (status) => findings.filter((f) => f.status === status).length;
+  const valid = count("VALID");
+  const total = findings.length;
+  return {
+    findings,
+    summary: {
+      valid,
+      re_anchorable: count("RE_ANCHORABLE"),
+      merged: count("MERGED"),
+      split: count("SPLIT"),
+      orphaned: count("ORPHANED"),
+      invalid_ratio: total === 0 ? 0 : Math.round((total - valid) / total * 1e12) / 1e12
+    },
+    judgments_fingerprint: fingerprints.size === 1 ? [...fingerprints][0] : null,
+    corpus_fingerprint: corpus.corpus_fingerprint ?? options.corpusFingerprint ?? null
+  };
+}
+function fix(judgments, result) {
+  const byQueryAndChunk = /* @__PURE__ */ new Map();
+  for (const finding of result.findings) {
+    byQueryAndChunk.set(`${finding.query_id}\0${finding.chunk_id ?? ""}`, finding);
+  }
+  let reanchored = 0;
+  const needsReview = [];
+  const out = judgments.map((j) => {
+    const finding = byQueryAndChunk.get(`${j.query_id}\0${j.chunk_id ?? ""}`);
+    if (!finding) return j;
+    if ((finding.status === "RE_ANCHORABLE" || finding.status === "MERGED") && finding.reanchor_to) {
+      reanchored++;
+      const next = { ...j, chunk_id: finding.reanchor_to };
+      if (result.corpus_fingerprint) next.corpus_fingerprint = result.corpus_fingerprint;
+      return next;
+    }
+    if (finding.status === "SPLIT" || finding.status === "ORPHANED") needsReview.push(finding);
+    if (finding.status === "VALID" && result.corpus_fingerprint) {
+      return { ...j, corpus_fingerprint: result.corpus_fingerprint };
+    }
+    return j;
+  });
+  return { judgments: out, reanchored, needsReview };
+}
+
+// src/metrics.ts
+function judgmentKey(j) {
+  return j.chunk_id ?? j.doc_uri;
+}
+function dedupe(ranking) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const key of ranking) {
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(key);
+  }
+  return out;
+}
+function dcg(gains) {
+  let total = 0;
+  for (let i = 0; i < gains.length; i++) total += gains[i] / Math.log2(i + 2);
+  return total;
+}
+function queryMetrics(relevance, ranking, options = {}) {
+  const k = options.k ?? 10;
+  const threshold = options.threshold ?? 1;
+  const topK = dedupe(ranking).slice(0, k);
+  const relevantKeys = new Set(
+    [...relevance.entries()].filter(([, r]) => r >= threshold).map(([key]) => key)
+  );
+  const hits = topK.filter((key) => relevantKeys.has(key));
+  const gains = topK.map((key) => 2 ** (relevance.get(key) ?? 0) - 1);
+  const idealGains = [...relevance.values()].map((r) => 2 ** r - 1).sort((a, b) => b - a).slice(0, k);
+  const idcg = dcg(idealGains);
+  let mrr = 0;
+  for (let i = 0; i < topK.length; i++) {
+    if (relevantKeys.has(topK[i])) {
+      mrr = 1 / (i + 1);
+      break;
+    }
+  }
+  let found = 0;
+  let apSum = 0;
+  for (let i = 0; i < topK.length; i++) {
+    if (relevantKeys.has(topK[i])) {
+      found++;
+      apSum += found / (i + 1);
+    }
+  }
+  return {
+    precision: hits.length / k,
+    recall: relevantKeys.size > 0 ? hits.length / relevantKeys.size : 0,
+    ndcg: idcg > 0 ? dcg(gains) / idcg : 0,
+    mrr,
+    ap: relevantKeys.size > 0 ? apSum / relevantKeys.size : 0,
+    hit_rate: hits.length > 0 ? 1 : 0
+  };
+}
+function relevanceByQuery(judgments) {
+  const out = /* @__PURE__ */ new Map();
+  for (const j of judgments) {
+    let inner = out.get(j.query_id);
+    if (!inner) {
+      inner = /* @__PURE__ */ new Map();
+      out.set(j.query_id, inner);
+    }
+    inner.set(judgmentKey(j), j.relevance);
+  }
+  return out;
+}
+var deterministic = (value) => ({
+  value: round(value),
+  n: 1,
+  deterministic: true
+});
+function round(value) {
+  return Math.round(value * 1e12) / 1e12;
+}
+function score(judgments, run, options = {}) {
+  const k = options.k ?? 10;
+  const threshold = options.threshold ?? 1;
+  const byQuery = relevanceByQuery(judgments);
+  const rankings = new Map(run.map((r) => [r.query_id, r.ranking]));
+  const perQuery = /* @__PURE__ */ new Map();
+  const missingQueries = [];
+  const queriesWithoutPositives = [];
+  const queriesWithDuplicates = [];
+  const scored = [];
+  for (const [queryId, relevance] of byQuery) {
+    const ranking = rankings.get(queryId);
+    if (!ranking) missingQueries.push(queryId);
+    else if (dedupe(ranking).length !== ranking.length) queriesWithDuplicates.push(queryId);
+    const metrics = queryMetrics(relevance, ranking ?? [], options);
+    perQuery.set(queryId, metrics);
+    if ([...relevance.values()].some((r) => r >= threshold)) scored.push(metrics);
+    else queriesWithoutPositives.push(queryId);
+  }
+  if (scored.length === 0) {
+    return {
+      metrics: {},
+      perQuery,
+      missingQueries,
+      queriesWithoutPositives,
+      queriesWithDuplicates
+    };
+  }
+  const mean = (pick) => scored.reduce((sum, m) => sum + pick(m), 0) / scored.length;
+  return {
+    metrics: {
+      [`precision@${k}`]: deterministic(mean((m) => m.precision)),
+      [`recall@${k}`]: deterministic(mean((m) => m.recall)),
+      [`ndcg@${k}`]: deterministic(mean((m) => m.ndcg)),
+      [`mrr@${k}`]: deterministic(mean((m) => m.mrr)),
+      [`map@${k}`]: deterministic(mean((m) => m.ap)),
+      [`hit_rate@${k}`]: deterministic(mean((m) => m.hit_rate))
+    },
+    perQuery,
+    missingQueries,
+    queriesWithoutPositives,
+    queriesWithDuplicates
+  };
+}
+function scoreByStratum(judgments, run, options = {}) {
+  const strata = /* @__PURE__ */ new Map();
+  for (const j of judgments) {
+    const name = j.stratum ?? "_unstratified";
+    const list = strata.get(name);
+    if (list) list.push(j);
+    else strata.set(name, [j]);
+  }
+  const out = {};
+  for (const [name, subset] of strata) {
+    const result = score(subset, run, options);
+    out[name] = {
+      n: result.perQuery.size - result.queriesWithoutPositives.length,
+      metrics: result.metrics
+    };
+  }
+  return out;
+}
+function worstStratum(perStratum, metric) {
+  let worst = null;
+  for (const [name, stratum] of Object.entries(perStratum)) {
+    if (stratum.n === 0) continue;
+    const measurement = stratum.metrics[metric];
+    if (!measurement) continue;
+    if (!worst || measurement.value < worst.value) {
+      worst = { name, value: measurement.value, n: stratum.n };
+    }
+  }
+  return worst;
+}
+
+// src/gate.ts
+function parseGate(expression) {
+  const parts = expression.split(":");
+  if (parts[0] === "worst-stratum") {
+    if (parts.length !== 3)
+      throw new Error(`gate '${expression}': expected worst-stratum:<metric>:<floor>`);
+    return {
+      raw: expression,
+      kind: "worst-stratum",
+      metric: parts[1],
+      threshold: parseNumber(parts[2], expression)
+    };
+  }
+  if (parts.length === 3 && parts[1] === "ci-lower") {
+    return {
+      raw: expression,
+      kind: "ci-lower",
+      metric: parts[0],
+      threshold: parseNumber(parts[2], expression)
+    };
+  }
+  if (parts.length !== 2) throw new Error(`gate '${expression}': expected <metric>:<threshold>`);
+  const value = parts[1];
+  return {
+    raw: expression,
+    kind: value.startsWith("-") || value.startsWith("+") ? "delta" : "absolute",
+    metric: parts[0],
+    threshold: parseNumber(value, expression)
+  };
+}
+var NUMBER = /^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/;
+function parseNumber(value, expression) {
+  if (!NUMBER.test(value)) throw new Error(`gate '${expression}': '${value}' is not a number`);
+  return Number.parseFloat(value);
+}
+function worseStatus(a, b) {
+  if (a === "FAIL" || b === "FAIL") return "FAIL";
+  if (a === "INDETERMINATE" || b === "INDETERMINATE") return "INDETERMINATE";
+  return "PASS";
+}
+function evaluateGates({ report: report2, baseline, gates }) {
+  const results = [];
+  const reasons = [];
+  for (const gate of gates) {
+    const result = evaluateGate(gate, report2, baseline);
+    results.push(result);
+    if (result.status === "FAIL" || result.status === "INDETERMINATE") {
+      reasons.push(describe(gate, result, report2));
+    }
+  }
+  const status = results.some((r) => r.status === "FAIL") ? "FAIL" : results.some((r) => r.status === "INDETERMINATE") ? "INDETERMINATE" : "PASS";
+  return { status, results, reasons };
+}
+function evaluateGate(gate, report2, baseline) {
+  if (gate.kind === "worst-stratum") {
+    if (!report2.per_stratum) {
+      return { expression: gate.raw, status: "INDETERMINATE" };
+    }
+    const worst = worstStratum(report2.per_stratum, gate.metric);
+    if (!worst) return { expression: gate.raw, status: "INDETERMINATE" };
+    return {
+      expression: gate.raw,
+      status: worst.value >= gate.threshold ? "PASS" : "FAIL",
+      observed: worst.value
+    };
+  }
+  const measurement = report2.metrics[gate.metric];
+  if (!measurement) return { expression: gate.raw, status: "INDETERMINATE" };
+  if (gate.kind === "ci-lower") {
+    if (!measurement.ci) {
+      return { expression: gate.raw, status: "INDETERMINATE", observed: measurement.value };
+    }
+    const lower = measurement.ci[0];
+    return {
+      expression: gate.raw,
+      status: lower >= gate.threshold ? "PASS" : "FAIL",
+      observed: lower
+    };
+  }
+  if (gate.kind === "delta") {
+    const previous = baseline?.metrics[gate.metric];
+    if (!previous) {
+      return {
+        expression: gate.raw,
+        status: "INDETERMINATE",
+        observed: measurement.value,
+        baseline: null
+      };
+    }
+    const delta = measurement.value - previous.value;
+    return {
+      expression: gate.raw,
+      status: delta >= gate.threshold ? "PASS" : "FAIL",
+      observed: measurement.value,
+      baseline: previous.value
+    };
+  }
+  return {
+    expression: gate.raw,
+    status: measurement.value >= gate.threshold ? "PASS" : "FAIL",
+    observed: measurement.value
+  };
+}
+function describe(gate, result, report2) {
+  if (result.status === "INDETERMINATE") {
+    const nothingScored = report2.judgments.queries_scored === 0 && result.observed === void 0;
+    if (nothingScored && gate.kind !== "worst-stratum") {
+      return `${gate.raw}: no query was scored, so '${gate.metric}' was not computed`;
+    }
+    if (gate.kind === "ci-lower") {
+      return `${gate.raw}: no confidence interval on '${gate.metric}', sample it more than once`;
+    }
+    if (gate.kind === "delta") return `${gate.raw}: no baseline value for '${gate.metric}'`;
+    if (gate.kind === "worst-stratum") {
+      const strata = Object.values(report2.per_stratum ?? {});
+      if (strata.length > 0 && strata.every((stratum) => stratum.n === 0)) {
+        return `${gate.raw}: no stratum was scored for '${gate.metric}'`;
+      }
+      return `${gate.raw}: no per-stratum data for '${gate.metric}'`;
+    }
+    return `${gate.raw}: metric '${gate.metric}' not present in the report`;
+  }
+  if (gate.kind === "delta") {
+    const from = result.baseline ?? 0;
+    const to = result.observed ?? 0;
+    const change = to - from;
+    const direction = change < 0 ? `fell ${(-change).toFixed(4)}` : `rose ${change.toFixed(4)}`;
+    return `${gate.metric} ${direction}, from ${from.toFixed(4)} to ${to.toFixed(4)}, outside ${gate.threshold}`;
+  }
+  if (gate.kind === "worst-stratum") {
+    return `worst stratum ${gate.metric} is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
+  }
+  if (gate.kind === "ci-lower") {
+    return `${gate.metric} lower bound is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
+  }
+  return `${gate.metric} is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
+}
+
+// src/judgments.ts
+function parseJsonl(content, label) {
+  const out = [];
+  const lines = content.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (line === "" || line.startsWith("//")) continue;
+    try {
+      out.push({ value: JSON.parse(line), line: i + 1 });
+    } catch (error) {
+      throw new Error(`${label}:${i + 1}: invalid JSON, ${error.message}`);
+    }
+  }
+  return out;
+}
+function parseJudgments(content, label = "judgments") {
+  const rows = parseJsonl(content, label);
+  for (const { value, line } of rows) {
+    if (typeof value.query_id !== "string" || value.query_id === "")
+      throw new Error(`${label}:${line}: missing query_id`);
+    if (typeof value.doc_uri !== "string" || value.doc_uri === "")
+      throw new Error(`${label}:${line}: missing doc_uri`);
+    if (!Number.isInteger(value.relevance) || value.relevance < 0)
+      throw new Error(`${label}:${line}: relevance must be a non-negative integer`);
+  }
+  return rows.map((row) => row.value);
+}
+function parseRun(content, label = "run") {
+  const rows = parseJsonl(content, label);
+  const firstSeen = /* @__PURE__ */ new Map();
+  for (const { value, line } of rows) {
+    if (typeof value.query_id !== "string" || value.query_id === "")
+      throw new Error(`${label}:${line}: missing query_id`);
+    if (!Array.isArray(value.ranking))
+      throw new Error(`${label}:${line}: ranking must be an array`);
+    for (let i = 0; i < value.ranking.length; i++) {
+      const key = value.ranking[i];
+      if (typeof key !== "string" || key === "")
+        throw new Error(`${label}:${line}: ranking[${i}] must be a non-empty string`);
+    }
+    const previous = firstSeen.get(value.query_id);
+    if (previous !== void 0)
+      throw new Error(
+        `${label}:${line}: duplicate entry for query ${value.query_id}, already on line ${previous}`
+      );
+    firstSeen.set(value.query_id, line);
+  }
+  return rows.map((row) => row.value);
+}
+function byCodePoint(a, b) {
+  const left = [...a];
+  const right = [...b];
+  for (let i = 0; i < Math.min(left.length, right.length); i++) {
+    const difference = (left[i].codePointAt(0) ?? 0) - (right[i].codePointAt(0) ?? 0);
+    if (difference !== 0) return difference;
+  }
+  return left.length - right.length;
+}
+var FIELD_ORDER = [
+  "query_id",
+  "query",
+  "doc_uri",
+  "chunk_id",
+  "text_sha",
+  "chunk_text",
+  "relevance",
+  "corpus_fingerprint",
+  "labeled_by",
+  "labeled_at",
+  "stratum",
+  "notes"
+];
+function serializeJudgments(judgments) {
+  const lines = judgments.map((judgment) => {
+    const ordered = {};
+    for (const field of FIELD_ORDER) {
+      if (judgment[field] !== void 0) ordered[field] = judgment[field];
+    }
+    for (const [key, value] of Object.entries(judgment)) {
+      if (!FIELD_ORDER.includes(key)) ordered[key] = value;
+    }
+    return JSON.stringify(ordered);
+  });
+  return `${lines.join("\n")}
+`;
+}
+function parseCorpus(content, label = "corpus") {
+  const parsed = JSON.parse(content);
+  if (!Array.isArray(parsed.chunks)) throw new Error(`${label}: missing chunks array`);
+  return parsed;
+}
+function validate(judgments) {
+  const issues = [];
+  const queries = /* @__PURE__ */ new Set();
+  const strata = {};
+  const fingerprints = /* @__PURE__ */ new Set();
+  const seen = /* @__PURE__ */ new Set();
+  const queryText = /* @__PURE__ */ new Map();
+  let humanLabels = 0;
+  let syntheticLabels = 0;
+  let positives = 0;
+  for (const j of judgments) {
+    queries.add(j.query_id);
+    if (j.relevance >= 1) positives++;
+    if (j.corpus_fingerprint) fingerprints.add(j.corpus_fingerprint);
+    if (j.labeled_by?.startsWith("human:")) humanLabels++;
+    else if (j.labeled_by?.startsWith("synthetic:")) syntheticLabels++;
+    const stratum = j.stratum ?? "_unstratified";
+    strata[stratum] = (strata[stratum] ?? 0) + 1;
+    const key = `${j.query_id}\0${j.chunk_id ?? j.doc_uri}`;
+    if (seen.has(key)) {
+      issues.push({
+        severity: "error",
+        code: "duplicate-label",
+        message: `duplicate judgment for query ${j.query_id} and target ${j.chunk_id ?? j.doc_uri}`
+      });
+    }
+    seen.add(key);
+    if (j.query) {
+      const existing = queryText.get(j.query_id);
+      if (existing !== void 0 && existing !== j.query) {
+        issues.push({
+          severity: "error",
+          code: "inconsistent-query-text",
+          message: `query ${j.query_id} has two different query strings`
+        });
+      }
+      queryText.set(j.query_id, j.query);
+    }
+    if (j.chunk_id && !/^c1:[0-9a-f]{32}$/.test(j.chunk_id)) {
+      issues.push({
+        severity: "error",
+        code: "bad-chunk-id",
+        message: `malformed chunk_id on query ${j.query_id}: ${j.chunk_id}`
+      });
+    }
+    if (j.text_sha && !/^t1:[0-9a-f]{32}$/.test(j.text_sha)) {
+      issues.push({
+        severity: "error",
+        code: "bad-text-sha",
+        message: `malformed text_sha on query ${j.query_id}: ${j.text_sha}`
+      });
+    }
+  }
+  for (const queryId of [...queries].sort(byCodePoint)) {
+    if (!queryText.has(queryId)) {
+      issues.push({
+        severity: "warning",
+        code: "missing-query-text",
+        message: `query ${queryId} has no query text on any row`
+      });
+    }
+  }
+  if (judgments.length === 0) {
+    issues.push({ severity: "error", code: "empty", message: "no judgments" });
+  }
+  if (positives === 0 && judgments.length > 0) {
+    issues.push({
+      severity: "error",
+      code: "no-positives",
+      message: "no judgment has relevance >= 1, so recall is undefined for every query"
+    });
+  }
+  if (humanLabels === 0 && syntheticLabels > 0) {
+    issues.push({
+      severity: "warning",
+      code: "no-human-labels",
+      message: "every label is synthetic. Without human labels you cannot measure judge calibration, and synthetic labels quietly become ground truth"
+    });
+  }
+  if (fingerprints.size > 1) {
+    issues.push({
+      severity: "warning",
+      code: "mixed-fingerprints",
+      message: `judgments span ${fingerprints.size} corpus fingerprints; run 'drift' before trusting any metric`
+    });
+  }
+  for (const [name, count] of Object.entries(strata).sort(([a], [b]) => byCodePoint(a, b))) {
+    if (name !== "_unstratified" && count < 5) {
+      issues.push({
+        severity: "warning",
+        code: "thin-stratum",
+        message: `stratum '${name}' has only ${count} labels, too few to gate on`
+      });
+    }
+  }
+  const withChunkId = judgments.filter((j) => j.chunk_id).length;
+  if (withChunkId > 0 && withChunkId < judgments.length) {
+    issues.push({
+      severity: "warning",
+      code: "mixed-granularity",
+      message: `${withChunkId}/${judgments.length} labels have chunk_id; the rest are document-level`
+    });
+  }
+  if (withChunkId === 0 && judgments.length > 0) {
+    issues.push({
+      severity: "warning",
+      code: "no-chunk-ids",
+      message: "no label has a chunk_id, so drift can only work at document granularity. See spec/chunk-id.md"
+    });
+  }
+  const fingerprint = fingerprints.size === 1 ? [...fingerprints][0] : null;
+  return {
+    issues,
+    queries: queries.size,
+    labels: judgments.length,
+    humanLabels,
+    syntheticLabels,
+    fingerprint,
+    strata,
+    ok: !issues.some((i) => i.severity === "error")
+  };
+}
+
+// src/report.ts
+var TOOL_NAME = "retrieval-eval";
+var TOOL_VERSION = "0.1.0";
+var SPEC_VERSION = "1";
+function buildReport(options) {
+  const { judgments, run, corpus, driftResult } = options;
+  const metricOptions = { k: options.k ?? 10, threshold: options.threshold ?? 1 };
+  const scored = score(judgments, run, metricOptions);
+  const validation = validate(judgments);
+  const hasStrata = judgments.some((j) => j.stratum !== void 0);
+  const corpusInfo = { fingerprint: corpus?.corpus_fingerprint ?? null };
+  if (corpus) {
+    corpusInfo.documents = new Set(corpus.chunks.map((c) => c.doc_uri)).size;
+    corpusInfo.chunks = corpus.chunks.length;
+  }
+  const reasons = [];
+  if (driftResult && driftResult.summary.invalid_ratio > 0) {
+    reasons.push(
+      `${Math.round(driftResult.summary.invalid_ratio * 100)}% of judgments no longer match the live corpus`
+    );
+  }
+  const missingAndScored = scored.missingQueries.filter(
+    (id) => !scored.queriesWithoutPositives.includes(id)
+  );
+  if (missingAndScored.length > 0) {
+    reasons.push(`${missingAndScored.length} judged queries had no run entry and scored zero`);
+  }
+  const queriesScored = validation.queries - scored.queriesWithoutPositives.length;
+  if (scored.queriesWithoutPositives.length > 0 && queriesScored > 0) {
+    reasons.push(
+      `${scored.queriesWithoutPositives.length} judged queries have no label at relevance >= ${metricOptions.threshold} and were excluded from the averages`
+    );
+  }
+  if (queriesScored === 0) {
+    reasons.push(
+      validation.queries === 0 ? "no judged queries, so no metric was computed" : `no query had a label at relevance >= ${metricOptions.threshold}, so no metric was computed`
+    );
+  }
+  if (scored.queriesWithDuplicates.length > 0) {
+    reasons.push(
+      `${scored.queriesWithDuplicates.length} queries repeated a key in their ranking; only the first occurrence of each was counted`
+    );
+  }
+  const validationErrors = validation.issues.filter((issue) => issue.severity === "error");
+  if (validationErrors.length > 0) {
+    const codes = [...new Set(validationErrors.map((issue) => issue.code))].sort(byCodePoint);
+    reasons.push(
+      `the judgment set is unsound (${codes.join(", ")}); run 'retrieval-eval validate' for detail. No metric computed from it can be trusted`
+    );
+  }
+  const report2 = {
+    spec_version: SPEC_VERSION,
+    tool: { name: TOOL_NAME, version: TOOL_VERSION },
+    generated_at: (options.now ?? /* @__PURE__ */ new Date()).toISOString(),
+    corpus: corpusInfo,
+    judgments: {
+      queries: validation.queries,
+      queries_scored: queriesScored,
+      labels: validation.labels,
+      fingerprint: validation.fingerprint,
+      human_labels: validation.humanLabels,
+      synthetic_labels: validation.syntheticLabels,
+      ...driftResult ? { drift: driftResult.summary } : {}
+    },
+    metrics: scored.metrics,
+    ...hasStrata ? { per_stratum: scoreByStratum(judgments, run, metricOptions) } : {},
+    verdict: {
+      status: queriesScored === 0 || validationErrors.length > 0 ? "INDETERMINATE" : "PASS",
+      reasons
+    }
+  };
+  return report2;
+}
+
+// src/help.ts
+var DOCS = "https://github.com/aton-of-data/retrieval-eval";
+var COMMANDS = ["drift", "score", "validate", "convert"];
+var ROOT_HELP = `retrieval-eval ${TOOL_VERSION}
+Does your retrieval find the right things, and are your labels still true?
+
+Usage
+  retrieval-eval <command> [options]
+
+Commands
+  drift      report which relevance labels survived a re-chunk, and re-anchor them
+  score      compute deterministic retrieval metrics and gate a build on them
+  validate   check a judgment set for problems before you trust its numbers
+  convert    move labels and runs between this format and TREC qrels
+
+Global options
+  -h, --help          show this help, or 'retrieval-eval <command> --help'
+  -v, --version       print the version and exit
+      --json          emit machine-readable JSON on stdout
+      --color <when>  auto, always or never (default auto; NO_COLOR is honored)
+
+Exit codes
+  0  success, every gate passed
+  1  a gate failed, a label decayed, or validation found an error
+  2  usage error, unreadable file, or malformed input
+
+Examples
+  retrieval-eval drift    --judgments judgments.jsonl --corpus corpus.json --fix
+  retrieval-eval score    --judgments judgments.jsonl --run hits.jsonl -k 5
+  retrieval-eval validate --judgments judgments.jsonl
+  retrieval-eval convert  --judgments judgments.jsonl --to qrels
+
+Docs  ${DOCS}`;
+var DRIFT_HELP = `retrieval-eval drift
+Report which relevance labels are still true against the live corpus.
+
+Usage
+  retrieval-eval drift --judgments <file> --corpus <file> [options]
+
+Options
+      --judgments <file>  the labels to check, JSONL
+      --corpus <file>     the live corpus, JSON
+      --fix               re-anchor recoverable labels in place, rewriting --judgments
+      --json              emit every finding as JSON
+      --color <when>      auto, always or never
+  -h, --help              show this help
+
+Label classes
+  VALID          the labeled chunk_id is still in the corpus
+  RE_ANCHORABLE  the labeled text moved to a new chunk_id, --fix recovers it
+  MERGED         the labeled text was absorbed into a coarser chunk, --fix recovers it
+  SPLIT          the labeled text now spans several chunks, a human must re-judge
+  ORPHANED       the labeled text or its document is gone
+
+--fix never touches SPLIT or ORPHANED labels. Guessing at them would fabricate ground truth,
+which is the failure this command exists to expose.
+
+Exit codes
+  0  every label still points at the text it was written for
+  1  at least one label has decayed
+  2  usage error, unreadable file, or malformed input
+
+Docs  ${DOCS}/blob/main/spec/drift.md`;
+var SCORE_HELP = `retrieval-eval score
+Compute deterministic retrieval metrics and gate a build on them.
+
+Usage
+  retrieval-eval score --judgments <file> --run <file> [options]
+
+Options
+      --judgments <file>  relevance labels, JSONL
+      --run <file>        ranked results per query, JSONL
+      --corpus <file>     also report judgment drift alongside the metrics
+  -k, --k <n>             rank cutoff for the @k metrics (default 10)
+      --threshold <n>     lowest relevance counted as relevant (default 1)
+      --gate <expr>       add a gate, repeatable; see below
+      --baseline <file>   a previous report, required by delta gates
+      --out <file>        write the JSON report to this file
+      --json              print the JSON report on stdout
+      --color <when>      auto, always or never
+  -h, --help              show this help
+
+Metrics
+  precision@k  recall@k  ndcg@k  mrr  map  hit_rate@k
+
+Gate expressions, repeatable
+  recall@5:0.8                 absolute floor
+  recall@5:-0.02               regression tolerance against --baseline
+  worst-stratum:recall@5:0.7   floor on the weakest query class, so averages cannot hide it
+  faithfulness:ci-lower:0.8    floor on the lower confidence bound
+
+Exit codes
+  0  every gate passed
+  1  a gate failed or could not be decided
+  2  usage error, unreadable file, or malformed input
+
+Docs  ${DOCS}`;
+var VALIDATE_HELP = `retrieval-eval validate
+Check a judgment set for problems before you trust its numbers.
+
+Usage
+  retrieval-eval validate --judgments <file> [options]
+
+Options
+      --judgments <file>  the labels to check, JSONL
+      --json              emit every issue as JSON
+      --color <when>      auto, always or never
+  -h, --help              show this help
+
+This reports more than schema conformance. A judgment set can parse cleanly and still be
+unsound: every label synthetic, labels spanning several corpus fingerprints, a stratum too thin
+to gate on, or no positive label at all, which leaves recall undefined.
+
+Exit codes
+  0  no errors, warnings may still be printed
+  1  at least one error
+  2  usage error, unreadable file, or malformed input
+
+Docs  ${DOCS}/blob/main/spec/README.md`;
+var CONVERT_HELP = `retrieval-eval convert
+Move labels and runs between this format and TREC qrels.
+
+Usage
+  retrieval-eval convert --judgments <file> --to qrels
+  retrieval-eval convert --run <file> --to trec-run
+  retrieval-eval convert --qrels <file> --to judgments [--as-chunk-ids]
+
+Options
+      --to <format>       qrels, trec-run or judgments
+      --judgments <file>  source labels, for --to qrels
+      --run <file>        source run, for --to trec-run
+      --qrels <file>      source qrels, for --to judgments
+      --as-chunk-ids      treat the qrels doc_id as a chunk_id rather than a doc_uri
+      --color <when>      auto, always or never
+  -h, --help              show this help
+
+A judgments file is a strict superset of qrels, so one conversion reaches trec_eval,
+ir_measures, pytrec_eval, BEIR and ir_datasets. Converted output goes to stdout.
+
+Exit codes
+  0  converted
+  2  usage error, unreadable file, or malformed input
+
+Docs  ${DOCS}`;
+var COMMAND_HELP = {
+  drift: DRIFT_HELP,
+  score: SCORE_HELP,
+  validate: VALIDATE_HELP,
+  convert: CONVERT_HELP
+};
+
+// src/qrels.ts
+function toQrels(judgments) {
+  const lines = judgments.map((j) => `${j.query_id} 0 ${j.chunk_id ?? j.doc_uri} ${j.relevance}`);
+  return `${lines.join("\n")}
+`;
+}
+function fromQrels(content, options = {}) {
+  const out = [];
+  const lines = content.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (line === "" || line.startsWith("#")) continue;
+    const parts = line.split(/\s+/);
+    if (parts.length < 3)
+      throw new Error(`qrels:${i + 1}: expected 3 or 4 fields, got ${parts.length}`);
+    const [queryId, second, third, fourth] = parts;
+    const docId = parts.length >= 4 ? third : second;
+    const relevance = parts.length >= 4 ? fourth : third;
+    const parsed = Number.parseInt(relevance, 10);
+    if (Number.isNaN(parsed)) {
+      if (out.length === 0 && /^query[-_ ]?id$/i.test(queryId)) continue;
+      throw new Error(`qrels:${i + 1}: relevance '${relevance}' is not an integer`);
+    }
+    const judgment = {
+      query_id: queryId,
+      doc_uri: docId,
+      // Negative relevance appears in some TREC collections; clamp to the schema's minimum.
+      relevance: Math.max(0, parsed)
+    };
+    if (options.asChunkIds) judgment.chunk_id = docId;
+    if (options.corpusFingerprint) judgment.corpus_fingerprint = options.corpusFingerprint;
+    if (options.labeledBy) judgment.labeled_by = options.labeledBy;
+    out.push(judgment);
+  }
+  return out;
+}
+function toTrecRun(run, runName = "retrieval-eval") {
+  const lines = [];
+  for (const entry of run) {
+    entry.ranking.forEach((docId, index) => {
+      const score2 = (entry.ranking.length - index).toFixed(4);
+      lines.push(`${entry.query_id} Q0 ${docId} ${index + 1} ${score2} ${runName}`);
+    });
+  }
+  return `${lines.join("\n")}
+`;
+}
+
+// src/ui.ts
+var CODES = {
+  reset: "\x1B[0m",
+  dim: "\x1B[2m",
+  bold: "\x1B[1m",
+  red: "\x1B[31m",
+  green: "\x1B[32m",
+  yellow: "\x1B[33m",
+  cyan: "\x1B[36m"
+};
+var enabled = false;
+function setColor(when, isTty, env) {
+  if (when === "never") enabled = false;
+  else if (when === "always") enabled = true;
+  else enabled = isTty && env.NO_COLOR === void 0;
+}
+function paint(color, text) {
+  return enabled ? `${CODES[color]}${text}${CODES.reset}` : text;
+}
+var num = (value) => value.toFixed(4);
+var pct = (value) => `${(value * 100).toFixed(0)}%`;
+function bar(value, width = 8) {
+  const clamped = Math.min(Math.max(value, 0), 1);
+  const filled = Math.round(clamped * width);
+  return "\u2588".repeat(filled) + "\u2591".repeat(width - filled);
+}
+function heading(text) {
+  return `
+  ${paint("dim", text)}
+`;
+}
+
+// src/cli.ts
+var processIo = {
+  out: (text) => process.stdout.write(text),
+  err: (text) => process.stderr.write(text),
+  isTty: process.stdout.isTTY === true,
+  env: process.env
+};
+var EXIT_OK = 0;
+var EXIT_FAILED = 1;
+var EXIT_USAGE = 2;
+var CliError = class extends Error {
+  constructor(message, hint) {
+    super(message);
+    this.hint = hint;
+  }
+  hint;
+};
+var OPTIONS = {
+  judgments: { type: "string" },
+  run: { type: "string" },
+  corpus: { type: "string" },
+  qrels: { type: "string" },
+  baseline: { type: "string" },
+  out: { type: "string" },
+  to: { type: "string" },
+  k: { type: "string", short: "k" },
+  threshold: { type: "string" },
+  color: { type: "string" },
+  gate: { type: "string", multiple: true },
+  fix: { type: "boolean", default: false },
+  "as-chunk-ids": { type: "boolean", default: false },
+  json: { type: "boolean", default: false },
+  help: { type: "boolean", short: "h", default: false },
+  version: { type: "boolean", short: "v", default: false }
+};
+function main(argv, io = processIo) {
+  let parsed;
+  try {
+    parsed = parseArgs({ args: argv, options: OPTIONS, allowPositionals: true });
+  } catch (error) {
+    const unknown = /Unknown option '([^']+)'/.exec(error.message);
+    const message = unknown ? `unknown option '${unknown[1]}'` : error.message;
+    return report(io, new CliError(message, "retrieval-eval --help"));
+  }
+  const { values, positionals } = parsed;
+  setColor("auto", io.isTty, io.env);
+  try {
+    setColor(resolveColor(values.color), io.isTty, io.env);
+    if (values.version) {
+      io.out(`${TOOL_VERSION}
+`);
+      return EXIT_OK;
+    }
+    const [first, second] = positionals;
+    const command = first === "help" ? second : first;
+    if (command === void 0) {
+      io.out(`${ROOT_HELP}
+`);
+      return values.help || first === "help" ? EXIT_OK : EXIT_USAGE;
+    }
+    if (!isCommand(command)) {
+      const near = closest(command);
+      throw new CliError(
+        `unknown command '${command}'`,
+        near ? `retrieval-eval ${near}` : "retrieval-eval --help"
+      );
+    }
+    if (values.help || first === "help") {
+      io.out(`${COMMAND_HELP[command]}
+`);
+      return EXIT_OK;
+    }
+    switch (command) {
+      case "drift":
+        return runDrift(values, io);
+      case "score":
+        return runScore(values, io);
+      case "validate":
+        return runValidate(values, io);
+      case "convert":
+        return runConvert(values, io);
+    }
+  } catch (error) {
+    return report(io, error);
+  }
+}
+function isCommand(value) {
+  return COMMANDS.includes(value);
+}
+function closest(typo) {
+  let best;
+  let bestDistance = Number.POSITIVE_INFINITY;
+  for (const candidate of COMMANDS) {
+    const distance = editDistance(typo, candidate);
+    if (distance < bestDistance) {
+      bestDistance = distance;
+      best = candidate;
+    }
+  }
+  return bestDistance <= 3 ? best : void 0;
+}
+function editDistance(a, b) {
+  let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
+  for (let i = 1; i <= a.length; i++) {
+    const current = [i];
+    for (let j = 1; j <= b.length; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      current[j] = Math.min(
+        current[j - 1] + 1,
+        previous[j] + 1,
+        previous[j - 1] + cost
+      );
+    }
+    previous = current;
+  }
+  return previous[b.length];
+}
+function resolveColor(value) {
+  if (value === void 0) return "auto";
+  if (value === "auto" || value === "always" || value === "never") return value;
+  throw new CliError(`--color expects auto, always or never, got '${value}'`);
+}
+function report(io, error) {
+  const message = error instanceof Error ? error.message : String(error);
+  io.err(`${paint("red", "error")}  ${message}
+`);
+  if (error instanceof CliError && error.hint !== void 0) {
+    io.err(`${paint("dim", "        try")} ${error.hint}
+`);
+  }
+  return EXIT_USAGE;
+}
+var READ_ERRORS = {
+  ENOENT: "no such file or directory",
+  EACCES: "permission denied",
+  EISDIR: "is a directory"
+};
+function read(path) {
+  try {
+    return readFileSync(path, "utf8");
+  } catch (error) {
+    const code = error.code ?? "";
+    throw new CliError(`cannot read ${path}: ${READ_ERRORS[code] ?? error.message}`);
+  }
+}
+function required(values, command, flags) {
+  const missing = flags.filter((flag) => values[flag] === void 0);
+  if (missing.length > 0) {
+    throw new CliError(
+      `${command} needs ${flags.map((flag) => `--${flag}`).join(" and ")}`,
+      `retrieval-eval ${command} --help`
+    );
+  }
+}
+function integer(raw, flag, fallback) {
+  if (raw === void 0) return fallback;
+  const value = Number.parseInt(raw, 10);
+  if (!Number.isInteger(value) || value < 1) {
+    throw new CliError(`${flag} expects a positive integer, got '${raw}'`);
+  }
+  return value;
+}
+function runDrift(values, io) {
+  required(values, "drift", ["judgments", "corpus"]);
+  const path = values.judgments;
+  const judgments = parseJudgments(read(path));
+  const corpus = parseCorpus(read(values.corpus));
+  const result = drift(judgments, corpus);
+  if (values.json) io.out(`${JSON.stringify(result, null, 2)}
+`);
+  else printDrift(io, result, judgments.length, values.fix === true);
+  if (values.fix) {
+    const fixed = fix(judgments, result);
+    writeFileSync(path, serializeJudgments(fixed.judgments), "utf8");
+    io.out(
+      `
+  ${paint("green", "fixed")}   re-anchored ${fixed.reanchored} label(s) in ${path}
+  ${paint("yellow", "review")}  ${fixed.needsReview.length} label(s) still need a human
+`
+    );
+  }
+  return result.summary.invalid_ratio > 0 ? EXIT_FAILED : EXIT_OK;
+}
+function printDrift(io, result, total, fixing) {
+  const s = result.summary;
+  const row = (mark, color, count, label, note) => `  ${paint(color, mark.padEnd(2))} ${String(count).padStart(3)}  ${label.padEnd(14)} ${paint("dim", note)}
+`;
+  let out = "\n";
+  out += `  ${total} judgments \xB7 labeled @ fingerprint ${paint("cyan", result.judgments_fingerprint ?? "unknown")}
+`;
+  out += `  live corpus    @ fingerprint ${paint("cyan", result.corpus_fingerprint ?? "unknown")}
+
+`;
+  out += row("ok", "green", s.valid, "VALID", "chunk_id still present");
+  out += row("!", "yellow", s.re_anchorable, "RE-ANCHORABLE", "text moved to a new chunk_id");
+  out += row("!", "yellow", s.merged, "MERGED", "text absorbed into a coarser chunk");
+  out += row("!", "yellow", s.split, "SPLIT", "labeled text now spans 2+ chunks");
+  out += row("x", "red", s.orphaned, "ORPHANED", "source text or document is gone");
+  if (s.invalid_ratio > 0) {
+    const recoverable = s.re_anchorable + s.merged;
+    const needsHuman = s.split + s.orphaned;
+    out += `
+  ${paint("bold", `${pct(s.invalid_ratio)} of your judgment set no longer matches the live corpus.`)}
+`;
+    out += `  ${paint("dim", "Any metric computed against it is measuring two changes at once.")}
+`;
+    out += `  ${paint("dim", `${recoverable} recoverable automatically \xB7 ${needsHuman} need a human`)}
+`;
+    if (recoverable > 0 && !fixing) {
+      out += `
+  ${paint("dim", "next")}    retrieval-eval drift --fix, to re-anchor the recoverable labels
+`;
+    }
+  } else {
+    out += `
+  ${paint("green", "Every label still points at the text it was written for.")}
+`;
+  }
+  io.out(out);
+}
+function runScore(values, io) {
+  required(values, "score", ["judgments", "run"]);
+  const judgments = parseJudgments(read(values.judgments));
+  const run = parseRun(read(values.run));
+  const k = integer(values.k, "-k", 10);
+  const threshold = integer(values.threshold, "--threshold", 1);
+  const corpus = values.corpus ? parseCorpus(read(values.corpus)) : void 0;
+  const driftResult = corpus ? drift(judgments, corpus) : void 0;
+  const report2 = buildReport({ judgments, run, k, threshold, corpus, driftResult });
+  const gates = (values.gate ?? []).map(parseGate);
+  if (gates.length > 0) {
+    const baseline = values.baseline ? JSON.parse(read(values.baseline)) : void 0;
+    const evaluated = evaluateGates({ report: report2, baseline, gates });
+    report2.verdict.status = worseStatus(report2.verdict.status, evaluated.status);
+    report2.verdict.gates = evaluated.results;
+    report2.verdict.reasons = [...report2.verdict.reasons, ...evaluated.reasons];
+  }
+  if (values.out) writeFileSync(values.out, `${JSON.stringify(report2, null, 2)}
+`, "utf8");
+  if (values.json) io.out(`${JSON.stringify(report2, null, 2)}
+`);
+  else printScore(io, report2);
+  return report2.verdict.status === "PASS" ? EXIT_OK : EXIT_FAILED;
+}
+function printScore(io, report2) {
+  let out = `
+  ${report2.judgments.queries} queries \xB7 ${report2.judgments.labels} labels`;
+  if (report2.judgments.human_labels !== void 0) {
+    out += paint(
+      "dim",
+      ` (${report2.judgments.human_labels} human, ${report2.judgments.synthetic_labels ?? 0} synthetic)`
+    );
+  }
+  out += "\n\n";
+  if (Object.keys(report2.metrics).length === 0) {
+    out += `  ${paint("dim", "no metric was computed")}
+`;
+  }
+  for (const [name, measurement] of Object.entries(report2.metrics)) {
+    const ci = measurement.ci ? paint("dim", `  [${num(measurement.ci[0])}, ${num(measurement.ci[1])}]`) : "";
+    out += `  ${name.padEnd(16)} ${num(measurement.value)}${ci}
+`;
+  }
+  if (report2.per_stratum) {
+    const names = Object.keys(report2.metrics);
+    const primary = names.find((name) => name.startsWith("recall@")) ?? names[0] ?? "";
+    const entries = Object.entries(report2.per_stratum).map(([name, stratum]) => ({
+      name,
+      n: stratum.n,
+      value: stratum.metrics[primary]?.value ?? 0
+    }));
+    const ranked = entries.filter((stratum) => stratum.n > 0).sort((a, b) => a.value - b.value || byCodePoint(a.name, b.name));
+    const unscored = entries.filter((stratum) => stratum.n === 0).sort((a, b) => byCodePoint(a.name, b.name));
+    out += heading(primary ? `${primary} by stratum` : "strata");
+    for (const [index, stratum] of ranked.entries()) {
+      const flag = index === 0 && ranked.length > 1 ? paint("yellow", "  worst") : "";
+      out += `  ${paint("dim", "\xB7")} ${stratum.name.padEnd(20)} ${num(stratum.value)}  ${paint("dim", bar(stratum.value))}  ${paint("dim", `n=${stratum.n}`)}${flag}
+`;
+    }
+    for (const stratum of unscored) {
+      out += `  ${paint("dim", "\xB7")} ${stratum.name.padEnd(20)} ${paint("dim", "not scored, no label at the relevance threshold")}
+`;
+    }
+  }
+  if (report2.judgments.drift && report2.judgments.drift.invalid_ratio > 0) {
+    out += `
+  ${paint("yellow", "warning")} ${pct(report2.judgments.drift.invalid_ratio)} of judgments no longer match the corpus
+`;
+  }
+  const gates = report2.verdict.gates ?? [];
+  if (gates.length > 0) {
+    out += heading("gates");
+    for (const gate of gates) {
+      const mark = gate.status === "PASS" ? paint("green", "ok") : gate.status === "FAIL" ? paint("red", "x ") : paint("yellow", "? ");
+      out += `  ${mark} ${gate.expression}
+`;
+    }
+  }
+  for (const reason of report2.verdict.reasons) {
+    out += `  ${paint("dim", `\u2192 ${reason}`)}
+`;
+  }
+  const color = report2.verdict.status === "PASS" ? "green" : report2.verdict.status === "FAIL" ? "red" : "yellow";
+  out += `
+  ${paint(color, paint("bold", report2.verdict.status))}
+`;
+  io.out(out);
+}
+function runValidate(values, io) {
+  required(values, "validate", ["judgments"]);
+  const result = validate(parseJudgments(read(values.judgments)));
+  if (values.json) {
+    io.out(`${JSON.stringify(result, null, 2)}
+`);
+    return result.ok ? EXIT_OK : EXIT_FAILED;
+  }
+  let out = `
+  ${result.labels} labels \xB7 ${result.queries} queries \xB7 ${Object.keys(result.strata).length} strata
+
+`;
+  if (result.issues.length === 0) {
+    out += `  ${paint("green", "ok")}      no issues
+`;
+  } else {
+    for (const issue of result.issues) {
+      const mark = issue.severity === "error" ? paint("red", "error  ") : paint("yellow", "warning");
+      out += `  ${mark} ${paint("dim", `[${issue.code}]`)} ${issue.message}
+`;
+    }
+  }
+  io.out(out);
+  return result.ok ? EXIT_OK : EXIT_FAILED;
+}
+function runConvert(values, io) {
+  const target = values.to;
+  if (target === void 0) {
+    throw new CliError(
+      "convert needs --to qrels, trec-run or judgments",
+      "retrieval-eval convert --help"
+    );
+  }
+  switch (target) {
+    case "judgments": {
+      required(values, "convert", ["qrels"]);
+      const judgments = fromQrels(read(values.qrels), {
+        asChunkIds: values["as-chunk-ids"] === true
+      });
+      io.out(serializeJudgments(judgments));
+      return EXIT_OK;
+    }
+    case "qrels": {
+      required(values, "convert", ["judgments"]);
+      io.out(toQrels(parseJudgments(read(values.judgments))));
+      return EXIT_OK;
+    }
+    case "trec-run": {
+      required(values, "convert", ["run"]);
+      io.out(toTrecRun(parseRun(read(values.run))));
+      return EXIT_OK;
+    }
+    default:
+      throw new CliError(
+        `unknown --to '${target}', expected qrels, trec-run or judgments`,
+        "retrieval-eval convert --help"
+      );
+  }
+}
+function isEntrypoint(entry) {
+  if (entry === void 0) return false;
+  try {
+    return import.meta.url === pathToFileURL(realpathSync(entry)).href;
+  } catch {
+    return false;
+  }
+}
+if (isEntrypoint(process.argv[1])) {
+  process.exit(main(process.argv.slice(2)));
+}
+export {
+  main
+};