retrieval-eval 0.1.0 → 0.1.1

package/README.md

@@ -291,7 +291,7 @@ anywhere, which is the point of a spec.
 
 ## Status
 
-**0.1.0, alpha.** The formats and the metric mathematics are what we intend to keep. The CLI
+**0.1.1, alpha.** The formats and the metric mathematics are what we intend to keep. The CLI
 surface may still move. The `c1:` and `t1:` hash prefixes exist so identity can be versioned
 without breaking existing judgment files.
 

package/dist/cli.js

@@ -658,7 +658,7 @@ function validate(judgments) {
 
 // src/report.ts
 var TOOL_NAME = "retrieval-eval";
-var TOOL_VERSION = "0.1.0";
+var TOOL_VERSION = "0.1.1";
 var SPEC_VERSION = "1";
 function buildReport(options) {
   const { judgments, run, corpus, driftResult } = options;

package/dist/index.d.ts

@@ -1,20 +1,3 @@
-/**
- * Normalize chunk text. Exactly three steps, in this order. Any deviation breaks
- * cross-language agreement with the Python implementation. See `spec/chunk-id.md`.
- */
-declare function normalize(text: string): string;
-/** Hash of the normalized text, so a judgment can describe itself without storing the text. */
-declare function textSha(text: string): string;
-interface ChunkIdInput {
-    docUri: string;
-    docRevision: string;
-    ordinal: number;
-    text: string;
-    chunkerFingerprint: string;
-}
-/** Content-addressed chunk id: a label points at text, not at a position in a list. */
-declare function chunkId(input: ChunkIdInput): string;
-
 /** One relevance judgment. A judgments file is JSONL: one of these per line. */
 interface Judgment {
     query_id: string;
@@ -127,6 +110,128 @@ interface DriftResult {
     corpus_fingerprint: string | null;
 }
 
+interface DriftOptions {
+    /** Corpus fingerprint to record when judgments carry none. */
+    corpusFingerprint?: string;
+}
+/**
+ * Classify every judgment against a live corpus.
+ *
+ * This is the question no other evaluation tool can answer: after you changed your chunker,
+ * which of your labels still mean what they meant when a human wrote them?
+ *
+ * - `VALID`          the labeled chunk_id is still present
+ * - `RE_ANCHORABLE`  the id is stale, but the exact text is still a live chunk
+ * - `MERGED`         the text was absorbed into a coarser chunk; re-anchors safely
+ * - `SPLIT`          the labeled text now spans two or more live chunks, so it needs re-judging
+ * - `ORPHANED`       the text or its document is gone
+ */
+declare function drift(judgments: Judgment[], corpus: Corpus, options?: DriftOptions): DriftResult;
+interface FixResult {
+    judgments: Judgment[];
+    reanchored: number;
+    /** Labels left untouched because they need a human: SPLIT and ORPHANED. */
+    needsReview: DriftFinding[];
+}
+/**
+ * Re-anchor the recoverable labels onto their new chunk_ids and stamp the new fingerprint.
+ *
+ * RE_ANCHORABLE and MERGED are recoverable: in both cases the text a human judged is still
+ * there. SPLIT and ORPHANED are deliberately left alone, because guessing at them would
+ * silently fabricate ground truth, the exact failure this tool exists to expose.
+ */
+declare function fix(judgments: Judgment[], result: DriftResult): FixResult;
+
+/**
+ * A gate expression. Four forms, because absolute thresholds get tuned until they pass:
+ *
+ *   recall@5:0.8              absolute floor
+ *   recall@5:-0.02            delta against a baseline (regression tolerance)
+ *   worst-stratum:recall@5:0.7  floor on the weakest query class
+ *   faithfulness:ci-lower:0.8   floor on the lower confidence bound, so judge noise cannot pass
+ */
+interface Gate {
+    raw: string;
+    kind: "absolute" | "delta" | "worst-stratum" | "ci-lower";
+    metric: string;
+    threshold: number;
+}
+declare function parseGate(expression: string): Gate;
+/**
+ * The more serious of two verdicts: FAIL beats INDETERMINATE beats PASS.
+ *
+ * A gate only knows about the numbers it was pointed at. It cannot clear a finding it never
+ * looked at, such as a judgment set `validate` rejects, so a passing gate never upgrades a
+ * verdict that was already worse.
+ */
+declare function worseStatus(a: Status, b: Status): Status;
+interface EvaluateGatesOptions {
+    report: Report;
+    baseline?: Report | undefined;
+    gates: Gate[];
+}
+declare function evaluateGates({ report, baseline, gates }: EvaluateGatesOptions): {
+    status: Status;
+    results: GateResult[];
+    reasons: string[];
+};
+
+/**
+ * Normalize chunk text. Exactly three steps, in this order. Any deviation breaks
+ * cross-language agreement with the Python implementation. See `spec/chunk-id.md`.
+ */
+declare function normalize(text: string): string;
+/** Hash of the normalized text, so a judgment can describe itself without storing the text. */
+declare function textSha(text: string): string;
+interface ChunkIdInput {
+    docUri: string;
+    docRevision: string;
+    ordinal: number;
+    text: string;
+    chunkerFingerprint: string;
+}
+/** Content-addressed chunk id: a label points at text, not at a position in a list. */
+declare function chunkId(input: ChunkIdInput): string;
+
+declare function parseJudgments(content: string, label?: string): Judgment[];
+/**
+ * Parse a run JSONL file.
+ *
+ * The run is the one input that used to be taken on trust, and an unchecked run is how a
+ * metric goes out of range: a ranking holding a non-string, or two entries claiming the same
+ * query, produce numbers that mean nothing and say nothing about it.
+ *
+ * Two entries for one query are rejected rather than resolved, because the file no longer says
+ * what the ranking for that query is, and picking one silently is a guess. A key repeated
+ * *within* one ranking is a different thing: the ranking is still unambiguous, so it is
+ * accepted here and counted once by `score`, which reports that it did.
+ */
+declare function parseRun(content: string, label?: string): RunEntry[];
+declare function serializeJudgments(judgments: Judgment[]): string;
+declare function parseCorpus(content: string, label?: string): Corpus;
+type Severity = "error" | "warning";
+interface ValidationIssue {
+    severity: Severity;
+    code: string;
+    message: string;
+}
+interface ValidationResult {
+    issues: ValidationIssue[];
+    queries: number;
+    labels: number;
+    humanLabels: number;
+    syntheticLabels: number;
+    fingerprint: string | null;
+    strata: Record<string, number>;
+    ok: boolean;
+}
+/**
+ * Structural and statistical sanity checks. The warnings matter as much as the errors: a
+ * judgment set with no positives, no human labels, or a two-query stratum will produce
+ * confident-looking numbers that mean nothing.
+ */
+declare function validate(judgments: Judgment[]): ValidationResult;
+
 interface MetricOptions {
     /** Cutoff for @k metrics. */
     k?: number;
@@ -230,45 +335,6 @@ declare function worstStratum(perStratum: Record<string, StratumScore>, metric:
     n: number;
 } | null;
 
-declare function parseJudgments(content: string, label?: string): Judgment[];
-/**
- * Parse a run JSONL file.
- *
- * The run is the one input that used to be taken on trust, and an unchecked run is how a
- * metric goes out of range: a ranking holding a non-string, or two entries claiming the same
- * query, produce numbers that mean nothing and say nothing about it.
- *
- * Two entries for one query are rejected rather than resolved, because the file no longer says
- * what the ranking for that query is, and picking one silently is a guess. A key repeated
- * *within* one ranking is a different thing: the ranking is still unambiguous, so it is
- * accepted here and counted once by `score`, which reports that it did.
- */
-declare function parseRun(content: string, label?: string): RunEntry[];
-declare function serializeJudgments(judgments: Judgment[]): string;
-declare function parseCorpus(content: string, label?: string): Corpus;
-type Severity = "error" | "warning";
-interface ValidationIssue {
-    severity: Severity;
-    code: string;
-    message: string;
-}
-interface ValidationResult {
-    issues: ValidationIssue[];
-    queries: number;
-    labels: number;
-    humanLabels: number;
-    syntheticLabels: number;
-    fingerprint: string | null;
-    strata: Record<string, number>;
-    ok: boolean;
-}
-/**
- * Structural and statistical sanity checks. The warnings matter as much as the errors: a
- * judgment set with no positives, no human labels, or a two-query stratum will produce
- * confident-looking numbers that mean nothing.
- */
-declare function validate(judgments: Judgment[]): ValidationResult;
-
 /**
  * TREC qrels: `query_id iteration doc_id relevance`, whitespace separated. Thirty years of
  * tooling reads this (trec_eval, ir_measures, BEIR, ir_datasets), which is why judgments are
@@ -295,74 +361,8 @@ declare function fromQrels(content: string, options?: FromQrelsOptions): Judgmen
 declare function toTrecRun(run: RunEntry[], runName?: string): string;
 declare function fromTrecRun(content: string): RunEntry[];
 
-interface DriftOptions {
-    /** Corpus fingerprint to record when judgments carry none. */
-    corpusFingerprint?: string;
-}
-/**
- * Classify every judgment against a live corpus.
- *
- * This is the question no other evaluation tool can answer: after you changed your chunker,
- * which of your labels still mean what they meant when a human wrote them?
- *
- * - `VALID`          the labeled chunk_id is still present
- * - `RE_ANCHORABLE`  the id is stale, but the exact text is still a live chunk
- * - `MERGED`         the text was absorbed into a coarser chunk; re-anchors safely
- * - `SPLIT`          the labeled text now spans two or more live chunks, so it needs re-judging
- * - `ORPHANED`       the text or its document is gone
- */
-declare function drift(judgments: Judgment[], corpus: Corpus, options?: DriftOptions): DriftResult;
-interface FixResult {
-    judgments: Judgment[];
-    reanchored: number;
-    /** Labels left untouched because they need a human: SPLIT and ORPHANED. */
-    needsReview: DriftFinding[];
-}
-/**
- * Re-anchor the recoverable labels onto their new chunk_ids and stamp the new fingerprint.
- *
- * RE_ANCHORABLE and MERGED are recoverable: in both cases the text a human judged is still
- * there. SPLIT and ORPHANED are deliberately left alone, because guessing at them would
- * silently fabricate ground truth, the exact failure this tool exists to expose.
- */
-declare function fix(judgments: Judgment[], result: DriftResult): FixResult;
-
-/**
- * A gate expression. Four forms, because absolute thresholds get tuned until they pass:
- *
- *   recall@5:0.8              absolute floor
- *   recall@5:-0.02            delta against a baseline (regression tolerance)
- *   worst-stratum:recall@5:0.7  floor on the weakest query class
- *   faithfulness:ci-lower:0.8   floor on the lower confidence bound, so judge noise cannot pass
- */
-interface Gate {
-    raw: string;
-    kind: "absolute" | "delta" | "worst-stratum" | "ci-lower";
-    metric: string;
-    threshold: number;
-}
-declare function parseGate(expression: string): Gate;
-/**
- * The more serious of two verdicts: FAIL beats INDETERMINATE beats PASS.
- *
- * A gate only knows about the numbers it was pointed at. It cannot clear a finding it never
- * looked at, such as a judgment set `validate` rejects, so a passing gate never upgrades a
- * verdict that was already worse.
- */
-declare function worseStatus(a: Status, b: Status): Status;
-interface EvaluateGatesOptions {
-    report: Report;
-    baseline?: Report | undefined;
-    gates: Gate[];
-}
-declare function evaluateGates({ report, baseline, gates }: EvaluateGatesOptions): {
-    status: Status;
-    results: GateResult[];
-    reasons: string[];
-};
-
 declare const TOOL_NAME = "retrieval-eval";
-declare const TOOL_VERSION = "0.1.0";
+declare const TOOL_VERSION = "0.1.1";
 declare const SPEC_VERSION: "1";
 interface BuildReportOptions extends MetricOptions {
     judgments: Judgment[];

package/dist/index.js

@@ -21,6 +21,144 @@ function chunkId(input) {
   return `c1:${h128(canonical)}`;
 }
 
+// src/drift.ts
+function buildIndex(corpus) {
+  const byChunkId = /* @__PURE__ */ new Map();
+  const byTextSha = /* @__PURE__ */ new Map();
+  const byDoc = /* @__PURE__ */ new Map();
+  for (const chunk of corpus.chunks) {
+    byChunkId.set(chunk.chunk_id, chunk);
+    const sha = chunk.text_sha ?? (chunk.text !== void 0 ? textSha(chunk.text) : void 0);
+    if (sha) {
+      const list = byTextSha.get(sha);
+      if (list) list.push(chunk);
+      else byTextSha.set(sha, [chunk]);
+    }
+    const docList = byDoc.get(chunk.doc_uri);
+    if (docList) docList.push(chunk);
+    else byDoc.set(chunk.doc_uri, [chunk]);
+  }
+  for (const list of byDoc.values()) {
+    list.sort((a, b) => (a.ordinal ?? 0) - (b.ordinal ?? 0));
+  }
+  return { byChunkId, byTextSha, byDoc };
+}
+function findSplit(labeledText, docChunks) {
+  const haystack = normalize(labeledText);
+  if (haystack === "") return null;
+  const covering = docChunks.filter((chunk) => {
+    if (chunk.text === void 0) return false;
+    const needle = normalize(chunk.text);
+    return needle !== "" && haystack.includes(needle);
+  });
+  return covering.length >= 2 ? covering : null;
+}
+function findMerged(labeledText, docChunks) {
+  const needle = normalize(labeledText);
+  if (needle === "") return null;
+  const containing = docChunks.filter(
+    (chunk) => chunk.text !== void 0 && normalize(chunk.text).includes(needle)
+  );
+  if (containing.length === 0) return null;
+  return containing.reduce(
+    (best, chunk) => normalize(chunk.text).length < normalize(best.text).length ? chunk : best
+  );
+}
+function drift(judgments, corpus, options = {}) {
+  const index = buildIndex(corpus);
+  const findings = [];
+  const fingerprints = /* @__PURE__ */ new Set();
+  for (const j of judgments) {
+    if (j.corpus_fingerprint) fingerprints.add(j.corpus_fingerprint);
+    const finding = {
+      query_id: j.query_id,
+      doc_uri: j.doc_uri,
+      status: "ORPHANED"
+    };
+    if (j.chunk_id !== void 0) finding.chunk_id = j.chunk_id;
+    const sha = j.text_sha ?? (j.chunk_text !== void 0 ? textSha(j.chunk_text) : void 0);
+    const docChunks = index.byDoc.get(j.doc_uri) ?? [];
+    if (j.chunk_id === void 0) {
+      finding.status = docChunks.length > 0 ? "VALID" : "ORPHANED";
+      findings.push(finding);
+      continue;
+    }
+    if (index.byChunkId.has(j.chunk_id)) {
+      finding.status = "VALID";
+      findings.push(finding);
+      continue;
+    }
+    if (sha) {
+      const matches = index.byTextSha.get(sha) ?? [];
+      const sameDoc = matches.filter((chunk) => chunk.doc_uri === j.doc_uri);
+      const target = sameDoc[0] ?? matches[0];
+      if (target) {
+        finding.status = "RE_ANCHORABLE";
+        finding.reanchor_to = target.chunk_id;
+        findings.push(finding);
+        continue;
+      }
+    }
+    if (j.chunk_text !== void 0) {
+      const split = findSplit(j.chunk_text, docChunks);
+      if (split) {
+        finding.status = "SPLIT";
+        finding.split_into = split.map((chunk) => chunk.chunk_id);
+        findings.push(finding);
+        continue;
+      }
+      const merged = findMerged(j.chunk_text, docChunks);
+      if (merged) {
+        finding.status = "MERGED";
+        finding.reanchor_to = merged.chunk_id;
+        findings.push(finding);
+        continue;
+      }
+    }
+    findings.push(finding);
+  }
+  const count = (status) => findings.filter((f) => f.status === status).length;
+  const valid = count("VALID");
+  const total = findings.length;
+  return {
+    findings,
+    summary: {
+      valid,
+      re_anchorable: count("RE_ANCHORABLE"),
+      merged: count("MERGED"),
+      split: count("SPLIT"),
+      orphaned: count("ORPHANED"),
+      invalid_ratio: total === 0 ? 0 : Math.round((total - valid) / total * 1e12) / 1e12
+    },
+    judgments_fingerprint: fingerprints.size === 1 ? [...fingerprints][0] : null,
+    corpus_fingerprint: corpus.corpus_fingerprint ?? options.corpusFingerprint ?? null
+  };
+}
+function fix(judgments, result) {
+  const byQueryAndChunk = /* @__PURE__ */ new Map();
+  for (const finding of result.findings) {
+    byQueryAndChunk.set(`${finding.query_id}\0${finding.chunk_id ?? ""}`, finding);
+  }
+  let reanchored = 0;
+  const needsReview = [];
+  const out = judgments.map((j) => {
+    const finding = byQueryAndChunk.get(`${j.query_id}\0${j.chunk_id ?? ""}`);
+    if (!finding) return j;
+    if ((finding.status === "RE_ANCHORABLE" || finding.status === "MERGED") && finding.reanchor_to) {
+      reanchored++;
+      const next = { ...j, chunk_id: finding.reanchor_to };
+      if (result.corpus_fingerprint) next.corpus_fingerprint = result.corpus_fingerprint;
+      return next;
+    }
+    if (finding.status === "SPLIT" || finding.status === "ORPHANED") needsReview.push(finding);
+    if (finding.status === "VALID" && result.corpus_fingerprint) {
+      return { ...j, corpus_fingerprint: result.corpus_fingerprint };
+    }
+    return j;
+  });
+  return { judgments: out, reanchored, needsReview };
+}
+
 // src/metrics.ts
 function judgmentKey(j) {
   return j.chunk_id ?? j.doc_uri;
@@ -310,72 +448,210 @@ function worstStratum(perStratum, metric) {
   return worst;
 }
 
-// src/judgments.ts
-function parseJsonl(content, label) {
-  const out = [];
-  const lines = content.split("\n");
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i].trim();
-    if (line === "" || line.startsWith("//")) continue;
-    try {
-      out.push({ value: JSON.parse(line), line: i + 1 });
-    } catch (error) {
-      throw new Error(`${label}:${i + 1}: invalid JSON, ${error.message}`);
-    }
+// src/gate.ts
+function parseGate(expression) {
+  const parts = expression.split(":");
+  if (parts[0] === "worst-stratum") {
+    if (parts.length !== 3)
+      throw new Error(`gate '${expression}': expected worst-stratum:<metric>:<floor>`);
+    return {
+      raw: expression,
+      kind: "worst-stratum",
+      metric: parts[1],
+      threshold: parseNumber(parts[2], expression)
+    };
   }
-  return out;
-}
-function parseJudgments(content, label = "judgments") {
-  const rows = parseJsonl(content, label);
-  for (const { value, line } of rows) {
-    if (typeof value.query_id !== "string" || value.query_id === "")
-      throw new Error(`${label}:${line}: missing query_id`);
-    if (typeof value.doc_uri !== "string" || value.doc_uri === "")
-      throw new Error(`${label}:${line}: missing doc_uri`);
-    if (!Number.isInteger(value.relevance) || value.relevance < 0)
-      throw new Error(`${label}:${line}: relevance must be a non-negative integer`);
+  if (parts.length === 3 && parts[1] === "ci-lower") {
+    return {
+      raw: expression,
+      kind: "ci-lower",
+      metric: parts[0],
+      threshold: parseNumber(parts[2], expression)
+    };
   }
-  return rows.map((row) => row.value);
+  if (parts.length !== 2) throw new Error(`gate '${expression}': expected <metric>:<threshold>`);
+  const value = parts[1];
+  return {
+    raw: expression,
+    kind: value.startsWith("-") || value.startsWith("+") ? "delta" : "absolute",
+    metric: parts[0],
+    threshold: parseNumber(value, expression)
+  };
 }
-function parseRun(content, label = "run") {
-  const rows = parseJsonl(content, label);
-  const firstSeen = /* @__PURE__ */ new Map();
-  for (const { value, line } of rows) {
-    if (typeof value.query_id !== "string" || value.query_id === "")
-      throw new Error(`${label}:${line}: missing query_id`);
-    if (!Array.isArray(value.ranking))
-      throw new Error(`${label}:${line}: ranking must be an array`);
-    for (let i = 0; i < value.ranking.length; i++) {
-      const key = value.ranking[i];
-      if (typeof key !== "string" || key === "")
-        throw new Error(`${label}:${line}: ranking[${i}] must be a non-empty string`);
+var NUMBER = /^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/;
+function parseNumber(value, expression) {
+  if (!NUMBER.test(value)) throw new Error(`gate '${expression}': '${value}' is not a number`);
+  return Number.parseFloat(value);
+}
+function worseStatus(a, b) {
+  if (a === "FAIL" || b === "FAIL") return "FAIL";
+  if (a === "INDETERMINATE" || b === "INDETERMINATE") return "INDETERMINATE";
+  return "PASS";
+}
+function evaluateGates({ report, baseline, gates }) {
+  const results = [];
+  const reasons = [];
+  for (const gate of gates) {
+    const result = evaluateGate(gate, report, baseline);
+    results.push(result);
+    if (result.status === "FAIL" || result.status === "INDETERMINATE") {
+      reasons.push(describe(gate, result, report));
     }
-    const previous = firstSeen.get(value.query_id);
-    if (previous !== void 0)
-      throw new Error(
-        `${label}:${line}: duplicate entry for query ${value.query_id}, already on line ${previous}`
-      );
-    firstSeen.set(value.query_id, line);
   }
-  return rows.map((row) => row.value);
+  const status = results.some((r) => r.status === "FAIL") ? "FAIL" : results.some((r) => r.status === "INDETERMINATE") ? "INDETERMINATE" : "PASS";
+  return { status, results, reasons };
 }
-function byCodePoint(a, b) {
-  const left = [...a];
-  const right = [...b];
-  for (let i = 0; i < Math.min(left.length, right.length); i++) {
-    const difference = (left[i].codePointAt(0) ?? 0) - (right[i].codePointAt(0) ?? 0);
-    if (difference !== 0) return difference;
+function evaluateGate(gate, report, baseline) {
+  if (gate.kind === "worst-stratum") {
+    if (!report.per_stratum) {
+      return { expression: gate.raw, status: "INDETERMINATE" };
+    }
+    const worst = worstStratum(report.per_stratum, gate.metric);
+    if (!worst) return { expression: gate.raw, status: "INDETERMINATE" };
+    return {
+      expression: gate.raw,
+      status: worst.value >= gate.threshold ? "PASS" : "FAIL",
+      observed: worst.value
+    };
   }
-  return left.length - right.length;
-}
-var FIELD_ORDER = [
-  "query_id",
-  "query",
-  "doc_uri",
-  "chunk_id",
-  "text_sha",
-  "chunk_text",
-  "relevance",
+  const measurement = report.metrics[gate.metric];
+  if (!measurement) return { expression: gate.raw, status: "INDETERMINATE" };
+  if (gate.kind === "ci-lower") {
+    if (!measurement.ci) {
+      return { expression: gate.raw, status: "INDETERMINATE", observed: measurement.value };
+    }
+    const lower = measurement.ci[0];
+    return {
+      expression: gate.raw,
+      status: lower >= gate.threshold ? "PASS" : "FAIL",
+      observed: lower
+    };
+  }
+  if (gate.kind === "delta") {
+    const previous = baseline?.metrics[gate.metric];
+    if (!previous) {
+      return {
+        expression: gate.raw,
+        status: "INDETERMINATE",
+        observed: measurement.value,
+        baseline: null
+      };
+    }
+    const delta = measurement.value - previous.value;
+    return {
+      expression: gate.raw,
+      status: delta >= gate.threshold ? "PASS" : "FAIL",
+      observed: measurement.value,
+      baseline: previous.value
+    };
+  }
+  return {
+    expression: gate.raw,
+    status: measurement.value >= gate.threshold ? "PASS" : "FAIL",
+    observed: measurement.value
+  };
+}
+function describe(gate, result, report) {
+  if (result.status === "INDETERMINATE") {
+    const nothingScored = report.judgments.queries_scored === 0 && result.observed === void 0;
+    if (nothingScored && gate.kind !== "worst-stratum") {
+      return `${gate.raw}: no query was scored, so '${gate.metric}' was not computed`;
+    }
+    if (gate.kind === "ci-lower") {
+      return `${gate.raw}: no confidence interval on '${gate.metric}', sample it more than once`;
+    }
+    if (gate.kind === "delta") return `${gate.raw}: no baseline value for '${gate.metric}'`;
+    if (gate.kind === "worst-stratum") {
+      const strata = Object.values(report.per_stratum ?? {});
+      if (strata.length > 0 && strata.every((stratum) => stratum.n === 0)) {
+        return `${gate.raw}: no stratum was scored for '${gate.metric}'`;
+      }
+      return `${gate.raw}: no per-stratum data for '${gate.metric}'`;
+    }
+    return `${gate.raw}: metric '${gate.metric}' not present in the report`;
+  }
+  if (gate.kind === "delta") {
+    const from = result.baseline ?? 0;
+    const to = result.observed ?? 0;
+    const change = to - from;
+    const direction = change < 0 ? `fell ${(-change).toFixed(4)}` : `rose ${change.toFixed(4)}`;
+    return `${gate.metric} ${direction}, from ${from.toFixed(4)} to ${to.toFixed(4)}, outside ${gate.threshold}`;
+  }
+  if (gate.kind === "worst-stratum") {
+    return `worst stratum ${gate.metric} is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
+  }
+  if (gate.kind === "ci-lower") {
+    return `${gate.metric} lower bound is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
+  }
+  return `${gate.metric} is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
+}
+
+// src/judgments.ts
+function parseJsonl(content, label) {
+  const out = [];
+  const lines = content.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (line === "" || line.startsWith("//")) continue;
+    try {
+      out.push({ value: JSON.parse(line), line: i + 1 });
+    } catch (error) {
+      throw new Error(`${label}:${i + 1}: invalid JSON, ${error.message}`);
+    }
+  }
+  return out;
+}
+function parseJudgments(content, label = "judgments") {
+  const rows = parseJsonl(content, label);
+  for (const { value, line } of rows) {
+    if (typeof value.query_id !== "string" || value.query_id === "")
+      throw new Error(`${label}:${line}: missing query_id`);
+    if (typeof value.doc_uri !== "string" || value.doc_uri === "")
+      throw new Error(`${label}:${line}: missing doc_uri`);
+    if (!Number.isInteger(value.relevance) || value.relevance < 0)
+      throw new Error(`${label}:${line}: relevance must be a non-negative integer`);
+  }
+  return rows.map((row) => row.value);
+}
+function parseRun(content, label = "run") {
+  const rows = parseJsonl(content, label);
+  const firstSeen = /* @__PURE__ */ new Map();
+  for (const { value, line } of rows) {
+    if (typeof value.query_id !== "string" || value.query_id === "")
+      throw new Error(`${label}:${line}: missing query_id`);
+    if (!Array.isArray(value.ranking))
+      throw new Error(`${label}:${line}: ranking must be an array`);
+    for (let i = 0; i < value.ranking.length; i++) {
+      const key = value.ranking[i];
+      if (typeof key !== "string" || key === "")
+        throw new Error(`${label}:${line}: ranking[${i}] must be a non-empty string`);
+    }
+    const previous = firstSeen.get(value.query_id);
+    if (previous !== void 0)
+      throw new Error(
+        `${label}:${line}: duplicate entry for query ${value.query_id}, already on line ${previous}`
+      );
+    firstSeen.set(value.query_id, line);
+  }
+  return rows.map((row) => row.value);
+}
+function byCodePoint(a, b) {
+  const left = [...a];
+  const right = [...b];
+  for (let i = 0; i < Math.min(left.length, right.length); i++) {
+    const difference = (left[i].codePointAt(0) ?? 0) - (right[i].codePointAt(0) ?? 0);
+    if (difference !== 0) return difference;
+  }
+  return left.length - right.length;
+}
+var FIELD_ORDER = [
+  "query_id",
+  "query",
+  "doc_uri",
+  "chunk_id",
+  "text_sha",
+  "chunk_text",
+  "relevance",
   "corpus_fingerprint",
   "labeled_by",
   "labeled_at",
@@ -589,285 +865,9 @@ function fromTrecRun(content) {
   }));
 }
 
-// src/drift.ts
-function buildIndex(corpus) {
-  const byChunkId = /* @__PURE__ */ new Map();
-  const byTextSha = /* @__PURE__ */ new Map();
-  const byDoc = /* @__PURE__ */ new Map();
-  for (const chunk of corpus.chunks) {
-    byChunkId.set(chunk.chunk_id, chunk);
-    const sha = chunk.text_sha ?? (chunk.text !== void 0 ? textSha(chunk.text) : void 0);
-    if (sha) {
-      const list = byTextSha.get(sha);
-      if (list) list.push(chunk);
-      else byTextSha.set(sha, [chunk]);
-    }
-    const docList = byDoc.get(chunk.doc_uri);
-    if (docList) docList.push(chunk);
-    else byDoc.set(chunk.doc_uri, [chunk]);
-  }
-  for (const list of byDoc.values()) {
-    list.sort((a, b) => (a.ordinal ?? 0) - (b.ordinal ?? 0));
-  }
-  return { byChunkId, byTextSha, byDoc };
-}
-function findSplit(labeledText, docChunks) {
-  const haystack = normalize(labeledText);
-  if (haystack === "") return null;
-  const covering = docChunks.filter((chunk) => {
-    if (chunk.text === void 0) return false;
-    const needle = normalize(chunk.text);
-    return needle !== "" && haystack.includes(needle);
-  });
-  return covering.length >= 2 ? covering : null;
-}
-function findMerged(labeledText, docChunks) {
-  const needle = normalize(labeledText);
-  if (needle === "") return null;
-  const containing = docChunks.filter(
-    (chunk) => chunk.text !== void 0 && normalize(chunk.text).includes(needle)
-  );
-  if (containing.length === 0) return null;
-  return containing.reduce(
-    (best, chunk) => normalize(chunk.text).length < normalize(best.text).length ? chunk : best
-  );
-}
-function drift(judgments, corpus, options = {}) {
-  const index = buildIndex(corpus);
-  const findings = [];
-  const fingerprints = /* @__PURE__ */ new Set();
-  for (const j of judgments) {
-    if (j.corpus_fingerprint) fingerprints.add(j.corpus_fingerprint);
-    const finding = {
-      query_id: j.query_id,
-      doc_uri: j.doc_uri,
-      status: "ORPHANED"
-    };
-    if (j.chunk_id !== void 0) finding.chunk_id = j.chunk_id;
-    const sha = j.text_sha ?? (j.chunk_text !== void 0 ? textSha(j.chunk_text) : void 0);
-    const docChunks = index.byDoc.get(j.doc_uri) ?? [];
-    if (j.chunk_id === void 0) {
-      finding.status = docChunks.length > 0 ? "VALID" : "ORPHANED";
-      findings.push(finding);
-      continue;
-    }
-    if (index.byChunkId.has(j.chunk_id)) {
-      finding.status = "VALID";
-      findings.push(finding);
-      continue;
-    }
-    if (sha) {
-      const matches = index.byTextSha.get(sha) ?? [];
-      const sameDoc = matches.filter((chunk) => chunk.doc_uri === j.doc_uri);
-      const target = sameDoc[0] ?? matches[0];
-      if (target) {
-        finding.status = "RE_ANCHORABLE";
-        finding.reanchor_to = target.chunk_id;
-        findings.push(finding);
-        continue;
-      }
-    }
-    if (j.chunk_text !== void 0) {
-      const split = findSplit(j.chunk_text, docChunks);
-      if (split) {
-        finding.status = "SPLIT";
-        finding.split_into = split.map((chunk) => chunk.chunk_id);
-        findings.push(finding);
-        continue;
-      }
-      const merged = findMerged(j.chunk_text, docChunks);
-      if (merged) {
-        finding.status = "MERGED";
-        finding.reanchor_to = merged.chunk_id;
-        findings.push(finding);
-        continue;
-      }
-    }
-    findings.push(finding);
-  }
-  const count = (status) => findings.filter((f) => f.status === status).length;
-  const valid = count("VALID");
-  const total = findings.length;
-  return {
-    findings,
-    summary: {
-      valid,
-      re_anchorable: count("RE_ANCHORABLE"),
-      merged: count("MERGED"),
-      split: count("SPLIT"),
-      orphaned: count("ORPHANED"),
-      invalid_ratio: total === 0 ? 0 : Math.round((total - valid) / total * 1e12) / 1e12
-    },
-    judgments_fingerprint: fingerprints.size === 1 ? [...fingerprints][0] : null,
-    corpus_fingerprint: corpus.corpus_fingerprint ?? options.corpusFingerprint ?? null
-  };
-}
-function fix(judgments, result) {
-  const byQueryAndChunk = /* @__PURE__ */ new Map();
-  for (const finding of result.findings) {
-    byQueryAndChunk.set(`${finding.query_id}\0${finding.chunk_id ?? ""}`, finding);
-  }
-  let reanchored = 0;
-  const needsReview = [];
-  const out = judgments.map((j) => {
-    const finding = byQueryAndChunk.get(`${j.query_id}\0${j.chunk_id ?? ""}`);
-    if (!finding) return j;
-    if ((finding.status === "RE_ANCHORABLE" || finding.status === "MERGED") && finding.reanchor_to) {
-      reanchored++;
-      const next = { ...j, chunk_id: finding.reanchor_to };
-      if (result.corpus_fingerprint) next.corpus_fingerprint = result.corpus_fingerprint;
-      return next;
-    }
-    if (finding.status === "SPLIT" || finding.status === "ORPHANED") needsReview.push(finding);
-    if (finding.status === "VALID" && result.corpus_fingerprint) {
-      return { ...j, corpus_fingerprint: result.corpus_fingerprint };
-    }
-    return j;
-  });
-  return { judgments: out, reanchored, needsReview };
-}
-
-// src/gate.ts
-function parseGate(expression) {
-  const parts = expression.split(":");
-  if (parts[0] === "worst-stratum") {
-    if (parts.length !== 3)
-      throw new Error(`gate '${expression}': expected worst-stratum:<metric>:<floor>`);
-    return {
-      raw: expression,
-      kind: "worst-stratum",
-      metric: parts[1],
-      threshold: parseNumber(parts[2], expression)
-    };
-  }
-  if (parts.length === 3 && parts[1] === "ci-lower") {
-    return {
-      raw: expression,
-      kind: "ci-lower",
-      metric: parts[0],
-      threshold: parseNumber(parts[2], expression)
-    };
-  }
-  if (parts.length !== 2) throw new Error(`gate '${expression}': expected <metric>:<threshold>`);
-  const value = parts[1];
-  return {
-    raw: expression,
-    kind: value.startsWith("-") || value.startsWith("+") ? "delta" : "absolute",
-    metric: parts[0],
-    threshold: parseNumber(value, expression)
-  };
-}
-var NUMBER = /^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/;
-function parseNumber(value, expression) {
-  if (!NUMBER.test(value)) throw new Error(`gate '${expression}': '${value}' is not a number`);
-  return Number.parseFloat(value);
-}
-function worseStatus(a, b) {
-  if (a === "FAIL" || b === "FAIL") return "FAIL";
-  if (a === "INDETERMINATE" || b === "INDETERMINATE") return "INDETERMINATE";
-  return "PASS";
-}
-function evaluateGates({ report, baseline, gates }) {
-  const results = [];
-  const reasons = [];
-  for (const gate of gates) {
-    const result = evaluateGate(gate, report, baseline);
-    results.push(result);
-    if (result.status === "FAIL" || result.status === "INDETERMINATE") {
-      reasons.push(describe(gate, result, report));
-    }
-  }
-  const status = results.some((r) => r.status === "FAIL") ? "FAIL" : results.some((r) => r.status === "INDETERMINATE") ? "INDETERMINATE" : "PASS";
-  return { status, results, reasons };
-}
-function evaluateGate(gate, report, baseline) {
-  if (gate.kind === "worst-stratum") {
-    if (!report.per_stratum) {
-      return { expression: gate.raw, status: "INDETERMINATE" };
-    }
-    const worst = worstStratum(report.per_stratum, gate.metric);
-    if (!worst) return { expression: gate.raw, status: "INDETERMINATE" };
-    return {
-      expression: gate.raw,
-      status: worst.value >= gate.threshold ? "PASS" : "FAIL",
-      observed: worst.value
-    };
-  }
-  const measurement = report.metrics[gate.metric];
-  if (!measurement) return { expression: gate.raw, status: "INDETERMINATE" };
-  if (gate.kind === "ci-lower") {
-    if (!measurement.ci) {
-      return { expression: gate.raw, status: "INDETERMINATE", observed: measurement.value };
-    }
-    const lower = measurement.ci[0];
-    return {
-      expression: gate.raw,
-      status: lower >= gate.threshold ? "PASS" : "FAIL",
-      observed: lower
-    };
-  }
-  if (gate.kind === "delta") {
-    const previous = baseline?.metrics[gate.metric];
-    if (!previous) {
-      return {
-        expression: gate.raw,
-        status: "INDETERMINATE",
-        observed: measurement.value,
-        baseline: null
-      };
-    }
-    const delta = measurement.value - previous.value;
-    return {
-      expression: gate.raw,
-      status: delta >= gate.threshold ? "PASS" : "FAIL",
-      observed: measurement.value,
-      baseline: previous.value
-    };
-  }
-  return {
-    expression: gate.raw,
-    status: measurement.value >= gate.threshold ? "PASS" : "FAIL",
-    observed: measurement.value
-  };
-}
-function describe(gate, result, report) {
-  if (result.status === "INDETERMINATE") {
-    const nothingScored = report.judgments.queries_scored === 0 && result.observed === void 0;
-    if (nothingScored && gate.kind !== "worst-stratum") {
-      return `${gate.raw}: no query was scored, so '${gate.metric}' was not computed`;
-    }
-    if (gate.kind === "ci-lower") {
-      return `${gate.raw}: no confidence interval on '${gate.metric}', sample it more than once`;
-    }
-    if (gate.kind === "delta") return `${gate.raw}: no baseline value for '${gate.metric}'`;
-    if (gate.kind === "worst-stratum") {
-      const strata = Object.values(report.per_stratum ?? {});
-      if (strata.length > 0 && strata.every((stratum) => stratum.n === 0)) {
-        return `${gate.raw}: no stratum was scored for '${gate.metric}'`;
-      }
-      return `${gate.raw}: no per-stratum data for '${gate.metric}'`;
-    }
-    return `${gate.raw}: metric '${gate.metric}' not present in the report`;
-  }
-  if (gate.kind === "delta") {
-    const from = result.baseline ?? 0;
-    const to = result.observed ?? 0;
-    const change = to - from;
-    const direction = change < 0 ? `fell ${(-change).toFixed(4)}` : `rose ${change.toFixed(4)}`;
-    return `${gate.metric} ${direction}, from ${from.toFixed(4)} to ${to.toFixed(4)}, outside ${gate.threshold}`;
-  }
-  if (gate.kind === "worst-stratum") {
-    return `worst stratum ${gate.metric} is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
-  }
-  if (gate.kind === "ci-lower") {
-    return `${gate.metric} lower bound is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
-  }
-  return `${gate.metric} is ${(result.observed ?? 0).toFixed(4)}, below ${gate.threshold}`;
-}
-
 // src/report.ts
 var TOOL_NAME = "retrieval-eval";
-var TOOL_VERSION = "0.1.0";
+var TOOL_VERSION = "0.1.1";
 var SPEC_VERSION = "1";
 function buildReport(options) {
   const { judgments, run, corpus, driftResult } = options;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "retrieval-eval",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Your RAG test suite decays every time you re-chunk. Drift detection that tells you which relevance labels are still true after a chunker or corpus change and re-anchors the ones it can, plus deterministic retrieval metrics and a portable, TREC qrels-compatible judgment format. Zero dependencies, no API key.",
   "keywords": [
     "rag",
@@ -41,7 +41,12 @@
     },
     "./package.json": "./package.json"
   },
-  "files": ["dist", "spec", "README.md", "LICENSE"],
+  "files": [
+    "dist",
+    "spec",
+    "README.md",
+    "LICENSE"
+  ],
   "engines": {
     "node": ">=20.11"
   },
@@ -55,10 +60,11 @@
   },
   "devDependencies": {
     "@types/node": "^22.9.0",
-    "@vitest/coverage-v8": "^2.1.4",
+    "@vitest/coverage-v8": "^4.1.11",
     "tsup": "^8.3.5",
-    "typescript": "^5.6.3",
-    "vitest": "^2.1.4"
+    "typescript": "^5.9.3",
+    "vite": "^8.3.0",
+    "vitest": "^4.1.11"
   },
   "repository": {
     "type": "git",