retrieval-eval 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,376 @@
+/**
+ * Normalize chunk text. Exactly three steps, in this order. Any deviation breaks
+ * cross-language agreement with the Python implementation. See `spec/chunk-id.md`.
+ */
+declare function normalize(text: string): string;
+/** Hash of the normalized text, so a judgment can describe itself without storing the text. */
+declare function textSha(text: string): string;
+interface ChunkIdInput {
+    docUri: string;
+    docRevision: string;
+    ordinal: number;
+    text: string;
+    chunkerFingerprint: string;
+}
+/** Content-addressed chunk id: a label points at text, not at a position in a list. */
+declare function chunkId(input: ChunkIdInput): string;
+
+/** One relevance judgment. A judgments file is JSONL: one of these per line. */
+interface Judgment {
+    query_id: string;
+    query?: string;
+    doc_uri: string;
+    chunk_id?: string;
+    text_sha?: string;
+    chunk_text?: string;
+    relevance: number;
+    corpus_fingerprint?: string;
+    labeled_by?: string;
+    labeled_at?: string;
+    stratum?: string;
+    notes?: string;
+    /** Unknown fields are preserved, never rejected. */
+    [key: string]: unknown;
+}
+/** One query's ranked result: chunk_ids, or doc_uris for document-level evaluation. */
+interface RunEntry {
+    query_id: string;
+    ranking: string[];
+}
+interface CorpusChunk {
+    chunk_id: string;
+    doc_uri: string;
+    doc_revision?: string;
+    ordinal?: number;
+    text?: string;
+    text_sha?: string;
+}
+interface Corpus {
+    corpus_fingerprint?: string;
+    chunker_fingerprint?: string;
+    chunks: CorpusChunk[];
+}
+interface Measurement {
+    value: number;
+    /** Samples. Greater than 1 for non-deterministic (LLM-judged) metrics. */
+    n?: number;
+    stdev?: number;
+    /** Required for any LLM-judged metric: a one-sample point estimate is a measurement lie. */
+    ci?: [number, number];
+    deterministic?: boolean;
+}
+type Status = "PASS" | "FAIL" | "INDETERMINATE";
+interface GateResult {
+    expression: string;
+    status: Status;
+    observed?: number;
+    baseline?: number | null;
+}
+interface Report {
+    spec_version: "1";
+    tool: {
+        name: string;
+        version: string;
+    };
+    generated_at: string;
+    corpus: {
+        fingerprint: string | null;
+        documents?: number;
+        chunks?: number;
+    };
+    judgments: {
+        queries: number;
+        /** Queries the averages were computed from: those with at least one label at the threshold. */
+        queries_scored: number;
+        labels: number;
+        fingerprint: string | null;
+        human_labels?: number;
+        synthetic_labels?: number;
+        drift?: DriftSummary;
+    };
+    metrics: Record<string, Measurement>;
+    /** Mandatory when judgments carry strata: averages hide broken query classes. */
+    per_stratum?: Record<string, {
+        n: number;
+        metrics: Record<string, Measurement>;
+    }>;
+    verdict: {
+        status: Status;
+        reasons: string[];
+        gates?: GateResult[];
+    };
+}
+type DriftStatus = "VALID" | "RE_ANCHORABLE" | "MERGED" | "SPLIT" | "ORPHANED";
+interface DriftFinding {
+    query_id: string;
+    doc_uri: string;
+    chunk_id?: string;
+    status: DriftStatus;
+    /** The live chunk_id this label should move to, when RE_ANCHORABLE or MERGED. */
+    reanchor_to?: string;
+    /** The live chunk_ids the labeled text now spans, when SPLIT. */
+    split_into?: string[];
+}
+interface DriftSummary {
+    valid: number;
+    re_anchorable: number;
+    merged: number;
+    split: number;
+    orphaned: number;
+    /** Share of labels that are not straightforwardly VALID. */
+    invalid_ratio: number;
+}
+interface DriftResult {
+    findings: DriftFinding[];
+    summary: DriftSummary;
+    judgments_fingerprint: string | null;
+    corpus_fingerprint: string | null;
+}
+
+interface MetricOptions {
+    /** Cutoff for @k metrics. */
+    k?: number;
+    /** Minimum graded relevance that counts as relevant. */
+    threshold?: number;
+}
+interface QueryMetrics {
+    precision: number;
+    recall: number;
+    ndcg: number;
+    mrr: number;
+    ap: number;
+    hit_rate: number;
+}
+/** Key used to match a judgment against a ranking entry: chunk_id when present, else doc_uri. */
+declare function judgmentKey(j: Judgment): string;
+/**
+ * Drop repeated keys from a ranking, keeping the first occurrence.
+ *
+ * A retriever that returns the same chunk twice is routine: a hybrid search merges two indexes,
+ * or a multi-query expansion unions its results, and nothing deduplicates. Counted naively the
+ * repeat scores as a second hit, which pushes `recall@k`, `map@k` and `ndcg@k` above 1.0 and
+ * can carry a failing run past a gate. A repeat is not new evidence, so only the first
+ * occurrence counts. `score` names the queries it happened to rather than correcting quietly.
+ */
+declare function dedupe(ranking: readonly string[]): string[];
+/**
+ * Metrics for a single query.
+ *
+ * `relevance` maps a ranking key to its graded relevance. Keys absent from the map are treated
+ * as relevance 0, so unjudged means not-relevant, the standard closed-world assumption. That is
+ * a real limitation of shallow judgment pools, and `validate` warns when pools look too thin.
+ */
+declare function queryMetrics(relevance: Map<string, number>, ranking: string[], options?: MetricOptions): QueryMetrics;
+/** Group judgments by query, as `query_id -> (ranking key -> relevance)`. */
+declare function relevanceByQuery(judgments: Judgment[]): Map<string, Map<string, number>>;
+/**
+ * Turn repeated samples of a non-deterministic metric into a measurement with error bars.
+ *
+ * LLM judges vary even at temperature 0: the same triple scored three times can come back 0.8,
+ * 1.0 and 0.6. A point estimate from one sample reports that spread as certainty, which is the
+ * most common measurement lie in RAG evaluation and the reason the report format carries `n`,
+ * `stdev` and `ci` at all.
+ *
+ * One sample yields no interval rather than a zero-width one, so a `ci-lower` gate over it is
+ * INDETERMINATE instead of quietly passing. The interval is a Student-t interval on the sample
+ * mean and is not clamped: a bound outside the metric's range is what the samples support.
+ */
+declare function summarize(samples: number[]): Measurement;
+interface ScoreResult {
+    metrics: Record<string, Measurement>;
+    perQuery: Map<string, QueryMetrics>;
+    /** Queries present in the judgments but missing from the run. */
+    missingQueries: string[];
+    /**
+     * Queries with no label at or above the threshold, excluded from the averages.
+     *
+     * Recall, nDCG, MRR and AP are all undefined when a query has nothing relevant to find.
+     * Scoring such a query as zero would quietly drag every average down and make a judgment set
+     * look like a retrieval failure, so they are excluded and counted instead.
+     */
+    queriesWithoutPositives: string[];
+    /**
+     * Queries whose ranking repeated a key. The metrics above already count each key once; this
+     * says so, because a correction nobody is told about is its own kind of wrong number.
+     */
+    queriesWithDuplicates: string[];
+}
+/**
+ * Macro-average metrics over queries.
+ *
+ * Queries with judgments but no run entry score zero rather than being dropped, because
+ * silently skipping them inflates every metric. Queries with no relevant label are excluded
+ * rather than scored zero, because there is nothing for retrieval to have found.
+ *
+ * Every metric is computed over the top `k` results and is named for it. `mrr@k` and `map@k`
+ * are reciprocal rank and average precision within that cutoff, not over an unbounded run:
+ * the name says so because a number that means something other than its name is how a report
+ * stops being trustworthy.
+ */
+declare function score(judgments: Judgment[], run: RunEntry[], options?: MetricOptions): ScoreResult;
+interface StratumScore {
+    n: number;
+    metrics: Record<string, Measurement>;
+}
+/**
+ * Per-stratum scores. A system can improve on the mean while failing an entire query class,
+ * so coverage across strata is reported rather than one average.
+ */
+declare function scoreByStratum(judgments: Judgment[], run: RunEntry[], options?: MetricOptions): Record<string, StratumScore>;
+/**
+ * The lowest-scoring stratum for a metric. This is what gates should watch.
+ *
+ * Strata with no scored query are skipped rather than treated as zero. A class whose labels are
+ * all below the relevance threshold has nothing for retrieval to have found, and reporting it as
+ * the worst class would fail a build over an empty set.
+ */
+declare function worstStratum(perStratum: Record<string, StratumScore>, metric: string): {
+    name: string;
+    value: number;
+    n: number;
+} | null;
+
+declare function parseJudgments(content: string, label?: string): Judgment[];
+/**
+ * Parse a run JSONL file.
+ *
+ * The run is the one input that used to be taken on trust, and an unchecked run is how a
+ * metric goes out of range: a ranking holding a non-string, or two entries claiming the same
+ * query, produce numbers that mean nothing and say nothing about it.
+ *
+ * Two entries for one query are rejected rather than resolved, because the file no longer says
+ * what the ranking for that query is, and picking one silently is a guess. A key repeated
+ * *within* one ranking is a different thing: the ranking is still unambiguous, so it is
+ * accepted here and counted once by `score`, which reports that it did.
+ */
+declare function parseRun(content: string, label?: string): RunEntry[];
+declare function serializeJudgments(judgments: Judgment[]): string;
+declare function parseCorpus(content: string, label?: string): Corpus;
+type Severity = "error" | "warning";
+interface ValidationIssue {
+    severity: Severity;
+    code: string;
+    message: string;
+}
+interface ValidationResult {
+    issues: ValidationIssue[];
+    queries: number;
+    labels: number;
+    humanLabels: number;
+    syntheticLabels: number;
+    fingerprint: string | null;
+    strata: Record<string, number>;
+    ok: boolean;
+}
+/**
+ * Structural and statistical sanity checks. The warnings matter as much as the errors: a
+ * judgment set with no positives, no human labels, or a two-query stratum will produce
+ * confident-looking numbers that mean nothing.
+ */
+declare function validate(judgments: Judgment[]): ValidationResult;
+
+/**
+ * TREC qrels: `query_id iteration doc_id relevance`, whitespace separated. Thirty years of
+ * tooling reads this (trec_eval, ir_measures, BEIR, ir_datasets), which is why judgments are
+ * a strict superset of it rather than a new idea.
+ */
+declare function toQrels(judgments: Judgment[]): string;
+interface FromQrelsOptions {
+    /** Treat the qrels doc_id as a chunk_id rather than a doc_uri. */
+    asChunkIds?: boolean;
+    corpusFingerprint?: string;
+    labeledBy?: string;
+}
+/**
+ * Read qrels in either shape that exists in the wild:
+ *
+ *   `query_id iteration doc_id relevance`  the TREC form, 4 columns
+ *   `query-id  corpus-id  score`           the BEIR form, 3 columns with a header row
+ *
+ * Accepting only the first means not being able to read BEIR or `ir_datasets` exports, which
+ * is most of the reason to speak qrels at all.
+ */
+declare function fromQrels(content: string, options?: FromQrelsOptions): Judgment[];
+/** TREC run format: `query_id iteration doc_id rank score run_name`. */
+declare function toTrecRun(run: RunEntry[], runName?: string): string;
+declare function fromTrecRun(content: string): RunEntry[];
+
+interface DriftOptions {
+    /** Corpus fingerprint to record when judgments carry none. */
+    corpusFingerprint?: string;
+}
+/**
+ * Classify every judgment against a live corpus.
+ *
+ * This is the question no other evaluation tool can answer: after you changed your chunker,
+ * which of your labels still mean what they meant when a human wrote them?
+ *
+ * - `VALID`          the labeled chunk_id is still present
+ * - `RE_ANCHORABLE`  the id is stale, but the exact text is still a live chunk
+ * - `MERGED`         the text was absorbed into a coarser chunk; re-anchors safely
+ * - `SPLIT`          the labeled text now spans two or more live chunks, so it needs re-judging
+ * - `ORPHANED`       the text or its document is gone
+ */
+declare function drift(judgments: Judgment[], corpus: Corpus, options?: DriftOptions): DriftResult;
+interface FixResult {
+    judgments: Judgment[];
+    reanchored: number;
+    /** Labels left untouched because they need a human: SPLIT and ORPHANED. */
+    needsReview: DriftFinding[];
+}
+/**
+ * Re-anchor the recoverable labels onto their new chunk_ids and stamp the new fingerprint.
+ *
+ * RE_ANCHORABLE and MERGED are recoverable: in both cases the text a human judged is still
+ * there. SPLIT and ORPHANED are deliberately left alone, because guessing at them would
+ * silently fabricate ground truth, the exact failure this tool exists to expose.
+ */
+declare function fix(judgments: Judgment[], result: DriftResult): FixResult;
+
+/**
+ * A gate expression. Four forms, because absolute thresholds get tuned until they pass:
+ *
+ *   recall@5:0.8              absolute floor
+ *   recall@5:-0.02            delta against a baseline (regression tolerance)
+ *   worst-stratum:recall@5:0.7  floor on the weakest query class
+ *   faithfulness:ci-lower:0.8   floor on the lower confidence bound, so judge noise cannot pass
+ */
+interface Gate {
+    raw: string;
+    kind: "absolute" | "delta" | "worst-stratum" | "ci-lower";
+    metric: string;
+    threshold: number;
+}
+declare function parseGate(expression: string): Gate;
+/**
+ * The more serious of two verdicts: FAIL beats INDETERMINATE beats PASS.
+ *
+ * A gate only knows about the numbers it was pointed at. It cannot clear a finding it never
+ * looked at, such as a judgment set `validate` rejects, so a passing gate never upgrades a
+ * verdict that was already worse.
+ */
+declare function worseStatus(a: Status, b: Status): Status;
+interface EvaluateGatesOptions {
+    report: Report;
+    baseline?: Report | undefined;
+    gates: Gate[];
+}
+declare function evaluateGates({ report, baseline, gates }: EvaluateGatesOptions): {
+    status: Status;
+    results: GateResult[];
+    reasons: string[];
+};
+
+declare const TOOL_NAME = "retrieval-eval";
+declare const TOOL_VERSION = "0.1.0";
+declare const SPEC_VERSION: "1";
+interface BuildReportOptions extends MetricOptions {
+    judgments: Judgment[];
+    run: RunEntry[];
+    corpus?: Corpus | undefined;
+    driftResult?: DriftResult | undefined;
+    now?: Date;
+}
+declare function buildReport(options: BuildReportOptions): Report;
+
+export { type BuildReportOptions, type ChunkIdInput, type Corpus, type CorpusChunk, type DriftFinding, type DriftOptions, type DriftResult, type DriftStatus, type DriftSummary, type EvaluateGatesOptions, type FixResult, type FromQrelsOptions, type Gate, type GateResult, type Judgment, type Measurement, type MetricOptions, type QueryMetrics, type Report, type RunEntry, SPEC_VERSION, type ScoreResult, type Severity, type Status, type StratumScore, TOOL_NAME, TOOL_VERSION, type ValidationIssue, type ValidationResult, buildReport, chunkId, dedupe, drift, evaluateGates, fix, fromQrels, fromTrecRun, judgmentKey, normalize, parseCorpus, parseGate, parseJudgments, parseRun, queryMetrics, relevanceByQuery, score, scoreByStratum, serializeJudgments, summarize, textSha, toQrels, toTrecRun, validate, worseStatus, worstStratum };