retrieval-eval 0.1.0

package/spec/fixtures/strata/expected.json

@@ -0,0 +1,40 @@
+{
+  "k": 3,
+  "metrics": {
+    "precision@3": 0.2,
+    "recall@3": 0.6,
+    "ndcg@3": 0.6,
+    "mrr@3": 0.6,
+    "map@3": 0.6,
+    "hit_rate@3": 0.6
+  },
+  "per_stratum": {
+    "billing": {
+      "n": 3,
+      "metrics": {
+        "precision@3": 0.333333333333,
+        "recall@3": 1.0,
+        "ndcg@3": 1.0,
+        "mrr@3": 1.0,
+        "map@3": 1.0,
+        "hit_rate@3": 1.0
+      }
+    },
+    "legal": {
+      "n": 2,
+      "metrics": {
+        "precision@3": 0.0,
+        "recall@3": 0.0,
+        "ndcg@3": 0.0,
+        "mrr@3": 0.0,
+        "map@3": 0.0,
+        "hit_rate@3": 0.0
+      }
+    }
+  },
+  "worst_stratum": {
+    "metric": "recall@3",
+    "name": "legal",
+    "value": 0.0
+  }
+}

package/spec/fixtures/strata/judgments.jsonl

@@ -0,0 +1,5 @@
+{"query_id": "b1", "query": "billing 1", "doc_uri": "bd1", "relevance": 1, "stratum": "billing", "labeled_by": "human:fixture"}
+{"query_id": "b2", "query": "billing 2", "doc_uri": "bd2", "relevance": 1, "stratum": "billing", "labeled_by": "human:fixture"}
+{"query_id": "b3", "query": "billing 3", "doc_uri": "bd3", "relevance": 1, "stratum": "billing", "labeled_by": "human:fixture"}
+{"query_id": "l1", "query": "legal 1", "doc_uri": "ld1", "relevance": 1, "stratum": "legal", "labeled_by": "human:fixture"}
+{"query_id": "l2", "query": "legal 2", "doc_uri": "ld2", "relevance": 1, "stratum": "legal", "labeled_by": "human:fixture"}

package/spec/fixtures/strata/run.jsonl

@@ -0,0 +1,5 @@
+{"query_id": "b1", "ranking": ["bd1", "x", "y"]}
+{"query_id": "b2", "ranking": ["bd2", "x", "y"]}
+{"query_id": "b3", "ranking": ["bd3", "x", "y"]}
+{"query_id": "l1", "ranking": ["x", "y", "z"]}
+{"query_id": "l2", "ranking": ["x", "y", "z"]}

package/spec/fixtures/stratum-order/expected.json

@@ -0,0 +1,30 @@
+{
+  "k": 3,
+  "threshold": 2,
+  "note": "Five strata chosen so that code-point order and locale collation disagree. Scored at threshold 2, Beta/alpha/m all reach recall 1.0 and tie, so the table falls back to the name; zeta and etude have nothing at the threshold and are listed after, unscored. JavaScript's localeCompare orders these by locale, which disagrees with Python and reorders them again on a machine with a different locale. Both implementations sort by Unicode code point, which is locale-independent and is what Python's sorted does.",
+  "scored_order": [
+    "Beta",
+    "alpha",
+    "m"
+  ],
+  "unscored_order": [
+    "zeta",
+    "étude"
+  ],
+  "per_stratum_n": {
+    "Beta": 1,
+    "alpha": 1,
+    "m": 1,
+    "zeta": 0,
+    "étude": 0
+  },
+  "metrics": {
+    "recall@3": 1.0,
+    "precision@3": 0.333333333333,
+    "ndcg@3": 1.0,
+    "mrr@3": 1.0,
+    "map@3": 1.0,
+    "hit_rate@3": 1.0
+  },
+  "locale_order_is_not_asserted": "Collation order varies by machine, which is the objection to it; only the code-point orders above are normative."
+}

package/spec/fixtures/stratum-order/judgments.jsonl

@@ -0,0 +1,5 @@
+{"query_id": "q1", "query": "beta query", "doc_uri": "d1", "relevance": 2, "labeled_by": "human:fixture", "stratum": "Beta"}
+{"query_id": "q2", "query": "alpha query", "doc_uri": "d2", "relevance": 2, "labeled_by": "human:fixture", "stratum": "alpha"}
+{"query_id": "q3", "query": "m query", "doc_uri": "d3", "relevance": 2, "labeled_by": "human:fixture", "stratum": "m"}
+{"query_id": "q4", "query": "zeta query", "doc_uri": "d4", "relevance": 1, "labeled_by": "human:fixture", "stratum": "zeta"}
+{"query_id": "q5", "query": "etude query", "doc_uri": "d5", "relevance": 1, "labeled_by": "human:fixture", "stratum": "étude"}

package/spec/fixtures/stratum-order/run.jsonl

@@ -0,0 +1,5 @@
+{"query_id": "q1", "ranking": ["d1"]}
+{"query_id": "q2", "ranking": ["d2"]}
+{"query_id": "q3", "ranking": ["d3"]}
+{"query_id": "q4", "ranking": ["d4"]}
+{"query_id": "q5", "ranking": ["d5"]}

package/spec/fixtures/summarize/expected.json

@@ -0,0 +1,53 @@
+{
+  "note": "95% Student-t interval on the sample mean, with the sample standard deviation (n-1). A single sample yields no interval at all, because one draw from a non-deterministic instrument is not evidence of a spread. The interval is not clamped: raising a negative lower bound to 0 would let a ci-lower gate pass.",
+  "cases": {
+    "five_samples_of_one_triple": {
+      "samples": [0.8, 1.0, 0.6, 0.9, 0.7],
+      "expected": {
+        "value": 0.8,
+        "n": 5,
+        "stdev": 0.158113883008,
+        "ci": [0.603675683852, 0.996324316148],
+        "deterministic": false
+      }
+    },
+    "single_sample_has_no_interval": {
+      "samples": [0.82],
+      "expected": {
+        "value": 0.82,
+        "n": 1,
+        "deterministic": false
+      }
+    },
+    "identical_samples_have_a_zero_width_interval": {
+      "samples": [0.9, 0.9, 0.9],
+      "expected": {
+        "value": 0.9,
+        "n": 3,
+        "stdev": 0.0,
+        "ci": [0.9, 0.9],
+        "deterministic": false
+      }
+    },
+    "interval_is_not_clamped_to_the_metric_range": {
+      "samples": [0.95, 1.0, 1.0, 0.99],
+      "expected": {
+        "value": 0.985,
+        "n": 4,
+        "stdev": 0.023804761428,
+        "ci": [0.947121312472, 1.022878687528],
+        "deterministic": false
+      }
+    },
+    "lower_bound_may_fall_below_zero": {
+      "samples": [0.0, 0.0, 1.0],
+      "expected": {
+        "value": 0.333333333333,
+        "n": 3,
+        "stdev": 0.57735026919,
+        "ci": [-1.10088424325, 1.767550909916],
+        "deterministic": false
+      }
+    }
+  }
+}

package/spec/fixtures/unsorted-queries/expected.json

@@ -0,0 +1,23 @@
+{
+  "note": "The query ids arrive unsorted and none carries query text, so every one raises `missing-query-text`. An ordering that reaches `--json` and the rendered report is part of what the two implementations must agree on, so both emit these sorted by Unicode code point rather than in file order. The strata are unsorted too, for the same reason: `thin-stratum` follows sorted order, not insertion order.",
+  "issue_order": [
+    "missing-query-text:q1",
+    "missing-query-text:q2",
+    "missing-query-text:q3",
+    "thin-stratum:alpha",
+    "thin-stratum:zeta",
+    "no-chunk-ids"
+  ],
+  "missing_query_text_order": [
+    "q1",
+    "q2",
+    "q3"
+  ],
+  "thin_stratum_order": [
+    "alpha",
+    "zeta"
+  ],
+  "ok": true,
+  "queries": 3,
+  "labels": 3
+}

package/spec/fixtures/unsorted-queries/judgments.jsonl

@@ -0,0 +1,3 @@
+{"query_id": "q3", "doc_uri": "d0", "relevance": 1, "labeled_by": "human:fixture"}
+{"query_id": "q1", "doc_uri": "d1", "relevance": 1, "labeled_by": "human:fixture", "stratum": "zeta"}
+{"query_id": "q2", "doc_uri": "d2", "relevance": 1, "labeled_by": "human:fixture", "stratum": "alpha"}

package/spec/fixtures/unsound-judgments/expected.json

@@ -0,0 +1,19 @@
+{
+  "k": 2,
+  "note": "Alice and Bob disagree about d1. `validate` calls that an error and exits non-zero, so `score` cannot look at the same file, resolve the conflict last-wins and report PASS. The verdict is INDETERMINATE: there is a number, but the ground truth it was computed from contradicts itself, and a gate that passes over it has proved nothing.",
+  "validate": {
+    "ok": false,
+    "error_codes": ["duplicate-label"]
+  },
+  "queries": 1,
+  "queries_scored": 1,
+  "verdict": "INDETERMINATE",
+  "reason": "the judgment set is unsound (duplicate-label); run 'retrieval-eval validate' for detail. No metric computed from it can be trusted",
+  "gate": {
+    "expression": "recall@2:0.1",
+    "gate_status": "PASS",
+    "note": "The gate itself passes. The verdict does not, because a gate cannot clear a finding it never looked at.",
+    "verdict": "INDETERMINATE",
+    "exit_code": 1
+  }
+}

package/spec/fixtures/unsound-judgments/judgments.jsonl

@@ -0,0 +1,3 @@
+{"query_id": "q1", "query": "how do I reset my password", "doc_uri": "d1", "relevance": 2, "labeled_by": "human:alice"}
+{"query_id": "q1", "query": "how do I reset my password", "doc_uri": "d1", "relevance": 0, "labeled_by": "human:bob"}
+{"query_id": "q1", "query": "how do I reset my password", "doc_uri": "d2", "relevance": 1, "labeled_by": "human:alice"}

package/spec/fixtures/unsound-judgments/run.jsonl

@@ -0,0 +1 @@
+{"query_id": "q1", "ranking": ["d1", "d2"]}

package/spec/judgments.schema.json

@@ -0,0 +1,54 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://retrieval-eval.dev/spec/judgments.schema.json",
+  "title": "retrieval-eval judgment",
+  "description": "One relevance judgment. A judgments file is JSONL: one of these per line.",
+  "type": "object",
+  "required": ["query_id", "doc_uri", "relevance"],
+  "properties": {
+    "query_id": { "type": "string", "minLength": 1, "description": "Stable id for the query." },
+    "query": {
+      "type": "string",
+      "description": "Query text. Required on at least one row per query_id."
+    },
+    "doc_uri": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Canonical source document uri."
+    },
+    "chunk_id": {
+      "type": "string",
+      "pattern": "^c1:[0-9a-f]{32}$",
+      "description": "Content-addressed chunk id. Omit for document-level judgments."
+    },
+    "text_sha": {
+      "type": "string",
+      "pattern": "^t1:[0-9a-f]{32}$",
+      "description": "Hash of the normalized labeled text. Enables drift re-anchoring without storing text."
+    },
+    "chunk_text": {
+      "type": "string",
+      "description": "The labeled text. Enables SPLIT detection. Omit for sensitive corpora."
+    },
+    "relevance": {
+      "type": "integer",
+      "minimum": 0,
+      "description": "Graded relevance. 0 = not relevant. TREC convention: 0-3."
+    },
+    "corpus_fingerprint": {
+      "type": "string",
+      "description": "State of corpus+chunker+embedder this judgment was made against."
+    },
+    "labeled_by": {
+      "type": "string",
+      "description": "Provenance, e.g. 'human:ana' or 'synthetic:claude-haiku-4-5'. Prefix with human: or synthetic:."
+    },
+    "labeled_at": { "type": "string", "format": "date-time" },
+    "stratum": {
+      "type": "string",
+      "description": "Query category, for coverage reporting instead of averages."
+    },
+    "notes": { "type": "string" }
+  },
+  "additionalProperties": true
+}

package/spec/report.schema.json

@@ -0,0 +1,227 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://retrieval-eval.dev/spec/report.schema.json",
+  "title": "retrieval-eval report",
+  "description": "Results any tool may emit and any CI or dashboard may ingest.",
+  "type": "object",
+  "required": [
+    "spec_version",
+    "tool",
+    "generated_at",
+    "corpus",
+    "judgments",
+    "metrics",
+    "verdict"
+  ],
+  "properties": {
+    "spec_version": {
+      "const": "1"
+    },
+    "tool": {
+      "type": "object",
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        }
+      }
+    },
+    "generated_at": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "corpus": {
+      "type": "object",
+      "properties": {
+        "fingerprint": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "documents": {
+          "type": "integer"
+        },
+        "chunks": {
+          "type": "integer"
+        }
+      }
+    },
+    "judgments": {
+      "type": "object",
+      "required": [
+        "queries",
+        "labels"
+      ],
+      "properties": {
+        "queries": {
+          "type": "integer"
+        },
+        "queries_scored": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Queries the averages were computed from: those with at least one label at or above the relevance threshold. Queries with nothing relevant to find are excluded rather than scored zero."
+        },
+        "labels": {
+          "type": "integer"
+        },
+        "fingerprint": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "human_labels": {
+          "type": "integer"
+        },
+        "synthetic_labels": {
+          "type": "integer"
+        },
+        "drift": {
+          "description": "Present when judgment validity was checked against a live corpus.",
+          "type": "object",
+          "properties": {
+            "valid": {
+              "type": "integer"
+            },
+            "re_anchorable": {
+              "type": "integer"
+            },
+            "merged": {
+              "type": "integer"
+            },
+            "split": {
+              "type": "integer"
+            },
+            "orphaned": {
+              "type": "integer"
+            },
+            "invalid_ratio": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 1
+            }
+          }
+        }
+      }
+    },
+    "metrics": {
+      "description": "Metric name to measurement. Every cutoff metric carries its cutoff in the name: precision@k, recall@k, ndcg@k, mrr@k, map@k, hit_rate@k.",
+      "type": "object",
+      "additionalProperties": {
+        "$ref": "#/$defs/measurement"
+      }
+    },
+    "per_stratum": {
+      "description": "MANDATORY when judgments carry strata. Averages hide broken query classes.",
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "required": [
+          "n"
+        ],
+        "properties": {
+          "n": {
+            "type": "integer"
+          },
+          "metrics": {
+            "type": "object",
+            "additionalProperties": {
+              "$ref": "#/$defs/measurement"
+            }
+          }
+        }
+      }
+    },
+    "verdict": {
+      "type": "object",
+      "required": [
+        "status"
+      ],
+      "properties": {
+        "status": {
+          "enum": [
+            "PASS",
+            "FAIL",
+            "INDETERMINATE"
+          ]
+        },
+        "reasons": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "gates": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": [
+              "expression",
+              "status"
+            ],
+            "properties": {
+              "expression": {
+                "type": "string"
+              },
+              "status": {
+                "enum": [
+                  "PASS",
+                  "FAIL",
+                  "INDETERMINATE"
+                ]
+              },
+              "observed": {
+                "type": "number"
+              },
+              "baseline": {
+                "type": [
+                  "number",
+                  "null"
+                ]
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "$defs": {
+    "measurement": {
+      "type": "object",
+      "required": [
+        "value"
+      ],
+      "properties": {
+        "value": {
+          "type": "number"
+        },
+        "n": {
+          "type": "integer",
+          "description": "Samples. >1 for non-deterministic (LLM-judged) metrics."
+        },
+        "stdev": {
+          "type": "number"
+        },
+        "ci": {
+          "description": "REQUIRED for any LLM-judged metric. A point estimate from one sample is a measurement lie.",
+          "type": "array",
+          "items": {
+            "type": "number"
+          },
+          "minItems": 2,
+          "maxItems": 2
+        },
+        "deterministic": {
+          "type": "boolean"
+        }
+      }
+    }
+  }
+}