evalcore 2.1.0__tar.gz → 2.2.0__tar.gz

{evalcore-2.1.0 → evalcore-2.2.0}/CHANGELOG.md

@@ -6,6 +6,28 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [2.2.0] - 2026-08-08
+
+### Added
+- `store.ScoreExporter`, a `runtime_checkable` Protocol naming the seam
+  `JsonlOutboxExporter` already occupied. `store.py` has always said to
+  "replace this class with a database client implementing the same
+  `export_scores` method", but the contract was a docstring sentence and
+  `**kwargs`, so an implementation had to duck-type a private shape. An
+  exporter belongs in the package that owns the store it targets - it is the
+  store that knows its own column types, null policy, and transport - and
+  swapping one for another is now a constructor line at the call site, so an
+  offline run and a live one share a code path.
+
+### Fixed
+- `examples/quickstart/graders.py` still used the one-argument `register` and
+  raised `TypeError` on import, so 2.1.0 shipped with its own bundled example
+  broken. `just test` does not run the example; `just test-all` does.
+- The README's Python API example called a `.export()` that does not exist and
+  passed `RunResult`s to `compare.compare` and `render_scorecard`, which take
+  `Scorecard`s. It now runs verbatim, and shows `grader_lookups` feeding the
+  exporter.
+
 ## [2.1.0] - 2026-08-07
 
 A grader declares what kind of check it is at registration, so a consumer
@@ -174,7 +196,8 @@ by semantic versioning: a breaking change to either means a 2.0.
   rating + ranking with judge agreement, Markdown/HTML reporters, JSON +
   column-store outbox, and content-hash provenance.
 
-[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.1.0...HEAD
+[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.2.0...HEAD
+[2.2.0]: https://github.com/scottpmiller/evalcore/compare/2.1.0...2.2.0
 [2.1.0]: https://github.com/scottpmiller/evalcore/compare/2.0.0...2.1.0
 [2.0.0]: https://github.com/scottpmiller/evalcore/compare/1.0.0...2.0.0
 [1.0.0]: https://github.com/scottpmiller/evalcore/compare/0.3.0...1.0.0

{evalcore-2.1.0 → evalcore-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalcore
-Version: 2.1.0
+Version: 2.2.0
 Summary: A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs.
 Project-URL: Homepage, https://github.com/scottpmiller/evalcore
 Project-URL: Repository, https://github.com/scottpmiller/evalcore
@@ -502,13 +502,30 @@ candidate = runner.run_suite_sync(suite, 'candidate', mode='replay',
                                   revision='abc123', created_at=now)
 # (async context: `await runner.run_suite(...)` is the same call)
 
-result = compare.compare(baseline, candidate, suite.thresholds)
-print(report.render_scorecard(candidate))
+# run_suite returns a RunResult: the scorecard plus every per-sample
+# result behind it. compare and the reporters take the scorecard.
+result = compare.compare(
+    baseline.scorecard, candidate.scorecard, suite.thresholds
+)
+print(report.render_scorecard(candidate.scorecard))
 print(report.render_comparison(result))
 
-store.write_scorecard('candidate.scorecard.json', candidate)
+store.write_scorecard('candidate.scorecard.json', candidate.scorecard)
 store.write_comparison('comparison.json', result)
-store.JsonlOutboxExporter('outbox.jsonl').export(candidate)
+
+# The rows. grader_lookups supplies the grader category and judge scale a
+# Score doesn't carry; the baseline half exports without the comparison,
+# since it was not itself gated.
+types, scales = store.grader_lookups(suite.graders)
+exporter = store.JsonlOutboxExporter('outbox.jsonl')
+exporter.export_scores(baseline, grader_types=types, judge_scales=scales)
+exporter.export_scores(
+    candidate,
+    result,
+    baseline_run_id=baseline.run_id,
+    grader_types=types,
+    judge_scales=scales,
+)
 
 raise SystemExit(0 if result.verdict != 'fail' else 1)
 ```
@@ -784,6 +801,19 @@ table without a mapping layer. A store that forbids nullable columns fills
 those nulls in at ingest, on its side of the seam. Swap the exporter for a real
 database client without touching the runner or any consumer.
 
+`ScoreExporter` is that seam, named: any object with an `export_scores(run,
+comparison, **kwargs)` method returning a count. `JsonlOutboxExporter` is the
+built-in one; an exporter that publishes to a real store belongs in the
+package that owns that store, since it is the store that knows its own column
+types, null policy, and transport. Swapping one for the other is a
+constructor line, so an offline run and a live one share a code path:
+
+```python
+exporter = store.JsonlOutboxExporter(path)   # offline
+exporter = KafkaOutboxExporter(...)          # live, from another package
+exporter.export_scores(run, comparison, grader_types=..., judge_scales=...)
+```
+
 ---
 
 ## The two extension seams (recap)

{evalcore-2.1.0 → evalcore-2.2.0}/README.md

@@ -470,13 +470,30 @@ candidate = runner.run_suite_sync(suite, 'candidate', mode='replay',
                                   revision='abc123', created_at=now)
 # (async context: `await runner.run_suite(...)` is the same call)
 
-result = compare.compare(baseline, candidate, suite.thresholds)
-print(report.render_scorecard(candidate))
+# run_suite returns a RunResult: the scorecard plus every per-sample
+# result behind it. compare and the reporters take the scorecard.
+result = compare.compare(
+    baseline.scorecard, candidate.scorecard, suite.thresholds
+)
+print(report.render_scorecard(candidate.scorecard))
 print(report.render_comparison(result))
 
-store.write_scorecard('candidate.scorecard.json', candidate)
+store.write_scorecard('candidate.scorecard.json', candidate.scorecard)
 store.write_comparison('comparison.json', result)
-store.JsonlOutboxExporter('outbox.jsonl').export(candidate)
+
+# The rows. grader_lookups supplies the grader category and judge scale a
+# Score doesn't carry; the baseline half exports without the comparison,
+# since it was not itself gated.
+types, scales = store.grader_lookups(suite.graders)
+exporter = store.JsonlOutboxExporter('outbox.jsonl')
+exporter.export_scores(baseline, grader_types=types, judge_scales=scales)
+exporter.export_scores(
+    candidate,
+    result,
+    baseline_run_id=baseline.run_id,
+    grader_types=types,
+    judge_scales=scales,
+)
 
 raise SystemExit(0 if result.verdict != 'fail' else 1)
 ```
@@ -752,6 +769,19 @@ table without a mapping layer. A store that forbids nullable columns fills
 those nulls in at ingest, on its side of the seam. Swap the exporter for a real
 database client without touching the runner or any consumer.
 
+`ScoreExporter` is that seam, named: any object with an `export_scores(run,
+comparison, **kwargs)` method returning a count. `JsonlOutboxExporter` is the
+built-in one; an exporter that publishes to a real store belongs in the
+package that owns that store, since it is the store that knows its own column
+types, null policy, and transport. Swapping one for the other is a
+constructor line, so an offline run and a live one share a code path:
+
+```python
+exporter = store.JsonlOutboxExporter(path)   # offline
+exporter = KafkaOutboxExporter(...)          # live, from another package
+exporter.export_scores(run, comparison, grader_types=..., judge_scales=...)
+```
+
 ---
 
 ## The two extension seams (recap)

{evalcore-2.1.0 → evalcore-2.2.0}/examples/quickstart/graders.py

@@ -26,7 +26,7 @@ _ACK = re.compile(
 )
 
 
-@base.register('acknowledges_customer')
+@base.register('acknowledges_customer', base.GraderType.HEURISTIC)
 class AcknowledgesCustomer:
     """The reply must acknowledge the customer, not just brush them off.
 
@@ -59,7 +59,7 @@ class AcknowledgesCustomer:
         ]
 
 
-@base.register('distinct_reply_rate')
+@base.register('distinct_reply_rate', base.GraderType.HEURISTIC)
 class DistinctReplyRate:
     """Fraction of cases whose reply is unique (mode-collapse check).
 

{evalcore-2.1.0 → evalcore-2.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evalcore"
-version = "2.1.0"
+version = "2.2.0"
 description = "A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs."
 readme = "README.md"
 requires-python = ">=3.11"

{evalcore-2.1.0 → evalcore-2.2.0}/src/evalcore/store.py

@@ -32,6 +32,7 @@ than model attribute names, so ``project`` is emitted as ``application``,
 
 import json
 import pathlib
+import typing
 
 from evalcore import graders, models
 
@@ -511,12 +512,56 @@ def score_rows(
     return rows
 
 
+@typing.runtime_checkable
+class ScoreExporter(typing.Protocol):
+    """Where a run's score rows go.
+
+    The seam between the engine and a results store. :func:`score_rows`
+    produces the rows; an exporter decides what happens to them. Implement
+    this to publish to a real store - a Kafka producer, a database client -
+    without the runner or any consumer changing.
+
+    An implementation lives with the store it targets, not here. It is the
+    store that knows its own column types, its null policy, and its
+    transport; the engine only knows the row.
+
+    Swapping one for another is a constructor line at the call site, so an
+    offline run and a live one share the same code path::
+
+        exporter = JsonlOutboxExporter(path)      # offline
+        exporter = KafkaOutboxExporter(...)       # live, from another package
+        exporter.export_scores(run, comparison, grader_types=..., ...)
+
+    """
+
+    def export_scores(
+        self,
+        run: models.RunResult,
+        comparison: models.Comparison | None = None,
+        **kwargs,
+    ) -> int:
+        """Export a run's rows and return how many were exported.
+
+        Args:
+            run: The completed run.
+            comparison: The gate result, on the candidate half of a gate;
+                left off the baseline half, which was not itself gated.
+            kwargs: Passed through to :func:`score_rows` - ``baseline_run_id``,
+                ``grader_types`` and ``judge_scales``.
+
+        Returns:
+            The number of rows exported.
+
+        """
+        ...
+
+
 class JsonlOutboxExporter:
     """Append score rows to a JSONL outbox for a shipper to drain.
 
     A no-network stand-in for direct ingestion: real deployments point a
-    shipper at this file, or replace this class with a database client
-    implementing the same ``export_scores`` method.
+    shipper at this file, or swap in another :class:`ScoreExporter` that
+    writes to the store directly.
     """
 
     def __init__(self, outbox_path: str | pathlib.Path):

{evalcore-2.1.0 → evalcore-2.2.0}/tests/test_store.py

@@ -338,6 +338,36 @@ class GraderLookupTests(unittest.TestCase):
                 name = 'non_empty'
 
 
+class ScoreExporterProtocolTests(unittest.TestCase):
+    """The seam another package implements to publish to a real store."""
+
+    def test_the_builtin_exporter_satisfies_it(self):
+        exporter = store.JsonlOutboxExporter('/dev/null')
+        self.assertIsInstance(exporter, store.ScoreExporter)
+
+    def test_an_outside_implementation_satisfies_it(self):
+        """A store-side exporter: same method, its own transport."""
+
+        class Collecting:
+            def __init__(self):
+                self.rows = []
+
+            def export_scores(self, run, comparison=None, **kwargs):
+                self.rows = store.score_rows(run, comparison, **kwargs)
+                return len(self.rows)
+
+        exporter = Collecting()
+        self.assertIsInstance(exporter, store.ScoreExporter)
+        self.assertEqual(exporter.export_scores(_run()), len(exporter.rows))
+
+    def test_missing_the_method_does_not(self):
+        class NotAnExporter:
+            def export(self, run):  # pragma: no cover - never called
+                return 0
+
+        self.assertNotIsInstance(NotAnExporter(), store.ScoreExporter)
+
+
 class ExporterTests(unittest.TestCase):
     def test_export_scores(self):
         with tempfile.TemporaryDirectory() as tmp:

{evalcore-2.1.0 → evalcore-2.2.0}/uv.lock

@@ -188,7 +188,7 @@ wheels = [
 
 [[package]]
 name = "evalcore"
-version = "2.1.0"
+version = "2.2.0"
 source = { editable = "." }
 dependencies = [
     { name = "pydantic" },