evalcore 2.0.0__tar.gz → 2.1.0__tar.gz

{evalcore-2.0.0 → evalcore-2.1.0}/CHANGELOG.md

@@ -6,6 +6,41 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [2.1.0] - 2026-08-07
+
+A grader declares what kind of check it is at registration, so a consumer
+plug-in is categorised the same way a built-in is.
+
+Shipped as a minor despite the signature change below. `register` is public,
+so the 1.0.0 policy would call this a major; it goes out as 2.1.0 as a
+deliberate exception, because the break is a one-line edit per grader that
+fails loudly at import.
+
+### Changed
+- **Breaking:** `graders.base.register` takes a required second argument,
+  `category`, a `graders.GraderType`. Every `@base.register('foo')` becomes
+  `@base.register('foo', base.GraderType.HEURISTIC)` or whichever member
+  applies; omitting it is a `TypeError` at import. Required rather than
+  defaulted on purpose - it is the only source of a row's `grader_type`, and
+  a default would be the value every grader forgets to override.
+- `store.grader_lookups` reads the category from the registry instead of a
+  closed table of built-in type names, so a plug-in that declares
+  `HEURISTIC` reports `heuristic` where it used to report `unknown`. Rows for
+  consumer graders change value in the `grader_type` column; nothing about
+  the row shape changes.
+
+### Added
+- `graders.GraderType`, a `StrEnum` over the closed set the results store's
+  `grader_type` column accepts: `unknown`, `heuristic`, `statistical`,
+  `llm_as_judge`, `trajectory`, `human`. A `StrEnum` so it needs no
+  serializer of its own on the way to a row.
+- `graders.category_of`, the registry lookup behind `grader_lookups`.
+
+### Removed
+- `store._GRADER_TYPES`, the private table the categories used to live in.
+  Keeping it alongside the registration argument would mean two sources for
+  one fact and a precedence rule between them.
+
 ## [2.0.0] - 2026-07-31
 
 Breaks both the public API and the outbox row shape, so it's a major per the
@@ -139,7 +174,9 @@ by semantic versioning: a breaking change to either means a 2.0.
   rating + ranking with judge agreement, Markdown/HTML reporters, JSON +
   column-store outbox, and content-hash provenance.
 
-[Unreleased]: https://github.com/scottpmiller/evalcore/compare/1.0.0...HEAD
+[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.1.0...HEAD
+[2.1.0]: https://github.com/scottpmiller/evalcore/compare/2.0.0...2.1.0
+[2.0.0]: https://github.com/scottpmiller/evalcore/compare/1.0.0...2.0.0
 [1.0.0]: https://github.com/scottpmiller/evalcore/compare/0.3.0...1.0.0
 [0.3.0]: https://github.com/scottpmiller/evalcore/compare/0.2.0...0.3.0
 [0.2.0]: https://github.com/scottpmiller/evalcore/compare/0.1.0...0.2.0

{evalcore-2.0.0 → evalcore-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalcore
-Version: 2.0.0
+Version: 2.1.0
 Summary: A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs.
 Project-URL: Homepage, https://github.com/scottpmiller/evalcore
 Project-URL: Repository, https://github.com/scottpmiller/evalcore

{evalcore-2.0.0 → evalcore-2.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evalcore"
-version = "2.0.0"
+version = "2.1.0"
 description = "A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs."
 readme = "README.md"
 requires-python = ">=3.11"

{evalcore-2.0.0 → evalcore-2.1.0}/src/evalcore/graders/__init__.py

@@ -20,5 +20,14 @@ from evalcore.graders import (
     judge,
     numeric,
 )
+from evalcore.graders.base import GraderType, category_of
 
-__all__ = ['base', 'classification', 'deterministic', 'judge', 'numeric']
+__all__ = [
+    'GraderType',
+    'base',
+    'category_of',
+    'classification',
+    'deterministic',
+    'judge',
+    'numeric',
+]

evalcore-2.1.0/src/evalcore/graders/base.py

@@ -0,0 +1,145 @@
+"""Grader protocols and the type registry.
+
+A grader spec is a plain dict from the suite config: ``{type, name, ...}``.
+``build_graders`` turns a list of specs into grader instances, split into the
+per-case and aggregate buckets the runner needs.
+"""
+
+import enum
+import typing
+
+from evalcore import models
+from evalcore.errors import ConfigError
+
+
+class GraderType(enum.StrEnum):
+    """What kind of check a grader performs, not which one.
+
+    A closed set, unlike the registry's ``type`` names, which any consumer
+    may extend. Declared once per grader at registration and read back by
+    ``store.grader_lookups``; a ``StrEnum`` so it needs no serializer of its
+    own on the way to a row.
+
+    The same set is spelled out in three other places, all of which have to
+    change together: ``GraderType`` in ``internal-eval-results``, the
+    ``grader_type`` ``Enum8`` in that repo's ``schema.sql``, and the deployed
+    DDL in ``schemata/clickhouse`` on GHE, which is the source of truth.
+
+    ``UNKNOWN`` exists for a producer with no registry behind it. Nothing in
+    evalcore emits it: ``register`` requires a category, so a grader that
+    reaches a suite has always declared one.
+
+    """
+
+    UNKNOWN = 'unknown'
+    HEURISTIC = 'heuristic'
+    STATISTICAL = 'statistical'
+    LLM_AS_JUDGE = 'llm_as_judge'
+    TRAJECTORY = 'trajectory'
+    HUMAN = 'human'
+
+
+@typing.runtime_checkable
+class Grader(typing.Protocol):
+    """Per-case grader. Scores are averaged across cases by the runner."""
+
+    name: str
+
+    def grade(
+        self, case: models.Case, output: models.Output
+    ) -> list[models.Score]: ...
+
+
+@typing.runtime_checkable
+class AggregateGrader(typing.Protocol):
+    """Whole-run grader for set-level metrics (P/R/F1, win-rate, ...)."""
+
+    name: str
+
+    def aggregate(
+        self, results: list[models.CaseResult]
+    ) -> list[models.Score]: ...
+
+
+_REGISTRY: dict[str, type] = {}
+
+_CATEGORIES: dict[str, GraderType] = {}
+
+
+def register(
+    type_name: str, category: GraderType
+) -> typing.Callable[[type], type]:
+    """Class decorator registering a grader under a suite-config ``type``.
+
+    ``category`` is required rather than defaulting, because it is the only
+    source of the row's ``grader_type`` and a default would be the value
+    every grader forgets to override. A grader's category belongs to its
+    implementation, not to a suite's use of it, so it is declared here and
+    not in the suite config.
+
+    Args:
+        type_name: The ``type`` a suite spec names to select this grader.
+        category: What kind of check it performs.
+
+    Returns:
+        The decorator.
+
+    Raises:
+        ConfigError: If ``type_name`` is already registered.
+
+    """
+
+    def _decorate(cls: type) -> type:
+        if type_name in _REGISTRY:
+            raise ConfigError(f'grader type {type_name!r} already registered')
+        _REGISTRY[type_name] = cls
+        _CATEGORIES[type_name] = GraderType(category)
+        return cls
+
+    return _decorate
+
+
+def category_of(type_name: str) -> GraderType:
+    """Return the category a grader type registered under.
+
+    Args:
+        type_name: The suite spec's ``type``.
+
+    Returns:
+        The declared category, or ``UNKNOWN`` for a type no plug-in has
+        registered. A suite naming one cannot run - ``build_graders``
+        raises - so ``UNKNOWN`` only reaches a row when a caller builds
+        rows without loading the plug-ins that produced them.
+
+    """
+    return _CATEGORIES.get(type_name, GraderType.UNKNOWN)
+
+
+def build_graders(
+    specs: list[dict],
+) -> tuple[list[Grader], list[AggregateGrader]]:
+    """Instantiate grader specs, partitioned into per-case and aggregate.
+
+    Each spec's ``type`` selects a registered class; remaining keys (minus
+    ``type``) are passed as keyword arguments to its constructor.
+    """
+    per_case: list[Grader] = []
+    aggregate: list[AggregateGrader] = []
+    for spec in specs:
+        spec = dict(spec)
+        type_name = spec.pop('type')
+        if type_name not in _REGISTRY:
+            raise ConfigError(
+                f'unknown grader type {type_name!r}; '
+                f'known: {sorted(_REGISTRY)}'
+            )
+        grader = _REGISTRY[type_name](**spec)
+        if isinstance(grader, AggregateGrader):
+            aggregate.append(grader)
+        elif isinstance(grader, Grader):
+            per_case.append(grader)
+        else:  # pragma: no cover - defensive
+            raise TypeError(
+                f'{type_name!r} is neither Grader nor AggregateGrader'
+            )
+    return per_case, aggregate

{evalcore-2.0.0 → evalcore-2.1.0}/src/evalcore/graders/classification.py

@@ -19,7 +19,7 @@ def _safe_div(numerator: float, denominator: float) -> float:
     return numerator / denominator if denominator else 0.0
 
 
-@base.register('classification')
+@base.register('classification', base.GraderType.STATISTICAL)
 class Classification:
     """Binary precision/recall/F1 + FN/FP rates over a labeled dataset."""
 

{evalcore-2.0.0 → evalcore-2.1.0}/src/evalcore/graders/deterministic.py

@@ -32,7 +32,7 @@ def _score(name: str, metric: str, case_id: str, ok: bool, detail: str):
     )
 
 
-@base.register('max_chars')
+@base.register('max_chars', base.GraderType.HEURISTIC)
 class MaxChars:
     """Assert a text field is at most ``maximum`` characters long."""
 
@@ -58,7 +58,7 @@ class MaxChars:
         ]
 
 
-@base.register('regex_absent')
+@base.register('regex_absent', base.GraderType.HEURISTIC)
 class RegexAbsent:
     """Assert a text field does NOT match ``pattern`` (e.g. no tokens)."""
 
@@ -78,7 +78,7 @@ class RegexAbsent:
         return [_score(self.name, self.name, case.id, ok, detail)]
 
 
-@base.register('regex_present')
+@base.register('regex_present', base.GraderType.HEURISTIC)
 class RegexPresent:
     """Assert a text field matches EVERY pattern in ``patterns`` (all-of).
 
@@ -111,7 +111,7 @@ class RegexPresent:
         return [_score(self.name, self.name, case.id, ok, detail)]
 
 
-@base.register('non_empty')
+@base.register('non_empty', base.GraderType.HEURISTIC)
 class NonEmpty:
     """Assert a field resolves to a non-empty value."""
 

{evalcore-2.0.0 → evalcore-2.1.0}/src/evalcore/graders/judge.py

@@ -289,7 +289,7 @@ def _load_images(refs_values: list) -> list[dict]:
     return images
 
 
-@base.register('llm_judge')
+@base.register('llm_judge', base.GraderType.LLM_AS_JUDGE)
 class RubricJudge:
     """Score an output's text on rubric dimensions with an LLM judge/panel.
 

{evalcore-2.0.0 → evalcore-2.1.0}/src/evalcore/graders/numeric.py

@@ -48,7 +48,7 @@ def _as_float(value) -> float | None:
         return None
 
 
-@base.register('numeric')
+@base.register('numeric', base.GraderType.HEURISTIC)
 class Numeric:
     """Surface numeric output fields as metrics, optionally range-checked."""
 

{evalcore-2.0.0 → evalcore-2.1.0}/src/evalcore/store.py

@@ -33,7 +33,7 @@ than model attribute names, so ``project`` is emitted as ``application``,
 import json
 import pathlib
 
-from evalcore import models
+from evalcore import graders, models
 
 
 def write_scorecard(
@@ -210,25 +210,18 @@ def read_preferences(path: str | pathlib.Path) -> list[models.Preference]:
     ]
 
 
-_GRADER_TYPES = {
-    'classification': 'statistical',
-    'llm_judge': 'llm_as_judge',
-    'max_chars': 'heuristic',
-    'non_empty': 'heuristic',
-    'numeric': 'heuristic',
-    'regex_absent': 'heuristic',
-    'regex_present': 'heuristic',
-}
-
-
 def grader_lookups(specs: list[dict]) -> tuple[dict[str, str], dict[str, int]]:
     """Grader name to category, and grader name to judge scale.
 
     A ``Score`` names the grader that emitted it but carries neither the
     grader's category nor the scale its raw judge points sit on, so the caller
-    builds both from ``suite.graders`` and hands them to :func:`score_rows`. A
-    grader type the registry does not know lands on ``'unknown'`` rather than
-    being guessed at.
+    builds both from ``suite.graders`` and hands them to :func:`score_rows`.
+
+    The category comes from the registry, where each grader declared it at
+    registration, so a consumer plug-in is categorised the same way a built-in
+    is. A type nothing has registered lands on ``'unknown'`` rather than being
+    guessed at, which in practice means the caller built rows without
+    importing the plug-ins: a suite naming an unregistered type cannot run.
     """
     types: dict[str, str] = {}
     scales: dict[str, int] = {}
@@ -237,7 +230,7 @@ def grader_lookups(specs: list[dict]) -> tuple[dict[str, str], dict[str, int]]:
         if not name:
             continue
         grader_type = spec.get('type', '')
-        types[name] = _GRADER_TYPES.get(grader_type, 'unknown')
+        types[name] = str(graders.category_of(grader_type))
         if grader_type == 'llm_judge':
             scales[name] = int(spec.get('scale', 5))
     return types, scales

{evalcore-2.0.0 → evalcore-2.1.0}/tests/test_runner.py

@@ -33,7 +33,7 @@ class _RecordingAdapter:
         self.closed = True
 
 
-@graders_base.register('_modewatch')
+@graders_base.register('_modewatch', graders_base.GraderType.HEURISTIC)
 class _ModeWatchGrader:
     """A per-case grader that records the run mode via set_mode."""
 
@@ -60,7 +60,7 @@ class _ModeWatchGrader:
         ]
 
 
-@graders_base.register('_pin')
+@graders_base.register('_pin', graders_base.GraderType.HEURISTIC)
 class _PinGrader:
     """A grader exposing a judge_version pin, like the LLM judge does."""
 

{evalcore-2.0.0 → evalcore-2.1.0}/tests/test_store.py

@@ -5,7 +5,7 @@ import pathlib
 import tempfile
 import unittest
 
-from evalcore import models, store
+from evalcore import errors, graders, models, store
 
 
 def _run(with_failure: bool = False):
@@ -315,6 +315,28 @@ class GraderLookupTests(unittest.TestCase):
         types, _ = store.grader_lookups([{'type': 'non_empty'}])
         self.assertEqual(types['non_empty'], 'heuristic')
 
+    def test_a_plugin_is_categorised_like_a_built_in(self):
+        """The point of the registry lookup: a consumer grader that declares
+        its category reads back as that category, not as 'unknown'.
+        """
+
+        @graders.base.register('_store_plugin', graders.GraderType.HEURISTIC)
+        class _Plugin:
+            name = '_store_plugin'
+
+            def grade(self, case, output):  # pragma: no cover - not run
+                return []
+
+        types, _ = store.grader_lookups([{'type': '_store_plugin'}])
+        self.assertEqual(types['_store_plugin'], 'heuristic')
+
+    def test_a_type_cannot_register_twice(self):
+        with self.assertRaises(errors.ConfigError):
+
+            @graders.base.register('non_empty', graders.GraderType.HEURISTIC)
+            class _Clash:
+                name = 'non_empty'
+
 
 class ExporterTests(unittest.TestCase):
     def test_export_scores(self):

{evalcore-2.0.0 → evalcore-2.1.0}/uv.lock

@@ -188,7 +188,7 @@ wheels = [
 
 [[package]]
 name = "evalcore"
-version = "2.0.0"
+version = "2.1.0"
 source = { editable = "." }
 dependencies = [
     { name = "pydantic" },

evalcore-2.0.0/src/evalcore/graders/base.py

@@ -1,78 +0,0 @@
-"""Grader protocols and the type registry.
-
-A grader spec is a plain dict from the suite config: ``{type, name, ...}``.
-``build_graders`` turns a list of specs into grader instances, split into the
-per-case and aggregate buckets the runner needs.
-"""
-
-import typing
-
-from evalcore import models
-from evalcore.errors import ConfigError
-
-
-@typing.runtime_checkable
-class Grader(typing.Protocol):
-    """Per-case grader. Scores are averaged across cases by the runner."""
-
-    name: str
-
-    def grade(
-        self, case: models.Case, output: models.Output
-    ) -> list[models.Score]: ...
-
-
-@typing.runtime_checkable
-class AggregateGrader(typing.Protocol):
-    """Whole-run grader for set-level metrics (P/R/F1, win-rate, ...)."""
-
-    name: str
-
-    def aggregate(
-        self, results: list[models.CaseResult]
-    ) -> list[models.Score]: ...
-
-
-_REGISTRY: dict[str, type] = {}
-
-
-def register(type_name: str) -> typing.Callable[[type], type]:
-    """Class decorator registering a grader under a suite-config ``type``."""
-
-    def _decorate(cls: type) -> type:
-        if type_name in _REGISTRY:
-            raise ConfigError(f'grader type {type_name!r} already registered')
-        _REGISTRY[type_name] = cls
-        return cls
-
-    return _decorate
-
-
-def build_graders(
-    specs: list[dict],
-) -> tuple[list[Grader], list[AggregateGrader]]:
-    """Instantiate grader specs, partitioned into per-case and aggregate.
-
-    Each spec's ``type`` selects a registered class; remaining keys (minus
-    ``type``) are passed as keyword arguments to its constructor.
-    """
-    per_case: list[Grader] = []
-    aggregate: list[AggregateGrader] = []
-    for spec in specs:
-        spec = dict(spec)
-        type_name = spec.pop('type')
-        if type_name not in _REGISTRY:
-            raise ConfigError(
-                f'unknown grader type {type_name!r}; '
-                f'known: {sorted(_REGISTRY)}'
-            )
-        grader = _REGISTRY[type_name](**spec)
-        if isinstance(grader, AggregateGrader):
-            aggregate.append(grader)
-        elif isinstance(grader, Grader):
-            per_case.append(grader)
-        else:  # pragma: no cover - defensive
-            raise TypeError(
-                f'{type_name!r} is neither Grader nor AggregateGrader'
-            )
-    return per_case, aggregate