structured-eval 0.2.0__tar.gz → 0.3.0__tar.gz

{structured_eval-0.2.0 → structured_eval-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: structured-eval
-Version: 0.2.0
+Version: 0.3.0
 Summary: The LLM Structured Output Evaluation Framework
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/kirillpechurin/structured-eval
@@ -33,12 +33,14 @@ Provides-Extra: diff
 Requires-Dist: deepdiff>=7.0.0; extra == "diff"
 Provides-Extra: align
 Requires-Dist: scipy>=1.13.0; extra == "align"
+Provides-Extra: litellm
+Requires-Dist: litellm>=1.58.0; extra == "litellm"
 Provides-Extra: deepeval
 Requires-Dist: deepeval>=3.0.0; extra == "deepeval"
 Provides-Extra: langsmith
 Requires-Dist: langsmith>=0.8.0; extra == "langsmith"
 Provides-Extra: all
-Requires-Dist: structured-eval[align,diff,fuzzy,jsonschema,rules,yaml]; extra == "all"
+Requires-Dist: structured-eval[align,diff,fuzzy,jsonschema,litellm,rules,yaml]; extra == "all"
 Dynamic: license-file
 
 # structured-eval
@@ -341,23 +343,42 @@ Learn more about strategies in
 Level L5 lets you catch hallucinations by checking each value against a source.
 Note that `expected` is not required for the computation.
 
-Learn more — [field faithfulness](docs/metrics/catalog/field_faithfulness.md).
+Attach a judge to the part of the document worth paying for, and an LLM rules on every field
+beneath it in **one** call — `supported` / `contradicted` / `not_stated`, with a reason:
 
 ```python
 from structured_eval import evaluate
-from structured_eval.models import EvalConfig
-from structured_eval.metrics import FieldFaithfulness
+from structured_eval.models import EvalConfig, ObjectFieldConfig
+from structured_eval.metrics import JudgeFaithfulness
 
 report = evaluate(
-    actual={"title": "Introduction to Python", "duration_hours": 40},
+    actual={
+        "course": "Introduction to Python",
+        "instructor": {"name": "Dr. Rivera", "title": "professor"},
+    },
     expected=None,
-    config=EvalConfig(metrics=[FieldFaithfulness()]),
-    source="Course: Introduction to Python. Duration: 12 hours.",
+    config=EvalConfig(fields={
+        "instructor": ObjectFieldConfig(
+            metrics=[JudgeFaithfulness(client="qwen/qwen3-235b-a22b-2507")]
+        )
+    }),
+    source="Introduction to Python is run by Dr. Rivera, a teaching assistant.",
 )
 
-report.metrics["field_faithfulness"].by_path  # {'title': 1.0, 'duration_hours': 0.0 ← 40 ≠ 12}
+result = report.field_scores["instructor"].metrics["judge_faithfulness"]
+float(result)                                   # 0.5 — one of the two fields is grounded
+result.extra["verdict"]["verdicts"]
+# [{'path': 'instructor.name',  'verdict': 'supported',    'reason': ''},
+#  {'path': 'instructor.title', 'verdict': 'contradicted', 'reason': 'the source says
+#                                                           teaching assistant'}]
 ```
 
+`course` is never judged — only the subtree you attached the judge to costs money.
+
+Learn more — [judge faithfulness](docs/metrics/catalog/judge-faithfulness.md) and
+[LLM clients](docs/core-concepts/llm-clients.md). For a free, deterministic floor there is
+also [field faithfulness](docs/metrics/catalog/field_faithfulness.md), a substring check.
+
 ### Logical consistency of values
 
 Level L6 offers an interface for describing cross-field business rules with a

{structured_eval-0.2.0 → structured_eval-0.3.0}/README.md

@@ -298,23 +298,42 @@ Learn more about strategies in
 Level L5 lets you catch hallucinations by checking each value against a source.
 Note that `expected` is not required for the computation.
 
-Learn more — [field faithfulness](docs/metrics/catalog/field_faithfulness.md).
+Attach a judge to the part of the document worth paying for, and an LLM rules on every field
+beneath it in **one** call — `supported` / `contradicted` / `not_stated`, with a reason:
 
 ```python
 from structured_eval import evaluate
-from structured_eval.models import EvalConfig
-from structured_eval.metrics import FieldFaithfulness
+from structured_eval.models import EvalConfig, ObjectFieldConfig
+from structured_eval.metrics import JudgeFaithfulness
 
 report = evaluate(
-    actual={"title": "Introduction to Python", "duration_hours": 40},
+    actual={
+        "course": "Introduction to Python",
+        "instructor": {"name": "Dr. Rivera", "title": "professor"},
+    },
     expected=None,
-    config=EvalConfig(metrics=[FieldFaithfulness()]),
-    source="Course: Introduction to Python. Duration: 12 hours.",
+    config=EvalConfig(fields={
+        "instructor": ObjectFieldConfig(
+            metrics=[JudgeFaithfulness(client="qwen/qwen3-235b-a22b-2507")]
+        )
+    }),
+    source="Introduction to Python is run by Dr. Rivera, a teaching assistant.",
 )
 
-report.metrics["field_faithfulness"].by_path  # {'title': 1.0, 'duration_hours': 0.0 ← 40 ≠ 12}
+result = report.field_scores["instructor"].metrics["judge_faithfulness"]
+float(result)                                   # 0.5 — one of the two fields is grounded
+result.extra["verdict"]["verdicts"]
+# [{'path': 'instructor.name',  'verdict': 'supported',    'reason': ''},
+#  {'path': 'instructor.title', 'verdict': 'contradicted', 'reason': 'the source says
+#                                                           teaching assistant'}]
 ```
 
+`course` is never judged — only the subtree you attached the judge to costs money.
+
+Learn more — [judge faithfulness](docs/metrics/catalog/judge-faithfulness.md) and
+[LLM clients](docs/core-concepts/llm-clients.md). For a free, deterministic floor there is
+also [field faithfulness](docs/metrics/catalog/field_faithfulness.md), a substring check.
+
 ### Logical consistency of values
 
 Level L6 offers an interface for describing cross-field business rules with a

{structured_eval-0.2.0 → structured_eval-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "structured-eval"
-version = "0.2.0"
+version = "0.3.0"
 description = "The LLM Structured Output Evaluation Framework"
 readme = "README.md"
 license = { text = "Apache-2.0" }
@@ -51,10 +51,13 @@ jsonschema = ["jsonschema>=4.20.0"]
 rules = ["jsonpath-ng>=1.6.0"]
 diff = ["deepdiff>=7.0.0"]
 align = ["scipy>=1.13.0"]
+litellm = ["litellm>=1.58.0"]
 deepeval = ["deepeval>=3.0.0"]
 langsmith = ["langsmith>=0.8.0"]
+# Every capability of structured-eval itself.
+# The host-framework adapters (deepeval / langsmith) are deliberately *not* here.
 all = [
-    "structured-eval[yaml,fuzzy,jsonschema,rules,diff,align]"
+    "structured-eval[yaml,fuzzy,jsonschema,rules,diff,align,litellm]"
 ]
 
 [dependency-groups]
@@ -65,14 +68,18 @@ dev = [
     "mypy>=1.10",
     "pre-commit>=4.6.0",
     "pyyaml>=6.0.1",
-    "rapidfuzz>=3.0",
+    "rapidfuzz>=3.0.0",
     "types-pyyaml>=6.0",
-    "jsonschema>=4.26.0",
-    "types-jsonschema>=4.26.0.20260518",
+    "jsonschema>=4.20.0",
+    "types-jsonschema>=4.20.0.20260518",
     "jsonpath-ng>=1.6",
-    "deepdiff>=9.1.0",
-    "scipy>=1.12",
-    "scipy-stubs>=1.12",
+    "deepdiff>=7.0.0",
+    "scipy>=1.13.0",
+    "scipy-stubs>=1.13.0",
+    "litellm>=1.58.0",
+    # Host-framework adapters, needed here so tests/integration/ runs instead of skipping.
+    "deepeval>=3.0.0",
+    "langsmith>=0.8.0",
 ]
 
 # ── Ruff ──────────────────────────────────────────────────────────────────────
@@ -82,6 +89,12 @@ line-length = 88
 target-version = "py312"
 
 [tool.ruff.lint]
+# Preview is on *only* to reach the three pydoclint rules below. Without
+# `explicit-preview-rules` it would also turn on every other preview rule in the
+# selected families — RUF069 alone fires 152 times on legitimate `score == 1.0`
+# comparisons. With it, only preview rules selected by exact code are enforced.
+preview = true
+explicit-preview-rules = true
 select = [
     "E",      # pycodestyle errors
     "W",      # pycodestyle warnings
@@ -98,6 +111,10 @@ select = [
     "T20",    # flake8-print (no print() in library code)
     "PT",     # flake8-pytest-style
     "PERF",   # performance anti-patterns
+    "D",      # pydocstyle (Google convention, see [tool.ruff.lint.pydocstyle])
+    "DOC201", # pydoclint: docstring omits a documented return value
+    "DOC402", # pydoclint: generator docstring omits Yields
+    "DOC501", # pydoclint: docstring omits a raised exception
     "PLR1714",
     "PLW2901", # select only non-noisy Pylint rules
 ]
@@ -118,10 +135,20 @@ indent-style = "space"
 skip-magic-trailing-comma = false
 line-ending = "auto"
 
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.pydoclint]
+# Google allows omitting `Returns:` when the summary already describes the
+# return value. A one-line docstring is exactly that case, and this codebase
+# leans on it heavily, so don't demand a section that would only restate it.
+ignore-one-line-docstrings = true
+
 [tool.ruff.lint.per-file-ignores]
 "tests/**" = [
     "ARG",     # unused args common in fixtures/parametrize
     "PLR2004", # magic numbers in test assertions are fine
+    "D103",    # a test's name and its `parametrize` ids are the documentation
 ]
 
 # ── Mypy ──────────────────────────────────────────────────────────────────────
@@ -137,7 +164,14 @@ warn_unreachable = true
 
 # Optional third-party deps without type stubs (lazy-imported behind extras).
 [[tool.mypy.overrides]]
-module = ["jsonpath_ng.*", "deepeval.*", "langsmith.*", "deepdiff.*", "rapidfuzz.*"]
+module = [
+    "jsonpath_ng.*",
+    "deepeval.*",
+    "langsmith.*",
+    "deepdiff.*",
+    "rapidfuzz.*",
+    "litellm.*",
+]
 ignore_missing_imports = true
 
 [[tool.mypy.overrides]]
@@ -161,7 +195,7 @@ addopts = "--import-mode=importlib"
 markers = [
     "unit: pure unit tests (single module, no engine)",
     "engine: end-to-end through the Evaluator",
-    "integration: host-framework adapters (deepeval/langsmith)",
+    "integration: real third-party libraries (deepeval/langsmith/litellm)",
     "golden: regression on dataset fixtures",
     "property: invariant / metamorphic tests over generated inputs",
 ]
@@ -174,7 +208,7 @@ branch = true
 
 [tool.coverage.report]
 show_missing = true
-fail_under = 90
+fail_under = 98
 omit = ["structured_eval/integrations/*"]
 exclude_lines = [
     "pragma: no cover",

structured_eval-0.3.0/structured_eval/__init__.py

@@ -0,0 +1,20 @@
+"""structured_eval — field-level evaluation of structured LLM outputs.
+
+The top level exposes only `evaluate`, `evaluate_batch` and
+`evaluate_consistency`. Everything else is imported from its own subsystem:
+
+- `structured_eval.models` — `Sample`, `EvalConfig`, `EvalReport` and the rest
+  of the data layer.
+- `structured_eval.metrics` — every metric, the base hierarchy, `resolve_metric`
+  and the rule DSL.
+- `structured_eval.alignment` / `.formats` / `.utils` — array alignment,
+  parsers, `flatten` / `structured_diff`.
+"""
+
+from structured_eval.api import evaluate, evaluate_batch, evaluate_consistency
+
+__all__ = [
+    "evaluate",
+    "evaluate_batch",
+    "evaluate_consistency",
+]

structured_eval-0.3.0/structured_eval/alignment/__init__.py

@@ -0,0 +1,29 @@
+"""Array alignment — pairing actual list items with expected ones.
+
+An aligner answers a single question: which actual element does each expected
+element correspond to? Array precision/recall/F1, per-element scores and
+cardinality all read that pairing:
+
+- `ArrayAligner` — the interface: implement `align` and you have a strategy.
+- `ByIndexAligner` — pairs by position.
+- `ByKeyAligner` — pairs on a matching key, greedily best-first.
+- `HungarianAligner` — optimal one-to-one assignment, behind the `align` extra.
+- `make_aligner` — builds the strategy an `ArrayFieldConfig` names.
+"""
+
+from structured_eval.alignment.base import ArrayAligner, key_value, keyable
+from structured_eval.alignment.by_index import ByIndexAligner
+from structured_eval.alignment.by_key import ByKeyAligner
+from structured_eval.alignment.factory import make_aligner
+from structured_eval.alignment.hungarian import HungarianAligner, Scorer
+
+__all__ = [
+    "ArrayAligner",
+    "ByIndexAligner",
+    "ByKeyAligner",
+    "HungarianAligner",
+    "Scorer",
+    "key_value",
+    "keyable",
+    "make_aligner",
+]

structured_eval-0.3.0/structured_eval/alignment/base.py

@@ -0,0 +1,141 @@
+"""The `ArrayAligner` interface and the key helpers keyed strategies share."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.utils.paths import MISSING, navigate
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Sequence
+
+    from structured_eval.models.nodes.array_node import ArrayMatchResult
+
+# Sentinel for a key that cannot be extracted (absent, or element not a dict).
+_MISSING_KEY = object()
+
+
+def key_value(element: Any, key: str | None) -> Any:
+    """The alignment key of an element: the whole element, or a named field.
+
+    Shared by every aligner that pairs on a key (`ByKeyAligner`,
+    `HungarianAligner`).
+
+    Args:
+        element: One array element.
+        key: Field path to key on, or `None` to key on the element itself.
+
+    Returns:
+        The element itself when `key` is `None`, the field's value when it is
+        there, `None` when the field is absent, and a private sentinel when
+        `key` is given but the element carries no fields at all. `keyable`
+        reads that sentinel back.
+
+    Example:
+        >>> from structured_eval.alignment import key_value
+        >>> key_value({"sku": "A-1", "qty": 2}, "sku")
+        'A-1'
+        >>> key_value({"sku": "A-1"}, "warehouse") is None   # field absent
+        True
+        >>> key_value("A-1", None)                           # the whole element
+        'A-1'
+    """
+    if key is None:
+        return element
+    if isinstance(element, dict):
+        value = navigate(element, key)
+        return None if value is MISSING else value
+    return _MISSING_KEY
+
+
+def keyable(values: Iterable[Any]) -> bool:
+    """Could every part of this key be extracted from its element?
+
+    `key_value` answers a sentinel for an element that carries no fields at all,
+    and two sentinels are the same object — scoring one against another reads as
+    a perfect match. Every such pair would tie at 1.0 and be claimed in index
+    order, degenerating keyed alignment into alignment by position.
+
+    An element with no key matches nothing, so the aligners ask this first.
+
+    Args:
+        values: The key values of one element, as `key_value` returned them.
+
+    Returns:
+        True when every value came from an element that could be keyed.
+
+    Example:
+        >>> from structured_eval.alignment import key_value, keyable
+        >>> keyable([key_value({"sku": "A-1"}, "sku")])
+        True
+        >>> keyable([key_value("A-1", "sku")])  # a scalar has no field to key on
+        False
+    """
+    return all(value is not _MISSING_KEY for value in values)
+
+
+def normalize_key(key: str | Sequence[str] | None, owner: str) -> list[str] | None:
+    """One key or many, as the list of field paths every keyed aligner works on.
+
+    A lone field name becomes a one-field list, so a single-field key is just
+    the degenerate composite key.
+
+    Args:
+        key: One field path, several of them, or `None`.
+        owner: The aligner's class name, used in the error message.
+
+    Returns:
+        The field paths as a list, or `None` — which passes through with its
+        meaning intact: key on the whole element.
+
+    Raises:
+        ValueError: If `key` is a sequence that names no field at all.
+    """
+    if key is None:
+        return None
+    if isinstance(key, str):
+        return [key]
+    fields = list(key)
+    if not fields:
+        raise ValueError(f"{owner}: key must name at least one field")
+    return fields
+
+
+class ArrayAligner(ABC):
+    """Maps actual array items onto expected ones (the only role of a matcher).
+
+    An aligner decides *who pairs with whom* and nothing else: value scoring of
+    the matched pairs happens later, in the array metrics. Implement `align`
+    and the strategy is complete.
+
+    Example:
+        >>> from typing import Any
+        >>> from structured_eval.alignment import ArrayAligner
+        >>> from structured_eval.models import ArrayMatchResult, ArrayStrategy
+        >>> class ReversedAligner(ArrayAligner):
+        ...     def align(self, expected: list[Any],
+        ...               actual: list[Any]) -> ArrayMatchResult:
+        ...         n = min(len(expected), len(actual))
+        ...         return ArrayMatchResult(
+        ...             strategy=ArrayStrategy.BY_INDEX,
+        ...             matched=[(i, len(actual) - 1 - i) for i in range(n)],
+        ...             missed=list(range(n, len(expected))),
+        ...             spurious=list(range(n, len(actual))),
+        ...         )
+        >>> ReversedAligner().align(["a", "b"], ["b", "a"]).matched
+        [(0, 1), (1, 0)]
+    """
+
+    @abstractmethod
+    def align(self, expected: list[Any], actual: list[Any]) -> ArrayMatchResult:
+        """Pair the actual elements with the expected ones.
+
+        Args:
+            expected: The expected list.
+            actual: The actual list, as the document has it.
+
+        Returns:
+            An `ArrayMatchResult` holding the `(expected_idx, actual_idx)` pairs
+            plus the unmatched expected (missed) and actual (spurious) indices.
+        """

structured_eval-0.3.0/structured_eval/alignment/by_index.py

@@ -0,0 +1,43 @@
+"""Positional array alignment — the `by_index` strategy."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from structured_eval.alignment.base import ArrayAligner
+from structured_eval.models.config import ArrayStrategy
+from structured_eval.models.nodes.array_node import ArrayMatchResult
+
+
+class ByIndexAligner(ArrayAligner):
+    """Pairs the i-th expected item with the i-th actual item.
+
+    For positionally significant lists — steps, time series, rankings. No key
+    is compared: the surplus of the longer side is simply unmatched.
+
+    Example:
+        >>> from structured_eval.alignment import ByIndexAligner
+        >>> result = ByIndexAligner().align(["a", "b", "c"], ["a", "x"])
+        >>> result.matched
+        [(0, 0), (1, 1)]
+        >>> result.missed, result.spurious
+        ([2], [])
+    """
+
+    def align(self, expected: list[Any], actual: list[Any]) -> ArrayMatchResult:
+        """Pair equal positions, leaving the longer side's tail unmatched.
+
+        Args:
+            expected: The expected list.
+            actual: The actual list.
+
+        Returns:
+            An `ArrayMatchResult` pairing `(i, i)` up to the shorter length.
+        """
+        n = min(len(expected), len(actual))
+        return ArrayMatchResult(
+            strategy=ArrayStrategy.BY_INDEX,
+            matched=[(i, i) for i in range(n)],
+            missed=list(range(n, len(expected))),
+            spurious=list(range(n, len(actual))),
+        )

structured_eval-0.3.0/structured_eval/alignment/by_key.py

@@ -0,0 +1,138 @@
+"""Key-based array alignment — the `by_key` strategy."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.alignment.base import (
+    ArrayAligner,
+    key_value,
+    keyable,
+    normalize_key,
+)
+from structured_eval.metrics.base import BaseMetric, resolve_metric
+from structured_eval.metrics.exact import ExactMatch
+from structured_eval.metrics.invoker import MetricInvoker
+from structured_eval.models.config import ArrayStrategy
+from structured_eval.models.nodes.array_node import ArrayMatchResult
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+
+class ByKeyAligner(ArrayAligner):
+    """Pairs items whose keys match, greedily best-first (generalized matching).
+
+    Extracts a key from each element — the `key` field, or the whole element
+    when `key` is `None` — compares keys with `key_metric` and pairs them when
+    the score clears `threshold`. Matching by value and matching by similarity
+    are both this strategy, differing only in the metric.
+
+    A composite `key` such as `["sku", "warehouse"]` scores each field with
+    `key_metric` and takes their mean, so with the default `ExactMatch` and
+    `threshold=1.0` every field must match, while a soft `key_metric` lets a
+    strong field carry a weaker one.
+
+    Pairing is **globally greedy**: every candidate pair clearing the threshold
+    is ranked by score, highest first, and claimed one-to-one. A soft key
+    therefore picks the strongest available partner rather than the first one
+    found, and the outcome does not depend on element order.
+
+    With an exact key every passing score ties at 1.0, and this reduces to
+    first-match. It is the cheap, scipy-free approximation of the optimal
+    assignment `HungarianAligner` computes.
+
+    Example:
+        >>> from structured_eval.alignment import ByKeyAligner
+        >>> expected = [{"sku": "A-1", "qty": 2}, {"sku": "B-2", "qty": 5}]
+        >>> actual = [{"sku": "B-2", "qty": 5}, {"sku": "C-3", "qty": 1}]
+        >>> result = ByKeyAligner(key="sku").align(expected, actual)
+        >>> result.matched          # B-2 pairs across the reordering
+        [(1, 0)]
+        >>> result.missed, result.spurious
+        ([0], [1])
+    """
+
+    def __init__(
+        self,
+        key: str | Sequence[str] | None = None,
+        key_metric: str | BaseMetric | None = None,
+        threshold: float = 1.0,
+    ):
+        """Set what the key is, how it is compared, and how close counts.
+
+        Args:
+            key: Field path to key on, several of them for a composite key, or
+                `None` to key on the whole element.
+            key_metric: Metric comparing two keys, by instance or registered
+                name. Defaults to `ExactMatch`.
+            threshold: Key score at which a pair may be claimed.
+
+        Raises:
+            ValueError: If `key` is a sequence that names no field at all.
+        """
+        self.key = normalize_key(key, self.__class__.__name__)
+        metric = ExactMatch() if key_metric is None else resolve_metric(key_metric)
+        self.scorer = MetricInvoker(metric)
+        self.threshold = threshold
+
+    def align(self, expected: list[Any], actual: list[Any]) -> ArrayMatchResult:
+        """Claim the best-scoring key pairs one-to-one, best first.
+
+        Args:
+            expected: The expected list.
+            actual: The actual list.
+
+        Returns:
+            An `ArrayMatchResult` whose pairs are reported in expected order.
+        """
+        # Score every (expected, actual) pair on its key; keep those clearing
+        # the threshold. Generated in (ei, ai) order so a stable sort breaks
+        # score ties by that order (→ exact-key matches reproduce first-match).
+        e_keys = [self._key_of(item) for item in expected]
+        a_keys = [self._key_of(item) for item in actual]
+        candidates: list[tuple[float, int, int]] = []
+        for ei, e_key in enumerate(e_keys):
+            for ai, a_key in enumerate(a_keys):
+                score = self._key_score(e_key, a_key)
+                if score >= self.threshold:
+                    candidates.append((score, ei, ai))
+        candidates.sort(key=lambda c: c[0], reverse=True)  # best first; ties keep order
+
+        used_e: set[int] = set()
+        used_a: set[int] = set()
+        matched: list[tuple[int, int]] = []
+        for _score, ei, ai in candidates:
+            if ei in used_e or ai in used_a:
+                continue
+            used_e.add(ei)
+            used_a.add(ai)
+            matched.append((ei, ai))
+        matched.sort()  # report pairs in expected order
+
+        missed = [ei for ei in range(len(expected)) if ei not in used_e]
+        spurious = [ai for ai in range(len(actual)) if ai not in used_a]
+        return ArrayMatchResult(
+            strategy=ArrayStrategy.BY_KEY,
+            matched=matched,
+            missed=missed,
+            spurious=spurious,
+        )
+
+    # ── key extraction & scoring ────────────────────────────────────────────
+
+    def _key_of(self, element: Any) -> list[Any]:
+        """The element's key: one value per configured field, or the element."""
+        if self.key is None:
+            return [element]
+        return [key_value(element, field) for field in self.key]
+
+    def _key_score(self, e_key: list[Any], a_key: list[Any]) -> float:
+        """Mean of the per-field key scores (a one-field key is that score)."""
+        if not (keyable(e_key) and keyable(a_key)):
+            return 0.0
+        total = sum(
+            self.scorer.scalar_on_values(a, e)
+            for e, a in zip(e_key, a_key, strict=True)
+        )
+        return total / len(e_key)

structured_eval-0.3.0/structured_eval/alignment/factory.py

@@ -0,0 +1,44 @@
+"""Construction of the aligner named by an array field's `strategy`."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.alignment.by_index import ByIndexAligner
+from structured_eval.alignment.by_key import ByKeyAligner
+from structured_eval.alignment.hungarian import HungarianAligner
+from structured_eval.models.config import ArrayStrategy
+
+if TYPE_CHECKING:
+    from structured_eval.alignment.base import ArrayAligner
+
+
+def make_aligner(
+    strategy: ArrayStrategy = ArrayStrategy.BY_INDEX,
+    params: dict[str, Any] | None = None,
+) -> ArrayAligner:
+    """Build the aligner for an array config's `strategy` from its `params`.
+
+    Args:
+        strategy: Which strategy to build.
+        params: That aligner's constructor arguments, by name. An unknown key
+            surfaces as a `TypeError` from the constructor itself.
+
+    Returns:
+        The aligner instance the strategy names.
+
+    Example:
+        >>> from structured_eval.alignment import make_aligner
+        >>> from structured_eval.models import ArrayStrategy
+        >>> type(make_aligner()).__name__
+        'ByIndexAligner'
+        >>> aligner = make_aligner(ArrayStrategy.BY_KEY, {"key": "sku"})
+        >>> aligner.align([{"sku": "A"}, {"sku": "B"}], [{"sku": "B"}]).matched
+        [(1, 0)]
+    """
+    params = params or {}
+    if strategy == ArrayStrategy.BY_INDEX:
+        return ByIndexAligner()
+    if strategy == ArrayStrategy.HUNGARIAN:
+        return HungarianAligner(**params)
+    return ByKeyAligner(**params)