structured-eval 0.1.1__tar.gz → 0.2.0__tar.gz

{structured_eval-0.1.1 → structured_eval-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: structured-eval
-Version: 0.1.1
+Version: 0.2.0
 Summary: The LLM Structured Output Evaluation Framework
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/kirillpechurin/structured-eval

{structured_eval-0.1.1 → structured_eval-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "structured-eval"
-version = "0.1.1"
+version = "0.2.0"
 description = "The LLM Structured Output Evaluation Framework"
 readme = "README.md"
 license = { text = "Apache-2.0" }

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/array_f1.py

@@ -19,8 +19,12 @@ class ArrayF1(ArrayMetric):
     name = "array_f1"
 
     def __init__(
-        self, threshold: float = 1.0, mode: stats.GradingMode = stats.GradingMode.HARD
+        self,
+        threshold: float = 1.0,
+        mode: stats.GradingMode = stats.GradingMode.HARD,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.threshold = threshold
         self.mode = stats.GradingMode(mode)
 

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/array_precision.py

@@ -21,8 +21,12 @@ class ArrayPrecision(ArrayMetric):
     name = "array_precision"
 
     def __init__(
-        self, threshold: float = 1.0, mode: stats.GradingMode = stats.GradingMode.HARD
+        self,
+        threshold: float = 1.0,
+        mode: stats.GradingMode = stats.GradingMode.HARD,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.threshold = threshold
         self.mode = stats.GradingMode(mode)
 

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/array_prf1.py

@@ -21,8 +21,12 @@ class ArrayPRF1(ArrayMetric):
     name = "array_prf1"
 
     def __init__(
-        self, threshold: float = 1.0, mode: stats.GradingMode = stats.GradingMode.HARD
+        self,
+        threshold: float = 1.0,
+        mode: stats.GradingMode = stats.GradingMode.HARD,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.threshold = threshold
         self.mode = stats.GradingMode(mode)
 

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/array_recall.py

@@ -20,8 +20,12 @@ class ArrayRecall(ArrayMetric):
     name = "array_recall"
 
     def __init__(
-        self, threshold: float = 1.0, mode: stats.GradingMode = stats.GradingMode.HARD
+        self,
+        threshold: float = 1.0,
+        mode: stats.GradingMode = stats.GradingMode.HARD,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.threshold = threshold
         self.mode = stats.GradingMode(mode)
 

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/base.py

@@ -30,12 +30,29 @@ class BaseMetric(ABC):  # noqa: B024 — registry root; subclasses define the in
 
     ``name`` is the key under which a scalar result lands in ``report.metrics``
     and ``FieldScore.metrics``. A metric that returns a ``dict`` instead writes
-    each of its keys directly (the ``name`` is then only a registry handle).
-    Declaring a subclass with a ``name`` registers it automatically.
+    each of its keys directly (the ``name`` is then only a registry handle, and
+    a per-instance override does not affect those keys). Declaring a subclass
+    with a ``name`` registers it automatically.
+
+    Passing ``name=`` at construction overrides the key **for that instance
+    only** — ``Numeric(tolerance=0.01, name="strict")``. Two instances of one
+    metric can then sit on the same node and report under distinct keys instead
+    of overwriting each other. The class registry, which backs name-string
+    resolution (``resolve_metric("numeric")``), is untouched.
+
+    Every metric defining its own ``__init__`` must accept ``name`` and forward
+    it here via ``super().__init__(name=name)``; ``test_metric_contracts.py``
+    enforces this across the registry.
     """
 
     name: str = ""
 
+    def __init__(self, name: str | None = None) -> None:
+        if name is not None:
+            if not name:
+                raise ValueError("metric name must be a non-empty string")
+            self.name = name
+
     def __init_subclass__(cls, **kwargs: Any) -> None:
         super().__init_subclass__(**kwargs)
         if n := getattr(cls, "name", None):

structured_eval-0.2.0/structured_eval/metrics/character_f1.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import re
+import string
+from collections import Counter
+from typing import Any
+
+from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.null import both_null
+
+_IGNORE_PUNCTUATION_CHARS = frozenset(string.punctuation)
+_IGNORE_WHITESPACE_REGEX = re.compile(r"\s+")
+
+
+class CharacterF1(FieldMetric):
+    """Character-overlap F1 for short free-text fields.
+
+    Characters are matched as a **multiset** (``Counter``), so repeated
+    characters contribute only as many times as they appear on both sides.
+    Precision and recall are computed over character counts, and their
+    harmonic mean is returned. String-only: if either side is not a ``str``
+    the score is ``0.0`` (no coercion) — except two ``None``s, which agree
+    (``1.0``; see ``metrics.utils.null``).
+
+    Normalization is applied to both sides before the comparison and each
+    step can be turned off independently::
+
+        CharacterF1(ignore_case=False)         # "AB" vs "ab" scores below 1.0
+        CharacterF1(ignore_punctuation=False)  # "," and "." count as characters
+        CharacterF1(ignore_whitespace=False)   # spaces count as characters
+
+    The defaults keep every normalization on. ``ignore_punctuation`` drops the
+    ASCII punctuation of ``string.punctuation`` — the same set :class:`TokenF1`
+    uses, so ``_`` is dropped and non-ASCII punctuation such as ``«»—`` is kept.
+    """
+
+    name = "character_f1"
+
+    def __init__(
+        self,
+        ignore_case: bool = True,
+        ignore_whitespace: bool = True,
+        ignore_punctuation: bool = True,
+        name: str | None = None,
+    ):
+        super().__init__(name=name)
+        self.ignore_case = ignore_case
+        self.ignore_whitespace = ignore_whitespace
+        self.ignore_punctuation = ignore_punctuation
+
+    def _characters(self, value: str) -> list[str]:
+        if self.ignore_case:
+            value = value.lower()
+        if self.ignore_punctuation:
+            value = "".join(ch for ch in value if ch not in _IGNORE_PUNCTUATION_CHARS)
+        if self.ignore_whitespace:
+            value = _IGNORE_WHITESPACE_REGEX.sub("", value)
+        return list(value)
+
+    def score(self, actual: Any, expected: Any) -> float:
+        if both_null(actual, expected):
+            return 1.0
+        if not (isinstance(actual, str) and isinstance(expected, str)):
+            return 0.0
+
+        a = self._characters(actual)
+        e = self._characters(expected)
+
+        if not a and not e:
+            return 1.0
+        if not a or not e:
+            return 0.0
+
+        same = sum((Counter(a) & Counter(e)).values())
+        if not same:
+            return 0.0
+
+        precision = same / len(a)
+        recall = same / len(e)
+
+        return 2 * precision * recall / (precision + recall)

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/composite_score.py

@@ -29,7 +29,8 @@ class CompositeScore(AnyNodeMetric):
 
     name = "composite_score"
 
-    def __init__(self, weights: dict[str, float]) -> None:
+    def __init__(self, weights: dict[str, float], name: str | None = None) -> None:
+        super().__init__(name=name)
         if not weights:
             raise ValueError("CompositeScore requires at least one metric weight")
         total = sum(weights.values())

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/date_distance_score.py

@@ -6,6 +6,7 @@ from typing import Any
 from pydantic import TypeAdapter
 
 from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.null import both_null
 
 
 def _to_date(value: Any) -> date | None:
@@ -34,17 +35,22 @@ class DateDistanceScore(FieldMetric):
     compared by their calendar date only (time-of-day is ignored).
 
     If either side cannot be read as a date — ``None``, an unparseable string,
-    or any non-date type — the score is ``0.0``.
+    or any non-date type — the score is ``0.0``. Two ``None``s are the exception
+    — no date was expected and none was given, so they agree (``1.0``; see
+    ``metrics.utils.null``).
     """
 
     name = "date_distance_score"
 
-    def __init__(self, max_days: int = 30) -> None:
+    def __init__(self, max_days: int = 30, name: str | None = None) -> None:
+        super().__init__(name=name)
         if max_days <= 0:
             raise ValueError("max_days must be greater than 0")
         self.max_days = max_days
 
     def score(self, actual: Any, expected: Any) -> float:
+        if both_null(actual, expected):
+            return 1.0
         if not isinstance(actual, (date, datetime)):
             actual = _to_date(actual)
         if not isinstance(expected, (date, datetime)):

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/exponential_numeric_score.py

@@ -4,6 +4,7 @@ import math
 from typing import Any
 
 from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.null import both_null
 from structured_eval.metrics.utils.number import parse_number
 
 
@@ -29,17 +30,21 @@ class ExponentialNumericScore(FieldMetric):
     :class:`NumericCloseness`, so numeric strings are graded too. The metric
     applies **only to numbers**: if either side isn't numeric (``None``, a
     non-numeric string, or a ``bool`` — ``True`` is not ``1``) the score is
-    ``0.0``.
+    ``0.0``. Two ``None``s are the exception — they agree (``1.0``; see
+    ``metrics.utils.null``).
     """
 
     name = "exponential_numeric_score"
 
-    def __init__(self, scale: float = 1.0) -> None:
+    def __init__(self, scale: float = 1.0, name: str | None = None) -> None:
+        super().__init__(name=name)
         if scale <= 0:
             raise ValueError("scale must be greater than 0")
         self.scale = scale
 
     def score(self, actual: Any, expected: Any) -> float:
+        if both_null(actual, expected):
+            return 1.0
         a = parse_number(actual)
         e = parse_number(expected)
         if a is None or e is None:

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/fuzzy.py

@@ -4,6 +4,7 @@ from enum import StrEnum
 from typing import Any
 
 from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.null import both_null
 
 
 class FuzzyMethod(StrEnum):
@@ -27,7 +28,8 @@ class Fuzzy(FieldMetric):
 
     ``normalize`` strips surrounding whitespace and lowercases before comparison.
     String-only: if either side is not a ``str`` the score is 0.0 (no coercion),
-    consistent with the other text metrics.
+    consistent with the other text metrics — except two ``None``s, which agree
+    (1.0; see ``metrics.utils.null``).
     """
 
     name = "fuzzy"
@@ -36,11 +38,15 @@ class Fuzzy(FieldMetric):
         self,
         method: FuzzyMethod = FuzzyMethod.TOKEN_SORT_RATIO,
         normalize: bool = True,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.method = FuzzyMethod(method)
         self.normalize = normalize
 
     def score(self, actual: Any, expected: Any) -> float:
+        if both_null(actual, expected):
+            return 1.0
         if not (isinstance(actual, str) and isinstance(expected, str)):
             return 0.0
         try:

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/levenshtein.py

@@ -12,5 +12,10 @@ class Levenshtein(Fuzzy):
 
     name = "levenshtein"
 
-    def __init__(self, method: FuzzyMethod = FuzzyMethod.RATIO, normalize: bool = True):
-        super().__init__(method=method, normalize=normalize)
+    def __init__(
+        self,
+        method: FuzzyMethod = FuzzyMethod.RATIO,
+        normalize: bool = True,
+        name: str | None = None,
+    ):
+        super().__init__(method=method, normalize=normalize, name=name)

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/numeric.py

@@ -4,6 +4,7 @@ from enum import StrEnum
 from typing import Any
 
 from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.null import both_null
 from structured_eval.metrics.utils.number import parse_number
 
 
@@ -32,6 +33,9 @@ class Numeric(FieldMetric):
     * ``relative_tolerance`` and/or ``absolute_tolerance`` — explicit bands; a
       value matches if it falls within *either* band. When either is supplied it
       takes precedence over ``tolerance``/``mode``.
+
+    Two ``None``s agree (1.0; see ``metrics.utils.null``); a one-sided ``None``
+    is 0.0.
     """
 
     name = "numeric"
@@ -42,13 +46,17 @@ class Numeric(FieldMetric):
         mode: NumericMode = NumericMode.RELATIVE,
         relative_tolerance: float | None = None,
         absolute_tolerance: float | None = None,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.tolerance = tolerance
         self.mode = NumericMode(mode)
         self.relative_tolerance = relative_tolerance
         self.absolute_tolerance = absolute_tolerance
 
     def score(self, actual: Any, expected: Any) -> float:
+        if both_null(actual, expected):
+            return 1.0
         a = parse_number(actual)
         e = parse_number(expected)
         if a is None or e is None:

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/numeric_closeness.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import Any
 
 from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.null import both_null
 from structured_eval.metrics.utils.number import parse_number
 
 
@@ -19,12 +20,15 @@ class NumericCloseness(FieldMetric):
     Values are parsed with the shared lenient numeric parser (same as
     :class:`Numeric`), so numeric strings are graded too. The metric applies
     **only to numbers**: if either side isn't numeric (``None``, a non-numeric
-    string, or a ``bool`` — ``True`` is not ``1``) the score is 0.0.
+    string, or a ``bool`` — ``True`` is not ``1``) the score is 0.0. Two
+    ``None``s are the exception — they agree (1.0; see ``metrics.utils.null``).
     """
 
     name = "numeric_closeness"
 
     def score(self, actual: Any, expected: Any) -> float:
+        if both_null(actual, expected):
+            return 1.0
         a = parse_number(actual)
         e = parse_number(expected)
         if a is None or e is None:

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/object_accuracy.py

@@ -31,7 +31,9 @@ class ObjectAccuracy(ObjectMetric):
         self,
         score_policy: dict[str, Any] | None = None,
         weight_mode: stats.WeightMode = stats.WeightMode.PROPORTIONAL,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.score_policy = score_policy
         self.weight_mode = stats.WeightMode(weight_mode)
 

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/object_f1.py

@@ -26,7 +26,9 @@ class ObjectF1(ObjectMetric):
         threshold: float | None = None,
         mode: stats.GradingMode = stats.GradingMode.HARD,
         weight_mode: stats.WeightMode = stats.WeightMode.PROPORTIONAL,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.score_policy = score_policy
         self.threshold = threshold
         self.mode = stats.GradingMode(mode)

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/object_precision.py

@@ -30,7 +30,9 @@ class ObjectPrecision(ObjectMetric):
         threshold: float | None = None,
         mode: stats.GradingMode = stats.GradingMode.HARD,
         weight_mode: stats.WeightMode = stats.WeightMode.PROPORTIONAL,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.score_policy = score_policy
         self.threshold = threshold
         self.mode = stats.GradingMode(mode)

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/object_prf1.py

@@ -26,7 +26,9 @@ class ObjectPRF1(ObjectMetric):
         threshold: float | None = None,
         mode: stats.GradingMode = stats.GradingMode.HARD,
         weight_mode: stats.WeightMode = stats.WeightMode.PROPORTIONAL,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.score_policy = score_policy
         self.threshold = threshold
         self.mode = stats.GradingMode(mode)

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/object_recall.py

@@ -25,7 +25,9 @@ class ObjectRecall(ObjectMetric):
         threshold: float | None = None,
         mode: stats.GradingMode = stats.GradingMode.HARD,
         weight_mode: stats.WeightMode = stats.WeightMode.PROPORTIONAL,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.score_policy = score_policy
         self.threshold = threshold
         self.mode = stats.GradingMode(mode)

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/object_type_validity.py

@@ -23,7 +23,8 @@ class ObjectTypeValidity(ObjectMetric):
 
     name = "object_type_validity"
 
-    def __init__(self) -> None:
+    def __init__(self, name: str | None = None) -> None:
+        super().__init__(name=name)
         self._type_match = MetricInvoker(TypeMatch())
 
     def compute(self, node: ObjectNode) -> float:

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/regex_match.py

@@ -4,6 +4,7 @@ import re
 from typing import Any
 
 from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.null import both_null
 
 
 class RegexMatch(FieldMetric):
@@ -11,7 +12,8 @@ class RegexMatch(FieldMetric):
 
     A **string-only** metric: if either side is not a ``str`` the score is
     ``0.0`` (use ``Numeric`` for numbers, ``ExactMatch`` for verbatim
-    equality). For two strings it applies, in order, optional ``lower`` and
+    equality) — except two ``None``s, which agree (``1.0``; see
+    ``metrics.utils.null``). For two strings it applies, in order, optional ``lower`` and
     ``strip``, then substitutes every match of ``pattern`` with ``repl``, and
     compares the results exactly.
 
@@ -31,7 +33,9 @@ class RegexMatch(FieldMetric):
         repl: str = " ",
         lower: bool = True,
         strip: bool = True,
+        name: str | None = None,
     ):
+        super().__init__(name=name)
         self.pattern = re.compile(pattern) if isinstance(pattern, str) else pattern
         self.repl = repl
         self.lower = lower
@@ -46,6 +50,8 @@ class RegexMatch(FieldMetric):
         return value.strip() if self.strip else value
 
     def score(self, actual: Any, expected: Any) -> float:
+        if both_null(actual, expected):
+            return 1.0
         if not (isinstance(actual, str) and isinstance(expected, str)):
             return 0.0
         return 1.0 if self._normalize(actual) == self._normalize(expected) else 0.0

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/rule_pass_rate/metric.py

@@ -21,7 +21,8 @@ class RulePassRate(RootMetric):
 
     name = "rule_pass_rate"
 
-    def __init__(self, rules: list[Any]):
+    def __init__(self, rules: list[Any], name: str | None = None):
+        super().__init__(name=name)
         self.rules = rules
         self.processor = RuleProcessor()
 

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/schema_validity/metric.py

@@ -21,7 +21,10 @@ class SchemaValidity(RootMetric):
 
     name = "schema_validity"
 
-    def __init__(self, schema: type[BaseModel] | dict[str, Any]):
+    def __init__(
+        self, schema: type[BaseModel] | dict[str, Any], name: str | None = None
+    ):
+        super().__init__(name=name)
         self.validator = SchemaValidator(schema)
 
     def compute(self, node: EvalNode) -> tuple[float, dict[str, Any]]:

structured_eval-0.2.0/structured_eval/metrics/token_f1.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+import re
+import string
+from collections import Counter
+from typing import Any
+
+from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.null import both_null
+
+_IGNORE_PUNCTUATION_CHARS = frozenset(string.punctuation)
+_IGNORE_ARTICLES_REGEX = re.compile(r"\b(a|an|the)\b", re.IGNORECASE)
+
+
+class TokenF1(FieldMetric):
+    """SQuAD-style token-overlap F1 — a default for free-text fields.
+
+    On its defaults this reproduces the ``f1_score`` of the official SQuAD v1.1
+    evaluation script: both sides go through the reference ``normalize_answer``
+    (lowercase, drop punctuation, drop the articles ``a``/``an``/``the``, collapse
+    whitespace), then tokens are matched as a **multiset** (``Counter``) — a
+    repeated token only helps as often as it appears on both sides, so
+    ``"the the cat"`` vs ``"the cat"`` is 0.8, not 1.0. Precision and recall are
+    over the token *counts*; their harmonic mean is the score.
+
+    Each normalization step can be turned off independently::
+
+        TokenF1(ignore_case=False)         # "AB" vs "ab" scores below 1.0
+        TokenF1(ignore_punctuation=False)  # "fox." and "fox" are distinct tokens
+        TokenF1(ignore_articles=False)     # "the" counts as a token like any other
+
+    Two deliberate departures from the reference script, both because this scores
+    fields rather than question answers: two empty strings score 1.0 (the script
+    returns 0.0, an empty answer being a failed answer), and a value that is not a
+    ``str`` scores 0.0 with no coercion — except two ``None``s, which agree (1.0;
+    see ``metrics.utils.null``).
+    """
+
+    name = "token_f1"
+
+    def __init__(
+        self,
+        ignore_case: bool = True,
+        ignore_punctuation: bool = True,
+        ignore_articles: bool = True,
+        name: str | None = None,
+    ):
+        super().__init__(name=name)
+        self.ignore_case = ignore_case
+        self.ignore_punctuation = ignore_punctuation
+        self.ignore_articles = ignore_articles
+
+    def _tokenize(self, value: str) -> list[str]:
+        if self.ignore_case:
+            value = value.lower()
+        if self.ignore_punctuation:
+            value = "".join(ch for ch in value if ch not in _IGNORE_PUNCTUATION_CHARS)
+        if self.ignore_articles:
+            value = _IGNORE_ARTICLES_REGEX.sub(" ", value)
+        return value.split()
+
+    def score(self, actual: Any, expected: Any) -> float:
+        if both_null(actual, expected):
+            return 1.0
+        if not (isinstance(actual, str) and isinstance(expected, str)):
+            return 0.0
+
+        a = self._tokenize(actual)
+        e = self._tokenize(expected)
+
+        if not a and not e:
+            return 1.0
+        if not a or not e:
+            return 0.0
+
+        same = sum((Counter(a) & Counter(e)).values())
+        if not same:
+            return 0.0
+
+        precision = same / len(a)
+        recall = same / len(e)
+
+        return 2 * precision * recall / (precision + recall)

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/url_match.py

@@ -4,6 +4,7 @@ from typing import Any
 from urllib.parse import parse_qsl, unquote, urlsplit, urlunsplit
 
 from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.null import both_null
 
 
 class UrlMatch(FieldMetric):
@@ -27,7 +28,9 @@ class UrlMatch(FieldMetric):
 
     Both sides must be non-empty strings that parse to a URL with a scheme and a
     host. Anything else — a non-string, an empty string, or a bare path with no
-    scheme/host — scores ``0.0``.
+    scheme/host — scores ``0.0``. Two ``None``s are the exception — no URL was
+    expected and none was given, so they agree (``1.0``; see
+    ``metrics.utils.null``).
     """
 
     name = "url_match"
@@ -38,7 +41,9 @@ class UrlMatch(FieldMetric):
         ignore_query: bool = False,
         ignore_fragment: bool = True,
         ignore_www: bool = True,
+        name: str | None = None,
     ) -> None:
+        super().__init__(name=name)
         self.ignore_query = ignore_query
         self.ignore_fragment = ignore_fragment
         self.ignore_www = ignore_www
@@ -75,6 +80,8 @@ class UrlMatch(FieldMetric):
         return (urlunsplit((scheme, netloc, path, query, fragment)),)
 
     def score(self, actual: Any, expected: Any) -> float:
+        if both_null(actual, expected):
+            return 1.0
         norm_actual = self._normalize(actual)
         norm_expected = self._normalize(expected)
         if norm_actual is None or norm_expected is None:

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval/metrics/utils/__init__.py

@@ -7,4 +7,6 @@ Clearly-scoped modules:
 * ``object_utils`` — turning an object's matched fields into the
   ``(score, threshold)`` pairs that ``calculate.prf_counts`` consumes.
 * ``array`` — the same for an array's aligned items, plus missing/spurious counts.
+* ``number`` — the lenient numeric parsing shared by the numeric field metrics.
+* ``null`` — the ``(None, None) → 1.0`` rule shared by the comparison field metrics.
 """

structured_eval-0.2.0/structured_eval/metrics/utils/null.py

@@ -0,0 +1,20 @@
+"""The ``(None, None) → 1.0`` rule shared by the comparison field metrics.
+
+The schemas under evaluation ask for ``null`` whenever a value is absent, so a
+null expectation met by a null answer is a *correct* answer. Without this gate a
+metric's type check (str / number / date) would reject the pair and score that
+right answer ``0.0``. Only ``None`` counts as null — an empty string, an empty
+list or a missing key are values, and are graded as such.
+
+A one-sided ``None`` stays a mismatch: a value was expected and nothing came
+back, or nothing was expected and a value was invented.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def both_null(actual: Any, expected: Any) -> bool:
+    """True when neither side has a value — the two agree."""
+    return actual is None and expected is None

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: structured-eval
-Version: 0.1.1
+Version: 0.2.0
 Summary: The LLM Structured Output Evaluation Framework
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/kirillpechurin/structured-eval

{structured_eval-0.1.1 → structured_eval-0.2.0}/structured_eval.egg-info/SOURCES.txt

@@ -77,6 +77,7 @@ structured_eval/metrics/schema_validity/validator.py
 structured_eval/metrics/utils/__init__.py
 structured_eval/metrics/utils/array.py
 structured_eval/metrics/utils/calculate.py
+structured_eval/metrics/utils/null.py
 structured_eval/metrics/utils/number.py
 structured_eval/metrics/utils/object_utils.py
 structured_eval/models/__init__.py

structured_eval-0.1.1/structured_eval/metrics/character_f1.py

@@ -1,50 +0,0 @@
-from __future__ import annotations
-
-import re
-from collections import Counter
-from typing import Any
-
-from structured_eval.metrics.base import FieldMetric
-
-_NON_WORD = re.compile(r"[^\w\s]")
-
-
-def _characters(value: Any) -> list[str]:
-    """Lowercase, drop punctuation and whitespace, split into characters."""
-    normalized = _NON_WORD.sub("", str(value).lower())
-    normalized = "".join(normalized.split())  # remove all whitespace
-    return list(normalized)
-
-
-class CharacterF1(FieldMetric):
-    """Character-overlap F1 for short free-text fields.
-
-    Characters are matched as a **multiset** (``Counter``), so repeated
-    characters contribute only as many times as they appear on both sides.
-    Precision and recall are computed over character counts, and their
-    harmonic mean is returned. String-only: if either side is not a ``str``
-    the score is ``0.0`` (no coercion).
-    """
-
-    name = "character_f1"
-
-    def score(self, actual: Any, expected: Any) -> float:
-        if not (isinstance(actual, str) and isinstance(expected, str)):
-            return 0.0
-
-        a = _characters(actual)
-        e = _characters(expected)
-
-        if not a and not e:
-            return 1.0
-        if not a or not e:
-            return 0.0
-
-        same = sum((Counter(a) & Counter(e)).values())
-        if not same:
-            return 0.0
-
-        precision = same / len(a)
-        recall = same / len(e)
-
-        return 2 * precision * recall / (precision + recall)

structured_eval-0.1.1/structured_eval/metrics/token_f1.py

@@ -1,44 +0,0 @@
-from __future__ import annotations
-
-import re
-from collections import Counter
-from typing import Any
-
-from structured_eval.metrics.base import FieldMetric
-
-_NON_WORD = re.compile(r"[^\w\s]")
-
-
-def _tokenize(value: Any) -> list[str]:
-    """Lowercase, drop punctuation, split on whitespace."""
-    return _NON_WORD.sub(" ", str(value).lower()).split()
-
-
-class TokenF1(FieldMetric):
-    """SQuAD-style token-overlap F1 — a default for free-text fields.
-
-    Tokens are matched as a **multiset** (``Counter``), counting shared tokens
-    with multiplicity exactly like the official SQuAD F1 — so a repeated token
-    only helps as often as it appears on both sides (``"the the cat"`` vs
-    ``"the cat"`` is 0.8, not 1.0). Precision and recall are over the token
-    *counts*; their harmonic mean is the score. String-only: if either side is
-    not a ``str`` the score is 0.0 (no coercion).
-    """
-
-    name = "token_f1"
-
-    def score(self, actual: Any, expected: Any) -> float:
-        if not (isinstance(actual, str) and isinstance(expected, str)):
-            return 0.0
-        a = _tokenize(actual)
-        e = _tokenize(expected)
-        if not a and not e:
-            return 1.0
-        if not a or not e:
-            return 0.0
-        same = sum((Counter(a) & Counter(e)).values())
-        if not same:
-            return 0.0
-        precision = same / len(a)
-        recall = same / len(e)
-        return 2 * precision * recall / (precision + recall)