auditkit 1.0.0__py3-none-any.whl

auditkit/metric.py

@@ -0,0 +1,199 @@
+"""Metrics: the internal, richer form of a scorer.
+
+A :class:`Metric` turns one sample plus one model output into one or more
+:class:`Score`. It knows which sample fields it needs (``required_fields``), so
+the runner can skip a metric that doesn't apply instead of crashing — an
+exact-match metric is silently skipped for an open-ended sample with no gold.
+The two here are deterministic string matchers; judge/code/security metrics
+arrive as later techniques but wear this same interface.
+"""
+
+from __future__ import annotations
+
+import re
+from abc import ABC, abstractmethod
+from typing import Any, Optional, Union
+
+from .registry import METRICS
+from .sample import Sample
+from .score import Score
+from .types import Direction, ScoreKind
+from ._identity_guard import warn_if_identity_incomplete
+
+
+class Metric(ABC):
+    """Score a model output against a sample."""
+
+    name: str = "metric"
+    kind: ScoreKind = ScoreKind.BENCHMARK
+    is_deterministic: bool = True
+    required_fields: frozenset[str] = frozenset()
+
+    def __init_subclass__(cls, **kwargs) -> None:
+        super().__init_subclass__(**kwargs)
+        warn_if_identity_incomplete(cls, Metric, "score")
+        if not hasattr(cls, "direction"):
+            raise TypeError(
+                f"{cls.__name__} must declare 'direction' (Direction.MAXIMIZE or "
+                f"Direction.MINIMIZE) -- required so RunComparison/compare_models() "
+                f"grade it in the right sense (a metric where lower is better, e.g. "
+                f"latency or error rate, graded as MAXIMIZE would report a regression "
+                f"as an improvement). Set it as a class attribute:\n\n"
+                f"    class {cls.__name__}(Metric):\n"
+                f"        direction = Direction.MAXIMIZE  # or Direction.MINIMIZE\n\n"
+                f"See docs/best_practices/scorers.md#direction for a full example."
+            )
+
+    def applicable(self, sample: Sample) -> bool:
+        """False when any required field is absent on the sample (→ skipped)."""
+        for field_name in self.required_fields:
+            if getattr(sample, field_name, None) is None:
+                return False
+        return True
+
+    @abstractmethod
+    def score(self, sample: Sample, output: str, context: Any = None) -> Union[Score, list[Score]]:
+        """Produce a Score (or list) for ``output`` against ``sample``."""
+        ...
+
+    async def ascore(self, sample: Sample, output: str, context: Any = None):
+        """Async entry point; deterministic metrics just defer to :meth:`score`."""
+        return self.score(sample, output, context)
+
+    def identity(self) -> dict:
+        """The config that defines this metric's scoring, for the run fingerprint.
+
+        Deterministic metrics are fully described by their name. Judges override
+        this to include the judge model + prompt + choices, so changing the judge
+        (a different "ruler") changes the run fingerprint and never silently
+        reuses a cached result scored by a different judge. ``__init_subclass__``
+        above warns at class-definition time when a subclass takes constructor
+        arguments but skips this override (unless it instead makes ``self.name``
+        itself parameter-derived, which already protects the fingerprint — see
+        ``_identity_guard.py``)."""
+        return {"name": self.name}
+
+
+@METRICS.register("exact_match")
+class ExactMatch(Metric):
+    """1.0 iff the output equals the target after trimming outer whitespace."""
+
+    name = "exact_match"
+    direction = Direction.MAXIMIZE
+    required_fields = frozenset({"target"})
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        hit = output.strip() == (sample.target or "").strip()
+        return Score(name=self.name, value=1.0 if hit else 0.0, kind=self.kind)
+
+
+_ARTICLE_RE = re.compile(r"\b(a|an|the)\b", re.IGNORECASE)
+_PUNCT_RE = re.compile(r"[^\w\s]")
+
+
+def _normalize(text: str) -> str:
+    """Lowercase, drop punctuation and articles, collapse whitespace."""
+    text = text.lower()
+    text = _PUNCT_RE.sub(" ", text)
+    text = _ARTICLE_RE.sub(" ", text)
+    return " ".join(text.split())
+
+
+@METRICS.register("quasi_exact_match")
+class QuasiExactMatch(Metric):
+    """Exact match after normalization (case, punctuation, articles, spacing)."""
+
+    name = "quasi_exact_match"
+    direction = Direction.MAXIMIZE
+    required_fields = frozenset({"target"})
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        hit = _normalize(output) == _normalize(sample.target or "")
+        return Score(name=self.name, value=1.0 if hit else 0.0, kind=self.kind)
+
+
+def _resolve_choice_index(value: Any, choices: list) -> Optional[int]:
+    """Resolve a target or model-output value to a 0-based index into ``choices``.
+
+    ``value`` legitimately shows up in three encodings across scenarios and
+    custom loaders: a bare letter ("B", only meaningful for <=26 choices), a
+    0-based index into ``choices`` (``1`` or ``"1"``), or the literal text of
+    the correct choice. All three are accepted here so ``Acc``/``AccNorm``
+    score correctly regardless of which one a given dataset uses, instead of
+    silently mis-scoring (or crashing on a non-str target, e.g. an int).
+
+    The canonical return value is a 0-based **index**, not a letter — letters
+    run out at 26 choices (``chr(65+26)`` is ``'['``, not a letter) and
+    collide once you reach lowercase range (index 32 -> ``'a'``, which
+    case-normalizes to the same key as index 0's ``'A'``). An index has no
+    such ceiling, so it's what both the adapter's prompt labels (see
+    :class:`~auditkit.adapter.MCQAdapter`) and this resolver key off of.
+    """
+    if value is None:
+        return None
+    choices = choices or []
+    if isinstance(value, str):
+        s = value.strip()
+        # Compare against *stripped* choices, not raw ones -- MCQ choices
+        # commonly carry a leading space (the correct convention for
+        # GPT-2/BPE-style loglikelihood scoring, so " Paris" tokenizes as a
+        # proper word-initial continuation of the prompt), and comparing an
+        # already-stripped candidate against unstripped choices can never
+        # match even on a perfect pick.
+        for i, c in enumerate(choices):
+            if isinstance(c, str) and c.strip() == s:
+                return i
+        if s.lstrip("-").isdigit():
+            idx = int(s)
+            if 0 <= idx < len(choices):
+                return idx
+        if len(s) == 1 and s.isalpha():
+            idx = ord(s.upper()) - ord("A")
+            if 0 <= idx < len(choices):
+                return idx
+    elif isinstance(value, int) and not isinstance(value, bool):
+        if 0 <= value < len(choices):
+            return value
+    return None
+
+
+def _target_index(sample: Sample) -> int:
+    idx = _resolve_choice_index(sample.target, sample.choices or [])
+    if idx is None:
+        raise ValueError(
+            f"cannot resolve sample.target={sample.target!r} against "
+            f"sample.choices={sample.choices!r}; expected a 0-based index, "
+            f"a choice letter (e.g. 'B', only for <=26 choices), or the "
+            f"exact text of the correct choice"
+        )
+    return idx
+
+
+@METRICS.register("acc")
+class Acc(Metric):
+    """1.0 iff the output resolves to the same choice index as the target."""
+
+    name = "acc"
+    direction = Direction.MAXIMIZE
+    required_fields = frozenset({"target", "choices"})
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        target_idx = _target_index(sample)
+        pred_idx = _resolve_choice_index(output, sample.choices or [])
+        hit = pred_idx is not None and pred_idx == target_idx
+        return Score(name=self.name, value=1.0 if hit else 0.0, kind=self.kind)
+
+
+@METRICS.register("acc_norm")
+class AccNorm(Metric):
+    """1.0 iff the predicted choice text maps to the correct target choice."""
+
+    name = "acc_norm"
+    direction = Direction.MAXIMIZE
+    required_fields = frozenset({"target", "choices"})
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        target_idx = _target_index(sample)
+        pred_idx = _resolve_choice_index(output, sample.choices or [])
+        hit = pred_idx is not None and pred_idx == target_idx
+        return Score(name=self.name, value=1.0 if hit else 0.0, kind=self.kind)

auditkit/metrics/README.md

@@ -0,0 +1,15 @@
+# Metrics
+
+Built-in metric families for evaluating model outputs:
+
+- **code** — Contains, Equals, F1Score, IsJson, Levenshtein, Regex, StartsWith, EndsWith, WordCount
+- **embedding** — CosineSimilarity, TokenOverlap, BM25Similarity
+- **generation** — Bleu, RogueL, ChrF, BertScore, Perplexity, WordErrorRate
+- **hallucination** — FactualConsistency
+- **judge** — JudgeMetric, LLMJudge, GEval, RubricItem, Factuality, ClosedQA, Relevance, BiasJudge
+- **pairwise** — WinRate, EloScore, PreferenceAccuracy
+- **perf** — LatencyStats, Throughput
+- **rag** — LexicalGroundedness, ContextCoverage, ContextOverlap, AnswerOverlap
+- **security** — DefconGrade, KeywordDetector, ThreatCategory
+- **toxicity** — ToxicityScore, RepresentationSkew, HateSpeechScore
+- **guard** — GuardJudge (safety scoring via a guard model, e.g. Llama Guard)

auditkit/metrics/code.py

@@ -0,0 +1,222 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from auditkit.metric import Metric
+from auditkit.registry import METRICS
+from auditkit.sample import Sample
+from auditkit.score import Score
+from auditkit.types import Direction, ScoreKind
+
+
+@METRICS.register("equals")
+class Equals(Metric):
+    kind = ScoreKind.CODE
+    direction = Direction.MAXIMIZE
+
+    def __init__(self, ignore_case: bool = False) -> None:
+        self._ignore_case = ignore_case
+        self.name = "equals_ci" if ignore_case else "equals"
+        self.required_fields = frozenset({"target"})
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        a = output.strip()
+        b = (sample.target or "").strip()
+        if self._ignore_case:
+            hit = a.lower() == b.lower()
+        else:
+            hit = a == b
+        return Score(name=self.name, value=1.0 if hit else 0.0, kind=self.kind)
+
+
+@METRICS.register("contains")
+class Contains(Metric):
+    kind = ScoreKind.CODE
+    direction = Direction.MAXIMIZE
+
+    def __init__(self, substring: str, ignore_case: bool = False) -> None:
+        self._substring = substring
+        self._ignore_case = ignore_case
+        prefix = "contains_ci" if ignore_case else "contains"
+        self.name = f"{prefix}({substring})"
+        self.required_fields = frozenset()
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        if self._ignore_case:
+            hit = self._substring.lower() in output.lower()
+        else:
+            hit = self._substring in output
+        return Score(name=self.name, value=1.0 if hit else 0.0, kind=self.kind)
+
+
+@METRICS.register("starts_with")
+class StartsWith(Metric):
+    kind = ScoreKind.CODE
+    direction = Direction.MAXIMIZE
+
+    def __init__(self, prefix: str, ignore_case: bool = False) -> None:
+        self._prefix = prefix
+        self._ignore_case = ignore_case
+        prefix_label = "startswith_ci" if ignore_case else "startswith"
+        self.name = f"{prefix_label}({prefix})"
+        self.required_fields = frozenset()
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        if self._ignore_case:
+            hit = output.lower().startswith(self._prefix.lower())
+        else:
+            hit = output.startswith(self._prefix)
+        return Score(name=self.name, value=1.0 if hit else 0.0, kind=self.kind)
+
+
+@METRICS.register("ends_with")
+class EndsWith(Metric):
+    kind = ScoreKind.CODE
+    direction = Direction.MAXIMIZE
+
+    def __init__(self, suffix: str, ignore_case: bool = False) -> None:
+        self._suffix = suffix
+        self._ignore_case = ignore_case
+        self.name = f"endswith({suffix})"
+        self.required_fields = frozenset()
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        if self._ignore_case:
+            hit = output.lower().endswith(self._suffix.lower())
+        else:
+            hit = output.endswith(self._suffix)
+        return Score(name=self.name, value=1.0 if hit else 0.0, kind=self.kind)
+
+
+@METRICS.register("regex")
+class Regex(Metric):
+    kind = ScoreKind.CODE
+    direction = Direction.MAXIMIZE
+
+    def __init__(self, pattern: str) -> None:
+        self._compiled = re.compile(pattern)
+        self.name = f"regex({pattern})"
+        self.required_fields = frozenset()
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        hit = bool(self._compiled.search(output))
+        return Score(name=self.name, value=1.0 if hit else 0.0, kind=self.kind)
+
+
+@METRICS.register("levenshtein")
+class Levenshtein(Metric):
+    kind = ScoreKind.CODE
+    direction = Direction.MAXIMIZE
+
+    def __init__(self) -> None:
+        self.name = "levenshtein"
+        self.required_fields = frozenset()
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        ref = sample.target or ""
+        if not output and not ref:
+            return Score(name=self.name, value=0.0, kind=self.kind)
+        distance = self._edit_distance(output, ref)
+        max_len = max(len(output), len(ref))
+        score_val = 1.0 - distance / max_len
+        return Score(name=self.name, value=score_val, kind=self.kind)
+
+    @staticmethod
+    def _edit_distance(a: str, b: str) -> int:
+        m, n = len(a), len(b)
+        prev = list(range(n + 1))
+        for i in range(1, m + 1):
+            curr = [i] * (n + 1)
+            for j in range(1, n + 1):
+                cost = 0 if a[i - 1] == b[j - 1] else 1
+                curr[j] = min(
+                    prev[j] + 1,
+                    curr[j - 1] + 1,
+                    prev[j - 1] + cost,
+                )
+            prev = curr
+        return prev[n]
+
+
+@METRICS.register("word_count")
+class WordCount(Metric):
+    kind = ScoreKind.CODE
+    direction = Direction.MAXIMIZE
+
+    def __init__(self, min_words: int = 0, max_words: int | None = None) -> None:
+        self._min = min_words
+        self._max = max_words
+        if min_words > 0 and max_words is not None:
+            self.name = f"wordcount({min_words}-{max_words})"
+        elif min_words > 0:
+            self.name = f"wordcount({min_words}-)"
+        elif max_words is not None:
+            self.name = f"wordcount(-{max_words})"
+        else:
+            self.name = f"wordcount({min_words}-)"
+        self.required_fields = frozenset()
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        count = len(output.split())
+        if count < self._min:
+            hit = False
+        elif self._max is not None and count > self._max:
+            hit = False
+        else:
+            hit = True
+        return Score(name=self.name, value=1.0 if hit else 0.0, kind=self.kind)
+
+
+@METRICS.register("is_json")
+class IsJson(Metric):
+    kind = ScoreKind.CODE
+    direction = Direction.MAXIMIZE
+
+    def __init__(self, require_keys: list[str] | None = None) -> None:
+        self._require_keys = require_keys
+        if require_keys:
+            keys_str = ",".join(require_keys)
+            self.name = f"is_json(keys={keys_str})"
+        else:
+            self.name = "is_json"
+        self.required_fields = frozenset()
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        try:
+            parsed = json.loads(output)
+        except (json.JSONDecodeError, ValueError):
+            return Score(name=self.name, value=0.0, kind=self.kind)
+        if self._require_keys is not None:
+            if not isinstance(parsed, dict):
+                return Score(name=self.name, value=0.0, kind=self.kind)
+            if not all(k in parsed for k in self._require_keys):
+                return Score(name=self.name, value=0.0, kind=self.kind)
+        return Score(name=self.name, value=1.0, kind=self.kind)
+
+
+@METRICS.register("f1_score")
+class F1Score(Metric):
+    kind = ScoreKind.CODE
+    direction = Direction.MAXIMIZE
+
+    def __init__(self) -> None:
+        self.name = "f1_score"
+        self.required_fields = frozenset({"target"})
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        output_tokens = output.split()
+        target_tokens = (sample.target or "").split()
+        if not output_tokens and not target_tokens:
+            return Score(name=self.name, value=0.0, kind=self.kind)
+        out_set = set(output_tokens)
+        tgt_set = set(target_tokens)
+        intersection = out_set & tgt_set
+        precision = len(intersection) / len(output_tokens) if output_tokens else 0.0
+        recall = len(intersection) / len(target_tokens) if target_tokens else 0.0
+        if precision == 0.0 and recall == 0.0:
+            f1 = 0.0
+        else:
+            f1 = 2.0 * precision * recall / (precision + recall)
+        return Score(name=self.name, value=f1, kind=self.kind)

auditkit/metrics/embedding.py

@@ -0,0 +1,131 @@
+"""Embedding-based similarity metrics."""
+from __future__ import annotations
+
+from typing import Any
+
+from auditkit.metric import Metric
+from auditkit.registry import METRICS
+from auditkit.sample import Sample
+from auditkit.score import Score
+from auditkit.types import Direction, ScoreKind
+from auditkit.errors import ExtraNotInstalled
+
+
+@METRICS.register("cosine_similarity")
+class CosineSimilarity(Metric):
+    """Embedding cosine similarity via plain ``transformers`` -- no
+    ``sentence-transformers`` dependency.
+
+    A previous version of this metric used the ``sentence-transformers``
+    package, which transitively imports ``transformers``' audio/video
+    processing modules and, in turn, ``torchcodec`` -- a real, live-
+    confirmed environment failure on Colab (``torch``/``torchcodec``
+    version mismatch, unrelated ``libavutil``/FFmpeg shared libraries
+    missing) that has nothing to do with text embeddings at all and took
+    this metric down with it. ``sentence-transformers`` was used for
+    exactly this one metric in the whole library (confirmed: no other
+    file imports it), so removing it entirely and reimplementing the
+    same computation directly on ``transformers.AutoModel`` -- already a
+    core dependency for :class:`EncoderJudge`/``HFGenModel``, and
+    confirmed *not* to trigger the ``torchcodec`` import path -- avoids
+    the whole problem rather than working around it.
+
+    Mean-pooling + L2-normalization is the exact recipe
+    ``sentence-transformers`` itself uses internally for MiniLM-family
+    checkpoints (documented on the model card) -- this reproduces the
+    same embeddings, not an approximation.
+    """
+
+    name = "cosine_similarity"
+    kind = ScoreKind.BENCHMARK
+    direction = Direction.MAXIMIZE
+    is_deterministic = False
+    required_fields = frozenset({"target"})
+
+    def __init__(self, model_name: str = "sentence-transformers/all-MiniLM-L6-v2") -> None:
+        self._model_name = model_name
+        self._model = None
+        self._tokenizer = None
+        self._torch = None
+
+    def identity(self) -> dict:
+        return {"name": self.name, "model_name": self._model_name}
+
+    def _ensure_model(self) -> None:
+        if self._model is not None:
+            return
+        try:
+            import torch
+            from transformers import AutoModel, AutoTokenizer
+        except ImportError:
+            raise ExtraNotInstalled("transformers", "pip install auditkit[transformers]")
+        self._torch = torch
+        self._tokenizer = AutoTokenizer.from_pretrained(self._model_name)
+        self._model = AutoModel.from_pretrained(self._model_name)
+        self._model.eval()
+
+    def _embed(self, text: str) -> Any:
+        torch = self._torch
+        inputs = self._tokenizer([text], padding=True, truncation=True, return_tensors="pt")
+        with torch.no_grad():
+            output = self._model(**inputs)
+        # Mean pooling over real (non-padding) token embeddings -- the
+        # attention mask zeroes out padding positions before averaging,
+        # exactly as sentence-transformers does for this checkpoint family.
+        token_embeddings = output.last_hidden_state
+        mask = inputs["attention_mask"].unsqueeze(-1).expand(token_embeddings.size()).float()
+        summed = (token_embeddings * mask).sum(dim=1)
+        counts = mask.sum(dim=1).clamp(min=1e-9)
+        embedding = (summed / counts)[0]
+        norm = embedding.norm(p=2)
+        if norm > 0:
+            embedding = embedding / norm
+        return embedding
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        self._ensure_model()
+        emb_out = self._embed(output)
+        emb_tgt = self._embed(sample.target or "")
+        similarity = float(self._torch.dot(emb_out, emb_tgt).item())
+        return Score(name=self.name, value=similarity, kind=self.kind)
+
+
+@METRICS.register("token_overlap")
+class TokenOverlap(Metric):
+    name = "token_overlap"
+    kind = ScoreKind.BENCHMARK
+    direction = Direction.MAXIMIZE
+    is_deterministic = True
+    required_fields = frozenset({"target"})
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        tokens_out = set(output.split())
+        tokens_tgt = set((sample.target or "").split())
+        if not tokens_out and not tokens_tgt:
+            return Score(name=self.name, value=0.0, kind=self.kind)
+        intersection = tokens_out & tokens_tgt
+        union = tokens_out | tokens_tgt
+        return Score(name=self.name, value=len(intersection) / len(union), kind=self.kind)
+
+
+@METRICS.register("bm25_similarity")
+class BM25Similarity(Metric):
+    name = "bm25_similarity"
+    kind = ScoreKind.BENCHMARK
+    direction = Direction.MAXIMIZE
+    is_deterministic = True
+    required_fields = frozenset({"target"})
+
+    def score(self, sample: Sample, output: str, context: Any = None) -> Score:
+        tokens_out = output.split()
+        tokens_tgt = (sample.target or "").split()
+        freq_out: dict[str, int] = {}
+        for t in tokens_out:
+            freq_out[t] = freq_out.get(t, 0) + 1
+        freq_tgt: dict[str, int] = {}
+        for t in tokens_tgt:
+            freq_tgt[t] = freq_tgt.get(t, 0) + 1
+        all_tokens = set(freq_out) | set(freq_tgt)
+        score_val = sum(min(freq_out.get(t, 0), freq_tgt.get(t, 0)) for t in all_tokens)
+        max_len = max(len(tokens_out), len(tokens_tgt), 1)
+        return Score(name=self.name, value=score_val / max_len, kind=self.kind)