auditkit 1.0.0__py3-none-any.whl

auditkit/runspec.py

@@ -0,0 +1,159 @@
+"""The reproducible description of a run: :class:`RunConfig` and :class:`RunSpec`.
+
+A ``RunSpec`` binds together everything a run needs — the scenario, the model,
+the adapter, the metrics, the config — and can distil itself to a stable
+:meth:`RunSpec.fingerprint`. Two runs with the same fingerprint are the same
+experiment; changing the model, the seed, the few-shot count, or the judge
+changes the fingerprint, which is what makes results comparable and cacheable.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from typing import Any, TYPE_CHECKING, Union
+
+if TYPE_CHECKING:  # avoid import cycles; these are only type hints here
+    from .adapter import Adapter
+    from .metric import Metric
+    from .model import Model
+    from .scenario import Scenario
+
+
+@dataclass
+class RunConfig:
+    """The knobs that shape a run (all optional, all with sane defaults)."""
+
+    num_fewshot: Union[int, None] = None
+    limit: Union[int, None] = None
+    # None (not 0): every other generation-affecting field here defaults to
+    # None so _gen_params() correctly skips it when unset. A concrete 0
+    # defeats that -- seed=0 was being force-included in every generation
+    # request whether the caller asked for it or not, and some providers
+    # (confirmed: at least one Groq model) reject the seed param outright,
+    # turning every request into a 400. Explicitly pass seed=0 if you want
+    # a real, reproducible seed sent to the backend.
+    seed: Union[int, None] = None
+    trials: int = 1
+    batch_size: Union[str, int] = "auto"
+    # Default 1 (not higher) because Runner._chunk() always splits requests
+    # into exactly `concurrency` chunks regardless of request count -- with
+    # fewer requests than concurrency, that produces empty chunks and wastes
+    # real model/API calls on them.
+    # concurrency<=1 takes Runner.execute()'s no-chunking path entirely, so
+    # this is the only value that's safe unconditionally.
+    concurrency: int = 1
+    split: SplitConfig | None = None
+    temperature: float = 0.0
+    top_p: float | None = None
+    top_k: int | None = None
+    max_tokens: int | None = None
+    stop_sequences: list[str] | None = None
+    presence_penalty: float | None = None
+    frequency_penalty: float | None = None
+    num_completions: int | None = None
+    best_of: int | None = None
+    timeout: float | None = None
+    max_retries: int = 3
+    retry_delay: float = 1.0
+    judge_model: Union[str, None] = None
+    judge_prompt_version: Union[str, None] = None
+    # On by default: RunResult.perf/model_size/token_usage are populated
+    # unless explicitly disabled. Set False to skip surfacing them (they are
+    # still measured for free during the run either way -- this only gates
+    # whether they're attached to the result; see Runner.run()) -- useful if
+    # local-backend model_size introspection's real, if best-effort, cost is
+    # unwanted. Part of the fingerprint (RunConfig.to_dict() is hashed
+    # whole), so toggling this forces a fresh run rather than replaying a
+    # cached perf-less result.
+    track_performance: bool = True
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {f.name: getattr(self, f.name) for f in self.__dataclass_fields__.values()}
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> RunConfig:
+        field_names = {f.name for f in cls.__dataclass_fields__.values()}
+        valid = {k: v for k, v in data.items() if k in field_names}
+        return cls(**valid)
+
+
+@dataclass
+class SplitConfig:
+    strategy: str = "sequential"
+    train_ratio: float = 0.0
+    val_ratio: float = 0.0
+    test_ratio: float = 1.0
+    fold: int = 0
+    # Fixed, not None: Runner._split_samples()'s "random"/"stratified" paths
+    # seed `random.Random(seed)` fresh on every call. `None` draws OS entropy
+    # each time, so reusing the identical SplitConfig (or RunConfig) across a
+    # baseline and a candidate evaluate() call -- the normal "same eval
+    # config" comparison pattern -- would silently reshuffle differently each
+    # time, breaking RunComparison's sample_id-based alignment (newly_wrong,
+    # significance) even though nothing about the config looked different.
+    # A caller who genuinely wants a different shuffle per run still can, by
+    # passing a different seed= explicitly.
+    seed: int | None = 0
+
+
+@dataclass
+class RunSpec:
+    """Everything a run needs, plus a stable fingerprint over what matters."""
+
+    scenario: "Scenario"
+    model: "Model"
+    adapter: "Adapter"
+    metrics: list["Metric"]
+    annotators: list[Any] = field(default_factory=list)
+    # Names an annotator (by its .name) whose context["extracted"] value
+    # should be scored instead of the raw output -- e.g. a RegexAnnotator
+    # pulling "42" out of "...FINAL ANSWER: 42". None (default): score the
+    # raw output, today's behavior, unchanged.
+    extracted_by: Union[str, None] = None
+    config: RunConfig = field(default_factory=RunConfig)
+    run_name: str = ""
+
+    def fingerprint(self) -> str:
+        """A stable sha256[:16] over the identity-defining parts of the run.
+
+        Hashes the *entire* config (via ``RunConfig.to_dict()``) rather than a
+        hand-picked subset of fields, so a run-affecting setting (``limit``,
+        ``batch_size``, a new field added later, ...) can never be silently
+        left out of the fingerprint the way ``limit`` previously was.
+        """
+        def _identity(obj: Any, default_attr: str) -> Any:
+            fn = getattr(obj, "identity", None)
+            if callable(fn):
+                return fn()
+            return getattr(obj, default_attr, type(obj).__name__)
+
+        key = {
+            "engine": "native",
+            "model": _identity(self.model, "name"),
+            "scenario": getattr(self.scenario, "name", type(self.scenario).__name__),
+            # identity() captures the *prompt* (adapter template/system prompt,
+            # judge prompt/choices), not just the type — so changing a prompt
+            # changes the fingerprint and can't return a stale cached result.
+            "adapter": _identity(self.adapter, "method"),
+            "metrics": sorted(
+                json.dumps(_identity(m, "name"), sort_keys=True, default=str)
+                for m in self.metrics
+            ),
+            # Annotators have no scoring effect unless extracted_by names
+            # one -- but hashing both unconditionally means changing a
+            # regex pattern (or which annotator feeds scoring) always
+            # invalidates the cache, never silently reuses a stale result
+            # scored under a different pattern.
+            "annotators": sorted(
+                json.dumps(_identity(a, "name"), sort_keys=True, default=str)
+                for a in self.annotators
+            ),
+            "extracted_by": self.extracted_by,
+            "run_name": self.run_name,
+            "config": self.config.to_dict(),
+        }
+        blob = json.dumps(key, sort_keys=True, default=str).encode("utf-8")
+        return hashlib.sha256(blob).hexdigest()[:16]

auditkit/sample.py

@@ -0,0 +1,40 @@
+"""The unit of evaluation: a :class:`Sample`.
+
+A sample is one thing to ask a model and (optionally) what a correct answer
+looks like. Most fields are optional and task-specific — an MCQ sample carries
+``choices``, a RAG sample carries ``retrieval_context`` — so one dataclass
+covers every task the spine handles.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Optional, Union
+
+from .types import TaskKind
+
+
+@dataclass
+class Sample:
+    """One evaluation item: what to ask, and (optionally) the reference answer."""
+
+    input: str
+    target: Optional[Union[str, int]] = None
+    id: Optional[str] = None
+    task: str = ""
+    kind: TaskKind = TaskKind.GENERATIVE
+    choices: Optional[list[str]] = None
+    retrieval_context: Optional[list[str]] = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    tags: list[str] = field(default_factory=list)
+    actual_output: Optional[str] = None
+
+    @property
+    def is_golden(self) -> bool:
+        """True when the sample carries a reference answer to score against."""
+        return self.target is not None
+
+    @property
+    def input_text(self) -> str:
+        """The input as plain text — what a text-only backend is prompted with."""
+        return self.input

auditkit/scenario.py

@@ -0,0 +1,88 @@
+"""Scenarios: a task and its data, yielding :class:`Sample`.
+
+Users pass a task name, a list of samples, or a callable to ``evaluate()``; the
+façade wraps whatever they pass into a :class:`Scenario` so the runner sees one
+shape. :class:`ListScenario` is the trivial wrapper around an in-memory list;
+:class:`CallableScenario` defers to a function that yields samples (a custom
+loader, a generator, a streaming source).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Iterable
+
+from .sample import Sample
+
+
+class Scenario(ABC):
+    """A task plus its data, as an iterable of samples."""
+
+    name: str = "scenario"
+
+    def __init__(self, name: str = "scenario", metadata: dict[str, Any] | None = None) -> None:
+        self.name = name
+        self.metadata: dict[str, Any] = metadata or {}
+
+    @abstractmethod
+    def samples(self) -> Iterable[Sample]:
+        ...
+
+
+class ListScenario(Scenario):
+    """A scenario backed by an in-memory list of samples."""
+
+    def __init__(self, samples: Iterable[Sample], name: str | None = None,
+                 metadata: dict[str, Any] | None = None) -> None:
+        samples = list(samples)
+        if name is None:
+            # Deliberately excludes s.id: Runner mutates a sample's id in
+            # place the first time it's scored (`if sample.id is None:
+            # sample.id = str(index)`), so including id here meant re-running
+            # the exact same Sample objects a second time produced a
+            # different auto-generated name -- and therefore a different
+            # RunSpec.fingerprint() -- for identical content, defeating the
+            # cache purely from that side effect. Identity for caching
+            # purposes should track the data, not a bookkeeping field that
+            # changes as a side effect of a prior run.
+            # Covers every field a built-in Adapter actually reads (input,
+            # target for all; choices for MCQAdapter; retrieval_context for
+            # RAGAdapter) -- input/target alone let two samples that differ
+            # only in retrieval_context or choices collide onto the same
+            # name/fingerprint and silently share a DiskCache entry.
+            #
+            # actual_output is included for the same reason: it's what
+            # PrecomputedModel (reads_actual_output=True) actually scores in
+            # the documented "generate once, score many times" flow
+            # (ak.generate() -> model="precomputed"). Without it, two
+            # datasets sharing the same input/target but carrying DIFFERENT
+            # generated text (e.g. two different models' real outputs, or a
+            # re-generation after a prompt change) hash to the identical
+            # name/fingerprint -- confirmed live: scoring "positive" (correct,
+            # 1.0) then "negative" (wrong, should be 0.0) against the same
+            # input/target silently returned the FIRST call's cached 1.0 for
+            # the second, wrong-answer sample.
+            h = hashlib.sha256(json.dumps(
+                [(s.input, s.target, s.choices, s.retrieval_context, s.actual_output) for s in samples],
+                default=str, sort_keys=True,
+            ).encode()).hexdigest()[:8]
+            name = f"inline_{len(samples)}_{h}"
+        super().__init__(name=name, metadata=metadata)
+        self._samples = samples
+
+    def samples(self) -> list[Sample]:
+        return list(self._samples)
+
+
+class CallableScenario(Scenario):
+    """A scenario that calls a function to produce samples each time it's read."""
+
+    def __init__(self, fn: Callable[[], Iterable[Sample]], name: str = "callable",
+                 metadata: dict[str, Any] | None = None) -> None:
+        super().__init__(name=name, metadata=metadata)
+        self._fn = fn
+
+    def samples(self) -> list[Sample]:
+        return list(self._fn())

auditkit/scenarios/README.md

@@ -0,0 +1,10 @@
+# Scenarios
+
+Built-in benchmark scenarios for standardized evaluation:
+
+- **MMLU** — Massive Multitask Language Understanding
+- **GSM8K** — Grade School Math
+- **ARC** — AI2 Reasoning Challenge
+- **HellaSwag** — Commonsense NLI
+- **HumanEval** — Code generation
+- **TruthfulQA** — Factuality and truthfulness

auditkit/scenarios/__init__.py

@@ -0,0 +1,4 @@
+"""Pre-built benchmark scenarios (optional extra: auditkit[interop])."""
+from __future__ import annotations
+
+from . import arc, gsm8k, hellaswag, humaneval, mmlu, truthfulqa  # noqa: F401 -- triggers @SCENARIOS.register decorators

auditkit/scenarios/arc.py

@@ -0,0 +1,33 @@
+"""ARC-Challenge benchmark scenario (lazy datasets import)."""
+from __future__ import annotations
+
+from auditkit.scenario import Scenario
+from auditkit.sample import Sample
+from auditkit.types import TaskKind
+from auditkit.errors import ExtraNotInstalled
+from auditkit.registry import SCENARIOS
+
+
+@SCENARIOS.register("arc")
+class ARCScenario(Scenario):
+    name = "arc"
+
+    def __init__(self, split: str = "test"):
+        self.split = split
+        super().__init__(name=f"arc_{split}")
+
+    def samples(self):
+        try:
+            import datasets
+        except ImportError:
+            raise ExtraNotInstalled("interop", "pip install auditkit[interop]")
+        data = datasets.load_dataset("arc", "ARC-Challenge", split=self.split)
+        return [
+            Sample(
+                input=row["question"],
+                choices=list(row["choices"]["text"]),
+                target=row["answerKey"],
+                kind=TaskKind.MCQ,
+            )
+            for row in data
+        ]

auditkit/scenarios/gsm8k.py

@@ -0,0 +1,32 @@
+"""GSM8K benchmark scenario (lazy datasets import)."""
+from __future__ import annotations
+
+from auditkit.scenario import Scenario
+from auditkit.sample import Sample
+from auditkit.types import TaskKind
+from auditkit.errors import ExtraNotInstalled
+from auditkit.registry import SCENARIOS
+
+
+@SCENARIOS.register("gsm8k")
+class GSM8KScenario(Scenario):
+    name = "gsm8k"
+
+    def __init__(self, split: str = "test"):
+        self.split = split
+        super().__init__(name=f"gsm8k_{split}")
+
+    def samples(self):
+        try:
+            import datasets
+        except ImportError:
+            raise ExtraNotInstalled("interop", "pip install auditkit[interop]")
+        data = datasets.load_dataset("gsm8k", "main", split=self.split)
+        return [
+            Sample(
+                input=row["question"],
+                target=row["answer"],
+                kind=TaskKind.GENERATIVE,
+            )
+            for row in data
+        ]

auditkit/scenarios/hellaswag.py

@@ -0,0 +1,33 @@
+"""HellaSwag benchmark scenario (lazy datasets import)."""
+from __future__ import annotations
+
+from auditkit.scenario import Scenario
+from auditkit.sample import Sample
+from auditkit.types import TaskKind
+from auditkit.errors import ExtraNotInstalled
+from auditkit.registry import SCENARIOS
+
+
+@SCENARIOS.register("hellaswag")
+class HellaSwagScenario(Scenario):
+    name = "hellaswag"
+
+    def __init__(self, split: str = "validation"):
+        self.split = split
+        super().__init__(name=f"hellaswag_{split}")
+
+    def samples(self):
+        try:
+            import datasets
+        except ImportError:
+            raise ExtraNotInstalled("interop", "pip install auditkit[interop]")
+        data = datasets.load_dataset("hellaswag", split=self.split)
+        return [
+            Sample(
+                input=row["ctx"],
+                choices=list(row["endings"]),
+                target=str(row["label"]),
+                kind=TaskKind.MCQ,
+            )
+            for row in data
+        ]

auditkit/scenarios/humaneval.py

@@ -0,0 +1,32 @@
+"""HumanEval benchmark scenario (lazy datasets import)."""
+from __future__ import annotations
+
+from auditkit.scenario import Scenario
+from auditkit.sample import Sample
+from auditkit.types import TaskKind
+from auditkit.errors import ExtraNotInstalled
+from auditkit.registry import SCENARIOS
+
+
+@SCENARIOS.register("humaneval")
+class HumanEvalScenario(Scenario):
+    name = "humaneval"
+
+    def __init__(self, split: str = "test"):
+        self.split = split
+        super().__init__(name=f"humaneval_{split}")
+
+    def samples(self):
+        try:
+            import datasets
+        except ImportError:
+            raise ExtraNotInstalled("interop", "pip install auditkit[interop]")
+        data = datasets.load_dataset("openai_humaneval", split=self.split)
+        return [
+            Sample(
+                input=row["prompt"],
+                target=row["canonical_solution"],
+                kind=TaskKind.CODE,
+            )
+            for row in data
+        ]

auditkit/scenarios/mmlu.py

@@ -0,0 +1,34 @@
+"""MMLU benchmark scenario (lazy datasets import)."""
+from __future__ import annotations
+
+from auditkit.scenario import Scenario
+from auditkit.sample import Sample
+from auditkit.types import TaskKind
+from auditkit.errors import ExtraNotInstalled
+from auditkit.registry import SCENARIOS
+
+
+@SCENARIOS.register("mmlu")
+class MMLUScenario(Scenario):
+    name = "mmlu"
+
+    def __init__(self, subject: str = "all", split: str = "test"):
+        self.subject = subject
+        self.split = split
+        super().__init__(name=f"mmlu_{subject}_{split}")
+
+    def samples(self):
+        try:
+            import datasets
+        except ImportError:
+            raise ExtraNotInstalled("interop", "pip install auditkit[interop]")
+        data = datasets.load_dataset("mmlu", self.subject, split=self.split)
+        return [
+            Sample(
+                input=row["question"],
+                choices=list(row["choices"]),
+                target=row["answer"],
+                kind=TaskKind.MCQ,
+            )
+            for row in data
+        ]

auditkit/scenarios/truthfulqa.py

@@ -0,0 +1,33 @@
+"""TruthfulQA benchmark scenario (lazy datasets import)."""
+from __future__ import annotations
+
+from auditkit.scenario import Scenario
+from auditkit.sample import Sample
+from auditkit.types import TaskKind
+from auditkit.errors import ExtraNotInstalled
+from auditkit.registry import SCENARIOS
+
+
+@SCENARIOS.register("truthfulqa")
+class TruthfulQAScenario(Scenario):
+    name = "truthfulqa"
+
+    def __init__(self, split: str = "validation"):
+        self.split = split
+        super().__init__(name=f"truthfulqa_{split}")
+
+    def samples(self):
+        try:
+            import datasets
+        except ImportError:
+            raise ExtraNotInstalled("interop", "pip install auditkit[interop]")
+        data = datasets.load_dataset("truthfulqa", "multiple_choice", split=self.split)
+        return [
+            Sample(
+                input=row["question"],
+                choices=row["mc1_targets"]["choices"],
+                target=row["mc1_targets"]["labels"].index(1) if 1 in row["mc1_targets"]["labels"] else "0",
+                kind=TaskKind.MCQ,
+            )
+            for row in data
+        ]

auditkit/score.py

@@ -0,0 +1,165 @@
+"""Scores, running statistics, and the pass@k / pass^k estimators.
+
+A :class:`Score` is one measurement of one sample (a value plus what it means).
+A :class:`Stat` accumulates many values into a mean/stderr/percentile summary
+without holding every value in memory longer than it needs to. :func:`pass_at_k`
+and :func:`pass_hat_k` turn "``c`` of ``n`` trials were correct" into the two
+numbers people actually want: capability (can it ever?) and reliability (does it
+every time?).
+"""
+
+from __future__ import annotations
+
+import math
+import threading
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+from .types import DataType, Direction, ScoreKind, Source
+
+
+@dataclass
+class Score:
+    """One measurement of one sample.
+
+    ``value`` is the number; the rest says how to read it. When ``threshold`` is
+    set, :attr:`passed` turns the number into a gate honoring ``direction`` —
+    higher-is-better for ``MAXIMIZE``, lower-is-better for ``MINIMIZE``.
+    """
+
+    name: str
+    value: float
+    kind: ScoreKind = ScoreKind.BENCHMARK
+    data_type: DataType = DataType.NUMERIC
+    direction: Direction = Direction.MAXIMIZE
+    label: Optional[str] = None
+    reason: Optional[str] = None
+    threshold: Optional[float] = None
+    weight: float = 1.0
+    source: Source = Source.SDK
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def passed(self) -> Optional[bool]:
+        """``True``/``False`` against ``threshold``, or ``None`` if ungated."""
+        if self.threshold is None:
+            return None
+        if self.direction == Direction.MINIMIZE:
+            return self.value <= self.threshold
+        return self.value >= self.threshold
+
+
+class Stat:
+    """A running summary of many values (mean, stderr, spread, percentiles).
+
+    Values are kept so percentiles can be computed exactly; everything else is
+    derived on read. Empty is safe: every reduction returns ``0.0``.
+    """
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self._values: list[float] = []
+        self._lock = threading.Lock()
+
+    def add(self, value: float) -> "Stat":
+        """Record one value; returns self so calls can be chained."""
+        with self._lock:
+            self._values.append(float(value))
+        return self
+
+    @property
+    def count(self) -> int:
+        return len(self._values)
+
+    @property
+    def mean(self) -> float:
+        return sum(self._values) / len(self._values) if self._values else 0.0
+
+    @property
+    def std(self) -> float:
+        """Population standard deviation (0.0 for fewer than two values)."""
+        n = len(self._values)
+        if n < 2:
+            return 0.0
+        mu = self.mean
+        return math.sqrt(sum((v - mu) ** 2 for v in self._values) / n)
+
+    @property
+    def stderr(self) -> float:
+        n = len(self._values)
+        if n < 2:
+            return 0.0
+        return self.std / math.sqrt(n)
+
+    @property
+    def min(self) -> float:
+        return min(self._values) if self._values else 0.0
+
+    @property
+    def max(self) -> float:
+        return max(self._values) if self._values else 0.0
+
+    def percentile(self, p: float) -> float:
+        """The ``p``-th percentile (0..100) by linear interpolation."""
+        if not self._values:
+            return 0.0
+        ordered = sorted(self._values)
+        if len(ordered) == 1:
+            return ordered[0]
+        rank = (p / 100.0) * (len(ordered) - 1)
+        lo = math.floor(rank)
+        hi = math.ceil(rank)
+        if lo == hi:
+            return ordered[lo]
+        frac = rank - lo
+        return ordered[lo] + (ordered[hi] - ordered[lo]) * frac
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "count": self.count,
+            "mean": self.mean,
+            "stderr": self.stderr,
+            "std": self.std,
+            "min": self.min,
+            "max": self.max,
+            "values": list(self._values),
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Stat:
+        s = cls(data.get("name", ""))
+        for v in data.get("values", []):
+            s.add(v)
+        return s
+
+
+def _check_draw(n: int, c: int, k: int) -> None:
+    if k > n:
+        raise ValueError(f"cannot draw k={k} from n={n} trials")
+    if c > n or c < 0:
+        raise ValueError(f"correct count c={c} out of range for n={n}")
+
+
+def pass_at_k(n: int, c: int, k: int) -> float:
+    """Capability: probability that a draw of ``k`` of ``n`` trials has >=1 correct.
+
+    ``1 - C(n-c, k) / C(n, k)`` — the unbiased HumanEval estimator.
+    """
+    _check_draw(n, c, k)
+    if c == 0:
+        return 0.0
+    if n - c < k:
+        return 1.0
+    return 1.0 - math.comb(n - c, k) / math.comb(n, k)
+
+
+def pass_hat_k(n: int, c: int, k: int) -> float:
+    """Reliability: probability that all ``k`` of a draw of ``k`` are correct.
+
+    ``C(c, k) / C(n, k)``.
+    """
+    _check_draw(n, c, k)
+    if c < k:
+        return 0.0
+    return math.comb(c, k) / math.comb(n, k)