auditkit 1.0.0__py3-none-any.whl

auditkit/README.md

@@ -0,0 +1,99 @@
+# AuditKIT — Source Layout
+
+```
+auditkit/
+├── __init__.py          # Public API exports, version, __all__
+├── __main__.py          # python -m auditkit entry point
+├── adapter.py           # Adapter ABC + 7 implementations (generation, chat, fewshot, etc.)
+├── annotator.py         # Annotator ABC for post-generation annotation
+├── api.py               # evaluate(), evaluate_many(), generate(), compare(), run_lmeval()
+├── cache.py             # DiskCache — fingerprint-keyed result cache
+├── cli.py               # argparse CLI: eval, init, list, redteam, compare
+├── comparison.py        # RunComparison, MetricDelta, TaskDelta, grade_delta
+├── diff.py              # RunDiff, DeltaGrade — comparison between runs
+├── errors.py            # AuditKitError, CapabilityError, ExtraNotInstalled, etc.
+├── evaluator.py         # Evaluator ABC for technique plugins
+├── experiment.py        # Experiment, ExperimentDB — run management
+├── lmeval_engine.py     # run_benchmark() — lm-evaluation-harness engine (extra)
+├── loaders.py           # load_csv, load_hf, load_croissant
+├── logs.py              # configure_logging
+├── metric.py            # Metric ABC + ExactMatch, QuasiExactMatch, Acc, AccNorm
+├── model_compare.py     # CompareResult, compare_models() — multi-model comparison
+├── registry.py          # ObjectSpec, Registry — SCENARIOS, ADAPTERS, MODELS, etc.
+├── report.py            # Prediction, RunResult — evaluation output
+├── report_format.py     # Report — markdown formatting
+├── router.py            # route_adapter() — auto-picks an Adapter from sample shape
+├── runner.py            # Runner — 5-stage evaluation pipeline
+├── runspec.py           # RunConfig, RunSpec, fingerprint
+├── sample.py            # Sample
+├── scenario.py          # Scenario ABC, ListScenario, CallableScenario
+├── score.py             # Score, Stat, pass_at_k, pass_hat_k
+├── scorers.py           # FunctionScorer, ScorerMetric, @scorer decorator
+├── scoring.py           # ScoreGate, WeightedSum
+├── types.py             # Enums: TaskKind, ScoreKind, DataType, etc.
+│
+├── metrics/             # metric families
+│   ├── code.py          # Contains, Equals, F1Score, IsJson, Levenshtein, etc.
+│   ├── embedding.py     # CosineSimilarity, TokenOverlap, BM25Similarity
+│   ├── generation.py    # Bleu, RogueL, ChrF, BertScore, Perplexity, WordErrorRate
+│   ├── hallucination.py # FactualConsistency
+│   ├── judge.py         # JudgeMetric, LLMJudge, GEval, RubricItem, Factuality, ClosedQA, Relevance
+│   ├── pairwise.py      # WinRate, EloScore, PreferenceAccuracy
+│   ├── perf.py          # LatencyStats, Throughput
+│   ├── rag.py           # LexicalGroundedness, ContextCoverage, ContextOverlap, AnswerOverlap
+│   ├── security.py      # DefconGrade, KeywordDetector, ThreatCategory
+│   └── toxicity.py      # ToxicityScore, RepresentationSkew, HateSpeechScore
+│
+├── model/               # Model backends
+│   ├── __init__.py      # Request, Generated, Result_, Model ABC, AutoModel, CallableModel, PrecomputedModel
+│   ├── anthropic.py     # Anthropic backend (extra)
+│   ├── api_gen.py       # Generic API backend (extra)
+│   ├── groq_gen.py      # Groq backend (extra)
+│   ├── hf_gen.py        # HuggingFace Transformers backend (extra)
+│   ├── lexsi.py         # Lexsi gateway backend (extra)
+│   ├── litellm_gen.py   # LiteLLM backend (extra)
+│   ├── openai.py        # OpenAI backend (extra)
+│   └── vllm_gen.py      # vLLM backend (extra)
+│
+├── redteam/             # Red teaming
+│   ├── __init__.py      # Probe, Detector, RedTeamRunner, RedTeamResult exports
+│   ├── probe.py         # Probe ABC + ProbeResult
+│   ├── detector.py      # Detector ABC + DetectorResult
+│   ├── runner.py        # RedTeamRunner, RedTeamResult — orchestration
+│   ├── probes/
+│   │   └── builtin.py   # PromptInjectionProbe, JailbreakProbe, EncodingProbe, RefusalProbe
+│   └── detectors/
+│       └── builtin.py   # KeywordDetector, RefusalDetector, InjectionSuccessDetector, etc.
+│
+└── scenarios/           # Built-in benchmark datasets
+    ├── arc.py           # AI2 Reasoning Challenge
+    ├── gsm8k.py         # Grade School Math 8K
+    ├── hellaswag.py     # HellaSwag
+    ├── humaneval.py     # HumanEval
+    ├── mmlu.py          # Massive Multitask Language Understanding
+    └── truthfulqa.py    # TruthfulQA
+```
+
+## Architecture (the spine)
+
+Every evaluation flows through the same 5-stage pipeline:
+
+```
+Dataset → Adapter → Model → Metrics → RunResult
+```
+
+1. Build requests — `Adapter.adapt(sample, config) → list[Request]`
+2. Execute — `Model.generate(requests) → list[Result_]`
+3. Annotate — `Annotator.annotate(sample, results) → dict`
+4. Score — `Metric.score(sample, output, context) → Score`
+5. Aggregate — `Stat.add(value)` → per-metric stats
+
+## Extending
+
+**Add a metric:** Create a class inheriting `Metric`, implement `score()`, export via `__init__.py`.
+
+**Add a model backend:** Create a file in `model/`, inherit `Model`, implement `generate()`, add prefix to `AutoModel._T1_BACKENDS`.
+
+**Add a probe:** Create a class inheriting `Probe`, implement `prompts()`, register in the probe registry.
+
+**Add a detector:** Create a class inheriting `Detector`, implement `detect()`, register in the detector registry.

auditkit/__init__.py

@@ -0,0 +1,177 @@
+"""auditkit — evaluate any model on any dataset and any task."""
+
+from __future__ import annotations
+
+__version__ = "1.0.0"
+
+from .api import run_lmeval, compare, evaluate, evaluate_many, generate, scorer
+from .lmeval_engine import BenchmarkEvaluator, map_model_spec, run_benchmark
+from .loaders import load_csv, load_hf, load_croissant
+from .model_compare import CompareResult, compare_models
+from .experiment import Experiment, ExperimentDB
+from .cache import DiskCache
+from .logs import configure_logging
+from .diff import DeltaGrade, RunDiff
+from .comparison import MetricDelta, RunComparison, TaskDelta, grade_delta
+from .registry import ADAPTERS, EVALUATORS, METRICS, SCENARIOS
+from . import scenarios  # noqa: F401 -- triggers @SCENARIOS.register decorators
+from .errors import AuditKitError, ExtraNotInstalled
+from .evaluator import Evaluator
+from .model import AutoModel
+from .scenario import CallableScenario, ListScenario, Scenario
+from .metrics.code import (
+    Contains, EndsWith, Equals, F1Score, IsJson, Levenshtein, Regex, StartsWith, WordCount,
+)
+from .metrics.embedding import BM25Similarity, CosineSimilarity, TokenOverlap
+from .metrics.generation import (
+    BertScore, Bleu, ChrF, Perplexity, RogueL, WordErrorRate,
+)
+from .metrics.encoder_judge import (
+    EncoderJudge, FactualityEncoderJudge, SentimentEncoderJudge,
+)
+from .metrics.hallucination import FactualConsistency
+from .metrics.judge import (
+    BiasJudge, ClosedQA, Factuality, GEval, JudgeMetric, LLMJudge, Relevance, RubricItem,
+)
+from .metrics.pairwise import EloScore, PreferenceAccuracy, WinRate
+from .metrics.toxicity import HateSpeechScore, RepresentationSkew, ToxicityScore
+from .metrics.guard import GuardJudge
+from .metrics.perf import LatencyStats, Throughput
+from .metrics.rag import AnswerOverlap, ContextCoverage, ContextOverlap, LexicalGroundedness
+from .metrics.security import DefconGrade, KeywordDetector, ThreatCategory
+from .adapter import (
+    Adapter, ChatAdapter, FewShotAdapter, GenerationAdapter, InstructionAdapter, MCQAdapter,
+    RAGAdapter, TemplateAdapter,
+)
+from .router import route_adapter
+from .annotator import Annotator, RegexAnnotator, LLMAnnotator, ThinkingStripAnnotator
+from .report import RunResult
+from .runspec import RunConfig
+from .report_format import Report
+from .sample import Sample
+from .score import Score, Stat, pass_at_k, pass_hat_k
+from .scoring import ScoreGate, WeightedSum
+from .types import (
+    Capability,
+    DataType,
+    Direction,
+    ScoreKind,
+    Source,
+    TaskKind,
+)
+
+#: Public alias of :class:`RunResult` — the object returned by :func:`evaluate`.
+Result = RunResult
+
+__all__ = [
+    "evaluate",
+    "evaluate_many",
+    "run_lmeval",
+    "generate",
+    "compare",
+    "scorer",
+    "BenchmarkEvaluator",
+    "run_benchmark",
+    "map_model_spec",
+    "load_csv",
+    "load_hf",
+    "load_croissant",
+    "Experiment",
+    "ExperimentDB",
+    "Sample",
+    "Score",
+    "Stat",
+    "Result",
+    "RunConfig",
+    "Adapter",
+    "GenerationAdapter",
+    "MCQAdapter",
+    "ChatAdapter",
+    "FewShotAdapter",
+    "InstructionAdapter",
+    "RAGAdapter",
+    "TemplateAdapter",
+    "route_adapter",
+    "AutoModel",
+    "Scenario",
+    "ListScenario",
+    "CallableScenario",
+    "ADAPTERS",
+    "METRICS",
+    "EVALUATORS",
+    "ScoreGate",
+    "WeightedSum",
+    "Equals",
+    "configure_logging",
+    "BertScore",
+    "Bleu",
+    "ChrF",
+    "Contains",
+    "Perplexity",
+    "StartsWith",
+    "EndsWith",
+    "Evaluator",
+    "Regex",
+    "Levenshtein",
+    "WordCount",
+    "IsJson",
+    "RogueL",
+    "WordErrorRate",
+    "F1Score",
+    "RunDiff",
+    "DeltaGrade",
+    "RunComparison",
+    "MetricDelta",
+    "TaskDelta",
+    "grade_delta",
+    "LexicalGroundedness",
+    "ContextCoverage",
+    "ContextOverlap",
+    "AnswerOverlap",
+    "JudgeMetric",
+    "RubricItem",
+    "GEval",
+    "LLMJudge",
+    "EncoderJudge",
+    "FactualityEncoderJudge",
+    "SentimentEncoderJudge",
+    "Factuality",
+    "ClosedQA",
+    "Relevance",
+    "BiasJudge",
+    "KeywordDetector",
+    "DefconGrade",
+    "DiskCache",
+    "ThreatCategory",
+    "LatencyStats",
+    "Throughput",
+    "Annotator",
+    "RegexAnnotator",
+    "LLMAnnotator",
+    "ThinkingStripAnnotator",
+    "Report",
+    "TaskKind",
+    "ScoreKind",
+    "DataType",
+    "Direction",
+    "Capability",
+    "Source",
+    "SCENARIOS",
+    "AuditKitError",
+    "ExtraNotInstalled",
+    "CosineSimilarity",
+    "TokenOverlap",
+    "BM25Similarity",
+    "FactualConsistency",
+    "ToxicityScore",
+    "RepresentationSkew",
+    "HateSpeechScore",
+    "GuardJudge",
+    "WinRate",
+    "EloScore",
+    "PreferenceAccuracy",
+    "pass_at_k",
+    "pass_hat_k",
+    "CompareResult",
+    "compare_models",
+]

auditkit/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

auditkit/_bootstrap.py

@@ -0,0 +1,77 @@
+"""Shared paired-bootstrap significance test.
+
+Both the model-comparison path (`model_compare.py`) and the experiment path
+(`experiment.py`) need the same test — a two-sided paired bootstrap over
+per-sample (baseline, candidate) score pairs — so it lives here once instead of
+being reimplemented (and drifting) in each. The old copies also aligned the two
+runs' scores by list *position*, which silently mispaired samples whenever the
+runs' prediction order diverged (partial failures, retries, caching); callers
+here align **by sample_id** via :func:`score_pairs`.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Any, Optional
+
+from .report import RunResult
+
+
+def score_pairs(
+    baseline: RunResult, candidate: RunResult, metric: Optional[str] = None
+) -> list[tuple[float, float]]:
+    """Per-sample ``(baseline_value, candidate_value)`` pairs, aligned by ``sample_id``.
+
+    ``metric=None`` uses each prediction's primary ``.score`` (works for every
+    engine, including lm-eval benchmark runs whose predictions don't carry a
+    per-metric ``metadata["scores"]`` breakdown). A metric name reads that named
+    score out of ``metadata["scores"]`` (native runs).
+    """
+    def by_id(run: RunResult) -> dict[str, float]:
+        out: dict[str, float] = {}
+        for p in run.predictions:
+            if metric is None:
+                if p.score is not None:
+                    out[p.sample_id] = float(p.score)
+            else:
+                for s in p.metadata.get("scores", []):
+                    if s.get("name") == metric and s.get("value") is not None:
+                        out[p.sample_id] = float(s["value"])
+                        break
+        return out
+
+    b, c = by_id(baseline), by_id(candidate)
+    return [(b[i], c[i]) for i in sorted(b.keys() & c.keys())]
+
+
+def paired_bootstrap(
+    pairs: list[tuple[float, float]], *, n_resamples: int = 1000, seed: int = 42
+) -> dict[str, Any]:
+    """Two-sided paired bootstrap over ``(baseline, candidate)`` value pairs.
+
+    Returns ``{n, mean_baseline, mean_candidate, delta, p_value, significant}``
+    where ``delta = mean_candidate - mean_baseline`` (positive ⇒ candidate scored
+    higher). Fewer than 2 pairs can't be resampled meaningfully → ``{"error": ...}``.
+    """
+    n = len(pairs)
+    if n < 2:
+        return {"error": "need at least 2 aligned sample pairs", "n": n}
+    diffs = [c - b for b, c in pairs]
+    mean_baseline = sum(b for b, _ in pairs) / n
+    mean_candidate = sum(c for _, c in pairs) / n
+    observed = sum(diffs) / n
+    rng = random.Random(seed)
+    count = 0
+    for _ in range(n_resamples):
+        resample = [rng.choice(diffs) for _ in range(n)]
+        if abs(sum(resample) / n) >= abs(observed):
+            count += 1
+    p = (count + 1) / (n_resamples + 1)
+    return {
+        "n": n,
+        "mean_baseline": round(mean_baseline, 6),
+        "mean_candidate": round(mean_candidate, 6),
+        "delta": round(observed, 6),
+        "p_value": round(p, 6),
+        "significant": p < 0.05,
+    }

auditkit/_identity_guard.py

@@ -0,0 +1,99 @@
+"""Shared class-definition-time guard against forgotten ``identity()``
+overrides, used by :class:`Annotator`, :class:`Adapter`, and :class:`Metric`.
+
+The bug this catches: a custom subclass takes real constructor config but
+never overrides the base ``identity()`` (which only ever returns
+``{"name": self.name}``) -- two differently-configured instances then
+silently produce the same ``RunSpec.fingerprint()``, and ``Runner.run()``
+can return a stale cached ``RunResult`` scored under a different config,
+with no error anywhere. Warning at class-definition time (via
+``__init_subclass__``) catches this as soon as the offending class is
+*defined* -- before anyone even instantiates it, let alone runs anything.
+
+One real false-positive this file exists to avoid: several built-in metrics
+(``Contains``, ``StartsWith``, ``KeywordDetector``, ...) never override
+``identity()`` either, but they're not buggy -- they assign a
+parameter-derived string to ``self.name`` inside ``__init__``
+(e.g. ``self.name = f"contains({substring})"``), which already makes the
+base ``identity()`` (``{"name": self.name}``) parameter-aware. Blindly
+warning on "no identity() override" alone would fire on every one of those.
+Detecting this safely means checking whether ``__init__`` assigns
+``self.name`` *without* instantiating the class (constructing an arbitrary
+user class as a side effect of merely defining it is unacceptable -- some
+constructors have required args with no defaults, and even when they don't,
+running arbitrary user `__init__` code at import time is not something a
+library should ever do implicitly). AST inspection of the already-loaded
+source is the only way to check this without executing anything.
+"""
+
+from __future__ import annotations
+
+import ast
+import inspect
+import textwrap
+import warnings
+from typing import Any
+
+
+def _assigns_self_attr(init_func: Any, attr: str) -> bool:
+    """True iff *init_func*'s body contains ``self.<attr> = ...`` anywhere
+    (a plain assignment, or a property setter reached via ``self.<attr> =``
+    -- both look identical in the AST).
+
+    Best-effort: classes defined somewhere ``inspect.getsource`` can't reach
+    (a REPL, ``exec``, certain notebook cells) fail open (return True, i.e.
+    "assume it's fine") rather than risk a false-positive warning on code
+    this check has no reliable way to inspect.
+    """
+    try:
+        source = textwrap.dedent(inspect.getsource(init_func))
+        tree = ast.parse(source)
+    except (OSError, TypeError, SyntaxError):
+        return True
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Assign):
+            continue
+        for target in node.targets:
+            if (
+                isinstance(target, ast.Attribute)
+                and target.attr == attr
+                and isinstance(target.value, ast.Name)
+                and target.value.id == "self"
+            ):
+                return True
+    return False
+
+
+def warn_if_identity_incomplete(
+    cls: type, base_cls: type, main_method: str, name_attr: str = "name",
+) -> None:
+    """Warn if *cls* (a fresh subclass of *base_cls*) takes real constructor
+    config but neither overrides ``identity()`` nor derives ``self.<name_attr>``
+    from it -- called from ``base_cls.__init_subclass__``. *name_attr* is
+    ``"name"`` for :class:`Metric`/:class:`Annotator`, ``"method"`` for
+    :class:`Adapter` (whichever attribute that base's ``identity()`` keys off).
+    """
+    if cls.identity is not base_cls.identity:
+        return  # this class (or a parent) already overrides identity()
+    init = cls.__dict__.get("__init__")
+    if init is None:
+        return  # no new __init__ defined here -- nothing new to guard
+    params = [
+        p for pname, p in inspect.signature(init).parameters.items()
+        if pname != "self" and p.kind not in (p.VAR_POSITIONAL, p.VAR_KEYWORD)
+    ]
+    if not params:
+        return
+    if _assigns_self_attr(init, name_attr):
+        return  # self.<name_attr> is parameter-derived -- base identity() is already fine
+    warnings.warn(
+        f"{cls.__name__} defines __init__ parameters "
+        f"({', '.join(p.name for p in params)}) but does not override "
+        f"{base_cls.__name__}.identity() (and doesn't derive self.{name_attr} "
+        f"from them either). Differently-configured instances will silently "
+        f"produce the same RunSpec.fingerprint() and can reuse a stale "
+        f"cached RunResult scored under a different config. Add an "
+        f"identity() override that includes every constructor argument "
+        f"affecting {main_method}()'s output.",
+        stacklevel=3,
+    )