clastogen 0.2.0__py3-none-any.whl

clastogen/__init__.py

@@ -0,0 +1,35 @@
+"""Clastogen: Mutation testing & statistical assertion framework for LLMs and Agents."""
+
+from importlib.metadata import version
+
+from clastogen.core.injection import override_prompt
+from clastogen.core.mutator import PromptMutator
+from clastogen.core.sprt import SPRT
+from clastogen.models import Mutant, PassRateResult, RegressionResult, SPRTConfig, SPRTResult
+from clastogen.stats import (
+    assert_no_regression,
+    assert_pass_rate,
+    compute_wilson_interval,
+    evaluate_pass_rate,
+    evaluate_regression,
+)
+from clastogen.types import Decision
+
+__all__ = [
+    "SPRT",
+    "Decision",
+    "Mutant",
+    "PassRateResult",
+    "PromptMutator",
+    "RegressionResult",
+    "SPRTConfig",
+    "SPRTResult",
+    "assert_no_regression",
+    "assert_pass_rate",
+    "compute_wilson_interval",
+    "evaluate_pass_rate",
+    "evaluate_regression",
+    "override_prompt",
+]
+
+__version__ = version("clastogen")

clastogen/core/injection.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import importlib
+import logging
+import sys
+import types
+from collections.abc import Generator
+from contextlib import ExitStack, contextmanager
+from functools import reduce
+from unittest.mock import patch
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_target(target_spec: str) -> tuple[object, str]:
+    """Resolves 'module:attr' or 'module:Class.attr' (dots after the colon walk nested attributes) into (owner, attr)."""
+    module_name, sep, attr_path = target_spec.partition(":")
+    if not sep:
+        raise ValueError(f"Target '{target_spec}' must be 'module:attr' or 'module:Class.attr'")
+    *owner_path, attr = attr_path.split(".")
+    owner = reduce(getattr, owner_path, importlib.import_module(module_name))
+    if not hasattr(owner, attr):
+        raise AttributeError(f"Object {owner!r} has no attribute '{attr}'")
+    return owner, attr
+
+
+def _aliasing_modules(owner: object, attr: str, original: object) -> list[types.ModuleType]:
+    """Modules other than owner that bound the same string under the same name, e.g. via `from agent import PROMPT`."""
+    if not isinstance(original, str):
+        return []
+    return [
+        mod
+        for mod in list(sys.modules.values())
+        if isinstance(mod, types.ModuleType) and mod is not owner and vars(mod).get(attr) is original
+    ]
+
+
+@contextmanager
+def override_prompt(target_spec: str, mutated_text: str) -> Generator[str, None, None]:
+    """Temporarily replaces the target attribute, and every same-name module alias of it, with mutated_text.
+
+    unittest.mock.patch.object restores descriptors such as @classmethod and removes shadow attributes
+    on classes that only inherited the target.
+    """
+    if not isinstance(mutated_text, str):
+        raise TypeError(f"mutated_text must be a string, got {type(mutated_text).__name__}")
+
+    owner, attr = resolve_target(target_spec)
+    original = getattr(owner, attr)
+    if original is None:
+        raise TypeError(f"Target '{target_spec}' resolved to None, expected a prompt string.")
+
+    with ExitStack() as stack:
+        stack.enter_context(patch.object(owner, attr, mutated_text))
+        aliases = _aliasing_modules(owner, attr, original)
+        for mod in aliases:
+            stack.enter_context(patch.object(mod, attr, mutated_text))
+        logger.debug("Injected prompt into target '%s' (%d module alias(es))", target_spec, len(aliases))
+        yield mutated_text

clastogen/core/mutator.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from decimal import Decimal
+from itertools import zip_longest
+
+from clastogen.models import Mutant
+
+
+class PromptMutator:
+    """Extracts load-bearing imperative constraints and generates targeted prompt mutants."""
+
+    RULE_PATTERN = re.compile(
+        r"\b(?:never|do not|don't|must not|mustn't|should not|shouldn't|cannot|can't|always|must|shall not|may not|avoid|only|requires?|required)\b",
+        re.IGNORECASE,
+    )
+
+    SENTENCE_SPLIT_PATTERN = re.compile(r"\n+|(?<!\be\.g)(?<!\bi\.e)(?<!\betc)[.?!]\s+")
+
+    NEGATION_REPLACEMENTS: tuple[tuple[re.Pattern[str], str], ...] = (
+        (re.compile(r"\bdo not\b", re.IGNORECASE), "do"),
+        (re.compile(r"\bdon't\b", re.IGNORECASE), "do"),
+        (re.compile(r"\bmust not\b", re.IGNORECASE), "must"),
+        (re.compile(r"\bmustn't\b", re.IGNORECASE), "must"),
+        (re.compile(r"\bnever\b", re.IGNORECASE), "always"),
+        (re.compile(r"\bcannot\b", re.IGNORECASE), "can"),
+        (re.compile(r"\bcan't\b", re.IGNORECASE), "can"),
+        (re.compile(r"\bshould not\b", re.IGNORECASE), "should"),
+        (re.compile(r"\bshouldn't\b", re.IGNORECASE), "should"),
+        (re.compile(r"\bshall not\b", re.IGNORECASE), "shall"),
+        (re.compile(r"\bmay not\b", re.IGNORECASE), "may"),
+        (re.compile(r"\balways\b", re.IGNORECASE), "never"),
+        (re.compile(r"\bmust\b", re.IGNORECASE), "must not"),
+        (re.compile(r"\bavoid\b", re.IGNORECASE), "prefer"),
+    )
+
+    NUMBER_PATTERN = re.compile(r"\b\d+(?:,\d{3})*(?:\.\d+)?\b")
+
+    def __init__(self, target_symbol: str = "SYSTEM_PROMPT") -> None:
+        self.target_symbol = target_symbol
+
+    def extract_candidate_rules(self, prompt: str) -> list[str]:
+        """Extracts candidate load-bearing sentences using linear string splitting and keyword search."""
+        if not isinstance(prompt, str):
+            raise TypeError(f"prompt must be a string, got {type(prompt).__name__}")
+        return [
+            cleaned
+            for line in self.SENTENCE_SPLIT_PATTERN.split(prompt)
+            if (cleaned := re.sub(r"^[-*]\s+", "", line.strip()))
+            and len(cleaned) > 10
+            and self.RULE_PATTERN.search(cleaned)
+        ]
+
+    def _delete_mutant(self, prompt: str, rule: str) -> Mutant | None:
+        """Removes rule with its bullet and trailing punctuation; None when the prompt is unchanged."""
+        clean_pat = re.compile(rf"[ \t]*(?:[-*]\s+)?{re.escape(rule)}[.?!]?([ \t]*)(\n?)")
+
+        def join(m: re.Match[str]) -> str:
+            if m.start() == 0 or prompt[m.start() - 1] == "\n":
+                return ""
+            return m[2] or ("" if m.end() == len(prompt) else " ")
+
+        mutated = clean_pat.sub(join, prompt).strip()
+        mutated = re.sub(r"\n\s*\.\s*\n", "\n", mutated)
+        mutated = re.sub(r"\n{3,}", "\n\n", mutated)
+        if mutated == prompt:
+            return None
+        return Mutant.create(
+            target_symbol=self.target_symbol,
+            operator_name="delete_constraint",
+            original_snippet=rule,
+            mutated_snippet="[DELETED]",
+            mutated_prompt=mutated,
+            description=f"Deleted load-bearing constraint: '{rule[:50]}...'",
+        )
+
+    def _invert_mutant(self, prompt: str, rule: str) -> Mutant | None:
+        """Flips the first matching negation or obligation keyword in rule; None when none applies."""
+        for pattern, replacement in self.NEGATION_REPLACEMENTS:
+            if pattern.search(rule):
+                inverted = pattern.sub(replacement.upper(), rule, count=1)
+                mutated = prompt.replace(rule, inverted)
+                if mutated != prompt:
+                    return Mutant.create(
+                        target_symbol=self.target_symbol,
+                        operator_name="invert_negation",
+                        original_snippet=rule,
+                        mutated_snippet=inverted,
+                        mutated_prompt=mutated,
+                        description=f"Inverted constraint: '{rule[:40]}' -> '{inverted[:40]}'",
+                    )
+        return None
+
+    def _threshold_mutant(self, prompt: str, rule: str) -> Mutant | None:
+        """Multiplies the first number in rule by 10 (a $50 limit becomes $500); None when rule has no number."""
+        match = self.NUMBER_PATTERN.search(rule)
+        if match is None:
+            return None
+        raw = match.group()
+        scaled = Decimal(raw.replace(",", "")) * 10
+        number = f"{scaled:,}" if "," in raw else str(scaled)
+        changed = rule[: match.start()] + number + rule[match.end() :]
+        mutated = prompt.replace(rule, changed)
+        if mutated == prompt:
+            return None
+        return Mutant.create(
+            target_symbol=self.target_symbol,
+            operator_name="change_threshold",
+            original_snippet=rule,
+            mutated_snippet=changed,
+            mutated_prompt=mutated,
+            description=f"Changed threshold: '{raw}' -> '{number}' in '{rule[:40]}'",
+        )
+
+    def generate_mutants(self, prompt: str, max_mutants: int) -> Sequence[Mutant]:
+        """Generates a bounded sequence of mutants (constraint deletions, negation inversions, threshold changes)."""
+        if not isinstance(prompt, str):
+            raise TypeError(f"prompt must be a string, got {type(prompt).__name__}")
+        if max_mutants < 1:
+            raise ValueError(f"max_mutants must be >= 1, got {max_mutants}")
+
+        candidates = list(dict.fromkeys(self.extract_candidate_rules(prompt)))
+        if not candidates:
+            return []
+
+        needed_rules = max(1, (max_mutants + 1) // 2)
+        if len(candidates) > needed_rules:
+            step = len(candidates) / needed_rules
+            selected_rules = [candidates[int(i * step)] for i in range(needed_rules)]
+        else:
+            selected_rules = candidates
+
+        operators = (self._delete_mutant, self._invert_mutant, self._threshold_mutant)
+        per_rule = [
+            [m for op in operators[i % 3 :] + operators[: i % 3] if (m := op(prompt, rule))]
+            for i, rule in enumerate(selected_rules)
+        ]
+        mutants = [m for group in zip_longest(*per_rule) for m in group if m]
+        return mutants[:max_mutants]

clastogen/core/sprt.py

@@ -0,0 +1,63 @@
+import inspect
+import logging
+import math
+from collections.abc import Callable, Sequence
+
+from clastogen.models import SPRTConfig, SPRTResult
+from clastogen.types import Decision
+
+logger = logging.getLogger(__name__)
+
+# Unmutated runs used to estimate p0; the Laplace estimate with n=10 keeps false KILLs on unchanged mutants low.
+BASELINE_RUNS = 10
+MIN_BASELINE_RATE = 0.80
+
+
+def check_outcome(raw: object) -> bool:
+    """Returns raw if it is a bool; rejects awaitables and other types with TypeError."""
+    if inspect.isawaitable(raw):
+        raise TypeError("evaluator returned an awaitable/coroutine; expected synchronous callable returning bool")
+    if not isinstance(raw, bool):
+        raise TypeError(f"evaluator must return bool, got {type(raw).__name__}")
+    return raw
+
+
+def estimate_p0(outcomes: Sequence[bool]) -> float:
+    """Estimates the baseline pass probability with Laplace's rule of succession, (s + 1) / (n + 2)."""
+    return (sum(outcomes) + 1) / (len(outcomes) + 2)
+
+
+def config_from_baseline(outcomes: Sequence[bool], *, delta: float, max_steps: int) -> SPRTConfig | None:
+    """Builds the SPRT config from baseline outcomes, or None when the raw pass rate is below MIN_BASELINE_RATE."""
+    if sum(outcomes) / len(outcomes) < MIN_BASELINE_RATE:
+        return None
+    return SPRTConfig.from_absolute_drop(p0=estimate_p0(outcomes), delta=delta, max_steps=max_steps)
+
+
+class SPRT:
+    """Wald's Sequential Probability Ratio Test engine for Bernoulli trials with early stopping."""
+
+    def __init__(self, config: SPRTConfig) -> None:
+        self.config = config
+        self.threshold_a = math.log((1.0 - config.beta) / config.alpha)
+        self.threshold_b = math.log(config.beta / (1.0 - config.alpha))
+        self.llr_pass = math.log(config.p1 / config.p0)
+        self.llr_fail = math.log((1.0 - config.p1) / (1.0 - config.p0))
+
+    def run_evaluator(self, evaluator: Callable[[], bool]) -> SPRTResult:
+        """Executes evaluator dynamically until a decision is reached or max_steps is exhausted."""
+        cumulative_llr = 0.0
+        decision = Decision.INCONCLUSIVE
+        step = 0
+        while step < self.config.max_steps:
+            step += 1
+            cumulative_llr += self.llr_pass if check_outcome(evaluator()) else self.llr_fail
+            if cumulative_llr >= self.threshold_a:
+                decision = Decision.KILLED
+                break
+            if cumulative_llr <= self.threshold_b:
+                decision = Decision.SURVIVED
+                break
+
+        logger.debug("SPRT %s after %d step(s) (LLR=%.3f)", decision.value, step, cumulative_llr)
+        return SPRTResult(decision=decision, sample_count=step, cumulative_llr=cumulative_llr)

clastogen/exceptions.py

@@ -0,0 +1,9 @@
+"""Raised inside a mutation trial and translated by the plugin into SKIPPED or ERROR statuses."""
+
+
+class TrialSkipped(Exception):
+    """The test called pytest.skip during a trial."""
+
+
+class TrialError(Exception):
+    """A trial could not produce a pass/fail outcome; the message is the recorded reason."""

clastogen/models.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+
+from clastogen.types import Decision, MutantStatus
+
+
+@dataclass(frozen=True)
+class Mutant:
+    """Stable, content-addressed mutant specification."""
+
+    id: str
+    target_symbol: str
+    operator_name: str
+    original_snippet: str
+    mutated_snippet: str
+    mutated_prompt: str
+    description: str
+
+    @classmethod
+    def create(
+        cls,
+        target_symbol: str,
+        operator_name: str,
+        original_snippet: str,
+        mutated_snippet: str,
+        mutated_prompt: str,
+        description: str,
+    ) -> Mutant:
+        """Constructs a mutant with a stable 12-character SHA-256 hash ID."""
+        content_key = f"{operator_name}:{target_symbol}:{original_snippet}->{mutated_snippet}"
+        stable_id = hashlib.sha256(content_key.encode("utf-8")).hexdigest()[:12]
+        return cls(
+            id=stable_id,
+            target_symbol=target_symbol,
+            operator_name=operator_name,
+            original_snippet=original_snippet,
+            mutated_snippet=mutated_snippet,
+            mutated_prompt=mutated_prompt,
+            description=description,
+        )
+
+
+@dataclass(frozen=True)
+class MutantExecution:
+    """Record of a single mutant evaluation by a test; sample_count and llr are None unless the SPRT ran."""
+
+    test_id: str
+    target: str
+    mutant_id: str
+    description: str
+    status: MutantStatus
+    sample_count: int | None = None
+    llr: float | None = None
+    error: str | None = None
+
+
+@dataclass(frozen=True)
+class BaselineRecord:
+    """Outcome of the unmutated baseline runs for a marked test; p0 is the Laplace estimate fed to the SPRT; error is set when a trial could not run cleanly."""
+
+    test_id: str
+    target: str
+    successes: int
+    runs: int
+    p0: float
+    stable: bool
+    error: str | None = None
+
+
+@dataclass(frozen=True)
+class MutationSummary:
+    """Order-independent aggregate of all mutant evaluations in a session."""
+
+    results: tuple[MutantExecution, ...]
+    counts: dict[MutantStatus, int]
+    score: float | None
+
+    @property
+    def total(self) -> int:
+        return len(self.results)
+
+
+@dataclass
+class ClastogenPluginState:
+    """Type-safe state stored in pytest config.stash."""
+
+    results: list[MutantExecution] = field(default_factory=list)
+    baselines: list[BaselineRecord] = field(default_factory=list)
+    summary: MutationSummary | None = None
+    killed_mutants: set[str] = field(default_factory=set)
+    suppressed_mutants: set[str] = field(default_factory=set)
+    marked_tests_count: int = 0
+
+
+@dataclass(frozen=True)
+class ClastogenParams:
+    """Arguments of @pytest.mark.clastogen; p0=None measures the baseline instead of assuming it."""
+
+    target: str
+    max_mutants: int = 5
+    delta: float = 0.30
+    p0: float | None = None
+    max_steps: int = 20
+
+    def __post_init__(self) -> None:
+        for name in ("max_mutants", "max_steps"):
+            value = getattr(self, name)
+            if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+                raise ValueError(f"{name} must be an integer >= 1, got {value!r}")
+        for name in ("delta", "p0"):
+            value = getattr(self, name)
+            if name == "p0" and value is None:
+                continue
+            if isinstance(value, bool) or not isinstance(value, int | float) or not 0.0 < value < 1.0:
+                raise ValueError(f"{name} must be a number in (0, 1), got {value!r}")
+
+
+@dataclass(frozen=True)
+class SPRTConfig:
+    """Configuration for Wald's Sequential Probability Ratio Test."""
+
+    alpha: float = 0.05
+    beta: float = 0.10
+    p0: float = 0.90
+    p1: float = 0.60
+    max_steps: int = 20
+
+    def __post_init__(self) -> None:
+        if not (0.0 < self.alpha < 1.0):
+            raise ValueError(f"alpha must be in (0, 1), got {self.alpha}")
+        if not (0.0 < self.beta < 1.0):
+            raise ValueError(f"beta must be in (0, 1), got {self.beta}")
+        if not (0.0 < self.p1 < self.p0 < 1.0):
+            raise ValueError(f"Requirement 0 < p1 < p0 < 1 violated: p0={self.p0}, p1={self.p1}")
+        if isinstance(self.max_steps, bool) or not isinstance(self.max_steps, int) or self.max_steps < 1:
+            raise ValueError(f"max_steps must be an integer >= 1, got {self.max_steps}")
+
+    @classmethod
+    def from_absolute_drop(
+        cls,
+        p0: float,
+        delta: float,
+        alpha: float = 0.05,
+        beta: float = 0.10,
+        max_steps: int = 20,
+    ) -> SPRTConfig:
+        """Derives p1 by subtracting delta from p0, clamping p0 to [0.02, 0.99] to prevent division by zero."""
+        clamped_p0 = min(max(p0, 0.02), 0.99)
+        p1 = max(clamped_p0 - delta, 0.01)
+        if p1 >= clamped_p0:
+            raise ValueError(f"Drop delta={delta} leaves p1={p1} >= p0={clamped_p0}")
+        return cls(alpha=alpha, beta=beta, p0=clamped_p0, p1=p1, max_steps=max_steps)
+
+
+@dataclass(frozen=True)
+class SPRTResult:
+    decision: Decision
+    sample_count: int
+    cumulative_llr: float
+
+
+@dataclass(frozen=True, slots=True)
+class PassRateResult:
+    """Outcome of a statistical pass rate assertion."""
+
+    passed: bool
+    observed_rate: float
+    sample_count: int
+    success_count: int
+    ci_lower: float
+    ci_upper: float
+    confidence: float
+    decided: bool
+    description: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class RegressionResult:
+    """Outcome of a one-sided test that a candidate evaluator regressed relative to a baseline."""
+
+    regressed: bool
+    p_value: float
+    test_name: str
+    baseline_rate: float
+    candidate_rate: float
+    sample_count: int
+    description: str | None = None