grudge-cli 0.1.0__py3-none-any.whl

grudge/shrinker/oracle.py

@@ -0,0 +1,230 @@
+"""Deterministic failure-reproduction oracles for cassette shrinking.
+
+The oracle is the seam between the pure shrink search and Phase 4 replay.  A
+candidate cassette is evaluated by running the live policy in the hermetic
+sandbox and asking a small, deterministic seed predicate whether the failure
+is still present.
+"""
+
+from __future__ import annotations
+
+import inspect
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, Protocol, runtime_checkable
+
+from grudge.errors import OracleError
+from grudge.replay.sandbox import run_agent
+from grudge.schema.cassette import Cassette
+from grudge.schema.predicate import (
+    CallShape,
+    Outcome,
+    PredicateSet,
+    TaintFlow,
+)
+from grudge.schema.trace import ToolResult, Trace
+
+
+@dataclass(frozen=True)
+class OracleResult:
+    """The result of one candidate evaluation."""
+
+    reproduced: bool
+    trace: Trace | None
+    reason: str
+
+
+@runtime_checkable
+class Oracle(Protocol):
+    """Protocol implemented by objects that can test a cassette candidate."""
+
+    calls: int
+    cache_hits: int
+
+    def reproduces(self, cassette: Cassette) -> OracleResult:
+        """Return whether ``cassette`` still reproduces the seed failure."""
+
+
+SeedPredicate = Callable[[Trace], bool] | CallShape | TaintFlow | Outcome | PredicateSet
+
+
+def _value_contains(value: Any, needle: Any) -> bool:
+    """Return whether ``needle`` occurs in a JSON-like value."""
+
+    if needle is None:
+        return False
+    if isinstance(value, Mapping):
+        return any(_value_contains(item, needle) for item in value.values())
+    if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray):
+        return any(_value_contains(item, needle) for item in value)
+    if isinstance(needle, str):
+        return needle in str(value)
+    return bool(value == needle)
+
+
+def _field(value: Any, name: str | None) -> Any:
+    """Read an optional field from a mapping, leaving scalar results intact."""
+
+    if name is None:
+        return value
+    if isinstance(value, Mapping):
+        return value.get(name)
+    return None
+
+
+def _schema_predicate(predicate: Any, trace: Trace) -> bool:
+    """Evaluate the Phase 1 predicate shapes used by the seed detectors."""
+
+    if isinstance(predicate, CallShape):
+        return any(
+            step.tool == predicate.tool
+            and all(
+                step.args.get(key) == value for key, value in predicate.args.items()
+            )
+            for step in trace.tool_calls()
+        )
+    if isinstance(predicate, TaintFlow):
+        calls = trace.tool_calls()
+        results = {
+            step.call_id: step for step in trace.steps if isinstance(step, ToolResult)
+        }
+        for source_index, source_call in enumerate(calls):
+            if source_call.tool != predicate.source:
+                continue
+            source_result = results.get(source_call.call_id)
+            if source_result is None:
+                continue
+            source_value = _field(source_result.content, predicate.source_field)
+            for sink_call in calls[source_index + 1 :]:
+                if sink_call.tool != predicate.sink:
+                    continue
+                sink_value = _field(sink_call.args, predicate.sink_field)
+                if _value_contains(sink_value, source_value):
+                    return True
+        return False
+    if isinstance(predicate, Outcome):
+        return trace.outcome == predicate.outcome
+    if isinstance(predicate, PredicateSet):
+        values = [_schema_predicate(item, trace) for item in predicate.predicates]
+        if predicate.kind == "all_of":
+            return all(values)
+        if predicate.kind == "any_of":
+            return any(values)
+        return not values[0]
+    raise TypeError(f"unsupported seed predicate: {type(predicate).__name__}")
+
+
+def _evaluate(predicate: Any, trace: Trace, cassette: Cassette) -> bool:
+    """Evaluate a callable, schema predicate, or small checker-like object."""
+
+    if isinstance(predicate, CallShape | TaintFlow | Outcome | PredicateSet):
+        return _schema_predicate(predicate, trace)
+    checker = getattr(predicate, "check", None)
+    if callable(checker):
+        result = checker(trace)
+        return bool(getattr(result, "violated", result))
+    evaluator = getattr(predicate, "evaluate", None)
+    if callable(evaluator):
+        result = evaluator(trace)
+        return bool(getattr(result, "violated", result))
+    if not callable(predicate):
+        raise TypeError(f"unsupported seed predicate: {type(predicate).__name__}")
+    try:
+        parameters = dict(inspect.signature(predicate).parameters)
+    except (TypeError, ValueError):
+        parameters = {}
+    positional = [
+        parameter
+        for parameter in parameters.values()
+        if parameter.kind
+        in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+    ]
+    if len(positional) >= 2:
+        return bool(predicate(trace, cassette))
+    return bool(predicate(trace))
+
+
+class PredicateOracle:
+    """Run an agent in the replay sandbox and memoize predicate outcomes."""
+
+    def __init__(
+        self,
+        agent_fn: Callable[..., Any],
+        seed_predicate: SeedPredicate | Any,
+        *,
+        llm: Any = None,
+        repeats: int = 1,
+        max_oracle_calls: int | None = 1000,
+        timeout: float = 2.0,
+        max_steps: int = 128,
+    ) -> None:
+        if repeats < 1:
+            raise ValueError("repeats must be at least 1")
+        if max_oracle_calls is not None and max_oracle_calls < 1:
+            raise ValueError("max_oracle_calls must be at least 1")
+        if timeout <= 0 or max_steps < 0:
+            raise ValueError("timeout must be positive and max_steps non-negative")
+        self.agent_fn = agent_fn
+        self.seed_predicate = seed_predicate
+        self.llm = llm
+        self.repeats = repeats
+        self.max_oracle_calls = max_oracle_calls
+        self.timeout = timeout
+        self.max_steps = max_steps
+        self.calls = 0
+        self.cache_hits = 0
+        self._cache: dict[str, OracleResult] = {}
+
+    def reproduces(self, cassette: Cassette) -> OracleResult:
+        """Evaluate ``cassette`` once, or return its memoized result."""
+
+        cache_key = cassette.content_hash
+        cached = self._cache.get(cache_key)
+        if cached is not None:
+            self.cache_hits += 1
+            return cached
+        if self.max_oracle_calls is not None and self.calls >= self.max_oracle_calls:
+            raise OracleError(
+                "oracle call budget exhausted: "
+                f"{self.calls} calls used, max_oracle_calls="
+                f"{self.max_oracle_calls}"
+            )
+
+        self.calls += 1
+        traces: list[Trace] = []
+        failures: list[str] = []
+        reproduced_runs = 0
+        for _ in range(self.repeats):
+            trace = run_agent(
+                self.agent_fn,
+                cassette,
+                self.llm,
+                timeout=self.timeout,
+                max_steps=self.max_steps,
+            )
+            traces.append(trace)
+            try:
+                reproduced = _evaluate(self.seed_predicate, trace, cassette)
+            except Exception as exc:
+                reproduced = False
+                failures.append(f"predicate error: {type(exc).__name__}: {exc}")
+            if reproduced:
+                reproduced_runs += 1
+            if not reproduced:
+                continue
+
+        if failures:
+            reason = failures[0]
+        else:
+            run_summary = f"{reproduced_runs}/{self.repeats}"
+            reason = f"seed predicate reproduced in {run_summary} runs"
+        result = OracleResult(
+            reproduced_runs == self.repeats,
+            traces[0] if traces else None,
+            reason,
+        )
+        self._cache[cache_key] = result
+        return result
+
+
+__all__ = ["Oracle", "OracleResult", "PredicateOracle", "SeedPredicate"]

grudge/shrinker/reducers.py

@@ -0,0 +1,331 @@
+"""Pure content reducers used after cassette-entry delta debugging.
+
+Each reducer accepts a JSON-compatible value (including a validated Pydantic
+model), makes copies before trying candidates, and keeps a candidate only when
+the supplied reproduction test still passes.  That makes the reducers safe to
+compose and keeps cassette validation at the model boundary.
+"""
+
+from __future__ import annotations
+
+import copy
+import re
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from typing import Any, Protocol
+
+from pydantic import BaseModel
+
+from grudge.shrinker.ddmin import minimize
+
+PathPart = str | int
+Path = tuple[PathPart, ...]
+ContentTest = Callable[[Any], bool]
+
+
+class ContentReducer(Protocol):
+    """The small common interface implemented by all content reducers."""
+
+    def reduce(self, value: Any, test: ContentTest) -> Any:
+        """Return the smallest accepted value this reducer can find."""
+
+
+def _passes(test: ContentTest, value: Any) -> bool:
+    """Treat structural lookup failures as a rejected candidate."""
+
+    try:
+        return bool(test(value))
+    except (AttributeError, IndexError, KeyError, TypeError):
+        return False
+
+
+def _children(value: Any) -> Iterable[tuple[PathPart, Any]]:
+    if isinstance(value, BaseModel):
+        for name in type(value).model_fields:
+            yield name, getattr(value, name)
+    elif isinstance(value, Mapping):
+        for key, child in value.items():
+            if isinstance(key, str | int):
+                yield key, child
+    elif isinstance(value, list):
+        yield from enumerate(value)
+
+
+def _paths(value: Any, prefix: Path = ()) -> Iterable[Path]:
+    for part, child in _children(value):
+        path = (*prefix, part)
+        yield path
+        yield from _paths(child, path)
+
+
+def _get_path(value: Any, path: Path) -> Any:
+    current = value
+    for part in path:
+        if isinstance(current, BaseModel):
+            current = getattr(current, str(part))
+        else:
+            current = current[part]
+    return current
+
+
+def _set_path(value: Any, path: Path, replacement: Any) -> None:
+    if not path:
+        raise ValueError("a reducer path must not be empty")
+    parent = _get_path(value, path[:-1])
+    part = path[-1]
+    if isinstance(parent, BaseModel):
+        setattr(parent, str(part), replacement)
+    else:
+        parent[part] = replacement
+
+
+def _candidate(value: Any, path: Path, replacement: Any) -> Any | None:
+    try:
+        result = copy.deepcopy(value)
+        _set_path(result, path, replacement)
+    except (AttributeError, KeyError, IndexError, TypeError, ValueError):
+        # Invalid assignments are expected for constrained Cassette fields
+        # (for example, replacing a required tool name with "").
+        return None
+    return result
+
+
+def _sentences(text: str) -> list[str]:
+    """Split prose without dropping its terminal punctuation."""
+
+    parts = re.split(r"(?<=[.!?])\s+", text)
+    return [part for part in parts if part]
+
+
+def _join_chunks(chunks: Sequence[str], *, lines: bool) -> str:
+    return ("" if lines else " ").join(chunks)
+
+
+def _tests_text(value: Any, path: Path, text: str, test: ContentTest) -> bool:
+    candidate = _candidate(value, path, text)
+    return candidate is not None and _passes(test, candidate)
+
+
+class TrimDocument:
+    """Remove irrelevant lines and sentences from text fields via ddmin."""
+
+    def reduce(self, value: Any, test: ContentTest) -> Any:
+        """Try every text field in stable traversal order."""
+
+        result = copy.deepcopy(value)
+        for path in list(_paths(result)):
+            try:
+                text = _get_path(result, path)
+            except (AttributeError, KeyError, IndexError, TypeError):
+                continue
+            if not isinstance(text, str) or len(text) < 2:
+                continue
+
+            lines = text.splitlines(keepends=True)
+            chunks = lines if len(lines) > 1 else _sentences(text)
+            if len(chunks) > 1:
+
+                def accepts_lines(
+                    candidate: list[str],
+                    base: Any = result,
+                    field_path: Path = path,
+                    line_mode: bool = len(lines) > 1,
+                ) -> bool:
+                    return _tests_text(
+                        base,
+                        field_path,
+                        _join_chunks(candidate, lines=line_mode),
+                        test,
+                    )
+
+                reduced_chunks, _ = minimize(
+                    chunks,
+                    accepts_lines,
+                )
+                replacement = _join_chunks(reduced_chunks, lines=len(lines) > 1)
+                candidate = _candidate(result, path, replacement)
+                if candidate is not None and _passes(test, candidate):
+                    result = candidate
+
+            # Once line noise is removed, sentence-level reduction can still
+            # remove irrelevant prose within the surviving line(s).
+            try:
+                current_text = _get_path(result, path)
+            except (AttributeError, KeyError, IndexError, TypeError):
+                continue
+            if not isinstance(current_text, str):
+                continue
+            sentences = _sentences(current_text)
+            if len(sentences) <= 1:
+                continue
+
+            def accepts_sentences(
+                candidate: list[str],
+                base: Any = result,
+                field_path: Path = path,
+            ) -> bool:
+                return _tests_text(
+                    base,
+                    field_path,
+                    _join_chunks(candidate, lines=False),
+                    test,
+                )
+
+            reduced_sentences, _ = minimize(
+                sentences,
+                accepts_sentences,
+            )
+            replacement = _join_chunks(reduced_sentences, lines=False)
+            candidate = _candidate(result, path, replacement)
+            if candidate is not None and _passes(test, candidate):
+                result = candidate
+        return result
+
+    apply = reduce
+
+    def __call__(self, value: Any, test: ContentTest) -> Any:
+        return self.reduce(value, test)
+
+
+def _neutral(value: Any) -> Any:
+    if isinstance(value, str):
+        return ""
+    if isinstance(value, bool):
+        return False
+    if isinstance(value, int | float):
+        return 0
+    if isinstance(value, list):
+        return []
+    if isinstance(value, dict):
+        return {}
+    return None
+
+
+class BlankField:
+    """Blank fields one at a time whenever reproduction remains sound."""
+
+    def __init__(self, fields: Sequence[str] | None = None) -> None:
+        self.fields = frozenset(fields) if fields is not None else None
+
+    def reduce(self, value: Any, test: ContentTest) -> Any:
+        result = copy.deepcopy(value)
+        for path in list(_paths(result)):
+            if self.fields is not None and str(path[-1]) not in self.fields:
+                continue
+            try:
+                old = _get_path(result, path)
+            except (AttributeError, KeyError, IndexError, TypeError):
+                continue
+            replacement = _neutral(old)
+            if replacement == old:
+                continue
+            candidate = _candidate(result, path, replacement)
+            if candidate is not None and _passes(test, candidate):
+                result = candidate
+        return result
+
+    apply = reduce
+
+    def __call__(self, value: Any, test: ContentTest) -> Any:
+        return self.reduce(value, test)
+
+
+_ID_TOKEN = re.compile(
+    r"\b(?P<prefix>email|user|account|order|ticket|message|document|file)"
+    r"[-_][A-Za-z0-9]+\b",
+    re.IGNORECASE,
+)
+
+
+def _id_tokens(value: Any) -> list[str]:
+    tokens: list[str] = []
+    for path in _paths(value):
+        try:
+            child = _get_path(value, path)
+        except (AttributeError, KeyError, IndexError, TypeError):
+            continue
+        if not isinstance(child, str):
+            continue
+        for match in _ID_TOKEN.finditer(child):
+            token = match.group(0)
+            if token not in tokens:
+                tokens.append(token)
+    return tokens
+
+
+def _replace_ids(value: Any, replacements: Mapping[str, str]) -> Any | None:
+    result = copy.deepcopy(value)
+    for path in list(_paths(result)):
+        try:
+            child = _get_path(result, path)
+        except (AttributeError, KeyError, IndexError, TypeError):
+            continue
+        if not isinstance(child, str):
+            continue
+        replacement = child
+        for old, new in replacements.items():
+            replacement = re.sub(
+                rf"(?<![A-Za-z0-9_-]){re.escape(old)}(?![A-Za-z0-9_-])",
+                new,
+                replacement,
+            )
+        if replacement != child:
+            try:
+                _set_path(result, path, replacement)
+            except (AttributeError, KeyError, IndexError, TypeError, ValueError):
+                return None
+    return result
+
+
+class NormalizeIds:
+    """Canonicalize stable entity IDs only when the reproduction test passes."""
+
+    def reduce(self, value: Any, test: ContentTest) -> Any:
+        tokens = _id_tokens(value)
+        if not tokens:
+            return copy.deepcopy(value)
+
+        counters: dict[str, int] = {}
+        replacements: dict[str, str] = {}
+        for token in tokens:
+            prefix = token.split("-", 1)[0].split("_", 1)[0].lower()
+            counters[prefix] = counters.get(prefix, 0) + 1
+            replacements[token] = f"{prefix}_{counters[prefix]}"
+
+        all_ids = _replace_ids(value, replacements)
+        if all_ids is not None and _passes(test, all_ids):
+            return all_ids
+
+        result = copy.deepcopy(value)
+        for token, replacement in replacements.items():
+            candidate = _replace_ids(result, {token: replacement})
+            if candidate is not None and _passes(test, candidate):
+                result = candidate
+        return result
+
+    apply = reduce
+
+    def __call__(self, value: Any, test: ContentTest) -> Any:
+        return self.reduce(value, test)
+
+
+def reduce_content(
+    value: Any,
+    test: ContentTest,
+    *,
+    reducers: Sequence[ContentReducer] | None = None,
+) -> Any:
+    """Apply the content-reducer pipeline in deterministic order."""
+
+    current = value
+    selected = reducers or (TrimDocument(), BlankField(), NormalizeIds())
+    for reducer in selected:
+        current = reducer.reduce(current, test)
+    return current
+
+
+__all__ = [
+    "BlankField",
+    "NormalizeIds",
+    "TrimDocument",
+    "reduce_content",
+]

grudge_cli-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Grudge contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.