agentverity 0.2.0__py3-none-any.whl

agentverity/__init__.py

@@ -0,0 +1,53 @@
+"""agentverity — measure-first testing for non-deterministic LLM agents.
+
+Before trusting any test suite, agentverity tells you whether your agent's
+verdict is stable enough to test against, and whether a passing relation is
+trivially satisfied by an indifferent agent. Two headline diagnostics:
+
+  1. **Verdict-stochasticity meter** — does the agent's decision flip across
+     identical reruns? If not, a frozen-output diff dominates and metamorphic
+     relations add little.
+  2. **Constant-gate-blindness detector** — does the agent return a near-constant
+     verdict across a diverse input set? If so, every relation passes
+     trivially and the suite is lying to you.
+
+Metamorphic relations are the vehicle; the diagnostics are the product.
+
+Quickstart::
+
+    from agentverity import run, from_callable
+    from agentverity.relations import builtin_relations
+
+    agent = from_callable(my_agent_fn)
+    result = run(agent, inputs=["hello", "world"], relations=builtin_relations())
+    print(result.summary())
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from agentverity.adapters import from_callable
+from agentverity.blindness import BlindnessResult, detect
+from agentverity.meter import MeterResult, measure
+from agentverity.observation import Observation
+from agentverity.relations import Relation, builtin_relations
+from agentverity.runner import RelationResult, RunConfig, RunResult, run
+
+__all__ = [
+    "BlindnessResult",
+    "MeterResult",
+    "Observation",
+    "Relation",
+    "RelationResult",
+    "RunConfig",
+    "RunResult",
+    "builtin_relations",
+    "detect",
+    "from_callable",
+    "measure",
+    "run",
+]
+
+try:
+    __version__ = version("agentverity")
+except PackageNotFoundError:
+    __version__ = "0+unknown"

agentverity/adapters/__init__.py

@@ -0,0 +1,14 @@
+"""Adapters turn a real agent into ``run(input) -> Observation``.
+
+Only ``callable`` is imported here (zero deps). Library adapters (Strands,
+LangGraph) are imported lazily from their own modules so the core installs
+without those libraries present.
+
+Available adapters:
+    - :func:`from_callable` — wrap any ``fn(input) -> str | dict | Observation``.
+    - :func:`from_strands` — wrap a Strands ``Agent`` (requires ``strands-agents``).
+"""
+
+from agentverity.adapters.callable_adapter import from_callable
+
+__all__ = ["from_callable"]

agentverity/adapters/callable_adapter.py

@@ -0,0 +1,43 @@
+"""Adapter for any bare callable agent: fn(input) -> str | dict | Observation.
+
+The most general adapter, and the one used for non-library agents and tests.
+Returns an AgentFn: (input:str) -> Observation.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from agentverity.observation import Observation
+
+
+def from_callable(fn: Callable[[str], Any], *,
+                  verdict_key: str | None = None,
+                  tools_key: str | None = None):
+    """Wrap `fn` so it yields an Observation.
+
+    fn may return:
+      - a str            -> becomes Observation.text
+      - an Observation   -> passed through
+      - a dict           -> read 'text'/'verdict'/'tools', or use verdict_key/
+                            tools_key to name the fields to pull out.
+    """
+    def run(x: str) -> Observation:
+        out = fn(x)
+        if isinstance(out, Observation):
+            return out
+        if isinstance(out, str):
+            return Observation(text=out)
+        if isinstance(out, dict):
+            text = str(out.get("text", ""))
+            verdict = out.get(verdict_key) if verdict_key else out.get("verdict")
+            raw_tools = out.get(tools_key) if tools_key else out.get("tools")
+            tools = tuple(raw_tools) if isinstance(raw_tools, (list, tuple)) else ()
+            return Observation(text=text,
+                               verdict=(str(verdict) if verdict is not None else None),
+                               tools=tools, raw=out)
+        # anything else: stringify into text, keep raw
+        return Observation(text=str(out), raw=out)
+
+    return run

agentverity/adapters/strands.py

@@ -0,0 +1,100 @@
+"""Strands adapter — wraps a Strands ``Agent`` into ``run(input) -> Observation``.
+
+Strands is an AWS-backed agent framework (``strands-agents`` on PyPI). This
+adapter calls the agent, extracts the final response text, the verdict (if
+the agent returns a structured response), and the ordered tool-call names
+from the message's tool-use blocks.
+
+The adapter is an optional import: the core library installs without
+``strands-agents``. Import this module only when working with Strands agents.
+
+Example::
+
+    from strands import Agent
+    from agentverity.adapters.strands import from_strands
+
+    agent = Agent(model="...", system_prompt="you are a gate")
+    fn = from_strands(agent)
+    obs = fn("should I share this?")
+    print(obs.text, obs.verdict, obs.tools)
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from agentverity.observation import Observation
+
+
+def from_strands(agent: Any) -> callable:
+    """Wrap a Strands ``Agent`` into ``run(input) -> Observation``.
+
+    Args:
+        agent: A ``strands.agent.Agent`` instance (must be callable with
+            ``agent(prompt: str)`` and return an ``AgentResult``).
+
+    Returns:
+        A function ``(input: str) -> Observation`` that calls the agent and
+        extracts the text, verdict, and tool trajectory from the result.
+    """
+
+    def run(x: str) -> Observation:
+        result = agent(x)
+        return _extract(result)
+
+    return run
+
+
+def _extract(result: Any) -> Observation:
+    """Extract an :class:`Observation` from a Strands ``AgentResult``.
+
+    Args:
+        result: A Strands ``AgentResult`` (must have ``.message`` with
+            ``content`` blocks).
+
+    Returns:
+        An :class:`Observation` with the final text, verdict (if any), and
+        the ordered list of tool names called.
+    """
+    message = getattr(result, "message", None)
+    if message is None:
+        return Observation(text=str(result), raw=result)
+
+    content = message.get("content", []) if isinstance(message, dict) else getattr(message, "content", [])
+
+    text_parts: list[str] = []
+    tool_names: list[str] = []
+
+    for block in content:
+        if isinstance(block, dict):
+            if "text" in block:
+                text_parts.append(str(block["text"]))
+            elif "toolUse" in block:
+                tool_use = block["toolUse"]
+                name = tool_use.get("name", "") if isinstance(tool_use, dict) else ""
+                if name:
+                    tool_names.append(name)
+            elif "toolResult" in block:
+                tool_result = block["toolResult"]
+                name = tool_result.get("name", "") if isinstance(tool_result, dict) else ""
+                if name:
+                    tool_names.append(name)
+        else:
+            text_parts.append(str(block))
+
+    text = "".join(text_parts) or str(result)
+    structured = getattr(result, "structured_output", None)
+    verdict = None
+    if structured is not None:
+        if hasattr(structured, "model_dump"):
+            dump = structured.model_dump()
+            verdict = str(dump.get("verdict", dump.get("decision", ""))) or None
+        elif isinstance(structured, dict):
+            verdict = str(structured.get("verdict", structured.get("decision", ""))) or None
+
+    return Observation(
+        text=text,
+        verdict=verdict,
+        tools=tuple(tool_names),
+        raw=result,
+    )

agentverity/blindness.py

@@ -0,0 +1,164 @@
+"""Constant-gate-blindness detector — the honesty instrument.
+
+A test suite can pass for the wrong reason: if the agent returns a near-constant
+verdict across a diverse input set, many invariance and monotonicity checks can
+be satisfied without exercising a decision boundary. A green suite can then say
+more about verdict skew than about whether the agent reasons correctly.
+
+This detector measures the agent's verdict **skew** on a probe set and warns
+when a pass is likely trivial. It is the feature that makes the framework
+honest about its own limits: it tells you when your passing metamorphic suite
+is lying to you.
+
+Example::
+
+    from agentverity.blindness import detect
+    from agentverity.adapters import from_callable
+
+    agent = from_callable(my_constant_agent)
+    result = detect(agent, inputs=["hello", "world", "foo", "bar"])
+    if result.blind:
+        print(result.warning)
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from typing import Any
+
+from agentverity.observation import Observation
+
+AgentFn = Callable[[str], Observation]
+
+
+@dataclass(frozen=True)
+class BlindnessResult:
+    """The outcome of a constant-gate-blindness scan.
+
+    Attributes:
+        inputs: Number of inputs probed.
+        layer: Which ``Observation`` layer was measured.
+        majority_verdict: The most common verdict value observed.
+        skew: The share of inputs that returned the majority verdict.
+        distinct: Number of distinct verdicts seen.
+        threshold: The skew threshold above which the gate is considered blind.
+    """
+
+    inputs: int
+    layer: str
+    majority_verdict: Any
+    skew: float
+    distinct: int
+    threshold: float
+
+    @property
+    def blind(self) -> bool:
+        """``True`` if verdict skew can make green relation results vacuous."""
+        return self.skew >= self.threshold
+
+    @property
+    def warning(self) -> str | None:
+        """A human-readable warning if the gate is blind, ``None`` otherwise."""
+        if not self.blind:
+            return None
+        return (
+            f"constant-gate blindness: agent returns "
+            f"{self.majority_verdict!r} on {self.skew:.0%} of inputs. "
+            f"Relation passes may be trivial because the probe set rarely "
+            f"exercises a decision boundary. A green suite is NOT evidence of "
+            f"correct behaviour here."
+        )
+
+
+def detect(
+    agent: AgentFn,
+    inputs: Iterable[str],
+    *,
+    layer: str = "verdict",
+    threshold: float = 0.9,
+) -> BlindnessResult:
+    """Detect whether an agent is near-constant and passes may be vacuous.
+
+    The agent is called once on each input and the verdict distribution is
+    measured. If a single verdict accounts for at least ``threshold`` of the
+    inputs, the gate is flagged as blind. High skew makes relation passes
+    vulnerable to being vacuous because few probes exercise a decision
+    boundary. The detector is a warning about suite power, not a correctness
+    judgement about the agent.
+
+    Args:
+        agent: An agent function ``run(input) -> Observation``.
+        inputs: An iterable of input strings to probe. Use a diverse set;
+            a narrow set inflates skew artificially.
+        layer: Which ``Observation`` layer to measure (``"verdict"``,
+            ``"text"``, or ``"tools"``).
+        threshold: The skew share (0–1) above which the gate is considered
+            blind (default 0.9, i.e. 90%).
+
+    Returns:
+        A :class:`BlindnessResult` with the skew, distinct-verdict count,
+        and blind flag.
+
+    Raises:
+        ValueError: If inputs is empty or threshold is outside ``(0, 1]``.
+    """
+    inputs = list(inputs)
+    if not inputs:
+        raise ValueError("inputs must not be empty")
+    if not 0 < threshold <= 1:
+        raise ValueError("threshold must be between 0 and 1")
+    return score([agent(x) for x in inputs], layer=layer, threshold=threshold)
+
+
+def score(
+    observations: Iterable[Observation],
+    *,
+    layer: str = "verdict",
+    threshold: float = 0.9,
+) -> BlindnessResult:
+    """Score already-collected observations for verdict skew.
+
+    :func:`detect` calls the agent and then delegates here. Call this directly
+    when the observations were gathered by an earlier phase, so a probe set does
+    not have to be re-run purely to count its verdict distribution.
+
+    Args:
+        observations: One :class:`Observation` per input, already collected.
+        layer: Which ``Observation`` layer to measure (``"verdict"``,
+            ``"text"``, or ``"tools"``).
+        threshold: The skew share (0–1) above which the gate is considered
+            blind (default 0.9, i.e. 90%).
+
+    Returns:
+        A :class:`BlindnessResult` with the skew, distinct-verdict count,
+        and blind flag.
+
+    Raises:
+        ValueError: If observations is empty or threshold is outside ``(0, 1]``.
+    """
+    observations = list(observations)
+    if not observations:
+        raise ValueError("observations must not be empty")
+    if not 0 < threshold <= 1:
+        raise ValueError("threshold must be between 0 and 1")
+    verdicts = [_hashable(obs.key(layer)) for obs in observations]
+    c = Counter(verdicts)
+    top, top_n = c.most_common(1)[0]
+    n = len(verdicts)
+    return BlindnessResult(
+        inputs=n,
+        layer=layer,
+        majority_verdict=top,
+        skew=top_n / n,
+        distinct=len(c),
+        threshold=threshold,
+    )
+
+
+def _hashable(v: Any) -> Any:
+    """Normalise a verdict value to a hashable comparison key."""
+    if isinstance(v, (list, tuple)):
+        return tuple(v)
+    return v.value if hasattr(v, "value") else v

agentverity/cli.py

@@ -0,0 +1,113 @@
+"""CLI entry point for agentverity.
+
+Usage::
+
+    agentverity run --agent mymod:build_agent --inputs seeds.txt
+    agentverity run --agent mymod:build_agent --inputs seeds.txt --k 10 --epsilon 0.02
+
+The ``--agent`` argument is a Python dotted path to a callable that returns
+the agent function (``fn() -> (str) -> Observation``). The ``--inputs``
+argument is a text file with one input per line.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import sys
+from collections.abc import Callable
+
+from agentverity.adapters.callable_adapter import from_callable
+from agentverity.runner import RunConfig, run
+
+
+def _load_agent(spec: str) -> Callable:
+    """Load an agent factory from a ``module:func`` spec."""
+    if ":" not in spec:
+        raise ValueError(
+            f"--agent must be 'module:func', got {spec!r}"
+        )
+    module_path, func_name = spec.split(":", 1)
+    module = importlib.import_module(module_path)
+    factory = getattr(module, func_name)
+    if not callable(factory):
+        raise TypeError(f"{spec!r} is not callable")
+    return factory
+
+
+def _load_inputs(path: str) -> list[str]:
+    """Load inputs from a text file, one per line, skipping blanks."""
+    with open(path) as f:
+        return [line.strip() for line in f if line.strip()]
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Run the agentverity CLI.
+
+    Returns:
+        0 if all relations hold and no blindness is detected, 1 otherwise.
+    """
+    parser = argparse.ArgumentParser(
+        prog="agentverity",
+        description="Measure-first testing for non-deterministic LLM agents.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    run_parser = sub.add_parser("run", help="Run the diagnostic suite on an agent.")
+    run_parser.add_argument(
+        "--agent", required=True,
+        help="Python dotted path to an agent factory: 'module:func'. "
+             "The factory is called with no args and must return a "
+             "callable (str) -> Observation.",
+    )
+    run_parser.add_argument(
+        "--inputs", required=True,
+        help="Path to a text file with one input per line.",
+    )
+    run_parser.add_argument("--k", type=int, default=5, help="Meter repeats per input (default 5).")
+    run_parser.add_argument("--epsilon", type=float, default=0.01, help="Meter epsilon (default 0.01).")
+    run_parser.add_argument(
+        "--blindness-threshold", type=float, default=0.9,
+        help="Blindness skew threshold (default 0.9).",
+    )
+    run_parser.add_argument("--layer", default="verdict", help="Observation layer to measure (default 'verdict').")
+    run_parser.add_argument(
+        "--no-meter", action="store_true",
+        help="Skip the verdict-stochasticity meter.",
+    )
+    run_parser.add_argument(
+        "--no-blindness", action="store_true",
+        help="Skip the constant-gate-blindness detector.",
+    )
+
+    args = parser.parse_args(argv)
+
+    if args.command == "run":
+        factory = _load_agent(args.agent)
+        # from_callable passes Observation objects through unchanged, so the
+        # CLI can adapt every factory without a side-effecting probe call.
+        agent_fn = from_callable(factory())
+        inputs = _load_inputs(args.inputs)
+        config = RunConfig(
+            k=args.k,
+            epsilon=args.epsilon,
+            blindness_threshold=args.blindness_threshold,
+            layer=args.layer,
+            run_meter=not args.no_meter,
+            run_blindness=not args.no_blindness,
+        )
+        result = run(agent_fn, inputs, config=config)
+        print(result.summary())
+
+        # Exit code: 1 if blind or any relation violated, 0 otherwise
+        if result.is_blind:
+            return 1
+        if any(rr.violated > 0 for rr in result.relation_results):
+            return 1
+        return 0
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

agentverity/meter.py

@@ -0,0 +1,204 @@
+"""Verdict-stochasticity meter — the measure-first instrument.
+
+Agents are non-deterministic, so before trusting any test you must know whether
+the agent's *decision* actually varies across identical reruns, and at which
+layer. Token-level variation (the final text wording) is common and mostly
+harmless; what matters for testing is whether the **verdict** (the categorical
+decision, or the tool trajectory) flips. If the verdict is stable and a
+trusted reference is available, frozen-baseline diffing is the more sensitive
+change detector.
+If it is stochastic, you need noise-robust relations and a measured baseline,
+not zero-tolerance assertions.
+
+The meter runs the agent ``k`` times on each unchanged input and reports a
+tri-state call with a Wilson confidence interval, so an underfunded probe is
+reported as ``"undecided"`` rather than mislabelled ``"deterministic"``.
+
+Example::
+
+    from agentverity.meter import measure
+    from agentverity.adapters import from_callable
+
+    agent = from_callable(my_agent_fn)
+    result = measure(agent, inputs=["hello", "world"], k=5)
+    print(result.call)       # "verdict-deterministic" / "verdict-stochastic" / "undecided"
+    print(result.advice)     # human-readable recommendation
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from typing import Any
+
+from agentverity.observation import Observation
+
+AgentFn = Callable[[str], Observation]
+
+
+def wilson_ci(successes: int, trials: int, z: float = 1.96) -> tuple[float, float]:
+    """Return the Wilson score interval for a binomial proportion.
+
+    Args:
+        successes: Number of successes observed.
+        trials: Total number of trials.
+        z: Z-value for the desired confidence level (default 1.96 for 95%).
+
+    Returns:
+        A ``(low, high)`` tuple with the lower and upper bounds of the
+        confidence interval. Returns ``(0.0, 0.0)`` if ``trials`` is zero.
+    """
+    if trials == 0:
+        return 0.0, 0.0
+    p = successes / trials
+    denom = 1 + z * z / trials
+    centre = (p + z * z / (2 * trials)) / denom
+    margin = z * math.sqrt((p * (1 - p) + z * z / (4 * trials)) / trials) / denom
+    return max(0.0, centre - margin), min(1.0, centre + margin)
+
+
+@dataclass(frozen=True)
+class MeterResult:
+    """The outcome of a verdict-stochasticity measurement.
+
+    Attributes:
+        layer: Which ``Observation`` layer was measured (``"verdict"``,
+            ``"text"``, or ``"tools"``).
+        epsilon: The flip-rate threshold below which a gate is considered
+            deterministic.
+        inputs: Number of distinct inputs probed.
+        repeats: Number of repeated calls per input (``k``).
+        pair_trials: Total independent, disjoint comparisons across all inputs.
+        pair_flips: Number of disjoint comparisons where the verdict differed.
+        inputs_with_flip: Number of inputs that showed at least one flip.
+        ci_low: Lower bound of the Wilson CI on the flip rate.
+        ci_high: Upper bound of the Wilson CI on the flip rate.
+    """
+
+    layer: str
+    epsilon: float
+    inputs: int
+    repeats: int
+    pair_trials: int
+    pair_flips: int
+    inputs_with_flip: int
+    ci_low: float
+    ci_high: float
+
+    @property
+    def flip_rate(self) -> float:
+        """The observed pairwise flip rate (``pair_flips / pair_trials``)."""
+        return self.pair_flips / self.pair_trials if self.pair_trials else 0.0
+
+    @property
+    def call(self) -> str:
+        """The tri-state classification.
+
+        Returns:
+            ``"verdict-stochastic"`` if the CI lower bound is above epsilon,
+            ``"verdict-deterministic"`` if the CI upper bound is below epsilon,
+            or ``"undecided (add repeats or inputs)"`` if the interval straddles
+            epsilon. A bare ``"deterministic"`` would conflate real stability
+            with an underpowered probe, so an interval straddling epsilon is
+            ``"undecided"``.
+        """
+        if self.ci_low > self.epsilon:
+            return "verdict-stochastic"
+        if self.ci_high < self.epsilon:
+            return "verdict-deterministic"
+        return "undecided (add repeats or inputs)"
+
+    @property
+    def advice(self) -> str:
+        """A human-readable recommendation based on the tri-state call."""
+        c = self.call
+        if c == "verdict-stochastic":
+            return ("verdict varies across identical runs: use noise-robust "
+                    "relations and compare violations to a measured baseline, "
+                    "not zero.")
+        if c == "verdict-deterministic":
+            return ("verdict is stable: prefer frozen-baseline diffing when "
+                    "a trusted reference is available.")
+        return "not enough evidence to choose an oracle; raise K or input count."
+
+
+def measure(
+    agent: AgentFn,
+    inputs: Iterable[str],
+    *,
+    k: int = 5,
+    layer: str = "verdict",
+    epsilon: float = 0.01,
+) -> MeterResult:
+    """Measure the verdict-stochasticity of an agent.
+
+    The agent is called ``k`` times on each input (with no transform applied).
+    Consecutive outputs are compared in disjoint pairs, giving
+    :math:`\\lfloor k/2 \\rfloor` independent comparisons per input. A Wilson
+    confidence interval on that flip rate determines the tri-state call.
+
+    Using every pair among the same ``k`` outputs would create
+    :math:`\\binom{k}{2}` dependent comparisons and make the interval look
+    more precise than the calls justify. An odd final repeat is retained for
+    ``inputs_with_flip`` but does not enter the confidence interval.
+
+    Args:
+        agent: An agent function ``run(input) -> Observation``.
+        inputs: An iterable of input strings to probe.
+        k: Number of repeated calls per input (must be >= 2). The confidence
+            interval uses ``floor(k / 2)`` disjoint pairs.
+        layer: Which ``Observation`` layer to measure (``"verdict"``,
+            ``"text"``, or ``"tools"``).
+        epsilon: The flip-rate threshold below which a gate is considered
+            deterministic (default 0.01, i.e. 1%).
+
+    Returns:
+        A :class:`MeterResult` with the flip rate, Wilson CI, and tri-state call.
+
+    Raises:
+        ValueError: If ``k`` is less than 2, inputs is empty, or epsilon is
+            outside ``(0, 1)``.
+    """
+    if k < 2:
+        raise ValueError("k must be >= 2 to compare repeated runs")
+    inputs = list(inputs)
+    if not inputs:
+        raise ValueError("inputs must not be empty")
+    if not 0 < epsilon < 1:
+        raise ValueError("epsilon must be between 0 and 1")
+    pair_trials = 0
+    pair_flips = 0
+    inputs_with_flip = 0
+    for x in inputs:
+        keys = [agent(x).key(layer) for _ in range(k)]
+        if len({_hashable(v) for v in keys}) > 1:
+            inputs_with_flip += 1
+        for i in range(0, k - 1, 2):
+            pair_trials += 1
+            if _hashable(keys[i]) != _hashable(keys[i + 1]):
+                pair_flips += 1
+    lo, hi = wilson_ci(pair_flips, pair_trials)
+    return MeterResult(
+        layer=layer,
+        epsilon=epsilon,
+        inputs=len(inputs),
+        repeats=k,
+        pair_trials=pair_trials,
+        pair_flips=pair_flips,
+        inputs_with_flip=inputs_with_flip,
+        ci_low=lo,
+        ci_high=hi,
+    )
+
+
+def _hashable(v: Any) -> Any:
+    """Normalise a verdict value to a hashable comparison key.
+
+    Verdicts may be strings, enums, or tuples (for tool trajectories).
+    Lists are converted to tuples; enum values are extracted via ``.value``
+    if present.
+    """
+    if isinstance(v, (list, tuple)):
+        return tuple(v)
+    return v.value if hasattr(v, "value") else v