faultbench 0.1.0__py3-none-any.whl

faultbench/__init__.py

@@ -0,0 +1,3 @@
+"""faultbench: fake, stateful worlds with fault injection for testing tool-using AI agents."""
+
+__version__ = "0.1.0"

faultbench/cli.py

@@ -0,0 +1,60 @@
+"""`faultbench` command line. `serve` runs a world.yaml over MCP (stdio or HTTP)."""
+
+import argparse
+import sys
+
+from faultbench import __version__
+from faultbench.faults import FaultProfile
+from faultbench.server import serve_http, serve_stdio
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="faultbench", description=__doc__)
+    parser.add_argument("--version", action="version", version=f"faultbench {__version__}")
+    sub = parser.add_subparsers(dest="command")
+
+    serve = sub.add_parser("serve", help="serve a world.yaml over MCP (stdio or HTTP)")
+    serve.add_argument("world", help="path to world.yaml")
+    serve.add_argument(
+        "--http", action="store_true", help="serve over streamable HTTP instead of stdio"
+    )
+    serve.add_argument("--host", default="127.0.0.1", help="HTTP host (default 127.0.0.1)")
+    serve.add_argument("--port", type=int, default=8000, help="HTTP port (default 8000)")
+    serve.add_argument("--faults", action="store_true", help="apply the world file's faults: block")
+    serve.add_argument("--run-index", type=int, default=0, help="run index for the fault RNG")
+    serve.add_argument("--state-file", help="mirror world state to this JSON file after each call")
+
+    args = parser.parse_args(argv)
+    if args.command == "serve":
+        return _serve(args)
+    parser.print_help()
+    return 0
+
+
+def _serve(args: argparse.Namespace) -> int:
+    faults = FaultProfile.from_world_file(args.world) if args.faults else None
+    if args.http:
+        print(
+            f"faultbench: serving {args.world} at http://{args.host}:{args.port}/mcp",
+            file=sys.stderr,
+        )
+        serve_http(
+            args.world,
+            host=args.host,
+            port=args.port,
+            faults=faults,
+            run_index=args.run_index,
+            state_file=args.state_file,
+        )
+    else:
+        serve_stdio(
+            args.world,
+            faults=faults,
+            run_index=args.run_index,
+            state_file=args.state_file,
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

faultbench/faults/__init__.py

@@ -0,0 +1,16 @@
+"""faultbench.faults — see docs/ARCHITECTURE.md."""
+
+from .clock import DEFAULT_NOW, FakeClock
+from .injector import FaultError, FaultTimeout, Injector
+from .profile import FaultProfile, FaultRule, parse_duration
+
+__all__ = [
+    "FakeClock",
+    "DEFAULT_NOW",
+    "FaultProfile",
+    "FaultRule",
+    "parse_duration",
+    "Injector",
+    "FaultTimeout",
+    "FaultError",
+]

faultbench/faults/clock.py

@@ -0,0 +1,32 @@
+"""FakeClock: the only source of time inside a world. Milestone 4.
+
+The engine, handlers, and injector read "now" only from a FakeClock, so time-dependent
+behaviour ("this order is 2 hours old") is testable without waiting. A FakeClock never
+advances on its own — a test moves it with `advance()` or `set()`.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+# A fixed default "now", so a clock with no explicit start is still deterministic.
+DEFAULT_NOW = datetime(2025, 6, 1)
+
+
+class FakeClock:
+    def __init__(self, now: datetime | None = None) -> None:
+        self._now = now or DEFAULT_NOW
+
+    def now(self) -> datetime:
+        return self._now
+
+    def advance(self, seconds: float) -> datetime:
+        self._now += timedelta(seconds=seconds)
+        return self._now
+
+    def set(self, when: datetime) -> datetime:
+        self._now = when
+        return self._now
+
+    def __repr__(self) -> str:
+        return f"FakeClock({self._now.isoformat()})"

faultbench/faults/injector.py

@@ -0,0 +1,151 @@
+"""Fault injector: wraps an operation call and applies latency + errors. Milestones 4, 7.
+
+Determinism: all randomness comes from `Random(f"{seed}:{run_index}")`, so the same profile,
+seed, and run index produce the same fault sequence across processes. Conditional faults and
+rate limits are deterministic (data- and clock-driven, no RNG).
+
+Order of checks per call: latency → conditional not-found → rate limit → probabilistic error.
+
+Error timing models real failures:
+  - `timeout`: the request reached the server and RAN (its side effect happened), but the
+    client gave up waiting. So the op executes, then a FaultTimeout is raised. A client that
+    retries will run it again — this is how a timeout becomes a double write.
+  - `http_500` / `http_429` / `not_found`: the request is rejected BEFORE running, so there
+    is no side effect.
+  - `empty_result` / `malformed_json`: the response is corrupted; the op does NOT run.
+
+Conditional (milestone 7):
+  - `not_found_if_newer_than: 2h`: a read of a record whose timestamp is newer than 2h before
+    `clock.now()` returns not_found — models sync lag (recent writes not yet visible).
+  - `rate_limit: {calls, per_seconds}`: after `calls` calls within the window, returns
+    http_429 until the window rolls forward (measured on the clock).
+"""
+
+from __future__ import annotations
+
+import random
+import sys
+import time
+from collections.abc import Callable
+from datetime import datetime
+from typing import Any
+
+from .profile import FaultProfile, FaultRule
+
+
+class FaultTimeout(Exception):
+    """The operation ran, but the client 'timed out' before getting the result."""
+
+
+class FaultError(Exception):
+    """The operation was rejected by an injected fault before it ran (or its response was
+    corrupted)."""
+
+    def __init__(self, kind: str, message: str | None = None) -> None:
+        self.kind = kind
+        super().__init__(message or f"injected {kind}")
+
+
+class Injector:
+    def __init__(
+        self,
+        profile: FaultProfile,
+        seed: int = 0,
+        run_index: int = 0,
+        clock: Any = None,
+        sleep: Callable[[float], None] = time.sleep,
+        timeout_seconds: float = 0.0,
+        on_event: Callable[[dict], None] | None = None,
+    ) -> None:
+        self.profile = profile
+        self.clock = clock
+        self._rng = random.Random(f"{seed}:{run_index}")
+        self._sleep = sleep
+        self._timeout_seconds = timeout_seconds
+        self._on_event = on_event
+        self._rl_calls: dict[str, list[float]] = {}  # service.op -> recent call times (rate limit)
+
+    def call(
+        self,
+        service: str,
+        op: str,
+        fn: Callable[[], Any],
+        on_decision: Callable[[str | None, float], None] | None = None,
+        record_time: datetime | None = None,
+    ) -> Any:
+        """Run `fn` under the fault rule for `service.op`.
+
+        `on_decision(fault_kind, latency_ms)` is called once the fault is decided, before the
+        operation runs — so a recorder can log what was injected even if the call then fails.
+        `record_time` is the target record's timestamp, used by conditional not-found rules.
+        """
+        rule = self.profile.resolve(service, op)
+        latency_ms = self._apply_latency(rule)
+        kind = self._conditional(rule, record_time) or self._rate_limited(service, op, rule)
+        if kind is None:
+            kind = self._roll(rule.errors)
+        if on_decision:
+            on_decision(kind, latency_ms)
+        self._emit(service, op, kind)
+        if kind is None:
+            return fn()
+        if kind == "timeout":
+            fn()  # side effect happens; the result is lost because the client "times out"
+            if self._timeout_seconds:
+                self._sleep(self._timeout_seconds)
+            raise FaultTimeout(f"{op} timed out after executing")
+        if kind == "empty_result":
+            return []
+        if kind == "malformed_json":
+            return "}{ not valid json"
+        raise FaultError(kind, f"{op}: injected {kind}")
+
+    # --- internals ---------------------------------------------------------------
+    def _conditional(self, rule: FaultRule, record_time: datetime | None) -> str | None:
+        """Return 'not_found' when the record is newer than the sync-lag window."""
+        window = rule.conditional.get("not_found_if_newer_than")
+        if window is None or record_time is None or self.clock is None:
+            return None
+        age = (self.clock.now() - record_time).total_seconds()
+        return "not_found" if age < window else None
+
+    def _rate_limited(self, service: str, op: str, rule: FaultRule) -> str | None:
+        """Return 'http_429' once more than `calls` calls happen within the window."""
+        limit = rule.rate_limit
+        if limit is None:
+            return None
+        now = self.clock.now().timestamp() if self.clock is not None else time.monotonic()
+        key = f"{service}.{op}"
+        recent = [t for t in self._rl_calls.get(key, []) if now - t < limit.per_seconds]
+        if len(recent) >= limit.calls:
+            self._rl_calls[key] = recent
+            return "http_429"
+        recent.append(now)
+        self._rl_calls[key] = recent
+        return None
+
+    def _apply_latency(self, rule: FaultRule) -> float:
+        if not rule.latency_ms:
+            return 0.0
+        lo, hi = rule.latency_ms
+        ms = self._rng.uniform(lo, hi)
+        self._sleep(ms / 1000.0)
+        return ms
+
+    def _roll(self, errors: dict[str, float]) -> str | None:
+        if not errors:
+            return None
+        r = self._rng.random()
+        cumulative = 0.0
+        for kind, prob in errors.items():  # insertion order (from YAML) => deterministic
+            cumulative += prob
+            if r < cumulative:
+                return kind
+        return None
+
+    def _emit(self, service: str, op: str, kind: str | None) -> None:
+        event = {"service": service, "op": op, "fault": kind}
+        if kind:
+            print(f"[faultbench:fault] {service}.{op} -> {kind}", file=sys.stderr)
+        if self._on_event:
+            self._on_event(event)

faultbench/faults/profile.py

@@ -0,0 +1,104 @@
+"""Fault profile: the parsed `faults:` block. Milestone 4.
+
+A profile maps a key to a rule. The key is either `default` (applies to every operation) or
+`service.operation` (applies to one). A rule carries a latency range and a map of error kind
+-> probability. `resolve(service, op)` merges the default rule with the specific one.
+
+Conditional errors (e.g. `not_found_if_newer_than: 2h`) and rate limits are parsed and kept
+but NOT enforced yet — that is milestone 7. Keeping them here means shop.yaml still loads.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import yaml
+from pydantic import BaseModel, Field
+
+# Probabilistic error kinds the injector knows how to produce (milestone 4).
+KNOWN_ERROR_KINDS = frozenset(
+    {"timeout", "http_500", "http_429", "malformed_json", "empty_result", "not_found"}
+)
+# Keys inside `errors:` that are conditions, not probabilities (enforced in milestone 7).
+CONDITIONAL_KEYS = frozenset({"not_found_if_newer_than"})
+
+_DURATION_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([smhd])\s*$")
+_UNIT_SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400}
+
+
+def parse_duration(text: str) -> float:
+    """ "2h" -> 7200.0 seconds."""
+    m = _DURATION_RE.match(text)
+    if not m:
+        raise ValueError(f"bad duration {text!r} (expected e.g. '30s', '2h', '1d')")
+    return float(m.group(1)) * _UNIT_SECONDS[m.group(2)]
+
+
+class RateLimit(BaseModel):
+    calls: int
+    per_seconds: float
+
+
+class FaultRule(BaseModel):
+    latency_ms: tuple[int, int] | None = None
+    errors: dict[str, float] = Field(default_factory=dict)  # kind -> probability
+    conditional: dict[str, float] = Field(default_factory=dict)  # cond -> seconds
+    rate_limit: RateLimit | None = None
+
+    @classmethod
+    def from_body(cls, body: dict) -> FaultRule:
+        latency = body.get("latency_ms")
+        if latency is not None:
+            latency = tuple(latency)
+            if len(latency) != 2 or latency[0] > latency[1]:
+                raise ValueError(f"latency_ms must be [lo, hi] with lo <= hi, got {latency}")
+        errors: dict[str, float] = {}
+        conditional: dict[str, float] = {}
+        for kind, value in (body.get("errors") or {}).items():
+            if kind in CONDITIONAL_KEYS:
+                conditional[kind] = parse_duration(str(value))
+            elif kind in KNOWN_ERROR_KINDS:
+                errors[kind] = float(value)
+            else:
+                raise ValueError(
+                    f"unknown error kind {kind!r} (known: {sorted(KNOWN_ERROR_KINDS)}, "
+                    f"conditional: {sorted(CONDITIONAL_KEYS)})"
+                )
+        total = sum(errors.values())
+        if total > 1.0 + 1e-9:
+            raise ValueError(f"error probabilities sum to {total} > 1.0")
+        rate_limit = RateLimit(**body["rate_limit"]) if body.get("rate_limit") else None
+        return cls(
+            latency_ms=latency, errors=errors, conditional=conditional, rate_limit=rate_limit
+        )
+
+
+class FaultProfile(BaseModel):
+    rules: dict[str, FaultRule] = Field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, faults: dict | None) -> FaultProfile:
+        rules = {key: FaultRule.from_body(body or {}) for key, body in (faults or {}).items()}
+        return cls(rules=rules)
+
+    @classmethod
+    def from_world_file(cls, path: str | Path) -> FaultProfile:
+        data = yaml.safe_load(Path(path).read_text())
+        return cls.from_dict(data.get("faults"))
+
+    def resolve(self, service: str, op: str) -> FaultRule:
+        """The effective rule for one operation: default merged with its specific rule."""
+        default = self.rules.get("default", FaultRule())
+        specific = self.rules.get(f"{service}.{op}")
+        if specific is None:
+            return default
+        return FaultRule(
+            latency_ms=specific.latency_ms or default.latency_ms,
+            errors={**default.errors, **specific.errors},
+            conditional={**default.conditional, **specific.conditional},
+            rate_limit=specific.rate_limit or default.rate_limit,
+        )
+
+    def is_empty(self) -> bool:
+        return not self.rules

faultbench/integrations/__init__.py

@@ -0,0 +1,6 @@
+"""Optional, framework-specific adapters for connecting an agent to a faultbench server.
+
+These are convenience only — faultbench's core is framework-neutral (it serves standard MCP).
+Import the adapter for your framework, e.g. `from faultbench.integrations.pydantic_ai import
+run_agent`. Each adapter needs its framework installed (an optional dependency).
+"""

faultbench/integrations/pydantic_ai.py

@@ -0,0 +1,55 @@
+"""Pydantic AI adapter: connect an agent to a faultbench server in one line.
+
+Needs Pydantic AI:  pip install 'faultbench[pydantic-ai]'
+
+`mcp` is whatever the pytest fixtures give you — an in-process server (`mcp_server`) or an
+HTTP URL (`mcp_url`); the same helpers work with both, so tests read the same either way.
+
+    from faultbench.integrations.pydantic_ai import run_agent
+
+    @pytest.mark.world("world.yaml")
+    async def test_refund(world, mcp_server):
+        order = world.orders.pick(status="delivered")
+        await run_agent("openai:gpt-5-mini", f"Refund order {order.id}", mcp=mcp_server,
+                        system_prompt="You are a refund agent.")
+        assert len(world.refunds.where(order_id=order.id)) == 1
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+try:
+    from pydantic_ai import Agent
+    from pydantic_ai.mcp import FastMCPClient, MCPToolset
+except ImportError as exc:  # pragma: no cover - exercised only without a suitable version
+    raise ImportError(
+        "faultbench.integrations.pydantic_ai needs a recent Pydantic AI (v2+, tested with "
+        "2.45+). Install it with: pip install 'faultbench[pydantic-ai]'"
+    ) from exc
+
+
+def toolset(mcp: Any) -> MCPToolset:
+    """A Pydantic AI toolset for a faultbench server. `mcp` is an in-process MCPServer (the
+    `mcp_server` fixture) or an HTTP URL string (the `mcp_url` fixture)."""
+    return MCPToolset(FastMCPClient(str(mcp) if not _is_server(mcp) else mcp))
+
+
+def agent(model: str, mcp: Any, *, system_prompt: str = "", **agent_kwargs: Any) -> Agent:
+    """A Pydantic AI Agent wired to a faultbench server."""
+    return Agent(model, system_prompt=system_prompt, toolsets=[toolset(mcp)], **agent_kwargs)
+
+
+async def run_agent(
+    model: str, prompt: str, *, mcp: Any, system_prompt: str = "", **agent_kwargs: Any
+) -> str:
+    """Run one request end to end against a faultbench server; return the agent's reply."""
+    the_agent = agent(model, mcp, system_prompt=system_prompt, **agent_kwargs)
+    async with the_agent:
+        result = await the_agent.run(prompt)
+    return result.output
+
+
+def _is_server(mcp: Any) -> bool:
+    # A URL/path is a str; anything else is treated as an in-process server object.
+    return not isinstance(mcp, str)