agentprobe-testing 0.5.0__tar.gz

agentprobe_testing-0.5.0/LICENSE

@@ -0,0 +1,109 @@
+Business Source License 1.1
+
+Parameters
+
+Licensor:             Agent-Probe
+Licensed Work:        AgentProbe
+                       The Licensed Work is (c) 2026 Agent-Probe
+Additional Use Grant:  You may use, copy, modify, and make production use of
+                       the Licensed Work for any purpose -- including testing,
+                       developing, and operating your own software agents --
+                       free of charge, provided that you do not offer the
+                       Licensed Work, or a modified or derivative version of
+                       it, to third parties as part of a hosted, managed, or
+                       embedded service that competes with a paid product or
+                       service offered by the Licensor (including, without
+                       limitation, hosted domain generation, hosted
+                       dashboards, or hosted test-run storage substantially
+                       similar to those offered at
+                       https://agentprobe-api.agentprobe.workers.dev).
+Change Date:           2030-01-01
+Change License:        Apache License, Version 2.0
+
+For information about alternative licensing arrangements for the Licensed
+Work, please open an issue at https://github.com/Agent-Probe/agentprobe.
+
+Notice
+
+The Business Source License (this document, or the "License") is not an
+Open Source license. However, the Licensed Work will eventually be made
+available under an Open Source License, as stated in this License.
+
+License text copyright (c) 2017 MariaDB Corporation Ab, All Rights
+Reserved. "Business Source License" is a trademark of MariaDB Corporation
+Ab.
+
+-----------------------------------------------------------------------------
+
+Business Source License 1.1
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create
+derivative works, redistribute, and make non-production use of the
+Licensed Work. The Licensor may make an Additional Use Grant, above,
+permitting limited production use.
+
+Effective on the Change Date, or the fourth anniversary of the first
+publicly available distribution of a specific version of the Licensed
+Work under this License, whichever comes first, the Licensor hereby
+grants you rights under the terms of the Change License, and the rights
+granted in the paragraph above terminate.
+
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or
+authorized resellers, or you must refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative
+works of the Licensed Work, are subject to this License. This License
+applies separately for each version of the Licensed Work and the Change
+Date may vary for each version of the Licensed Work released by
+Licensor.
+
+You must conspicuously display this License on each original or
+modified copy of the Licensed Work. If you receive the Licensed Work in
+original or modified form from a third party, the terms and conditions
+set forth in this License apply to your use of that work.
+
+Any use of the Licensed Work in violation of this License will
+automatically terminate your rights under this License for the current
+and all other versions of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or
+logo of Licensor as expressly required by this License).
+
+TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS
+PROVIDED ON AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES
+AND CONDITIONS, EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION)
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
+NON-INFRINGEMENT, AND TITLE.
+
+MariaDB hereby grants you permission to use this License's text to
+license your works, and to refer to it using the trademark "Business
+Source License", as long as you comply with the Covenants of Licensor
+below.
+
+Covenants of Licensor
+
+In consideration of the right to use this License's text and the
+"Business Source License" name and trademark, Licensor covenants to
+MariaDB, and to all other recipients of the licensed work to be
+provided by Licensor:
+
+1. To specify as the Change License the GPL Version 2.0 or any later
+   version, or a license that is compatible with GPL Version 2.0 or a
+   later version, where "compatible" means that software provided
+   under the Change License can be included in a program with software
+   provided under GPL Version 2.0 or a later version. Licensor may
+   specify additional Change Licenses without limitation.
+
+2. To either: (a) specify an additional grant of rights to use that
+   does not impose any additional restriction on the right granted in
+   this License, as the Additional Use Grant, or (b) insert the text
+   "None".
+
+3. To specify a Change Date.
+
+4. Not to modify this License in any other way.

agentprobe_testing-0.5.0/PKG-INFO

@@ -0,0 +1,127 @@
+Metadata-Version: 2.4
+Name: agentprobe-testing
+Version: 0.5.0
+Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
+License: Business Source License 1.1
+        
+        Parameters
+        
+        Licensor:             Agent-Probe
+        Licensed Work:        AgentProbe
+                               The Licensed Work is (c) 2026 Agent-Probe
+        Additional Use Grant:  You may use, copy, modify, and make production use of
+                               the Licensed Work for any purpose -- including testing,
+                               developing, and operating your own software agents --
+                               free of charge, provided that you do not offer the
+                               Licensed Work, or a modified or derivative version of
+                               it, to third parties as part of a hosted, managed, or
+                               embedded service that competes with a paid product or
+                               service offered by the Licensor (including, without
+                               limitation, hosted domain generation, hosted
+                               dashboards, or hosted test-run storage substantially
+                               similar to those offered at
+                               https://agentprobe-api.agentprobe.workers.dev).
+        Change Date:           2030-01-01
+        Change License:        Apache License, Version 2.0
+        
+        For information about alternative licensing arrangements for the Licensed
+        Work, please open an issue at https://github.com/Agent-Probe/agentprobe.
+        
+        Notice
+        
+        The Business Source License (this document, or the "License") is not an
+        Open Source license. However, the Licensed Work will eventually be made
+        available under an Open Source License, as stated in this License.
+        
+        License text copyright (c) 2017 MariaDB Corporation Ab, All Rights
+        Reserved. "Business Source License" is a trademark of MariaDB Corporation
+        Ab.
+        
+        -----------------------------------------------------------------------------
+        
+        Business Source License 1.1
+        
+        Terms
+        
+        The Licensor hereby grants you the right to copy, modify, create
+        derivative works, redistribute, and make non-production use of the
+        Licensed Work. The Licensor may make an Additional Use Grant, above,
+        permitting limited production use.
+        
+        Effective on the Change Date, or the fourth anniversary of the first
+        publicly available distribution of a specific version of the Licensed
+        Work under this License, whichever comes first, the Licensor hereby
+        grants you rights under the terms of the Change License, and the rights
+        granted in the paragraph above terminate.
+        
+        If your use of the Licensed Work does not comply with the requirements
+        currently in effect as described in this License, you must purchase a
+        commercial license from the Licensor, its affiliated entities, or
+        authorized resellers, or you must refrain from using the Licensed Work.
+        
+        All copies of the original and modified Licensed Work, and derivative
+        works of the Licensed Work, are subject to this License. This License
+        applies separately for each version of the Licensed Work and the Change
+        Date may vary for each version of the Licensed Work released by
+        Licensor.
+        
+        You must conspicuously display this License on each original or
+        modified copy of the Licensed Work. If you receive the Licensed Work in
+        original or modified form from a third party, the terms and conditions
+        set forth in this License apply to your use of that work.
+        
+        Any use of the Licensed Work in violation of this License will
+        automatically terminate your rights under this License for the current
+        and all other versions of the Licensed Work.
+        
+        This License does not grant you any right in any trademark or logo of
+        Licensor or its affiliates (provided that you may use a trademark or
+        logo of Licensor as expressly required by this License).
+        
+        TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS
+        PROVIDED ON AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES
+        AND CONDITIONS, EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION)
+        WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
+        NON-INFRINGEMENT, AND TITLE.
+        
+        MariaDB hereby grants you permission to use this License's text to
+        license your works, and to refer to it using the trademark "Business
+        Source License", as long as you comply with the Covenants of Licensor
+        below.
+        
+        Covenants of Licensor
+        
+        In consideration of the right to use this License's text and the
+        "Business Source License" name and trademark, Licensor covenants to
+        MariaDB, and to all other recipients of the licensed work to be
+        provided by Licensor:
+        
+        1. To specify as the Change License the GPL Version 2.0 or any later
+           version, or a license that is compatible with GPL Version 2.0 or a
+           later version, where "compatible" means that software provided
+           under the Change License can be included in a program with software
+           provided under GPL Version 2.0 or a later version. Licensor may
+           specify additional Change Licenses without limitation.
+        
+        2. To either: (a) specify an additional grant of rights to use that
+           does not impose any additional restriction on the right granted in
+           this License, as the Additional Use Grant, or (b) insert the text
+           "None".
+        
+        3. To specify a Change Date.
+        
+        4. Not to modify this License in any other way.
+        
+Project-URL: Repository, https://github.com/Agent-Probe/agentprobe
+Project-URL: Changelog, https://github.com/Agent-Probe/agentprobe/blob/main/CHANGELOG.md
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.11
+License-File: LICENSE
+Requires-Dist: anthropic>=1.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
+Dynamic: license-file

agentprobe_testing-0.5.0/README.md

@@ -0,0 +1,275 @@
+# AgentProbe
+
+[![CI](https://github.com/Agent-Probe/agentprobe/actions/workflows/ci.yml/badge.svg)](https://github.com/Agent-Probe/agentprobe/actions/workflows/ci.yml)
+
+Chaos-testing for LLM agents: a second agent (the **Injector**) watches
+your agent's live trajectory — its actual tool calls, its actual
+commitments — and adaptively decides where to break something, instead of
+firing perturbations on a fixed schedule authored in advance. See
+[PRIOR_ART.md](PRIOR_ART.md) for how this differs from τ²-bench, Gaia2,
+and STAGE-Claw, all of which script their perturbations offline.
+
+Six injection kinds: a failed tool call (`TOOL_ERROR`), a record that
+silently changes after your agent already read it (`STALE_READ`), a new
+fact that contradicts an earlier one (`CONTRADICTION`), information that
+only matters after an irreversible action already happened (`LATE_INFO`),
+a second equally-valid candidate record appearing (`AMBIGUITY`), and
+adversarial content disguised as ordinary data (`PROMPT_INJECTION`).
+
+## Quickstart
+
+```bash
+pip install -e .
+python examples/free_zero_cost_demo.py   # no API key needed, no network calls at all
+```
+
+That runs a real (deterministic, not scripted) decision-tree agent
+against a hardcoded chaos injector across both domains this repo ships
+with, prints a pass/fail report, and saves a step-by-step HTML report
+next to itself. Nothing here costs money.
+
+To see it with real Claude agents on both sides (Target reasoning live,
+Injector deciding live), across both domains this repo ships with, which
+does cost a small amount of real API usage:
+
+```bash
+python examples/live_llm_test.py   # needs ANTHROPIC_API_KEY
+```
+
+For a harder test subject than the baseline `AccessControlAgent` --
+`ComplexAccessControlAgent` follows an explicit multi-factor decision
+framework (sensitivity vs. clearance vs. justification quality, required
+policy lookups before granting anything high-sensitivity, a mandatory
+re-check right before committing, and explicit prompt-injection-defense
+instructions) instead of a short generic prompt. Same cost profile as
+`live_llm_test.py`, scoped to one scenario per class (clean/split/decoy/
+trap) so it's cheap to run and still exercises everything that framework
+claims:
+
+```bash
+python examples/test_complex_agent.py   # needs ANTHROPIC_API_KEY
+```
+
+## As a library
+
+The fastest way to test your own agent — write one function, get a report:
+
+```python
+import agentprobe as ap
+
+def my_agent(task: str, history: list[dict]) -> ap.AgentAction:
+    # call your real agent here; history is everything observed so far
+    # this run, as {"tool_name", "tool_args", "result", "ok"} dicts
+    ...
+    return ap.AgentAction(kind="tool_call", tool_name="...", tool_args={...})
+    # or: return ap.AgentAction(kind="final_answer", text="...")
+
+report = ap.quick_test(
+    ap.TICKET_SCENARIOS_BY_ID["clean-1"],
+    my_agent,
+    injector=ap.HardcodedToolErrorInjector(offsets=[0], tool_name="close_ticket"),
+)
+print(report.render())
+```
+
+No class to write, no manual state tracking — `quick_test` calls `my_agent`
+fresh for both the clean and chaos runs, so there's nothing to reset
+between them either. This only tests *ticket-support* scenarios though
+(`ap.TICKET_SCENARIOS_BY_ID`) — meaningfully testing a different business's
+agent means authoring a new [`Domain`](agentprobe/domain.py) for it first
+(see "Domains" below); `quick_test` removes the harness boilerplate,
+not the need to describe what your agent's tools and correct answers are.
+
+For full control (multi-scenario runs, custom Report fields, manual state
+in your agent), the same thing without the convenience wrapper:
+
+```python
+import agentprobe as ap
+
+injector = ap.HardcodedToolErrorInjector(offsets=[0], tool_name="close_ticket")
+clean, chaos = ap.run_robustness_pair(
+    ap.TICKET_SCENARIOS_BY_ID["clean-1"],
+    target_factory=lambda: ap.RuleBasedAgent(),  # zero API cost
+    injector=injector,
+)
+print(ap.Report(
+    mode="robustness", injector_model="hardcoded", target_model="rule_based",
+    clean_trajectories=[clean], chaos_trajectories=[chaos],
+).render())
+```
+
+`import agentprobe` re-exports the pieces most programs need directly —
+see [`agentprobe/__init__.py`](agentprobe/__init__.py) for the full
+surface. The `access_control` domain lives at
+`agentprobe.domains.access_control` rather than being re-exported at the
+top level, to keep this surface to the one domain most people reach for
+first.
+
+Beyond one-scenario `quick_test()`, see [USAGE.md](USAGE.md) for: running
+a whole domain at once (`quick_test_all`), a one-line pytest integration
+(`assert_passes`), catching regressions between uploaded runs
+(`check_regression`), and replaying a specific past run's exact chaos
+sequence against a new agent version (`replay_run`) — plus a worked
+end-to-end CI regression-gate example combining all four.
+
+## CLI
+
+```bash
+agentprobe run [options]
+```
+
+Key options (`agentprobe run --help` for the full list):
+
+| Flag | What it controls |
+|---|---|
+| `--domain {ticket,access_control}` | Which business domain's world/tools/scenarios to run against |
+| `--target {claude,scripted,rule_based}` | `claude`: real Claude tool-calling loop. `scripted`: no-op stub for exercising the harness. `rule_based`: a real deterministic decision tree, zero API cost |
+| `--injector {none,hardcoded,model,replay}` | `hardcoded`: fires one kind on a named tool at a fixed offset, zero cost. `model`: a live Claude Injector reasoning about the trajectory. `replay`: re-fires a previously recorded sequence for free |
+| `--mode {robustness,recovery}` | `robustness`: a clean control run + a chaos run from the same seed. `recovery`: one run, chaos mid-trajectory, no control |
+| `--scenario ID` | Run a single scenario instead of the whole registry |
+| `--list-scenarios` | Print every scenario in `--domain` as JSON and exit without running anything -- for building a CI test matrix |
+| `--classify` | Run the response classifier (HANDLED/IGNORED/MISREAD/OVERREACTED/LOOPED) on every valid injection |
+| `--max-total-cost-usd N` | Stop starting new scenarios once combined Target + Injector spend for this invocation reaches `N` -- a whole-run safety net, separate from each scenario's own much smaller `max_cost_usd` |
+| `--detailed` | Print the full scenario-by-scenario, step-by-step breakdown to the terminal |
+| `--html-out PATH` | Save a self-contained HTML report (always includes the step-by-step detail, collapsed per scenario) |
+| `--json-out PATH` | Save the full trajectory data, for `agentprobe diff` later |
+| `--use-playbook` | Learn from and contribute to a local cross-run history of which triggers actually fire (see "Data moat" below) |
+
+Other subcommands: `agentprobe diff before.json after.json
+--fail-on-regression` (CI gate), `agentprobe score-agreement
+labeled.json` (classifier hand-labeling agreement), `agentprobe
+apply-feedback feedback.json` (fold human feedback into the playbook).
+
+## Domains
+
+A **domain** is a business world (entities + tools + business rules) the
+Injector and Target both operate on. This repo ships two:
+
+- **ticket** (default): customer support tickets — refund/close/reply/
+  escalate over tickets, orders, customers.
+- **access_control**: employees requesting access to internal systems —
+  grant/close/reply/escalate over requests, resources, employees.
+
+The actual chaos-injection engine (trigger timing, the six injection
+kinds, the reachability checker, the Playbook) doesn't know or care which
+domain it's running — see [`agentprobe/domain.py`](agentprobe/domain.py)
+for the `Domain` bundle that's the seam between them, and
+[`agentprobe/domains/access_control/`](agentprobe/domains/access_control/)
+for a complete worked second example (entities, toolkit, scenarios, a
+live Claude agent, a free rule-based agent, and its own Injector system
+prompt). Adding a third domain means writing those same pieces for your
+own world — the engine underneath is already generic.
+
+## Generating a domain instead of writing one
+
+Writing `agentprobe/domain.py`'s `Domain` bundle by hand (entities, tools,
+business rules) is real, unavoidable work — see "Domains" above. If you'd
+rather not write it yourself, [agentprobe-api.agentprobe.workers.dev](https://agentprobe-api.agentprobe.workers.dev)
+can generate one from your existing tool schemas plus a plain-English
+description of your business rules, and hand you back a key:
+
+```python
+import agentprobe as ap
+
+fetched = ap.fetch_domain("joeblow-ai-agents_bd0725d86767c7481581fc41")
+report = ap.quick_test(
+    fetched.scenarios["wrong-order-among-several"],
+    lambda: ap.TargetAgent(system_prompt="You are JoeBlow's order support agent."),
+    domain=fetched.domain,
+)
+print(report.render())
+```
+
+**Trust disclaimer — read this before using `fetch_domain`:** the domain
+behind a key is Python code an LLM generated from your description, not
+code AgentProbe's authors reviewed. `fetch_domain` downloads and *executes*
+that code locally, and prints a warning with a `review_url` every time it
+runs — open that link and skim the generated business logic (what counts
+as "already refunded," what's irreversible, what a precondition checks)
+before trusting any report it produces. A subtly wrong rule doesn't error;
+it just makes your test pass without proving anything real about your
+agent. Only fetch a key you generated for your own business, the same
+rule as never installing a package from someone you don't trust.
+
+## The website: accounts, personal API keys, and a dashboard
+
+[agentprobe-api.agentprobe.workers.dev](https://agentprobe-api.agentprobe.workers.dev)
+is the real site, not just the API — sign in with GitHub and you get:
+
+- **Your Keys** — every domain you've generated (via `/v1/generate` with
+  your API key attached), click-to-copy, so you don't have to keep your
+  own notes of which key was which business.
+- **Your Runs** — a history of test runs you've uploaded, each with its
+  full report viewable in the browser.
+- **API access** (on the Dashboard) — create a personal key
+  (`ap_live_...`, shown once at creation) to authenticate the two things
+  above from your own scripts.
+
+Attribute a generated domain to your account by passing the key as a
+bearer token:
+
+```bash
+curl -X POST https://agentprobe-api.agentprobe.workers.dev/v1/generate \
+  -H "Authorization: Bearer ap_live_..." \
+  -H "content-type: application/json" \
+  -d '{"name": "...", "tool_schemas": [...], "business_description": "..."}'
+```
+
+And upload a finished run so it shows up under Your Runs by passing the
+same key to `quick_test()`:
+
+```python
+report = ap.quick_test(
+    scenario, my_agent, domain=fetched.domain,
+    upload_api_key="ap_live_...",       # from Your Dashboard
+    domain_key=fetched_key,             # optional -- links the run back to its domain
+)
+```
+
+Both are entirely optional — omit `upload_api_key` (the default) and
+`quick_test()` stays fully local, same as always. See
+[`server/README.md`](server/README.md) for how the site itself is built
+and deployed (Cloudflare Worker + D1 + GitHub OAuth).
+
+## Cost
+
+Nothing about this harness requires API spend to develop or test against:
+`--target scripted` / `--target rule_based` and `--injector none` /
+`--injector hardcoded` make zero network calls between them. Live testing
+(`--target claude`, `--injector model`, `--classify`) uses the real
+Anthropic API and costs real money — check `agentprobe/llm.py`'s pricing
+table for current per-model rates, and a run's own report always prints
+its actual cost breakdown (target/injector/classifier/total).
+
+## Data moat
+
+`--use-playbook` accumulates cross-run outcome data (which trigger
+actually fires for a given injection kind and scenario shape) into a
+local JSON file, surfaced back into the Injector's prompt as an advisory
+hint on future runs — see [`agentprobe/playbook.py`](agentprobe/playbook.py).
+`--export-for-feedback` / `agentprobe apply-feedback` let a human add
+free-text judgment (what was good, what should change, what was missing)
+about a run's injections into the same file — see
+[`agentprobe/feedback.py`](agentprobe/feedback.py). Both are keyed by
+structural scenario shape, not by scenario/customer content, so they
+don't touch a BYOK customer's actual data isolation.
+
+## Testing
+
+```bash
+pytest tests/ -q
+```
+
+600+ tests, all free — no test in this suite makes a live API call
+(anything that would is mocked). Contributions should keep it that way.
+
+The website/API server (`server/`) has its own separate suite, same rule:
+
+```bash
+cd server && node --test test/
+```
+
+90+ tests against fake D1/KV/fetch — no real Cloudflare account, network
+access, or secrets needed for either suite. Both run in CI on every push.
+
+See [CHANGELOG.md](CHANGELOG.md) for what's new in each version.

agentprobe_testing-0.5.0/agentprobe/__init__.py

@@ -0,0 +1,104 @@
+"""AgentProbe: chaos-testing for LLM agents, via a second agent (the
+Injector) that watches a Target's live trajectory and adaptively decides
+where to break something -- see README.md for the full pitch.
+
+This re-exports the pieces most programs actually need so a simple case
+doesn't require knowing the submodule layout:
+
+    import agentprobe as ap
+
+    injector = ap.HardcodedToolErrorInjector(offsets=[0], tool_name="close_ticket")
+    clean, chaos = ap.run_robustness_pair(
+        ap.TICKET_SCENARIOS_BY_ID["clean-1"],
+        target_factory=lambda: ap.RuleBasedAgent(),  # zero API cost
+        injector=injector,
+    )
+    print(ap.Report(
+        mode="robustness", injector_model="hardcoded", target_model="rule_based",
+        clean_trajectories=[clean], chaos_trajectories=[chaos],
+    ).render())
+
+Everything here is free (RuleBasedAgent + HardcodedToolErrorInjector make
+no network calls at all). Swap in TargetAgent/ModelInjector for the real,
+billed thing.
+
+A second domain (access-control) ships in agentprobe.domains.access_control
+rather than being re-exported here, to keep this top-level surface to the
+one domain most people reach for first -- see domain.py for the seam that
+lets you plug in a third.
+"""
+
+from __future__ import annotations
+
+from agentprobe.agents.base import Agent, AgentAction
+from agentprobe.agents.rule_based import RuleBasedAgent
+from agentprobe.agents.scripted import ScriptedAgent
+from agentprobe.agents.target_agent import TargetAgent
+from agentprobe.domain import Domain, TICKET_DOMAIN
+from agentprobe.injection import AppliedInjection, Injection, InjectionKind, injection_was_triggered
+from agentprobe.injector import (
+    HardcodedToolErrorInjector,
+    Injector,
+    ModelInjector,
+    NullInjector,
+    RecordingInjector,
+    ReplayInjector,
+)
+from agentprobe.quickstart import assert_passes, quick_test, quick_test_all, wrap_agent
+from agentprobe.registry import (
+    FetchedDomain,
+    RegressionResult,
+    check_regression,
+    fetch_domain,
+    get_latest_run,
+    replay_run,
+    upload_run,
+)
+from agentprobe.report import InjectionRecord, Report
+from agentprobe.runner import run_recovery, run_robustness_pair
+from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
+from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
+from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
+
+__version__ = "0.5.0"
+
+__all__ = [
+    "Agent",
+    "AgentAction",
+    "AppliedInjection",
+    "CommitPattern",
+    "Domain",
+    "FactPattern",
+    "FetchedDomain",
+    "GoalSpec",
+    "HardcodedToolErrorInjector",
+    "Injection",
+    "InjectionKind",
+    "InjectionRecord",
+    "Injector",
+    "ModelInjector",
+    "NullInjector",
+    "RecordingInjector",
+    "RegressionResult",
+    "ReplayInjector",
+    "Report",
+    "RuleBasedAgent",
+    "Scenario",
+    "ScriptedAgent",
+    "TICKET_DOMAIN",
+    "TICKET_SCENARIOS",
+    "TICKET_SCENARIOS_BY_ID",
+    "TargetAgent",
+    "assert_passes",
+    "check_regression",
+    "fetch_domain",
+    "get_latest_run",
+    "injection_was_triggered",
+    "quick_test",
+    "quick_test_all",
+    "replay_run",
+    "run_recovery",
+    "run_robustness_pair",
+    "upload_run",
+    "wrap_agent",
+]

agentprobe_testing-0.5.0/agentprobe/agents/base.py

@@ -0,0 +1,32 @@
+"""Agent interface the runner drives. An agent is a small state machine:
+start a task, propose an action, observe the result, repeat."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Literal, Optional
+
+
+@dataclass
+class AgentAction:
+    kind: Literal["tool_call", "final_answer"]
+    tool_name: Optional[str] = None
+    tool_args: Optional[dict[str, Any]] = None
+    text: Optional[str] = None
+    latency_s: float = 0.0
+    cost_usd: float = 0.0
+
+
+class Agent(ABC):
+    @abstractmethod
+    def start(self, task: str, tool_schemas: list[dict]) -> None:
+        """Reset internal state for a new scenario."""
+
+    @abstractmethod
+    def next_action(self) -> AgentAction:
+        """Propose the next step: a tool call or a final answer."""
+
+    @abstractmethod
+    def observe(self, tool_name: str, tool_args: dict, result: Any, ok: bool) -> None:
+        """Feed back the result of the tool call just executed."""