agentprobe-testing 0.5.0__tar.gz → 0.5.2__tar.gz

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.5.0
+Version: 0.5.2
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.2}/README.md

@@ -18,6 +18,19 @@ adversarial content disguised as ordinary data (`PROMPT_INJECTION`).
 
 ## Quickstart
 
+Using this in your own project (no need to clone this repo):
+
+```bash
+pip install agentprobe-testing
+```
+
+`import agentprobe as ap` afterward, same as everywhere else in this
+README — the PyPI *distribution* name is `agentprobe-testing`, but the
+Python package you import is still plain `agentprobe`.
+
+To run the examples in this repo (or work on the library itself), clone
+it and install from source instead:
+
 ```bash
 pip install -e .
 python examples/free_zero_cost_demo.py   # no API key needed, no network calls at all
@@ -105,6 +118,12 @@ surface. The `access_control` domain lives at
 top level, to keep this surface to the one domain most people reach for
 first.
 
+Testing your own agent instead of the built-in `TargetAgent`? Pass any
+plain `(task, history) -> AgentAction` function as `agent` — no subclass
+needed, `quick_test`/`wrap_agent` handle the wiring, and if your agent
+already runs as a service you can POST to, `ap.http_agent(url)` builds
+that function for you (see [USAGE.md](USAGE.md)).
+
 Beyond one-scenario `quick_test()`, see [USAGE.md](USAGE.md) for: running
 a whole domain at once (`quick_test_all`), a one-line pytest integration
 (`assert_passes`), catching regressions between uploaded runs
@@ -166,7 +185,10 @@ Writing `agentprobe/domain.py`'s `Domain` bundle by hand (entities, tools,
 business rules) is real, unavoidable work — see "Domains" above. If you'd
 rather not write it yourself, [agentprobe-api.agentprobe.workers.dev](https://agentprobe-api.agentprobe.workers.dev)
 can generate one from your existing tool schemas plus a plain-English
-description of your business rules, and hand you back a key:
+description of your business rules, and hand you back a key — either via
+[/generate](https://agentprobe-api.agentprobe.workers.dev/generate) on the
+site (sign in, fill in a form, get the key and the generated source back
+right on the page — no `curl` needed) or the raw API:
 
 ```python
 import agentprobe as ap
@@ -204,6 +226,19 @@ is the real site, not just the API — sign in with GitHub and you get:
 - **API access** (on the Dashboard) — create a personal key
   (`ap_live_...`, shown once at creation) to authenticate the two things
   above from your own scripts.
+- **`/generate`** — the same domain generation as `POST /v1/generate`,
+  as a web form instead of `curl` — fill it in, get back the key and the
+  generated source to review right on the page.
+- **`/run`** — execute a real chaos test from the website itself, no
+  terminal needed: paste a domain key, pick a scenario from a real
+  dropdown (or "All scenarios" to run every one, combined into a single
+  report), point at your agent's HTTP endpoint and your own Anthropic API
+  key, and it runs in the background — the page comes back immediately so
+  you can go do something else on the site while it finishes, and it
+  shows up under Your Runs once it does. Free to use — you're billed by
+  Anthropic for your own API usage, the same as running it yourself. See
+  [`runner/README.md`](runner/README.md) for how this actually executes server-side (the real `agentprobe-testing`
+  package, not a reimplementation).
 
 Attribute a generated domain to your account by passing the key as a
 bearer token:
@@ -269,7 +304,7 @@ The website/API server (`server/`) has its own separate suite, same rule:
 cd server && node --test test/
 ```
 
-90+ tests against fake D1/KV/fetch — no real Cloudflare account, network
+160+ tests against fake D1/KV/fetch — no real Cloudflare account, network
 access, or secrets needed for either suite. Both run in CI on every push.
 
 See [CHANGELOG.md](CHANGELOG.md) for what's new in each version.

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.2}/agentprobe/__init__.py

@@ -44,7 +44,7 @@ from agentprobe.injector import (
     RecordingInjector,
     ReplayInjector,
 )
-from agentprobe.quickstart import assert_passes, quick_test, quick_test_all, wrap_agent
+from agentprobe.quickstart import assert_passes, http_agent, quick_test, quick_test_all, wrap_agent
 from agentprobe.registry import (
     FetchedDomain,
     RegressionResult,
@@ -60,7 +60,7 @@ from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "0.5.0"
+__version__ = "0.5.2"
 
 __all__ = [
     "Agent",
@@ -93,6 +93,7 @@ __all__ = [
     "check_regression",
     "fetch_domain",
     "get_latest_run",
+    "http_agent",
     "injection_was_triggered",
     "quick_test",
     "quick_test_all",

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.2}/agentprobe/quickstart.py

@@ -9,6 +9,9 @@ what to do next, test it against one scenario."
 
 from __future__ import annotations
 
+import json
+import time
+import urllib.request
 from typing import Any, Callable, Optional
 
 from agentprobe.agents.base import Agent, AgentAction
@@ -57,6 +60,64 @@ def wrap_agent(decide_fn: DecideFn) -> Callable[[], Agent]:
     return lambda: _FunctionAgent(decide_fn)
 
 
+HttpRequester = Callable[[str, dict, dict], dict]  # (url, body, headers) -> response dict
+
+
+def _default_http_requester(url: str, body: dict, headers: dict) -> dict:
+    data = json.dumps(body).encode("utf-8")
+    request = urllib.request.Request(
+        url,
+        data=data,
+        method="POST",
+        headers={"Content-Type": "application/json", "User-Agent": "agentprobe-client/0.5.0", **headers},
+    )
+    with urllib.request.urlopen(request, timeout=30) as resp:  # noqa: S310 -- caller-supplied url, same trust model as any target agent's own API
+        return json.loads(resp.read().decode("utf-8"))
+
+
+def http_agent(
+    url: str,
+    headers: Optional[dict] = None,
+    requester: Optional[HttpRequester] = None,
+) -> Callable[[], Agent]:
+    """Adapt a hosted HTTP agent into a target_factory for quick_test()/
+    run_robustness_pair() -- for when your real agent already runs as a
+    service you can POST to, instead of a Python function/object living in
+    this same process. No custom decide_fn to write: this builds one for
+    you and wraps it exactly like wrap_agent() does.
+
+    On every step, POSTs `{"task": <str>, "history": [...]}` as JSON to
+    `url` -- `history` is the same shape wrap_agent's decide_fn receives, a
+    list of `{"tool_name", "tool_args", "result", "ok"}` dicts covering
+    everything observed so far this run. Expects back JSON shaped like
+    either:
+
+        {"tool_name": "...", "tool_args": {...}}   -- call this tool next
+        {"final_answer": "..."}                     -- done, this is the reply
+
+    Raises ValueError if a response matches neither shape, rather than
+    guessing what was meant. Pass `headers` (e.g.
+    `{"Authorization": "Bearer ..."}`) for an endpoint that needs auth, and
+    `requester` in tests instead of hitting the real network -- see
+    tests/test_quickstart.py.
+    """
+    requester = requester if requester is not None else _default_http_requester
+    resolved_headers = headers or {}
+
+    def decide_fn(task: str, history: list[dict[str, Any]]) -> AgentAction:
+        response = requester(url, {"task": task, "history": history}, resolved_headers)
+        if "final_answer" in response:
+            return AgentAction(kind="final_answer", text=response["final_answer"])
+        if "tool_name" in response and "tool_args" in response:
+            return AgentAction(kind="tool_call", tool_name=response["tool_name"], tool_args=response["tool_args"])
+        raise ValueError(
+            f"http_agent: response from {url} must include either 'final_answer' or "
+            f"both 'tool_name' and 'tool_args', got keys {sorted(response.keys())}"
+        )
+
+    return wrap_agent(decide_fn)
+
+
 def quick_test(
     scenario: Scenario,
     agent: Callable[[], Agent] | DecideFn,
@@ -149,6 +210,7 @@ def quick_test_all(
     upload_api_key: Optional[str] = None,
     domain_key: Optional[str] = None,
     record_injections: bool = False,
+    max_duration_seconds: Optional[float] = None,
 ) -> Report:
     """Run every scenario in `scenarios` (a dict[str, Scenario] -- e.g.
     TICKET_SCENARIOS or a FetchedDomain's .scenarios -- or a plain list of
@@ -168,6 +230,16 @@ def quick_test_all(
     run (so Your Runs shows them individually, the same granularity as
     calling quick_test() per scenario and uploading each) -- the Report
     this function returns is still the combined summary across all of them.
+
+    `max_duration_seconds`, if given, stops starting new scenarios once
+    that much wall-clock time has passed since the call began -- for a
+    caller with its own hard execution ceiling (e.g. a serverless platform
+    that kills a job outright past some limit, which can't be caught or
+    cleaned up after), a report covering fewer scenarios than asked for is
+    far better than getting killed mid-scenario with nothing to show for
+    it. The returned Report's `_scenarios_run`/`_scenarios_total`
+    attributes say whether this happened, so a caller can tell the
+    difference between "ran everything" and "stopped early."
     """
     items = list(scenarios.items()) if isinstance(scenarios, dict) else [(s.id, s) for s in scenarios]
     if not items:
@@ -186,8 +258,13 @@ def quick_test_all(
     all_records: list[InjectionRecord] = []
     total_injector_cost = 0.0
     combined_injector_model: Optional[str] = None
+    start = time.monotonic()
+    scenarios_run = 0
 
     for _scenario_id, scenario in items:
+        if max_duration_seconds is not None and scenarios_run > 0 and (time.monotonic() - start) >= max_duration_seconds:
+            break  # already have at least one scenario's worth of real results -- stop here, not mid-scenario
+        scenarios_run += 1
         injector = make_injector()
         resolved_injector_model = injector_model if injector_model is not None else type(injector).__name__
         if combined_injector_model is None:
@@ -229,7 +306,7 @@ def quick_test_all(
             recorded = [serialize_armed_injection(a) for a in recording.recorded] if recording is not None else None
             upload_run(scenario_report, api_key=upload_api_key, domain_key=domain_key, recorded_injections=recorded)
 
-    return Report(
+    report = Report(
         mode=mode,
         injector_model=combined_injector_model or "none",
         target_model=target_model,
@@ -238,6 +315,9 @@ def quick_test_all(
         injection_records=all_records,
         injector_cost_usd=total_injector_cost,
     )
+    report._scenarios_run = len(all_chaos)
+    report._scenarios_total = len(items)
+    return report
 
 
 def assert_passes(report: Report, save_html_on_failure: Optional[str] = None) -> None:

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.2}/agentprobe_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.5.0
+Version: 0.5.2
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentprobe-testing"
-version = "0.5.0"
+version = "0.5.2"
 description = "Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance."
 requires-python = ">=3.11"
 dependencies = ["anthropic>=1.0.0", "python-dotenv>=1.0.0"]

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.2}/tests/test_quickstart.py

@@ -3,7 +3,7 @@ import pytest
 from agentprobe.agents.base import AgentAction
 from agentprobe.agents.rule_based import RuleBasedAgent
 from agentprobe.injector import HardcodedToolErrorInjector, Injector, NullInjector
-from agentprobe.quickstart import assert_passes, quick_test, quick_test_all, wrap_agent
+from agentprobe.quickstart import assert_passes, http_agent, quick_test, quick_test_all, wrap_agent
 from agentprobe.report import Report
 from agentprobe.scenarios.registry import BY_ID
 
@@ -52,6 +52,81 @@ def test_wrap_agent_gives_each_call_its_own_history_not_shared_across_instances(
     assert action.tool_name == "get_ticket"  # fresh history, not polluted by `first`
 
 
+def make_requester(responses):
+    """Returns each queued response in order, on every call -- the same
+    scripted-sequence pattern ScriptedAgent uses, since http_agent's whole
+    point is standing in for a real remote agent step by step."""
+    calls = []
+    responses = list(responses)
+
+    def requester(url, body, headers):
+        calls.append({"url": url, "body": body, "headers": headers})
+        return responses.pop(0)
+
+    requester.calls = calls
+    return requester
+
+
+def test_http_agent_drives_a_full_run_from_scripted_json_responses():
+    # mode="recovery" -- a robustness-mode run calls the target_factory
+    # twice (clean then chaos), which would drain this same scripted
+    # response queue twice over; recovery mode's single trajectory keeps
+    # this test's response list simple and matched 1:1 to real steps.
+    scenario = BY_ID["clean-1"]
+    requester = make_requester(
+        [
+            {"tool_name": "get_ticket", "tool_args": {"id": "T-1001"}},
+            {"tool_name": "issue_refund", "tool_args": {"ticket_id": "T-1001", "amount": 49.99}},
+            {"tool_name": "close_ticket", "tool_args": {"ticket_id": "T-1001", "resolution": "refunded"}},
+            {"final_answer": "done"},
+        ]
+    )
+
+    report = quick_test(scenario, http_agent("https://example.com/agent", requester=requester), injector=NullInjector(), mode="recovery")
+
+    assert report._compute()["chaos_passed"] == 1
+    assert requester.calls[0]["url"] == "https://example.com/agent"
+
+
+def test_http_agent_sends_task_and_history_as_the_request_body():
+    scenario = BY_ID["clean-1"]
+    requester = make_requester(
+        [
+            {"tool_name": "get_ticket", "tool_args": {"id": "T-1001"}},
+            {"final_answer": "done"},
+        ]
+    )
+
+    quick_test(scenario, http_agent("https://example.com/agent", requester=requester), injector=NullInjector(), mode="recovery")
+
+    first_call, second_call = requester.calls
+    assert first_call["body"]["task"] == "Handle ticket T-1001."
+    assert first_call["body"]["history"] == []
+    assert second_call["body"]["history"][0]["tool_name"] == "get_ticket"
+
+
+def test_http_agent_passes_through_custom_headers():
+    scenario = BY_ID["clean-1"]
+    requester = make_requester([{"final_answer": "done"}])
+
+    quick_test(
+        scenario,
+        http_agent("https://example.com/agent", headers={"Authorization": "Bearer secret"}, requester=requester),
+        injector=NullInjector(),
+        mode="recovery",
+    )
+
+    assert requester.calls[0]["headers"] == {"Authorization": "Bearer secret"}
+
+
+def test_http_agent_raises_a_clear_error_on_a_response_matching_neither_shape():
+    scenario = BY_ID["clean-1"]
+    requester = make_requester([{"something_else": "not a valid response"}])
+
+    with pytest.raises(ValueError, match="must include either 'final_answer' or"):
+        quick_test(scenario, http_agent("https://example.com/agent", requester=requester), injector=NullInjector(), mode="recovery")
+
+
 def test_quick_test_runs_a_plain_function_end_to_end_and_reports_pass():
     scenario = BY_ID["clean-1"]
     report = quick_test(scenario, _clean1_decide, injector=NullInjector())
@@ -198,6 +273,29 @@ def test_quick_test_all_recovery_mode_has_no_clean_trajectories():
     assert len(report.chaos_trajectories) == 2
 
 
+def test_quick_test_all_reports_scenarios_run_and_total_when_nothing_stops_it_early():
+    scenarios = {"clean-1": BY_ID["clean-1"], "clean-2": BY_ID["clean-2"]}
+    report = quick_test_all(scenarios, lambda: RuleBasedAgent())
+    assert report._scenarios_run == 2
+    assert report._scenarios_total == 2
+
+
+def test_quick_test_all_stops_early_past_max_duration_seconds_but_always_runs_at_least_one():
+    # A caller with its own hard execution ceiling (e.g. a serverless
+    # platform that kills the whole job outright past some limit, with no
+    # chance to clean up) needs a report covering fewer scenarios instead
+    # of getting killed mid-run with nothing to show. max_duration_seconds=0
+    # means "already over budget" as soon as any real time has passed --
+    # deterministic regardless of how fast RuleBasedAgent actually runs,
+    # since real wall-clock time (time.monotonic()) always advances by a
+    # nonzero amount between the loop's start and its first elapsed check.
+    scenarios = {"clean-1": BY_ID["clean-1"], "clean-2": BY_ID["clean-2"], "clean-1-again": BY_ID["clean-1"]}
+    report = quick_test_all(scenarios, lambda: RuleBasedAgent(), max_duration_seconds=0)
+    assert report._scenarios_run == 1  # never zero -- see the "always run at least one" guard
+    assert report._scenarios_total == 3
+    assert report._compute()["n"] == 1  # the returned Report itself only covers what actually ran
+
+
 def test_quick_test_all_uploads_each_scenario_as_its_own_run():
     import agentprobe.registry as registry_module