agentprobe-testing 0.5.1__tar.gz → 0.5.2__tar.gz

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.5.1
+Version: 0.5.2
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.2}/README.md

@@ -185,7 +185,10 @@ Writing `agentprobe/domain.py`'s `Domain` bundle by hand (entities, tools,
 business rules) is real, unavoidable work — see "Domains" above. If you'd
 rather not write it yourself, [agentprobe-api.agentprobe.workers.dev](https://agentprobe-api.agentprobe.workers.dev)
 can generate one from your existing tool schemas plus a plain-English
-description of your business rules, and hand you back a key:
+description of your business rules, and hand you back a key — either via
+[/generate](https://agentprobe-api.agentprobe.workers.dev/generate) on the
+site (sign in, fill in a form, get the key and the generated source back
+right on the page — no `curl` needed) or the raw API:
 
 ```python
 import agentprobe as ap
@@ -223,6 +226,19 @@ is the real site, not just the API — sign in with GitHub and you get:
 - **API access** (on the Dashboard) — create a personal key
   (`ap_live_...`, shown once at creation) to authenticate the two things
   above from your own scripts.
+- **`/generate`** — the same domain generation as `POST /v1/generate`,
+  as a web form instead of `curl` — fill it in, get back the key and the
+  generated source to review right on the page.
+- **`/run`** — execute a real chaos test from the website itself, no
+  terminal needed: paste a domain key, pick a scenario from a real
+  dropdown (or "All scenarios" to run every one, combined into a single
+  report), point at your agent's HTTP endpoint and your own Anthropic API
+  key, and it runs in the background — the page comes back immediately so
+  you can go do something else on the site while it finishes, and it
+  shows up under Your Runs once it does. Free to use — you're billed by
+  Anthropic for your own API usage, the same as running it yourself. See
+  [`runner/README.md`](runner/README.md) for how this actually executes server-side (the real `agentprobe-testing`
+  package, not a reimplementation).
 
 Attribute a generated domain to your account by passing the key as a
 bearer token:
@@ -288,7 +304,7 @@ The website/API server (`server/`) has its own separate suite, same rule:
 cd server && node --test test/
 ```
 
-90+ tests against fake D1/KV/fetch — no real Cloudflare account, network
+160+ tests against fake D1/KV/fetch — no real Cloudflare account, network
 access, or secrets needed for either suite. Both run in CI on every push.
 
 See [CHANGELOG.md](CHANGELOG.md) for what's new in each version.

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.2}/agentprobe/__init__.py

@@ -60,7 +60,7 @@ from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "0.5.1"
+__version__ = "0.5.2"
 
 __all__ = [
     "Agent",

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.2}/agentprobe/quickstart.py

@@ -10,6 +10,7 @@ what to do next, test it against one scenario."
 from __future__ import annotations
 
 import json
+import time
 import urllib.request
 from typing import Any, Callable, Optional
 
@@ -209,6 +210,7 @@ def quick_test_all(
     upload_api_key: Optional[str] = None,
     domain_key: Optional[str] = None,
     record_injections: bool = False,
+    max_duration_seconds: Optional[float] = None,
 ) -> Report:
     """Run every scenario in `scenarios` (a dict[str, Scenario] -- e.g.
     TICKET_SCENARIOS or a FetchedDomain's .scenarios -- or a plain list of
@@ -228,6 +230,16 @@ def quick_test_all(
     run (so Your Runs shows them individually, the same granularity as
     calling quick_test() per scenario and uploading each) -- the Report
     this function returns is still the combined summary across all of them.
+
+    `max_duration_seconds`, if given, stops starting new scenarios once
+    that much wall-clock time has passed since the call began -- for a
+    caller with its own hard execution ceiling (e.g. a serverless platform
+    that kills a job outright past some limit, which can't be caught or
+    cleaned up after), a report covering fewer scenarios than asked for is
+    far better than getting killed mid-scenario with nothing to show for
+    it. The returned Report's `_scenarios_run`/`_scenarios_total`
+    attributes say whether this happened, so a caller can tell the
+    difference between "ran everything" and "stopped early."
     """
     items = list(scenarios.items()) if isinstance(scenarios, dict) else [(s.id, s) for s in scenarios]
     if not items:
@@ -246,8 +258,13 @@ def quick_test_all(
     all_records: list[InjectionRecord] = []
     total_injector_cost = 0.0
     combined_injector_model: Optional[str] = None
+    start = time.monotonic()
+    scenarios_run = 0
 
     for _scenario_id, scenario in items:
+        if max_duration_seconds is not None and scenarios_run > 0 and (time.monotonic() - start) >= max_duration_seconds:
+            break  # already have at least one scenario's worth of real results -- stop here, not mid-scenario
+        scenarios_run += 1
         injector = make_injector()
         resolved_injector_model = injector_model if injector_model is not None else type(injector).__name__
         if combined_injector_model is None:
@@ -289,7 +306,7 @@ def quick_test_all(
             recorded = [serialize_armed_injection(a) for a in recording.recorded] if recording is not None else None
             upload_run(scenario_report, api_key=upload_api_key, domain_key=domain_key, recorded_injections=recorded)
 
-    return Report(
+    report = Report(
         mode=mode,
         injector_model=combined_injector_model or "none",
         target_model=target_model,
@@ -298,6 +315,9 @@ def quick_test_all(
         injection_records=all_records,
         injector_cost_usd=total_injector_cost,
     )
+    report._scenarios_run = len(all_chaos)
+    report._scenarios_total = len(items)
+    return report
 
 
 def assert_passes(report: Report, save_html_on_failure: Optional[str] = None) -> None:

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.2}/agentprobe_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.5.1
+Version: 0.5.2
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentprobe-testing"
-version = "0.5.1"
+version = "0.5.2"
 description = "Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance."
 requires-python = ">=3.11"
 dependencies = ["anthropic>=1.0.0", "python-dotenv>=1.0.0"]

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.2}/tests/test_quickstart.py

@@ -273,6 +273,29 @@ def test_quick_test_all_recovery_mode_has_no_clean_trajectories():
     assert len(report.chaos_trajectories) == 2
 
 
+def test_quick_test_all_reports_scenarios_run_and_total_when_nothing_stops_it_early():
+    scenarios = {"clean-1": BY_ID["clean-1"], "clean-2": BY_ID["clean-2"]}
+    report = quick_test_all(scenarios, lambda: RuleBasedAgent())
+    assert report._scenarios_run == 2
+    assert report._scenarios_total == 2
+
+
+def test_quick_test_all_stops_early_past_max_duration_seconds_but_always_runs_at_least_one():
+    # A caller with its own hard execution ceiling (e.g. a serverless
+    # platform that kills the whole job outright past some limit, with no
+    # chance to clean up) needs a report covering fewer scenarios instead
+    # of getting killed mid-run with nothing to show. max_duration_seconds=0
+    # means "already over budget" as soon as any real time has passed --
+    # deterministic regardless of how fast RuleBasedAgent actually runs,
+    # since real wall-clock time (time.monotonic()) always advances by a
+    # nonzero amount between the loop's start and its first elapsed check.
+    scenarios = {"clean-1": BY_ID["clean-1"], "clean-2": BY_ID["clean-2"], "clean-1-again": BY_ID["clean-1"]}
+    report = quick_test_all(scenarios, lambda: RuleBasedAgent(), max_duration_seconds=0)
+    assert report._scenarios_run == 1  # never zero -- see the "always run at least one" guard
+    assert report._scenarios_total == 3
+    assert report._compute()["n"] == 1  # the returned Report itself only covers what actually ran
+
+
 def test_quick_test_all_uploads_each_scenario_as_its_own_run():
     import agentprobe.registry as registry_module