agentprobe-testing 0.5.1__tar.gz → 0.5.3__tar.gz

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.5.1
+Version: 0.5.3
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.3}/README.md

@@ -185,15 +185,18 @@ Writing `agentprobe/domain.py`'s `Domain` bundle by hand (entities, tools,
 business rules) is real, unavoidable work — see "Domains" above. If you'd
 rather not write it yourself, [agentprobe-api.agentprobe.workers.dev](https://agentprobe-api.agentprobe.workers.dev)
 can generate one from your existing tool schemas plus a plain-English
-description of your business rules, and hand you back a key:
+description of your business rules, and hand you back a key — either via
+[/generate](https://agentprobe-api.agentprobe.workers.dev/generate) on the
+site (sign in, fill in a form, get the key and the generated source back
+right on the page — no `curl` needed) or the raw API:
 
 ```python
 import agentprobe as ap
 
-fetched = ap.fetch_domain("joeblow-ai-agents_bd0725d86767c7481581fc41")
+fetched = ap.fetch_domain("meridian-ai-agents_bd0725d86767c7481581fc41")
 report = ap.quick_test(
     fetched.scenarios["wrong-order-among-several"],
-    lambda: ap.TargetAgent(system_prompt="You are JoeBlow's order support agent."),
+    lambda: ap.TargetAgent(system_prompt="You are Meridian's order support agent."),
     domain=fetched.domain,
 )
 print(report.render())
@@ -210,19 +213,34 @@ it just makes your test pass without proving anything real about your
 agent. Only fetch a key you generated for your own business, the same
 rule as never installing a package from someone you don't trust.
 
-## The website: accounts, personal API keys, and a dashboard
+## The website: sign in and run a real test in your browser
 
 [agentprobe-api.agentprobe.workers.dev](https://agentprobe-api.agentprobe.workers.dev)
 is the real site, not just the API — sign in with GitHub and you get:
 
-- **Your Keys** — every domain you've generated (via `/v1/generate` with
-  your API key attached), click-to-copy, so you don't have to keep your
-  own notes of which key was which business.
-- **Your Runs** — a history of test runs you've uploaded, each with its
-  full report viewable in the browser.
-- **API access** (on the Dashboard) — create a personal key
-  (`ap_live_...`, shown once at creation) to authenticate the two things
-  above from your own scripts.
+- **`/run`** — one flow, describing to running: describe your business in
+  plain English (same generation as `POST /v1/generate`, as a web form
+  instead of `curl`), and it writes a real test domain for it, then falls
+  straight into picking a scenario from a real dropdown (or "All
+  scenarios" to run every one, combined into a single report), naming the
+  run, and pointing at your agent's HTTP endpoint plus your own Anthropic
+  API key. It runs in the background — the page comes back immediately so
+  you can go do something else on the site while it finishes, and it
+  shows up under Your Runs once it does. Free to use — you're billed by
+  Anthropic for your own API usage, the same as running it yourself. See
+  [`runner/README.md`](runner/README.md) for how this actually executes server-side (the real `agentprobe-testing`
+  package, not a reimplementation).
+- **Your Runs** — a history of test runs you've uploaded or executed,
+  each with its full report viewable in the browser, and a Rerun link
+  that jumps straight back into `/run`'s scenario/agent step for any run
+  that was started from the site.
+
+There's no separate domain-key page on the site anymore — a domain
+generated through `/run` is only ever referenced internally, not
+something you need to copy around. A personal API key
+(`ap_live_...`) for scripting the CLI/library against your account, if
+your account has one, is issued separately from the website (there's no
+self-serve "create a key" page).
 
 Attribute a generated domain to your account by passing the key as a
 bearer token:
@@ -240,7 +258,7 @@ same key to `quick_test()`:
 ```python
 report = ap.quick_test(
     scenario, my_agent, domain=fetched.domain,
-    upload_api_key="ap_live_...",       # from Your Dashboard
+    upload_api_key="ap_live_...",       # a personal key issued for your account
     domain_key=fetched_key,             # optional -- links the run back to its domain
 )
 ```
@@ -288,7 +306,7 @@ The website/API server (`server/`) has its own separate suite, same rule:
 cd server && node --test test/
 ```
 
-90+ tests against fake D1/KV/fetch — no real Cloudflare account, network
+160+ tests against fake D1/KV/fetch — no real Cloudflare account, network
 access, or secrets needed for either suite. Both run in CI on every push.
 
 See [CHANGELOG.md](CHANGELOG.md) for what's new in each version.

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.3}/agentprobe/__init__.py

@@ -60,7 +60,7 @@ from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "0.5.1"
+__version__ = "0.5.3"
 
 __all__ = [
     "Agent",

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.3}/agentprobe/quickstart.py

@@ -10,6 +10,7 @@ what to do next, test it against one scenario."
 from __future__ import annotations
 
 import json
+import time
 import urllib.request
 from typing import Any, Callable, Optional
 
@@ -135,7 +136,7 @@ def quick_test(
     or a plain decide_fn(task, history) -> AgentAction, which gets wrap_agent()
     applied automatically -- either way, nothing else here changes.
 
-    Pass `upload_api_key` (a personal key from /dashboard) to also upload the
+    Pass `upload_api_key` (a personal key issued for your account) to also upload the
     finished report to your AgentProbe account, so it shows up under Your
     Runs -- see registry.upload_run for what this does and its failure
     behavior. Pass `domain_key` too if `domain` came from fetch_domain(), so
@@ -209,6 +210,7 @@ def quick_test_all(
     upload_api_key: Optional[str] = None,
     domain_key: Optional[str] = None,
     record_injections: bool = False,
+    max_duration_seconds: Optional[float] = None,
 ) -> Report:
     """Run every scenario in `scenarios` (a dict[str, Scenario] -- e.g.
     TICKET_SCENARIOS or a FetchedDomain's .scenarios -- or a plain list of
@@ -228,6 +230,16 @@ def quick_test_all(
     run (so Your Runs shows them individually, the same granularity as
     calling quick_test() per scenario and uploading each) -- the Report
     this function returns is still the combined summary across all of them.
+
+    `max_duration_seconds`, if given, stops starting new scenarios once
+    that much wall-clock time has passed since the call began -- for a
+    caller with its own hard execution ceiling (e.g. a serverless platform
+    that kills a job outright past some limit, which can't be caught or
+    cleaned up after), a report covering fewer scenarios than asked for is
+    far better than getting killed mid-scenario with nothing to show for
+    it. The returned Report's `_scenarios_run`/`_scenarios_total`
+    attributes say whether this happened, so a caller can tell the
+    difference between "ran everything" and "stopped early."
     """
     items = list(scenarios.items()) if isinstance(scenarios, dict) else [(s.id, s) for s in scenarios]
     if not items:
@@ -246,8 +258,13 @@ def quick_test_all(
     all_records: list[InjectionRecord] = []
     total_injector_cost = 0.0
     combined_injector_model: Optional[str] = None
+    start = time.monotonic()
+    scenarios_run = 0
 
     for _scenario_id, scenario in items:
+        if max_duration_seconds is not None and scenarios_run > 0 and (time.monotonic() - start) >= max_duration_seconds:
+            break  # already have at least one scenario's worth of real results -- stop here, not mid-scenario
+        scenarios_run += 1
         injector = make_injector()
         resolved_injector_model = injector_model if injector_model is not None else type(injector).__name__
         if combined_injector_model is None:
@@ -289,7 +306,7 @@ def quick_test_all(
             recorded = [serialize_armed_injection(a) for a in recording.recorded] if recording is not None else None
             upload_run(scenario_report, api_key=upload_api_key, domain_key=domain_key, recorded_injections=recorded)
 
-    return Report(
+    report = Report(
         mode=mode,
         injector_model=combined_injector_model or "none",
         target_model=target_model,
@@ -298,6 +315,9 @@ def quick_test_all(
         injection_records=all_records,
         injector_cost_usd=total_injector_cost,
     )
+    report._scenarios_run = len(all_chaos)
+    report._scenarios_total = len(items)
+    return report
 
 
 def assert_passes(report: Report, save_html_on_failure: Optional[str] = None) -> None:

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.3}/agentprobe/registry.py

@@ -125,11 +125,11 @@ def upload_run(
     recorded_injections: Optional[list[dict]] = None,
 ) -> None:
     """Upload a finished quick_test() report to your AgentProbe account (POST
-    /v1/runs) so it shows up under Your Runs on the dashboard. `api_key` is a
-    personal key created on /dashboard -- create one there, it's never shown
-    again after creation so save it somewhere real (an env var, not a
-    committed file). `report` must cover exactly one scenario, same as every
-    report quick_test() itself produces.
+    /v1/runs) so it shows up under Your Runs on the website. `api_key` is a
+    personal key issued for your account -- it's never shown again after
+    issuance so save it somewhere real (an env var, not a committed file).
+    `report` must cover exactly one scenario, same as every report
+    quick_test() itself produces.
 
     Pass `recorded_injections` (a list of agentprobe.injector.
     serialize_armed_injection() dicts) to make the uploaded run replayable
@@ -274,7 +274,7 @@ def replay_run(
     target_model: str = "custom",
 ) -> "Report":
     """Replay the exact chaos sequence recorded for `run_id` (an id from
-    your dashboard's Your Runs table, or a RegressionResult.previous_run_id)
+    the website's Your Runs table, or a RegressionResult.previous_run_id)
     against a new agent or model version, instead of paying for a fresh
     ModelInjector decision sequence -- only the Target's own calls repeat.
 

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.3}/agentprobe/report.py

@@ -564,14 +564,14 @@ class Report:
     {classification_html}
   </section>
   <section>
-    <h2>By injection kind &mdash; handled rate</h2>
+    <h2>By injection kind: handled rate</h2>
     {kind_table}
   </section>
   <section>
-    <h2>Scenario detail &mdash; step by step</h2>
+    <h2>Scenario detail: step by step</h2>
     {scenario_detail_html}
   </section>
-  <footer>Generated by AgentProbe &mdash; adaptive chaos-testing for LLM agents.</footer>
+  <footer>Generated by AgentProbe. Adaptive chaos-testing for LLM agents.</footer>
 </div>
 </body></html>"""
 
@@ -621,7 +621,7 @@ class Report:
             notes = ""
             for a in traj.injections:
                 note_class = "injection-note" if a.valid else "injection-note injection-note--invalid"
-                validity = "" if a.valid else " &mdash; <strong>INVALID</strong>"
+                validity = "" if a.valid else " (<strong>INVALID</strong>)"
                 notes += (
                     f"<div class='{note_class}'>&#9889; injection fired: <strong>{esc(a.injection.kind.value)}</strong> "
                     f"@ step {a.fired_at_step} (trigger: <code>{esc(a.trigger_kind)}</code>){validity}"

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.3}/agentprobe_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.5.1
+Version: 0.5.3
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentprobe-testing"
-version = "0.5.1"
+version = "0.5.3"
 description = "Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance."
 requires-python = ">=3.11"
 dependencies = ["anthropic>=1.0.0", "python-dotenv>=1.0.0"]

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.3}/tests/test_quickstart.py

@@ -273,6 +273,29 @@ def test_quick_test_all_recovery_mode_has_no_clean_trajectories():
     assert len(report.chaos_trajectories) == 2
 
 
+def test_quick_test_all_reports_scenarios_run_and_total_when_nothing_stops_it_early():
+    scenarios = {"clean-1": BY_ID["clean-1"], "clean-2": BY_ID["clean-2"]}
+    report = quick_test_all(scenarios, lambda: RuleBasedAgent())
+    assert report._scenarios_run == 2
+    assert report._scenarios_total == 2
+
+
+def test_quick_test_all_stops_early_past_max_duration_seconds_but_always_runs_at_least_one():
+    # A caller with its own hard execution ceiling (e.g. a serverless
+    # platform that kills the whole job outright past some limit, with no
+    # chance to clean up) needs a report covering fewer scenarios instead
+    # of getting killed mid-run with nothing to show. max_duration_seconds=0
+    # means "already over budget" as soon as any real time has passed --
+    # deterministic regardless of how fast RuleBasedAgent actually runs,
+    # since real wall-clock time (time.monotonic()) always advances by a
+    # nonzero amount between the loop's start and its first elapsed check.
+    scenarios = {"clean-1": BY_ID["clean-1"], "clean-2": BY_ID["clean-2"], "clean-1-again": BY_ID["clean-1"]}
+    report = quick_test_all(scenarios, lambda: RuleBasedAgent(), max_duration_seconds=0)
+    assert report._scenarios_run == 1  # never zero -- see the "always run at least one" guard
+    assert report._scenarios_total == 3
+    assert report._compute()["n"] == 1  # the returned Report itself only covers what actually ran
+
+
 def test_quick_test_all_uploads_each_scenario_as_its_own_run():
     import agentprobe.registry as registry_module
 

{agentprobe_testing-0.5.1 → agentprobe_testing-0.5.3}/tests/test_registry.py

@@ -175,9 +175,9 @@ def test_upload_run_includes_domain_key_when_given():
     report = quick_test(scenario, _clean1_decide)
     poster = make_poster()
 
-    upload_run(report, api_key="ap_live_test123", domain_key="joeblow_1234", poster=poster)
+    upload_run(report, api_key="ap_live_test123", domain_key="meridian_1234", poster=poster)
 
-    assert poster.calls[0]["body"]["domain_key"] == "joeblow_1234"
+    assert poster.calls[0]["body"]["domain_key"] == "meridian_1234"
 
 
 def test_upload_run_rejects_a_report_covering_more_or_fewer_than_one_scenario():