agentprobe-testing 0.5.0__tar.gz → 0.5.1__tar.gz

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.5.0
+Version: 0.5.1
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.1}/README.md

@@ -18,6 +18,19 @@ adversarial content disguised as ordinary data (`PROMPT_INJECTION`).
 
 ## Quickstart
 
+Using this in your own project (no need to clone this repo):
+
+```bash
+pip install agentprobe-testing
+```
+
+`import agentprobe as ap` afterward, same as everywhere else in this
+README — the PyPI *distribution* name is `agentprobe-testing`, but the
+Python package you import is still plain `agentprobe`.
+
+To run the examples in this repo (or work on the library itself), clone
+it and install from source instead:
+
 ```bash
 pip install -e .
 python examples/free_zero_cost_demo.py   # no API key needed, no network calls at all
@@ -105,6 +118,12 @@ surface. The `access_control` domain lives at
 top level, to keep this surface to the one domain most people reach for
 first.
 
+Testing your own agent instead of the built-in `TargetAgent`? Pass any
+plain `(task, history) -> AgentAction` function as `agent` — no subclass
+needed, `quick_test`/`wrap_agent` handle the wiring, and if your agent
+already runs as a service you can POST to, `ap.http_agent(url)` builds
+that function for you (see [USAGE.md](USAGE.md)).
+
 Beyond one-scenario `quick_test()`, see [USAGE.md](USAGE.md) for: running
 a whole domain at once (`quick_test_all`), a one-line pytest integration
 (`assert_passes`), catching regressions between uploaded runs

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.1}/agentprobe/__init__.py

@@ -44,7 +44,7 @@ from agentprobe.injector import (
     RecordingInjector,
     ReplayInjector,
 )
-from agentprobe.quickstart import assert_passes, quick_test, quick_test_all, wrap_agent
+from agentprobe.quickstart import assert_passes, http_agent, quick_test, quick_test_all, wrap_agent
 from agentprobe.registry import (
     FetchedDomain,
     RegressionResult,
@@ -60,7 +60,7 @@ from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 
 __all__ = [
     "Agent",
@@ -93,6 +93,7 @@ __all__ = [
     "check_regression",
     "fetch_domain",
     "get_latest_run",
+    "http_agent",
     "injection_was_triggered",
     "quick_test",
     "quick_test_all",

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.1}/agentprobe/quickstart.py

@@ -9,6 +9,8 @@ what to do next, test it against one scenario."
 
 from __future__ import annotations
 
+import json
+import urllib.request
 from typing import Any, Callable, Optional
 
 from agentprobe.agents.base import Agent, AgentAction
@@ -57,6 +59,64 @@ def wrap_agent(decide_fn: DecideFn) -> Callable[[], Agent]:
     return lambda: _FunctionAgent(decide_fn)
 
 
+HttpRequester = Callable[[str, dict, dict], dict]  # (url, body, headers) -> response dict
+
+
+def _default_http_requester(url: str, body: dict, headers: dict) -> dict:
+    data = json.dumps(body).encode("utf-8")
+    request = urllib.request.Request(
+        url,
+        data=data,
+        method="POST",
+        headers={"Content-Type": "application/json", "User-Agent": "agentprobe-client/0.5.0", **headers},
+    )
+    with urllib.request.urlopen(request, timeout=30) as resp:  # noqa: S310 -- caller-supplied url, same trust model as any target agent's own API
+        return json.loads(resp.read().decode("utf-8"))
+
+
+def http_agent(
+    url: str,
+    headers: Optional[dict] = None,
+    requester: Optional[HttpRequester] = None,
+) -> Callable[[], Agent]:
+    """Adapt a hosted HTTP agent into a target_factory for quick_test()/
+    run_robustness_pair() -- for when your real agent already runs as a
+    service you can POST to, instead of a Python function/object living in
+    this same process. No custom decide_fn to write: this builds one for
+    you and wraps it exactly like wrap_agent() does.
+
+    On every step, POSTs `{"task": <str>, "history": [...]}` as JSON to
+    `url` -- `history` is the same shape wrap_agent's decide_fn receives, a
+    list of `{"tool_name", "tool_args", "result", "ok"}` dicts covering
+    everything observed so far this run. Expects back JSON shaped like
+    either:
+
+        {"tool_name": "...", "tool_args": {...}}   -- call this tool next
+        {"final_answer": "..."}                     -- done, this is the reply
+
+    Raises ValueError if a response matches neither shape, rather than
+    guessing what was meant. Pass `headers` (e.g.
+    `{"Authorization": "Bearer ..."}`) for an endpoint that needs auth, and
+    `requester` in tests instead of hitting the real network -- see
+    tests/test_quickstart.py.
+    """
+    requester = requester if requester is not None else _default_http_requester
+    resolved_headers = headers or {}
+
+    def decide_fn(task: str, history: list[dict[str, Any]]) -> AgentAction:
+        response = requester(url, {"task": task, "history": history}, resolved_headers)
+        if "final_answer" in response:
+            return AgentAction(kind="final_answer", text=response["final_answer"])
+        if "tool_name" in response and "tool_args" in response:
+            return AgentAction(kind="tool_call", tool_name=response["tool_name"], tool_args=response["tool_args"])
+        raise ValueError(
+            f"http_agent: response from {url} must include either 'final_answer' or "
+            f"both 'tool_name' and 'tool_args', got keys {sorted(response.keys())}"
+        )
+
+    return wrap_agent(decide_fn)
+
+
 def quick_test(
     scenario: Scenario,
     agent: Callable[[], Agent] | DecideFn,

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.1}/agentprobe_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.5.0
+Version: 0.5.1
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentprobe-testing"
-version = "0.5.0"
+version = "0.5.1"
 description = "Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance."
 requires-python = ">=3.11"
 dependencies = ["anthropic>=1.0.0", "python-dotenv>=1.0.0"]

{agentprobe_testing-0.5.0 → agentprobe_testing-0.5.1}/tests/test_quickstart.py

@@ -3,7 +3,7 @@ import pytest
 from agentprobe.agents.base import AgentAction
 from agentprobe.agents.rule_based import RuleBasedAgent
 from agentprobe.injector import HardcodedToolErrorInjector, Injector, NullInjector
-from agentprobe.quickstart import assert_passes, quick_test, quick_test_all, wrap_agent
+from agentprobe.quickstart import assert_passes, http_agent, quick_test, quick_test_all, wrap_agent
 from agentprobe.report import Report
 from agentprobe.scenarios.registry import BY_ID
 
@@ -52,6 +52,81 @@ def test_wrap_agent_gives_each_call_its_own_history_not_shared_across_instances(
     assert action.tool_name == "get_ticket"  # fresh history, not polluted by `first`
 
 
+def make_requester(responses):
+    """Returns each queued response in order, on every call -- the same
+    scripted-sequence pattern ScriptedAgent uses, since http_agent's whole
+    point is standing in for a real remote agent step by step."""
+    calls = []
+    responses = list(responses)
+
+    def requester(url, body, headers):
+        calls.append({"url": url, "body": body, "headers": headers})
+        return responses.pop(0)
+
+    requester.calls = calls
+    return requester
+
+
+def test_http_agent_drives_a_full_run_from_scripted_json_responses():
+    # mode="recovery" -- a robustness-mode run calls the target_factory
+    # twice (clean then chaos), which would drain this same scripted
+    # response queue twice over; recovery mode's single trajectory keeps
+    # this test's response list simple and matched 1:1 to real steps.
+    scenario = BY_ID["clean-1"]
+    requester = make_requester(
+        [
+            {"tool_name": "get_ticket", "tool_args": {"id": "T-1001"}},
+            {"tool_name": "issue_refund", "tool_args": {"ticket_id": "T-1001", "amount": 49.99}},
+            {"tool_name": "close_ticket", "tool_args": {"ticket_id": "T-1001", "resolution": "refunded"}},
+            {"final_answer": "done"},
+        ]
+    )
+
+    report = quick_test(scenario, http_agent("https://example.com/agent", requester=requester), injector=NullInjector(), mode="recovery")
+
+    assert report._compute()["chaos_passed"] == 1
+    assert requester.calls[0]["url"] == "https://example.com/agent"
+
+
+def test_http_agent_sends_task_and_history_as_the_request_body():
+    scenario = BY_ID["clean-1"]
+    requester = make_requester(
+        [
+            {"tool_name": "get_ticket", "tool_args": {"id": "T-1001"}},
+            {"final_answer": "done"},
+        ]
+    )
+
+    quick_test(scenario, http_agent("https://example.com/agent", requester=requester), injector=NullInjector(), mode="recovery")
+
+    first_call, second_call = requester.calls
+    assert first_call["body"]["task"] == "Handle ticket T-1001."
+    assert first_call["body"]["history"] == []
+    assert second_call["body"]["history"][0]["tool_name"] == "get_ticket"
+
+
+def test_http_agent_passes_through_custom_headers():
+    scenario = BY_ID["clean-1"]
+    requester = make_requester([{"final_answer": "done"}])
+
+    quick_test(
+        scenario,
+        http_agent("https://example.com/agent", headers={"Authorization": "Bearer secret"}, requester=requester),
+        injector=NullInjector(),
+        mode="recovery",
+    )
+
+    assert requester.calls[0]["headers"] == {"Authorization": "Bearer secret"}
+
+
+def test_http_agent_raises_a_clear_error_on_a_response_matching_neither_shape():
+    scenario = BY_ID["clean-1"]
+    requester = make_requester([{"something_else": "not a valid response"}])
+
+    with pytest.raises(ValueError, match="must include either 'final_answer' or"):
+        quick_test(scenario, http_agent("https://example.com/agent", requester=requester), injector=NullInjector(), mode="recovery")
+
+
 def test_quick_test_runs_a_plain_function_end_to_end_and_reports_pass():
     scenario = BY_ID["clean-1"]
     report = quick_test(scenario, _clean1_decide, injector=NullInjector())