agentprobe-testing 0.5.0__py3-none-any.whl

agentprobe/reachability.py

@@ -0,0 +1,196 @@
+"""The reachability checker: pure, deterministic, no LLM.
+
+`reachable()` is a comparison against an authored answer key, never a
+judgment call. If you find yourself wanting a model here, the GoalSpec is
+underspecified -- fix the spec, don't add a judge.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Callable, Optional
+
+from agentprobe.scenario import CommitPattern, GoalSpec
+from agentprobe.trajectory import CallRecord, Reachability, ReachabilityStatus
+from agentprobe.world import WorldState
+
+if TYPE_CHECKING:
+    from agentprobe.domain import EntityWorld
+
+_MISSING = object()
+
+# The parameter slot a domain's own precondition_checker fills in (see
+# domain.py's Domain.precondition_checker) -- generic on the world type,
+# since different domains pass different world classes here. Only
+# default_precondition_violation (the ticket domain's own implementation,
+# below) is concretely typed to WorldState, because it's the one function
+# in this file that actually is ticket-specific.
+PreconditionChecker = Callable[[str, dict, "EntityWorld"], Optional[str]]
+
+
+def default_precondition_violation(tool: str, args: dict, world: WorldState) -> str | None:
+    """The ticket-support domain's precondition rules -- mirrors the
+    precondition checks in tools.Toolkit, without mutating state or
+    raising. Used to detect condition 3 (a required commit's precondition
+    was destroyed by an earlier commit) before the agent ever attempts it.
+
+    A different Domain (see domain.py) supplies its own checker with its
+    own tool names and business rules instead of this one -- reachable()
+    takes it as a parameter for exactly that reason.
+    """
+    if tool == "issue_refund":
+        ticket = world.tickets.get(args.get("ticket_id"))
+        if ticket is None:
+            return "ticket does not exist"
+        if ticket.status == "closed":
+            return "ticket is closed"
+        if any(r["ticket_id"] == ticket.id for r in world.refunds_issued):
+            return "ticket already refunded"
+        return None
+    if tool == "close_ticket":
+        ticket = world.tickets.get(args.get("ticket_id"))
+        if ticket is None:
+            return "ticket does not exist"
+        if ticket.status == "closed":
+            return "ticket already closed"
+        return None
+    if tool in ("send_reply", "escalate"):
+        ticket = world.tickets.get(args.get("ticket_id"))
+        if ticket is None:
+            return "ticket does not exist"
+        return None
+    return None
+
+
+def _find_wrong_arg_commit(
+    calls: list[CallRecord], patterns: list[CommitPattern], entity_id_arg: str = "ticket_id"
+) -> CallRecord | None:
+    """A call is a 'wrong-argument' execution of a required commit if it
+    targets the *same entity* (identified by `entity_id_arg`, e.g.
+    "ticket_id") as a required commit on that tool, but with different
+    other args (wrong amount, wrong tier, ...).
+
+    Deliberately scoped to same-entity calls: a commit on some other,
+    unrelated entity that happens to share a tool name (e.g. closing an
+    already-resolved decoy ticket) is not an attempt at the required
+    commit at all, and is governed by forbidden_commits, not this check.
+    """
+    sensitive_tools = {p.tool for p in patterns}
+    for call in calls:
+        if not call.ok or not call.is_commit:
+            continue
+        if call.tool_name not in sensitive_tools:
+            continue
+        call_entity_id = call.args.get(entity_id_arg)
+        relevant_patterns = [
+            p
+            for p in patterns
+            if p.tool == call.tool_name and p.args.get(entity_id_arg) == call_entity_id
+        ]
+        if not relevant_patterns:
+            continue
+        if not any(p.matches(call.tool_name, call.args) for p in relevant_patterns):
+            return call
+    return None
+
+
+def reachable(
+    world_state: "EntityWorld",
+    calls: list[CallRecord],
+    goal: GoalSpec,
+    precondition_checker: PreconditionChecker = default_precondition_violation,
+    entity_id_arg: str = "ticket_id",
+) -> Reachability:
+    committed_ok = [c for c in calls if c.ok and c.is_commit]
+    read_ok = [c for c in calls if c.ok and not c.is_commit]
+
+    # Condition 1: a forbidden commit was executed.
+    forbidden_hits = [
+        c for c in committed_ok
+        if any(p.matches(c.tool_name, c.args) for p in goal.forbidden_commits)
+    ]
+    if forbidden_hits:
+        c = forbidden_hits[0]
+        return Reachability(
+            status=ReachabilityStatus.UNREACHABLE,
+            reason=f"forbidden commit executed: {c.tool_name}({c.args})",
+            forbidden_commits_hit=len(forbidden_hits),
+        )
+
+    # Condition 2: a required commit was executed with wrong arguments.
+    wrong_arg = _find_wrong_arg_commit(committed_ok, goal.required_commits, entity_id_arg)
+    if wrong_arg is not None:
+        return Reachability(
+            status=ReachabilityStatus.UNREACHABLE,
+            reason=(
+                f"required commit executed with wrong arguments: "
+                f"{wrong_arg.tool_name}({wrong_arg.args})"
+            ),
+        )
+
+    satisfied_commit_idx = {
+        i
+        for i, p in enumerate(goal.required_commits)
+        if any(p.matches(c.tool_name, c.args) for c in committed_ok)
+    }
+    outstanding = [
+        p for i, p in enumerate(goal.required_commits) if i not in satisfied_commit_idx
+    ]
+
+    satisfied_fact_ids = {
+        f.id for f in goal.required_facts if any(f.matches(c.tool_name, c.args) for c in read_ok)
+    }
+    facts_unknown = [f for f in goal.required_facts if f.id not in satisfied_fact_ids]
+
+    # Condition 3: an outstanding required commit's precondition was
+    # destroyed by something that already happened.
+    for p in outstanding:
+        violation = precondition_checker(p.tool, p.args, world_state)
+        if violation is not None:
+            return Reachability(
+                status=ReachabilityStatus.UNREACHABLE,
+                reason=(
+                    f"precondition for required commit {p.tool}({p.args}) "
+                    f"destroyed: {violation}"
+                ),
+                required_commits_outstanding=len(outstanding),
+                required_facts_unknown=len(facts_unknown),
+            )
+
+    if not outstanding and not facts_unknown:
+        return Reachability(status=ReachabilityStatus.ACHIEVED)
+
+    return Reachability(
+        status=ReachabilityStatus.REACHABLE,
+        required_commits_outstanding=len(outstanding),
+        required_facts_unknown=len(facts_unknown),
+    )
+
+
+def check_budget_exhausted(
+    goal: GoalSpec, world_state: "EntityWorld", calls: list[CallRecord], reach: Reachability
+) -> str | None:
+    """Condition 4, checked by the runner once budget is spent: required
+    commits outstanding with no steps left. Kept separate from reachable()
+    itself since it needs the scenario's budget, not just calls-so-far.
+    """
+    if reach.status != ReachabilityStatus.ACHIEVED and reach.required_commits_outstanding > 0:
+        return (
+            f"budget exhausted with {reach.required_commits_outstanding} "
+            f"required commit(s) outstanding"
+        )
+    return None
+
+
+def check_final_answer_contradiction(goal: GoalSpec, final_answer: str | None) -> str | None:
+    """Condition 5: the agent's final natural-language answer contradicts
+    the goal (claims a resolution the authored answer key rules out).
+    """
+    if not final_answer:
+        return None
+    lowered = final_answer.lower()
+    for bad in goal.final_answer_must_not_contain:
+        if bad.lower() in lowered:
+            return f"final answer contains forbidden claim: {bad!r}"
+    return None
+
+

agentprobe/registry.py

@@ -0,0 +1,313 @@
+"""fetch_domain(): load a Domain generated server-side from your own tool
+schemas + a plain-English description of your business rules (the
+"generate a domain" flow on agentprobe.dev), instead of hand-writing
+agentprobe/domain.py yourself.
+
+SECURITY: this downloads and *executes* Python source from AgentProbe's
+server under the key you were given. Only fetch a key you generated for
+your own business -- never one someone else handed you, same rule as never
+installing a package from someone you don't trust. And review the
+generated business logic (the review_url printed on every fetch) before
+trusting its results: an LLM can get a precondition subtly wrong -- e.g.
+treating an already-granted request as grantable again -- and a wrong rule
+makes a test pass without proving anything real about your agent.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import sys
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Callable, Optional
+
+from agentprobe.domain import Domain, TICKET_DOMAIN
+from agentprobe.scenario import Scenario
+
+if TYPE_CHECKING:
+    from agentprobe.report import Report
+
+DEFAULT_BASE_URL = "https://agentprobe-api.agentprobe.workers.dev"
+
+Fetcher = Callable[[str], dict]
+Poster = Callable[[str, dict, str], dict]  # (url, json_body, api_key) -> response dict
+AuthGetter = Callable[[str, str], dict]  # (url, api_key) -> response dict
+
+
+@dataclass
+class FetchedDomain:
+    domain: Domain
+    scenarios: dict[str, Scenario]
+    review_url: Optional[str] = None
+
+
+def _default_fetcher(url: str) -> dict:
+    # Cloudflare's default bot protection blocks urllib's stock User-Agent
+    # ("Python-urllib/3.x") outright -- a real client identifier, same idea
+    # as any other SDK sending its own name/version, gets through.
+    request = urllib.request.Request(url, headers={"User-Agent": "agentprobe-client/0.5.0"})
+    with urllib.request.urlopen(request, timeout=10) as resp:  # noqa: S310 -- https enforced by fetch_domain below
+        return json.loads(resp.read().decode("utf-8"))
+
+
+def fetch_domain(key: str, base_url: str = DEFAULT_BASE_URL, fetcher: Optional[Fetcher] = None) -> FetchedDomain:
+    """Fetch and load the Domain stored under `key`. Pass `fetcher` (a
+    callable url -> dict) in tests instead of hitting the real network --
+    see tests/test_registry.py.
+    """
+    if not (base_url.startswith("https://") or base_url.startswith("http://localhost")):
+        raise ValueError("fetch_domain requires an https:// base_url (http://localhost is allowed for local testing)")
+    if not key:
+        raise ValueError("fetch_domain requires a non-empty key")
+
+    fetcher = fetcher if fetcher is not None else _default_fetcher
+    payload = fetcher(f"{base_url}/v1/domains/{key}")
+
+    if "source" not in payload:
+        raise ValueError(f"domain {key!r}: server response missing 'source'")
+    review_url = payload.get("review_url")
+
+    print(
+        f"[agentprobe] fetch_domain({key!r}): about to execute server-generated Python. "
+        "This is AI-generated business logic that has NOT been reviewed by AgentProbe -- "
+        "skim it before trusting any results it produces"
+        + (f": {review_url}" if review_url else " (no review_url was provided)."),
+        file=sys.stderr,
+    )
+
+    namespace: dict = {}
+    exec(compile(payload["source"], f"<agentprobe domain {key}>", "exec"), namespace)  # noqa: S102
+
+    domain = namespace.get("DOMAIN")
+    if not isinstance(domain, Domain):
+        raise ValueError(f"domain {key!r}: generated source did not define a top-level DOMAIN: Domain")
+
+    # Observed real generation bug: the model sometimes writes a top-level
+    # INJECTOR_SYSTEM_PROMPT string but references it as `None  # set below`
+    # inside the (frozen, unpatchable-after-construction) Domain(...) call
+    # instead of defining it first and passing it directly. Patch it back on
+    # here rather than losing a paid-for generation to an ordering mistake.
+    injector_prompt = namespace.get("INJECTOR_SYSTEM_PROMPT")
+    if domain.injector_system_prompt is None and isinstance(injector_prompt, str):
+        domain = dataclasses.replace(domain, injector_system_prompt=injector_prompt)
+
+    scenarios = namespace.get("SCENARIOS", {})
+    if not isinstance(scenarios, dict) or not all(isinstance(s, Scenario) for s in scenarios.values()):
+        raise ValueError(f"domain {key!r}: generated source's SCENARIOS must be a dict[str, Scenario]")
+
+    return FetchedDomain(domain=domain, scenarios=scenarios, review_url=review_url)
+
+
+def _default_poster(url: str, body: dict, api_key: str) -> dict:
+    data = json.dumps(body).encode("utf-8")
+    request = urllib.request.Request(
+        url,
+        data=data,
+        method="POST",
+        headers={
+            "User-Agent": "agentprobe-client/0.5.0",
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        },
+    )
+    with urllib.request.urlopen(request, timeout=10) as resp:  # noqa: S310 -- https enforced by upload_run below
+        return json.loads(resp.read().decode("utf-8"))
+
+
+def upload_run(
+    report: Report,
+    api_key: str,
+    domain_key: Optional[str] = None,
+    base_url: str = DEFAULT_BASE_URL,
+    poster: Optional[Poster] = None,
+    recorded_injections: Optional[list[dict]] = None,
+) -> None:
+    """Upload a finished quick_test() report to your AgentProbe account (POST
+    /v1/runs) so it shows up under Your Runs on the dashboard. `api_key` is a
+    personal key created on /dashboard -- create one there, it's never shown
+    again after creation so save it somewhere real (an env var, not a
+    committed file). `report` must cover exactly one scenario, same as every
+    report quick_test() itself produces.
+
+    Pass `recorded_injections` (a list of agentprobe.injector.
+    serialize_armed_injection() dicts) to make the uploaded run replayable
+    later via replay_run(). Omit it (the default) and this auto-detects a
+    recording quick_test(..., record_injections=True) attached to `report`
+    itself -- the common case where you called quick_test() WITHOUT
+    upload_api_key so you could run check_regression() first, then upload
+    this same report yourself once you knew whether to. Explicitly passing
+    recorded_injections=[] uploads with no recording even if one is
+    attached, same as before this auto-detection existed.
+
+    Silent about network failures beyond raising -- this is meant to run
+    after your real test already finished, so a failed upload shouldn't be
+    confused with a failed test. Catch the exception yourself if you'd
+    rather not have an upload failure interrupt a script.
+    """
+    if not (base_url.startswith("https://") or base_url.startswith("http://localhost")):
+        raise ValueError("upload_run requires an https:// base_url (http://localhost is allowed for local testing)")
+    if not api_key:
+        raise ValueError("upload_run requires a non-empty api_key")
+
+    c = report._compute()
+    if c["n"] != 1:
+        raise ValueError(f"upload_run expects a single-scenario report (got {c['n']}) -- same shape quick_test() always produces")
+
+    scenario_id = report.chaos_trajectories[0].scenario_id
+    body = {
+        "scenario_id": scenario_id,
+        "mode": report.mode,
+        "chaos_passed": bool(c["chaos_passed"]),
+        "clean_passed": bool(c["clean_passed"]) if c["clean_passed"] is not None else None,
+        "target_model": report.target_model,
+        "injector_model": report.injector_model,
+        "cost_usd": c["target_cost_usd"] + c["injector_cost_usd"] + c["classifier_cost_usd"],
+        "domain_key": domain_key,
+        "report_html": report.render_html(),
+    }
+    if recorded_injections is None:
+        recorded_injections = getattr(report, "_recorded_injections", None)
+    if recorded_injections:
+        body["recorded_injections"] = recorded_injections
+
+    poster = poster if poster is not None else _default_poster
+    poster(f"{base_url}/v1/runs", body, api_key)
+
+
+def _default_auth_getter(url: str, api_key: str) -> dict:
+    request = urllib.request.Request(
+        url, headers={"User-Agent": "agentprobe-client/0.5.0", "Authorization": f"Bearer {api_key}"}
+    )
+    with urllib.request.urlopen(request, timeout=10) as resp:  # noqa: S310 -- https enforced by callers below
+        return json.loads(resp.read().decode("utf-8"))
+
+
+def get_latest_run(
+    scenario_id: str,
+    api_key: str,
+    base_url: str = DEFAULT_BASE_URL,
+    fetcher: Optional[AuthGetter] = None,
+) -> dict:
+    """Fetch the most recent uploaded run for `scenario_id` on your account
+    (GET /v1/runs/latest). Returns {"found": False} if nothing's been
+    uploaded for that scenario_id yet, else {"found": True, "id",
+    "chaos_passed", "clean_passed", "created_at", ...} -- see
+    check_regression() for the higher-level version of this that also does
+    the comparison for you.
+    """
+    if not (base_url.startswith("https://") or base_url.startswith("http://localhost")):
+        raise ValueError("get_latest_run requires an https:// base_url (http://localhost is allowed for local testing)")
+    if not api_key:
+        raise ValueError("get_latest_run requires a non-empty api_key")
+    if not scenario_id:
+        raise ValueError("get_latest_run requires a non-empty scenario_id")
+
+    fetcher = fetcher if fetcher is not None else _default_auth_getter
+    url = f"{base_url}/v1/runs/latest?scenario_id={urllib.parse.quote(scenario_id, safe='')}"
+    return fetcher(url, api_key)
+
+
+@dataclass(frozen=True)
+class RegressionResult:
+    previous_run_id: int
+    previous_created_at: str
+    previous_chaos_passed: bool
+    current_chaos_passed: bool
+    regressed: bool
+    """True only when the previous uploaded run passed and this one didn't
+    -- the one direction worth flagging automatically. A chaos pass rate can
+    legitimately flip either way between two single-scenario runs (a
+    ModelInjector doesn't reproduce exactly -- see ReplayInjector's
+    docstring), so "any different result" would be too noisy to act on."""
+    improved: bool
+    """True when the previous run failed and this one passed."""
+
+
+def check_regression(
+    report: Report,
+    api_key: str,
+    base_url: str = DEFAULT_BASE_URL,
+    fetcher: Optional[AuthGetter] = None,
+) -> Optional[RegressionResult]:
+    """Compare a freshly-finished single-scenario Report (same shape
+    upload_run() expects) against the most recent run PREVIOUSLY uploaded
+    for the same scenario_id on your account, so you can catch "this used to
+    pass, now it doesn't" without hand-tracking history yourself.
+
+    Call this BEFORE uploading the new report (e.g. before quick_test(...,
+    upload_api_key=...) or your own upload_run() call) -- otherwise "the
+    most recent run" on the server is this same report and there's nothing
+    to compare against. Returns None the first time you test a given
+    scenario_id (nothing uploaded yet), not an error.
+    """
+    c = report._compute()
+    if c["n"] != 1:
+        raise ValueError(f"check_regression expects a single-scenario report (got {c['n']}) -- same shape quick_test() always produces")
+
+    scenario_id = report.chaos_trajectories[0].scenario_id
+    payload = get_latest_run(scenario_id, api_key, base_url=base_url, fetcher=fetcher)
+    if not payload.get("found"):
+        return None
+
+    current_passed = bool(c["chaos_passed"])
+    previous_passed = bool(payload["chaos_passed"])
+    return RegressionResult(
+        previous_run_id=payload["id"],
+        previous_created_at=payload["created_at"],
+        previous_chaos_passed=previous_passed,
+        current_chaos_passed=current_passed,
+        regressed=previous_passed and not current_passed,
+        improved=current_passed and not previous_passed,
+    )
+
+
+def replay_run(
+    run_id: int,
+    api_key: str,
+    scenario: Scenario,
+    agent,
+    domain: Domain = TICKET_DOMAIN,
+    base_url: str = DEFAULT_BASE_URL,
+    fetcher: Optional[AuthGetter] = None,
+    target_model: str = "custom",
+) -> "Report":
+    """Replay the exact chaos sequence recorded for `run_id` (an id from
+    your dashboard's Your Runs table, or a RegressionResult.previous_run_id)
+    against a new agent or model version, instead of paying for a fresh
+    ModelInjector decision sequence -- only the Target's own calls repeat.
+
+    Only works for a run uploaded with quick_test(..., record_injections=
+    True); raises ValueError for a run with no recorded sequence.
+
+    `scenario` and `domain` must be the SAME scenario/domain object the
+    original run used -- this function has no way to look that up
+    server-side (the server only stores the injection sequence, not the
+    scenario itself), so pass whatever your original script used.
+    """
+    if not (base_url.startswith("https://") or base_url.startswith("http://localhost")):
+        raise ValueError("replay_run requires an https:// base_url (http://localhost is allowed for local testing)")
+    if not api_key:
+        raise ValueError("replay_run requires a non-empty api_key")
+
+    fetcher = fetcher if fetcher is not None else _default_auth_getter
+    payload = fetcher(f"{base_url}/v1/runs/{run_id}", api_key)
+    recorded = payload.get("recorded_injections")
+    if not recorded:
+        raise ValueError(f"run {run_id} has no recorded injection sequence -- it wasn't uploaded with record_injections=True")
+
+    from agentprobe.injector import ReplayInjector, deserialize_armed_injection
+    from agentprobe.quickstart import quick_test
+
+    armed = [deserialize_armed_injection(d) for d in recorded]
+    injector = ReplayInjector(armed)
+    return quick_test(
+        scenario,
+        agent,
+        injector=injector,
+        mode="robustness",
+        domain=domain,
+        target_model=target_model,
+        injector_model=f"replay:{run_id}",
+    )