agentprobe-testing 0.8.2__tar.gz → 0.8.4__tar.gz

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.8.2
+Version: 0.8.4
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/agentprobe/__init__.py

@@ -60,7 +60,7 @@ from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "0.8.2"
+__version__ = "0.8.4"
 
 __all__ = [
     "Agent",

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/agentprobe/llm.py

@@ -15,20 +15,72 @@ from __future__ import annotations
 
 from typing import Any
 
+import anthropic._streaming as _anthropic_streaming
+
 _TEMPERATURE_SUPPORTED: dict[str, bool] = {}
 
 
+def _patch_sse_decoder_for_memoryview_chunks() -> None:
+    """anthropic._streaming.SSEDecoder calls `chunk.splitlines()` directly on
+    whatever the underlying HTTP iterator yields, assuming it's always
+    `bytes`. Under Cloudflare's Python Worker runtime (Pyodide/emscripten),
+    the httpx2-jsfetch transport that bridges JS's fetch API yields
+    `memoryview` chunks instead -- confirmed live: every single streamed
+    Injector call failed with AttributeError('memoryview' object has no
+    attribute 'splitlines') the moment create_deterministic() switched from
+    create() to stream() to dodge a separate idle-timeout bug. `bytes(chunk)`
+    is a no-op for already-bytes chunks and a cheap copy for memoryview, so
+    coercing at the one boundary where raw chunks enter the SSE decoder
+    fixes this without reimplementing SSE parsing ourselves. Wrapped in
+    try/except so an anthropic SDK upgrade that moves this internal method
+    fails quiet (falls back to the original, unpatched bug) rather than
+    breaking import entirely.
+    """
+    try:
+        decoder_cls = _anthropic_streaming.SSEDecoder
+        original_iter_chunks = decoder_cls._iter_chunks
+        original_aiter_chunks = decoder_cls._aiter_chunks
+
+        def _iter_chunks(self, iterator):
+            return original_iter_chunks(self, (bytes(chunk) for chunk in iterator))
+
+        async def _aiter_chunks(self, iterator):
+            async def _coerced():
+                async for chunk in iterator:
+                    yield bytes(chunk)
+
+            async for sse_chunk in original_aiter_chunks(self, _coerced()):
+                yield sse_chunk
+
+        decoder_cls._iter_chunks = _iter_chunks
+        decoder_cls._aiter_chunks = _aiter_chunks
+    except AttributeError:
+        pass
+
+
+_patch_sse_decoder_for_memoryview_chunks()
+
+
 def create_deterministic(client: Any, **kwargs: Any) -> Any:
+    # Streamed, not create(): a non-streaming call blocks with zero bytes on
+    # the wire until the whole completion is ready, which under Cloudflare's
+    # Python Worker runtime (where the Injector actually runs) gets read as
+    # an idle connection and killed as a timeout -- confirmed live, this
+    # exact symptom (APITimeoutError on every single decide() call, even a
+    # trivial first step) is what the same non-streaming pattern already
+    # caused for domain generation, fixed there the same way.
     model = kwargs["model"]
     if _TEMPERATURE_SUPPORTED.get(model, True):
         try:
-            return client.messages.create(**kwargs, extra_body={"temperature": 0})
+            with client.messages.stream(**kwargs, extra_body={"temperature": 0}) as stream:
+                return stream.get_final_message()
         except Exception as e:
             if "temperature" in str(e).lower():
                 _TEMPERATURE_SUPPORTED[model] = False
             else:
                 raise
-    return client.messages.create(**kwargs)
+    with client.messages.stream(**kwargs) as stream:
+        return stream.get_final_message()
 
 
 _PRICE_PER_TOKEN: dict[str, tuple[float, float]] = {

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/agentprobe_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.8.2
+Version: 0.8.4
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentprobe-testing"
-version = "0.8.2"
+version = "0.8.4"
 description = "Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance."
 requires-python = ">=3.11"
 dependencies = ["anthropic>=1.0.0", "python-dotenv>=1.0.0"]

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/tests/test_access_control_agent.py

@@ -1,5 +1,7 @@
 from types import SimpleNamespace
 
+from conftest import stream_from_create
+
 from agentprobe.domains.access_control.agent import AccessControlAgent
 
 
@@ -15,6 +17,7 @@ def test_next_action_returns_tool_call_and_uses_cached_system(monkeypatch):
             captured.update(kwargs)
             tool_use = SimpleNamespace(type="tool_use", id="tu_1", name="get_request", input={"id": "AR-1"})
             return SimpleNamespace(content=[tool_use], usage=_usage())
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -41,6 +44,7 @@ def test_next_action_returns_final_answer_when_no_tool_use(monkeypatch):
         def create(self, **kwargs):
             text_block = SimpleNamespace(type="text", text="All done, access granted.")
             return SimpleNamespace(content=[text_block], usage=_usage())
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -63,6 +67,7 @@ def test_next_action_prices_cost_by_the_agents_own_model_not_a_fixed_rate(monkey
         def create(self, **kwargs):
             tool_use = SimpleNamespace(type="tool_use", id="tu_1", name="get_request", input={"id": "AR-1"})
             return SimpleNamespace(content=[tool_use], usage=_usage(input_tokens=1000, output_tokens=1000))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -98,6 +103,7 @@ def test_next_action_includes_cache_write_cost_from_a_real_shaped_response(monke
                     input_tokens=100, output_tokens=50, cache_creation_input_tokens=5000, cache_read_input_tokens=0
                 ),
             )
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -120,6 +126,7 @@ def test_observe_appends_tool_result_and_marks_errors(monkeypatch):
         def create(self, **kwargs):
             tool_use = SimpleNamespace(type="tool_use", id="tu_1", name="get_request", input={"id": "AR-1"})
             return SimpleNamespace(content=[tool_use], usage=_usage())
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/tests/test_classifier.py

@@ -1,5 +1,7 @@
 from types import SimpleNamespace
 
+from conftest import stream_from_create
+
 from agentprobe.classifier import ResponseClassification, _render_classification_prompt, classify_response
 from agentprobe.injection import AppliedInjection, Injection, InjectionKind, ResponseSignature
 from agentprobe.trajectory import Reachability, ReachabilityStatus, Step
@@ -68,6 +70,7 @@ def test_classify_response_parses_forced_tool_call(monkeypatch):
                 input={"classification": "HANDLED", "rationale": "retried and succeeded"},
             )
             return SimpleNamespace(content=[tool_use], usage=SimpleNamespace(input_tokens=100, output_tokens=20))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -91,6 +94,7 @@ def test_classify_response_raises_clear_error_when_no_tool_use_block(monkeypatch
         def create(self, **kwargs):
             text_block = SimpleNamespace(type="text", text="I refuse to classify this.")
             return SimpleNamespace(content=[text_block], stop_reason="end_turn")
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -116,6 +120,7 @@ def test_classify_response_raises_clear_error_when_tool_call_missing_fields(monk
                 input={"classification": "HANDLED"},  # missing "rationale"
             )
             return SimpleNamespace(content=[tool_use], stop_reason="max_tokens")
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/tests/test_complex_access_control_agent.py

@@ -1,5 +1,7 @@
 from types import SimpleNamespace
 
+from conftest import stream_from_create
+
 from agentprobe.domains.access_control.complex_agent import ComplexAccessControlAgent
 
 
@@ -15,6 +17,7 @@ def test_next_action_returns_tool_call_and_uses_cached_system(monkeypatch):
             captured.update(kwargs)
             tool_use = SimpleNamespace(type="tool_use", id="tu_1", name="get_request", input={"id": "AR-1"})
             return SimpleNamespace(content=[tool_use], usage=_usage())
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -40,6 +43,7 @@ def test_next_action_returns_final_answer_when_no_tool_use(monkeypatch):
         def create(self, **kwargs):
             text_block = SimpleNamespace(type="text", text="Escalated per policy, did not grant directly.")
             return SimpleNamespace(content=[text_block], usage=_usage())
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -62,6 +66,7 @@ def test_observe_appends_tool_result_and_marks_errors(monkeypatch):
         def create(self, **kwargs):
             tool_use = SimpleNamespace(type="tool_use", id="tu_1", name="get_request", input={"id": "AR-1"})
             return SimpleNamespace(content=[tool_use], usage=_usage())
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/tests/test_injector.py

@@ -2,6 +2,8 @@ from types import SimpleNamespace
 
 import pytest
 
+from conftest import stream_from_create
+
 from agentprobe.injection import ArmedInjection, CommitPattern, Injection, InjectionKind, ResponseSignature, Trigger
 from agentprobe.injector import (
     HardcodedToolErrorInjector,
@@ -113,6 +115,7 @@ def test_model_injector_declined_for_budget_stays_zero_when_floor_never_bites(mo
     class FakeMessages:
         def create(self, **kwargs):
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -133,6 +136,7 @@ def test_model_injector_accumulates_cost_across_calls_but_not_when_declined_for_
                 content=[SimpleNamespace(type="tool_use", name="wait", input={})],
                 usage=SimpleNamespace(input_tokens=100, output_tokens=50),
             )
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -172,6 +176,7 @@ def test_model_injector_includes_cache_write_cost_from_a_real_shaped_response(mo
                     input_tokens=100, output_tokens=50, cache_creation_input_tokens=5000, cache_read_input_tokens=0
                 ),
             )
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -200,6 +205,7 @@ def test_free_policy_offers_all_kinds_and_never_forces(monkeypatch):
         def create(self, **kwargs):
             captured.update(kwargs)
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -230,6 +236,7 @@ def test_free_policy_ignores_remaining_step_budget_forcing():
         def create(self, **kwargs):
             captured.update(kwargs)
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -262,6 +269,7 @@ def test_forced_coverage_policy_never_offers_wait_while_a_kind_is_pending(monkey
                 },
             )
             return SimpleNamespace(content=[tool_use], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -285,6 +293,7 @@ def test_decision_log_records_every_call_including_waits_and_declines():
     class FakeMessages:
         def create(self, **kwargs):
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -311,6 +320,7 @@ def test_decision_log_records_declined_for_budget_without_calling_the_model():
         def create(self, **kwargs):
             call_count["n"] += 1
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -338,6 +348,7 @@ def test_model_injector_wait_returns_none(monkeypatch):
         def create(self, **kwargs):
             wait_call = SimpleNamespace(type="tool_use", name="wait", input={})
             return SimpleNamespace(content=[wait_call], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -358,6 +369,7 @@ def test_model_injector_restricts_kind_enum_to_first_uncovered_kind(monkeypatch)
         def create(self, **kwargs):
             captured.update(kwargs)
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -387,6 +399,7 @@ def test_model_injector_offers_next_uncovered_kind_once_others_are_used_fired_or
         def create(self, **kwargs):
             captured.update(kwargs)
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -412,6 +425,7 @@ def test_model_injector_skips_without_calling_the_model_when_only_followup_sensi
         def create(self, **kwargs):
             call_count["n"] += 1
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -440,6 +454,7 @@ def test_model_injector_excludes_followup_sensitive_kinds_from_free_choice_when_
         def create(self, **kwargs):
             captured.update(kwargs)
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -491,6 +506,7 @@ def test_model_injector_forces_injection_when_budget_cant_cover_remaining_kinds(
                 },
             )
             return SimpleNamespace(content=[tool_use], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -535,6 +551,7 @@ def test_model_injector_forces_after_one_wait_on_same_pending_kind(monkeypatch):
                 },
             )
             return SimpleNamespace(content=[tool_use], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -565,6 +582,7 @@ def test_model_injector_allows_free_choice_once_all_kinds_covered(monkeypatch):
         def create(self, **kwargs):
             captured.update(kwargs)
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -607,6 +625,7 @@ def test_model_injector_inject_returns_armed_injection_with_parsed_trigger(monke
                 },
             )
             return SimpleNamespace(content=[tool_use], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -654,6 +673,7 @@ def test_model_injector_parses_after_commit_trigger(monkeypatch):
                 },
             )
             return SimpleNamespace(content=[tool_use], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -731,6 +751,7 @@ def test_model_injector_parses_on_any_commit_trigger(monkeypatch):
                 },
             )
             return SimpleNamespace(content=[tool_use], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -763,6 +784,7 @@ def test_model_injector_raises_on_missing_trigger_field(monkeypatch):
                 },
             )
             return SimpleNamespace(content=[tool_use], usage=SimpleNamespace(input_tokens=100, output_tokens=50), stop_reason="end_turn")
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -863,6 +885,7 @@ def test_model_injector_includes_playbook_hint_when_data_exists(tmp_path, monkey
                 content=[SimpleNamespace(type="tool_use", name="wait", input={})],
                 usage=SimpleNamespace(input_tokens=100, output_tokens=50),
             )
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -890,6 +913,7 @@ def test_model_injector_omits_playbook_hint_when_no_playbook_configured(monkeypa
                 content=[SimpleNamespace(type="tool_use", name="wait", input={})],
                 usage=SimpleNamespace(input_tokens=100, output_tokens=50),
             )
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -922,6 +946,7 @@ def test_model_injector_omits_playbook_hint_when_insufficient_data(tmp_path, mon
                 content=[SimpleNamespace(type="tool_use", name="wait", input={})],
                 usage=SimpleNamespace(input_tokens=100, output_tokens=50),
             )
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -960,6 +985,7 @@ def test_model_injector_playbook_hint_includes_customer_feedback_notes(tmp_path,
                 content=[SimpleNamespace(type="tool_use", name="wait", input={})],
                 usage=SimpleNamespace(input_tokens=100, output_tokens=50),
             )
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -998,6 +1024,7 @@ def test_model_injector_playbook_hint_surfaces_feedback_even_below_fire_rate_min
                 content=[SimpleNamespace(type="tool_use", name="wait", input={})],
                 usage=SimpleNamespace(input_tokens=100, output_tokens=50),
             )
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -1041,6 +1068,7 @@ def test_model_injector_uses_a_custom_system_prompt_when_supplied(monkeypatch):
         def create(self, **kwargs):
             captured.update(kwargs)
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -1068,6 +1096,7 @@ def test_model_injector_default_system_prompt_is_the_ticket_domain_one(monkeypat
         def create(self, **kwargs):
             captured.update(kwargs)
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -1089,6 +1118,7 @@ def test_model_injector_uses_custom_all_tools_and_commit_tools_in_the_schema(mon
         def create(self, **kwargs):
             captured.update(kwargs)
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -1116,6 +1146,7 @@ def test_model_injector_uses_custom_entity_types_in_the_world_snapshot(monkeypat
         def create(self, **kwargs):
             captured.update(kwargs)
             return SimpleNamespace(content=[SimpleNamespace(type="tool_use", name="wait", input={})], usage=SimpleNamespace(input_tokens=100, output_tokens=50))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/tests/test_llm.py

@@ -4,16 +4,33 @@ import agentprobe.llm as llm_module
 from agentprobe.llm import cacheable_system, compute_cost_usd, cost_per_token, create_deterministic, response_cost_usd
 
 
+class FakeMessageStream:
+    """Mimics anthropic's MessageStreamManager just enough for
+    create_deterministic's `with ... as stream: stream.get_final_message()`."""
+
+    def __init__(self, result):
+        self._result = result
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_info):
+        return False
+
+    def get_final_message(self):
+        return self._result
+
+
 class FakeMessages:
     def __init__(self, reject_temperature=False):
         self.reject_temperature = reject_temperature
         self.calls = []
 
-    def create(self, **kwargs):
+    def stream(self, **kwargs):
         self.calls.append(kwargs)
         if self.reject_temperature and "extra_body" in kwargs:
             raise ValueError("`temperature` is deprecated for this model.")
-        return "ok"
+        return FakeMessageStream("ok")
 
 
 class FakeClient:
@@ -50,7 +67,7 @@ def test_falls_back_and_caches_when_temperature_rejected():
 
 def test_unrelated_errors_are_not_swallowed():
     class FailingMessages:
-        def create(self, **kwargs):
+        def stream(self, **kwargs):
             raise RuntimeError("network blip")
 
     class FailingClient:

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/tests/test_target_agent.py

@@ -1,5 +1,7 @@
 from types import SimpleNamespace
 
+from conftest import stream_from_create
+
 from agentprobe.agents.target_agent import TargetAgent
 
 
@@ -15,6 +17,7 @@ def test_next_action_returns_tool_call_and_uses_cached_system(monkeypatch):
             captured.update(kwargs)
             tool_use = SimpleNamespace(type="tool_use", id="tu_1", name="get_ticket", input={"id": "T-1"})
             return SimpleNamespace(content=[tool_use], usage=_usage())
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -44,6 +47,7 @@ def test_custom_system_prompt_overrides_the_default(monkeypatch):
             captured.update(kwargs)
             tool_use = SimpleNamespace(type="tool_use", id="tu_1", name="get_ticket", input={"id": "T-1"})
             return SimpleNamespace(content=[tool_use], usage=_usage())
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -67,6 +71,7 @@ def test_next_action_returns_final_answer_when_no_tool_use(monkeypatch):
         def create(self, **kwargs):
             text_block = SimpleNamespace(type="text", text="All done, refunded you.")
             return SimpleNamespace(content=[text_block], usage=_usage())
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -89,6 +94,7 @@ def test_next_action_prices_cost_by_the_agents_own_model_not_a_fixed_rate(monkey
         def create(self, **kwargs):
             tool_use = SimpleNamespace(type="tool_use", id="tu_1", name="get_ticket", input={"id": "T-1"})
             return SimpleNamespace(content=[tool_use], usage=_usage(input_tokens=1000, output_tokens=1000))
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -114,6 +120,7 @@ def test_observe_appends_tool_result_and_marks_errors(monkeypatch):
         def create(self, **kwargs):
             tool_use = SimpleNamespace(type="tool_use", id="tu_1", name="get_ticket", input={"id": "T-1"})
             return SimpleNamespace(content=[tool_use], usage=_usage())
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -141,6 +148,7 @@ def test_receive_message_appends_a_plain_user_turn(monkeypatch):
         def create(self, **kwargs):
             tool_use = SimpleNamespace(type="tool_use", id="tu_1", name="get_ticket", input={"id": "T-1"})
             return SimpleNamespace(content=[tool_use], usage=_usage())
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):

{agentprobe_testing-0.8.2 → agentprobe_testing-0.8.4}/tests/test_validate_scenarios.py

@@ -5,6 +5,8 @@ these tests exist to catch that.
 
 from types import SimpleNamespace
 
+from conftest import stream_from_create
+
 from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS
 from agentprobe.validate_scenarios import (
@@ -239,6 +241,7 @@ def test_verify_warnings_adds_model_note_without_changing_severity(monkeypatch):
                 input={"consistent": True, "explanation": "200.0 * 0.5 = 100.0, matches."},
             )
             return SimpleNamespace(content=[tool_use])
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -274,6 +277,7 @@ def test_verify_warnings_reports_inconsistent_without_promoting_to_broken(monkey
                 input={"consistent": False, "explanation": "200.0 * 0.5 = 100.0, not 60.0."},
             )
             return SimpleNamespace(content=[tool_use])
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -307,6 +311,7 @@ def test_verify_warnings_raises_clear_error_when_no_tool_use_block(monkeypatch):
         def create(self, **kwargs):
             text_block = SimpleNamespace(type="text", text="I refuse.")
             return SimpleNamespace(content=[text_block], stop_reason="end_turn")
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):
@@ -328,6 +333,7 @@ def test_verify_warnings_raises_clear_error_on_missing_fields(monkeypatch):
         def create(self, **kwargs):
             tool_use = SimpleNamespace(type="tool_use", input={"consistent": True})  # missing "explanation"
             return SimpleNamespace(content=[tool_use], stop_reason="max_tokens")
+        stream = stream_from_create(create)
 
     class FakeClient:
         def __init__(self):