simpleaudit 0.2.0__tar.gz → 0.2.1__tar.gz

{simpleaudit-0.2.0/simpleaudit.egg-info → simpleaudit-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: simpleaudit
-Version: 0.2.0
+Version: 0.2.1
 Summary: Lightweight AI Safety Auditing Framework
 Author: SimpleAudit Contributors
 License: MIT
@@ -162,10 +162,10 @@ results.save("./my_audit_results/audit_results.json")
 **💡 View results interactively:**
 ```bash
 # Option 1: Run directly with uvx (no installation needed, requires uv)
-uvx simpleaudit[visualize] serve --results_dir ./my_audit_results
+uvx 'simpleaudit[visualize]' serve --results_dir ./my_audit_results
 
 # Option 2: Install and run locally
-pip install simpleaudit[visualize]
+pip install 'simpleaudit[visualize]'
 simpleaudit serve --results_dir ./my_audit_results
 ```
 This will spin-up a local web server to explore results with scenario details. 👉 [Check for live demo.](https://simulamet-simpleauditvisualization.hf.space)
@@ -173,7 +173,14 @@ See [visualization/README.md](https://github.com/kelkalot/simpleaudit/blob/main/
 
 To share results as a single self-contained HTML file (no server, no JSON upload), use `simpleaudit export-html ./audit_results.json` or the **Download HTML** button in the visualizer.
 
-> **Note:** Option 1 requires [`uv`](https://pypi.org/project/uv/) to be installed ([install guide](https://docs.astral.sh/uv/getting-started/installation/)).
+> **Note:** Option 1 requires [`uv`](https://pypi.org/project/uv/) to be installed ([install guide](https://docs.astral.sh/uv/getting-started/installation/)). The package spec is quoted so your shell doesn't glob-expand `[visualize]`.
+
+<Troubleshooting>
+If the command fails:
+- **`Address already in use`** — port 8000 is taken. Use `--port 8080` or stop the other process.
+- **`uvx: command not found`** — install `uv` first (`curl -LsSf https://astral.sh/uv/install.sh | sh`).
+- **Weird import/module error** — stale cache. Run `uv cache clean` and retry.
+</Troubleshooting>
 
 [![simpleaudit-visualization-ui](https://github.com/user-attachments/assets/f9bbb891-a847-48d4-85d6-6d6d99c9e017)](https://github.com/kelkalot/simpleaudit/blob/main/simpleaudit/visualization/README.md)
 

{simpleaudit-0.2.0 → simpleaudit-0.2.1}/README.md

@@ -120,10 +120,10 @@ results.save("./my_audit_results/audit_results.json")
 **💡 View results interactively:**
 ```bash
 # Option 1: Run directly with uvx (no installation needed, requires uv)
-uvx simpleaudit[visualize] serve --results_dir ./my_audit_results
+uvx 'simpleaudit[visualize]' serve --results_dir ./my_audit_results
 
 # Option 2: Install and run locally
-pip install simpleaudit[visualize]
+pip install 'simpleaudit[visualize]'
 simpleaudit serve --results_dir ./my_audit_results
 ```
 This will spin-up a local web server to explore results with scenario details. 👉 [Check for live demo.](https://simulamet-simpleauditvisualization.hf.space)
@@ -131,7 +131,14 @@ See [visualization/README.md](https://github.com/kelkalot/simpleaudit/blob/main/
 
 To share results as a single self-contained HTML file (no server, no JSON upload), use `simpleaudit export-html ./audit_results.json` or the **Download HTML** button in the visualizer.
 
-> **Note:** Option 1 requires [`uv`](https://pypi.org/project/uv/) to be installed ([install guide](https://docs.astral.sh/uv/getting-started/installation/)).
+> **Note:** Option 1 requires [`uv`](https://pypi.org/project/uv/) to be installed ([install guide](https://docs.astral.sh/uv/getting-started/installation/)). The package spec is quoted so your shell doesn't glob-expand `[visualize]`.
+
+<Troubleshooting>
+If the command fails:
+- **`Address already in use`** — port 8000 is taken. Use `--port 8080` or stop the other process.
+- **`uvx: command not found`** — install `uv` first (`curl -LsSf https://astral.sh/uv/install.sh | sh`).
+- **Weird import/module error** — stale cache. Run `uv cache clean` and retry.
+</Troubleshooting>
 
 [![simpleaudit-visualization-ui](https://github.com/user-attachments/assets/f9bbb891-a847-48d4-85d6-6d6d99c9e017)](https://github.com/kelkalot/simpleaudit/blob/main/simpleaudit/visualization/README.md)
 

{simpleaudit-0.2.0 → simpleaudit-0.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "simpleaudit"
-version = "0.2.0"
+version = "0.2.1"
 description = "Lightweight AI Safety Auditing Framework"
 readme = "README.md"
 license = {text = "MIT"}

{simpleaudit-0.2.0 → simpleaudit-0.2.1}/simpleaudit/cli.py

@@ -71,8 +71,17 @@ def main():
             results_dir = "."
             print("⚠️  Warning: --results_dir not specified, using current directory '.'")
             print("   Recommended: explicitly set --results_dir to avoid confusion\n")
-        
-        start_server(results_dir, args.host, args.port)
+
+        try:
+            start_server(results_dir, args.host, args.port)
+        except OSError as exc:
+            if "Address already in use" in str(exc) or "Errno 48" in str(exc):
+                print(f"Error: port {args.port} is already in use.")
+                print(f"Try a different port: simpleaudit serve --results_dir {results_dir} --port 8080")
+                print("Or stop the process currently using that port.\n")
+            else:
+                raise
+            sys.exit(1)
     elif args.command == "export-html":
         try:
             from .visualization.server import export_standalone_html

{simpleaudit-0.2.0 → simpleaudit-0.2.1}/simpleaudit/experiment.py

@@ -305,6 +305,7 @@ class AuditExperiment:
         max_turns: Optional[int],
         language: str,
         max_workers: int,
+        on_turn: Optional[Callable[[int, int, str], None]] = None,
     ) -> AuditResults:
         """Execute one rep with auto-retry on ERROR. Returns the final result."""
         attempts = 1 + self.max_retries_per_rep
@@ -316,6 +317,7 @@ class AuditExperiment:
                 max_turns=max_turns,
                 language=language,
                 max_workers=max_workers,
+                on_turn=on_turn,
             )
             if not any(r.severity == "ERROR" for r in result):
                 break
@@ -335,6 +337,7 @@ class AuditExperiment:
         scenario: Dict[str, Any],
         max_turns: Optional[int] = None,
         language: str = "English",
+        on_turn: Optional[Callable[[int, int, str], None]] = None,
     ) -> List[AuditResult]:
         """Run a single scenario N times for one model.
 
@@ -348,6 +351,10 @@ class AuditExperiment:
             scenario: A single scenario dict.
             max_turns: Override for max conversation turns.
             language: Language for probe generation.
+            on_turn: Optional callback fired at each phase boundary with
+                ``(turn_index, max_turns, role)`` where role is "auditor",
+                "target", or "judge". Called synchronously from within the
+                asyncio event loop.
 
         Returns:
             List of :class:`AuditResult`, one per completed rep. May be
@@ -397,7 +404,8 @@ class AuditExperiment:
 
             # Execute with auto-retry
             rep_result = await self._run_single_rep(
-                merged, [scenario], max_turns, language, max_workers=1
+                merged, [scenario], max_turns, language, max_workers=1,
+                on_turn=on_turn,
             )
 
             # Persist

{simpleaudit-0.2.0 → simpleaudit-0.2.1}/simpleaudit/model_auditor.py

@@ -15,6 +15,7 @@ import asyncio
 import json
 import re
 import threading
+import traceback
 import warnings
 from datetime import date
 from importlib.metadata import PackageNotFoundError
@@ -309,6 +310,7 @@ class ModelAuditor:
         target_params: Optional[Dict[str, Any]] = None,
         judge_params: Optional[Dict[str, Any]] = None,
         auditor_params: Optional[Dict[str, Any]] = None,
+        on_turn: Optional[Callable[[int, int, str], None]] = None,
     ):
         if max_retries < 0:
             raise ValueError(f"max_retries must be >= 0, got {max_retries}")
@@ -319,6 +321,7 @@ class ModelAuditor:
         self.json_format = json_format
         self.max_retries = max_retries
         self.retry_backoff = retry_backoff
+        self.on_turn = on_turn
         self.judge_fields = judge_fields
         self.params = params
         self.target_params = target_params
@@ -489,6 +492,26 @@ class ModelAuditor:
                     message = f"[{name}] {message}"
                 tqdm.write(message)
 
+    def _fire_on_turn(
+        self,
+        turn_index: int,
+        max_turns: int,
+        role: str,
+        callback: Optional[Callable[[int, int, str], None]] = None,
+    ) -> None:
+        """Invoke an on_turn callback if set. Never raises.
+
+        Uses ``callback`` when provided, otherwise falls back to the
+        construction-time ``self.on_turn``. A raising callback is logged,
+        not propagated — a progress observer must never fail an audit.
+        """
+        cb = callback if callback is not None else self.on_turn
+        if cb is not None:
+            try:
+                cb(turn_index, max_turns, role)
+            except Exception:
+                self._log(f"on_turn callback failed:\n{traceback.format_exc()}")
+
     @staticmethod
     def strip_thinking(text: str) -> str:
         # Remove complete <think>...</think> / <thinking>...</thinking> blocks.
@@ -809,6 +832,7 @@ Evaluate this conversation and respond with this exact JSON structure:
         target_params: Optional[Dict[str, Any]] = None,
         judge_params: Optional[Dict[str, Any]] = None,
         auditor_params: Optional[Dict[str, Any]] = None,
+        on_turn: Optional[Callable[[int, int, str], None]] = None,
     ) -> AuditResult:
         turns = max_turns or self.max_turns
         base = {**(self.params or {}), **(params or {})}
@@ -816,6 +840,9 @@ Evaluate this conversation and respond with this exact JSON structure:
         effective_judge = {**base, **(self.judge_params or {}), **(judge_params or {})}
         effective_auditor = {**base, **(self.auditor_params or {}), **(auditor_params or {})}
 
+        # A per-call on_turn overrides one set at construction time; either may be used.
+        effective_on_turn = on_turn if on_turn is not None else self.on_turn
+
         mode_str = " (Parallel)" if (max_workers or 1) > 1 else ""
         self._log(f"--- Started Scenario: {name}{mode_str} ---")
 
@@ -858,6 +885,7 @@ Evaluate this conversation and respond with this exact JSON structure:
                     auditor_input_tokens += a_in
                     auditor_output_tokens += a_out
                     probe = ModelAuditor.strip_thinking(probe)
+                    self._fire_on_turn(turn, turns, "auditor", effective_on_turn)
 
                 probe_preview = probe[:80] + "..." if len(probe) > 80 else probe
                 self._log(f"PROBE: {probe_preview}", name=name)
@@ -885,6 +913,7 @@ Evaluate this conversation and respond with this exact JSON structure:
                 target_input_tokens += t_in
                 target_output_tokens += t_out
                 response = ModelAuditor.strip_thinking(response)
+                self._fire_on_turn(turn, turns, "target", effective_on_turn)
 
                 response_preview = response[:80] + "..." if len(response) > 80 else response
                 self._log(f"TARGET: {response_preview}", name=name)
@@ -927,6 +956,9 @@ Evaluate this conversation and respond with this exact JSON structure:
                 )
                 judge_input_tokens += j_in
                 judge_output_tokens += j_out
+                # The judge runs once after all turns complete; report it against
+                # the final turn index so consumers can place it at the end.
+                self._fire_on_turn(turns - 1, turns, "judge", effective_on_turn)
                 if fell_back and isinstance(judgment, dict):
                     judgment["judge_fallback"] = "default"
             except Exception as exc:
@@ -989,6 +1021,7 @@ Evaluate this conversation and respond with this exact JSON structure:
         target_params: Optional[Dict[str, Any]] = None,
         judge_params: Optional[Dict[str, Any]] = None,
         auditor_params: Optional[Dict[str, Any]] = None,
+        on_turn: Optional[Callable[[int, int, str], None]] = None,
     ) -> AuditResults:
         if max_workers < 1:
             raise ValueError(
@@ -1058,6 +1091,7 @@ Evaluate this conversation and respond with this exact JSON structure:
                         target_params=target_params,
                         judge_params=judge_params,
                         auditor_params=auditor_params,
+                        on_turn=on_turn,
                     )
                 except Exception as exc:
                     # Don't let one failing scenario abort the whole batch and
@@ -1114,6 +1148,7 @@ Evaluate this conversation and respond with this exact JSON structure:
         target_params: Optional[Dict[str, Any]] = None,
         judge_params: Optional[Dict[str, Any]] = None,
         auditor_params: Optional[Dict[str, Any]] = None,
+        on_turn: Optional[Callable[[int, int, str], None]] = None,
     ) -> AuditResults:
         try:
             asyncio.get_running_loop()
@@ -1128,6 +1163,7 @@ Evaluate this conversation and respond with this exact JSON structure:
                     target_params=target_params,
                     judge_params=judge_params,
                     auditor_params=auditor_params,
+                    on_turn=on_turn,
                 )
             )
         msg = "ModelAuditor.run() cannot be called from an active event loop. Use await <object>.run_async()."

{simpleaudit-0.2.0 → simpleaudit-0.2.1/simpleaudit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: simpleaudit
-Version: 0.2.0
+Version: 0.2.1
 Summary: Lightweight AI Safety Auditing Framework
 Author: SimpleAudit Contributors
 License: MIT
@@ -162,10 +162,10 @@ results.save("./my_audit_results/audit_results.json")
 **💡 View results interactively:**
 ```bash
 # Option 1: Run directly with uvx (no installation needed, requires uv)
-uvx simpleaudit[visualize] serve --results_dir ./my_audit_results
+uvx 'simpleaudit[visualize]' serve --results_dir ./my_audit_results
 
 # Option 2: Install and run locally
-pip install simpleaudit[visualize]
+pip install 'simpleaudit[visualize]'
 simpleaudit serve --results_dir ./my_audit_results
 ```
 This will spin-up a local web server to explore results with scenario details. 👉 [Check for live demo.](https://simulamet-simpleauditvisualization.hf.space)
@@ -173,7 +173,14 @@ See [visualization/README.md](https://github.com/kelkalot/simpleaudit/blob/main/
 
 To share results as a single self-contained HTML file (no server, no JSON upload), use `simpleaudit export-html ./audit_results.json` or the **Download HTML** button in the visualizer.
 
-> **Note:** Option 1 requires [`uv`](https://pypi.org/project/uv/) to be installed ([install guide](https://docs.astral.sh/uv/getting-started/installation/)).
+> **Note:** Option 1 requires [`uv`](https://pypi.org/project/uv/) to be installed ([install guide](https://docs.astral.sh/uv/getting-started/installation/)). The package spec is quoted so your shell doesn't glob-expand `[visualize]`.
+
+<Troubleshooting>
+If the command fails:
+- **`Address already in use`** — port 8000 is taken. Use `--port 8080` or stop the other process.
+- **`uvx: command not found`** — install `uv` first (`curl -LsSf https://astral.sh/uv/install.sh | sh`).
+- **Weird import/module error** — stale cache. Run `uv cache clean` and retry.
+</Troubleshooting>
 
 [![simpleaudit-visualization-ui](https://github.com/user-attachments/assets/f9bbb891-a847-48d4-85d6-6d6d99c9e017)](https://github.com/kelkalot/simpleaudit/blob/main/simpleaudit/visualization/README.md)
 

{simpleaudit-0.2.0 → simpleaudit-0.2.1}/tests/test_experiment_streaming.py

@@ -555,3 +555,76 @@ class TestExperimentEvent:
         assert e.rep_index == 2
         assert e.total_reps == 5
         assert e.result.severity == "pass"
+
+
+# ---------------------------------------------------------------------------
+# on_turn forwarding through run_scenario_reps
+# ---------------------------------------------------------------------------
+
+class TestOnTurnForwarding:
+    """Tests that on_turn is forwarded through run_scenario_reps."""
+
+    def test_on_turn_forwarded_to_each_rep(self):
+        """on_turn should be called for each rep's execution."""
+        calls = []
+
+        def on_turn(turn_index, max_turns, role):
+            calls.append((turn_index, max_turns, role))
+
+        with patch.object(ModelAuditor, '_create_anyllm_client', return_value=MagicMock()):
+            exp = AuditExperiment(
+                models=[{"model": "test-model", "provider": "openai"}],
+                judge_model="judge",
+                judge_provider="openai",
+                show_progress=False,
+                n_repetitions=2,
+            )
+
+            # Mock run_async to capture on_turn
+            async def fake_run_async(self_a, scenarios, **kwargs):
+                # Verify on_turn was passed
+                assert 'on_turn' in kwargs
+                if kwargs['on_turn']:
+                    # Simulate a single turn with target phase
+                    kwargs['on_turn'](0, 1, "target")
+                return _make_results(["pass"])
+
+            with patch.object(ModelAuditor, 'run_async', new=fake_run_async):
+                results = asyncio.run(exp.run_scenario_reps(
+                    model_index=0,
+                    scenario={"name": "test", "description": "desc"},
+                    on_turn=on_turn,
+                ))
+
+        # Should have created 2 reps
+        assert len(results) == 2
+        # on_turn should have been called (at least once per rep)
+        assert len(calls) >= 2
+
+    def test_on_turn_not_forwarded_when_none(self):
+        """When on_turn is None, it should not cause errors."""
+        captured_kwargs = []
+
+        async def fake_run_async(self_a, scenarios, **kwargs):
+            captured_kwargs.append(kwargs)
+            return _make_results(["pass"])
+
+        with patch.object(ModelAuditor, '_create_anyllm_client', return_value=MagicMock()), \
+             patch.object(ModelAuditor, 'run_async', new=fake_run_async):
+            
+            exp = AuditExperiment(
+                models=[{"model": "test-model", "provider": "openai"}],
+                judge_model="judge",
+                judge_provider="openai",
+                show_progress=False,
+                n_repetitions=1,
+            )
+
+            asyncio.run(exp.run_scenario_reps(
+                model_index=0,
+                scenario={"name": "test", "description": "desc"},
+            ))
+
+        # on_turn should not be in kwargs or should be None
+        for kwargs in captured_kwargs:
+            assert 'on_turn' not in kwargs or kwargs['on_turn'] is None

{simpleaudit-0.2.0 → simpleaudit-0.2.1}/tests/test_model_auditor.py

@@ -6,7 +6,7 @@ Run with: pytest tests/test_model_auditor.py -v
 
 import asyncio
 import pytest
-from unittest.mock import Mock, patch, MagicMock
+from unittest.mock import AsyncMock, Mock, patch, MagicMock
 
 from simpleaudit import ModelAuditor, get_scenarios, list_scenario_packs
 # Check for optional provider dependencies
@@ -354,3 +354,171 @@ def test_model_auditor_judge_fields_none_keeps_default():
         )
         assert auditor.judge_fields is None
         assert auditor.judge_response_schema is None  # no explicit schema set
+
+
+# ---------------------------------------------------------------------------
+# on_turn callback
+# ---------------------------------------------------------------------------
+
+class TestOnTurnCallback:
+    """Tests for the on_turn callback in run_scenario."""
+
+    def _make_auditor(self, **kwargs):
+        """Create a ModelAuditor with mocked LLM clients."""
+        with patch.object(ModelAuditor, "_create_anyllm_client", return_value=MagicMock()):
+            return ModelAuditor(
+                model="test-model",
+                provider="openai",
+                judge_model="judge-model",
+                judge_provider="openai",
+                api_key="test-key",
+                show_progress=False,
+                verbose=False,
+                **kwargs,
+            )
+
+    def test_on_turn_called_at_each_phase_exact_sequence(self):
+        """on_turn fires in the exact phase order with correct turn indices.
+
+        With a test_prompt, turn 0 skips probe generation (auditor), so the
+        sequence is: turn0 target -> turn1 auditor -> turn1 target -> judge.
+        The judge runs once after all turns, reported against the final index.
+        """
+        calls = []
+
+        def on_turn(turn_index, max_turns, role):
+            calls.append((turn_index, max_turns, role))
+
+        auditor = self._make_auditor()
+
+        with patch.object(ModelAuditor, "_generate_probe_async", new_callable=AsyncMock) as mock_probe, \
+             patch.object(ModelAuditor, "_call_async", new_callable=AsyncMock) as mock_call, \
+             patch.object(ModelAuditor, "_judge_conversation_async", new_callable=AsyncMock) as mock_judge:
+
+            mock_probe.return_value = ("probe text", 10, 20)
+            mock_call.return_value = ("response text", 30, 40)
+            mock_judge.return_value = ({"severity": "pass"}, 50, 60)
+
+            asyncio.run(auditor.run_scenario(
+                name="test-scenario",
+                description="test description",
+                expected_behavior=["should do X"],
+                test_prompt="initial prompt",
+                max_turns=2,
+                language="English",
+                on_turn=on_turn,
+            ))
+
+        assert calls == [
+            (0, 2, "target"),   # turn 0: test_prompt sent verbatim, no probe
+            (1, 2, "auditor"),  # turn 1: probe generated
+            (1, 2, "target"),   # turn 1: target responds
+            (1, 2, "judge"),    # judge runs once after all turns
+        ]
+
+    def test_on_turn_set_in_init_fires_without_per_call_arg(self):
+        """A callback set at construction time must fire even when run_scenario
+        is called without an explicit on_turn argument (regression: the local
+        param previously shadowed self.on_turn, so init-set callbacks never ran)."""
+        calls = []
+
+        def on_turn(turn_index, max_turns, role):
+            calls.append((turn_index, max_turns, role))
+
+        auditor = self._make_auditor(on_turn=on_turn)
+
+        with patch.object(ModelAuditor, "_generate_probe_async", new_callable=AsyncMock) as mock_probe, \
+             patch.object(ModelAuditor, "_call_async", new_callable=AsyncMock) as mock_call, \
+             patch.object(ModelAuditor, "_judge_conversation_async", new_callable=AsyncMock) as mock_judge:
+
+            mock_probe.return_value = ("probe", 10, 20)
+            mock_call.return_value = ("response", 30, 40)
+            mock_judge.return_value = ({"severity": "pass"}, 50, 60)
+
+            # No on_turn passed here — it must come from __init__.
+            asyncio.run(auditor.run_scenario(
+                name="test",
+                description="desc",
+                max_turns=1,
+                language="English",
+            ))
+
+        assert len(calls) >= 1
+        assert any(role == "target" for _, _, role in calls)
+
+    def test_on_turn_per_call_overrides_init(self):
+        """A per-call on_turn takes precedence over the construction-time one."""
+        init_calls = []
+        call_calls = []
+
+        auditor = self._make_auditor(on_turn=lambda *a: init_calls.append(a))
+
+        with patch.object(ModelAuditor, "_generate_probe_async", new_callable=AsyncMock) as mock_probe, \
+             patch.object(ModelAuditor, "_call_async", new_callable=AsyncMock) as mock_call, \
+             patch.object(ModelAuditor, "_judge_conversation_async", new_callable=AsyncMock) as mock_judge:
+
+            mock_probe.return_value = ("probe", 10, 20)
+            mock_call.return_value = ("response", 30, 40)
+            mock_judge.return_value = ({"severity": "pass"}, 50, 60)
+
+            asyncio.run(auditor.run_scenario(
+                name="test",
+                description="desc",
+                max_turns=1,
+                language="English",
+                on_turn=lambda *a: call_calls.append(a),
+            ))
+
+        assert len(call_calls) >= 1
+        assert init_calls == []  # init callback must NOT fire when per-call given
+
+    def test_on_turn_not_called_when_none(self):
+        """When on_turn is None, no callback should fire (no error)."""
+        auditor = self._make_auditor()
+
+        with patch.object(ModelAuditor, "_generate_probe_async", new_callable=AsyncMock) as mock_probe, \
+             patch.object(ModelAuditor, "_call_async", new_callable=AsyncMock) as mock_call, \
+             patch.object(ModelAuditor, "_judge_conversation_async", new_callable=AsyncMock) as mock_judge:
+
+            mock_probe.return_value = ("probe", 10, 20)
+            mock_call.return_value = ("response", 30, 40)
+            mock_judge.return_value = ({"severity": "pass"}, 50, 60)
+
+            # Should not raise
+            result = asyncio.run(auditor.run_scenario(
+                name="test",
+                description="desc",
+                max_turns=1,
+                language="English",
+                on_turn=None,
+            ))
+
+    def test_on_turn_receives_correct_turn_index(self):
+        """turn_index should increment correctly across turns."""
+        calls = []
+
+        def on_turn(turn_index, max_turns, role):
+            calls.append((turn_index, role))
+
+        auditor = self._make_auditor()
+
+        with patch.object(ModelAuditor, "_generate_probe_async", new_callable=AsyncMock) as mock_probe, \
+             patch.object(ModelAuditor, "_call_async", new_callable=AsyncMock) as mock_call, \
+             patch.object(ModelAuditor, "_judge_conversation_async", new_callable=AsyncMock) as mock_judge:
+
+            mock_probe.return_value = ("probe", 10, 20)
+            mock_call.return_value = ("response", 30, 40)
+            mock_judge.return_value = ({"severity": "pass"}, 50, 60)
+
+            asyncio.run(auditor.run_scenario(
+                name="test",
+                description="desc",
+                max_turns=3,
+                language="English",
+                on_turn=on_turn,
+            ))
+
+        # Extract turn indices for target calls
+        target_calls = [(idx, role) for idx, role in calls if role == "target"]
+        turn_indices = [idx for idx, _ in target_calls]
+        assert turn_indices == [0, 1, 2]  # One target call per turn