mcpbr 0.5.2__tar.gz → 0.5.4__tar.gz

{mcpbr-0.5.2 → mcpbr-0.5.4}/.claude-plugin/marketplace.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
   "name": "mcpbr",
-  "version": "0.5.2",
+  "version": "0.5.4",
   "description": "mcpbr - MCP Benchmark Runner plugin marketplace",
   "owner": {
     "name": "mcpbr Contributors",
@@ -11,7 +11,7 @@
     {
       "name": "mcpbr",
       "description": "Expert benchmark runner for MCP servers using mcpbr. Handles Docker checks, config generation, and result parsing.",
-      "version": "0.5.2",
+      "version": "0.5.4",
       "author": {
         "name": "mcpbr Contributors"
       },

{mcpbr-0.5.2 → mcpbr-0.5.4}/.claude-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@greynewell/mcpbr-claude-plugin",
-  "version": "0.5.2",
+  "version": "0.5.4",
   "description": "Claude Code plugin for mcpbr - Expert benchmark runner for MCP servers with specialized skills",
   "keywords": [
     "claude-code",

{mcpbr-0.5.2 → mcpbr-0.5.4}/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "mcpbr",
-  "version": "0.5.2",
+  "version": "0.5.4",
   "description": "Expert benchmark runner for MCP servers using mcpbr. Handles Docker checks, config generation, and result parsing.",
   "schema_version": "1.0"
 }

{mcpbr-0.5.2 → mcpbr-0.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mcpbr
-Version: 0.5.2
+Version: 0.5.4
 Summary: Model Context Protocol Benchmark Runner - evaluate MCP servers against software engineering benchmarks
 Project-URL: Homepage, https://github.com/greynewell/mcpbr
 Project-URL: Repository, https://github.com/greynewell/mcpbr

{mcpbr-0.5.2 → mcpbr-0.5.4}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@greynewell/mcpbr",
-  "version": "0.5.2",
+  "version": "0.5.4",
   "description": "Model Context Protocol Benchmark Runner - CLI tool for evaluating MCP servers",
   "keywords": [
     "mcpbr",

{mcpbr-0.5.2 → mcpbr-0.5.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "mcpbr"
-version = "0.5.2"
+version = "0.5.4"
 description = "Model Context Protocol Benchmark Runner - evaluate MCP servers against software engineering benchmarks"
 readme = "README.md"
 license = "MIT"

{mcpbr-0.5.2 → mcpbr-0.5.4}/src/mcpbr/__init__.py

@@ -3,4 +3,4 @@
 A benchmark runner for evaluating MCP servers against SWE-bench tasks.
 """
 
-__version__ = "0.5.1"
+__version__ = "0.5.4"

{mcpbr-0.5.2 → mcpbr-0.5.4}/src/mcpbr/evaluation.py

@@ -93,43 +93,52 @@ async def apply_patch(
 
     workdir = workdir or env.workdir
 
-    # Reset repository to clean state before applying patch
-    # The agent modified files directly, so we need to restore HEAD state
-    await env.exec_command("git reset --hard HEAD", timeout=30, workdir=workdir)
-    await env.exec_command("git clean -fd", timeout=30, workdir=workdir)
-
-    await env.write_file("fix.patch", patch, workdir=workdir)
+    # Use longer timeouts for git operations — under concurrent load,
+    # Docker exec can be slow and 30s is insufficient (#399).
+    try:
+        # Reset repository to clean state before applying patch
+        # The agent modified files directly, so we need to restore HEAD state
+        await env.exec_command("git reset --hard HEAD", timeout=120, workdir=workdir)
+        await env.exec_command("git clean -fd", timeout=120, workdir=workdir)
 
-    exit_code, stdout, stderr = await env.exec_command(
-        "git apply --check fix.patch",
-        timeout=30,
-        workdir=workdir,
-    )
+        await env.write_file("fix.patch", patch, workdir=workdir)
 
-    if exit_code != 0:
-        exit_code2, stdout2, stderr2 = await env.exec_command(
-            "git apply --check -3 fix.patch",
-            timeout=30,
-            workdir=workdir,
-        )
-        if exit_code2 != 0:
-            return False, f"Patch does not apply: {stderr or stderr2}"
         exit_code, stdout, stderr = await env.exec_command(
-            "git apply -3 fix.patch",
-            timeout=30,
-            workdir=workdir,
-        )
-    else:
-        exit_code, stdout, stderr = await env.exec_command(
-            "git apply fix.patch",
-            timeout=30,
+            "git apply --check fix.patch",
+            timeout=120,
             workdir=workdir,
         )
 
-    if exit_code != 0:
-        return False, f"Failed to apply patch: {stderr}"
+        if exit_code != 0:
+            exit_code2, stdout2, stderr2 = await env.exec_command(
+                "git apply --check -3 fix.patch",
+                timeout=120,
+                workdir=workdir,
+            )
+            if exit_code2 != 0:
+                return False, f"Patch does not apply: {stderr or stderr2}"
+            exit_code, stdout, stderr = await env.exec_command(
+                "git apply -3 fix.patch",
+                timeout=120,
+                workdir=workdir,
+            )
+        else:
+            exit_code, stdout, stderr = await env.exec_command(
+                "git apply fix.patch",
+                timeout=120,
+                workdir=workdir,
+            )
 
-    return True, ""
+        if exit_code != 0:
+            return False, f"Failed to apply patch: {stderr}"
+
+        return True, ""
+
+    except (TimeoutError, asyncio.TimeoutError):
+        # Catch exec_command timeouts here so they don't bubble up as
+        # asyncio.TimeoutError to the harness, which would misclassify
+        # this as an agent/eval timeout (#399).
+        return False, "Docker exec timed out during patch application"
 
 
 async def run_tests(
@@ -282,38 +291,43 @@ async def _apply_test_patch(
 
     workdir = workdir or env.workdir
 
-    await env.write_file("test.patch", test_patch, workdir=workdir)
-
-    exit_code, stdout, stderr = await env.exec_command(
-        "git apply --check test.patch",
-        timeout=30,
-        workdir=workdir,
-    )
+    try:
+        await env.write_file("test.patch", test_patch, workdir=workdir)
 
-    if exit_code != 0:
         exit_code, stdout, stderr = await env.exec_command(
-            "git apply --check -3 test.patch",
-            timeout=30,
+            "git apply --check test.patch",
+            timeout=120,
             workdir=workdir,
         )
+
+        if exit_code != 0:
+            exit_code, stdout, stderr = await env.exec_command(
+                "git apply --check -3 test.patch",
+                timeout=120,
+                workdir=workdir,
+            )
+            if exit_code != 0:
+                return True, ""
+            exit_code, stdout, stderr = await env.exec_command(
+                "git apply -3 test.patch",
+                timeout=120,
+                workdir=workdir,
+            )
+        else:
+            exit_code, stdout, stderr = await env.exec_command(
+                "git apply test.patch",
+                timeout=120,
+                workdir=workdir,
+            )
+
         if exit_code != 0:
             return True, ""
-        exit_code, stdout, stderr = await env.exec_command(
-            "git apply -3 test.patch",
-            timeout=30,
-            workdir=workdir,
-        )
-    else:
-        exit_code, stdout, stderr = await env.exec_command(
-            "git apply test.patch",
-            timeout=30,
-            workdir=workdir,
-        )
 
-    if exit_code != 0:
         return True, ""
 
-    return True, ""
+    except (TimeoutError, asyncio.TimeoutError):
+        # Don't let exec timeouts bubble up to the harness (#399)
+        return True, ""
 
 
 async def evaluate_patch(
@@ -356,7 +370,14 @@ async def evaluate_patch(
 
     # Skip dependency installation for pre-built images (already done)
     if not env.uses_prebuilt:
-        await _install_dependencies(env)
+        try:
+            await _install_dependencies(env)
+        except (TimeoutError, asyncio.TimeoutError):
+            return EvaluationResult(
+                resolved=False,
+                patch_applied=True,
+                error="Docker exec timed out during dependency installation",
+            )
 
     repo = task.get("repo")
 

{mcpbr-0.5.2 → mcpbr-0.5.4}/src/mcpbr/harness.py

@@ -1,6 +1,7 @@
 """Main evaluation harness orchestrating parallel task execution."""
 
 import asyncio
+import logging
 import time
 from dataclasses import dataclass
 from datetime import datetime, timezone
@@ -29,6 +30,7 @@ from .pricing import calculate_cost
 from .profiler import PerformanceProfiler
 
 console = Console()
+logger = logging.getLogger(__name__)
 
 
 class SimpleNamespace:
@@ -56,6 +58,57 @@ def dict_to_namespace(data: Any) -> Any:
         return data
 
 
+# -- Cold-start mitigation helpers (#401) ------------------------------------
+
+# Seconds between each task launch in the first concurrent batch.
+_STAGGER_INTERVAL = 1.0
+
+
+def _stagger_delay(task_index: int, max_concurrent: int) -> float:
+    """Return the startup delay for a task to avoid cold-start contention.
+
+    Only the first batch (indices 0 .. max_concurrent-1) is staggered.
+    The very first task starts immediately; subsequent tasks in the batch
+    get an increasing delay so Docker image pulls and container creation
+    don't all hit at once.
+
+    Args:
+        task_index: Zero-based index of the task in launch order.
+        max_concurrent: Semaphore size / max parallelism.
+
+    Returns:
+        Delay in seconds (0.0 means start immediately).
+    """
+    if max_concurrent <= 1:
+        return 0.0
+    # Only stagger the first batch
+    if task_index >= max_concurrent:
+        return 0.0
+    return task_index * _STAGGER_INTERVAL
+
+
+def _should_retry_zero_iteration(result: dict[str, Any]) -> bool:
+    """Check whether a task result indicates a cold-start failure worth retrying.
+
+    A cold-start failure is characterised by zero iterations AND zero tokens
+    AND a timeout status — the agent never actually ran.
+
+    Args:
+        result: Single-run result dict from _run_mcp_evaluation or _run_baseline_evaluation.
+
+    Returns:
+        True if the result looks like a cold-start failure.
+    """
+    if result.get("status") != "timeout":
+        return False
+    if result.get("iterations", -1) != 0:
+        return False
+    tokens = result.get("tokens", {})
+    if tokens.get("input", -1) != 0 or tokens.get("output", -1) != 0:
+        return False
+    return True
+
+
 @dataclass
 class TaskResult:
     """Result for a single task."""
@@ -302,6 +355,24 @@ async def run_single_task(
                     mcp_server_config=config.mcp_server_a,
                     server_name="server_a",
                 )
+                # Retry once on cold-start failure (#401)
+                if result.mcp_server_a and _should_retry_zero_iteration(result.mcp_server_a):
+                    logger.info(
+                        "Retrying MCP server_a task %s (zero-iteration cold-start)", instance_id
+                    )
+                    result.mcp_server_a = await _run_mcp_evaluation(
+                        task,
+                        config,
+                        docker_manager,
+                        benchmark,
+                        verbose,
+                        verbosity,
+                        mcp_log_writer_a if mcp_log_writer_a else log_file,
+                        cache,
+                        mcp_logs_dir,
+                        mcp_server_config=config.mcp_server_a,
+                        server_name="server_a",
+                    )
             finally:
                 if mcp_log_writer_a:
                     mcp_log_writer_a.close()
@@ -324,6 +395,24 @@ async def run_single_task(
                     mcp_server_config=config.mcp_server_b,
                     server_name="server_b",
                 )
+                # Retry once on cold-start failure (#401)
+                if result.mcp_server_b and _should_retry_zero_iteration(result.mcp_server_b):
+                    logger.info(
+                        "Retrying MCP server_b task %s (zero-iteration cold-start)", instance_id
+                    )
+                    result.mcp_server_b = await _run_mcp_evaluation(
+                        task,
+                        config,
+                        docker_manager,
+                        benchmark,
+                        verbose,
+                        verbosity,
+                        mcp_log_writer_b if mcp_log_writer_b else log_file,
+                        cache,
+                        mcp_logs_dir,
+                        mcp_server_config=config.mcp_server_b,
+                        server_name="server_b",
+                    )
             finally:
                 if mcp_log_writer_b:
                     mcp_log_writer_b.close()
@@ -344,6 +433,20 @@ async def run_single_task(
                     cache,
                     mcp_logs_dir,
                 )
+                # Retry once on cold-start failure (#401)
+                if result.mcp and _should_retry_zero_iteration(result.mcp):
+                    logger.info("Retrying MCP task %s (zero-iteration cold-start)", instance_id)
+                    result.mcp = await _run_mcp_evaluation(
+                        task,
+                        config,
+                        docker_manager,
+                        benchmark,
+                        verbose,
+                        verbosity,
+                        mcp_log_writer if mcp_log_writer else log_file,
+                        cache,
+                        mcp_logs_dir,
+                    )
             finally:
                 if mcp_log_writer:
                     mcp_log_writer.close()
@@ -363,6 +466,19 @@ async def run_single_task(
                 baseline_log_writer if baseline_log_writer else log_file,
                 cache,
             )
+            # Retry once on cold-start failure (#401)
+            if result.baseline and _should_retry_zero_iteration(result.baseline):
+                logger.info("Retrying baseline task %s (zero-iteration cold-start)", instance_id)
+                result.baseline = await _run_baseline_evaluation(
+                    task,
+                    config,
+                    docker_manager,
+                    benchmark,
+                    verbose,
+                    verbosity,
+                    baseline_log_writer if baseline_log_writer else log_file,
+                    cache,
+                )
         finally:
             if baseline_log_writer:
                 baseline_log_writer.close()
@@ -539,7 +655,15 @@ async def _run_mcp_evaluation(
         if env:
             # Track Docker teardown time
             teardown_start = time.time()
-            await env.cleanup()
+            try:
+                await asyncio.wait_for(env.cleanup(), timeout=60)
+            except (asyncio.TimeoutError, Exception) as cleanup_err:
+                logger.warning("Container cleanup failed for MCP task: %s", cleanup_err)
+                try:
+                    if hasattr(env, "container") and env.container:
+                        env.container.remove(force=True)
+                except Exception:
+                    pass
             if profiler:
                 teardown_end = time.time()
                 profiler.record_docker_teardown(teardown_end - teardown_start)
@@ -695,7 +819,15 @@ async def _run_baseline_evaluation(
         if env:
             # Track Docker teardown time
             teardown_start = time.time()
-            await env.cleanup()
+            try:
+                await asyncio.wait_for(env.cleanup(), timeout=60)
+            except (asyncio.TimeoutError, Exception) as cleanup_err:
+                logger.warning("Container cleanup failed for baseline task: %s", cleanup_err)
+                try:
+                    if hasattr(env, "container") and env.container:
+                        env.container.remove(force=True)
+                except Exception:
+                    pass
             if profiler:
                 teardown_end = time.time()
                 profiler.record_docker_teardown(teardown_end - teardown_start)
@@ -1013,9 +1145,10 @@ async def run_evaluation(
     semaphore = asyncio.Semaphore(config.max_concurrent)
     budget_exceeded = False
     current_cost = 0.0
+    _task_launch_counter = 0
 
     async def run_with_semaphore(task: dict[str, Any]) -> TaskResult | None:
-        nonlocal current_cost, budget_exceeded
+        nonlocal current_cost, budget_exceeded, _task_launch_counter
 
         # Check budget before running task
         if config.budget and current_cost >= config.budget:
@@ -1023,6 +1156,15 @@ async def run_evaluation(
             return None
 
         async with semaphore:
+            # Stagger first-batch launches to avoid cold-start contention (#401).
+            # Delay is inside the semaphore so the sleeping task holds its slot
+            # and later tasks cannot leapfrog ahead of the first batch.
+            my_index = _task_launch_counter
+            _task_launch_counter += 1
+            delay = _stagger_delay(my_index, config.max_concurrent)
+            if delay > 0:
+                await asyncio.sleep(delay)
+
             result = await run_single_task(
                 task,
                 config,

{mcpbr-0.5.2 → mcpbr-0.5.4}/src/mcpbr/infrastructure/azure.py

@@ -17,6 +17,7 @@ except ImportError:
 
 from rich.console import Console
 
+from .. import __version__
 from ..config import HarnessConfig
 from .base import InfrastructureProvider
 
@@ -342,9 +343,9 @@ class AzureProvider(InfrastructureProvider):
         else:
             console.print("[green]✓ Node.js installed[/green]")
 
-        # Step 4: Install mcpbr
-        console.print("[cyan]Installing mcpbr...[/cyan]")
-        step4_cmd = f"python{py_ver} -m pip install mcpbr"
+        # Step 4: Install mcpbr (pin to local version)
+        console.print(f"[cyan]Installing mcpbr=={__version__}...[/cyan]")
+        step4_cmd = f"python{py_ver} -m pip install mcpbr=={__version__}"
         exit_code, _stdout, stderr = await self._ssh_exec(step4_cmd, timeout=300)
         if exit_code != 0:
             console.print(f"[yellow]⚠ mcpbr install issues: {stderr[:300]}[/yellow]")

mcpbr-0.5.4/tests/test_cold_start.py

@@ -0,0 +1,130 @@
+"""Tests for cold-start staggering and zero-iteration retry logic."""
+
+import asyncio
+
+import pytest
+
+from mcpbr.harness import TaskResult, _should_retry_zero_iteration, _stagger_delay
+
+
+class TestStaggeredStarts:
+    """Verify that concurrent task launches are staggered to avoid cold-start failures."""
+
+    @pytest.mark.asyncio
+    async def test_tasks_are_staggered(self) -> None:
+        """First-batch tasks should not all start at the same instant.
+
+        When max_concurrent > 1, the semaphore wrapper should insert a small
+        delay between task launches so Docker isn't overwhelmed by simultaneous
+        image pulls and container startups.
+        """
+        launch_times: list[float] = []
+        loop = asyncio.get_running_loop()
+
+        async def fake_run_single_task(task):
+            launch_times.append(loop.time())
+            await asyncio.sleep(0.05)  # Simulate brief work
+            return TaskResult(instance_id=f"task-{len(launch_times)}")
+
+        tasks = [{"instance_id": f"task-{i}"} for i in range(5)]
+        max_concurrent = 5  # All 5 could start at once without staggering
+
+        semaphore = asyncio.Semaphore(max_concurrent)
+        task_counter = 0
+
+        async def run_with_semaphore(task):
+            nonlocal task_counter
+            async with semaphore:
+                my_index = task_counter
+                task_counter += 1
+                delay = _stagger_delay(my_index, max_concurrent)
+                if delay > 0:
+                    await asyncio.sleep(delay)
+                return await fake_run_single_task(task)
+
+        async_tasks = [asyncio.create_task(run_with_semaphore(t)) for t in tasks]
+        await asyncio.gather(*async_tasks)
+
+        assert len(launch_times) == 5
+
+        # The first and last task should be separated by at least some delay
+        spread = launch_times[-1] - launch_times[0]
+        assert spread > 0.1, (
+            f"Tasks launched with only {spread:.3f}s spread — expected staggering to space them out"
+        )
+
+    @pytest.mark.asyncio
+    async def test_stagger_delay_values(self) -> None:
+        """_stagger_delay should return increasing delays for the first batch."""
+        # First task: no delay
+        assert _stagger_delay(0, max_concurrent=5) == 0.0
+
+        # Subsequent first-batch tasks: increasing delay
+        d1 = _stagger_delay(1, max_concurrent=5)
+        d2 = _stagger_delay(2, max_concurrent=5)
+        assert d1 > 0
+        assert d2 > d1
+
+        # Tasks beyond the first batch: no delay
+        assert _stagger_delay(5, max_concurrent=5) == 0.0
+        assert _stagger_delay(10, max_concurrent=5) == 0.0
+
+    @pytest.mark.asyncio
+    async def test_stagger_delay_single_concurrent(self) -> None:
+        """With max_concurrent=1, no staggering is needed."""
+        assert _stagger_delay(0, max_concurrent=1) == 0.0
+        assert _stagger_delay(1, max_concurrent=1) == 0.0
+
+
+class TestZeroIterationRetry:
+    """Verify that _should_retry_zero_iteration detects cold-start failures."""
+
+    @pytest.mark.asyncio
+    async def test_detects_cold_start_failure(self) -> None:
+        """Zero iterations + zero tokens + timeout = cold-start failure."""
+        zero_iter_result = {
+            "resolved": False,
+            "patch_applied": False,
+            "status": "timeout",
+            "error": "Timeout",
+            "tokens": {"input": 0, "output": 0},
+            "iterations": 0,
+            "tool_calls": 0,
+            "cost": 0.0,
+            "runtime_seconds": 236.0,
+        }
+        assert _should_retry_zero_iteration(zero_iter_result) is True
+
+    @pytest.mark.asyncio
+    async def test_completed_task_not_retried(self) -> None:
+        """A task that completed successfully should never be retried."""
+        good_result = {
+            "resolved": True,
+            "status": "completed",
+            "iterations": 20,
+            "tokens": {"input": 10000, "output": 5000},
+        }
+        assert _should_retry_zero_iteration(good_result) is False
+
+    @pytest.mark.asyncio
+    async def test_nonzero_iteration_timeout_not_retried(self) -> None:
+        """A timeout with real iterations is a genuine timeout, not cold-start."""
+        real_timeout = {
+            "resolved": False,
+            "status": "timeout",
+            "iterations": 5,
+            "tokens": {"input": 3000, "output": 1500},
+        }
+        assert _should_retry_zero_iteration(real_timeout) is False
+
+    @pytest.mark.asyncio
+    async def test_non_timeout_error_not_retried(self) -> None:
+        """Zero iterations from a non-timeout error should not trigger retry."""
+        error_result = {
+            "resolved": False,
+            "status": "error",
+            "error": "Something broke",
+            "iterations": 0,
+            "tokens": {"input": 0, "output": 0},
+        }
+        assert _should_retry_zero_iteration(error_result) is False

{mcpbr-0.5.2 → mcpbr-0.5.4}/tests/test_mcp_logging.py

@@ -61,7 +61,8 @@ class TestMCPLogging:
                 1,
                 "",
                 "npx: command not found",
-            ),  # MCP registration fails
+            ),  # .mcp.json write fails
+            (0, "", ""),  # chown .mcp.json
             (0, "", ""),  # cleanup temp files
         ]
 
@@ -87,7 +88,7 @@ class TestMCPLogging:
 
         # Verify registration failure was caught
         assert result.success is False
-        assert "MCP server registration failed" in result.error
+        assert "MCP config write failed" in result.error
         assert "npx: command not found" in result.error
 
         # Verify cleanup was called
@@ -110,6 +111,7 @@ class TestMCPLogging:
             (0, "", ""),  # env file write
             (0, "", ""),  # chown env
             (1, "Server starting...\nInitialization failed", "Error: Missing API key"),
+            (0, "", ""),  # chown .mcp.json
             (0, "", ""),  # cleanup
         ]
 
@@ -133,10 +135,10 @@ class TestMCPLogging:
                 task_id="test_id",
             )
 
-        # Verify both stderr and stdout are in error message
+        # Verify stderr is in error message and stdout is captured separately
         assert "Error: Missing API key" in result.error
-        assert "Server starting" in result.error or "Initialization failed" in result.error
         assert result.stdout is not None
+        assert "Server starting" in result.stdout or "Initialization failed" in result.stdout
 
     @pytest.mark.asyncio
     async def test_mcp_timeout_cleanup(self, harness: ClaudeCodeHarness) -> None:
@@ -177,8 +179,7 @@ class TestMCPLogging:
 
         # Verify timeout was caught
         assert result.success is False
-        assert "timed out after 60s" in result.error
-        assert "failed to start or is hanging" in result.error
+        assert "Failed to write MCP configuration file" in result.error
 
         # Verify cleanup was called
         cleanup_calls = [
@@ -255,9 +256,10 @@ Debug: Cache miss for /workspace/"""
             (0, "", ""),  # chown prompt
             (0, "", ""),  # env file write
             (0, "", ""),  # chown env
-            (0, "MCP server registered successfully", ""),  # MCP registration
-            (0, "", ""),  # MCP server remove (cleanup)
-            (0, "", ""),  # rm temp files (cleanup)
+            (0, "MCP server registered successfully", ""),  # .mcp.json write
+            (0, "", ""),  # chown .mcp.json
+            (0, "", ""),  # rm .mcp.json (exit_code != 0 path)
+            (0, "", ""),  # rm temp files (finally cleanup)
         ]
 
         # Mock streaming execution with our test output