inspect-test-utils 0.2.0__py3-none-any.whl

inspect_test_utils/__init__.py

@@ -0,0 +1,37 @@
+"""Reusable testing framework for Inspect AI evaluation tasks.
+
+This framework provides:
+- Model-level mocking (HardcodedModelAPI)
+- Solver-level hardcoded execution
+- Test scorers and tasks
+- Pytest fixtures and assertion helpers
+- Eval runner for integration tests
+"""
+
+from inspect_test_utils.assertions import (
+    assert_contains,
+    assert_eval_score,
+    assert_files_exist,
+    assert_score_in_range,
+)
+from inspect_test_utils.eval_runner import (
+    EvalTestResult,
+    run_eval_test,
+)
+from inspect_test_utils.solvers import (
+    hardcoded_bash_solver,
+    hardcoded_python_solver,
+    inspection_solver,
+)
+
+__all__ = [
+    "assert_contains",
+    "assert_eval_score",
+    "assert_files_exist",
+    "assert_score_in_range",
+    "EvalTestResult",
+    "hardcoded_bash_solver",
+    "hardcoded_python_solver",
+    "inspection_solver",
+    "run_eval_test",
+]

inspect_test_utils/_registry.py

@@ -0,0 +1,39 @@
+from inspect_test_utils.hardcoded import hardcoded
+from inspect_test_utils.mockllm import mockllm_wrapper
+from inspect_test_utils.scanners import (
+    model_roles_scanner,
+    suspicious_behaviour,
+    word_counter,
+)
+from inspect_test_utils.tasks import (
+    configurable_sandbox,
+    guess_number,
+    guess_number_keep_guessing,
+    hardcoded_score,
+    network_sandbox,
+    say_hello,
+    say_hello_with_tools,
+    sometimes_fails_scoring,
+    sometimes_fails_setup,
+    timeout,
+    uses_model_roles,
+)
+
+__all__ = [
+    "configurable_sandbox",
+    "guess_number",
+    "guess_number_keep_guessing",
+    "hardcoded",
+    "hardcoded_score",
+    "mockllm_wrapper",
+    "model_roles_scanner",
+    "network_sandbox",
+    "say_hello",
+    "say_hello_with_tools",
+    "sometimes_fails_scoring",
+    "sometimes_fails_setup",
+    "suspicious_behaviour",
+    "timeout",
+    "uses_model_roles",
+    "word_counter",
+]

inspect_test_utils/assertions.py

@@ -0,0 +1,114 @@
+"""Assertion helpers for Inspect AI evaluation tests.
+
+These functions provide convenient assertions for common test patterns.
+"""
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from inspect_test_utils.eval_runner import EvalTestResult
+
+
+def assert_eval_score(
+    result: "EvalTestResult",
+    expected: float,
+    tolerance: float = 0.01,
+    *,
+    message: str | None = None,
+) -> None:
+    """Assert that the eval score matches the expected value.
+
+    Args:
+        result: The EvalTestResult from run_eval_test.
+        expected: Expected score value.
+        tolerance: Acceptable deviation from expected score.
+        message: Optional custom failure message.
+
+    Raises:
+        AssertionError: If score doesn't match within tolerance.
+    """
+    actual = result.score
+    if actual is None:
+        raise AssertionError(f"No score returned. Error: {result.error}")
+
+    if abs(actual - expected) > tolerance:
+        msg = message or f"Expected score {expected}, got {actual}"
+        if result.explanation:
+            msg += f"\nExplanation: {result.explanation}"
+        raise AssertionError(msg)
+
+
+def assert_score_in_range(
+    result: "EvalTestResult",
+    min_score: float,
+    max_score: float,
+    *,
+    message: str | None = None,
+) -> None:
+    """Assert that the eval score is within a range.
+
+    Args:
+        result: The EvalTestResult from run_eval_test.
+        min_score: Minimum acceptable score (inclusive).
+        max_score: Maximum acceptable score (inclusive).
+        message: Optional custom failure message.
+
+    Raises:
+        AssertionError: If score is outside the range.
+    """
+    actual = result.score
+    if actual is None:
+        raise AssertionError(f"No score returned. Error: {result.error}")
+
+    if not (min_score <= actual <= max_score):
+        msg = message or f"Expected score in [{min_score}, {max_score}], got {actual}"
+        if result.explanation:
+            msg += f"\nExplanation: {result.explanation}"
+        raise AssertionError(msg)
+
+
+def assert_files_exist(
+    files: list[str],
+    actual_files: list[str],
+    *,
+    message: str | None = None,
+) -> None:
+    """Assert that expected files exist in the actual file list.
+
+    Args:
+        files: List of expected file paths/names.
+        actual_files: List of actual files found.
+        message: Optional custom failure message.
+
+    Raises:
+        AssertionError: If any expected file is missing.
+    """
+    missing = [f for f in files if f not in actual_files]
+    if missing:
+        msg = message or f"Missing files: {missing}"
+        msg += f"\nFound: {actual_files}"
+        raise AssertionError(msg)
+
+
+def assert_contains(
+    needle: str,
+    haystack: str,
+    *,
+    message: str | None = None,
+) -> None:
+    """Assert that a string contains a substring.
+
+    Args:
+        needle: String to search for.
+        haystack: String to search in.
+        message: Optional custom failure message.
+
+    Raises:
+        AssertionError: If needle not found in haystack.
+    """
+    if needle not in haystack:
+        msg = message or f"Expected to find '{needle}' in output"
+        # Show a preview of the haystack
+        preview = haystack[:500] + "..." if len(haystack) > 500 else haystack
+        msg += f"\nActual output:\n{preview}"
+        raise AssertionError(msg)

inspect_test_utils/eval_runner.py

@@ -0,0 +1,164 @@
+"""Eval runner helpers for testing Inspect AI evaluations.
+
+Provides a simplified interface for running evals in tests with custom solvers.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from inspect_ai import Task, eval
+from inspect_ai.log import EvalLog
+from inspect_ai.solver import Solver
+
+
+@dataclass
+class EvalTestResult:
+    """Result of running an eval test.
+
+    Attributes:
+        success: Whether the eval completed without errors.
+        score: The score from the first sample (if available).
+        scores: All scores from the eval (for multi-sample tests).
+        explanation: Score explanation (if available).
+        metadata: Score metadata (if available).
+        log: The full EvalLog for detailed inspection.
+        error: Error message if the eval failed.
+    """
+
+    success: bool
+    score: float | None = None
+    scores: list[float] = field(default_factory=list)
+    explanation: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    log: EvalLog | None = None
+    error: str | None = None
+
+    @classmethod
+    def from_log(cls, log: EvalLog) -> "EvalTestResult":
+        """Create an EvalTestResult from an EvalLog."""
+        if log.status != "success":
+            return cls(
+                success=False,
+                log=log,
+                error=f"Eval status: {log.status}. Error: {log.error}",
+            )
+
+        # Extract scores from samples
+        scores: list[float] = []
+        explanation: str | None = None
+        metadata: dict[str, Any] = {}
+
+        if log.samples:
+            for sample in log.samples:
+                if sample.scores:
+                    for score_name, score in sample.scores.items():
+                        if score.value is not None:
+                            if isinstance(score.value, (int, float)):
+                                scores.append(float(score.value))
+                            # Capture first explanation/metadata for convenience
+                            if explanation is None:
+                                explanation = score.explanation
+                            if not metadata:
+                                metadata = score.metadata or {}
+
+        return cls(
+            success=True,
+            score=scores[0] if scores else None,
+            scores=scores,
+            explanation=explanation,
+            metadata=metadata,
+            log=log,
+        )
+
+
+def run_eval_test(
+    task: Task | Callable[[], Task],
+    solver: Solver | list[Solver] | None = None,
+    *,
+    limit: int | None = 1,
+    model: str = "mockllm/model",
+    model_args: dict[str, Any] | None = None,
+    sandbox_cleanup: bool = True,
+    message_limit: int | None = None,
+    **eval_kwargs: Any,
+) -> EvalTestResult:
+    """Run an eval for testing and return a simplified result.
+
+    This is the main entry point for running eval tests. It:
+    - Runs the eval with the provided solver
+    - Uses mockllm by default (no API calls)
+    - Returns a simplified result for assertions
+
+    Args:
+        task: The Task or task function to evaluate.
+        solver: Custom solver(s) to use. If None, uses the task's default solver.
+        limit: Number of samples to run (default 1 for fast tests).
+        model: Model to use (default mockllm/model for no API calls).
+            Use "hardcoded/test" with model_args for HardcodedModelAPI.
+        model_args: Arguments passed to the model provider. For HardcodedModelAPI:
+            - tool_calls: List of commands (strings) or HardcodedToolCall dicts
+            - repetitions: Number of times to cycle through commands (default 1)
+            - answer: Final answer content (default "done")
+        sandbox_cleanup: Whether to cleanup sandbox after test (default True).
+        message_limit: Override message_limit for the task.
+        **eval_kwargs: Additional kwargs passed to eval().
+
+    Returns:
+        EvalTestResult with score and metadata.
+
+    Example using hardcoded solver (bypasses model):
+        result = run_eval_test(
+            code_repair,
+            solver=hardcoded_bash_solver(["sed -i 's/bug/fix/' file.py"]),
+        )
+
+    Example using HardcodedModelAPI (tests full agent loop):
+        result = run_eval_test(
+            code_repair,
+            model="hardcoded/test",
+            model_args={"tool_calls": ["sed -i 's/bug/fix/' file.py"]},
+        )
+    """
+    task_instance = task() if callable(task) else task
+
+    if message_limit is not None:
+        task_instance = Task(
+            dataset=task_instance.dataset,
+            solver=task_instance.solver if solver is None else solver,
+            scorer=task_instance.scorer,
+            message_limit=message_limit,
+            sandbox=task_instance.sandbox,
+            metadata=task_instance.metadata,
+        )
+    elif solver is not None:
+        task_instance = Task(
+            dataset=task_instance.dataset,
+            solver=solver,
+            scorer=task_instance.scorer,
+            message_limit=task_instance.message_limit,
+            sandbox=task_instance.sandbox,
+            metadata=task_instance.metadata,
+        )
+
+    try:
+        call_kwargs: dict[str, Any] = {
+            "limit": limit,
+            "sandbox_cleanup": sandbox_cleanup,
+            **eval_kwargs,
+        }
+        if model_args is not None:
+            call_kwargs["model_args"] = model_args
+
+        logs = eval(
+            task_instance,
+            model=model,
+            **call_kwargs,
+        )
+
+        if not logs:
+            return EvalTestResult(success=False, error="No eval logs returned")
+
+        return EvalTestResult.from_log(logs[0])
+
+    except Exception as e:
+        return EvalTestResult(success=False, error=str(e))

inspect_test_utils/fixtures.py

@@ -0,0 +1,100 @@
+"""Pytest fixtures and configuration for Inspect AI eval testing.
+
+This module provides pytest plugins and fixtures for testing Inspect AI evals.
+Register it in your conftest.py:
+
+    pytest_plugins = ["inspect_test_utils.fixtures"]
+
+Or import fixtures directly:
+
+    from inspect_test_utils.fixtures import skip_sandbox
+"""
+
+import os
+from collections.abc import Generator
+
+import pytest
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    """Register custom markers."""
+    config.addinivalue_line(
+        "markers", "sandbox: marks tests as requiring Docker sandbox (may be slow)"
+    )
+    config.addinivalue_line("markers", "slow: marks tests as slow running")
+    config.addinivalue_line(
+        "markers",
+        "compose(path): specifies the compose.yaml path for sandbox tests",
+    )
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    """Add custom command line options."""
+    parser.addoption(
+        "--skip-sandbox",
+        action="store_true",
+        default=False,
+        help="Skip tests that require Docker sandbox",
+    )
+    parser.addoption(
+        "--run-slow",
+        action="store_true",
+        default=False,
+        help="Run tests marked as slow",
+    )
+
+
+def pytest_collection_modifyitems(
+    config: pytest.Config, items: list[pytest.Item]
+) -> None:
+    """Modify test collection based on markers and options."""
+    skip_sandbox_marker = pytest.mark.skip(reason="--skip-sandbox specified")
+    skip_slow_marker = pytest.mark.skip(reason="need --run-slow option to run")
+
+    for item in items:
+        # Skip sandbox tests if --skip-sandbox is specified
+        if "sandbox" in item.keywords and config.getoption("--skip-sandbox"):
+            item.add_marker(skip_sandbox_marker)
+
+        # Skip slow tests unless --run-slow is specified
+        if "slow" in item.keywords and not config.getoption("--run-slow"):
+            item.add_marker(skip_slow_marker)
+
+
+@pytest.fixture
+def task_root() -> str:
+    """Return the path to the tasks directory."""
+    # Find the tasks directory relative to the test file
+    return os.path.join(os.path.dirname(__file__), "..", "tasks")
+
+
+@pytest.fixture
+def sandbox_timeout() -> int:
+    """Default timeout for sandbox operations in seconds."""
+    return 60
+
+
+# Marker shortcuts for use in tests
+skip_sandbox = pytest.mark.skipif(
+    os.environ.get("SKIP_SANDBOX", "").lower() in ("1", "true", "yes"),
+    reason="SKIP_SANDBOX environment variable is set",
+)
+
+
+@pytest.fixture
+def requires_docker() -> Generator[None, None, None]:
+    """Fixture that skips test if Docker is not available."""
+    import subprocess
+
+    try:
+        result = subprocess.run(
+            ["docker", "info"],
+            capture_output=True,
+            timeout=10,
+        )
+        if result.returncode != 0:
+            pytest.skip("Docker is not running")
+    except (FileNotFoundError, subprocess.TimeoutExpired):
+        pytest.skip("Docker is not available")
+
+    yield

inspect_test_utils/hardcoded.py

@@ -0,0 +1,191 @@
+import json
+import random
+from asyncio import sleep
+from collections.abc import Callable
+from typing import Any, TypedDict, override
+
+import inspect_ai._util.constants
+from inspect_ai.model import (
+    ChatCompletionChoice,
+    ChatMessage,
+    ChatMessageAssistant,
+    GenerateConfig,
+    ModelAPI,
+    ModelCall,
+    ModelOutput,
+    ModelUsage,
+    modelapi,
+)
+from inspect_ai.tool import ToolCall, ToolInfo, ToolChoice
+
+
+class HardcodedToolCall(TypedDict):
+    tool_name: str
+    tool_args: dict[str, Any]
+
+
+class HardcodedModelAPI(ModelAPI):
+    def __init__(
+        self,
+        model_name: str,
+        base_url: str | None = None,
+        api_key: str | None = None,
+        config: GenerateConfig = GenerateConfig(),
+        tool_calls: list[HardcodedToolCall] | str | list[str] | None = None,
+        repetitions: int = 1,
+        answer: str = "done",
+        delay: float = 0.0,
+        concurrency: int = inspect_ai._util.constants.DEFAULT_MAX_CONNECTIONS,
+        failure_rate: float = 0.0,
+        input_tokens: int = 100,
+        output_tokens: int = 50,
+    ):
+        super().__init__(
+            model_name=model_name, base_url=base_url, api_key=api_key, config=config
+        )
+        self.tool_calls = self._parse_tool_calls(tool_calls)
+        self.repetitions = repetitions
+        self.answer = answer
+        self.delay = delay
+        self.concurrency = concurrency
+        self.failure_rate = failure_rate
+        self.input_tokens = input_tokens
+        self.output_tokens = output_tokens
+
+    def _parse_tool_calls(
+        self, tool_calls: list[HardcodedToolCall] | str | list[str] | None
+    ) -> list[HardcodedToolCall]:
+        if tool_calls is None:
+            return []
+
+        # Handle empty list early
+        if isinstance(tool_calls, list) and len(tool_calls) == 0:
+            return []
+
+        # Try to parse JSON if it's a list of strings (could be JSON fragments)
+        if (
+            isinstance(tool_calls, list)
+            and len(tool_calls) > 0
+            and isinstance(tool_calls[0], str)
+        ):
+            try:
+                tool_calls = json.loads("[" + ",".join(tool_calls) + "]")
+            except json.JSONDecodeError:
+                pass
+        elif isinstance(tool_calls, str):
+            try:
+                tool_calls = json.loads(tool_calls)
+            except json.JSONDecodeError:
+                pass
+
+        if isinstance(tool_calls, str):
+            tool_calls = [tool_calls]
+        if len(tool_calls) == 0:
+            return []
+        if isinstance(tool_calls[0], str):
+            return [
+                HardcodedToolCall(tool_name="bash", tool_args={"cmd": cmd})
+                for cmd in tool_calls
+            ]
+        for tool_call in tool_calls:
+            if not isinstance(tool_call, dict):
+                raise ValueError(f"Invalid tool call: {tool_call}")
+            if "tool_name" not in tool_call or "tool_args" not in tool_call:
+                raise ValueError(f"Invalid tool call: {tool_call}")
+            if not isinstance(tool_call.get("tool_args"), dict):
+                raise ValueError(f"Invalid tool_args (must be dict): {tool_call}")
+        return tool_calls
+
+    def max_connections(self) -> int:
+        return self.concurrency
+
+    @override
+    async def generate(
+        self,
+        input: list[ChatMessage],
+        tools: list[ToolInfo],
+        tool_choice: ToolChoice,
+        config: GenerateConfig,
+        record_call: Callable[[ModelCall], None] | None = None,
+    ) -> ModelOutput | tuple[ModelOutput | Exception, ModelCall]:
+        index = sum(1 for m in input if m.role == "assistant")
+        next_tool_call_index = (
+            int(index) % len(self.tool_calls) if self.tool_calls else 0
+        )
+        repetition_count = int(index) // len(self.tool_calls) if self.tool_calls else 1
+        next_tool_call = (
+            self.tool_calls[next_tool_call_index]
+            if next_tool_call_index < len(self.tool_calls)
+            else None
+        )
+
+        model_call = ModelCall.create(
+            request={"hardcoded": "test"}, response=None, filter=None, time=None
+        )
+        if record_call:
+            record_call(model_call)
+
+        if self.delay > 0:
+            await sleep(self.delay)
+
+        if random.random() < self.failure_rate:
+            model_call.response = {"failure": "test"}
+            try:
+                raise Exception("Failure")
+            except Exception as e:
+                return e, model_call
+
+        if repetition_count >= self.repetitions:
+            submit_tool = next((tool for tool in tools if tool.name == "submit"), None)
+            if submit_tool is None:
+                message = ChatMessageAssistant(content=self.answer)
+            else:
+                message = ChatMessageAssistant(
+                    content="I will now submit my answer.",
+                    tool_calls=[
+                        ToolCall(
+                            id="hardcoded_submit",
+                            function=submit_tool.name,
+                            arguments={"answer": self.answer},
+                        )
+                    ],
+                )
+
+            choice = ChatCompletionChoice(
+                message=message,
+                stop_reason="stop",
+            )
+        else:
+            tool_name = next_tool_call["tool_name"]
+            tool_args = next_tool_call["tool_args"]
+
+            message = ChatMessageAssistant(
+                content=f"Executing {tool_name} with args: {tool_args}",
+                tool_calls=[
+                    ToolCall(
+                        id=f"hardcoded_{index}",
+                        function=tool_name,
+                        arguments=tool_args,
+                    )
+                ],
+            )
+            choice = ChatCompletionChoice(message=message)
+
+        model_call.response = {"test": "hardcoded"}
+        return ModelOutput(
+            model="hardcoded",
+            choices=[choice],
+            usage=ModelUsage(
+                input_tokens=self.input_tokens,
+                output_tokens=self.output_tokens,
+                total_tokens=self.input_tokens + self.output_tokens,
+            ),
+        ), model_call
+
+    def should_retry(self, ex: Exception) -> bool:
+        return True
+
+
+@modelapi(name="hardcoded")
+def hardcoded() -> type[ModelAPI]:
+    return HardcodedModelAPI

inspect_test_utils/mockllm.py

@@ -0,0 +1,48 @@
+from typing import Any, Iterable
+
+from inspect_ai.model import (
+    GenerateConfig,
+    ModelAPI,
+    ModelInfo,
+    ModelOutput,
+    modelapi,
+    set_model_info,
+)
+from inspect_ai.model._providers.mockllm import MockLLM
+from pydantic import TypeAdapter
+
+
+class MockLLMWrapper(MockLLM):
+    """A simple MockLLM wrapper that parses custom model outputs given as a dict. Useful
+    when configuring mockllm from an eval set config.
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        base_url: str | None = None,
+        api_key: str | None = None,
+        config: GenerateConfig = GenerateConfig(),
+        custom_outputs: Iterable[ModelOutput] | None = None,
+        **model_args: dict[str, Any],
+    ) -> None:
+        parsed_outputs = (
+            TypeAdapter(list[ModelOutput]).validate_python(custom_outputs)
+            if custom_outputs
+            else None
+        )
+        super().__init__(model_name, base_url, api_key, config, parsed_outputs, **model_args)
+
+        # Need to register this so cost tracking works
+        set_model_info(
+            f"mockllm_wrapper/{self.model_name}",
+            ModelInfo()
+        )
+
+    def canonical_name(self):
+        return f"mockllm_wrapper/{self.model_name}"
+
+
+@modelapi(name="mockllm_wrapper")
+def mockllm_wrapper() -> type[ModelAPI]:
+    return MockLLMWrapper