pytest-inspect-evals 0.1.0__py3-none-any.whl

pytest_inspect_evals/__init__.py

@@ -0,0 +1,10 @@
+"""pytest plugin with shared test gates, fixtures and helpers for Inspect AI evaluations"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+# The version is set once, in pyproject.toml; this reads it back from the
+# installed package metadata.
+try:
+    __version__ = version("pytest-inspect-evals")
+except PackageNotFoundError:  # a source tree that was never installed
+    __version__ = "unknown"

pytest_inspect_evals/_hf.py

@@ -0,0 +1,69 @@
+"""Hugging Face handling for tests marked huggingface."""
+
+import os
+from importlib.util import find_spec
+
+import pytest
+
+
+def hf_disable_tokenizer_parallelism() -> None:
+    """Disable HF tokenizers parallelism to avoid segfaults."""
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+def hf_configure_logging() -> None:
+    """Raise datasets and huggingface_hub logging to INFO when datasets is installed."""
+    if find_spec("datasets") is None:
+        return
+    import datasets
+    import huggingface_hub
+
+    datasets.logging.set_verbosity_info()
+    huggingface_hub.logging.set_verbosity_info()
+
+
+def hf_apply_collection_markers(items: list[pytest.Item]) -> None:
+    hf_items = [item for item in items if item.get_closest_marker("huggingface") is not None]
+
+    if not hf_items:
+        return
+
+    if not os.environ.get("HF_TOKEN", "").strip():
+        for item in hf_items:
+            item.add_marker(pytest.mark.skip(reason=f"HF_TOKEN not set ({item.name})"))
+
+    else:
+        for item in hf_items:
+            item.add_marker(pytest.mark.flaky(reruns=2, reruns_delay=60.0))
+
+
+def _is_hf_gated_dataset_failure(
+    item: pytest.Item, call: pytest.CallInfo, report: pytest.TestReport
+) -> bool:
+    return (
+        report.when == "call"
+        and report.failed
+        and item.get_closest_marker("huggingface") is not None
+        and bool(call.excinfo)
+        and is_gated_dataset_exception(call.excinfo.value)  # type: ignore
+    )
+
+
+def hf_convert_gated_failure_to_skip(
+    item: pytest.Item, call: pytest.CallInfo, report: pytest.TestReport
+) -> None:
+    if _is_hf_gated_dataset_failure(item, call, report):
+        report.outcome = "skipped"
+        line_number = item.reportinfo()[1] or 0
+        report.longrepr = (
+            str(item.fspath),
+            line_number,
+            f"Skipped: Gated dataset ({item.name})",
+        )
+
+
+def is_gated_dataset_exception(exc: BaseException) -> bool:
+    exc_name = type(exc).__name__
+    return exc_name == "GatedRepoError" or (
+        exc_name == "DatasetNotFoundError" and "gated dataset" in str(exc).lower()
+    )

pytest_inspect_evals/_windows.py

@@ -0,0 +1,25 @@
+"""Skip tests that can't run on Windows."""
+
+import sys
+
+import pytest
+
+
+def windows_skip_unsupported_tests(items: list[pytest.Item], platform: str | None = None) -> None:
+    """Skip POSIX-only and Docker tests on Windows."""
+    if (platform or sys.platform) != "win32":
+        return
+
+    for item in items:
+        if item.get_closest_marker("posix_only") is not None:
+            item.add_marker(
+                pytest.mark.skip(
+                    reason=f"Skipping {item.name}: test requires POSIX system (not supported on Windows)"
+                )
+            )
+        if item.get_closest_marker("docker") is not None:
+            item.add_marker(
+                pytest.mark.skip(
+                    reason=f"Skipping {item.name}: Docker tests are not supported on Windows CI"
+                )
+            )

pytest_inspect_evals/assertions.py

@@ -0,0 +1,145 @@
+"""Assertions for Inspect task structure and eval results."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from inspect_ai import Task
+from inspect_ai import eval as inspect_eval
+from inspect_ai.dataset import MemoryDataset, Sample
+from inspect_ai.log import EvalLog
+from inspect_ai.scorer import Scorer
+
+from pytest_inspect_evals.solvers import mock_solver_with_output
+
+DisplayType = Literal["full", "conversation", "rich", "plain", "log", "none"]
+
+
+def assert_task_structure(task: Task, *, sandbox_type: str | None = None) -> None:
+    """Assert that a Task has dataset, solver, and scorer.
+
+    Args:
+        task: The Task object to validate.
+        sandbox_type: If set, also assert task.sandbox.type matches.
+    """
+    assert task.dataset is not None, "task.dataset is None"
+    assert task.solver is not None, "task.solver is None"
+    assert task.scorer is not None, "task.scorer is None"
+
+    if sandbox_type is not None:
+        assert task.sandbox is not None, (
+            f"task.sandbox is None, expected sandbox_type='{sandbox_type}'"
+        )
+        assert task.sandbox.type == sandbox_type, (
+            f"Expected sandbox type '{sandbox_type}', got '{task.sandbox.type}'"
+        )
+
+
+def assert_eval_success(
+    log: EvalLog,
+    *,
+    require_scorer: bool = True,
+    expected_metric: str | None = None,
+    min_samples: int | None = None,
+    expected_sample_count: int | None = None,
+) -> None:
+    """Assert that an eval completed successfully with valid results.
+
+    Args:
+        log: The EvalLog from running an evaluation.
+        require_scorer: If True (default), assert scores are non-empty.
+            Set to False for tasks that intentionally have no scorer.
+        expected_metric: If set, assert this metric exists in the first score.
+        min_samples: If set, assert at least this many samples.
+        expected_sample_count: If set, assert exactly this many samples.
+    """
+    assert log.status == "success", f"Eval failed with status '{log.status}': {log.error}"
+    assert log.results is not None, "log.results is None"
+    assert log.results.scores is not None, "log.results.scores is None"
+    if require_scorer:
+        assert len(log.results.scores) > 0, "log.results.scores is empty"
+
+    if expected_metric is not None:
+        assert expected_metric in log.results.scores[0].metrics, (
+            f"Expected metric '{expected_metric}' not found. "
+            f"Available: {list(log.results.scores[0].metrics.keys())}"
+        )
+
+    if min_samples is not None:
+        assert log.samples is not None, "log.samples is None"
+        assert len(log.samples) >= min_samples, (
+            f"Expected at least {min_samples} samples, got {len(log.samples)}"
+        )
+
+    if expected_sample_count is not None:
+        assert log.samples is not None, "log.samples is None"
+        assert len(log.samples) == expected_sample_count, (
+            f"Expected {expected_sample_count} samples, got {len(log.samples)}"
+        )
+
+
+def run_single_sample_eval(
+    sample: Sample,
+    scorer: Scorer,
+    solver_output: str,
+    *,
+    epochs: int = 1,
+    display: DisplayType = "none",
+    model: str = "mockllm/model",
+) -> EvalLog:
+    """Run an evaluation on a single sample for testing.
+
+    Args:
+        sample: The Sample to evaluate.
+        scorer: The scorer to use.
+        solver_output: The fixed output the mock solver should produce.
+        epochs: Number of epochs to run.
+        display: Display mode.
+        model: Model identifier.
+
+    Returns:
+        The EvalLog from the evaluation run.
+    """
+    task = Task(
+        dataset=MemoryDataset([sample]),
+        solver=[mock_solver_with_output(solver_output)],
+        scorer=scorer,
+    )
+
+    [log] = inspect_eval(
+        tasks=task,
+        model=model,
+        epochs=epochs,
+        display=display,
+    )
+
+    return log
+
+
+def get_metric_value(
+    log: EvalLog,
+    metric_name: str,
+    scorer_index: int = 0,
+) -> int | float:
+    """Extract a metric value from an EvalLog.
+
+    Args:
+        log: The EvalLog to extract from.
+        metric_name: The name of the metric to extract.
+        scorer_index: Index of the scorer in log.results.scores.
+
+    Returns:
+        The metric value.
+    """
+    assert log.results is not None, "log.results is None"
+    assert log.results.scores is not None, "log.results.scores is None"
+    assert len(log.results.scores) > scorer_index, (
+        f"scorer_index {scorer_index} out of range, only {len(log.results.scores)} scorers"
+    )
+
+    metrics = log.results.scores[scorer_index].metrics
+    assert metric_name in metrics, (
+        f"Metric '{metric_name}' not found. Available: {list(metrics.keys())}"
+    )
+
+    return metrics[metric_name].value

pytest_inspect_evals/gates.py

@@ -0,0 +1,97 @@
+"""Opt-in test gates: markers whose tests are skipped unless switched on."""
+
+import os
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+import pytest
+
+TRUTHY = frozenset({"1", "true", "yes", "on"})
+
+
+def truthy(value: str | None) -> bool:
+    """Return whether an env var value switches a gate on."""
+    return (value or "").strip().lower() in TRUTHY
+
+
+@dataclass(frozen=True)
+class Gate:
+    """A marker that is skipped unless an env var, CLI flag or ini option enables it."""
+
+    marker: str
+    cli_flag: str
+    env_var: str
+    ini_option: str
+    description: str
+
+    @property
+    def reason(self) -> str:
+        return (
+            f"{self.marker} tests disabled (set {self.env_var}=1, pass {self.cli_flag}, "
+            f"or set {self.ini_option} = true in the pytest config)"
+        )
+
+
+GATES: tuple[Gate, ...] = (
+    Gate(
+        "slow",
+        "--runslow",
+        "RUN_SLOW_TESTS",
+        "inspect_evals_slow",
+        "marks tests that are slow to run",
+    ),
+    Gate(
+        "dataset_download",
+        "--dataset-download",
+        "RUN_DATASET_DOWNLOAD_TESTS",
+        "inspect_evals_dataset_download",
+        "marks tests that download datasets",
+    ),
+    Gate(
+        "k8s",
+        "--runk8s",
+        "RUN_K8S_TESTS",
+        "inspect_evals_k8s",
+        "marks tests requiring k8s sandbox support",
+    ),
+    Gate(
+        "gpu",
+        "--rungpu",
+        "RUN_GPU_TESTS",
+        "inspect_evals_gpu",
+        "marks tests that need an NVIDIA GPU in the sandbox host (Docker or k8s)",
+    ),
+)
+
+
+def skip_if_marker_present(
+    config: pytest.Config,
+    items: Iterable[pytest.Item],
+    marker: str,
+    cli_flag: str,
+    env_var: str,
+    reason: str,
+    *,
+    default_enabled: bool = False,
+    ini_option: str | None = None,
+) -> None:
+    """Skip tests carrying `marker` unless the gate is switched on.
+
+    Priority: env var, then CLI flag, then ini option (when given), then
+    `default_enabled`. An empty env var counts as unset.
+    """
+    env_val = os.environ.get(env_var, "").strip()
+    if env_val:
+        enabled = truthy(env_val)
+    elif config.getoption(cli_flag):
+        enabled = True
+    elif ini_option is not None:
+        enabled = bool(config.getini(ini_option))
+    else:
+        enabled = default_enabled
+    if enabled:
+        return
+    skip_mark = pytest.mark.skip(reason=reason)
+    for item in items:
+        if item.get_closest_marker(marker) is not None:
+            item.add_marker(skip_mark)