pytest-inspect-evals 0.1.0__tar.gz

pytest_inspect_evals-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Matt Fisher
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pytest_inspect_evals-0.1.0/PKG-INFO

@@ -0,0 +1,148 @@
+Metadata-Version: 2.4
+Name: pytest-inspect-evals
+Version: 0.1.0
+Summary: pytest plugin with shared test gates, fixtures and helpers for Inspect AI evaluations
+Author: Matt Fisher
+Author-email: Matt Fisher <m@ttfisher.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Framework :: Pytest
+Requires-Dist: inspect-ai>=0.3.262
+Requires-Dist: numpy>=1.26.0
+Requires-Dist: pydantic>=2.10.0
+Requires-Dist: pytest>=8.0
+Requires-Dist: pytest-rerunfailures>=15
+Requires-Dist: requests>=2.32.0
+Requires-Dist: urllib3>=2.0
+Requires-Dist: datasets>=4.8.5 ; extra == 'huggingface'
+Requires-Python: >=3.11
+Project-URL: Homepage, https://github.com/Generality-Labs/pytest-inspect-evals
+Project-URL: Repository, https://github.com/Generality-Labs/pytest-inspect-evals
+Project-URL: Issues, https://github.com/Generality-Labs/pytest-inspect-evals/issues
+Provides-Extra: huggingface
+Description-Content-Type: text/markdown
+
+# pytest-inspect-evals
+
+A pytest plugin with the test gates, fixtures and helpers shared by [Inspect AI](https://inspect.aisi.org.uk/) evaluation repos. It started as the shared test code in [inspect_evals](https://github.com/UKGovernmentBEIS/inspect_evals). Installing it replaces a copied `conftest.py` and `tests/utils/` with one pinned dependency.
+
+## Install
+
+```bash
+uv add --dev pytest-inspect-evals
+# or, to use the Hugging Face dataset helpers:
+uv add --dev "pytest-inspect-evals[huggingface]"
+```
+
+The plugin loads automatically. There is nothing to add to `conftest.py`.
+
+## Test gates
+
+Four markers mark tests that are skipped unless you switch them on.
+
+| Marker             | CLI flag             | Env var                      | Pytest config option             | Default |
+| ------------------ | -------------------- | ---------------------------- | -------------------------------- | ------- |
+| `slow`             | `--runslow`          | `RUN_SLOW_TESTS`             | `inspect_evals_slow`             | off     |
+| `dataset_download` | `--dataset-download` | `RUN_DATASET_DOWNLOAD_TESTS` | `inspect_evals_dataset_download` | off     |
+| `k8s`              | `--runk8s`           | `RUN_K8S_TESTS`              | `inspect_evals_k8s`              | off     |
+| `gpu`              | `--rungpu`           | `RUN_GPU_TESTS`              | `inspect_evals_gpu`              | off     |
+
+The env var wins when it is set to a non-empty value: `1`, `true`, `yes` or `on` switch the gate on, anything else switches it off. Otherwise the CLI flag switches it on. Otherwise the config option decides, and the default is off. An empty env var counts as unset.
+
+To run a gate by default in your repo, set its option in `pyproject.toml`:
+
+```toml
+[tool.pytest.ini_options]
+inspect_evals_dataset_download = true
+```
+
+Three more markers have fixed behaviour:
+
+- `huggingface`: skipped when `HF_TOKEN` is unset or blank. Otherwise the test is retried up to twice, 60 seconds apart, and a gated-dataset error (`GatedRepoError`, or `DatasetNotFoundError` mentioning a gated dataset) is reported as a skip.
+- `docker` and `posix_only`: skipped on Windows.
+
+The plugin registers all seven markers, so they work under `--strict-markers` without any config.
+
+## Adding your own gate
+
+`skip_if_marker_present` is the function the plugin uses. Call it from your `conftest.py` for repo-specific gates:
+
+```python
+from pytest_inspect_evals.gates import skip_if_marker_present
+
+
+def pytest_addoption(parser):
+    parser.addoption("--custom-smoke", action="store_true", default=False)
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "custom_smoke: tests that call a live service")
+
+
+def pytest_collection_modifyitems(config, items):
+    skip_if_marker_present(
+        config,
+        items,
+        marker="custom_smoke",
+        cli_flag="--custom-smoke",
+        env_var="RUN_CUSTOM_SMOKE_TESTS",
+        reason="custom_smoke tests disabled (set RUN_CUSTOM_SMOKE_TESTS=1 or pass --custom-smoke)",
+    )
+```
+
+## Fixtures
+
+`set_model_roles` returns a context manager that sets model roles, for testing scorers that call a grader model:
+
+```python
+def test_judge_scorer(set_model_roles):
+    with set_model_roles(grader=get_model("mockllm/model")):
+        ...
+```
+
+`mock_docker_sandbox` stubs the Docker sandbox lifecycle, so an eval with `sandbox="docker"` runs without Docker. Combine it with mocked `sandbox().exec` results to test a scorer end to end:
+
+```python
+def test_e2e(mock_docker_sandbox):
+    [log] = eval(my_task(), model="mockllm/model")
+    assert log.status == "success"
+```
+
+## Helpers
+
+- `pytest_inspect_evals.assertions`: `assert_task_structure`, `assert_eval_success`, `run_single_sample_eval`, `get_metric_value`.
+- `pytest_inspect_evals.solvers`: `mock_solver_with_output`, `run_command`.
+- `pytest_inspect_evals.sandbox`: `MockExecResult`, `create_sandbox_tool_task`, `assert_sandbox_test_passed`.
+- `pytest_inspect_evals.metrics`: `run_metrics`, `assert_agreeing_epochs_change_nothing`, `MetricEvalError`. These run custom metrics through the real epoch reducer, the way an eval does.
+- `pytest_inspect_evals.huggingface` (needs the `huggingface` extra): `assert_huggingface_dataset_structure`, `get_dataset_infos_dict`, `assert_huggingface_dataset_is_valid`, `assert_dataset_contains_subsets`, `assert_dataset_has_columns`. These check a dataset's schema through the Hugging Face dataset viewer API without downloading it.
+
+## Migrating from a copied conftest
+
+If your `conftest.py` came from inspect_evals or inspect-evals-template, remove what the plugin now provides:
+
+- the `pytest_addoption` lines for `--runslow`, `--dataset-download`, `--runk8s` and `--rungpu`;
+- the matching skip logic in `pytest_collection_modifyitems`;
+- the seven markers above from your `markers =` config (keeping them is harmless).
+
+If you keep the options, pytest stops at startup with `argparse.ArgumentError: argument --runslow: conflicting option string: --runslow`.
+
+## Private inspect_ai APIs
+
+Two fixtures use inspect_ai internals that have no public equivalent: `set_model_roles` calls `inspect_ai.model._model.init_model_roles`, and `mock_docker_sandbox` patches `inspect_ai.util._sandbox.docker.docker.DockerSandboxEnvironment`. A future inspect_ai release could break them.
+
+## Development
+
+```bash
+uv sync
+uv run pre-commit install   # optional: run the lint stack on every commit
+uv run pytest
+uv run basedpyright src
+```
+
+Each pull request adds a changelog fragment rather than editing `CHANGELOG.md`, so concurrent PRs don't conflict. Run `uv run scriv create`, uncomment the sections that apply in the new file under `changelog.d/`, and commit it with the change, or delete it if the change needs no entry. To release, run the **Prepare release** workflow from the Actions tab. It bumps `version` in `pyproject.toml`, collects the fragments into `CHANGELOG.md`, and opens a release pull request, whose CI starts once you click **Approve workflows to run** on it; merging it tags the release. It needs _Settings → Actions → General_ → **Allow GitHub Actions to create and approve pull requests**. By hand, the same is `uv version --bump minor` and `uv run scriv collect`.
+
+Linting (ruff, [zizmor](https://docs.zizmor.sh/), mdformat) runs via [pre-commit](https://pre-commit.com); CI runs the same stack plus basedpyright and pytest via the shared [`python-ci`](https://github.com/Generality-Labs/python-project-template) reusable workflow.
+
+## Releasing
+
+See [RELEASING.md](RELEASING.md).

pytest_inspect_evals-0.1.0/README.md

@@ -0,0 +1,124 @@
+# pytest-inspect-evals
+
+A pytest plugin with the test gates, fixtures and helpers shared by [Inspect AI](https://inspect.aisi.org.uk/) evaluation repos. It started as the shared test code in [inspect_evals](https://github.com/UKGovernmentBEIS/inspect_evals). Installing it replaces a copied `conftest.py` and `tests/utils/` with one pinned dependency.
+
+## Install
+
+```bash
+uv add --dev pytest-inspect-evals
+# or, to use the Hugging Face dataset helpers:
+uv add --dev "pytest-inspect-evals[huggingface]"
+```
+
+The plugin loads automatically. There is nothing to add to `conftest.py`.
+
+## Test gates
+
+Four markers mark tests that are skipped unless you switch them on.
+
+| Marker             | CLI flag             | Env var                      | Pytest config option             | Default |
+| ------------------ | -------------------- | ---------------------------- | -------------------------------- | ------- |
+| `slow`             | `--runslow`          | `RUN_SLOW_TESTS`             | `inspect_evals_slow`             | off     |
+| `dataset_download` | `--dataset-download` | `RUN_DATASET_DOWNLOAD_TESTS` | `inspect_evals_dataset_download` | off     |
+| `k8s`              | `--runk8s`           | `RUN_K8S_TESTS`              | `inspect_evals_k8s`              | off     |
+| `gpu`              | `--rungpu`           | `RUN_GPU_TESTS`              | `inspect_evals_gpu`              | off     |
+
+The env var wins when it is set to a non-empty value: `1`, `true`, `yes` or `on` switch the gate on, anything else switches it off. Otherwise the CLI flag switches it on. Otherwise the config option decides, and the default is off. An empty env var counts as unset.
+
+To run a gate by default in your repo, set its option in `pyproject.toml`:
+
+```toml
+[tool.pytest.ini_options]
+inspect_evals_dataset_download = true
+```
+
+Three more markers have fixed behaviour:
+
+- `huggingface`: skipped when `HF_TOKEN` is unset or blank. Otherwise the test is retried up to twice, 60 seconds apart, and a gated-dataset error (`GatedRepoError`, or `DatasetNotFoundError` mentioning a gated dataset) is reported as a skip.
+- `docker` and `posix_only`: skipped on Windows.
+
+The plugin registers all seven markers, so they work under `--strict-markers` without any config.
+
+## Adding your own gate
+
+`skip_if_marker_present` is the function the plugin uses. Call it from your `conftest.py` for repo-specific gates:
+
+```python
+from pytest_inspect_evals.gates import skip_if_marker_present
+
+
+def pytest_addoption(parser):
+    parser.addoption("--custom-smoke", action="store_true", default=False)
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "custom_smoke: tests that call a live service")
+
+
+def pytest_collection_modifyitems(config, items):
+    skip_if_marker_present(
+        config,
+        items,
+        marker="custom_smoke",
+        cli_flag="--custom-smoke",
+        env_var="RUN_CUSTOM_SMOKE_TESTS",
+        reason="custom_smoke tests disabled (set RUN_CUSTOM_SMOKE_TESTS=1 or pass --custom-smoke)",
+    )
+```
+
+## Fixtures
+
+`set_model_roles` returns a context manager that sets model roles, for testing scorers that call a grader model:
+
+```python
+def test_judge_scorer(set_model_roles):
+    with set_model_roles(grader=get_model("mockllm/model")):
+        ...
+```
+
+`mock_docker_sandbox` stubs the Docker sandbox lifecycle, so an eval with `sandbox="docker"` runs without Docker. Combine it with mocked `sandbox().exec` results to test a scorer end to end:
+
+```python
+def test_e2e(mock_docker_sandbox):
+    [log] = eval(my_task(), model="mockllm/model")
+    assert log.status == "success"
+```
+
+## Helpers
+
+- `pytest_inspect_evals.assertions`: `assert_task_structure`, `assert_eval_success`, `run_single_sample_eval`, `get_metric_value`.
+- `pytest_inspect_evals.solvers`: `mock_solver_with_output`, `run_command`.
+- `pytest_inspect_evals.sandbox`: `MockExecResult`, `create_sandbox_tool_task`, `assert_sandbox_test_passed`.
+- `pytest_inspect_evals.metrics`: `run_metrics`, `assert_agreeing_epochs_change_nothing`, `MetricEvalError`. These run custom metrics through the real epoch reducer, the way an eval does.
+- `pytest_inspect_evals.huggingface` (needs the `huggingface` extra): `assert_huggingface_dataset_structure`, `get_dataset_infos_dict`, `assert_huggingface_dataset_is_valid`, `assert_dataset_contains_subsets`, `assert_dataset_has_columns`. These check a dataset's schema through the Hugging Face dataset viewer API without downloading it.
+
+## Migrating from a copied conftest
+
+If your `conftest.py` came from inspect_evals or inspect-evals-template, remove what the plugin now provides:
+
+- the `pytest_addoption` lines for `--runslow`, `--dataset-download`, `--runk8s` and `--rungpu`;
+- the matching skip logic in `pytest_collection_modifyitems`;
+- the seven markers above from your `markers =` config (keeping them is harmless).
+
+If you keep the options, pytest stops at startup with `argparse.ArgumentError: argument --runslow: conflicting option string: --runslow`.
+
+## Private inspect_ai APIs
+
+Two fixtures use inspect_ai internals that have no public equivalent: `set_model_roles` calls `inspect_ai.model._model.init_model_roles`, and `mock_docker_sandbox` patches `inspect_ai.util._sandbox.docker.docker.DockerSandboxEnvironment`. A future inspect_ai release could break them.
+
+## Development
+
+```bash
+uv sync
+uv run pre-commit install   # optional: run the lint stack on every commit
+uv run pytest
+uv run basedpyright src
+```
+
+Each pull request adds a changelog fragment rather than editing `CHANGELOG.md`, so concurrent PRs don't conflict. Run `uv run scriv create`, uncomment the sections that apply in the new file under `changelog.d/`, and commit it with the change, or delete it if the change needs no entry. To release, run the **Prepare release** workflow from the Actions tab. It bumps `version` in `pyproject.toml`, collects the fragments into `CHANGELOG.md`, and opens a release pull request, whose CI starts once you click **Approve workflows to run** on it; merging it tags the release. It needs _Settings → Actions → General_ → **Allow GitHub Actions to create and approve pull requests**. By hand, the same is `uv version --bump minor` and `uv run scriv collect`.
+
+Linting (ruff, [zizmor](https://docs.zizmor.sh/), mdformat) runs via [pre-commit](https://pre-commit.com); CI runs the same stack plus basedpyright and pytest via the shared [`python-ci`](https://github.com/Generality-Labs/python-project-template) reusable workflow.
+
+## Releasing
+
+See [RELEASING.md](RELEASING.md).

pytest_inspect_evals-0.1.0/pyproject.toml

@@ -0,0 +1,121 @@
+[build-system]
+requires = ["uv_build>=0.12.22,<0.13.0"]
+build-backend = "uv_build"
+
+[project]
+name = "pytest-inspect-evals"
+version = "0.1.0"
+description = "pytest plugin with shared test gates, fixtures and helpers for Inspect AI evaluations"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.11"
+classifiers = ["Framework :: Pytest"]
+dependencies = [
+    "inspect_ai>=0.3.262",
+    "numpy>=1.26.0",
+    "pydantic>=2.10.0",
+    "pytest>=8.0",
+    "pytest-rerunfailures>=15",
+    "requests>=2.32.0",
+    "urllib3>=2.0",
+]
+
+[[project.authors]]
+name = "Matt Fisher"
+email = "m@ttfisher.com"
+
+[project.optional-dependencies]
+huggingface = ["datasets>=4.8.5"]
+
+[project.entry-points.pytest11]
+pytest_inspect_evals = "pytest_inspect_evals.plugin"
+
+[project.urls]
+Homepage = "https://github.com/Generality-Labs/pytest-inspect-evals"
+Repository = "https://github.com/Generality-Labs/pytest-inspect-evals"
+Issues = "https://github.com/Generality-Labs/pytest-inspect-evals/issues"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-cov>=7.0.0",
+    "basedpyright>=1.39",
+    "pre-commit>=4.0.0",
+    "scriv>=1.8",
+    "datasets>=4.8.5",
+    "pytest-asyncio>=1.0",
+    "mdformat>=1.0.0",
+    "mdformat-gfm>=1.0.0",
+]
+
+[tool.scriv]
+format = "md"
+fragment_directory = "changelog.d"
+categories = [
+    "Added",
+    "Changed",
+    "Deprecated",
+    "Removed",
+    "Fixed",
+    "Security",
+]
+skip_fragments = "[A-Z]*"
+new_fragment_template = "file: TEMPLATE.md"
+entry_title_template = "[{{ version }}] - {{ date.strftime('%Y-%m-%d') }}"
+md_header_level = "2"
+md_html_anchors = false
+compact_fragments = true
+version = "literal: pyproject.toml: project.version"
+
+[tool.ruff]
+line-length = 100
+src = ["src"]
+target-version = "py311"
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "W",
+    "F",
+    "I",
+    "UP",
+    "B",
+    "SIM",
+    "D",
+    "C4",
+    "PT",
+    "PIE",
+    "DTZ",
+    "ISC",
+    "ASYNC",
+    "N",
+    "FURB",
+    "RUF",
+    "PLE",
+    "PLW",
+]
+ignore = [
+    "E501",
+    "D10",
+    "D415",
+    "ISC001",
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["D"]
+
+[tool.basedpyright]
+pythonVersion = "3.11"
+typeCheckingMode = "standard"
+reportMissingTypeStubs = false
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-rA --durations=10 --color=yes --cov=src --cov-report=term-missing"
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+filterwarnings = ['ignore:The configuration option "asyncio_default_fixture_loop_scope" is unset:pytest.PytestDeprecationWarning']

pytest_inspect_evals-0.1.0/pyproject.toml.orig

@@ -0,0 +1,127 @@
+[build-system]
+# uv's own backend. It builds src/pytest_inspect_evals/ with no further config.
+# Keep the upper bound at the next uv minor, as `uv init` writes it, and raise
+# both bounds when moving to a new uv minor.
+requires = ["uv_build>=0.12.22,<0.13.0"]
+build-backend = "uv_build"
+
+[project]
+name = "pytest-inspect-evals"
+version = "0.1.0"
+description = "pytest plugin with shared test gates, fixtures and helpers for Inspect AI evaluations"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"] # uv_build ships only the license files named here
+authors = [{ name = "Matt Fisher", email = "m@ttfisher.com" }]
+requires-python = ">=3.11"
+classifiers = ["Framework :: Pytest"]
+dependencies = [
+    "inspect_ai>=0.3.262",
+    "numpy>=1.26.0",
+    "pydantic>=2.10.0",
+    "pytest>=8.0",
+    "pytest-rerunfailures>=15",
+    "requests>=2.32.0",
+    "urllib3>=2.0",
+]
+
+[project.optional-dependencies]
+# Only pytest_inspect_evals.huggingface needs this.
+huggingface = ["datasets>=4.8.5"]
+
+[project.entry-points.pytest11]
+pytest_inspect_evals = "pytest_inspect_evals.plugin"
+
+[project.urls]
+Homepage = "https://github.com/Generality-Labs/pytest-inspect-evals"
+Repository = "https://github.com/Generality-Labs/pytest-inspect-evals"
+Issues = "https://github.com/Generality-Labs/pytest-inspect-evals/issues"
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-cov>=7.0.0",
+    "basedpyright>=1.39",
+    "pre-commit>=4.0.0",
+    "scriv>=1.8",
+    "datasets>=4.8.5",
+    "pytest-asyncio>=1.0",
+    "mdformat>=1.0.0",
+    "mdformat-gfm>=1.0.0",
+]
+
+
+[tool.scriv]
+# One fragment per pull request in changelog.d/, collected into CHANGELOG.md
+# at release time, so concurrent PRs never edit the same lines of the changelog.
+format = "md"
+fragment_directory = "changelog.d"
+categories = ["Added", "Changed", "Deprecated", "Removed", "Fixed", "Security"]
+skip_fragments = "[A-Z]*" # TEMPLATE.md
+new_fragment_template = "file: TEMPLATE.md"
+entry_title_template = "[{{ version }}] - {{ date.strftime('%Y-%m-%d') }}"
+md_header_level = "2"
+md_html_anchors = false
+# Without this, two fragments in one category leave a blank line between them,
+# which makes the list loose and fails mdformat on the release commit.
+compact_fragments = true
+# The version's single source is [project] version above, which
+# `uv version --bump` (and the Prepare release workflow) rewrites.
+version = "literal: pyproject.toml: project.version"
+
+[tool.ruff]
+line-length = 100
+src = ["src"]
+target-version = "py311"
+
+[tool.ruff.lint]
+select = [
+    "E",     # pycodestyle errors
+    "W",     # pycodestyle warnings
+    "F",     # pyflakes
+    "I",     # isort
+    "UP",    # pyupgrade
+    "B",     # flake8-bugbear
+    "SIM",   # flake8-simplify
+    "D",     # pydocstyle
+    "C4",    # flake8-comprehensions
+    "PT",    # flake8-pytest-style
+    "PIE",   # flake8-pie
+    "DTZ",   # flake8-datetimez (timezone-aware datetimes)
+    "ISC",   # implicit-str-concat (catches missing commas)
+    "ASYNC", # flake8-async
+    "N",     # pep8-naming
+    "FURB",  # refurb (modernization)
+    "RUF",   # ruff-specific rules
+    "PLE",   # pylint errors
+    "PLW",   # pylint warnings
+]
+ignore = [
+    "E501",   # line-too-long — the formatter owns line length
+    "D10",    # missing docstrings — don't force them
+    "D415",   # first-line punctuation — the module docstring is the project
+    # description, whose punctuation varies
+    "ISC001", # single-line implicit concat — conflicts with the formatter
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["D"] # tests don't need docstring linting
+
+[tool.basedpyright]
+pythonVersion = "3.11"
+# "standard", not "strict": v0.1 moves code verbatim from inspect_evals, which
+# was checked with mypy. Tighten once the code is ours.
+typeCheckingMode = "standard"
+reportMissingTypeStubs = false # third-party libs are often untyped
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-rA --durations=10 --color=yes --cov=src --cov-report=term-missing"
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+filterwarnings = [
+    # pytester's throwaway test dirs have no pytest-asyncio config of their own.
+    'ignore:The configuration option "asyncio_default_fixture_loop_scope" is unset:pytest.PytestDeprecationWarning',
+]

pytest_inspect_evals-0.1.0/src/pytest_inspect_evals/__init__.py

@@ -0,0 +1,10 @@
+"""pytest plugin with shared test gates, fixtures and helpers for Inspect AI evaluations"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+# The version is set once, in pyproject.toml; this reads it back from the
+# installed package metadata.
+try:
+    __version__ = version("pytest-inspect-evals")
+except PackageNotFoundError:  # a source tree that was never installed
+    __version__ = "unknown"

pytest_inspect_evals-0.1.0/src/pytest_inspect_evals/_hf.py

@@ -0,0 +1,69 @@
+"""Hugging Face handling for tests marked huggingface."""
+
+import os
+from importlib.util import find_spec
+
+import pytest
+
+
+def hf_disable_tokenizer_parallelism() -> None:
+    """Disable HF tokenizers parallelism to avoid segfaults."""
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+def hf_configure_logging() -> None:
+    """Raise datasets and huggingface_hub logging to INFO when datasets is installed."""
+    if find_spec("datasets") is None:
+        return
+    import datasets
+    import huggingface_hub
+
+    datasets.logging.set_verbosity_info()
+    huggingface_hub.logging.set_verbosity_info()
+
+
+def hf_apply_collection_markers(items: list[pytest.Item]) -> None:
+    hf_items = [item for item in items if item.get_closest_marker("huggingface") is not None]
+
+    if not hf_items:
+        return
+
+    if not os.environ.get("HF_TOKEN", "").strip():
+        for item in hf_items:
+            item.add_marker(pytest.mark.skip(reason=f"HF_TOKEN not set ({item.name})"))
+
+    else:
+        for item in hf_items:
+            item.add_marker(pytest.mark.flaky(reruns=2, reruns_delay=60.0))
+
+
+def _is_hf_gated_dataset_failure(
+    item: pytest.Item, call: pytest.CallInfo, report: pytest.TestReport
+) -> bool:
+    return (
+        report.when == "call"
+        and report.failed
+        and item.get_closest_marker("huggingface") is not None
+        and bool(call.excinfo)
+        and is_gated_dataset_exception(call.excinfo.value)  # type: ignore
+    )
+
+
+def hf_convert_gated_failure_to_skip(
+    item: pytest.Item, call: pytest.CallInfo, report: pytest.TestReport
+) -> None:
+    if _is_hf_gated_dataset_failure(item, call, report):
+        report.outcome = "skipped"
+        line_number = item.reportinfo()[1] or 0
+        report.longrepr = (
+            str(item.fspath),
+            line_number,
+            f"Skipped: Gated dataset ({item.name})",
+        )
+
+
+def is_gated_dataset_exception(exc: BaseException) -> bool:
+    exc_name = type(exc).__name__
+    return exc_name == "GatedRepoError" or (
+        exc_name == "DatasetNotFoundError" and "gated dataset" in str(exc).lower()
+    )

pytest_inspect_evals-0.1.0/src/pytest_inspect_evals/_windows.py

@@ -0,0 +1,25 @@
+"""Skip tests that can't run on Windows."""
+
+import sys
+
+import pytest
+
+
+def windows_skip_unsupported_tests(items: list[pytest.Item], platform: str | None = None) -> None:
+    """Skip POSIX-only and Docker tests on Windows."""
+    if (platform or sys.platform) != "win32":
+        return
+
+    for item in items:
+        if item.get_closest_marker("posix_only") is not None:
+            item.add_marker(
+                pytest.mark.skip(
+                    reason=f"Skipping {item.name}: test requires POSIX system (not supported on Windows)"
+                )
+            )
+        if item.get_closest_marker("docker") is not None:
+            item.add_marker(
+                pytest.mark.skip(
+                    reason=f"Skipping {item.name}: Docker tests are not supported on Windows CI"
+                )
+            )