inspect-evals-lint 0.1.0__tar.gz

inspect_evals_lint-0.1.0/.claude/hooks/session-start.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -euo pipefail
+
+if [ "${CLAUDE_CODE_REMOTE:-}" != "true" ]; then
+  exit 0
+fi
+
+cd "$CLAUDE_PROJECT_DIR"
+
+# Install the project + dev tooling (pytest, basedpyright, pre-commit) into the uv-managed venv.
+uv sync
+
+# Installs the git hook (so `git commit` runs it automatically) and
+# pre-warms hook environments/cache. Pre-existing findings shouldn't block
+# session start, so this step is non-blocking - failures surface as normal
+# output for the agent to see and address, not a broken session.
+uv run pre-commit install
+uv run pre-commit run --all-files || true

inspect_evals_lint-0.1.0/.claude/settings.json

@@ -0,0 +1,14 @@
+{
+  "hooks": {
+    "SessionStart": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/session-start.sh"
+          }
+        ]
+      }
+    ]
+  }
+}

inspect_evals_lint-0.1.0/.copier-answers.yml

@@ -0,0 +1,18 @@
+# This file is auto-generated and updated by Copier; do not edit by hand.
+# Run `copier update` to pull in template changes.
+_commit: v1.8.1
+_src_path: gh:Generality-Labs/python-project-template
+author_email: m@ttfisher.com
+author_name: Matt Fisher
+github_owner: Generality-Labs
+license: MIT
+project_description: 'Static checks for Inspect AI evaluations: structure, tests,
+    best practices and sandbox pinning'
+project_kind: library
+project_name: inspect-evals-lint
+publish_to_pypi: true
+python_version: '3.11'
+use_coverage_gate: false
+use_frontend: false
+use_template_update: true
+use_typos: true

inspect_evals_lint-0.1.0/.github/dependabot.yml

@@ -0,0 +1,30 @@
+version: 2
+updates:
+  # Bump the SHA-pinned GitHub Actions (opens PRs with changelogs + the new pin).
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    cooldown:
+      default-days: 7 # don't adopt a release until it's had time to be vetted/yanked
+    groups:
+      actions:
+        patterns: ["*"]
+    ignore:
+      # Our own reusable workflows are referenced by moving major tag on
+      # purpose (see .github/zizmor.yml), so template CI fixes flow in
+      # without a bump PR per release. Left un-ignored, Dependabot rewrites
+      # `@v1` to a fixed `@v1.x.y` and re-bumps it on every release — and
+      # `copier update` restores the moving tag, so the two fight forever.
+      - dependency-name: "Generality-Labs/*"
+
+  # Keep Python dependencies (pyproject.toml + uv.lock) current.
+  - package-ecosystem: uv
+    directory: /
+    schedule:
+      interval: weekly
+    cooldown:
+      default-days: 7
+    groups:
+      python:
+        patterns: ["*"]

inspect_evals_lint-0.1.0/.github/repo-settings.json

@@ -0,0 +1,10 @@
+{
+  "delete_branch_on_merge": true,
+  "allow_auto_merge": false,
+  "allow_squash_merge": true,
+  "allow_merge_commit": true,
+  "allow_rebase_merge": true,
+  "has_wiki": false,
+  "has_projects": false,
+  "web_commit_signoff_required": false
+}

inspect_evals_lint-0.1.0/.github/rulesets/main.json

@@ -0,0 +1,40 @@
+{
+  "name": "protect-main",
+  "target": "branch",
+  "enforcement": "active",
+  "conditions": {
+    "ref_name": {
+      "include": ["~DEFAULT_BRANCH"],
+      "exclude": []
+    }
+  },
+  "bypass_actors": [
+    {
+      "actor_id": 5,
+      "actor_type": "RepositoryRole",
+      "bypass_mode": "always"
+    }
+  ],
+  "rules": [
+    { "type": "deletion" },
+    { "type": "non_fast_forward" },
+    {
+      "type": "pull_request",
+      "parameters": {
+        "required_approving_review_count": 0,
+        "dismiss_stale_reviews_on_push": false,
+        "require_code_owner_review": false,
+        "require_last_push_approval": false,
+        "required_review_thread_resolution": false
+      }
+    },
+    {
+      "type": "required_status_checks",
+      "parameters": {
+        "strict_required_status_checks_policy": false,
+        "required_status_checks": [
+          { "context": "ci / Lint, type-check, and test" }        ]
+      }
+    }
+  ]
+}

inspect_evals_lint-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,20 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  ci:
+    # Moving major tag on purpose (see .github/zizmor.yml): we own this repo,
+    # so CI fixes flow in without a bump PR per template release. Dependabot
+    # is told to leave it alone in .github/dependabot.yml — don't "helpfully"
+    # pin this to an exact version, or copier update will just revert it.
+    uses: Generality-Labs/python-project-template/.github/workflows/python-ci.yml@v1
+    with:
+      python-version: "3.11"

inspect_evals_lint-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,36 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: publish
+  cancel-in-progress: false
+
+jobs:
+  publish:
+    name: Build and publish to PyPI
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write # required for PyPI trusted publishing (OIDC, no API token)
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
+        with:
+          python-version: "3.11"
+          enable-cache: false # no caching in the privileged publish job (cache-poisoning surface)
+
+      - name: Build sdist + wheel
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

inspect_evals_lint-0.1.0/.github/workflows/template-update.yml

@@ -0,0 +1,29 @@
+name: Template Update
+
+# Pulls scaffolded-file changes from the template via `copier update`, opening
+# a PR only when something actually changed. Changes to the shared reusable
+# workflows need no run here — those are pinned `@v1` and propagate on their
+# own when the tag moves.
+#
+# To update to something other than `v1`, run `uvx copier update` locally with
+# `--vcs-ref`.
+on:
+  schedule:
+    - cron: "0 6 * * 1" # Mondays, 06:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  template-update:
+    uses: Generality-Labs/python-project-template/.github/workflows/template-update.yml@v1
+    permissions:
+      contents: write # push the branch holding the template update
+      pull-requests: write # open the PR for it
+    with:
+      # Set true, and pass the secret below, to have Claude attempt the merge
+      # conflicts copier can't resolve on its own.
+      resolve-conflicts-with-claude: false
+    # secrets:
+    #   anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}

inspect_evals_lint-0.1.0/.github/zizmor.yml

@@ -0,0 +1,9 @@
+# Allow tag/branch refs for our own reusable workflows and actions (we control
+# those repos and pin them to a moving major tag on purpose); still require full
+# commit-SHA pins for third-party actions.
+rules:
+  unpinned-uses:
+    config:
+      policies:
+        "Generality-Labs/*": ref-pin
+        "*": hash-pin

inspect_evals_lint-0.1.0/.gitignore

@@ -0,0 +1,43 @@
+# Byte-compiled
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Virtual environments
+.venv/
+venv/
+
+# Packaging / build artifacts
+build/
+dist/
+*.egg-info/
+*.egg
+
+# Tool caches
+.pytest_cache/
+.ruff_cache/
+
+# Coverage
+.coverage
+.coverage.*
+coverage.xml
+htmlcov/
+
+# Local env / secrets — never commit API keys
+.env
+.env.*
+!.env.example
+
+# Claude Code per-user overrides (the shared .claude/settings.json is committed)
+.claude/settings.local.json
+
+# Eval / run output
+logs/
+
+# Editors
+.idea/
+*.swp
+
+# OS
+.DS_Store
+Thumbs.db

inspect_evals_lint-0.1.0/.mdformat.toml

@@ -0,0 +1,3 @@
+# mdformat configuration — https://mdformat.readthedocs.io/en/stable/users/configuration_file.html
+wrap = "no"    # one line per paragraph (no reflow); keeps prose diffs minimal
+number = true  # ordered lists render 1. 2. 3. instead of all 1.

inspect_evals_lint-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,91 @@
+# Latest Python version we support — keeps hook envs consistent.
+default_language_version:
+  python: python3.12
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # v6.0.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-json
+      - id: check-yaml
+      - id: debug-statements
+      - id: detect-private-key
+      - id: end-of-file-fixer
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: c59bba8fb259db0fec2bbb77ad8ba51ea7341b56 # v0.15.20
+    hooks:
+      - id: ruff-check
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: b7359babf83a050dcd7bf892538f3df09c3d4ba6 # 0.11.29
+    hooks:
+      - id: uv-lock # keep uv.lock in sync with pyproject.toml
+
+  - repo: https://github.com/rhysd/actionlint
+    rev: 914e7df21a07ef503a81201c76d2b11c789d3fca # v1.7.12
+    hooks:
+      - id: actionlint # GitHub Actions correctness (complements zizmor's security audits)
+
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: 745eface02aef23e168a8afb6b5737818efbea95 # v0.11.0.1
+    hooks:
+      - id: shellcheck # lints shell scripts (e.g. .claude/hooks/session-start.sh)
+        # Pinned independently of default_language_version above, and it must
+        # stay below 3.13 when that default moves up. shellcheck-py ships no
+        # wheel in its git repo, so pre-commit builds it from source, and that
+        # build downloads the binary with a bare urllib.request.urlopen. Python
+        # 3.13 turns on ssl.VERIFY_X509_STRICT by default, which rejects a CA
+        # certificate carrying no keyUsage extension -- as the MITM proxies in
+        # front of some sandboxes do. The hook environment then can't be built at
+        # all, so `git commit` fails there, not merely `pre-commit run`. SKIP= is
+        # no help: it applies to `pre-commit run`, not to installation.
+        #
+        # The interpreter builds this hook's environment and nothing else, so it
+        # has no bearing on what shellcheck checks. The durable fix is a local
+        # hook using the prebuilt wheel from PyPI -- no build, no download -- at
+        # the cost of `pre-commit autoupdate` no longer tracking it.
+        language_version: python3.12
+
+  - repo: https://github.com/crate-ci/typos
+    rev: bee27e3a4fd1ea2111cf90ab89cd076c870fce14 # v1.48.0
+    hooks:
+      # Report only. The hook's own default args are
+      # [--write-changes, --force-exclude], which rewrite source rather than
+      # reporting it: anything the checker mistakes for a misspelling gets
+      # renamed silently, including identifiers and domain vocabulary. A
+      # correction needs a human to approve it, so --write-changes is dropped.
+      - id: typos # spell-check code, comments, and docs
+        args: [--force-exclude]
+
+  - repo: https://github.com/zizmorcore/zizmor-pre-commit
+    rev: e3eebf65325ccc992422292cb7a4baee967cf815 # v1.26.1
+    hooks:
+      # Several of zizmor's audits (artipacked, impostor-commit,
+      # known-vulnerable-actions, stale-action-refs, ref-version-mismatch, ...)
+      # need to query the GitHub API for upstream Actions metadata, which isn't
+      # reliably available in every environment this runs in (sandboxes,
+      # restricted networks). Split into two hooks so local runs always get
+      # zizmor's offline coverage (most of its ruleset) without depending on
+      # network access, while CI - which has real network + a real GH_TOKEN -
+      # additionally runs the online-only audits via `--hook-stage manual`.
+      # `--persona=auditor` is zizmor's most thorough mode; findings at any
+      # severity fail the run.
+      - id: zizmor
+        name: zizmor (offline checks)
+        args: [--no-online-audits, --persona=auditor]
+      - id: zizmor
+        name: zizmor (online checks - GitHub API, CI only)
+        args: [--persona=auditor]
+        stages: [manual]
+
+  - repo: https://github.com/executablebooks/mdformat
+    rev: 82912cdaea4fb830f751504486a7879c70526547 # 1.0.0
+    hooks:
+      - id: mdformat
+        # Formatting config (wrap, number, ...) lives in .mdformat.toml.
+        additional_dependencies:
+          - mdformat-gfm # padded/aligned tables + GFM extensions (task lists, strikethrough)

inspect_evals_lint-0.1.0/.python-version

@@ -0,0 +1 @@
+3.11

inspect_evals_lint-0.1.0/CHANGELOG.md

@@ -0,0 +1,24 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-09-16
+
+### Added
+
+- Initial extraction of the `autolint` checks from [inspect_evals](https://github.com/UKGovernmentBEIS/inspect_evals) `tools/run_autolint/` as an installable package with an `inspect-evals-lint` CLI.
+- `[tool.inspect-evals-lint]` configuration with `monorepo` and `template` layout presets covering source root, tests root, import prefix, registry mode (`module`, `entry-points`, `none`), non-eval directories, required `eval.yaml` fields, isolated package directory, disabled checks and the sandbox image allowlist.
+- Repository root discovery from the nearest configured `pyproject.toml`, with `--root` and `--preset` overrides.
+- Malformed `eval.yaml`, unparsable main files and unreadable test files now produce `fail` results instead of crashing the run.
+- `register` layout preset for single-evaluation upstream repositories, built from three new configuration keys: `tests-layout = "flat"` accepts test files directly under the tests root when `<tests-root>/<eval>/` is absent (and skips `tests_init` there), `readme-location = "repo-root"` accepts the repository's top-level `README.md`, and `eval-yaml-required = false` turns a missing `eval.yaml` into a skip while still validating one that is present.
+- `main_file` and `init_exports` accept `tasks.py` as the module holding the `@task` functions, as an alternative to `<eval_name>.py`.
+- `--json` writes results as a single JSON document to stdout (progress goes to stderr), with paths relative to the repository root, for badges and other tooling. Each result carries its `category` (the CHECKS.md section), also available as `runner.CHECK_CATEGORIES`. `render_json`, `report_to_dict` and `reports_to_dict` expose the same in the Python API.
+
+### Fixed
+
+- `external_dependencies` compares distribution names in PEP 503 normalised form, so `import inspect_ai` is satisfied by `dependencies = ["inspect-ai"]`. Before, the check failed on that spelling whenever the linter ran in an environment without the package installed. Requirement strings using `!=`, `~=`, `@` or `,` are also parsed correctly.
+- The notice about a missing `[tool.inspect-evals-lint]` table no longer has its brackets swallowed as console markup.

inspect_evals_lint-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Matt Fisher
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

inspect_evals_lint-0.1.0/PKG-INFO

@@ -0,0 +1,116 @@
+Metadata-Version: 2.5
+Name: inspect-evals-lint
+Version: 0.1.0
+Summary: Static checks for Inspect AI evaluations: structure, tests, best practices and sandbox pinning
+Project-URL: Homepage, https://github.com/Generality-Labs/inspect-evals-lint
+Project-URL: Repository, https://github.com/Generality-Labs/inspect-evals-lint
+Project-URL: Issues, https://github.com/Generality-Labs/inspect-evals-lint/issues
+Author-email: Matt Fisher <m@ttfisher.com>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Description-Content-Type: text/markdown
+
+# inspect-evals-lint
+
+Static checks for [Inspect AI](https://inspect.aisi.org.uk/) evaluations: file structure, test coverage conventions, best practices and sandbox image pinning.
+
+These checks began life as the `autolint` tool inside [inspect_evals](https://github.com/UKGovernmentBEIS/inspect_evals). They are packaged here so any repository of Inspect evaluations can run the same checks, including standalone repos built from the [inspect-evals-template](https://github.com/Generality-Labs/inspect-evals-template) and submitted to the inspect_evals register.
+
+Nothing is imported or executed from the evaluation being checked. Every check is static analysis over Python source (via `ast`), `eval.yaml`, compose files and `pyproject.toml`.
+
+## Install
+
+```bash
+uv add --dev inspect-evals-lint
+# or
+pip install inspect-evals-lint
+```
+
+## Usage
+
+```bash
+inspect-evals-lint <eval_name>            # one evaluation
+inspect-evals-lint --all-evals            # every evaluation in the repo
+inspect-evals-lint --all-evals --summary-only
+inspect-evals-lint --check-summary        # per-check compliance across evals
+inspect-evals-lint <eval_name> --check registry
+inspect-evals-lint --all-evals --json > lint.json
+inspect-evals-lint --list-checks
+```
+
+`--json` writes one document to stdout and sends progress to stderr, so the output can be piped straight into other tooling. It carries `passed`, run-wide `summary` counts and, per evaluation, every check's `status`, `category` (the [CHECKS.md](docs/CHECKS.md) section: `file_structure`, `code_quality`, `tests` or `best_practices`), `message`, `file` (relative to the repository root when possible) and `line`.
+
+The repository root is the nearest `pyproject.toml` carrying a `[tool.inspect-evals-lint]` table (falling back to the nearest `pyproject.toml`, then the current directory). Pass `--root` to override.
+
+Exit codes: `0` all checks passed (warnings, skips and suppressions count as passing), `1` at least one check failed, `2` usage or configuration error.
+
+## Configuration
+
+Configuration lives in `pyproject.toml`. Pick a layout preset and override any field:
+
+```toml
+[tool.inspect-evals-lint]
+preset = "template"   # or "monorepo" / "register"
+```
+
+| Key                         | `template` preset                                | `monorepo` preset                | `register` preset | Meaning                                                                                                  |
+| --------------------------- | ------------------------------------------------ | -------------------------------- | ----------------- | -------------------------------------------------------------------------------------------------------- |
+| `source-root`               | `src`                                            | `src/inspect_evals`              | `src`             | Directory with one sub-directory per evaluation                                                          |
+| `tests-root`                | `tests`                                          | `tests`                          | `tests`           | Directory holding `<tests-root>/<eval>/`                                                                 |
+| `tests-layout`              | `per-eval`                                       | `per-eval`                       | `flat`            | `flat` also accepts test files directly under `tests-root` when `<tests-root>/<eval>/` is absent         |
+| `readme-location`           | `eval-dir`                                       | `eval-dir`                       | `repo-root`       | `repo-root` also accepts the repository's top-level `README.md`                                          |
+| `eval-yaml-required`        | `true`                                           | `true`                           | `false`           | Whether a missing `eval.yaml` fails (a present one is always validated)                                  |
+| `import-prefix`             | `""`                                             | `inspect_evals`                  | `""`              | Dotted prefix evaluations import under                                                                   |
+| `registry`                  | `entry-points`                                   | `module`                         | `entry-points`    | `entry-points` reads `[project.entry-points.inspect_ai]`; `module` greps a registry module; `none` skips |
+| `registry-module`           | unset                                            | `src/inspect_evals/_registry.py` | unset             | Required when `registry = "module"`                                                                      |
+| `non-eval-dirs`             | `["utils", "examples"]`                          | `["utils"]`                      | same as template  | Sub-directories of `source-root` that are not evaluations                                                |
+| `eval-yaml-required-fields` | `title, description, group, contributors, tasks` | same                             | same              | Keys every `eval.yaml` must define                                                                       |
+| `isolated-packages-dir`     | unset                                            | `packages`                       | unset             | Per-eval `pyproject.toml` directory for isolated dependency sets                                         |
+| `disabled-checks`           | `[]`                                             | `[]`                             | `[]`              | Checks that never run                                                                                    |
+| `sandbox-image-allowlist`   | `{}`                                             | `{}`                             | `{}`              | `{ eval = ["image/ref"] }` pairs allowed to stay unpinned (warn, not fail)                               |
+
+Without a `[tool.inspect-evals-lint]` table the `template` preset is used. `--preset` overrides the table for one run.
+
+The `register` preset is for an upstream repository listed in the [inspect_evals register](https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/register/README.md): one evaluation, tests directly under `tests/`, the README at the repository root, and metadata held by the register entry rather than an `eval.yaml` in the repo. Run it from outside the repo with `inspect-evals-lint --root <clone> --preset register --all-evals --json`.
+
+## Suppressing a check
+
+- Line: `# noautolint: <check_name>` on the offending line (checks that report per-site results: `private_api_imports`, `get_model_location`).
+- File: `# noautolint-file: <check_name>` within the first ten lines of a file.
+- Directory: a `.noautolint` file in a sub-directory listing check names, one per line. Files under that directory are also excluded from AST-based checks.
+- Evaluation: a `.noautolint` file in the evaluation directory listing check names.
+
+## Checks
+
+See [docs/CHECKS.md](docs/CHECKS.md) for the full list with the reasoning behind each check.
+
+## Python API
+
+```python
+from pathlib import Path
+from inspect_evals_lint import lint_evaluation, load_config, get_all_eval_names
+
+root = Path(".")
+config = load_config(root)
+for name in get_all_eval_names(root, config):
+    report = lint_evaluation(root, name, config)
+    print(name, report.passed(), report.summary())
+```
+
+## Development
+
+```bash
+uv sync
+uv run pre-commit install   # optional: run the lint stack on every commit
+uv run pytest
+uv run basedpyright src
+```
+
+Linting (ruff, [zizmor](https://docs.zizmor.sh/), mdformat) runs via [pre-commit](https://pre-commit.com); CI runs the same stack plus basedpyright and pytest via the shared [`python-ci`](https://github.com/Generality-Labs/python-project-template) reusable workflow.
+
+## Releasing
+
+See [RELEASING.md](RELEASING.md).

inspect_evals_lint-0.1.0/README.md

@@ -0,0 +1,101 @@
+# inspect-evals-lint
+
+Static checks for [Inspect AI](https://inspect.aisi.org.uk/) evaluations: file structure, test coverage conventions, best practices and sandbox image pinning.
+
+These checks began life as the `autolint` tool inside [inspect_evals](https://github.com/UKGovernmentBEIS/inspect_evals). They are packaged here so any repository of Inspect evaluations can run the same checks, including standalone repos built from the [inspect-evals-template](https://github.com/Generality-Labs/inspect-evals-template) and submitted to the inspect_evals register.
+
+Nothing is imported or executed from the evaluation being checked. Every check is static analysis over Python source (via `ast`), `eval.yaml`, compose files and `pyproject.toml`.
+
+## Install
+
+```bash
+uv add --dev inspect-evals-lint
+# or
+pip install inspect-evals-lint
+```
+
+## Usage
+
+```bash
+inspect-evals-lint <eval_name>            # one evaluation
+inspect-evals-lint --all-evals            # every evaluation in the repo
+inspect-evals-lint --all-evals --summary-only
+inspect-evals-lint --check-summary        # per-check compliance across evals
+inspect-evals-lint <eval_name> --check registry
+inspect-evals-lint --all-evals --json > lint.json
+inspect-evals-lint --list-checks
+```
+
+`--json` writes one document to stdout and sends progress to stderr, so the output can be piped straight into other tooling. It carries `passed`, run-wide `summary` counts and, per evaluation, every check's `status`, `category` (the [CHECKS.md](docs/CHECKS.md) section: `file_structure`, `code_quality`, `tests` or `best_practices`), `message`, `file` (relative to the repository root when possible) and `line`.
+
+The repository root is the nearest `pyproject.toml` carrying a `[tool.inspect-evals-lint]` table (falling back to the nearest `pyproject.toml`, then the current directory). Pass `--root` to override.
+
+Exit codes: `0` all checks passed (warnings, skips and suppressions count as passing), `1` at least one check failed, `2` usage or configuration error.
+
+## Configuration
+
+Configuration lives in `pyproject.toml`. Pick a layout preset and override any field:
+
+```toml
+[tool.inspect-evals-lint]
+preset = "template"   # or "monorepo" / "register"
+```
+
+| Key                         | `template` preset                                | `monorepo` preset                | `register` preset | Meaning                                                                                                  |
+| --------------------------- | ------------------------------------------------ | -------------------------------- | ----------------- | -------------------------------------------------------------------------------------------------------- |
+| `source-root`               | `src`                                            | `src/inspect_evals`              | `src`             | Directory with one sub-directory per evaluation                                                          |
+| `tests-root`                | `tests`                                          | `tests`                          | `tests`           | Directory holding `<tests-root>/<eval>/`                                                                 |
+| `tests-layout`              | `per-eval`                                       | `per-eval`                       | `flat`            | `flat` also accepts test files directly under `tests-root` when `<tests-root>/<eval>/` is absent         |
+| `readme-location`           | `eval-dir`                                       | `eval-dir`                       | `repo-root`       | `repo-root` also accepts the repository's top-level `README.md`                                          |
+| `eval-yaml-required`        | `true`                                           | `true`                           | `false`           | Whether a missing `eval.yaml` fails (a present one is always validated)                                  |
+| `import-prefix`             | `""`                                             | `inspect_evals`                  | `""`              | Dotted prefix evaluations import under                                                                   |
+| `registry`                  | `entry-points`                                   | `module`                         | `entry-points`    | `entry-points` reads `[project.entry-points.inspect_ai]`; `module` greps a registry module; `none` skips |
+| `registry-module`           | unset                                            | `src/inspect_evals/_registry.py` | unset             | Required when `registry = "module"`                                                                      |
+| `non-eval-dirs`             | `["utils", "examples"]`                          | `["utils"]`                      | same as template  | Sub-directories of `source-root` that are not evaluations                                                |
+| `eval-yaml-required-fields` | `title, description, group, contributors, tasks` | same                             | same              | Keys every `eval.yaml` must define                                                                       |
+| `isolated-packages-dir`     | unset                                            | `packages`                       | unset             | Per-eval `pyproject.toml` directory for isolated dependency sets                                         |
+| `disabled-checks`           | `[]`                                             | `[]`                             | `[]`              | Checks that never run                                                                                    |
+| `sandbox-image-allowlist`   | `{}`                                             | `{}`                             | `{}`              | `{ eval = ["image/ref"] }` pairs allowed to stay unpinned (warn, not fail)                               |
+
+Without a `[tool.inspect-evals-lint]` table the `template` preset is used. `--preset` overrides the table for one run.
+
+The `register` preset is for an upstream repository listed in the [inspect_evals register](https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/register/README.md): one evaluation, tests directly under `tests/`, the README at the repository root, and metadata held by the register entry rather than an `eval.yaml` in the repo. Run it from outside the repo with `inspect-evals-lint --root <clone> --preset register --all-evals --json`.
+
+## Suppressing a check
+
+- Line: `# noautolint: <check_name>` on the offending line (checks that report per-site results: `private_api_imports`, `get_model_location`).
+- File: `# noautolint-file: <check_name>` within the first ten lines of a file.
+- Directory: a `.noautolint` file in a sub-directory listing check names, one per line. Files under that directory are also excluded from AST-based checks.
+- Evaluation: a `.noautolint` file in the evaluation directory listing check names.
+
+## Checks
+
+See [docs/CHECKS.md](docs/CHECKS.md) for the full list with the reasoning behind each check.
+
+## Python API
+
+```python
+from pathlib import Path
+from inspect_evals_lint import lint_evaluation, load_config, get_all_eval_names
+
+root = Path(".")
+config = load_config(root)
+for name in get_all_eval_names(root, config):
+    report = lint_evaluation(root, name, config)
+    print(name, report.passed(), report.summary())
+```
+
+## Development
+
+```bash
+uv sync
+uv run pre-commit install   # optional: run the lint stack on every commit
+uv run pytest
+uv run basedpyright src
+```
+
+Linting (ruff, [zizmor](https://docs.zizmor.sh/), mdformat) runs via [pre-commit](https://pre-commit.com); CI runs the same stack plus basedpyright and pytest via the shared [`python-ci`](https://github.com/Generality-Labs/python-project-template) reusable workflow.
+
+## Releasing
+
+See [RELEASING.md](RELEASING.md).