scperteval 0.1.0__tar.gz

scperteval-0.1.0/.github/workflows/build.yaml

@@ -0,0 +1,26 @@
+name: Check Build
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  package:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          filter: blob:none
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Build package
+        run: uv build
+      - name: Check package
+        run: uvx twine check --strict dist/*.whl

scperteval-0.1.0/.github/workflows/lint.yaml

@@ -0,0 +1,32 @@
+name: Lint
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          filter: blob:none
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Install lint deps
+        run: uv sync --group lint
+      - name: Ruff lint
+        run: uv run ruff check .
+      - name: Ruff format check
+        run: uv run ruff format --check .
+      - name: mypy
+        run: uv run mypy src/scperteval
+      - name: pyright
+        run: uv run pyright src/scperteval

scperteval-0.1.0/.github/workflows/notebooks.yaml

@@ -0,0 +1,41 @@
+name: Notebooks
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  execute:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          # hatch-vcs derives the package version from git tags; fetch full history + tags
+          # so `uv sync` (which builds/installs the package) resolves a real version.
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Install docs deps
+        run: uv sync --group docs
+      - name: Execute tutorial notebooks (guard against stale/broken output)
+        # Docs render the committed (already-executed) notebooks, so re-run them here to
+        # catch a tutorial that no longer runs against the current API. Output is discarded:
+        # this only checks that the notebook still executes, not that its values match.
+        #
+        # Notebooks that download large real datasets (hundreds of MB to GBs) are skipped here
+        # via SKIP — they are executed by hand and committed with their outputs.
+        run: |
+          SKIP="02_preparing_a_dataset.ipynb"
+          for nb in docs/notebooks/*.ipynb; do
+            case " $SKIP " in
+              *" $(basename "$nb") "*) echo "skipping $nb (downloads large datasets; not run in CI)"; continue;;
+            esac
+            echo "executing $nb"
+            uv run jupyter nbconvert --to notebook --execute --stdout "$nb" > /dev/null
+          done

scperteval-0.1.0/.github/workflows/release.yaml

@@ -0,0 +1,30 @@
+name: Release
+
+on:
+  release:
+    types: [published]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  release:
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/scperteval
+    permissions:
+      id-token: write  # required for trusted publishing
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          filter: blob:none
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Build package
+        run: uv build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

scperteval-0.1.0/.github/workflows/test.yaml

@@ -0,0 +1,150 @@
+name: Test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "0 5 1,15 * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Get the test environment from hatch as defined in pyproject.toml.
+  # This ensures that the pyproject.toml is the single point of truth for test definitions and the same tests are
+  # run locally and on continuous integration.
+  # Check [[tool.hatch.envs.hatch-test.matrix]] in pyproject.toml and https://hatch.pypa.io/latest/environment/ for
+  # more details.
+  get-environments:
+    runs-on: ubuntu-slim
+    outputs:
+      envs: ${{ steps.get-envs.outputs.envs }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          filter: blob:none
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Get test environments
+        id: get-envs
+        run: |
+          ENVS_JSON=$(uvx hatch env show --json | jq -c 'to_entries
+            | map(
+                select(.key | startswith("hatch-test"))
+                | {
+                    name: .key,
+                    label: (if (.key | contains("pre")) then .key + " (PRE-RELEASE DEPENDENCIES)" else .key end),
+                    python: .value.python
+                  }
+              )')
+          echo "envs=${ENVS_JSON}" | tee $GITHUB_OUTPUT
+
+  # Run tests through hatch. Spawns a separate runner for each environment defined in the hatch matrix obtained above.
+  test:
+    needs: get-environments
+    permissions:
+       id-token: write  # for codecov OIDC
+       contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        env: ${{ fromJSON(needs.get-environments.outputs.envs) }}
+
+    name: ${{ matrix.env.label }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ contains(matrix.env.name, 'pre') }}  # make "all-green" pass even if pre-release job fails
+
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          filter: blob:none
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.env.python }}
+      - name: create hatch environment
+        run: uvx hatch env create ${{ matrix.env.name }}
+      - name: list all all installed package versions
+        run: uvx hatch run ${{ matrix.env.name }}:uv pip list
+      - name: run tests using hatch
+        env:
+          MPLBACKEND: agg
+          PLATFORM: ${{ matrix.os }}
+          DISPLAY: :42
+        run: uvx hatch run ${{ matrix.env.name }}:run-cov -v --color=yes -n auto
+      - name: generate coverage report
+        run: |
+          # See https://coverage.readthedocs.io/page/config.html#run-patch
+          test -f .coverage || uvx hatch run ${{ matrix.env.name }}:cov-combine
+          uvx hatch run ${{ matrix.env.name }}:cov-report   # report visibly
+          uvx hatch run ${{ matrix.env.name }}:coverage xml # create report for upload
+      - name: Upload coverage
+        uses: codecov/codecov-action@v6
+        with:
+          # Don't fail the job if the upload fails: the repo isn't activated in the
+          # Virtual-Cell-Research-Community Codecov org yet, so OIDC tokenless upload is
+          # rejected. Flip back to true once Codecov is set up (see CONTRIBUTORS/PR notes).
+          fail_ci_if_error: false
+          use_oidc: true
+
+  # Install the *exact* lower bounds declared in [project.dependencies] and run the suite against
+  # them. The floors encode real, bisected compatibility limits (see the comment above the
+  # dependency list in pyproject.toml); without this job they are unverifiable claims that drift
+  # out of date silently. Runs on the oldest supported Python for the same reason.
+  floors:
+    name: Tests pass at the declared dependency floors
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          filter: blob:none
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.11"
+      - name: Pin every dependency to its declared floor
+        run: |
+          python - <<'PY' > floors.txt
+          import pathlib, re, tomllib
+          project = tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]
+          deps = project["dependencies"] + project["optional-dependencies"]["sinkhorn"]
+          for dep in deps:
+              m = re.fullmatch(r"([A-Za-z0-9._-]+)>=([0-9][^,;\s]*)", dep.strip())
+              if not m:
+                  raise SystemExit(f"dependency {dep!r} is not a simple `name>=version` floor")
+              print(f"{m[1]}=={m[2]}")
+          PY
+          cat floors.txt
+      - name: Install at the floors
+        # the floors go in first, then the package with --no-deps so it cannot pull them back up
+        run: |
+          uv venv --python 3.11
+          uv pip install -r floors.txt "pytest>=8"
+          uv pip install --no-deps .
+      - name: Run tests
+        env:
+          MPLBACKEND: agg
+        run: uv run --no-sync pytest -v --color=yes
+
+  # Check that all tests defined above pass. This makes it easy to set a single "required" test in branch
+  # protection instead of having to update it frequently. See https://github.com/re-actors/alls-green#why.
+  check:
+    name: Tests pass in all hatch environments
+    if: always()
+    needs:
+      - get-environments
+      - test
+      - floors
+    runs-on: ubuntu-latest
+    steps:
+        - uses: re-actors/alls-green@release/v1
+          with:
+            jobs: ${{ toJSON(needs) }}

scperteval-0.1.0/.gitignore

@@ -0,0 +1,30 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.venv/
+venv/
+
+# Environment
+uv.lock
+.envrc
+requirements-local.txt
+
+# Run outputs
+results/
+
+# Local research/scratch work — kept out of git and out of the sdist (hatchling's sdist
+# includes anything not ignored here, so an untracked scratch file would otherwise ship).
+debug_api.py
+stress_empirical/
+
+# Editor / OS
+.idea/
+.vscode/
+.DS_Store
+
+# docs
+docs/generated/
+docs/_build/

scperteval-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+# Run `pre-commit install` once; hooks then run on every commit.
+# Update pinned revs with `pre-commit autoupdate`.
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.13
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/tox-dev/pyproject-fmt
+    rev: "v2.25.1"
+    hooks:
+      - id: pyproject-fmt
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: check-toml
+      - id: check-merge-conflict
+      - id: check-added-large-files
+        args: [--maxkb=1024]

scperteval-0.1.0/.readthedocs.yaml

@@ -0,0 +1,16 @@
+# https://docs.readthedocs.io/en/stable/config-file/v2.html
+version: 2
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.12"
+  jobs:
+    create_environment:
+      - asdf plugin add uv
+      - asdf install uv latest
+      - asdf global uv latest
+    build:
+      html:
+        - uv sync --group docs
+        - uv run sphinx-build -M html docs docs/_build -W
+        - mv docs/_build $READTHEDOCS_OUTPUT

scperteval-0.1.0/CHANGELOG.md

@@ -0,0 +1,62 @@
+# Changelog
+
+## Unreleased
+
+## 0.1.0
+
+First release of scPertEval — reference implementations of single-cell perturbation evaluation
+protocols, usable from the command line and as a native Python API.
+
+scPertEval accompanies [Schäfer et al. (2026), *Towards Principled Evaluation of Single-Cell
+Perturbation Prediction Models*](https://doi.org/10.64898/2026.07.23.740433).
+
+### Features
+
+- **22 evaluation protocols** in one declarative table, spanning three groups: `pseudobulk`
+  (Pearson/MSE/weighted-MSE variants over full, top-k and DEG feature spaces, plus
+  cross-perturbation retrieval rank), `distributional` (unbiased MMD, energy distance, and
+  Sinkhorn W2 over top-k/PCA spaces), and `de` (AUPRC, AUROC, top-k overlap).
+- **Three actions over the same catalog** — `calibrate` (score a protocol against empirical
+  positive/negative controls per perturbation, reporting DRF or BDS), `score` (score model
+  predictions against ground truth), and `de` (export per-gene differential expression).
+- **Two calibrators**: Dynamic Range Fraction (DRF) and Bound Discrimination Score (BDS).
+- **Native Python API** (`prepare` / `calibrate` / `score` / `de`) that reads and indexes a
+  dataset once and reuses it across calls, alongside the `scperteval` CLI.
+- **Extension points** for new protocols, metrics, feature spaces, DE methods, control sources,
+  and calibrators — each a registered function plus one table row.
+
+### Packaging
+
+- `torch` and `geomloss` are an optional `sinkhorn` extra rather than base dependencies, so
+  `pip install scperteval` stays light. Bulk protocol selections (`-p all`, `-p distributional`)
+  skip the Sinkhorn protocols with a warning when the extra is absent; naming one explicitly
+  raises an error that points at `pip install "scperteval[sinkhorn]"`.
+- Ships a `py.typed` marker, so downstream users get the package's type hints.
+- The version is derived from the git tag by `hatch-vcs`; see `RELEASE.md`.
+
+### A note on the dependency floors
+
+The lower bounds in `pyproject.toml` are bisected, not guessed — each is the oldest version the
+full test suite passes on, and the `floors` CI job reinstalls exactly those pins on every push so
+they cannot drift. Both ends of the supported range are verified: the declared floors
+(anndata 0.12.7 / numpy 2.0 / pandas 2.2.2 / torch 2.4, Python 3.11) and current releases
+(anndata 0.13 / numpy 2.4 / pandas 3.0 / torch 2.13, Python 3.14) both pass, which is why
+dependencies are specified as open-ended floors with no upper caps.
+
+What sets the floors, in case someone tries to lower them:
+
+- **`numpy>=2`** is the dominant constraint. Under numpy 1.x, `illico`'s asymptotic-Wilcoxon
+  path (the `MWU` DE method) derives its test count as a float and formats it with `:,d`,
+  raising `Unknown format code 'd' for object of type 'float'`.
+- **`h5py>=3.11`** and **`scikit-learn>=1.5`** follow from that: earlier releases ship numpy-1.x
+  ABI wheels and abort at import with `numpy.dtype size changed`.
+- **`torch>=2.4`** for the same reason — 2.1–2.3 compute correct results against numpy 2 but
+  bury every run in `A module that was compiled using NumPy 1.x` warnings.
+- **`anndata>=0.12.7`** — below it the suite cannot write its own h5ad fixtures (anndata refuses
+  `pd.arrays.StringArray` without `settings.allow_write_nullable_strings`), and at 0.11.x and
+  older it is a genuine runtime break: `illico` reaches for
+  `anndata._core.sparse_dataset._CSCDataset`, which does not exist yet.
+- **`pandas>=2.2.2`** and **`scipy>=1.13`** are inherited from `illico`'s own requirements;
+  anything lower is unsatisfiable. Note `illico` 0.6.0's declared `anndata>=0.10.8` is looser
+  than what it actually needs, which is why scPertEval pins these from testing rather than
+  relying on the transitive constraint.

scperteval-0.1.0/CITATION.cff

@@ -0,0 +1,66 @@
+cff-version: 1.2.0
+message: "If you use scPertEval, please cite the paper it accompanies (preferred-citation)."
+title: scPertEval
+abstract: >-
+  Evaluation Protocols for Perturbation Studies: reference implementations of single-cell
+  perturbation evaluation protocols, with per-metric DRF/BDS calibration against empirical
+  positive and negative controls on a single preprocessed dataset.
+type: software
+license: MIT
+repository-code: "https://github.com/Virtual-Cell-Research-Community/scPertEval"
+url: "https://scperteval.readthedocs.io/"
+keywords:
+  - single-cell
+  - perturb-seq
+  - perturbation
+  - differential-expression
+  - benchmark
+  - evaluation
+  - bioinformatics
+# The authors of the paper that introduces scPertEval, in publication order.
+authors:
+  - family-names: Schäfer
+    given-names: Philipp S. L.
+    affiliation: >-
+      Institute for Computational Biomedicine, Heidelberg University and Heidelberg University
+      Hospital
+  - family-names: Reid
+    given-names: Kendall A.
+    affiliation: >-
+      Bioinformatics Program, Center for Biotechnology Education, Johns Hopkins University
+  - family-names: Boldyga
+    given-names: Zach
+    affiliation: Independent Researcher
+  - family-names: Aksu
+    given-names: Ekin D.
+    affiliation: >-
+      Department of Computational Molecular Biology, Max Planck Institute for Molecular Genetics
+  - family-names: Hakem
+    given-names: Hugo
+    affiliation: >-
+      European Molecular Biology Laboratory, European Bioinformatics Institute (EMBL-EBI)
+  - family-names: Saez-Rodriguez
+    given-names: Julio
+    affiliation: >-
+      Institute for Computational Biomedicine, Heidelberg University and Heidelberg University
+      Hospital
+preferred-citation:
+  type: article
+  title: Towards Principled Evaluation of Single-Cell Perturbation Prediction Models
+  journal: bioRxiv
+  year: 2026
+  doi: 10.64898/2026.07.23.740433
+  url: "https://www.biorxiv.org/content/10.64898/2026.07.23.740433v1"
+  authors:
+    - family-names: Schäfer
+      given-names: Philipp S. L.
+    - family-names: Reid
+      given-names: Kendall A.
+    - family-names: Boldyga
+      given-names: Zach
+    - family-names: Aksu
+      given-names: Ekin D.
+    - family-names: Hakem
+      given-names: Hugo
+    - family-names: Saez-Rodriguez
+      given-names: Julio

scperteval-0.1.0/CONTRIBUTORS.md

@@ -0,0 +1,31 @@
+# Contributing to scPertEval
+
+scPertEval is meant to be a shared catalog of evaluation protocols, so contributions are
+welcome. There are a few paths, depending on what you're changing.
+
+## New evaluation protocol implementations — open a Pull Request
+
+If you're adding a protocol (a new metric, or a new combination of an existing metric with
+a space / centering / controls), **open a PR directly.** This is the common case and the
+whole point of the project. See [Create a protocol](https://github.com/Virtual-Cell-Research-Community/scPertEval/blob/main/docs/user-guide/protocols.md#create-a-protocol) for the
+two-step pattern (a pure function in `src/scperteval/protocols/metrics.py` plus a row in
+`src/scperteval/protocols/table.py`). Adding a new building block (feature space, DE method, control
+source, calibrator) the same way is also welcome as a PR.
+
+Please include:
+- a one-line reference to the source paper/method the protocol comes from, where applicable;
+- the protocol added to the table and runnable via `scperteval calibrate ... -p <name>`.
+
+## Bugs or changes to core code — open an Issue first
+
+If you've found a bug, or want to change shared/core behavior (the runner, the context
+engine, the reference/sampling logic, the calibrators, or the scoring semantics),
+**open an Issue and discuss it first** before sending a PR. Core changes affect every
+protocol's results, so we want to agree on the approach before implementation.
+
+## Tutorials and notebooks
+
+For tutorials and more in-depth examples, consider adding a notebook under `docs/notebooks/`.
+Commit it as an executed notebook (outputs saved) so the rendered docs show real output, then
+add it to the toctree in `docs/tutorials.md`. The `Notebooks` CI workflow re-executes every
+notebook against the current API to guard against tutorials that no longer run.

scperteval-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Virtual Cell Research Community
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.