perturbguard 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- perturbguard-1.0.0/.github/workflows/ci.yml +35 -0
- perturbguard-1.0.0/CHANGELOG.md +17 -0
- perturbguard-1.0.0/LICENSE +21 -0
- perturbguard-1.0.0/MANIFEST.in +7 -0
- perturbguard-1.0.0/PKG-INFO +263 -0
- perturbguard-1.0.0/README.md +224 -0
- perturbguard-1.0.0/configs/default.yaml +19 -0
- perturbguard-1.0.0/configs/jorge_example.yaml +10 -0
- perturbguard-1.0.0/configs/lincs_example.yaml +11 -0
- perturbguard-1.0.0/configs/sciplex_example.yaml +15 -0
- perturbguard-1.0.0/docs/advanced_features.md +16 -0
- perturbguard-1.0.0/docs/assumptions.md +14 -0
- perturbguard-1.0.0/docs/beta_test_report.md +15 -0
- perturbguard-1.0.0/docs/example_workflows.md +59 -0
- perturbguard-1.0.0/docs/input_contract.md +34 -0
- perturbguard-1.0.0/docs/release_checklist.md +17 -0
- perturbguard-1.0.0/pyproject.toml +74 -0
- perturbguard-1.0.0/scripts/__init__.py +1 -0
- perturbguard-1.0.0/scripts/download_norman_2019.py +12 -0
- perturbguard-1.0.0/scripts/make_demo_subset.py +26 -0
- perturbguard-1.0.0/scripts/run_smoke_matrix.py +224 -0
- perturbguard-1.0.0/setup.cfg +4 -0
- perturbguard-1.0.0/src/perturbguard/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/adversarial/__init__.py +3 -0
- perturbguard-1.0.0/src/perturbguard/adversarial/tests.py +49 -0
- perturbguard-1.0.0/src/perturbguard/benchmark/__init__.py +3 -0
- perturbguard-1.0.0/src/perturbguard/benchmark/manifest.py +53 -0
- perturbguard-1.0.0/src/perturbguard/claims/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/claims/claim_checker.py +49 -0
- perturbguard-1.0.0/src/perturbguard/cli/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/cli/main.py +350 -0
- perturbguard-1.0.0/src/perturbguard/compare/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/compare/datasets.py +23 -0
- perturbguard-1.0.0/src/perturbguard/design/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/design/checker.py +43 -0
- perturbguard-1.0.0/src/perturbguard/design/power.py +61 -0
- perturbguard-1.0.0/src/perturbguard/evaluate/__init__.py +3 -0
- perturbguard-1.0.0/src/perturbguard/evaluate/model.py +146 -0
- perturbguard-1.0.0/src/perturbguard/io/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/io/config.py +33 -0
- perturbguard-1.0.0/src/perturbguard/io/config_wizard.py +51 -0
- perturbguard-1.0.0/src/perturbguard/io/schema.py +58 -0
- perturbguard-1.0.0/src/perturbguard/io/split_loader.py +67 -0
- perturbguard-1.0.0/src/perturbguard/leakage/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/leakage/combination_leakage.py +74 -0
- perturbguard-1.0.0/src/perturbguard/leakage/leakage_graph.py +29 -0
- perturbguard-1.0.0/src/perturbguard/leakage/pathway_leakage.py +43 -0
- perturbguard-1.0.0/src/perturbguard/leakage/split_balance.py +62 -0
- perturbguard-1.0.0/src/perturbguard/leakage/split_leakage.py +47 -0
- perturbguard-1.0.0/src/perturbguard/qc/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/qc/cell_count.py +91 -0
- perturbguard-1.0.0/src/perturbguard/qc/confounding.py +90 -0
- perturbguard-1.0.0/src/perturbguard/qc/control_balance.py +24 -0
- perturbguard-1.0.0/src/perturbguard/qc/dataset_validator.py +61 -0
- perturbguard-1.0.0/src/perturbguard/qc/guide_consistency.py +43 -0
- perturbguard-1.0.0/src/perturbguard/qc/matched_controls.py +29 -0
- perturbguard-1.0.0/src/perturbguard/qc/metadata_shortcut.py +41 -0
- perturbguard-1.0.0/src/perturbguard/qc/perturbation_summary.py +33 -0
- perturbguard-1.0.0/src/perturbguard/qc/target_effect.py +65 -0
- perturbguard-1.0.0/src/perturbguard/qc/target_mapping.py +110 -0
- perturbguard-1.0.0/src/perturbguard/repair/__init__.py +3 -0
- perturbguard-1.0.0/src/perturbguard/repair/anndata.py +44 -0
- perturbguard-1.0.0/src/perturbguard/reports/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/reports/dataset_card.py +50 -0
- perturbguard-1.0.0/src/perturbguard/reports/html.py +108 -0
- perturbguard-1.0.0/src/perturbguard/reports/plots.py +43 -0
- perturbguard-1.0.0/src/perturbguard/reports/recommendations.py +38 -0
- perturbguard-1.0.0/src/perturbguard/reports/summary.py +23 -0
- perturbguard-1.0.0/src/perturbguard/simulate/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/simulate/synthetic_anndata.py +79 -0
- perturbguard-1.0.0/src/perturbguard/splitting/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/splitting/strategies.py +88 -0
- perturbguard-1.0.0/src/perturbguard/streaming/__init__.py +3 -0
- perturbguard-1.0.0/src/perturbguard/streaming/profile.py +27 -0
- perturbguard-1.0.0/src/perturbguard/utils/__init__.py +1 -0
- perturbguard-1.0.0/src/perturbguard/utils/controls.py +19 -0
- perturbguard-1.0.0/src/perturbguard/utils/matrix.py +16 -0
- perturbguard-1.0.0/src/perturbguard/utils/status.py +17 -0
- perturbguard-1.0.0/src/perturbguard.egg-info/PKG-INFO +263 -0
- perturbguard-1.0.0/src/perturbguard.egg-info/SOURCES.txt +96 -0
- perturbguard-1.0.0/src/perturbguard.egg-info/dependency_links.txt +1 -0
- perturbguard-1.0.0/src/perturbguard.egg-info/entry_points.txt +2 -0
- perturbguard-1.0.0/src/perturbguard.egg-info/requires.txt +19 -0
- perturbguard-1.0.0/src/perturbguard.egg-info/top_level.txt +1 -0
- perturbguard-1.0.0/tests/test_bug_hunt_edge_cases.py +263 -0
- perturbguard-1.0.0/tests/test_cli_completion.py +178 -0
- perturbguard-1.0.0/tests/test_hardening.py +132 -0
- perturbguard-1.0.0/tests/test_mvp.py +146 -0
- perturbguard-1.0.0/tests/test_production_readiness.py +94 -0
- perturbguard-1.0.0/tests/test_top10_features.py +220 -0
- perturbguard-1.0.0/tests/test_v02_qc.py +30 -0
- perturbguard-1.0.0/tests/test_v03_quality.py +39 -0
- perturbguard-1.0.0/tests/test_v04_splits_claims.py +42 -0
- perturbguard-1.0.0/tests/test_v05_reports_demo.py +31 -0
- perturbguard-1.0.0/tests/test_v06_stretch.py +46 -0
- perturbguard-1.0.0/workflows/snakemake/Snakefile +13 -0
- perturbguard-1.0.0/workflows/snakemake/config.yaml +1 -0
- perturbguard-1.0.0/workflows/snakemake/profiles/slurm/config.yaml +3 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
test:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
strategy:
|
|
11
|
+
matrix:
|
|
12
|
+
python-version: ["3.11", "3.12"]
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: actions/setup-python@v5
|
|
16
|
+
with:
|
|
17
|
+
python-version: ${{ matrix.python-version }}
|
|
18
|
+
- name: Install
|
|
19
|
+
run: |
|
|
20
|
+
python -m pip install --upgrade pip
|
|
21
|
+
python -m pip install -e ".[dev]"
|
|
22
|
+
- name: Lint
|
|
23
|
+
run: ruff check .
|
|
24
|
+
- name: Test
|
|
25
|
+
run: pytest -q
|
|
26
|
+
- name: Build package
|
|
27
|
+
run: python -m build
|
|
28
|
+
- name: Install wheel smoke
|
|
29
|
+
shell: bash
|
|
30
|
+
run: |
|
|
31
|
+
python -m venv /tmp/pg-wheel
|
|
32
|
+
source /tmp/pg-wheel/bin/activate
|
|
33
|
+
python -m pip install --upgrade pip
|
|
34
|
+
python -m pip install dist/*.whl
|
|
35
|
+
perturbguard --help
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 1.0.0
|
|
4
|
+
|
|
5
|
+
- First stable PerturbGuard release.
|
|
6
|
+
- Adds the full guardrail loop for perturbation benchmarking: config inference, AnnData repair, validation, audit, split generation, claim checking, adversarial leakage checks, model prediction evaluation, benchmark manifest validation, large-file profiling, and dataset-card generation.
|
|
7
|
+
- Adds interactive HTML reports with status summary cards, searchable tables, status filters, plot links, and clearer PASS/WARNING/FAIL/SUPPORTED/UNSUPPORTED styling.
|
|
8
|
+
- Adds target mapping audits for gene, drug class, pathway/class, missing, and unmapped targets, with optional user-supplied perturbation-to-target maps.
|
|
9
|
+
- Adds power/design planning checks for planned cells, controls, replicate support, and batch support.
|
|
10
|
+
- Expands CLI smoke testing across synthetic scenarios, real public datasets, split strategies, claim types, design checks, benchmark manifests, and the advanced guardrail commands.
|
|
11
|
+
- Packages configs, docs, and Snakemake workflow assets in release artifacts.
|
|
12
|
+
|
|
13
|
+
## 0.1.0
|
|
14
|
+
|
|
15
|
+
- Initial production-hardening line for PerturbGuard.
|
|
16
|
+
- Adds CLI commands for validation, audit, split generation, claim checking, dataset comparison, design checks, and synthetic simulation.
|
|
17
|
+
- Adds schema mapping, synthetic and real-data smoke workflows, split leakage checks, QC tables, recommendations, HTML reports, and plot artifacts.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 PerturbGuard contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: perturbguard
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: QC, leakage detection, split design, and claim validation for single-cell perturbation studies.
|
|
5
|
+
Author: PerturbGuard contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/perturbguard/perturbguard
|
|
8
|
+
Project-URL: Documentation, https://github.com/perturbguard/perturbguard#readme
|
|
9
|
+
Project-URL: Issues, https://github.com/perturbguard/perturbguard/issues
|
|
10
|
+
Keywords: single-cell,perturb-seq,quality-control,leakage,anndata
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: anndata>=0.10
|
|
21
|
+
Requires-Dist: numpy
|
|
22
|
+
Requires-Dist: pandas
|
|
23
|
+
Requires-Dist: scipy
|
|
24
|
+
Requires-Dist: scikit-learn
|
|
25
|
+
Requires-Dist: statsmodels
|
|
26
|
+
Requires-Dist: networkx
|
|
27
|
+
Requires-Dist: typer
|
|
28
|
+
Requires-Dist: rich
|
|
29
|
+
Requires-Dist: pyyaml
|
|
30
|
+
Requires-Dist: jinja2
|
|
31
|
+
Requires-Dist: plotly
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: build; extra == "dev"
|
|
34
|
+
Requires-Dist: pytest; extra == "dev"
|
|
35
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
36
|
+
Requires-Dist: ruff; extra == "dev"
|
|
37
|
+
Requires-Dist: mypy; extra == "dev"
|
|
38
|
+
Dynamic: license-file
|
|
39
|
+
|
|
40
|
+
# PerturbGuard
|
|
41
|
+
|
|
42
|
+
[](https://github.com/prithvirajanR/perturbguard/actions/workflows/ci.yml)
|
|
43
|
+
[](https://www.python.org/)
|
|
44
|
+
[](https://github.com/prithvirajanR/perturbguard/releases/tag/v1.0.0)
|
|
45
|
+
[](LICENSE)
|
|
46
|
+
|
|
47
|
+
**FastQC-style guardrails for single-cell perturbation datasets, splits, claims, and model benchmarks.**
|
|
48
|
+
|
|
49
|
+
PerturbGuard helps you answer the uncomfortable but necessary question:
|
|
50
|
+
|
|
51
|
+
> Is this perturbation benchmark actually valid, or did leakage, controls, confounding, weak target effects, or split design make it look better than it is?
|
|
52
|
+
|
|
53
|
+
It works with AnnData `.h5ad` files and produces interactive HTML reports, machine-readable CSV tables, JSON summaries, and Markdown dataset cards.
|
|
54
|
+
|
|
55
|
+
## Why Use It
|
|
56
|
+
|
|
57
|
+
Perturbation models are easy to overclaim. A model can appear to generalize because the split leaked perturbations, a batch variable predicts the label, controls are missing, target effects failed, or a drug target was treated as a single gene when it is really a pathway/class.
|
|
58
|
+
|
|
59
|
+
PerturbGuard audits those failure modes before you trust a dataset, split, claim, or benchmark.
|
|
60
|
+
|
|
61
|
+
## Install
|
|
62
|
+
|
|
63
|
+
From GitHub:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
pip install "perturbguard @ git+https://github.com/prithvirajanR/perturbguard.git@v1.0.0"
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
For local development:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
git clone https://github.com/prithvirajanR/perturbguard.git
|
|
73
|
+
cd perturbguard
|
|
74
|
+
pip install -e ".[dev]"
|
|
75
|
+
pytest -q
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Quickstart
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
perturbguard simulate --scenario batch_confounded --out data/demo/batch_confounded.h5ad
|
|
82
|
+
perturbguard audit --data data/demo/batch_confounded.h5ad --out results/demo_audit
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Open:
|
|
86
|
+
|
|
87
|
+
```text
|
|
88
|
+
results/demo_audit/report.html
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
You get a searchable report with status filters, summary cards, linked plots, recommendations, and CSV tables under `results/demo_audit/tables/`.
|
|
92
|
+
|
|
93
|
+
## Common Workflows
|
|
94
|
+
|
|
95
|
+
### Audit a raw public dataset
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
perturbguard infer-config --data data/raw.h5ad --out configs/inferred.yaml
|
|
99
|
+
perturbguard repair --data data/raw.h5ad --config configs/inferred.yaml --out data/repaired.h5ad
|
|
100
|
+
perturbguard audit --data data/repaired.h5ad --config configs/inferred.yaml --out results/audit
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### Generate and validate a split
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
perturbguard split \
|
|
107
|
+
--data data/repaired.h5ad \
|
|
108
|
+
--strategy leave-perturbation-out \
|
|
109
|
+
--out results/split
|
|
110
|
+
|
|
111
|
+
perturbguard claim \
|
|
112
|
+
--data data/repaired.h5ad \
|
|
113
|
+
--split results/split/split.csv \
|
|
114
|
+
--claim unseen_perturbation \
|
|
115
|
+
--out results/claim
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### Evaluate model predictions
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
perturbguard evaluate \
|
|
122
|
+
--data data/repaired.h5ad \
|
|
123
|
+
--predictions predictions.csv \
|
|
124
|
+
--out results/evaluation
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## Feature Map
|
|
128
|
+
|
|
129
|
+
| Area | What PerturbGuard Does |
|
|
130
|
+
| --- | --- |
|
|
131
|
+
| AnnData validation | Checks loadability, matrix presence, cell/gene counts, controls, perturbation metadata, duplicate obs/var names, and malformed files. |
|
|
132
|
+
| Config inference | Suggests YAML schema mappings from common metadata aliases in messy public datasets. |
|
|
133
|
+
| Repair | Writes normalized `.h5ad` files with canonical metadata, unique indices, inferred controls, and repair action logs. |
|
|
134
|
+
| QC audit | Audits perturbation support, controls, cell counts, confounding, metadata shortcuts, target effects, guide consistency, and target mapping. |
|
|
135
|
+
| Split generation | Creates random, leave-target-gene-out, leave-perturbation-out, metadata holdout, strict combination, and seen-component combination splits. |
|
|
136
|
+
| Claim checking | Verifies whether a split supports claims like unseen perturbation, unseen target gene, or unseen combinations. |
|
|
137
|
+
| Leakage detection | Detects train/val/test leakage, combination leakage, unsupported claims, split imbalance, and invalid split labels. |
|
|
138
|
+
| Model evaluation | Audits prediction CSVs for overall metrics, per-group performance, and confidence calibration. |
|
|
139
|
+
| Adversarial checks | Tests whether metadata-only shortcuts can predict perturbation identity. |
|
|
140
|
+
| Target mapping | Classifies targets as measured genes, drug target classes, pathway/class annotations, missing, or unmapped. |
|
|
141
|
+
| Design planning | Checks planned cells, controls, replicate support, and batch support before running an experiment. |
|
|
142
|
+
| Benchmark manifests | Validates dataset/split/claim/model/metrics manifests and runs claim-support checks. |
|
|
143
|
+
| Large files | Profiles `.h5ad` files in backed mode before expensive audits. |
|
|
144
|
+
| Dataset cards | Generates Markdown dataset cards with audit counts, uses, and limitations. |
|
|
145
|
+
| Reports | Writes interactive HTML reports, plot links, CSV tables, `summary.json`, and recommendations. |
|
|
146
|
+
|
|
147
|
+
## CLI Commands
|
|
148
|
+
|
|
149
|
+
```text
|
|
150
|
+
perturbguard simulate
|
|
151
|
+
perturbguard validate
|
|
152
|
+
perturbguard audit
|
|
153
|
+
perturbguard split
|
|
154
|
+
perturbguard claim
|
|
155
|
+
perturbguard evaluate
|
|
156
|
+
perturbguard repair
|
|
157
|
+
perturbguard infer-config
|
|
158
|
+
perturbguard target-map
|
|
159
|
+
perturbguard compare-datasets
|
|
160
|
+
perturbguard design-check
|
|
161
|
+
perturbguard power-check
|
|
162
|
+
perturbguard benchmark-check
|
|
163
|
+
perturbguard profile-large
|
|
164
|
+
perturbguard adversarial-check
|
|
165
|
+
perturbguard dataset-card
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
## Input Formats
|
|
169
|
+
|
|
170
|
+
### AnnData
|
|
171
|
+
|
|
172
|
+
Required for full audits:
|
|
173
|
+
|
|
174
|
+
- `perturbation`
|
|
175
|
+
- `is_control`, or enough configured control labels to infer controls
|
|
176
|
+
|
|
177
|
+
Recommended:
|
|
178
|
+
|
|
179
|
+
- `target_gene`
|
|
180
|
+
- `guide_id`
|
|
181
|
+
- `perturbation_type`
|
|
182
|
+
- `batch`
|
|
183
|
+
- `replicate`
|
|
184
|
+
- `cell_type`
|
|
185
|
+
- `donor`
|
|
186
|
+
- `plate`
|
|
187
|
+
- `timepoint`
|
|
188
|
+
- `dose`
|
|
189
|
+
|
|
190
|
+
### Prediction CSV
|
|
191
|
+
|
|
192
|
+
Required for `perturbguard evaluate`:
|
|
193
|
+
|
|
194
|
+
- `cell_id`
|
|
195
|
+
- `y_true`
|
|
196
|
+
- `y_pred`
|
|
197
|
+
|
|
198
|
+
Optional:
|
|
199
|
+
|
|
200
|
+
- `confidence`
|
|
201
|
+
|
|
202
|
+
### Target Mapping CSV
|
|
203
|
+
|
|
204
|
+
Recommended for `perturbguard target-map`:
|
|
205
|
+
|
|
206
|
+
- `perturbation`
|
|
207
|
+
- `target`
|
|
208
|
+
- optional `target_type`
|
|
209
|
+
- optional `source`
|
|
210
|
+
|
|
211
|
+
### Benchmark Manifest
|
|
212
|
+
|
|
213
|
+
```yaml
|
|
214
|
+
dataset: data/repaired.h5ad
|
|
215
|
+
split: results/split/split.csv
|
|
216
|
+
claim: unseen_perturbation
|
|
217
|
+
model:
|
|
218
|
+
name: my-model
|
|
219
|
+
metrics:
|
|
220
|
+
- accuracy
|
|
221
|
+
- macro_f1
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
## Real Data Smoke Test
|
|
225
|
+
|
|
226
|
+
PerturbGuard has been smoke-tested locally on public example AnnData files for SciPlex, Jorge, and LINCS-style perturbation datasets.
|
|
227
|
+
|
|
228
|
+
Example SciPlex run:
|
|
229
|
+
|
|
230
|
+
```powershell
|
|
231
|
+
Invoke-WebRequest `
|
|
232
|
+
-Uri "https://huggingface.co/cyclopeta/PerturbNet_reproduce/resolve/main/example_data/sciplex_example.h5ad" `
|
|
233
|
+
-OutFile data/real/sciplex_example.h5ad
|
|
234
|
+
|
|
235
|
+
perturbguard audit `
|
|
236
|
+
--data data/real/sciplex_example.h5ad `
|
|
237
|
+
--config configs/sciplex_example.yaml `
|
|
238
|
+
--out results/real/sciplex_audit
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
Full smoke matrix:
|
|
242
|
+
|
|
243
|
+
```bash
|
|
244
|
+
python scripts/run_smoke_matrix.py --include-real
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
## What It Is Not
|
|
248
|
+
|
|
249
|
+
PerturbGuard is not a perturbation prediction model and does not claim biological truth for unmeasured perturbations. It is a guardrail system: it tells you when your dataset, split, controls, metadata, or benchmark claim may not support the conclusion you want to draw.
|
|
250
|
+
|
|
251
|
+
## Release
|
|
252
|
+
|
|
253
|
+
Current stable release: `1.0.0`
|
|
254
|
+
|
|
255
|
+
Built and verified with:
|
|
256
|
+
|
|
257
|
+
- Python 3.11+
|
|
258
|
+
- AnnData
|
|
259
|
+
- pandas / NumPy / SciPy
|
|
260
|
+
- scikit-learn
|
|
261
|
+
- Plotly
|
|
262
|
+
- Typer
|
|
263
|
+
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
# PerturbGuard
|
|
2
|
+
|
|
3
|
+
[](https://github.com/prithvirajanR/perturbguard/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.python.org/)
|
|
5
|
+
[](https://github.com/prithvirajanR/perturbguard/releases/tag/v1.0.0)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
**FastQC-style guardrails for single-cell perturbation datasets, splits, claims, and model benchmarks.**
|
|
9
|
+
|
|
10
|
+
PerturbGuard helps you answer the uncomfortable but necessary question:
|
|
11
|
+
|
|
12
|
+
> Is this perturbation benchmark actually valid, or did leakage, controls, confounding, weak target effects, or split design make it look better than it is?
|
|
13
|
+
|
|
14
|
+
It works with AnnData `.h5ad` files and produces interactive HTML reports, machine-readable CSV tables, JSON summaries, and Markdown dataset cards.
|
|
15
|
+
|
|
16
|
+
## Why Use It
|
|
17
|
+
|
|
18
|
+
Perturbation models are easy to overclaim. A model can appear to generalize because the split leaked perturbations, a batch variable predicts the label, controls are missing, target effects failed, or a drug target was treated as a single gene when it is really a pathway/class.
|
|
19
|
+
|
|
20
|
+
PerturbGuard audits those failure modes before you trust a dataset, split, claim, or benchmark.
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
From GitHub:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install "perturbguard @ git+https://github.com/prithvirajanR/perturbguard.git@v1.0.0"
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
For local development:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
git clone https://github.com/prithvirajanR/perturbguard.git
|
|
34
|
+
cd perturbguard
|
|
35
|
+
pip install -e ".[dev]"
|
|
36
|
+
pytest -q
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Quickstart
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
perturbguard simulate --scenario batch_confounded --out data/demo/batch_confounded.h5ad
|
|
43
|
+
perturbguard audit --data data/demo/batch_confounded.h5ad --out results/demo_audit
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Open:
|
|
47
|
+
|
|
48
|
+
```text
|
|
49
|
+
results/demo_audit/report.html
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
You get a searchable report with status filters, summary cards, linked plots, recommendations, and CSV tables under `results/demo_audit/tables/`.
|
|
53
|
+
|
|
54
|
+
## Common Workflows
|
|
55
|
+
|
|
56
|
+
### Audit a raw public dataset
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
perturbguard infer-config --data data/raw.h5ad --out configs/inferred.yaml
|
|
60
|
+
perturbguard repair --data data/raw.h5ad --config configs/inferred.yaml --out data/repaired.h5ad
|
|
61
|
+
perturbguard audit --data data/repaired.h5ad --config configs/inferred.yaml --out results/audit
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### Generate and validate a split
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
perturbguard split \
|
|
68
|
+
--data data/repaired.h5ad \
|
|
69
|
+
--strategy leave-perturbation-out \
|
|
70
|
+
--out results/split
|
|
71
|
+
|
|
72
|
+
perturbguard claim \
|
|
73
|
+
--data data/repaired.h5ad \
|
|
74
|
+
--split results/split/split.csv \
|
|
75
|
+
--claim unseen_perturbation \
|
|
76
|
+
--out results/claim
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Evaluate model predictions
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
perturbguard evaluate \
|
|
83
|
+
--data data/repaired.h5ad \
|
|
84
|
+
--predictions predictions.csv \
|
|
85
|
+
--out results/evaluation
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Feature Map
|
|
89
|
+
|
|
90
|
+
| Area | What PerturbGuard Does |
|
|
91
|
+
| --- | --- |
|
|
92
|
+
| AnnData validation | Checks loadability, matrix presence, cell/gene counts, controls, perturbation metadata, duplicate obs/var names, and malformed files. |
|
|
93
|
+
| Config inference | Suggests YAML schema mappings from common metadata aliases in messy public datasets. |
|
|
94
|
+
| Repair | Writes normalized `.h5ad` files with canonical metadata, unique indices, inferred controls, and repair action logs. |
|
|
95
|
+
| QC audit | Audits perturbation support, controls, cell counts, confounding, metadata shortcuts, target effects, guide consistency, and target mapping. |
|
|
96
|
+
| Split generation | Creates random, leave-target-gene-out, leave-perturbation-out, metadata holdout, strict combination, and seen-component combination splits. |
|
|
97
|
+
| Claim checking | Verifies whether a split supports claims like unseen perturbation, unseen target gene, or unseen combinations. |
|
|
98
|
+
| Leakage detection | Detects train/val/test leakage, combination leakage, unsupported claims, split imbalance, and invalid split labels. |
|
|
99
|
+
| Model evaluation | Audits prediction CSVs for overall metrics, per-group performance, and confidence calibration. |
|
|
100
|
+
| Adversarial checks | Tests whether metadata-only shortcuts can predict perturbation identity. |
|
|
101
|
+
| Target mapping | Classifies targets as measured genes, drug target classes, pathway/class annotations, missing, or unmapped. |
|
|
102
|
+
| Design planning | Checks planned cells, controls, replicate support, and batch support before running an experiment. |
|
|
103
|
+
| Benchmark manifests | Validates dataset/split/claim/model/metrics manifests and runs claim-support checks. |
|
|
104
|
+
| Large files | Profiles `.h5ad` files in backed mode before expensive audits. |
|
|
105
|
+
| Dataset cards | Generates Markdown dataset cards with audit counts, uses, and limitations. |
|
|
106
|
+
| Reports | Writes interactive HTML reports, plot links, CSV tables, `summary.json`, and recommendations. |
|
|
107
|
+
|
|
108
|
+
## CLI Commands
|
|
109
|
+
|
|
110
|
+
```text
|
|
111
|
+
perturbguard simulate
|
|
112
|
+
perturbguard validate
|
|
113
|
+
perturbguard audit
|
|
114
|
+
perturbguard split
|
|
115
|
+
perturbguard claim
|
|
116
|
+
perturbguard evaluate
|
|
117
|
+
perturbguard repair
|
|
118
|
+
perturbguard infer-config
|
|
119
|
+
perturbguard target-map
|
|
120
|
+
perturbguard compare-datasets
|
|
121
|
+
perturbguard design-check
|
|
122
|
+
perturbguard power-check
|
|
123
|
+
perturbguard benchmark-check
|
|
124
|
+
perturbguard profile-large
|
|
125
|
+
perturbguard adversarial-check
|
|
126
|
+
perturbguard dataset-card
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
## Input Formats
|
|
130
|
+
|
|
131
|
+
### AnnData
|
|
132
|
+
|
|
133
|
+
Required for full audits:
|
|
134
|
+
|
|
135
|
+
- `perturbation`
|
|
136
|
+
- `is_control`, or enough configured control labels to infer controls
|
|
137
|
+
|
|
138
|
+
Recommended:
|
|
139
|
+
|
|
140
|
+
- `target_gene`
|
|
141
|
+
- `guide_id`
|
|
142
|
+
- `perturbation_type`
|
|
143
|
+
- `batch`
|
|
144
|
+
- `replicate`
|
|
145
|
+
- `cell_type`
|
|
146
|
+
- `donor`
|
|
147
|
+
- `plate`
|
|
148
|
+
- `timepoint`
|
|
149
|
+
- `dose`
|
|
150
|
+
|
|
151
|
+
### Prediction CSV
|
|
152
|
+
|
|
153
|
+
Required for `perturbguard evaluate`:
|
|
154
|
+
|
|
155
|
+
- `cell_id`
|
|
156
|
+
- `y_true`
|
|
157
|
+
- `y_pred`
|
|
158
|
+
|
|
159
|
+
Optional:
|
|
160
|
+
|
|
161
|
+
- `confidence`
|
|
162
|
+
|
|
163
|
+
### Target Mapping CSV
|
|
164
|
+
|
|
165
|
+
Recommended for `perturbguard target-map`:
|
|
166
|
+
|
|
167
|
+
- `perturbation`
|
|
168
|
+
- `target`
|
|
169
|
+
- optional `target_type`
|
|
170
|
+
- optional `source`
|
|
171
|
+
|
|
172
|
+
### Benchmark Manifest
|
|
173
|
+
|
|
174
|
+
```yaml
|
|
175
|
+
dataset: data/repaired.h5ad
|
|
176
|
+
split: results/split/split.csv
|
|
177
|
+
claim: unseen_perturbation
|
|
178
|
+
model:
|
|
179
|
+
name: my-model
|
|
180
|
+
metrics:
|
|
181
|
+
- accuracy
|
|
182
|
+
- macro_f1
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
## Real Data Smoke Test
|
|
186
|
+
|
|
187
|
+
PerturbGuard has been smoke-tested locally on public example AnnData files for SciPlex, Jorge, and LINCS-style perturbation datasets.
|
|
188
|
+
|
|
189
|
+
Example SciPlex run:
|
|
190
|
+
|
|
191
|
+
```powershell
|
|
192
|
+
Invoke-WebRequest `
|
|
193
|
+
-Uri "https://huggingface.co/cyclopeta/PerturbNet_reproduce/resolve/main/example_data/sciplex_example.h5ad" `
|
|
194
|
+
-OutFile data/real/sciplex_example.h5ad
|
|
195
|
+
|
|
196
|
+
perturbguard audit `
|
|
197
|
+
--data data/real/sciplex_example.h5ad `
|
|
198
|
+
--config configs/sciplex_example.yaml `
|
|
199
|
+
--out results/real/sciplex_audit
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
Full smoke matrix:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
python scripts/run_smoke_matrix.py --include-real
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
## What It Is Not
|
|
209
|
+
|
|
210
|
+
PerturbGuard is not a perturbation prediction model and does not claim biological truth for unmeasured perturbations. It is a guardrail system: it tells you when your dataset, split, controls, metadata, or benchmark claim may not support the conclusion you want to draw.
|
|
211
|
+
|
|
212
|
+
## Release
|
|
213
|
+
|
|
214
|
+
Current stable release: `1.0.0`
|
|
215
|
+
|
|
216
|
+
Built and verified with:
|
|
217
|
+
|
|
218
|
+
- Python 3.11+
|
|
219
|
+
- AnnData
|
|
220
|
+
- pandas / NumPy / SciPy
|
|
221
|
+
- scikit-learn
|
|
222
|
+
- Plotly
|
|
223
|
+
- Typer
|
|
224
|
+
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
schema:
|
|
2
|
+
perturbation: perturbation
|
|
3
|
+
is_control: is_control
|
|
4
|
+
target_gene: target_gene
|
|
5
|
+
guide_id: guide_id
|
|
6
|
+
perturbation_type: perturbation_type
|
|
7
|
+
cell_type: cell_type
|
|
8
|
+
batch: batch
|
|
9
|
+
replicate: replicate
|
|
10
|
+
controls:
|
|
11
|
+
labels: ["control", "ctrl", "NTC", "non-targeting", "DMSO", "vehicle"]
|
|
12
|
+
min_controls_per_group: 30
|
|
13
|
+
cell_count:
|
|
14
|
+
min_cells_warning: 50
|
|
15
|
+
min_cells_fail: 20
|
|
16
|
+
confounding:
|
|
17
|
+
metadata_columns: ["batch", "replicate", "donor", "plate", "timepoint"]
|
|
18
|
+
cramers_v_warning: 0.3
|
|
19
|
+
cramers_v_fail: 0.5
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
schema:
|
|
2
|
+
perturbation: product_name_clean
|
|
3
|
+
is_control: vehicle
|
|
4
|
+
target_gene: target
|
|
5
|
+
perturbation_type: perturb_string_processed
|
|
6
|
+
cell_type: cell_type
|
|
7
|
+
batch: culture_plate
|
|
8
|
+
replicate: replicate
|
|
9
|
+
timepoint: time_point
|
|
10
|
+
dose: dose
|
|
11
|
+
n_counts: n_counts
|
|
12
|
+
pct_mito: percent_mito
|
|
13
|
+
|
|
14
|
+
controls:
|
|
15
|
+
labels: ["DMSO", "vehicle", "control", "ctrl"]
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Advanced Features
|
|
2
|
+
|
|
3
|
+
PerturbGuard now includes ten higher-level guardrails beyond the original audit core.
|
|
4
|
+
|
|
5
|
+
1. `evaluate`: audits model predictions overall, by metadata group, and by optional confidence calibration.
|
|
6
|
+
2. `repair`: writes a normalized AnnData file with canonical metadata, unique indices, and inferred controls where possible.
|
|
7
|
+
3. Interactive HTML reports: reports include search, status filters, summary cards, and plot links.
|
|
8
|
+
4. `target-map`: classifies target annotations as measured genes, drug classes, pathways/classes, missing, or unmapped.
|
|
9
|
+
5. `power-check`: checks planned cells, controls, replicate support, and batch support before an experiment.
|
|
10
|
+
6. `benchmark-check`: validates a benchmark manifest and checks whether the declared claim is supported by the split.
|
|
11
|
+
7. `profile-large`: opens `.h5ad` files in backed mode and reports shape/storage metadata before expensive audits.
|
|
12
|
+
8. `adversarial-check`: asks whether technical metadata can predict perturbation identity.
|
|
13
|
+
9. `infer-config`: suggests a YAML schema mapping from common metadata aliases.
|
|
14
|
+
10. `dataset-card`: writes a Markdown summary of dataset size, controls, audit statuses, supported uses, and limitations.
|
|
15
|
+
|
|
16
|
+
These checks are guardrails. They help prevent invalid benchmark claims and brittle analyses, but they do not replace biological validation or model-specific evaluation.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Statistical Assumptions And Thresholds
|
|
2
|
+
|
|
3
|
+
PerturbGuard reports measurable audit risks. It does not prove biological truth.
|
|
4
|
+
|
|
5
|
+
Default thresholds are intentionally conservative:
|
|
6
|
+
|
|
7
|
+
- Low perturbation support: warning below 50 cells, fail below 20 cells.
|
|
8
|
+
- Batch or metadata confounding: warning at Cramer's V >= 0.3, fail at >= 0.5.
|
|
9
|
+
- Dominant metadata concentration: warning at 70%, fail at 90%.
|
|
10
|
+
- Metadata-only shortcut baseline: warning at balanced accuracy >= 0.4, fail at >= 0.6.
|
|
11
|
+
- Guide consistency: warning at median correlation <= 0.2, fail at <= 0.0.
|
|
12
|
+
|
|
13
|
+
These are screening thresholds, not universal biological cutoffs. Users should tune them for
|
|
14
|
+
experiment size, modality, cell type diversity, dose design, and intended claim.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Beta Test Report
|
|
2
|
+
|
|
3
|
+
PerturbGuard has been smoke-tested on synthetic perturbation scenarios and public example
|
|
4
|
+
AnnData files from the PerturbNet reproduction repository.
|
|
5
|
+
|
|
6
|
+
Real-data cases:
|
|
7
|
+
|
|
8
|
+
| Dataset | File | Purpose |
|
|
9
|
+
|---|---|---|
|
|
10
|
+
| SciPlex | `data/real/sciplex_example.h5ad` | Drug perturbation metadata, dose/time/cell-type fields |
|
|
11
|
+
| Jorge | `data/real/Jorge_example.h5ad` | Variant perturbation labels and dataset-specific controls |
|
|
12
|
+
| LINCS | `data/real/lincs_example.h5ad` | Drug perturbations, many batches/cell IDs, provided holdouts |
|
|
13
|
+
|
|
14
|
+
The real-data tests verify that schema mapping, missing-control handling, target classification,
|
|
15
|
+
audit table writing, HTML reports, plots, and provided split claim checks complete without crashes.
|