perturbguard 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. perturbguard-1.0.0/.github/workflows/ci.yml +35 -0
  2. perturbguard-1.0.0/CHANGELOG.md +17 -0
  3. perturbguard-1.0.0/LICENSE +21 -0
  4. perturbguard-1.0.0/MANIFEST.in +7 -0
  5. perturbguard-1.0.0/PKG-INFO +263 -0
  6. perturbguard-1.0.0/README.md +224 -0
  7. perturbguard-1.0.0/configs/default.yaml +19 -0
  8. perturbguard-1.0.0/configs/jorge_example.yaml +10 -0
  9. perturbguard-1.0.0/configs/lincs_example.yaml +11 -0
  10. perturbguard-1.0.0/configs/sciplex_example.yaml +15 -0
  11. perturbguard-1.0.0/docs/advanced_features.md +16 -0
  12. perturbguard-1.0.0/docs/assumptions.md +14 -0
  13. perturbguard-1.0.0/docs/beta_test_report.md +15 -0
  14. perturbguard-1.0.0/docs/example_workflows.md +59 -0
  15. perturbguard-1.0.0/docs/input_contract.md +34 -0
  16. perturbguard-1.0.0/docs/release_checklist.md +17 -0
  17. perturbguard-1.0.0/pyproject.toml +74 -0
  18. perturbguard-1.0.0/scripts/__init__.py +1 -0
  19. perturbguard-1.0.0/scripts/download_norman_2019.py +12 -0
  20. perturbguard-1.0.0/scripts/make_demo_subset.py +26 -0
  21. perturbguard-1.0.0/scripts/run_smoke_matrix.py +224 -0
  22. perturbguard-1.0.0/setup.cfg +4 -0
  23. perturbguard-1.0.0/src/perturbguard/__init__.py +1 -0
  24. perturbguard-1.0.0/src/perturbguard/adversarial/__init__.py +3 -0
  25. perturbguard-1.0.0/src/perturbguard/adversarial/tests.py +49 -0
  26. perturbguard-1.0.0/src/perturbguard/benchmark/__init__.py +3 -0
  27. perturbguard-1.0.0/src/perturbguard/benchmark/manifest.py +53 -0
  28. perturbguard-1.0.0/src/perturbguard/claims/__init__.py +1 -0
  29. perturbguard-1.0.0/src/perturbguard/claims/claim_checker.py +49 -0
  30. perturbguard-1.0.0/src/perturbguard/cli/__init__.py +1 -0
  31. perturbguard-1.0.0/src/perturbguard/cli/main.py +350 -0
  32. perturbguard-1.0.0/src/perturbguard/compare/__init__.py +1 -0
  33. perturbguard-1.0.0/src/perturbguard/compare/datasets.py +23 -0
  34. perturbguard-1.0.0/src/perturbguard/design/__init__.py +1 -0
  35. perturbguard-1.0.0/src/perturbguard/design/checker.py +43 -0
  36. perturbguard-1.0.0/src/perturbguard/design/power.py +61 -0
  37. perturbguard-1.0.0/src/perturbguard/evaluate/__init__.py +3 -0
  38. perturbguard-1.0.0/src/perturbguard/evaluate/model.py +146 -0
  39. perturbguard-1.0.0/src/perturbguard/io/__init__.py +1 -0
  40. perturbguard-1.0.0/src/perturbguard/io/config.py +33 -0
  41. perturbguard-1.0.0/src/perturbguard/io/config_wizard.py +51 -0
  42. perturbguard-1.0.0/src/perturbguard/io/schema.py +58 -0
  43. perturbguard-1.0.0/src/perturbguard/io/split_loader.py +67 -0
  44. perturbguard-1.0.0/src/perturbguard/leakage/__init__.py +1 -0
  45. perturbguard-1.0.0/src/perturbguard/leakage/combination_leakage.py +74 -0
  46. perturbguard-1.0.0/src/perturbguard/leakage/leakage_graph.py +29 -0
  47. perturbguard-1.0.0/src/perturbguard/leakage/pathway_leakage.py +43 -0
  48. perturbguard-1.0.0/src/perturbguard/leakage/split_balance.py +62 -0
  49. perturbguard-1.0.0/src/perturbguard/leakage/split_leakage.py +47 -0
  50. perturbguard-1.0.0/src/perturbguard/qc/__init__.py +1 -0
  51. perturbguard-1.0.0/src/perturbguard/qc/cell_count.py +91 -0
  52. perturbguard-1.0.0/src/perturbguard/qc/confounding.py +90 -0
  53. perturbguard-1.0.0/src/perturbguard/qc/control_balance.py +24 -0
  54. perturbguard-1.0.0/src/perturbguard/qc/dataset_validator.py +61 -0
  55. perturbguard-1.0.0/src/perturbguard/qc/guide_consistency.py +43 -0
  56. perturbguard-1.0.0/src/perturbguard/qc/matched_controls.py +29 -0
  57. perturbguard-1.0.0/src/perturbguard/qc/metadata_shortcut.py +41 -0
  58. perturbguard-1.0.0/src/perturbguard/qc/perturbation_summary.py +33 -0
  59. perturbguard-1.0.0/src/perturbguard/qc/target_effect.py +65 -0
  60. perturbguard-1.0.0/src/perturbguard/qc/target_mapping.py +110 -0
  61. perturbguard-1.0.0/src/perturbguard/repair/__init__.py +3 -0
  62. perturbguard-1.0.0/src/perturbguard/repair/anndata.py +44 -0
  63. perturbguard-1.0.0/src/perturbguard/reports/__init__.py +1 -0
  64. perturbguard-1.0.0/src/perturbguard/reports/dataset_card.py +50 -0
  65. perturbguard-1.0.0/src/perturbguard/reports/html.py +108 -0
  66. perturbguard-1.0.0/src/perturbguard/reports/plots.py +43 -0
  67. perturbguard-1.0.0/src/perturbguard/reports/recommendations.py +38 -0
  68. perturbguard-1.0.0/src/perturbguard/reports/summary.py +23 -0
  69. perturbguard-1.0.0/src/perturbguard/simulate/__init__.py +1 -0
  70. perturbguard-1.0.0/src/perturbguard/simulate/synthetic_anndata.py +79 -0
  71. perturbguard-1.0.0/src/perturbguard/splitting/__init__.py +1 -0
  72. perturbguard-1.0.0/src/perturbguard/splitting/strategies.py +88 -0
  73. perturbguard-1.0.0/src/perturbguard/streaming/__init__.py +3 -0
  74. perturbguard-1.0.0/src/perturbguard/streaming/profile.py +27 -0
  75. perturbguard-1.0.0/src/perturbguard/utils/__init__.py +1 -0
  76. perturbguard-1.0.0/src/perturbguard/utils/controls.py +19 -0
  77. perturbguard-1.0.0/src/perturbguard/utils/matrix.py +16 -0
  78. perturbguard-1.0.0/src/perturbguard/utils/status.py +17 -0
  79. perturbguard-1.0.0/src/perturbguard.egg-info/PKG-INFO +263 -0
  80. perturbguard-1.0.0/src/perturbguard.egg-info/SOURCES.txt +96 -0
  81. perturbguard-1.0.0/src/perturbguard.egg-info/dependency_links.txt +1 -0
  82. perturbguard-1.0.0/src/perturbguard.egg-info/entry_points.txt +2 -0
  83. perturbguard-1.0.0/src/perturbguard.egg-info/requires.txt +19 -0
  84. perturbguard-1.0.0/src/perturbguard.egg-info/top_level.txt +1 -0
  85. perturbguard-1.0.0/tests/test_bug_hunt_edge_cases.py +263 -0
  86. perturbguard-1.0.0/tests/test_cli_completion.py +178 -0
  87. perturbguard-1.0.0/tests/test_hardening.py +132 -0
  88. perturbguard-1.0.0/tests/test_mvp.py +146 -0
  89. perturbguard-1.0.0/tests/test_production_readiness.py +94 -0
  90. perturbguard-1.0.0/tests/test_top10_features.py +220 -0
  91. perturbguard-1.0.0/tests/test_v02_qc.py +30 -0
  92. perturbguard-1.0.0/tests/test_v03_quality.py +39 -0
  93. perturbguard-1.0.0/tests/test_v04_splits_claims.py +42 -0
  94. perturbguard-1.0.0/tests/test_v05_reports_demo.py +31 -0
  95. perturbguard-1.0.0/tests/test_v06_stretch.py +46 -0
  96. perturbguard-1.0.0/workflows/snakemake/Snakefile +13 -0
  97. perturbguard-1.0.0/workflows/snakemake/config.yaml +1 -0
  98. perturbguard-1.0.0/workflows/snakemake/profiles/slurm/config.yaml +3 -0
@@ -0,0 +1,35 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ pull_request:
6
+
7
+ jobs:
8
+ test:
9
+ runs-on: ubuntu-latest
10
+ strategy:
11
+ matrix:
12
+ python-version: ["3.11", "3.12"]
13
+ steps:
14
+ - uses: actions/checkout@v4
15
+ - uses: actions/setup-python@v5
16
+ with:
17
+ python-version: ${{ matrix.python-version }}
18
+ - name: Install
19
+ run: |
20
+ python -m pip install --upgrade pip
21
+ python -m pip install -e ".[dev]"
22
+ - name: Lint
23
+ run: ruff check .
24
+ - name: Test
25
+ run: pytest -q
26
+ - name: Build package
27
+ run: python -m build
28
+ - name: Install wheel smoke
29
+ shell: bash
30
+ run: |
31
+ python -m venv /tmp/pg-wheel
32
+ source /tmp/pg-wheel/bin/activate
33
+ python -m pip install --upgrade pip
34
+ python -m pip install dist/*.whl
35
+ perturbguard --help
@@ -0,0 +1,17 @@
1
+ # Changelog
2
+
3
+ ## 1.0.0
4
+
5
+ - First stable PerturbGuard release.
6
+ - Adds the full guardrail loop for perturbation benchmarking: config inference, AnnData repair, validation, audit, split generation, claim checking, adversarial leakage checks, model prediction evaluation, benchmark manifest validation, large-file profiling, and dataset-card generation.
7
+ - Adds interactive HTML reports with status summary cards, searchable tables, status filters, plot links, and clearer PASS/WARNING/FAIL/SUPPORTED/UNSUPPORTED styling.
8
+ - Adds target mapping audits for gene, drug class, pathway/class, missing, and unmapped targets, with optional user-supplied perturbation-to-target maps.
9
+ - Adds power/design planning checks for planned cells, controls, replicate support, and batch support.
10
+ - Expands CLI smoke testing across synthetic scenarios, real public datasets, split strategies, claim types, design checks, benchmark manifests, and the advanced guardrail commands.
11
+ - Packages configs, docs, and Snakemake workflow assets in release artifacts.
12
+
13
+ ## 0.1.0
14
+
15
+ - Initial production-hardening line for PerturbGuard.
16
+ - Adds CLI commands for validation, audit, split generation, claim checking, dataset comparison, design checks, and synthetic simulation.
17
+ - Adds schema mapping, synthetic and real-data smoke workflows, split leakage checks, QC tables, recommendations, HTML reports, and plot artifacts.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 PerturbGuard contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,7 @@
1
+ include LICENSE
2
+ include CHANGELOG.md
3
+ recursive-include configs *.yaml
4
+ recursive-include docs *.md
5
+ recursive-include scripts *.py
6
+ recursive-include workflows *
7
+ recursive-include .github *.yml
@@ -0,0 +1,263 @@
1
+ Metadata-Version: 2.4
2
+ Name: perturbguard
3
+ Version: 1.0.0
4
+ Summary: QC, leakage detection, split design, and claim validation for single-cell perturbation studies.
5
+ Author: PerturbGuard contributors
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/perturbguard/perturbguard
8
+ Project-URL: Documentation, https://github.com/perturbguard/perturbguard#readme
9
+ Project-URL: Issues, https://github.com/perturbguard/perturbguard/issues
10
+ Keywords: single-cell,perturb-seq,quality-control,leakage,anndata
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
17
+ Requires-Python: >=3.11
18
+ Description-Content-Type: text/markdown
19
+ License-File: LICENSE
20
+ Requires-Dist: anndata>=0.10
21
+ Requires-Dist: numpy
22
+ Requires-Dist: pandas
23
+ Requires-Dist: scipy
24
+ Requires-Dist: scikit-learn
25
+ Requires-Dist: statsmodels
26
+ Requires-Dist: networkx
27
+ Requires-Dist: typer
28
+ Requires-Dist: rich
29
+ Requires-Dist: pyyaml
30
+ Requires-Dist: jinja2
31
+ Requires-Dist: plotly
32
+ Provides-Extra: dev
33
+ Requires-Dist: build; extra == "dev"
34
+ Requires-Dist: pytest; extra == "dev"
35
+ Requires-Dist: pytest-cov; extra == "dev"
36
+ Requires-Dist: ruff; extra == "dev"
37
+ Requires-Dist: mypy; extra == "dev"
38
+ Dynamic: license-file
39
+
40
+ # PerturbGuard
41
+
42
+ [![CI](https://github.com/prithvirajanR/perturbguard/actions/workflows/ci.yml/badge.svg)](https://github.com/prithvirajanR/perturbguard/actions/workflows/ci.yml)
43
+ [![Python](https://img.shields.io/badge/python-3.11%2B-blue)](https://www.python.org/)
44
+ [![Release](https://img.shields.io/badge/release-1.0.0-green)](https://github.com/prithvirajanR/perturbguard/releases/tag/v1.0.0)
45
+ [![License](https://img.shields.io/badge/license-MIT-lightgrey)](LICENSE)
46
+
47
+ **FastQC-style guardrails for single-cell perturbation datasets, splits, claims, and model benchmarks.**
48
+
49
+ PerturbGuard helps you answer the uncomfortable but necessary question:
50
+
51
+ > Is this perturbation benchmark actually valid, or did leakage, controls, confounding, weak target effects, or split design make it look better than it is?
52
+
53
+ It works with AnnData `.h5ad` files and produces interactive HTML reports, machine-readable CSV tables, JSON summaries, and Markdown dataset cards.
54
+
55
+ ## Why Use It
56
+
57
+ Perturbation models are easy to overclaim. A model can appear to generalize because the split leaked perturbations, a batch variable predicts the label, controls are missing, target effects failed, or a drug target was treated as a single gene when it is really a pathway/class.
58
+
59
+ PerturbGuard audits those failure modes before you trust a dataset, split, claim, or benchmark.
60
+
61
+ ## Install
62
+
63
+ From GitHub:
64
+
65
+ ```bash
66
+ pip install "perturbguard @ git+https://github.com/prithvirajanR/perturbguard.git@v1.0.0"
67
+ ```
68
+
69
+ For local development:
70
+
71
+ ```bash
72
+ git clone https://github.com/prithvirajanR/perturbguard.git
73
+ cd perturbguard
74
+ pip install -e ".[dev]"
75
+ pytest -q
76
+ ```
77
+
78
+ ## Quickstart
79
+
80
+ ```bash
81
+ perturbguard simulate --scenario batch_confounded --out data/demo/batch_confounded.h5ad
82
+ perturbguard audit --data data/demo/batch_confounded.h5ad --out results/demo_audit
83
+ ```
84
+
85
+ Open:
86
+
87
+ ```text
88
+ results/demo_audit/report.html
89
+ ```
90
+
91
+ You get a searchable report with status filters, summary cards, linked plots, recommendations, and CSV tables under `results/demo_audit/tables/`.
92
+
93
+ ## Common Workflows
94
+
95
+ ### Audit a raw public dataset
96
+
97
+ ```bash
98
+ perturbguard infer-config --data data/raw.h5ad --out configs/inferred.yaml
99
+ perturbguard repair --data data/raw.h5ad --config configs/inferred.yaml --out data/repaired.h5ad
100
+ perturbguard audit --data data/repaired.h5ad --config configs/inferred.yaml --out results/audit
101
+ ```
102
+
103
+ ### Generate and validate a split
104
+
105
+ ```bash
106
+ perturbguard split \
107
+ --data data/repaired.h5ad \
108
+ --strategy leave-perturbation-out \
109
+ --out results/split
110
+
111
+ perturbguard claim \
112
+ --data data/repaired.h5ad \
113
+ --split results/split/split.csv \
114
+ --claim unseen_perturbation \
115
+ --out results/claim
116
+ ```
117
+
118
+ ### Evaluate model predictions
119
+
120
+ ```bash
121
+ perturbguard evaluate \
122
+ --data data/repaired.h5ad \
123
+ --predictions predictions.csv \
124
+ --out results/evaluation
125
+ ```
126
+
127
+ ## Feature Map
128
+
129
+ | Area | What PerturbGuard Does |
130
+ | --- | --- |
131
+ | AnnData validation | Checks loadability, matrix presence, cell/gene counts, controls, perturbation metadata, duplicate obs/var names, and malformed files. |
132
+ | Config inference | Suggests YAML schema mappings from common metadata aliases in messy public datasets. |
133
+ | Repair | Writes normalized `.h5ad` files with canonical metadata, unique indices, inferred controls, and repair action logs. |
134
+ | QC audit | Audits perturbation support, controls, cell counts, confounding, metadata shortcuts, target effects, guide consistency, and target mapping. |
135
+ | Split generation | Creates random, leave-target-gene-out, leave-perturbation-out, metadata holdout, strict combination, and seen-component combination splits. |
136
+ | Claim checking | Verifies whether a split supports claims like unseen perturbation, unseen target gene, or unseen combinations. |
137
+ | Leakage detection | Detects train/val/test leakage, combination leakage, unsupported claims, split imbalance, and invalid split labels. |
138
+ | Model evaluation | Audits prediction CSVs for overall metrics, per-group performance, and confidence calibration. |
139
+ | Adversarial checks | Tests whether metadata-only shortcuts can predict perturbation identity. |
140
+ | Target mapping | Classifies targets as measured genes, drug target classes, pathway/class annotations, missing, or unmapped. |
141
+ | Design planning | Checks planned cells, controls, replicate support, and batch support before running an experiment. |
142
+ | Benchmark manifests | Validates dataset/split/claim/model/metrics manifests and runs claim-support checks. |
143
+ | Large files | Profiles `.h5ad` files in backed mode before expensive audits. |
144
+ | Dataset cards | Generates Markdown dataset cards with audit counts, uses, and limitations. |
145
+ | Reports | Writes interactive HTML reports, plot links, CSV tables, `summary.json`, and recommendations. |
146
+
147
+ ## CLI Commands
148
+
149
+ ```text
150
+ perturbguard simulate
151
+ perturbguard validate
152
+ perturbguard audit
153
+ perturbguard split
154
+ perturbguard claim
155
+ perturbguard evaluate
156
+ perturbguard repair
157
+ perturbguard infer-config
158
+ perturbguard target-map
159
+ perturbguard compare-datasets
160
+ perturbguard design-check
161
+ perturbguard power-check
162
+ perturbguard benchmark-check
163
+ perturbguard profile-large
164
+ perturbguard adversarial-check
165
+ perturbguard dataset-card
166
+ ```
167
+
168
+ ## Input Formats
169
+
170
+ ### AnnData
171
+
172
+ Required for full audits:
173
+
174
+ - `perturbation`
175
+ - `is_control`, or enough configured control labels to infer controls
176
+
177
+ Recommended:
178
+
179
+ - `target_gene`
180
+ - `guide_id`
181
+ - `perturbation_type`
182
+ - `batch`
183
+ - `replicate`
184
+ - `cell_type`
185
+ - `donor`
186
+ - `plate`
187
+ - `timepoint`
188
+ - `dose`
189
+
190
+ ### Prediction CSV
191
+
192
+ Required for `perturbguard evaluate`:
193
+
194
+ - `cell_id`
195
+ - `y_true`
196
+ - `y_pred`
197
+
198
+ Optional:
199
+
200
+ - `confidence`
201
+
202
+ ### Target Mapping CSV
203
+
204
+ Recommended for `perturbguard target-map`:
205
+
206
+ - `perturbation`
207
+ - `target`
208
+ - optional `target_type`
209
+ - optional `source`
210
+
211
+ ### Benchmark Manifest
212
+
213
+ ```yaml
214
+ dataset: data/repaired.h5ad
215
+ split: results/split/split.csv
216
+ claim: unseen_perturbation
217
+ model:
218
+ name: my-model
219
+ metrics:
220
+ - accuracy
221
+ - macro_f1
222
+ ```
223
+
224
+ ## Real Data Smoke Test
225
+
226
+ PerturbGuard has been smoke-tested locally on public example AnnData files for SciPlex, Jorge, and LINCS-style perturbation datasets.
227
+
228
+ Example SciPlex run:
229
+
230
+ ```powershell
231
+ Invoke-WebRequest `
232
+ -Uri "https://huggingface.co/cyclopeta/PerturbNet_reproduce/resolve/main/example_data/sciplex_example.h5ad" `
233
+ -OutFile data/real/sciplex_example.h5ad
234
+
235
+ perturbguard audit `
236
+ --data data/real/sciplex_example.h5ad `
237
+ --config configs/sciplex_example.yaml `
238
+ --out results/real/sciplex_audit
239
+ ```
240
+
241
+ Full smoke matrix:
242
+
243
+ ```bash
244
+ python scripts/run_smoke_matrix.py --include-real
245
+ ```
246
+
247
+ ## What It Is Not
248
+
249
+ PerturbGuard is not a perturbation prediction model and does not claim biological truth for unmeasured perturbations. It is a guardrail system: it tells you when your dataset, split, controls, metadata, or benchmark claim may not support the conclusion you want to draw.
250
+
251
+ ## Release
252
+
253
+ Current stable release: `1.0.0`
254
+
255
+ Built and verified with:
256
+
257
+ - Python 3.11+
258
+ - AnnData
259
+ - pandas / NumPy / SciPy
260
+ - scikit-learn
261
+ - Plotly
262
+ - Typer
263
+
@@ -0,0 +1,224 @@
1
+ # PerturbGuard
2
+
3
+ [![CI](https://github.com/prithvirajanR/perturbguard/actions/workflows/ci.yml/badge.svg)](https://github.com/prithvirajanR/perturbguard/actions/workflows/ci.yml)
4
+ [![Python](https://img.shields.io/badge/python-3.11%2B-blue)](https://www.python.org/)
5
+ [![Release](https://img.shields.io/badge/release-1.0.0-green)](https://github.com/prithvirajanR/perturbguard/releases/tag/v1.0.0)
6
+ [![License](https://img.shields.io/badge/license-MIT-lightgrey)](LICENSE)
7
+
8
+ **FastQC-style guardrails for single-cell perturbation datasets, splits, claims, and model benchmarks.**
9
+
10
+ PerturbGuard helps you answer the uncomfortable but necessary question:
11
+
12
+ > Is this perturbation benchmark actually valid, or did leakage, controls, confounding, weak target effects, or split design make it look better than it is?
13
+
14
+ It works with AnnData `.h5ad` files and produces interactive HTML reports, machine-readable CSV tables, JSON summaries, and Markdown dataset cards.
15
+
16
+ ## Why Use It
17
+
18
+ Perturbation models are easy to overclaim. A model can appear to generalize because the split leaked perturbations, a batch variable predicts the label, controls are missing, target effects failed, or a drug target was treated as a single gene when it is really a pathway/class.
19
+
20
+ PerturbGuard audits those failure modes before you trust a dataset, split, claim, or benchmark.
21
+
22
+ ## Install
23
+
24
+ From GitHub:
25
+
26
+ ```bash
27
+ pip install "perturbguard @ git+https://github.com/prithvirajanR/perturbguard.git@v1.0.0"
28
+ ```
29
+
30
+ For local development:
31
+
32
+ ```bash
33
+ git clone https://github.com/prithvirajanR/perturbguard.git
34
+ cd perturbguard
35
+ pip install -e ".[dev]"
36
+ pytest -q
37
+ ```
38
+
39
+ ## Quickstart
40
+
41
+ ```bash
42
+ perturbguard simulate --scenario batch_confounded --out data/demo/batch_confounded.h5ad
43
+ perturbguard audit --data data/demo/batch_confounded.h5ad --out results/demo_audit
44
+ ```
45
+
46
+ Open:
47
+
48
+ ```text
49
+ results/demo_audit/report.html
50
+ ```
51
+
52
+ You get a searchable report with status filters, summary cards, linked plots, recommendations, and CSV tables under `results/demo_audit/tables/`.
53
+
54
+ ## Common Workflows
55
+
56
+ ### Audit a raw public dataset
57
+
58
+ ```bash
59
+ perturbguard infer-config --data data/raw.h5ad --out configs/inferred.yaml
60
+ perturbguard repair --data data/raw.h5ad --config configs/inferred.yaml --out data/repaired.h5ad
61
+ perturbguard audit --data data/repaired.h5ad --config configs/inferred.yaml --out results/audit
62
+ ```
63
+
64
+ ### Generate and validate a split
65
+
66
+ ```bash
67
+ perturbguard split \
68
+ --data data/repaired.h5ad \
69
+ --strategy leave-perturbation-out \
70
+ --out results/split
71
+
72
+ perturbguard claim \
73
+ --data data/repaired.h5ad \
74
+ --split results/split/split.csv \
75
+ --claim unseen_perturbation \
76
+ --out results/claim
77
+ ```
78
+
79
+ ### Evaluate model predictions
80
+
81
+ ```bash
82
+ perturbguard evaluate \
83
+ --data data/repaired.h5ad \
84
+ --predictions predictions.csv \
85
+ --out results/evaluation
86
+ ```
87
+
88
+ ## Feature Map
89
+
90
+ | Area | What PerturbGuard Does |
91
+ | --- | --- |
92
+ | AnnData validation | Checks loadability, matrix presence, cell/gene counts, controls, perturbation metadata, duplicate obs/var names, and malformed files. |
93
+ | Config inference | Suggests YAML schema mappings from common metadata aliases in messy public datasets. |
94
+ | Repair | Writes normalized `.h5ad` files with canonical metadata, unique indices, inferred controls, and repair action logs. |
95
+ | QC audit | Audits perturbation support, controls, cell counts, confounding, metadata shortcuts, target effects, guide consistency, and target mapping. |
96
+ | Split generation | Creates random, leave-target-gene-out, leave-perturbation-out, metadata holdout, strict combination, and seen-component combination splits. |
97
+ | Claim checking | Verifies whether a split supports claims like unseen perturbation, unseen target gene, or unseen combinations. |
98
+ | Leakage detection | Detects train/val/test leakage, combination leakage, unsupported claims, split imbalance, and invalid split labels. |
99
+ | Model evaluation | Audits prediction CSVs for overall metrics, per-group performance, and confidence calibration. |
100
+ | Adversarial checks | Tests whether metadata-only shortcuts can predict perturbation identity. |
101
+ | Target mapping | Classifies targets as measured genes, drug target classes, pathway/class annotations, missing, or unmapped. |
102
+ | Design planning | Checks planned cells, controls, replicate support, and batch support before running an experiment. |
103
+ | Benchmark manifests | Validates dataset/split/claim/model/metrics manifests and runs claim-support checks. |
104
+ | Large files | Profiles `.h5ad` files in backed mode before expensive audits. |
105
+ | Dataset cards | Generates Markdown dataset cards with audit counts, uses, and limitations. |
106
+ | Reports | Writes interactive HTML reports, plot links, CSV tables, `summary.json`, and recommendations. |
107
+
108
+ ## CLI Commands
109
+
110
+ ```text
111
+ perturbguard simulate
112
+ perturbguard validate
113
+ perturbguard audit
114
+ perturbguard split
115
+ perturbguard claim
116
+ perturbguard evaluate
117
+ perturbguard repair
118
+ perturbguard infer-config
119
+ perturbguard target-map
120
+ perturbguard compare-datasets
121
+ perturbguard design-check
122
+ perturbguard power-check
123
+ perturbguard benchmark-check
124
+ perturbguard profile-large
125
+ perturbguard adversarial-check
126
+ perturbguard dataset-card
127
+ ```
128
+
129
+ ## Input Formats
130
+
131
+ ### AnnData
132
+
133
+ Required for full audits:
134
+
135
+ - `perturbation`
136
+ - `is_control`, or enough configured control labels to infer controls
137
+
138
+ Recommended:
139
+
140
+ - `target_gene`
141
+ - `guide_id`
142
+ - `perturbation_type`
143
+ - `batch`
144
+ - `replicate`
145
+ - `cell_type`
146
+ - `donor`
147
+ - `plate`
148
+ - `timepoint`
149
+ - `dose`
150
+
151
+ ### Prediction CSV
152
+
153
+ Required for `perturbguard evaluate`:
154
+
155
+ - `cell_id`
156
+ - `y_true`
157
+ - `y_pred`
158
+
159
+ Optional:
160
+
161
+ - `confidence`
162
+
163
+ ### Target Mapping CSV
164
+
165
+ Recommended for `perturbguard target-map`:
166
+
167
+ - `perturbation`
168
+ - `target`
169
+ - optional `target_type`
170
+ - optional `source`
171
+
172
+ ### Benchmark Manifest
173
+
174
+ ```yaml
175
+ dataset: data/repaired.h5ad
176
+ split: results/split/split.csv
177
+ claim: unseen_perturbation
178
+ model:
179
+ name: my-model
180
+ metrics:
181
+ - accuracy
182
+ - macro_f1
183
+ ```
184
+
185
+ ## Real Data Smoke Test
186
+
187
+ PerturbGuard has been smoke-tested locally on public example AnnData files for SciPlex, Jorge, and LINCS-style perturbation datasets.
188
+
189
+ Example SciPlex run:
190
+
191
+ ```powershell
192
+ Invoke-WebRequest `
193
+ -Uri "https://huggingface.co/cyclopeta/PerturbNet_reproduce/resolve/main/example_data/sciplex_example.h5ad" `
194
+ -OutFile data/real/sciplex_example.h5ad
195
+
196
+ perturbguard audit `
197
+ --data data/real/sciplex_example.h5ad `
198
+ --config configs/sciplex_example.yaml `
199
+ --out results/real/sciplex_audit
200
+ ```
201
+
202
+ Full smoke matrix:
203
+
204
+ ```bash
205
+ python scripts/run_smoke_matrix.py --include-real
206
+ ```
207
+
208
+ ## What It Is Not
209
+
210
+ PerturbGuard is not a perturbation prediction model and does not claim biological truth for unmeasured perturbations. It is a guardrail system: it tells you when your dataset, split, controls, metadata, or benchmark claim may not support the conclusion you want to draw.
211
+
212
+ ## Release
213
+
214
+ Current stable release: `1.0.0`
215
+
216
+ Built and verified with:
217
+
218
+ - Python 3.11+
219
+ - AnnData
220
+ - pandas / NumPy / SciPy
221
+ - scikit-learn
222
+ - Plotly
223
+ - Typer
224
+
@@ -0,0 +1,19 @@
1
+ schema:
2
+ perturbation: perturbation
3
+ is_control: is_control
4
+ target_gene: target_gene
5
+ guide_id: guide_id
6
+ perturbation_type: perturbation_type
7
+ cell_type: cell_type
8
+ batch: batch
9
+ replicate: replicate
10
+ controls:
11
+ labels: ["control", "ctrl", "NTC", "non-targeting", "DMSO", "vehicle"]
12
+ min_controls_per_group: 30
13
+ cell_count:
14
+ min_cells_warning: 50
15
+ min_cells_fail: 20
16
+ confounding:
17
+ metadata_columns: ["batch", "replicate", "donor", "plate", "timepoint"]
18
+ cramers_v_warning: 0.3
19
+ cramers_v_fail: 0.5
@@ -0,0 +1,10 @@
1
+ schema:
2
+ perturbation: mutation_name
3
+ is_control: UTR_cond
4
+ target_gene: mutation_name
5
+ guide_id: mutation_poi
6
+ perturbation_type: UTR_cond
7
+ batch: mutation_poi
8
+
9
+ controls:
10
+ labels: ["WT", "not mutated"]
@@ -0,0 +1,11 @@
1
+ schema:
2
+ perturbation: pert_iname
3
+ target_gene: pert_id
4
+ perturbation_type: pert_type
5
+ cell_type: cell_id
6
+ batch: batch_ident
7
+ timepoint: pert_time
8
+ dose: fake_dose
9
+
10
+ controls:
11
+ labels: ["DMSO", "vehicle", "control", "ctrl"]
@@ -0,0 +1,15 @@
1
+ schema:
2
+ perturbation: product_name_clean
3
+ is_control: vehicle
4
+ target_gene: target
5
+ perturbation_type: perturb_string_processed
6
+ cell_type: cell_type
7
+ batch: culture_plate
8
+ replicate: replicate
9
+ timepoint: time_point
10
+ dose: dose
11
+ n_counts: n_counts
12
+ pct_mito: percent_mito
13
+
14
+ controls:
15
+ labels: ["DMSO", "vehicle", "control", "ctrl"]
@@ -0,0 +1,16 @@
1
+ # Advanced Features
2
+
3
+ PerturbGuard now includes ten higher-level guardrails beyond the original audit core.
4
+
5
+ 1. `evaluate`: audits model predictions overall, by metadata group, and by optional confidence calibration.
6
+ 2. `repair`: writes a normalized AnnData file with canonical metadata, unique indices, and inferred controls where possible.
7
+ 3. Interactive HTML reports: reports include search, status filters, summary cards, and plot links.
8
+ 4. `target-map`: classifies target annotations as measured genes, drug classes, pathways/classes, missing, or unmapped.
9
+ 5. `power-check`: checks planned cells, controls, replicate support, and batch support before an experiment.
10
+ 6. `benchmark-check`: validates a benchmark manifest and checks whether the declared claim is supported by the split.
11
+ 7. `profile-large`: opens `.h5ad` files in backed mode and reports shape/storage metadata before expensive audits.
12
+ 8. `adversarial-check`: asks whether technical metadata can predict perturbation identity.
13
+ 9. `infer-config`: suggests a YAML schema mapping from common metadata aliases.
14
+ 10. `dataset-card`: writes a Markdown summary of dataset size, controls, audit statuses, supported uses, and limitations.
15
+
16
+ These checks are guardrails. They help prevent invalid benchmark claims and brittle analyses, but they do not replace biological validation or model-specific evaluation.
@@ -0,0 +1,14 @@
1
+ # Statistical Assumptions And Thresholds
2
+
3
+ PerturbGuard reports measurable audit risks. It does not prove biological truth.
4
+
5
+ Default thresholds are intentionally conservative:
6
+
7
+ - Low perturbation support: warning below 50 cells, fail below 20 cells.
8
+ - Batch or metadata confounding: warning at Cramer's V >= 0.3, fail at >= 0.5.
9
+ - Dominant metadata concentration: warning at 70%, fail at 90%.
10
+ - Metadata-only shortcut baseline: warning at balanced accuracy >= 0.4, fail at >= 0.6.
11
+ - Guide consistency: warning at median correlation <= 0.2, fail at <= 0.0.
12
+
13
+ These are screening thresholds, not universal biological cutoffs. Users should tune them for
14
+ experiment size, modality, cell type diversity, dose design, and intended claim.
@@ -0,0 +1,15 @@
1
+ # Beta Test Report
2
+
3
+ PerturbGuard has been smoke-tested on synthetic perturbation scenarios and public example
4
+ AnnData files from the PerturbNet reproduction repository.
5
+
6
+ Real-data cases:
7
+
8
+ | Dataset | File | Purpose |
9
+ |---|---|---|
10
+ | SciPlex | `data/real/sciplex_example.h5ad` | Drug perturbation metadata, dose/time/cell-type fields |
11
+ | Jorge | `data/real/Jorge_example.h5ad` | Variant perturbation labels and dataset-specific controls |
12
+ | LINCS | `data/real/lincs_example.h5ad` | Drug perturbations, many batches/cell IDs, provided holdouts |
13
+
14
+ The real-data tests verify that schema mapping, missing-control handling, target classification,
15
+ audit table writing, HTML reports, plots, and provided split claim checks complete without crashes.