eca-pp 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eca_pp-0.4.0/PKG-INFO +281 -0
- eca_pp-0.4.0/README.md +241 -0
- eca_pp-0.4.0/pyproject.toml +40 -0
- eca_pp-0.4.0/setup.cfg +4 -0
- eca_pp-0.4.0/src/eca_pp/__init__.py +3 -0
- eca_pp-0.4.0/src/eca_pp/_harness_claude.py +196 -0
- eca_pp-0.4.0/src/eca_pp/_harness_deepseek.py +552 -0
- eca_pp-0.4.0/src/eca_pp/_harness_openai.py +390 -0
- eca_pp-0.4.0/src/eca_pp/agent.py +116 -0
- eca_pp-0.4.0/src/eca_pp/core/__init__.py +2 -0
- eca_pp-0.4.0/src/eca_pp/core/atomic_io.py +45 -0
- eca_pp-0.4.0/src/eca_pp/core/colspec.py +62 -0
- eca_pp-0.4.0/src/eca_pp/core/columns.py +49 -0
- eca_pp-0.4.0/src/eca_pp/core/result.py +71 -0
- eca_pp-0.4.0/src/eca_pp/core/run_outputs.py +42 -0
- eca_pp-0.4.0/src/eca_pp/core/values.py +28 -0
- eca_pp-0.4.0/src/eca_pp/harness.py +241 -0
- eca_pp-0.4.0/src/eca_pp/identify_columns/__init__.py +1 -0
- eca_pp-0.4.0/src/eca_pp/identify_columns/__main__.py +3 -0
- eca_pp-0.4.0/src/eca_pp/identify_columns/cli.py +846 -0
- eca_pp-0.4.0/src/eca_pp/identify_columns/obsprofile.py +212 -0
- eca_pp-0.4.0/src/eca_pp/identify_columns/policies.py +256 -0
- eca_pp-0.4.0/src/eca_pp/probe/__init__.py +1 -0
- eca_pp-0.4.0/src/eca_pp/probe/__main__.py +3 -0
- eca_pp-0.4.0/src/eca_pp/probe/cli.py +356 -0
- eca_pp-0.4.0/src/eca_pp/standardize/__init__.py +1 -0
- eca_pp-0.4.0/src/eca_pp/standardize/__main__.py +3 -0
- eca_pp-0.4.0/src/eca_pp/standardize/build.py +65 -0
- eca_pp-0.4.0/src/eca_pp/standardize/cli.py +338 -0
- eca_pp-0.4.0/src/eca_pp/standardize/countsloc.py +231 -0
- eca_pp-0.4.0/src/eca_pp/standardize/harmonize.py +107 -0
- eca_pp-0.4.0/src/eca_pp/standardize/qc.py +131 -0
- eca_pp-0.4.0/src/eca_pp/standardize/species.py +157 -0
- eca_pp-0.4.0/src/eca_pp.egg-info/PKG-INFO +281 -0
- eca_pp-0.4.0/src/eca_pp.egg-info/SOURCES.txt +49 -0
- eca_pp-0.4.0/src/eca_pp.egg-info/dependency_links.txt +1 -0
- eca_pp-0.4.0/src/eca_pp.egg-info/entry_points.txt +4 -0
- eca_pp-0.4.0/src/eca_pp.egg-info/requires.txt +38 -0
- eca_pp-0.4.0/src/eca_pp.egg-info/top_level.txt +1 -0
- eca_pp-0.4.0/tests/test_agent.py +283 -0
- eca_pp-0.4.0/tests/test_columns.py +102 -0
- eca_pp-0.4.0/tests/test_float_groups.py +23 -0
- eca_pp-0.4.0/tests/test_harness_openai.py +358 -0
- eca_pp-0.4.0/tests/test_identify_columns.py +518 -0
- eca_pp-0.4.0/tests/test_obsprofile.py +117 -0
- eca_pp-0.4.0/tests/test_pre_gate.py +26 -0
- eca_pp-0.4.0/tests/test_probe.py +141 -0
- eca_pp-0.4.0/tests/test_run_outputs.py +27 -0
- eca_pp-0.4.0/tests/test_species_logging.py +20 -0
- eca_pp-0.4.0/tests/test_standardize.py +160 -0
- eca_pp-0.4.0/tests/test_standardize_v02.py +277 -0
eca_pp-0.4.0/PKG-INFO
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: eca-pp
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Standardized single-cell preprocessing for the Ensemble Cell Atlas ecosystem
|
|
5
|
+
Project-URL: Repository, https://github.com/chansigit/eca-pp
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: numpy>=1.22
|
|
9
|
+
Requires-Dist: scipy>=1.7
|
|
10
|
+
Requires-Dist: h5py>=3.8
|
|
11
|
+
Requires-Dist: anndata>=0.8
|
|
12
|
+
Requires-Dist: stancounts>=0.3
|
|
13
|
+
Requires-Dist: stangene>=0.5
|
|
14
|
+
Provides-Extra: test
|
|
15
|
+
Requires-Dist: pytest>=7.0; extra == "test"
|
|
16
|
+
Provides-Extra: llm
|
|
17
|
+
Requires-Dist: deepseek-harness-sdk>=0.1.2a3; extra == "llm"
|
|
18
|
+
Requires-Dist: mcp>=1.27; extra == "llm"
|
|
19
|
+
Requires-Dist: PyYAML>=6; extra == "llm"
|
|
20
|
+
Requires-Dist: uvicorn>=0.30; extra == "llm"
|
|
21
|
+
Requires-Dist: anyio>=4.0; extra == "llm"
|
|
22
|
+
Provides-Extra: probe
|
|
23
|
+
Requires-Dist: scanpy>=1.10; extra == "probe"
|
|
24
|
+
Requires-Dist: harmonypy<0.3,>=0.0.9; extra == "probe"
|
|
25
|
+
Requires-Dist: scikit-learn>=1.1; extra == "probe"
|
|
26
|
+
Requires-Dist: leidenalg>=0.10; extra == "probe"
|
|
27
|
+
Provides-Extra: agent
|
|
28
|
+
Requires-Dist: openai-agents<1,>=0.22; extra == "agent"
|
|
29
|
+
Requires-Dist: deepseek-harness-sdk>=0.1.2a3; extra == "agent"
|
|
30
|
+
Requires-Dist: mcp>=1.27; extra == "agent"
|
|
31
|
+
Requires-Dist: PyYAML>=6; extra == "agent"
|
|
32
|
+
Requires-Dist: uvicorn>=0.30; extra == "agent"
|
|
33
|
+
Requires-Dist: anyio>=4.0; extra == "agent"
|
|
34
|
+
Provides-Extra: claude
|
|
35
|
+
Requires-Dist: claude-agent-sdk>=0.2.152; extra == "claude"
|
|
36
|
+
Requires-Dist: anyio>=4.0; extra == "claude"
|
|
37
|
+
Provides-Extra: openai
|
|
38
|
+
Requires-Dist: openai-agents<1,>=0.22; extra == "openai"
|
|
39
|
+
Requires-Dist: anyio>=4.0; extra == "openai"
|
|
40
|
+
|
|
41
|
+
<p align="center">
|
|
42
|
+
<img src="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-logo.svg" alt="ECA-PP logo: scattered cells organized into a consistent atlas" width="160" height="160">
|
|
43
|
+
</p>
|
|
44
|
+
|
|
45
|
+
<h1 align="center">ECA-PP: Standardized Single-Cell Preprocessing</h1>
|
|
46
|
+
|
|
47
|
+
<p align="center">
|
|
48
|
+
<strong>Give real-world single-cell data a consistent starting point.</strong>
|
|
49
|
+
</p>
|
|
50
|
+
|
|
51
|
+
<p align="center">
|
|
52
|
+
<a href="pyproject.toml"><img src="https://img.shields.io/badge/Version-0.4.0-258B81?style=flat" alt="Project version 0.4.0"></a>
|
|
53
|
+
<a href="pyproject.toml"><img src="https://img.shields.io/badge/Python-3.10%2B-3776AB?style=flat&logo=python&logoColor=white" alt="Python 3.10 or newer"></a>
|
|
54
|
+
<a href="#where-it-fits"><img src="https://img.shields.io/badge/Input-H5AD-64748B?style=flat" alt="Starts from H5AD data"></a>
|
|
55
|
+
<a href="https://github.com/chansigit/eca-rsi"><img src="https://img.shields.io/badge/Ecosystem-Ensemble_Cell_Atlas-258B81?style=flat" alt="Part of the Ensemble Cell Atlas ecosystem"></a>
|
|
56
|
+
</p>
|
|
57
|
+
|
|
58
|
+
<p align="center">
|
|
59
|
+
<a href="#why-standardize-before-analysis">🔬 Why ECA-PP?</a>
|
|
60
|
+
|
|
61
|
+
<a href="#what-you-get">What you get</a>
|
|
62
|
+
|
|
63
|
+
<a href="#try-it">🚀 Try it</a>
|
|
64
|
+
|
|
65
|
+
<a href="#faq">FAQ</a>
|
|
66
|
+
|
|
67
|
+
<a href="#further-reading">📖 Further reading</a>
|
|
68
|
+
</p>
|
|
69
|
+
|
|
70
|
+
<br>
|
|
71
|
+
|
|
72
|
+
ECA-PP prepares published single-cell RNA sequencing datasets for reuse. Starting
|
|
73
|
+
from an **H5AD file**, it finds expression counts, standardizes gene names,
|
|
74
|
+
calculates quality-control measurements, and identifies useful batch and
|
|
75
|
+
cell-type metadata. You get prepared data and a record of the evidence behind
|
|
76
|
+
each decision.
|
|
77
|
+
|
|
78
|
+
ECA-PP belongs to the **Ensemble Cell Atlas (ECA)** ecosystem. It handles routine
|
|
79
|
+
preprocessing for **[ECA-RSI](https://github.com/chansigit/eca-rsi)**
|
|
80
|
+
(Recursive Self-Improvement), ECA's automated data governance system, so that
|
|
81
|
+
system can focus on quality assessment, annotation, and iterative refinement.
|
|
82
|
+
You can also use ECA-PP independently in your own pipeline.
|
|
83
|
+
|
|
84
|
+
<br>
|
|
85
|
+
|
|
86
|
+
## Why standardize before analysis?
|
|
87
|
+
|
|
88
|
+
Public single-cell data reflects the choices of many different authors.
|
|
89
|
+
Expression matrices arrive as text tables, sparse matrix files, Seurat objects,
|
|
90
|
+
or H5AD files, often with metadata in separate supplements. Converting them into
|
|
91
|
+
one format solves only part of the problem.
|
|
92
|
+
|
|
93
|
+
Inside an H5AD file, expression values may be raw, normalized, or scaled. Gene
|
|
94
|
+
names may mix old symbols and Ensembl IDs. Sample and cell-type columns may use
|
|
95
|
+
unfamiliar names, duplicate one another, or contain missing values. Even existing
|
|
96
|
+
QC measurements may have been calculated using different gene sets.
|
|
97
|
+
|
|
98
|
+
ECA-PP gives these recurring problems a shared treatment:
|
|
99
|
+
|
|
100
|
+
- **Consistent preprocessing across studies.** Counts checks, gene mapping, and
|
|
101
|
+
QC use a common implementation. QC and normalized expression are calculated
|
|
102
|
+
on the same final gene set.
|
|
103
|
+
|
|
104
|
+
- **Metadata choices supported by data.** Small integration trials test batch
|
|
105
|
+
candidates for improved mixing and preservation of cell-type structure.
|
|
106
|
+
Technical and donor factors take priority over biological conditions.
|
|
107
|
+
|
|
108
|
+
- **Automation you can inspect.** Model suggestions pass programmatic checks;
|
|
109
|
+
built-in rules keep column identification moving when a model is unavailable.
|
|
110
|
+
Decisions, changes, and unresolved questions are recorded for review.
|
|
111
|
+
|
|
112
|
+
<br>
|
|
113
|
+
|
|
114
|
+
## What you get
|
|
115
|
+
|
|
116
|
+
- **A standardized dataset.** An H5AD with counts, normalized expression,
|
|
117
|
+
standardized gene names, QC measurements, and preserved author metadata.
|
|
118
|
+
|
|
119
|
+
- **Guidance for downstream analysis.** Proposed batch and existing cell-type
|
|
120
|
+
columns, evidence for each selection, and an assessment of whether batch
|
|
121
|
+
correction is needed.
|
|
122
|
+
|
|
123
|
+
- **A record of what happened.** Each step writes a `result.json` describing its
|
|
124
|
+
outcome, changes, and issues that need attention. Your source file stays unchanged.
|
|
125
|
+
|
|
126
|
+
<br>
|
|
127
|
+
|
|
128
|
+
## Where it fits
|
|
129
|
+
|
|
130
|
+
General-purpose agents can usually download files, unpack archives, and script
|
|
131
|
+
conversions into `.h5ad`, the AnnData format used by Scanpy. **ECA-PP starts at
|
|
132
|
+
H5AD**, where decisions about counts, gene identity, QC, and batch structure need
|
|
133
|
+
domain-specific standards applied consistently across studies.
|
|
134
|
+
|
|
135
|
+
| Stage | Responsibility |
|
|
136
|
+
| --- | --- |
|
|
137
|
+
| Upstream tools or agents | Gather published files and convert them into H5AD. |
|
|
138
|
+
| ECA-PP | Standardize the data and evaluate metadata using shared rules and recorded evidence. |
|
|
139
|
+
| ECA-RSI | Coordinate subsequent quality review, annotation, and iterative refinement. |
|
|
140
|
+
|
|
141
|
+
ECA-PP currently prepares data and evaluates metadata. It does not assign new
|
|
142
|
+
biological cell-type labels, filter individual low-quality cells or doublets,
|
|
143
|
+
or produce a final integrated atlas.
|
|
144
|
+
|
|
145
|
+
Follow the main path below; the lower branches show required review, dataset
|
|
146
|
+
rejection, and the trial loop used to evaluate metadata. Click the diagram for
|
|
147
|
+
an interactive version with search, zoom, and guided views.
|
|
148
|
+
|
|
149
|
+
<p align="center">
|
|
150
|
+
<a href="https://raw.githack.com/chansigit/eca-pp/main/docs/workflow.html">
|
|
151
|
+
<picture>
|
|
152
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-workflow-dark.svg">
|
|
153
|
+
<img src="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-workflow-light.svg" alt="ECA-PP workflow: input H5AD, standardization, prepared data, metadata identification with integration trials, recorded decisions, and handoff to ECA-RSI. Unresolved standardization choices need review; unsuitable inputs are rejected." width="1100">
|
|
154
|
+
</picture>
|
|
155
|
+
</a>
|
|
156
|
+
</p>
|
|
157
|
+
|
|
158
|
+
<br>
|
|
159
|
+
|
|
160
|
+
## Try it
|
|
161
|
+
|
|
162
|
+
### 1. Install
|
|
163
|
+
|
|
164
|
+
Use Python 3.10 or newer, preferably in a dedicated environment. ECA-PP runs on CPUs.
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
pip install "eca-pp[probe,openai]==0.4.0"
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
This also installs `stancounts` and `stangene`, the counts-recovery and gene-mapping
|
|
171
|
+
dependencies. To work on ECA-PP itself, clone this repository and use
|
|
172
|
+
`pip install ".[probe,openai]"` instead.
|
|
173
|
+
|
|
174
|
+
### 2. Standardize your dataset
|
|
175
|
+
|
|
176
|
+
Replace `your-data.h5ad` with your input file:
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
eca-pp-standardize your-data.h5ad -o results/standardize
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Open **`results/standardize/result.json`** to check the outcome. A successful run
|
|
183
|
+
produces `standardized.h5ad`. If a required choice is unresolved, such as the
|
|
184
|
+
species, the report explains what needs clarification before you continue.
|
|
185
|
+
|
|
186
|
+
<details>
|
|
187
|
+
<summary>Dataset size checks and gene filtering</summary>
|
|
188
|
+
|
|
189
|
+
The default checks require at least **100 cells and 5,000 detected genes across
|
|
190
|
+
the whole dataset**. A gene is detected if it has a nonzero count in at least one
|
|
191
|
+
cell. **This is not a requirement for each cell to express 5,000 genes.**
|
|
192
|
+
|
|
193
|
+
By default, features that cannot be mapped to a canonical gene are removed from
|
|
194
|
+
the output. Use `--keep-unmapped` to retain them. The dataset-level gene threshold
|
|
195
|
+
is checked again after gene mapping and filtering.
|
|
196
|
+
|
|
197
|
+
See the [tutorial's options](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md#4-常用参数) for adjusting size checks
|
|
198
|
+
or specifying the species and counts layer.
|
|
199
|
+
|
|
200
|
+
</details>
|
|
201
|
+
|
|
202
|
+
### 3. Identify batch and cell-type columns
|
|
203
|
+
|
|
204
|
+
After successful standardization, run:
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
eca-pp-identify-columns results/standardize/standardized.h5ad \
|
|
208
|
+
-o results/columns
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
For AI-assisted decisions, set `ARK_API_KEY` in your environment before running
|
|
212
|
+
this command. The default uses Doubao Turbo through the OpenAI Agents SDK.
|
|
213
|
+
Without model credentials, ECA-PP uses built-in rules and integration trials.
|
|
214
|
+
|
|
215
|
+
Read **`results/columns/result.json`** for the selected columns and supporting
|
|
216
|
+
evidence. For example, ECA-PP may identify a sequencing channel as the batch but
|
|
217
|
+
conclude that the cells are already sufficiently mixed and correction is unnecessary.
|
|
218
|
+
|
|
219
|
+
<details>
|
|
220
|
+
<summary>Find the output files</summary>
|
|
221
|
+
|
|
222
|
+
| File | Contents |
|
|
223
|
+
| --- | --- |
|
|
224
|
+
| `results/standardize/standardized.h5ad` | Prepared expression data, gene identifiers, and QC measurements. |
|
|
225
|
+
| `results/standardize/result.json` | Input checks, counts source, species, gene changes, and review notes. |
|
|
226
|
+
| `results/columns/result.json` | Selected metadata, correction assessment, decisions, and trial results. |
|
|
227
|
+
| `results/columns/batch.tsv` | Batch labels when the selected grouping is derived from barcodes or multiple columns; created only when needed. |
|
|
228
|
+
|
|
229
|
+
</details>
|
|
230
|
+
|
|
231
|
+
<br>
|
|
232
|
+
|
|
233
|
+
## FAQ
|
|
234
|
+
|
|
235
|
+
### Does ECA-PP change my original data?
|
|
236
|
+
|
|
237
|
+
The source file stays unchanged. ECA-PP writes a separate dataset, preserves
|
|
238
|
+
author metadata, and backs up fields it replaces. Gene mapping and filtering
|
|
239
|
+
changes are recorded. Standardization may reject a whole dataset that fails its
|
|
240
|
+
checks, but does not remove individual cells from an accepted dataset.
|
|
241
|
+
|
|
242
|
+
### What does an empty batch or cell-type result mean?
|
|
243
|
+
|
|
244
|
+
`null` means no suitable column was selected. This can be a valid outcome when
|
|
245
|
+
evidence is insufficient. A selected batch with `correction: "unnecessary"`
|
|
246
|
+
means the evidence did not support correcting it. Neither conclusion should be
|
|
247
|
+
confused with an execution error; check the outcome and reasons in `result.json`.
|
|
248
|
+
|
|
249
|
+
### Do I need an AI model?
|
|
250
|
+
|
|
251
|
+
Standardization runs locally by default; optional AI assistance is available for
|
|
252
|
+
unresolved species inference. Column identification uses a model when configured
|
|
253
|
+
and falls back to built-in rules when it is unavailable. See the
|
|
254
|
+
[model configuration guide](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md#8-识别批次列--细胞类型列identify-columns)
|
|
255
|
+
for backend and model choices.
|
|
256
|
+
|
|
257
|
+
### Can I recover counts from normalized data?
|
|
258
|
+
|
|
259
|
+
ECA-PP uses [stancounts](https://github.com/chansigit/stancounts) to recover counts
|
|
260
|
+
from supported transformed inputs when possible. Recovery depends on the data;
|
|
261
|
+
unsupported or ambiguous cases are reported rather than silently treated as raw counts.
|
|
262
|
+
|
|
263
|
+
<br>
|
|
264
|
+
|
|
265
|
+
## Further reading
|
|
266
|
+
|
|
267
|
+
**For users:** the [hands-on tutorial](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md) walks through a real
|
|
268
|
+
mouse dataset, result interpretation, common options, model setup, and reruns.
|
|
269
|
+
The tutorial is currently in Chinese. Each command also provides `--help`.
|
|
270
|
+
|
|
271
|
+
**For developers:** the [standardization specification](https://github.com/chansigit/eca-pp/blob/main/docs/standardize-spec.md)
|
|
272
|
+
and [column-identification specification](https://github.com/chansigit/eca-pp/blob/main/docs/identify-columns-spec.md) describe
|
|
273
|
+
methods, interfaces, and tests. Download the
|
|
274
|
+
[interactive architecture diagram](https://github.com/chansigit/eca-pp/blob/main/docs/architecture.html) to open it in a browser.
|
|
275
|
+
|
|
276
|
+
**Continue in the ECA ecosystem:** [ECA-RSI](https://github.com/chansigit/eca-rsi)
|
|
277
|
+
coordinates downstream analysis, including sample-level QC and annotation with
|
|
278
|
+
[OSP](https://github.com/chansigit/osp).
|
|
279
|
+
|
|
280
|
+
For questions, unexpected results, or feature requests,
|
|
281
|
+
[open an issue](https://github.com/chansigit/eca-pp/issues).
|
eca_pp-0.4.0/README.md
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-logo.svg" alt="ECA-PP logo: scattered cells organized into a consistent atlas" width="160" height="160">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">ECA-PP: Standardized Single-Cell Preprocessing</h1>
|
|
6
|
+
|
|
7
|
+
<p align="center">
|
|
8
|
+
<strong>Give real-world single-cell data a consistent starting point.</strong>
|
|
9
|
+
</p>
|
|
10
|
+
|
|
11
|
+
<p align="center">
|
|
12
|
+
<a href="pyproject.toml"><img src="https://img.shields.io/badge/Version-0.4.0-258B81?style=flat" alt="Project version 0.4.0"></a>
|
|
13
|
+
<a href="pyproject.toml"><img src="https://img.shields.io/badge/Python-3.10%2B-3776AB?style=flat&logo=python&logoColor=white" alt="Python 3.10 or newer"></a>
|
|
14
|
+
<a href="#where-it-fits"><img src="https://img.shields.io/badge/Input-H5AD-64748B?style=flat" alt="Starts from H5AD data"></a>
|
|
15
|
+
<a href="https://github.com/chansigit/eca-rsi"><img src="https://img.shields.io/badge/Ecosystem-Ensemble_Cell_Atlas-258B81?style=flat" alt="Part of the Ensemble Cell Atlas ecosystem"></a>
|
|
16
|
+
</p>
|
|
17
|
+
|
|
18
|
+
<p align="center">
|
|
19
|
+
<a href="#why-standardize-before-analysis">🔬 Why ECA-PP?</a>
|
|
20
|
+
|
|
21
|
+
<a href="#what-you-get">What you get</a>
|
|
22
|
+
|
|
23
|
+
<a href="#try-it">🚀 Try it</a>
|
|
24
|
+
|
|
25
|
+
<a href="#faq">FAQ</a>
|
|
26
|
+
|
|
27
|
+
<a href="#further-reading">📖 Further reading</a>
|
|
28
|
+
</p>
|
|
29
|
+
|
|
30
|
+
<br>
|
|
31
|
+
|
|
32
|
+
ECA-PP prepares published single-cell RNA sequencing datasets for reuse. Starting
|
|
33
|
+
from an **H5AD file**, it finds expression counts, standardizes gene names,
|
|
34
|
+
calculates quality-control measurements, and identifies useful batch and
|
|
35
|
+
cell-type metadata. You get prepared data and a record of the evidence behind
|
|
36
|
+
each decision.
|
|
37
|
+
|
|
38
|
+
ECA-PP belongs to the **Ensemble Cell Atlas (ECA)** ecosystem. It handles routine
|
|
39
|
+
preprocessing for **[ECA-RSI](https://github.com/chansigit/eca-rsi)**
|
|
40
|
+
(Recursive Self-Improvement), ECA's automated data governance system, so that
|
|
41
|
+
system can focus on quality assessment, annotation, and iterative refinement.
|
|
42
|
+
You can also use ECA-PP independently in your own pipeline.
|
|
43
|
+
|
|
44
|
+
<br>
|
|
45
|
+
|
|
46
|
+
## Why standardize before analysis?
|
|
47
|
+
|
|
48
|
+
Public single-cell data reflects the choices of many different authors.
|
|
49
|
+
Expression matrices arrive as text tables, sparse matrix files, Seurat objects,
|
|
50
|
+
or H5AD files, often with metadata in separate supplements. Converting them into
|
|
51
|
+
one format solves only part of the problem.
|
|
52
|
+
|
|
53
|
+
Inside an H5AD file, expression values may be raw, normalized, or scaled. Gene
|
|
54
|
+
names may mix old symbols and Ensembl IDs. Sample and cell-type columns may use
|
|
55
|
+
unfamiliar names, duplicate one another, or contain missing values. Even existing
|
|
56
|
+
QC measurements may have been calculated using different gene sets.
|
|
57
|
+
|
|
58
|
+
ECA-PP gives these recurring problems a shared treatment:
|
|
59
|
+
|
|
60
|
+
- **Consistent preprocessing across studies.** Counts checks, gene mapping, and
|
|
61
|
+
QC use a common implementation. QC and normalized expression are calculated
|
|
62
|
+
on the same final gene set.
|
|
63
|
+
|
|
64
|
+
- **Metadata choices supported by data.** Small integration trials test batch
|
|
65
|
+
candidates for improved mixing and preservation of cell-type structure.
|
|
66
|
+
Technical and donor factors take priority over biological conditions.
|
|
67
|
+
|
|
68
|
+
- **Automation you can inspect.** Model suggestions pass programmatic checks;
|
|
69
|
+
built-in rules keep column identification moving when a model is unavailable.
|
|
70
|
+
Decisions, changes, and unresolved questions are recorded for review.
|
|
71
|
+
|
|
72
|
+
<br>
|
|
73
|
+
|
|
74
|
+
## What you get
|
|
75
|
+
|
|
76
|
+
- **A standardized dataset.** An H5AD with counts, normalized expression,
|
|
77
|
+
standardized gene names, QC measurements, and preserved author metadata.
|
|
78
|
+
|
|
79
|
+
- **Guidance for downstream analysis.** Proposed batch and existing cell-type
|
|
80
|
+
columns, evidence for each selection, and an assessment of whether batch
|
|
81
|
+
correction is needed.
|
|
82
|
+
|
|
83
|
+
- **A record of what happened.** Each step writes a `result.json` describing its
|
|
84
|
+
outcome, changes, and issues that need attention. Your source file stays unchanged.
|
|
85
|
+
|
|
86
|
+
<br>
|
|
87
|
+
|
|
88
|
+
## Where it fits
|
|
89
|
+
|
|
90
|
+
General-purpose agents can usually download files, unpack archives, and script
|
|
91
|
+
conversions into `.h5ad`, the AnnData format used by Scanpy. **ECA-PP starts at
|
|
92
|
+
H5AD**, where decisions about counts, gene identity, QC, and batch structure need
|
|
93
|
+
domain-specific standards applied consistently across studies.
|
|
94
|
+
|
|
95
|
+
| Stage | Responsibility |
|
|
96
|
+
| --- | --- |
|
|
97
|
+
| Upstream tools or agents | Gather published files and convert them into H5AD. |
|
|
98
|
+
| ECA-PP | Standardize the data and evaluate metadata using shared rules and recorded evidence. |
|
|
99
|
+
| ECA-RSI | Coordinate subsequent quality review, annotation, and iterative refinement. |
|
|
100
|
+
|
|
101
|
+
ECA-PP currently prepares data and evaluates metadata. It does not assign new
|
|
102
|
+
biological cell-type labels, filter individual low-quality cells or doublets,
|
|
103
|
+
or produce a final integrated atlas.
|
|
104
|
+
|
|
105
|
+
Follow the main path below; the lower branches show required review, dataset
|
|
106
|
+
rejection, and the trial loop used to evaluate metadata. Click the diagram for
|
|
107
|
+
an interactive version with search, zoom, and guided views.
|
|
108
|
+
|
|
109
|
+
<p align="center">
|
|
110
|
+
<a href="https://raw.githack.com/chansigit/eca-pp/main/docs/workflow.html">
|
|
111
|
+
<picture>
|
|
112
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-workflow-dark.svg">
|
|
113
|
+
<img src="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-workflow-light.svg" alt="ECA-PP workflow: input H5AD, standardization, prepared data, metadata identification with integration trials, recorded decisions, and handoff to ECA-RSI. Unresolved standardization choices need review; unsuitable inputs are rejected." width="1100">
|
|
114
|
+
</picture>
|
|
115
|
+
</a>
|
|
116
|
+
</p>
|
|
117
|
+
|
|
118
|
+
<br>
|
|
119
|
+
|
|
120
|
+
## Try it
|
|
121
|
+
|
|
122
|
+
### 1. Install
|
|
123
|
+
|
|
124
|
+
Use Python 3.10 or newer, preferably in a dedicated environment. ECA-PP runs on CPUs.
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
pip install "eca-pp[probe,openai]==0.4.0"
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
This also installs `stancounts` and `stangene`, the counts-recovery and gene-mapping
|
|
131
|
+
dependencies. To work on ECA-PP itself, clone this repository and use
|
|
132
|
+
`pip install ".[probe,openai]"` instead.
|
|
133
|
+
|
|
134
|
+
### 2. Standardize your dataset
|
|
135
|
+
|
|
136
|
+
Replace `your-data.h5ad` with your input file:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
eca-pp-standardize your-data.h5ad -o results/standardize
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Open **`results/standardize/result.json`** to check the outcome. A successful run
|
|
143
|
+
produces `standardized.h5ad`. If a required choice is unresolved, such as the
|
|
144
|
+
species, the report explains what needs clarification before you continue.
|
|
145
|
+
|
|
146
|
+
<details>
|
|
147
|
+
<summary>Dataset size checks and gene filtering</summary>
|
|
148
|
+
|
|
149
|
+
The default checks require at least **100 cells and 5,000 detected genes across
|
|
150
|
+
the whole dataset**. A gene is detected if it has a nonzero count in at least one
|
|
151
|
+
cell. **This is not a requirement for each cell to express 5,000 genes.**
|
|
152
|
+
|
|
153
|
+
By default, features that cannot be mapped to a canonical gene are removed from
|
|
154
|
+
the output. Use `--keep-unmapped` to retain them. The dataset-level gene threshold
|
|
155
|
+
is checked again after gene mapping and filtering.
|
|
156
|
+
|
|
157
|
+
See the [tutorial's options](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md#4-常用参数) for adjusting size checks
|
|
158
|
+
or specifying the species and counts layer.
|
|
159
|
+
|
|
160
|
+
</details>
|
|
161
|
+
|
|
162
|
+
### 3. Identify batch and cell-type columns
|
|
163
|
+
|
|
164
|
+
After successful standardization, run:
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
eca-pp-identify-columns results/standardize/standardized.h5ad \
|
|
168
|
+
-o results/columns
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
For AI-assisted decisions, set `ARK_API_KEY` in your environment before running
|
|
172
|
+
this command. The default uses Doubao Turbo through the OpenAI Agents SDK.
|
|
173
|
+
Without model credentials, ECA-PP uses built-in rules and integration trials.
|
|
174
|
+
|
|
175
|
+
Read **`results/columns/result.json`** for the selected columns and supporting
|
|
176
|
+
evidence. For example, ECA-PP may identify a sequencing channel as the batch but
|
|
177
|
+
conclude that the cells are already sufficiently mixed and correction is unnecessary.
|
|
178
|
+
|
|
179
|
+
<details>
|
|
180
|
+
<summary>Find the output files</summary>
|
|
181
|
+
|
|
182
|
+
| File | Contents |
|
|
183
|
+
| --- | --- |
|
|
184
|
+
| `results/standardize/standardized.h5ad` | Prepared expression data, gene identifiers, and QC measurements. |
|
|
185
|
+
| `results/standardize/result.json` | Input checks, counts source, species, gene changes, and review notes. |
|
|
186
|
+
| `results/columns/result.json` | Selected metadata, correction assessment, decisions, and trial results. |
|
|
187
|
+
| `results/columns/batch.tsv` | Batch labels when the selected grouping is derived from barcodes or multiple columns; created only when needed. |
|
|
188
|
+
|
|
189
|
+
</details>
|
|
190
|
+
|
|
191
|
+
<br>
|
|
192
|
+
|
|
193
|
+
## FAQ
|
|
194
|
+
|
|
195
|
+
### Does ECA-PP change my original data?
|
|
196
|
+
|
|
197
|
+
The source file stays unchanged. ECA-PP writes a separate dataset, preserves
|
|
198
|
+
author metadata, and backs up fields it replaces. Gene mapping and filtering
|
|
199
|
+
changes are recorded. Standardization may reject a whole dataset that fails its
|
|
200
|
+
checks, but does not remove individual cells from an accepted dataset.
|
|
201
|
+
|
|
202
|
+
### What does an empty batch or cell-type result mean?
|
|
203
|
+
|
|
204
|
+
`null` means no suitable column was selected. This can be a valid outcome when
|
|
205
|
+
evidence is insufficient. A selected batch with `correction: "unnecessary"`
|
|
206
|
+
means the evidence did not support correcting it. Neither conclusion should be
|
|
207
|
+
confused with an execution error; check the outcome and reasons in `result.json`.
|
|
208
|
+
|
|
209
|
+
### Do I need an AI model?
|
|
210
|
+
|
|
211
|
+
Standardization runs locally by default; optional AI assistance is available for
|
|
212
|
+
unresolved species inference. Column identification uses a model when configured
|
|
213
|
+
and falls back to built-in rules when it is unavailable. See the
|
|
214
|
+
[model configuration guide](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md#8-识别批次列--细胞类型列identify-columns)
|
|
215
|
+
for backend and model choices.
|
|
216
|
+
|
|
217
|
+
### Can I recover counts from normalized data?
|
|
218
|
+
|
|
219
|
+
ECA-PP uses [stancounts](https://github.com/chansigit/stancounts) to recover counts
|
|
220
|
+
from supported transformed inputs when possible. Recovery depends on the data;
|
|
221
|
+
unsupported or ambiguous cases are reported rather than silently treated as raw counts.
|
|
222
|
+
|
|
223
|
+
<br>
|
|
224
|
+
|
|
225
|
+
## Further reading
|
|
226
|
+
|
|
227
|
+
**For users:** the [hands-on tutorial](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md) walks through a real
|
|
228
|
+
mouse dataset, result interpretation, common options, model setup, and reruns.
|
|
229
|
+
The tutorial is currently in Chinese. Each command also provides `--help`.
|
|
230
|
+
|
|
231
|
+
**For developers:** the [standardization specification](https://github.com/chansigit/eca-pp/blob/main/docs/standardize-spec.md)
|
|
232
|
+
and [column-identification specification](https://github.com/chansigit/eca-pp/blob/main/docs/identify-columns-spec.md) describe
|
|
233
|
+
methods, interfaces, and tests. Download the
|
|
234
|
+
[interactive architecture diagram](https://github.com/chansigit/eca-pp/blob/main/docs/architecture.html) to open it in a browser.
|
|
235
|
+
|
|
236
|
+
**Continue in the ECA ecosystem:** [ECA-RSI](https://github.com/chansigit/eca-rsi)
|
|
237
|
+
coordinates downstream analysis, including sample-level QC and annotation with
|
|
238
|
+
[OSP](https://github.com/chansigit/osp).
|
|
239
|
+
|
|
240
|
+
For questions, unexpected results, or feature requests,
|
|
241
|
+
[open an issue](https://github.com/chansigit/eca-pp/issues).
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "eca-pp"
|
|
7
|
+
version = "0.4.0"
|
|
8
|
+
description = "Standardized single-cell preprocessing for the Ensemble Cell Atlas ecosystem"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"numpy>=1.22",
|
|
13
|
+
"scipy>=1.7",
|
|
14
|
+
"h5py>=3.8",
|
|
15
|
+
"anndata>=0.8",
|
|
16
|
+
"stancounts>=0.3",
|
|
17
|
+
"stangene>=0.5",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.optional-dependencies]
|
|
21
|
+
test = ["pytest>=7.0"]
|
|
22
|
+
llm = ["deepseek-harness-sdk>=0.1.2a3", "mcp>=1.27", "PyYAML>=6", "uvicorn>=0.30", "anyio>=4.0"]
|
|
23
|
+
probe = ["scanpy>=1.10", "harmonypy>=0.0.9,<0.3", "scikit-learn>=1.1",
|
|
24
|
+
"leidenalg>=0.10"]
|
|
25
|
+
agent = ["openai-agents>=0.22,<1", "deepseek-harness-sdk>=0.1.2a3",
|
|
26
|
+
"mcp>=1.27", "PyYAML>=6", "uvicorn>=0.30", "anyio>=4.0"]
|
|
27
|
+
claude = ["claude-agent-sdk>=0.2.152", "anyio>=4.0"]
|
|
28
|
+
openai = ["openai-agents>=0.22,<1", "anyio>=4.0"]
|
|
29
|
+
# 下期: doublets = ["scrublet"], gpu = ["scvi-tools"] …
|
|
30
|
+
|
|
31
|
+
[project.scripts]
|
|
32
|
+
eca-pp-standardize = "eca_pp.standardize.cli:cli"
|
|
33
|
+
eca-pp-integration-probe = "eca_pp.probe.cli:cli"
|
|
34
|
+
eca-pp-identify-columns = "eca_pp.identify_columns.cli:cli"
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Repository = "https://github.com/chansigit/eca-pp"
|
|
38
|
+
|
|
39
|
+
[tool.setuptools.packages.find]
|
|
40
|
+
where = ["src"]
|
eca_pp-0.4.0/setup.cfg
ADDED