eca-pp 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. eca_pp-0.4.0/PKG-INFO +281 -0
  2. eca_pp-0.4.0/README.md +241 -0
  3. eca_pp-0.4.0/pyproject.toml +40 -0
  4. eca_pp-0.4.0/setup.cfg +4 -0
  5. eca_pp-0.4.0/src/eca_pp/__init__.py +3 -0
  6. eca_pp-0.4.0/src/eca_pp/_harness_claude.py +196 -0
  7. eca_pp-0.4.0/src/eca_pp/_harness_deepseek.py +552 -0
  8. eca_pp-0.4.0/src/eca_pp/_harness_openai.py +390 -0
  9. eca_pp-0.4.0/src/eca_pp/agent.py +116 -0
  10. eca_pp-0.4.0/src/eca_pp/core/__init__.py +2 -0
  11. eca_pp-0.4.0/src/eca_pp/core/atomic_io.py +45 -0
  12. eca_pp-0.4.0/src/eca_pp/core/colspec.py +62 -0
  13. eca_pp-0.4.0/src/eca_pp/core/columns.py +49 -0
  14. eca_pp-0.4.0/src/eca_pp/core/result.py +71 -0
  15. eca_pp-0.4.0/src/eca_pp/core/run_outputs.py +42 -0
  16. eca_pp-0.4.0/src/eca_pp/core/values.py +28 -0
  17. eca_pp-0.4.0/src/eca_pp/harness.py +241 -0
  18. eca_pp-0.4.0/src/eca_pp/identify_columns/__init__.py +1 -0
  19. eca_pp-0.4.0/src/eca_pp/identify_columns/__main__.py +3 -0
  20. eca_pp-0.4.0/src/eca_pp/identify_columns/cli.py +846 -0
  21. eca_pp-0.4.0/src/eca_pp/identify_columns/obsprofile.py +212 -0
  22. eca_pp-0.4.0/src/eca_pp/identify_columns/policies.py +256 -0
  23. eca_pp-0.4.0/src/eca_pp/probe/__init__.py +1 -0
  24. eca_pp-0.4.0/src/eca_pp/probe/__main__.py +3 -0
  25. eca_pp-0.4.0/src/eca_pp/probe/cli.py +356 -0
  26. eca_pp-0.4.0/src/eca_pp/standardize/__init__.py +1 -0
  27. eca_pp-0.4.0/src/eca_pp/standardize/__main__.py +3 -0
  28. eca_pp-0.4.0/src/eca_pp/standardize/build.py +65 -0
  29. eca_pp-0.4.0/src/eca_pp/standardize/cli.py +338 -0
  30. eca_pp-0.4.0/src/eca_pp/standardize/countsloc.py +231 -0
  31. eca_pp-0.4.0/src/eca_pp/standardize/harmonize.py +107 -0
  32. eca_pp-0.4.0/src/eca_pp/standardize/qc.py +131 -0
  33. eca_pp-0.4.0/src/eca_pp/standardize/species.py +157 -0
  34. eca_pp-0.4.0/src/eca_pp.egg-info/PKG-INFO +281 -0
  35. eca_pp-0.4.0/src/eca_pp.egg-info/SOURCES.txt +49 -0
  36. eca_pp-0.4.0/src/eca_pp.egg-info/dependency_links.txt +1 -0
  37. eca_pp-0.4.0/src/eca_pp.egg-info/entry_points.txt +4 -0
  38. eca_pp-0.4.0/src/eca_pp.egg-info/requires.txt +38 -0
  39. eca_pp-0.4.0/src/eca_pp.egg-info/top_level.txt +1 -0
  40. eca_pp-0.4.0/tests/test_agent.py +283 -0
  41. eca_pp-0.4.0/tests/test_columns.py +102 -0
  42. eca_pp-0.4.0/tests/test_float_groups.py +23 -0
  43. eca_pp-0.4.0/tests/test_harness_openai.py +358 -0
  44. eca_pp-0.4.0/tests/test_identify_columns.py +518 -0
  45. eca_pp-0.4.0/tests/test_obsprofile.py +117 -0
  46. eca_pp-0.4.0/tests/test_pre_gate.py +26 -0
  47. eca_pp-0.4.0/tests/test_probe.py +141 -0
  48. eca_pp-0.4.0/tests/test_run_outputs.py +27 -0
  49. eca_pp-0.4.0/tests/test_species_logging.py +20 -0
  50. eca_pp-0.4.0/tests/test_standardize.py +160 -0
  51. eca_pp-0.4.0/tests/test_standardize_v02.py +277 -0
eca_pp-0.4.0/PKG-INFO ADDED
@@ -0,0 +1,281 @@
1
+ Metadata-Version: 2.4
2
+ Name: eca-pp
3
+ Version: 0.4.0
4
+ Summary: Standardized single-cell preprocessing for the Ensemble Cell Atlas ecosystem
5
+ Project-URL: Repository, https://github.com/chansigit/eca-pp
6
+ Requires-Python: >=3.10
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: numpy>=1.22
9
+ Requires-Dist: scipy>=1.7
10
+ Requires-Dist: h5py>=3.8
11
+ Requires-Dist: anndata>=0.8
12
+ Requires-Dist: stancounts>=0.3
13
+ Requires-Dist: stangene>=0.5
14
+ Provides-Extra: test
15
+ Requires-Dist: pytest>=7.0; extra == "test"
16
+ Provides-Extra: llm
17
+ Requires-Dist: deepseek-harness-sdk>=0.1.2a3; extra == "llm"
18
+ Requires-Dist: mcp>=1.27; extra == "llm"
19
+ Requires-Dist: PyYAML>=6; extra == "llm"
20
+ Requires-Dist: uvicorn>=0.30; extra == "llm"
21
+ Requires-Dist: anyio>=4.0; extra == "llm"
22
+ Provides-Extra: probe
23
+ Requires-Dist: scanpy>=1.10; extra == "probe"
24
+ Requires-Dist: harmonypy<0.3,>=0.0.9; extra == "probe"
25
+ Requires-Dist: scikit-learn>=1.1; extra == "probe"
26
+ Requires-Dist: leidenalg>=0.10; extra == "probe"
27
+ Provides-Extra: agent
28
+ Requires-Dist: openai-agents<1,>=0.22; extra == "agent"
29
+ Requires-Dist: deepseek-harness-sdk>=0.1.2a3; extra == "agent"
30
+ Requires-Dist: mcp>=1.27; extra == "agent"
31
+ Requires-Dist: PyYAML>=6; extra == "agent"
32
+ Requires-Dist: uvicorn>=0.30; extra == "agent"
33
+ Requires-Dist: anyio>=4.0; extra == "agent"
34
+ Provides-Extra: claude
35
+ Requires-Dist: claude-agent-sdk>=0.2.152; extra == "claude"
36
+ Requires-Dist: anyio>=4.0; extra == "claude"
37
+ Provides-Extra: openai
38
+ Requires-Dist: openai-agents<1,>=0.22; extra == "openai"
39
+ Requires-Dist: anyio>=4.0; extra == "openai"
40
+
41
+ <p align="center">
42
+ <img src="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-logo.svg" alt="ECA-PP logo: scattered cells organized into a consistent atlas" width="160" height="160">
43
+ </p>
44
+
45
+ <h1 align="center">ECA-PP: Standardized Single-Cell Preprocessing</h1>
46
+
47
+ <p align="center">
48
+ <strong>Give real-world single-cell data a consistent starting point.</strong>
49
+ </p>
50
+
51
+ <p align="center">
52
+ <a href="pyproject.toml"><img src="https://img.shields.io/badge/Version-0.4.0-258B81?style=flat" alt="Project version 0.4.0"></a>
53
+ <a href="pyproject.toml"><img src="https://img.shields.io/badge/Python-3.10%2B-3776AB?style=flat&amp;logo=python&amp;logoColor=white" alt="Python 3.10 or newer"></a>
54
+ <a href="#where-it-fits"><img src="https://img.shields.io/badge/Input-H5AD-64748B?style=flat" alt="Starts from H5AD data"></a>
55
+ <a href="https://github.com/chansigit/eca-rsi"><img src="https://img.shields.io/badge/Ecosystem-Ensemble_Cell_Atlas-258B81?style=flat" alt="Part of the Ensemble Cell Atlas ecosystem"></a>
56
+ </p>
57
+
58
+ <p align="center">
59
+ <a href="#why-standardize-before-analysis">🔬 Why ECA-PP?</a>
60
+ &nbsp;&nbsp;&nbsp;
61
+ <a href="#what-you-get">What you get</a>
62
+ &nbsp;&nbsp;&nbsp;
63
+ <a href="#try-it">🚀 Try it</a>
64
+ &nbsp;&nbsp;&nbsp;
65
+ <a href="#faq">FAQ</a>
66
+ &nbsp;&nbsp;&nbsp;
67
+ <a href="#further-reading">📖 Further reading</a>
68
+ </p>
69
+
70
+ <br>
71
+
72
+ ECA-PP prepares published single-cell RNA sequencing datasets for reuse. Starting
73
+ from an **H5AD file**, it finds expression counts, standardizes gene names,
74
+ calculates quality-control measurements, and identifies useful batch and
75
+ cell-type metadata. You get prepared data and a record of the evidence behind
76
+ each decision.
77
+
78
+ ECA-PP belongs to the **Ensemble Cell Atlas (ECA)** ecosystem. It handles routine
79
+ preprocessing for **[ECA-RSI](https://github.com/chansigit/eca-rsi)**
80
+ (Recursive Self-Improvement), ECA's automated data governance system, so that
81
+ system can focus on quality assessment, annotation, and iterative refinement.
82
+ You can also use ECA-PP independently in your own pipeline.
83
+
84
+ <br>
85
+
86
+ ## Why standardize before analysis?
87
+
88
+ Public single-cell data reflects the choices of many different authors.
89
+ Expression matrices arrive as text tables, sparse matrix files, Seurat objects,
90
+ or H5AD files, often with metadata in separate supplements. Converting them into
91
+ one format solves only part of the problem.
92
+
93
+ Inside an H5AD file, expression values may be raw, normalized, or scaled. Gene
94
+ names may mix old symbols and Ensembl IDs. Sample and cell-type columns may use
95
+ unfamiliar names, duplicate one another, or contain missing values. Even existing
96
+ QC measurements may have been calculated using different gene sets.
97
+
98
+ ECA-PP gives these recurring problems a shared treatment:
99
+
100
+ - **Consistent preprocessing across studies.** Counts checks, gene mapping, and
101
+ QC use a common implementation. QC and normalized expression are calculated
102
+ on the same final gene set.
103
+
104
+ - **Metadata choices supported by data.** Small integration trials test batch
105
+ candidates for improved mixing and preservation of cell-type structure.
106
+ Technical and donor factors take priority over biological conditions.
107
+
108
+ - **Automation you can inspect.** Model suggestions pass programmatic checks;
109
+ built-in rules keep column identification moving when a model is unavailable.
110
+ Decisions, changes, and unresolved questions are recorded for review.
111
+
112
+ <br>
113
+
114
+ ## What you get
115
+
116
+ - **A standardized dataset.** An H5AD with counts, normalized expression,
117
+ standardized gene names, QC measurements, and preserved author metadata.
118
+
119
+ - **Guidance for downstream analysis.** Proposed batch and existing cell-type
120
+ columns, evidence for each selection, and an assessment of whether batch
121
+ correction is needed.
122
+
123
+ - **A record of what happened.** Each step writes a `result.json` describing its
124
+ outcome, changes, and issues that need attention. Your source file stays unchanged.
125
+
126
+ <br>
127
+
128
+ ## Where it fits
129
+
130
+ General-purpose agents can usually download files, unpack archives, and script
131
+ conversions into `.h5ad`, the AnnData format used by Scanpy. **ECA-PP starts at
132
+ H5AD**, where decisions about counts, gene identity, QC, and batch structure need
133
+ domain-specific standards applied consistently across studies.
134
+
135
+ | Stage | Responsibility |
136
+ | --- | --- |
137
+ | Upstream tools or agents | Gather published files and convert them into H5AD. |
138
+ | ECA-PP | Standardize the data and evaluate metadata using shared rules and recorded evidence. |
139
+ | ECA-RSI | Coordinate subsequent quality review, annotation, and iterative refinement. |
140
+
141
+ ECA-PP currently prepares data and evaluates metadata. It does not assign new
142
+ biological cell-type labels, filter individual low-quality cells or doublets,
143
+ or produce a final integrated atlas.
144
+
145
+ Follow the main path below; the lower branches show required review, dataset
146
+ rejection, and the trial loop used to evaluate metadata. Click the diagram for
147
+ an interactive version with search, zoom, and guided views.
148
+
149
+ <p align="center">
150
+ <a href="https://raw.githack.com/chansigit/eca-pp/main/docs/workflow.html">
151
+ <picture>
152
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-workflow-dark.svg">
153
+ <img src="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-workflow-light.svg" alt="ECA-PP workflow: input H5AD, standardization, prepared data, metadata identification with integration trials, recorded decisions, and handoff to ECA-RSI. Unresolved standardization choices need review; unsuitable inputs are rejected." width="1100">
154
+ </picture>
155
+ </a>
156
+ </p>
157
+
158
+ <br>
159
+
160
+ ## Try it
161
+
162
+ ### 1. Install
163
+
164
+ Use Python 3.10 or newer, preferably in a dedicated environment. ECA-PP runs on CPUs.
165
+
166
+ ```bash
167
+ pip install "eca-pp[probe,openai]==0.4.0"
168
+ ```
169
+
170
+ This also installs `stancounts` and `stangene`, the counts-recovery and gene-mapping
171
+ dependencies. To work on ECA-PP itself, clone this repository and use
172
+ `pip install ".[probe,openai]"` instead.
173
+
174
+ ### 2. Standardize your dataset
175
+
176
+ Replace `your-data.h5ad` with your input file:
177
+
178
+ ```bash
179
+ eca-pp-standardize your-data.h5ad -o results/standardize
180
+ ```
181
+
182
+ Open **`results/standardize/result.json`** to check the outcome. A successful run
183
+ produces `standardized.h5ad`. If a required choice is unresolved, such as the
184
+ species, the report explains what needs clarification before you continue.
185
+
186
+ <details>
187
+ <summary>Dataset size checks and gene filtering</summary>
188
+
189
+ The default checks require at least **100 cells and 5,000 detected genes across
190
+ the whole dataset**. A gene is detected if it has a nonzero count in at least one
191
+ cell. **This is not a requirement for each cell to express 5,000 genes.**
192
+
193
+ By default, features that cannot be mapped to a canonical gene are removed from
194
+ the output. Use `--keep-unmapped` to retain them. The dataset-level gene threshold
195
+ is checked again after gene mapping and filtering.
196
+
197
+ See the [tutorial's options](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md#4-常用参数) for adjusting size checks
198
+ or specifying the species and counts layer.
199
+
200
+ </details>
201
+
202
+ ### 3. Identify batch and cell-type columns
203
+
204
+ After successful standardization, run:
205
+
206
+ ```bash
207
+ eca-pp-identify-columns results/standardize/standardized.h5ad \
208
+ -o results/columns
209
+ ```
210
+
211
+ For AI-assisted decisions, set `ARK_API_KEY` in your environment before running
212
+ this command. The default uses Doubao Turbo through the OpenAI Agents SDK.
213
+ Without model credentials, ECA-PP uses built-in rules and integration trials.
214
+
215
+ Read **`results/columns/result.json`** for the selected columns and supporting
216
+ evidence. For example, ECA-PP may identify a sequencing channel as the batch but
217
+ conclude that the cells are already sufficiently mixed and correction is unnecessary.
218
+
219
+ <details>
220
+ <summary>Find the output files</summary>
221
+
222
+ | File | Contents |
223
+ | --- | --- |
224
+ | `results/standardize/standardized.h5ad` | Prepared expression data, gene identifiers, and QC measurements. |
225
+ | `results/standardize/result.json` | Input checks, counts source, species, gene changes, and review notes. |
226
+ | `results/columns/result.json` | Selected metadata, correction assessment, decisions, and trial results. |
227
+ | `results/columns/batch.tsv` | Batch labels when the selected grouping is derived from barcodes or multiple columns; created only when needed. |
228
+
229
+ </details>
230
+
231
+ <br>
232
+
233
+ ## FAQ
234
+
235
+ ### Does ECA-PP change my original data?
236
+
237
+ The source file stays unchanged. ECA-PP writes a separate dataset, preserves
238
+ author metadata, and backs up fields it replaces. Gene mapping and filtering
239
+ changes are recorded. Standardization may reject a whole dataset that fails its
240
+ checks, but does not remove individual cells from an accepted dataset.
241
+
242
+ ### What does an empty batch or cell-type result mean?
243
+
244
+ `null` means no suitable column was selected. This can be a valid outcome when
245
+ evidence is insufficient. A selected batch with `correction: "unnecessary"`
246
+ means the evidence did not support correcting it. Neither conclusion should be
247
+ confused with an execution error; check the outcome and reasons in `result.json`.
248
+
249
+ ### Do I need an AI model?
250
+
251
+ Standardization runs locally by default; optional AI assistance is available for
252
+ unresolved species inference. Column identification uses a model when configured
253
+ and falls back to built-in rules when it is unavailable. See the
254
+ [model configuration guide](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md#8-识别批次列--细胞类型列identify-columns)
255
+ for backend and model choices.
256
+
257
+ ### Can I recover counts from normalized data?
258
+
259
+ ECA-PP uses [stancounts](https://github.com/chansigit/stancounts) to recover counts
260
+ from supported transformed inputs when possible. Recovery depends on the data;
261
+ unsupported or ambiguous cases are reported rather than silently treated as raw counts.
262
+
263
+ <br>
264
+
265
+ ## Further reading
266
+
267
+ **For users:** the [hands-on tutorial](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md) walks through a real
268
+ mouse dataset, result interpretation, common options, model setup, and reruns.
269
+ The tutorial is currently in Chinese. Each command also provides `--help`.
270
+
271
+ **For developers:** the [standardization specification](https://github.com/chansigit/eca-pp/blob/main/docs/standardize-spec.md)
272
+ and [column-identification specification](https://github.com/chansigit/eca-pp/blob/main/docs/identify-columns-spec.md) describe
273
+ methods, interfaces, and tests. Download the
274
+ [interactive architecture diagram](https://github.com/chansigit/eca-pp/blob/main/docs/architecture.html) to open it in a browser.
275
+
276
+ **Continue in the ECA ecosystem:** [ECA-RSI](https://github.com/chansigit/eca-rsi)
277
+ coordinates downstream analysis, including sample-level QC and annotation with
278
+ [OSP](https://github.com/chansigit/osp).
279
+
280
+ For questions, unexpected results, or feature requests,
281
+ [open an issue](https://github.com/chansigit/eca-pp/issues).
eca_pp-0.4.0/README.md ADDED
@@ -0,0 +1,241 @@
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-logo.svg" alt="ECA-PP logo: scattered cells organized into a consistent atlas" width="160" height="160">
3
+ </p>
4
+
5
+ <h1 align="center">ECA-PP: Standardized Single-Cell Preprocessing</h1>
6
+
7
+ <p align="center">
8
+ <strong>Give real-world single-cell data a consistent starting point.</strong>
9
+ </p>
10
+
11
+ <p align="center">
12
+ <a href="pyproject.toml"><img src="https://img.shields.io/badge/Version-0.4.0-258B81?style=flat" alt="Project version 0.4.0"></a>
13
+ <a href="pyproject.toml"><img src="https://img.shields.io/badge/Python-3.10%2B-3776AB?style=flat&amp;logo=python&amp;logoColor=white" alt="Python 3.10 or newer"></a>
14
+ <a href="#where-it-fits"><img src="https://img.shields.io/badge/Input-H5AD-64748B?style=flat" alt="Starts from H5AD data"></a>
15
+ <a href="https://github.com/chansigit/eca-rsi"><img src="https://img.shields.io/badge/Ecosystem-Ensemble_Cell_Atlas-258B81?style=flat" alt="Part of the Ensemble Cell Atlas ecosystem"></a>
16
+ </p>
17
+
18
+ <p align="center">
19
+ <a href="#why-standardize-before-analysis">🔬 Why ECA-PP?</a>
20
+ &nbsp;&nbsp;&nbsp;
21
+ <a href="#what-you-get">What you get</a>
22
+ &nbsp;&nbsp;&nbsp;
23
+ <a href="#try-it">🚀 Try it</a>
24
+ &nbsp;&nbsp;&nbsp;
25
+ <a href="#faq">FAQ</a>
26
+ &nbsp;&nbsp;&nbsp;
27
+ <a href="#further-reading">📖 Further reading</a>
28
+ </p>
29
+
30
+ <br>
31
+
32
+ ECA-PP prepares published single-cell RNA sequencing datasets for reuse. Starting
33
+ from an **H5AD file**, it finds expression counts, standardizes gene names,
34
+ calculates quality-control measurements, and identifies useful batch and
35
+ cell-type metadata. You get prepared data and a record of the evidence behind
36
+ each decision.
37
+
38
+ ECA-PP belongs to the **Ensemble Cell Atlas (ECA)** ecosystem. It handles routine
39
+ preprocessing for **[ECA-RSI](https://github.com/chansigit/eca-rsi)**
40
+ (Recursive Self-Improvement), ECA's automated data governance system, so that
41
+ system can focus on quality assessment, annotation, and iterative refinement.
42
+ You can also use ECA-PP independently in your own pipeline.
43
+
44
+ <br>
45
+
46
+ ## Why standardize before analysis?
47
+
48
+ Public single-cell data reflects the choices of many different authors.
49
+ Expression matrices arrive as text tables, sparse matrix files, Seurat objects,
50
+ or H5AD files, often with metadata in separate supplements. Converting them into
51
+ one format solves only part of the problem.
52
+
53
+ Inside an H5AD file, expression values may be raw, normalized, or scaled. Gene
54
+ names may mix old symbols and Ensembl IDs. Sample and cell-type columns may use
55
+ unfamiliar names, duplicate one another, or contain missing values. Even existing
56
+ QC measurements may have been calculated using different gene sets.
57
+
58
+ ECA-PP gives these recurring problems a shared treatment:
59
+
60
+ - **Consistent preprocessing across studies.** Counts checks, gene mapping, and
61
+ QC use a common implementation. QC and normalized expression are calculated
62
+ on the same final gene set.
63
+
64
+ - **Metadata choices supported by data.** Small integration trials test batch
65
+ candidates for improved mixing and preservation of cell-type structure.
66
+ Technical and donor factors take priority over biological conditions.
67
+
68
+ - **Automation you can inspect.** Model suggestions pass programmatic checks;
69
+ built-in rules keep column identification moving when a model is unavailable.
70
+ Decisions, changes, and unresolved questions are recorded for review.
71
+
72
+ <br>
73
+
74
+ ## What you get
75
+
76
+ - **A standardized dataset.** An H5AD with counts, normalized expression,
77
+ standardized gene names, QC measurements, and preserved author metadata.
78
+
79
+ - **Guidance for downstream analysis.** Proposed batch and existing cell-type
80
+ columns, evidence for each selection, and an assessment of whether batch
81
+ correction is needed.
82
+
83
+ - **A record of what happened.** Each step writes a `result.json` describing its
84
+ outcome, changes, and issues that need attention. Your source file stays unchanged.
85
+
86
+ <br>
87
+
88
+ ## Where it fits
89
+
90
+ General-purpose agents can usually download files, unpack archives, and script
91
+ conversions into `.h5ad`, the AnnData format used by Scanpy. **ECA-PP starts at
92
+ H5AD**, where decisions about counts, gene identity, QC, and batch structure need
93
+ domain-specific standards applied consistently across studies.
94
+
95
+ | Stage | Responsibility |
96
+ | --- | --- |
97
+ | Upstream tools or agents | Gather published files and convert them into H5AD. |
98
+ | ECA-PP | Standardize the data and evaluate metadata using shared rules and recorded evidence. |
99
+ | ECA-RSI | Coordinate subsequent quality review, annotation, and iterative refinement. |
100
+
101
+ ECA-PP currently prepares data and evaluates metadata. It does not assign new
102
+ biological cell-type labels, filter individual low-quality cells or doublets,
103
+ or produce a final integrated atlas.
104
+
105
+ Follow the main path below; the lower branches show required review, dataset
106
+ rejection, and the trial loop used to evaluate metadata. Click the diagram for
107
+ an interactive version with search, zoom, and guided views.
108
+
109
+ <p align="center">
110
+ <a href="https://raw.githack.com/chansigit/eca-pp/main/docs/workflow.html">
111
+ <picture>
112
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-workflow-dark.svg">
113
+ <img src="https://raw.githubusercontent.com/chansigit/eca-pp/main/assets/eca-pp-workflow-light.svg" alt="ECA-PP workflow: input H5AD, standardization, prepared data, metadata identification with integration trials, recorded decisions, and handoff to ECA-RSI. Unresolved standardization choices need review; unsuitable inputs are rejected." width="1100">
114
+ </picture>
115
+ </a>
116
+ </p>
117
+
118
+ <br>
119
+
120
+ ## Try it
121
+
122
+ ### 1. Install
123
+
124
+ Use Python 3.10 or newer, preferably in a dedicated environment. ECA-PP runs on CPUs.
125
+
126
+ ```bash
127
+ pip install "eca-pp[probe,openai]==0.4.0"
128
+ ```
129
+
130
+ This also installs `stancounts` and `stangene`, the counts-recovery and gene-mapping
131
+ dependencies. To work on ECA-PP itself, clone this repository and use
132
+ `pip install ".[probe,openai]"` instead.
133
+
134
+ ### 2. Standardize your dataset
135
+
136
+ Replace `your-data.h5ad` with your input file:
137
+
138
+ ```bash
139
+ eca-pp-standardize your-data.h5ad -o results/standardize
140
+ ```
141
+
142
+ Open **`results/standardize/result.json`** to check the outcome. A successful run
143
+ produces `standardized.h5ad`. If a required choice is unresolved, such as the
144
+ species, the report explains what needs clarification before you continue.
145
+
146
+ <details>
147
+ <summary>Dataset size checks and gene filtering</summary>
148
+
149
+ The default checks require at least **100 cells and 5,000 detected genes across
150
+ the whole dataset**. A gene is detected if it has a nonzero count in at least one
151
+ cell. **This is not a requirement for each cell to express 5,000 genes.**
152
+
153
+ By default, features that cannot be mapped to a canonical gene are removed from
154
+ the output. Use `--keep-unmapped` to retain them. The dataset-level gene threshold
155
+ is checked again after gene mapping and filtering.
156
+
157
+ See the [tutorial's options](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md#4-常用参数) for adjusting size checks
158
+ or specifying the species and counts layer.
159
+
160
+ </details>
161
+
162
+ ### 3. Identify batch and cell-type columns
163
+
164
+ After successful standardization, run:
165
+
166
+ ```bash
167
+ eca-pp-identify-columns results/standardize/standardized.h5ad \
168
+ -o results/columns
169
+ ```
170
+
171
+ For AI-assisted decisions, set `ARK_API_KEY` in your environment before running
172
+ this command. The default uses Doubao Turbo through the OpenAI Agents SDK.
173
+ Without model credentials, ECA-PP uses built-in rules and integration trials.
174
+
175
+ Read **`results/columns/result.json`** for the selected columns and supporting
176
+ evidence. For example, ECA-PP may identify a sequencing channel as the batch but
177
+ conclude that the cells are already sufficiently mixed and correction is unnecessary.
178
+
179
+ <details>
180
+ <summary>Find the output files</summary>
181
+
182
+ | File | Contents |
183
+ | --- | --- |
184
+ | `results/standardize/standardized.h5ad` | Prepared expression data, gene identifiers, and QC measurements. |
185
+ | `results/standardize/result.json` | Input checks, counts source, species, gene changes, and review notes. |
186
+ | `results/columns/result.json` | Selected metadata, correction assessment, decisions, and trial results. |
187
+ | `results/columns/batch.tsv` | Batch labels when the selected grouping is derived from barcodes or multiple columns; created only when needed. |
188
+
189
+ </details>
190
+
191
+ <br>
192
+
193
+ ## FAQ
194
+
195
+ ### Does ECA-PP change my original data?
196
+
197
+ The source file stays unchanged. ECA-PP writes a separate dataset, preserves
198
+ author metadata, and backs up fields it replaces. Gene mapping and filtering
199
+ changes are recorded. Standardization may reject a whole dataset that fails its
200
+ checks, but does not remove individual cells from an accepted dataset.
201
+
202
+ ### What does an empty batch or cell-type result mean?
203
+
204
+ `null` means no suitable column was selected. This can be a valid outcome when
205
+ evidence is insufficient. A selected batch with `correction: "unnecessary"`
206
+ means the evidence did not support correcting it. Neither conclusion should be
207
+ confused with an execution error; check the outcome and reasons in `result.json`.
208
+
209
+ ### Do I need an AI model?
210
+
211
+ Standardization runs locally by default; optional AI assistance is available for
212
+ unresolved species inference. Column identification uses a model when configured
213
+ and falls back to built-in rules when it is unavailable. See the
214
+ [model configuration guide](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md#8-识别批次列--细胞类型列identify-columns)
215
+ for backend and model choices.
216
+
217
+ ### Can I recover counts from normalized data?
218
+
219
+ ECA-PP uses [stancounts](https://github.com/chansigit/stancounts) to recover counts
220
+ from supported transformed inputs when possible. Recovery depends on the data;
221
+ unsupported or ambiguous cases are reported rather than silently treated as raw counts.
222
+
223
+ <br>
224
+
225
+ ## Further reading
226
+
227
+ **For users:** the [hands-on tutorial](https://github.com/chansigit/eca-pp/blob/main/docs/tutorial.md) walks through a real
228
+ mouse dataset, result interpretation, common options, model setup, and reruns.
229
+ The tutorial is currently in Chinese. Each command also provides `--help`.
230
+
231
+ **For developers:** the [standardization specification](https://github.com/chansigit/eca-pp/blob/main/docs/standardize-spec.md)
232
+ and [column-identification specification](https://github.com/chansigit/eca-pp/blob/main/docs/identify-columns-spec.md) describe
233
+ methods, interfaces, and tests. Download the
234
+ [interactive architecture diagram](https://github.com/chansigit/eca-pp/blob/main/docs/architecture.html) to open it in a browser.
235
+
236
+ **Continue in the ECA ecosystem:** [ECA-RSI](https://github.com/chansigit/eca-rsi)
237
+ coordinates downstream analysis, including sample-level QC and annotation with
238
+ [OSP](https://github.com/chansigit/osp).
239
+
240
+ For questions, unexpected results, or feature requests,
241
+ [open an issue](https://github.com/chansigit/eca-pp/issues).
@@ -0,0 +1,40 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "eca-pp"
7
+ version = "0.4.0"
8
+ description = "Standardized single-cell preprocessing for the Ensemble Cell Atlas ecosystem"
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ dependencies = [
12
+ "numpy>=1.22",
13
+ "scipy>=1.7",
14
+ "h5py>=3.8",
15
+ "anndata>=0.8",
16
+ "stancounts>=0.3",
17
+ "stangene>=0.5",
18
+ ]
19
+
20
+ [project.optional-dependencies]
21
+ test = ["pytest>=7.0"]
22
+ llm = ["deepseek-harness-sdk>=0.1.2a3", "mcp>=1.27", "PyYAML>=6", "uvicorn>=0.30", "anyio>=4.0"]
23
+ probe = ["scanpy>=1.10", "harmonypy>=0.0.9,<0.3", "scikit-learn>=1.1",
24
+ "leidenalg>=0.10"]
25
+ agent = ["openai-agents>=0.22,<1", "deepseek-harness-sdk>=0.1.2a3",
26
+ "mcp>=1.27", "PyYAML>=6", "uvicorn>=0.30", "anyio>=4.0"]
27
+ claude = ["claude-agent-sdk>=0.2.152", "anyio>=4.0"]
28
+ openai = ["openai-agents>=0.22,<1", "anyio>=4.0"]
29
+ # 下期: doublets = ["scrublet"], gpu = ["scvi-tools"] …
30
+
31
+ [project.scripts]
32
+ eca-pp-standardize = "eca_pp.standardize.cli:cli"
33
+ eca-pp-integration-probe = "eca_pp.probe.cli:cli"
34
+ eca-pp-identify-columns = "eca_pp.identify_columns.cli:cli"
35
+
36
+ [project.urls]
37
+ Repository = "https://github.com/chansigit/eca-pp"
38
+
39
+ [tool.setuptools.packages.find]
40
+ where = ["src"]
eca_pp-0.4.0/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,3 @@
1
+ """Standalone, orchestrator-free curation steps (spec: docs/standardize-spec.md)."""
2
+
3
+ __version__ = "0.4.0"