osp-sc 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- osp_sc-0.1.0/LICENSE +21 -0
- osp_sc-0.1.0/PKG-INFO +105 -0
- osp_sc-0.1.0/README.md +85 -0
- osp_sc-0.1.0/osp/__init__.py +63 -0
- osp_sc-0.1.0/osp/__main__.py +54 -0
- osp_sc-0.1.0/osp/_decontx/__init__.py +38 -0
- osp_sc-0.1.0/osp/_decontx/_core.py +224 -0
- osp_sc-0.1.0/osp/_decontx/_dirichlet.py +97 -0
- osp_sc-0.1.0/osp/_decontx/decontx.py +363 -0
- osp_sc-0.1.0/osp/annotate.py +533 -0
- osp_sc-0.1.0/osp/cluster.py +625 -0
- osp_sc-0.1.0/osp/qc.py +730 -0
- osp_sc-0.1.0/osp/report.py +633 -0
- osp_sc-0.1.0/osp_sc.egg-info/PKG-INFO +105 -0
- osp_sc-0.1.0/osp_sc.egg-info/SOURCES.txt +18 -0
- osp_sc-0.1.0/osp_sc.egg-info/dependency_links.txt +1 -0
- osp_sc-0.1.0/osp_sc.egg-info/requires.txt +10 -0
- osp_sc-0.1.0/osp_sc.egg-info/top_level.txt +1 -0
- osp_sc-0.1.0/pyproject.toml +27 -0
- osp_sc-0.1.0/setup.cfg +4 -0
osp_sc-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 chansigit
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
osp_sc-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: osp-sc
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: One-sample-pipeline: per-sample scRNA-seq QC, clustering, DEG and self-contained HTML report
|
|
5
|
+
Author: chansigit
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: scanpy
|
|
11
|
+
Requires-Dist: igraph
|
|
12
|
+
Requires-Dist: pandas
|
|
13
|
+
Requires-Dist: numpy
|
|
14
|
+
Requires-Dist: scipy
|
|
15
|
+
Requires-Dist: matplotlib
|
|
16
|
+
Requires-Dist: scikit-learn
|
|
17
|
+
Provides-Extra: agent
|
|
18
|
+
Requires-Dist: claude-agent-sdk; extra == "agent"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# osp — one-sample-pipeline
|
|
22
|
+
|
|
23
|
+
Single-sample scRNA-seq QC → clustering/DEG → self-contained HTML report, with
|
|
24
|
+
an optional Claude-agent step that proposes cell-type annotations and QC
|
|
25
|
+
actions from the report and the cluster marker tables.
|
|
26
|
+
|
|
27
|
+
Strictly single-sample by design — one sample per run, no cross-sample batch
|
|
28
|
+
integration. Loop over samples in an outer driver (e.g. a Slurm job array);
|
|
29
|
+
treat integration as a separate downstream step.
|
|
30
|
+
|
|
31
|
+
## Install
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install osp-sc # PyPI name; `import osp` / `python -m osp`
|
|
35
|
+
# with the optional annotation agent (needs claude-agent-sdk + claude CLI credentials):
|
|
36
|
+
pip install "osp-sc[agent]" # + claude-agent-sdk for --annotate
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Quick usage
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from osp import run_one_sample_pipeline, generate_report
|
|
43
|
+
|
|
44
|
+
ad_fo = adata[adata.obs["sample"] == "FO"]
|
|
45
|
+
run_one_sample_pipeline(ad_fo, sample_label="FO", outdir="osp_out/FO")
|
|
46
|
+
generate_report("osp_out/FO")
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Stepwise calls, if you want more control:
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from osp import qc_one_sample, cluster_and_deg, deg_two_groups
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
- `qc_one_sample` — QC only (flags cells, drops nothing)
|
|
56
|
+
- `cluster_and_deg` — clustering/DEG/PAGA on QC-passed data
|
|
57
|
+
- `deg_two_groups` — Wilcoxon DEG between any two cell groups, for ad hoc comparisons outside the main pipeline
|
|
58
|
+
|
|
59
|
+
## Command line
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
python -m osp data.h5ad --sample FO --outdir osp_out # full pipeline + report
|
|
63
|
+
python -m osp data.h5ad --sample FO --outdir osp_out --annotate --model claude-sonnet-5
|
|
64
|
+
python -m osp.report osp_out # rebuild the report only
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
See `examples/run_one_sample.py` for a driver that loads a large h5ad in
|
|
68
|
+
backed mode and pulls out one sample (for per-sample Slurm array tasks), and
|
|
69
|
+
`examples/submit_array.sbatch` for the job-array template.
|
|
70
|
+
|
|
71
|
+
## Conventions
|
|
72
|
+
|
|
73
|
+
- **Raw counts convention**: if `adata.layers["counts"]` exists, `X` is swapped
|
|
74
|
+
for it at the start of both the QC and clustering stages — this makes the
|
|
75
|
+
pipeline robust to inputs where `X` already holds normalized values with
|
|
76
|
+
raw counts kept in a layer (common in released h5ad files).
|
|
77
|
+
- **QC is flag-only**: `qc_one_sample` never drops cells; `low_quality` is a
|
|
78
|
+
column, filtering is the caller's decision.
|
|
79
|
+
- **DecontX degeneracy guard**: DecontX's own UMAP+DBSCAN init can collapse on
|
|
80
|
+
samples where the dominant cell lineage's transcriptome resembles the
|
|
81
|
+
ambient RNA pool (all contamination pinned near 1, or the init shattering
|
|
82
|
+
into 100+ tiny clusters). When detected, `qc_one_sample` automatically
|
|
83
|
+
re-runs DecontX with an explicit coarse-leiden clustering; check
|
|
84
|
+
`summary["decontx_z_source"]` (`"internal"` vs `"leiden_fallback"`) and the
|
|
85
|
+
Ambient Contamination section of the report.
|
|
86
|
+
- **MAD-outlier assumption**: the adaptive per-sample QC thresholds (`nmads`
|
|
87
|
+
MADs around the median) assume a roughly regular within-sample
|
|
88
|
+
distribution. On samples with unusually shallow depth or heavy ambient
|
|
89
|
+
contamination this assumption can break — a naturally low-complexity but
|
|
90
|
+
perfectly healthy population (e.g. neutrophils, dominated by a handful of
|
|
91
|
+
granule genes) can get its `pct_counts_in_top_20_genes` MAD range squeezed
|
|
92
|
+
and be flagged en masse. The QC report's "MAD keep-ranges" table exists
|
|
93
|
+
specifically so this is visible instead of silent — a suspiciously tight
|
|
94
|
+
range, or one metric dominating the fail counts, is a signal to check which
|
|
95
|
+
cell types are being flagged before trusting the calls.
|
|
96
|
+
|
|
97
|
+
## Sherlock / HPC notes
|
|
98
|
+
|
|
99
|
+
- Never run the pipeline on a login node — submit through Slurm or use an
|
|
100
|
+
interactive allocation.
|
|
101
|
+
- For large h5ad files, load in backed mode and subset to one sample before
|
|
102
|
+
bringing it into memory (see `examples/run_one_sample.py`).
|
|
103
|
+
- `osp.annotate` is intentionally not imported by `osp/__init__.py` — it
|
|
104
|
+
depends on the optional `claude-agent-sdk`; import it explicitly
|
|
105
|
+
(`from osp.annotate import propose_annotation`) only when you need it.
|
osp_sc-0.1.0/README.md
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# osp — one-sample-pipeline
|
|
2
|
+
|
|
3
|
+
Single-sample scRNA-seq QC → clustering/DEG → self-contained HTML report, with
|
|
4
|
+
an optional Claude-agent step that proposes cell-type annotations and QC
|
|
5
|
+
actions from the report and the cluster marker tables.
|
|
6
|
+
|
|
7
|
+
Strictly single-sample by design — one sample per run, no cross-sample batch
|
|
8
|
+
integration. Loop over samples in an outer driver (e.g. a Slurm job array);
|
|
9
|
+
treat integration as a separate downstream step.
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install osp-sc # PyPI name; `import osp` / `python -m osp`
|
|
15
|
+
# with the optional annotation agent (needs claude-agent-sdk + claude CLI credentials):
|
|
16
|
+
pip install "osp-sc[agent]" # + claude-agent-sdk for --annotate
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Quick usage
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
from osp import run_one_sample_pipeline, generate_report
|
|
23
|
+
|
|
24
|
+
ad_fo = adata[adata.obs["sample"] == "FO"]
|
|
25
|
+
run_one_sample_pipeline(ad_fo, sample_label="FO", outdir="osp_out/FO")
|
|
26
|
+
generate_report("osp_out/FO")
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Stepwise calls, if you want more control:
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from osp import qc_one_sample, cluster_and_deg, deg_two_groups
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
- `qc_one_sample` — QC only (flags cells, drops nothing)
|
|
36
|
+
- `cluster_and_deg` — clustering/DEG/PAGA on QC-passed data
|
|
37
|
+
- `deg_two_groups` — Wilcoxon DEG between any two cell groups, for ad hoc comparisons outside the main pipeline
|
|
38
|
+
|
|
39
|
+
## Command line
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
python -m osp data.h5ad --sample FO --outdir osp_out # full pipeline + report
|
|
43
|
+
python -m osp data.h5ad --sample FO --outdir osp_out --annotate --model claude-sonnet-5
|
|
44
|
+
python -m osp.report osp_out # rebuild the report only
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
See `examples/run_one_sample.py` for a driver that loads a large h5ad in
|
|
48
|
+
backed mode and pulls out one sample (for per-sample Slurm array tasks), and
|
|
49
|
+
`examples/submit_array.sbatch` for the job-array template.
|
|
50
|
+
|
|
51
|
+
## Conventions
|
|
52
|
+
|
|
53
|
+
- **Raw counts convention**: if `adata.layers["counts"]` exists, `X` is swapped
|
|
54
|
+
for it at the start of both the QC and clustering stages — this makes the
|
|
55
|
+
pipeline robust to inputs where `X` already holds normalized values with
|
|
56
|
+
raw counts kept in a layer (common in released h5ad files).
|
|
57
|
+
- **QC is flag-only**: `qc_one_sample` never drops cells; `low_quality` is a
|
|
58
|
+
column, filtering is the caller's decision.
|
|
59
|
+
- **DecontX degeneracy guard**: DecontX's own UMAP+DBSCAN init can collapse on
|
|
60
|
+
samples where the dominant cell lineage's transcriptome resembles the
|
|
61
|
+
ambient RNA pool (all contamination pinned near 1, or the init shattering
|
|
62
|
+
into 100+ tiny clusters). When detected, `qc_one_sample` automatically
|
|
63
|
+
re-runs DecontX with an explicit coarse-leiden clustering; check
|
|
64
|
+
`summary["decontx_z_source"]` (`"internal"` vs `"leiden_fallback"`) and the
|
|
65
|
+
Ambient Contamination section of the report.
|
|
66
|
+
- **MAD-outlier assumption**: the adaptive per-sample QC thresholds (`nmads`
|
|
67
|
+
MADs around the median) assume a roughly regular within-sample
|
|
68
|
+
distribution. On samples with unusually shallow depth or heavy ambient
|
|
69
|
+
contamination this assumption can break — a naturally low-complexity but
|
|
70
|
+
perfectly healthy population (e.g. neutrophils, dominated by a handful of
|
|
71
|
+
granule genes) can get its `pct_counts_in_top_20_genes` MAD range squeezed
|
|
72
|
+
and be flagged en masse. The QC report's "MAD keep-ranges" table exists
|
|
73
|
+
specifically so this is visible instead of silent — a suspiciously tight
|
|
74
|
+
range, or one metric dominating the fail counts, is a signal to check which
|
|
75
|
+
cell types are being flagged before trusting the calls.
|
|
76
|
+
|
|
77
|
+
## Sherlock / HPC notes
|
|
78
|
+
|
|
79
|
+
- Never run the pipeline on a login node — submit through Slurm or use an
|
|
80
|
+
interactive allocation.
|
|
81
|
+
- For large h5ad files, load in backed mode and subset to one sample before
|
|
82
|
+
bringing it into memory (see `examples/run_one_sample.py`).
|
|
83
|
+
- `osp.annotate` is intentionally not imported by `osp/__init__.py` — it
|
|
84
|
+
depends on the optional `claude-agent-sdk`; import it explicitly
|
|
85
|
+
(`from osp.annotate import propose_annotation`) only when you need it.
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""OSP (one-sample-pipeline): single-sample scRNA-seq QC → clustering/DEG →
|
|
2
|
+
self-contained HTML report.
|
|
3
|
+
|
|
4
|
+
Strictly single-sample by design — one sample per run, no cross-sample batch
|
|
5
|
+
integration; loop over samples in an outer driver, and treat integration as a
|
|
6
|
+
separate downstream step.
|
|
7
|
+
|
|
8
|
+
Entry points:
|
|
9
|
+
from osp import run_one_sample_pipeline, generate_report
|
|
10
|
+
|
|
11
|
+
ad_fo = adata[adata.obs["sample"] == "FO"]
|
|
12
|
+
run_one_sample_pipeline(ad_fo, sample_label="FO", outdir="osp_out/FO")
|
|
13
|
+
generate_report("osp_out/FO")
|
|
14
|
+
|
|
15
|
+
Stepwise calls / utilities:
|
|
16
|
+
qc_one_sample QC only (flags cells, drops nothing) — see osp.qc
|
|
17
|
+
cluster_and_deg clustering/DEG/PAGA on QC-passed data — see osp.cluster
|
|
18
|
+
deg_two_groups wilcoxon DEG between any two cell groups — see osp.cluster
|
|
19
|
+
|
|
20
|
+
Command line:
|
|
21
|
+
python -m osp data.h5ad --sample FO --outdir osp_out # full pipeline + report
|
|
22
|
+
python -m osp.qc data.h5ad --sample FO # QC only
|
|
23
|
+
python -m osp.report osp_out # report only
|
|
24
|
+
python -m osp.annotate osp_out --species mouse --tissue "bone marrow"
|
|
25
|
+
# Claude annotation agent
|
|
26
|
+
# (needs claude-agent-sdk)
|
|
27
|
+
|
|
28
|
+
osp.annotate is intentionally not imported here — it depends on the optional
|
|
29
|
+
claude-agent-sdk; use `from osp.annotate import propose_annotation` when
|
|
30
|
+
needed.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from .qc import (
|
|
34
|
+
DISSOCIATION_GENES_HS,
|
|
35
|
+
SPECIES_GENE_PATTERNS,
|
|
36
|
+
assert_single_sample,
|
|
37
|
+
cluster_order,
|
|
38
|
+
decontx_top_genes,
|
|
39
|
+
qc_one_sample,
|
|
40
|
+
)
|
|
41
|
+
from .cluster import (
|
|
42
|
+
DEFAULT_QC_PCA_COVARIATES,
|
|
43
|
+
QC_OVERLAY_COLS,
|
|
44
|
+
cluster_and_deg,
|
|
45
|
+
deg_two_groups,
|
|
46
|
+
run_one_sample_pipeline,
|
|
47
|
+
)
|
|
48
|
+
from .report import generate_report
|
|
49
|
+
|
|
50
|
+
__all__ = [
|
|
51
|
+
"DEFAULT_QC_PCA_COVARIATES",
|
|
52
|
+
"DISSOCIATION_GENES_HS",
|
|
53
|
+
"QC_OVERLAY_COLS",
|
|
54
|
+
"SPECIES_GENE_PATTERNS",
|
|
55
|
+
"assert_single_sample",
|
|
56
|
+
"cluster_and_deg",
|
|
57
|
+
"cluster_order",
|
|
58
|
+
"decontx_top_genes",
|
|
59
|
+
"deg_two_groups",
|
|
60
|
+
"generate_report",
|
|
61
|
+
"qc_one_sample",
|
|
62
|
+
"run_one_sample_pipeline",
|
|
63
|
+
]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""python -m osp: single-sample QC → clustering → DEG → HTML report, end to end.
|
|
2
|
+
|
|
3
|
+
With --annotate, the Claude annotation agent (osp.annotate, needs the
|
|
4
|
+
optional claude-agent-sdk) runs afterwards and its proposal is folded into
|
|
5
|
+
the report.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
|
|
10
|
+
import scanpy as sc
|
|
11
|
+
|
|
12
|
+
from .cluster import run_one_sample_pipeline
|
|
13
|
+
from .report import generate_report, write_report_context
|
|
14
|
+
|
|
15
|
+
parser = argparse.ArgumentParser(prog="osp", description=__doc__)
|
|
16
|
+
parser.add_argument("h5ad_path")
|
|
17
|
+
parser.add_argument("--sample-col", default="sample")
|
|
18
|
+
parser.add_argument("--sample", required=True, help="sample name to run on its own")
|
|
19
|
+
parser.add_argument("--outdir", default="osp_out")
|
|
20
|
+
parser.add_argument("--no-scrublet", action="store_true")
|
|
21
|
+
parser.add_argument("--resolution", type=float, default=1.0)
|
|
22
|
+
parser.add_argument("--annotate", action="store_true",
|
|
23
|
+
help="after the pipeline, run the Claude annotation agent and refresh the report")
|
|
24
|
+
parser.add_argument("--species", default=None, help="context passed to --annotate")
|
|
25
|
+
parser.add_argument("--tissue", default=None, help="context passed to --annotate")
|
|
26
|
+
parser.add_argument("--language", default="English", help='annotation output language (default "English")')
|
|
27
|
+
parser.add_argument("--model", default=None, help='model for --annotate, e.g. "claude-fable-5" / "claude-sonnet-5"')
|
|
28
|
+
parser.add_argument("--effort", default=None, choices=["low", "medium", "high", "xhigh", "max"],
|
|
29
|
+
help="reasoning effort for --annotate (models that support it)")
|
|
30
|
+
parser.add_argument("--report-context", default=None, metavar="TEXT",
|
|
31
|
+
help="where this sample sits, for the report title (e.g. the analysis unit name)")
|
|
32
|
+
args = parser.parse_args()
|
|
33
|
+
write_report_context(args.outdir, args.report_context)
|
|
34
|
+
|
|
35
|
+
adata = sc.read_h5ad(args.h5ad_path)
|
|
36
|
+
sub = adata[adata.obs[args.sample_col] == args.sample]
|
|
37
|
+
_, _, cluster_summary, *_ = run_one_sample_pipeline(
|
|
38
|
+
sub,
|
|
39
|
+
sample_label=args.sample,
|
|
40
|
+
sample_col=args.sample_col,
|
|
41
|
+
qc_kwargs={"run_scrublet": not args.no_scrublet},
|
|
42
|
+
cluster_kwargs={"resolutions": (args.resolution,), "primary_resolution": args.resolution},
|
|
43
|
+
outdir=args.outdir,
|
|
44
|
+
)
|
|
45
|
+
print(cluster_summary)
|
|
46
|
+
print(f"report: {generate_report(args.outdir)}")
|
|
47
|
+
|
|
48
|
+
if args.annotate:
|
|
49
|
+
from .annotate import propose_annotation
|
|
50
|
+
|
|
51
|
+
propose_annotation(
|
|
52
|
+
args.outdir, species=args.species, tissue=args.tissue, language=args.language,
|
|
53
|
+
model=args.model, effort=args.effort,
|
|
54
|
+
)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""osp._decontx -- vendored DecontX implementation.
|
|
2
|
+
|
|
3
|
+
Vendored (not an external dependency) from ``pydecontx`` /
|
|
4
|
+
``py-decontx`` (https://github.com/omicverse/py-decontx), itself a
|
|
5
|
+
pure-Python port of the Bioconductor ``decontX`` package (Yang et al.,
|
|
6
|
+
*Genome Biology* 2020). Licensed Apache License 2.0 -- see
|
|
7
|
+
../../THIRD_PARTY_NOTICES.md for the full license text and attribution.
|
|
8
|
+
|
|
9
|
+
DecontX models each cell's observed UMI counts as a Bayesian
|
|
10
|
+
two-component multinomial mixture of a *native* gene distribution
|
|
11
|
+
``phi`` and a *contamination* distribution ``eta`` (a weighted blend of
|
|
12
|
+
every other population). Inference is by variational EM and yields a
|
|
13
|
+
per-cell contamination fraction and a decontaminated count matrix.
|
|
14
|
+
|
|
15
|
+
Internal to osp -- only :func:`decontx_one_sample.qc_one_sample` (via
|
|
16
|
+
``osp.qc``) calls into this subpackage; it is not part of the public
|
|
17
|
+
osp API.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from ._core import (
|
|
22
|
+
calculate_native_matrix,
|
|
23
|
+
decontx_em,
|
|
24
|
+
decontx_initialize,
|
|
25
|
+
decontx_loglik,
|
|
26
|
+
)
|
|
27
|
+
from ._dirichlet import fit_dirichlet
|
|
28
|
+
from .decontx import DecontXResult, decontx
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"decontx",
|
|
32
|
+
"DecontXResult",
|
|
33
|
+
"decontx_initialize",
|
|
34
|
+
"decontx_em",
|
|
35
|
+
"decontx_loglik",
|
|
36
|
+
"calculate_native_matrix",
|
|
37
|
+
"fit_dirichlet",
|
|
38
|
+
]
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"""Core variational-EM routines for DecontX.
|
|
2
|
+
|
|
3
|
+
Vendored from ``pydecontx`` (Apache-2.0; see ../../THIRD_PARTY_NOTICES.md),
|
|
4
|
+
itself a pure-Python / numpy / scipy port of the C++ inner loops of the
|
|
5
|
+
Bioconductor ``decontX`` package (``src/DecontX.cpp``):
|
|
6
|
+
|
|
7
|
+
* :func:`decontx_initialize` -- initial native (``phi``) and contamination
|
|
8
|
+
(``eta``) gene distributions from a random ``theta``.
|
|
9
|
+
* :func:`decontx_em` -- one variational-EM step updating ``phi``, ``eta``,
|
|
10
|
+
``theta`` and (optionally) the Dirichlet hyper-parameter ``delta``.
|
|
11
|
+
* :func:`decontx_loglik` -- the two-component multinomial log-likelihood.
|
|
12
|
+
* :func:`calculate_native_matrix` -- the decontaminated count matrix.
|
|
13
|
+
|
|
14
|
+
All matrices follow the R convention: ``counts`` is genes-by-cells
|
|
15
|
+
(rows = genes, columns = cells); ``phi`` / ``eta`` are genes-by-clusters.
|
|
16
|
+
``z`` holds 1-based integer cluster labels (one per cell).
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
import scipy.sparse as sp
|
|
22
|
+
|
|
23
|
+
from ._dirichlet import fit_dirichlet
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"decontx_initialize",
|
|
27
|
+
"decontx_em",
|
|
28
|
+
"decontx_loglik",
|
|
29
|
+
"calculate_native_matrix",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _as_csc(counts) -> sp.csc_matrix:
|
|
34
|
+
"""Return ``counts`` as a float CSC sparse matrix (genes x cells)."""
|
|
35
|
+
if sp.issparse(counts):
|
|
36
|
+
return counts.tocsc().astype(float)
|
|
37
|
+
return sp.csc_matrix(np.asarray(counts, dtype=float))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def decontx_initialize(counts, theta, z, pseudocount: float = 1e-20):
|
|
41
|
+
"""Initialise the native (``phi``) and contamination (``eta``) matrices.
|
|
42
|
+
|
|
43
|
+
Port of the C++ ``decontXInitialize``. ``phi[:, k]`` accumulates
|
|
44
|
+
``theta_j * counts`` over all cells ``j`` assigned to cluster ``k``;
|
|
45
|
+
``eta`` is the row-sum complement (every *other* cluster's signal),
|
|
46
|
+
and both are column-normalised to proportions.
|
|
47
|
+
|
|
48
|
+
Parameters
|
|
49
|
+
----------
|
|
50
|
+
counts : (G, C) array or sparse matrix
|
|
51
|
+
Gene-by-cell UMI counts.
|
|
52
|
+
theta : (C,) array
|
|
53
|
+
Initial native proportion for each cell.
|
|
54
|
+
z : (C,) int array
|
|
55
|
+
1-based cluster label for each cell.
|
|
56
|
+
pseudocount : float
|
|
57
|
+
Added to every cell of ``phi``/``eta`` before normalising.
|
|
58
|
+
|
|
59
|
+
Returns
|
|
60
|
+
-------
|
|
61
|
+
dict with keys ``phi`` and ``eta`` -- (G, K) numpy arrays.
|
|
62
|
+
"""
|
|
63
|
+
counts = _as_csc(counts)
|
|
64
|
+
theta = np.asarray(theta, dtype=float)
|
|
65
|
+
z = np.asarray(z, dtype=int)
|
|
66
|
+
G, C = counts.shape
|
|
67
|
+
K = int(z.max())
|
|
68
|
+
|
|
69
|
+
phi = np.full((G, K), pseudocount, dtype=float)
|
|
70
|
+
indptr, indices, data = counts.indptr, counts.indices, counts.data
|
|
71
|
+
for j in range(C):
|
|
72
|
+
k = z[j] - 1
|
|
73
|
+
start, end = indptr[j], indptr[j + 1]
|
|
74
|
+
rows = indices[start:end]
|
|
75
|
+
vals = data[start:end] * theta[j]
|
|
76
|
+
np.add.at(phi[:, k], rows, vals)
|
|
77
|
+
|
|
78
|
+
phi_rowsum = phi.sum(axis=1)
|
|
79
|
+
eta = phi_rowsum[:, None] - phi
|
|
80
|
+
|
|
81
|
+
phi = phi / phi.sum(axis=0, keepdims=True)
|
|
82
|
+
eta = eta / eta.sum(axis=0, keepdims=True)
|
|
83
|
+
return {"phi": phi, "eta": eta}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def decontx_em(counts, counts_colsums, theta, eta, phi, z,
|
|
87
|
+
estimate_eta: bool = True, estimate_delta: bool = True,
|
|
88
|
+
delta=(10.0, 10.0), pseudocount: float = 1e-20):
|
|
89
|
+
"""One variational-EM update of the DecontX model.
|
|
90
|
+
|
|
91
|
+
Port of the C++ ``decontXEM``. For every observed transcript the
|
|
92
|
+
variational native/contaminant responsibility is
|
|
93
|
+
|
|
94
|
+
``p_native = (phi[i,k] + pc) * (theta[j] + pc)``
|
|
95
|
+
``p_contam = (eta[i,k] + pc) * (1 - theta[j] + pc)``
|
|
96
|
+
``normp = p_native / (p_native + p_contam)``
|
|
97
|
+
|
|
98
|
+
(the non-log form -- exact for a two-component mixture and what the
|
|
99
|
+
C++ code uses). The native mass ``normp * x`` is accumulated into the
|
|
100
|
+
new ``phi`` by cluster; ``eta`` is the row-sum complement. ``theta``
|
|
101
|
+
is then the posterior mean of a Beta/Dirichlet with concentration
|
|
102
|
+
``delta``, which is itself re-estimated by :func:`fit_dirichlet`.
|
|
103
|
+
|
|
104
|
+
Returns a dict with the updated ``phi``, ``eta``, ``theta``,
|
|
105
|
+
``delta`` and the per-cell ``contamination`` fraction.
|
|
106
|
+
"""
|
|
107
|
+
counts = _as_csc(counts)
|
|
108
|
+
theta = np.asarray(theta, dtype=float)
|
|
109
|
+
counts_colsums = np.asarray(counts_colsums, dtype=float)
|
|
110
|
+
phi = np.asarray(phi, dtype=float)
|
|
111
|
+
eta = np.asarray(eta, dtype=float)
|
|
112
|
+
z = np.asarray(z, dtype=int)
|
|
113
|
+
delta = np.asarray(delta, dtype=float)
|
|
114
|
+
|
|
115
|
+
G, C = counts.shape
|
|
116
|
+
K = phi.shape[1]
|
|
117
|
+
|
|
118
|
+
new_phi = np.zeros((G, K), dtype=float)
|
|
119
|
+
native_total = np.zeros(C, dtype=float)
|
|
120
|
+
|
|
121
|
+
indptr, indices, data = counts.indptr, counts.indices, counts.data
|
|
122
|
+
for j in range(C):
|
|
123
|
+
k = z[j] - 1
|
|
124
|
+
start, end = indptr[j], indptr[j + 1]
|
|
125
|
+
if start == end:
|
|
126
|
+
continue
|
|
127
|
+
rows = indices[start:end]
|
|
128
|
+
x = data[start:end]
|
|
129
|
+
p_native = (phi[rows, k] + pseudocount) * (theta[j] + pseudocount)
|
|
130
|
+
p_contam = (eta[rows, k] + pseudocount) * (1.0 - theta[j] + pseudocount)
|
|
131
|
+
normp = p_native / (p_native + p_contam)
|
|
132
|
+
px = normp * x
|
|
133
|
+
np.add.at(new_phi[:, k], rows, px)
|
|
134
|
+
native_total[j] = px.sum()
|
|
135
|
+
|
|
136
|
+
if estimate_eta:
|
|
137
|
+
phi_rowsum = new_phi.sum(axis=1)
|
|
138
|
+
new_eta = phi_rowsum[:, None] - new_phi
|
|
139
|
+
else:
|
|
140
|
+
new_eta = eta
|
|
141
|
+
|
|
142
|
+
new_phi = new_phi / new_phi.sum(axis=0, keepdims=True)
|
|
143
|
+
if estimate_eta:
|
|
144
|
+
new_eta = new_eta / new_eta.sum(axis=0, keepdims=True)
|
|
145
|
+
|
|
146
|
+
# Update theta (and optionally its Dirichlet hyper-parameter delta).
|
|
147
|
+
contamination_prop = (counts_colsums - native_total) / counts_colsums
|
|
148
|
+
native_prop = 1.0 - contamination_prop
|
|
149
|
+
new_delta = delta
|
|
150
|
+
if estimate_delta:
|
|
151
|
+
theta_raw = np.column_stack([native_prop, contamination_prop])
|
|
152
|
+
new_delta = fit_dirichlet(theta_raw)["alpha"]
|
|
153
|
+
|
|
154
|
+
new_theta = (native_total + new_delta[0]) / (counts_colsums
|
|
155
|
+
+ np.sum(new_delta))
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
"phi": new_phi,
|
|
159
|
+
"eta": new_eta,
|
|
160
|
+
"theta": new_theta,
|
|
161
|
+
"delta": new_delta,
|
|
162
|
+
"contamination": contamination_prop,
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def decontx_loglik(counts, theta, eta, phi, z, pseudocount: float = 1e-20):
|
|
167
|
+
"""Two-component multinomial log-likelihood of the DecontX model.
|
|
168
|
+
|
|
169
|
+
Port of the C++ ``decontXLogLik``:
|
|
170
|
+
``ll = sum_{i,j} x_{ij} * log(phi*theta + eta*(1-theta) + pc)``.
|
|
171
|
+
"""
|
|
172
|
+
counts = _as_csc(counts)
|
|
173
|
+
theta = np.asarray(theta, dtype=float)
|
|
174
|
+
phi = np.asarray(phi, dtype=float)
|
|
175
|
+
eta = np.asarray(eta, dtype=float)
|
|
176
|
+
z = np.asarray(z, dtype=int)
|
|
177
|
+
|
|
178
|
+
loglik = 0.0
|
|
179
|
+
indptr, indices, data = counts.indptr, counts.indices, counts.data
|
|
180
|
+
for j in range(counts.shape[1]):
|
|
181
|
+
k = z[j] - 1
|
|
182
|
+
start, end = indptr[j], indptr[j + 1]
|
|
183
|
+
if start == end:
|
|
184
|
+
continue
|
|
185
|
+
rows = indices[start:end]
|
|
186
|
+
x = data[start:end]
|
|
187
|
+
mix = (phi[rows, k] * theta[j]
|
|
188
|
+
+ eta[rows, k] * (1.0 - theta[j]) + pseudocount)
|
|
189
|
+
loglik += float(np.sum(x * np.log(mix)))
|
|
190
|
+
return loglik
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def calculate_native_matrix(counts, theta, eta, phi, z,
|
|
194
|
+
pseudocount: float = 1e-20) -> sp.csc_matrix:
|
|
195
|
+
"""Return the decontaminated (native) count matrix.
|
|
196
|
+
|
|
197
|
+
Port of the C++ ``calculateNativeMatrix``: each observed entry is
|
|
198
|
+
scaled by its variational native responsibility ``normp``. Values
|
|
199
|
+
may be non-integer; round for integer counts.
|
|
200
|
+
"""
|
|
201
|
+
counts = _as_csc(counts).copy()
|
|
202
|
+
theta = np.asarray(theta, dtype=float)
|
|
203
|
+
phi = np.asarray(phi, dtype=float)
|
|
204
|
+
eta = np.asarray(eta, dtype=float)
|
|
205
|
+
z = np.asarray(z, dtype=int)
|
|
206
|
+
|
|
207
|
+
indptr, indices, data = counts.indptr, counts.indices, counts.data
|
|
208
|
+
out = data.copy()
|
|
209
|
+
for j in range(counts.shape[1]):
|
|
210
|
+
k = z[j] - 1
|
|
211
|
+
start, end = indptr[j], indptr[j + 1]
|
|
212
|
+
if start == end:
|
|
213
|
+
continue
|
|
214
|
+
rows = indices[start:end]
|
|
215
|
+
p_native = np.log(phi[rows, k] + pseudocount) + np.log(theta[j]
|
|
216
|
+
+ pseudocount)
|
|
217
|
+
p_contam = np.log(eta[rows, k] + pseudocount) + np.log(
|
|
218
|
+
1.0 - theta[j] + pseudocount)
|
|
219
|
+
normp = np.exp(p_native) / (np.exp(p_contam) + np.exp(p_native))
|
|
220
|
+
out[start:end] = data[start:end] * normp
|
|
221
|
+
|
|
222
|
+
native = sp.csc_matrix((out, indices.copy(), indptr.copy()),
|
|
223
|
+
shape=counts.shape)
|
|
224
|
+
return native
|