gotmap-benchmark 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gotmap_benchmark-0.2.0/PKG-INFO +130 -0
- gotmap_benchmark-0.2.0/README.md +106 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark/__init__.py +20 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark/cli.py +45 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark/datasets.py +81 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark/methods.py +315 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark/metrics.py +122 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark/pipeline.py +171 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark.egg-info/PKG-INFO +130 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark.egg-info/SOURCES.txt +14 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark.egg-info/dependency_links.txt +1 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark.egg-info/entry_points.txt +2 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark.egg-info/requires.txt +15 -0
- gotmap_benchmark-0.2.0/gotmap_benchmark.egg-info/top_level.txt +1 -0
- gotmap_benchmark-0.2.0/pyproject.toml +37 -0
- gotmap_benchmark-0.2.0/setup.cfg +4 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gotmap-benchmark
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Benchmark pipeline for gotmap (query-to-reference atlas mapping) vs baselines.
|
|
5
|
+
Author: gotmap developers
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ZengFLab/gotmap-benchmark
|
|
8
|
+
Requires-Python: >=3.9
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
Requires-Dist: numpy>=1.21
|
|
11
|
+
Requires-Dist: scipy>=1.8
|
|
12
|
+
Requires-Dist: pandas>=1.3
|
|
13
|
+
Requires-Dist: scikit-learn>=1.0
|
|
14
|
+
Requires-Dist: scanpy>=1.9
|
|
15
|
+
Requires-Dist: gotmap>=0.2.0
|
|
16
|
+
Requires-Dist: grassmann-ot>=0.3.1
|
|
17
|
+
Requires-Dist: matplotlib>=3.5
|
|
18
|
+
Requires-Dist: seaborn>=0.11
|
|
19
|
+
Provides-Extra: all
|
|
20
|
+
Requires-Dist: scvi-tools>=0.20; extra == "all"
|
|
21
|
+
Requires-Dist: harmonypy>=0.0.6; extra == "all"
|
|
22
|
+
Requires-Dist: bbknn>=1.5; extra == "all"
|
|
23
|
+
Requires-Dist: rpy2>=3.5; extra == "all"
|
|
24
|
+
|
|
25
|
+
# gotmap-benchmark
|
|
26
|
+
|
|
27
|
+
**Benchmark pipeline for gotmap** — compare query-to-reference atlas mapping methods
|
|
28
|
+
on real and synthetic single-cell data and get quantitative results + plots out of the box.
|
|
29
|
+
|
|
30
|
+
> Positioning: this benchmark evaluates **query → reference mapping quality**, not
|
|
31
|
+
> symmetric batch mixing. It is designed for the gotmap use case (map a query onto a
|
|
32
|
+
> fixed atlas), not to compete with integration-tool comparisons.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install gotmap grassmann-ot scanpy
|
|
40
|
+
# optional, for more baselines:
|
|
41
|
+
pip install scvi-tools harmonypy bbknn
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Quick start (Python)
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
import scanpy as sc
|
|
50
|
+
from gotmap_benchmark import BenchmarkPipeline
|
|
51
|
+
|
|
52
|
+
adata = sc.datasets.pbmc68k_reduced()
|
|
53
|
+
if "X_pca" not in adata.obsm:
|
|
54
|
+
sc.pp.normalize_total(adata); sc.pp.log1p(adata); sc.pp.pca(adata, n_comps=30)
|
|
55
|
+
|
|
56
|
+
label_key = "bulk_labels" if "bulk_labels" in adata.obs.columns else "louvain"
|
|
57
|
+
if label_key not in adata.obs.columns:
|
|
58
|
+
sc.tl.louvain(adata, resolution=0.8, key_added="louvain"); label_key = "louvain"
|
|
59
|
+
|
|
60
|
+
ref, query = sc.pp.train_test_split(adata, test_size=0.3, random_state=42,
|
|
61
|
+
stratify=label_key)
|
|
62
|
+
|
|
63
|
+
bench = BenchmarkPipeline(ref, query, label_key=label_key, use_rep="X_pca",
|
|
64
|
+
methods=["gotmap_nn", "gotmap_knn", "direct_knn"])
|
|
65
|
+
bench.run()
|
|
66
|
+
bench.save_results("benchmark_metrics.csv")
|
|
67
|
+
bench.plot(save="benchmark_plot.png")
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Quick start (CLI)
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
gotmap-benchmark --ref ref.h5ad --query query.h5ad --label cell_type \
|
|
74
|
+
--use-rep X_pca --methods gotmap_nn direct_knn \
|
|
75
|
+
--output ./results
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## Included methods
|
|
81
|
+
|
|
82
|
+
| Key | Method | Notes |
|
|
83
|
+
|-----|--------|-------|
|
|
84
|
+
| `gotmap_nn` | gotmap, neural-network mapping | uses anchors + gamma |
|
|
85
|
+
| `gotmap_knn` | gotmap, kNN mapping | non-parametric |
|
|
86
|
+
| `scArches` | scVI + scArches | needs scvi-tools |
|
|
87
|
+
| `harmony_project` | Harmony project | needs R harmony + rpy2 |
|
|
88
|
+
| `BBKNN_align` | BBKNN | symmetric, then extracts query |
|
|
89
|
+
| `direct_knn` | kNN (no correction) | **baseline** |
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## Metrics reported
|
|
94
|
+
|
|
95
|
+
- **Label Transfer Accuracy** / **Weighted F1** — ground-truth label recovery
|
|
96
|
+
- **kNN Preservation (kNP)** — overlap of reference-neighbourhoods before/after mapping
|
|
97
|
+
- **Cell-type Silhouette** — separation of types in mapped space
|
|
98
|
+
- **Mean Confidence** / **Mapping Rate** — placement certainty
|
|
99
|
+
- **Runtime (s)**
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## Output
|
|
104
|
+
|
|
105
|
+
- `benchmark_metrics.csv` — one row per method
|
|
106
|
+
- `benchmark_plot.png` — horizontal bar chart per metric
|
|
107
|
+
- `bench.results_` — full per-method outputs (mapped AnnData, labels, confidence)
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## Project layout
|
|
112
|
+
|
|
113
|
+
```
|
|
114
|
+
gotmap-benchmark/
|
|
115
|
+
├── pyproject.toml
|
|
116
|
+
├── README.md
|
|
117
|
+
├── gotmap_benchmark/
|
|
118
|
+
│ ├── __init__.py
|
|
119
|
+
│ ├── cli.py # command-line entry point
|
|
120
|
+
│ ├── pipeline.py # BenchmarkPipeline + plotting
|
|
121
|
+
│ ├── methods.py # method wrappers
|
|
122
|
+
│ ├── metrics.py # evaluation metrics
|
|
123
|
+
│ └── datasets.py # dataset loaders + ref/query splitting
|
|
124
|
+
└── examples/
|
|
125
|
+
└── run_pbmc_benchmark.py
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
License: MIT
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# gotmap-benchmark
|
|
2
|
+
|
|
3
|
+
**Benchmark pipeline for gotmap** — compare query-to-reference atlas mapping methods
|
|
4
|
+
on real and synthetic single-cell data and get quantitative results + plots out of the box.
|
|
5
|
+
|
|
6
|
+
> Positioning: this benchmark evaluates **query → reference mapping quality**, not
|
|
7
|
+
> symmetric batch mixing. It is designed for the gotmap use case (map a query onto a
|
|
8
|
+
> fixed atlas), not to compete with integration-tool comparisons.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install gotmap grassmann-ot scanpy
|
|
16
|
+
# optional, for more baselines:
|
|
17
|
+
pip install scvi-tools harmonypy bbknn
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## Quick start (Python)
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
import scanpy as sc
|
|
26
|
+
from gotmap_benchmark import BenchmarkPipeline
|
|
27
|
+
|
|
28
|
+
adata = sc.datasets.pbmc68k_reduced()
|
|
29
|
+
if "X_pca" not in adata.obsm:
|
|
30
|
+
sc.pp.normalize_total(adata); sc.pp.log1p(adata); sc.pp.pca(adata, n_comps=30)
|
|
31
|
+
|
|
32
|
+
label_key = "bulk_labels" if "bulk_labels" in adata.obs.columns else "louvain"
|
|
33
|
+
if label_key not in adata.obs.columns:
|
|
34
|
+
sc.tl.louvain(adata, resolution=0.8, key_added="louvain"); label_key = "louvain"
|
|
35
|
+
|
|
36
|
+
ref, query = sc.pp.train_test_split(adata, test_size=0.3, random_state=42,
|
|
37
|
+
stratify=label_key)
|
|
38
|
+
|
|
39
|
+
bench = BenchmarkPipeline(ref, query, label_key=label_key, use_rep="X_pca",
|
|
40
|
+
methods=["gotmap_nn", "gotmap_knn", "direct_knn"])
|
|
41
|
+
bench.run()
|
|
42
|
+
bench.save_results("benchmark_metrics.csv")
|
|
43
|
+
bench.plot(save="benchmark_plot.png")
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Quick start (CLI)
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
gotmap-benchmark --ref ref.h5ad --query query.h5ad --label cell_type \
|
|
50
|
+
--use-rep X_pca --methods gotmap_nn direct_knn \
|
|
51
|
+
--output ./results
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## Included methods
|
|
57
|
+
|
|
58
|
+
| Key | Method | Notes |
|
|
59
|
+
|-----|--------|-------|
|
|
60
|
+
| `gotmap_nn` | gotmap, neural-network mapping | uses anchors + gamma |
|
|
61
|
+
| `gotmap_knn` | gotmap, kNN mapping | non-parametric |
|
|
62
|
+
| `scArches` | scVI + scArches | needs scvi-tools |
|
|
63
|
+
| `harmony_project` | Harmony project | needs R harmony + rpy2 |
|
|
64
|
+
| `BBKNN_align` | BBKNN | symmetric, then extracts query |
|
|
65
|
+
| `direct_knn` | kNN (no correction) | **baseline** |
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## Metrics reported
|
|
70
|
+
|
|
71
|
+
- **Label Transfer Accuracy** / **Weighted F1** — ground-truth label recovery
|
|
72
|
+
- **kNN Preservation (kNP)** — overlap of reference-neighbourhoods before/after mapping
|
|
73
|
+
- **Cell-type Silhouette** — separation of types in mapped space
|
|
74
|
+
- **Mean Confidence** / **Mapping Rate** — placement certainty
|
|
75
|
+
- **Runtime (s)**
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
## Output
|
|
80
|
+
|
|
81
|
+
- `benchmark_metrics.csv` — one row per method
|
|
82
|
+
- `benchmark_plot.png` — horizontal bar chart per metric
|
|
83
|
+
- `bench.results_` — full per-method outputs (mapped AnnData, labels, confidence)
|
|
84
|
+
|
|
85
|
+
---
|
|
86
|
+
|
|
87
|
+
## Project layout
|
|
88
|
+
|
|
89
|
+
```
|
|
90
|
+
gotmap-benchmark/
|
|
91
|
+
├── pyproject.toml
|
|
92
|
+
├── README.md
|
|
93
|
+
├── gotmap_benchmark/
|
|
94
|
+
│ ├── __init__.py
|
|
95
|
+
│ ├── cli.py # command-line entry point
|
|
96
|
+
│ ├── pipeline.py # BenchmarkPipeline + plotting
|
|
97
|
+
│ ├── methods.py # method wrappers
|
|
98
|
+
│ ├── metrics.py # evaluation metrics
|
|
99
|
+
│ └── datasets.py # dataset loaders + ref/query splitting
|
|
100
|
+
└── examples/
|
|
101
|
+
└── run_pbmc_benchmark.py
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
---
|
|
105
|
+
|
|
106
|
+
License: MIT
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""gotmap-benchmark: benchmark pipeline for query-to-reference atlas mapping."""
|
|
2
|
+
|
|
3
|
+
from .pipeline import BenchmarkPipeline, run_benchmark
|
|
4
|
+
from .methods import METHODS, run_gotmap_nn, run_gotmap_knn
|
|
5
|
+
from .metrics import compute_knn_preservation, compute_mapping_confidence
|
|
6
|
+
from .datasets import load_dataset, split_reference_query
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"BenchmarkPipeline",
|
|
10
|
+
"run_benchmark",
|
|
11
|
+
"METHODS",
|
|
12
|
+
"run_gotmap_nn",
|
|
13
|
+
"run_gotmap_knn",
|
|
14
|
+
"compute_knn_preservation",
|
|
15
|
+
"compute_mapping_confidence",
|
|
16
|
+
"load_dataset",
|
|
17
|
+
"split_reference_query",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
__version__ = "0.2.0"
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Command-line interface for gotmap-benchmark."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import os
|
|
7
|
+
import scanpy as sc
|
|
8
|
+
from .pipeline import BenchmarkPipeline
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def main():
|
|
12
|
+
p = argparse.ArgumentParser(
|
|
13
|
+
description="gotmap benchmark: compare query-to-reference mapping methods.")
|
|
14
|
+
p.add_argument("--ref", required=True, help="Path to reference .h5ad")
|
|
15
|
+
p.add_argument("--query", required=True, help="Path to query .h5ad")
|
|
16
|
+
p.add_argument("--label", default=None, help="obs column with ground-truth labels")
|
|
17
|
+
p.add_argument("--use-rep", default="X_pca", help="obsm key (e.g. X_pca, X_scVI)")
|
|
18
|
+
p.add_argument("--methods", nargs="*", default=None,
|
|
19
|
+
help="Subset of methods to run")
|
|
20
|
+
p.add_argument("--output", default="./benchmark_results",
|
|
21
|
+
help="Output directory")
|
|
22
|
+
p.add_argument("--k", type=int, default=15, help="k for kNN preservation")
|
|
23
|
+
p.add_argument("--quiet", action="store_true", help="Suppress per-method output")
|
|
24
|
+
args = p.parse_args()
|
|
25
|
+
|
|
26
|
+
os.makedirs(args.output, exist_ok=True)
|
|
27
|
+
ref = sc.read_h5ad(args.ref)
|
|
28
|
+
query = sc.read_h5ad(args.query)
|
|
29
|
+
|
|
30
|
+
bench = BenchmarkPipeline(ref, query, label_key=args.label, use_rep=args.use_rep,
|
|
31
|
+
methods=args.methods)
|
|
32
|
+
bench.run(k_neighbors=args.k, verbose=not args.quiet)
|
|
33
|
+
|
|
34
|
+
if not bench.metrics_.empty:
|
|
35
|
+
bench.save_results(os.path.join(args.output, "benchmark_metrics.csv"))
|
|
36
|
+
bench.plot(save=os.path.join(args.output, "benchmark_plot.png"))
|
|
37
|
+
print("\nFinal metrics:\n", bench.metrics_.to_string(index=False))
|
|
38
|
+
if bench.errors_:
|
|
39
|
+
print("\nFailed methods:")
|
|
40
|
+
for m, e in bench.errors_.items():
|
|
41
|
+
print(f" - {m}: {e}")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
if __name__ == "__main__":
|
|
45
|
+
main()
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Dataset loaders and reference/query splitting utilities."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import scanpy as sc
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def load_dataset(name, **kwargs):
|
|
9
|
+
"""
|
|
10
|
+
Load a built-in dataset by name.
|
|
11
|
+
|
|
12
|
+
Supported:
|
|
13
|
+
'pbmc68k' -> scanpy.datasets.pbmc68k_reduced
|
|
14
|
+
'paul15' -> scanpy.datasets.paul15
|
|
15
|
+
'blobs' -> synthetic Gaussian blobs (multi-batch, with labels)
|
|
16
|
+
|
|
17
|
+
Returns an AnnData ready for split_reference_query().
|
|
18
|
+
"""
|
|
19
|
+
if name == "pbmc68k":
|
|
20
|
+
return sc.datasets.pbmc68k_reduced()
|
|
21
|
+
if name == "paul15":
|
|
22
|
+
return sc.datasets.paul15()
|
|
23
|
+
if name == "blobs":
|
|
24
|
+
return _make_synthetic_blobs(**kwargs)
|
|
25
|
+
raise ValueError(f"Unknown dataset '{name}'. Available: pbmc68k, paul15, blobs")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _make_synthetic_blobs(n_per_blob=300, n_blobs=5, n_genes=200, seed=0):
|
|
29
|
+
"""Synthetic multi-cluster, multi-batch data for smoke tests."""
|
|
30
|
+
import numpy as np
|
|
31
|
+
rng = np.random.default_rng(seed)
|
|
32
|
+
from scipy.stats import norm
|
|
33
|
+
|
|
34
|
+
X, labs, batches = [], [], []
|
|
35
|
+
for b in range(2): # two batches
|
|
36
|
+
for c in range(n_blobs):
|
|
37
|
+
centre = rng.normal(0, 5, n_genes)
|
|
38
|
+
# batch effect: shift batch 1
|
|
39
|
+
shift = 3.0 if b == 1 else 0.0
|
|
40
|
+
expr = centre + norm.rvs(loc=shift, scale=1.5, size=(n_per_blob, n_genes), random_state=rng)
|
|
41
|
+
X.append(expr)
|
|
42
|
+
labs.extend([f"cluster_{c}"] * n_per_blob)
|
|
43
|
+
batches.extend([f"batch_{b}"] * n_per_blob)
|
|
44
|
+
import anndata
|
|
45
|
+
X = np.vstack(X)
|
|
46
|
+
# ensure no zeros after normalisation -> no NaNs in log
|
|
47
|
+
X = np.clip(X, 1e-3, None)
|
|
48
|
+
adata = anndata.AnnData(X)
|
|
49
|
+
adata.obs["cell_type"] = labs
|
|
50
|
+
adata.obs["batch"] = batches
|
|
51
|
+
sc.pp.normalize_total(adata)
|
|
52
|
+
sc.pp.log1p(adata)
|
|
53
|
+
sc.pp.pca(adata, n_comps=min(20, X.shape[1] - 1))
|
|
54
|
+
return adata
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def split_reference_query(adata, label_key="cell_type", batch_key="batch",
|
|
58
|
+
ref_batch=None, query_batch=None, stratify=True,
|
|
59
|
+
seed=42):
|
|
60
|
+
"""
|
|
61
|
+
Split an AnnData into a reference atlas and a query.
|
|
62
|
+
|
|
63
|
+
If ref_batch / query_batch are given, split by that categorical.
|
|
64
|
+
Otherwise split by cells (stratified by label so both sides have all types).
|
|
65
|
+
"""
|
|
66
|
+
if batch_key in adata.obs.columns and ref_batch and query_batch:
|
|
67
|
+
ref = adata[adata.obs[batch_key] == ref_batch].copy()
|
|
68
|
+
query = adata[adata.obs[batch_key] == query_batch].copy()
|
|
69
|
+
else:
|
|
70
|
+
from sklearn.model_selection import train_test_split
|
|
71
|
+
idx = np.arange(adata.n_obs)
|
|
72
|
+
if stratify and label_key in adata.obs.columns:
|
|
73
|
+
train_idx, test_idx = train_test_split(
|
|
74
|
+
idx, test_size=0.3, random_state=seed,
|
|
75
|
+
stratify=adata.obs[label_key])
|
|
76
|
+
else:
|
|
77
|
+
train_idx, test_idx = train_test_split(idx, test_size=0.3, random_state=seed)
|
|
78
|
+
ref = adata[train_idx].copy()
|
|
79
|
+
query = adata[test_idx].copy()
|
|
80
|
+
|
|
81
|
+
return ref, query
|
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
"""Wrappers for each mapping method.
|
|
2
|
+
|
|
3
|
+
Every wrapper returns a dict with at least:
|
|
4
|
+
- 'query_mapped': AnnData with obsm['X_mapped'] (n_query x n_dims)
|
|
5
|
+
- 'method': str
|
|
6
|
+
Optional keys:
|
|
7
|
+
- 'predicted_labels': pd.Series or np.ndarray aligned to query
|
|
8
|
+
- 'confidence': np.ndarray aligned to query
|
|
9
|
+
- 'info': dict with extra metadata (model, runtime subtotals, etc.)
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import time
|
|
15
|
+
import warnings
|
|
16
|
+
import numpy as np
|
|
17
|
+
import pandas as pd
|
|
18
|
+
import scanpy as sc
|
|
19
|
+
from sklearn.neighbors import KNeighborsClassifier, NearestNeighbors
|
|
20
|
+
|
|
21
|
+
warnings.filterwarnings("ignore")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# ---------------------------------------------------------------------------
|
|
25
|
+
# gotmap
|
|
26
|
+
# ---------------------------------------------------------------------------
|
|
27
|
+
|
|
28
|
+
_GOTMAP_LOADED = {"GOTMap": None}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _import_gotmap():
|
|
32
|
+
"""Import gotmap.GOTMap robustly, even if sys.path[0] shadows the package."""
|
|
33
|
+
if _GOTMAP_LOADED["GOTMap"] is not None:
|
|
34
|
+
return _GOTMAP_LOADED["GOTMap"]
|
|
35
|
+
|
|
36
|
+
import importlib, importlib.util, runpy, sys, os
|
|
37
|
+
|
|
38
|
+
# 1. Try the normal import first
|
|
39
|
+
try:
|
|
40
|
+
mod = importlib.import_module("gotmap")
|
|
41
|
+
if hasattr(mod, "GOTMap"):
|
|
42
|
+
_GOTMAP_LOADED["GOTMap"] = mod.GOTMap
|
|
43
|
+
return mod.GOTMap
|
|
44
|
+
except Exception:
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
# 2. Locate the real 'gotmap' package on disk
|
|
48
|
+
search_dirs = [os.getcwd()] + sys.path
|
|
49
|
+
init_path = None
|
|
50
|
+
for d in search_dirs:
|
|
51
|
+
if not d or not os.path.isdir(d):
|
|
52
|
+
continue
|
|
53
|
+
cand = os.path.join(d, "gotmap", "__init__.py")
|
|
54
|
+
if os.path.exists(cand):
|
|
55
|
+
init_path = cand
|
|
56
|
+
break
|
|
57
|
+
if init_path is None:
|
|
58
|
+
# also search one level up (e.g. gotmap_pkg/gotmap/__init__.py)
|
|
59
|
+
for d in search_dirs:
|
|
60
|
+
if not d or not os.path.isdir(d):
|
|
61
|
+
continue
|
|
62
|
+
for sub in os.listdir(d):
|
|
63
|
+
subdir = os.path.join(d, sub, "gotmap", "__init__.py")
|
|
64
|
+
if os.path.exists(subdir):
|
|
65
|
+
init_path = subdir
|
|
66
|
+
break
|
|
67
|
+
if init_path:
|
|
68
|
+
break
|
|
69
|
+
|
|
70
|
+
if init_path is None:
|
|
71
|
+
raise ImportError(
|
|
72
|
+
"gotmap is not importable and could not be located on disk. "
|
|
73
|
+
"Install it with 'pip install gotmap'.")
|
|
74
|
+
|
|
75
|
+
pkg_dir = os.path.dirname(init_path)
|
|
76
|
+
|
|
77
|
+
# 3. Load every submodule under that package via runpy so that
|
|
78
|
+
# 'from gotmap import ...' inside them resolves correctly.
|
|
79
|
+
sys.path.insert(0, os.path.dirname(pkg_dir)) # make 'gotmap' importable
|
|
80
|
+
try:
|
|
81
|
+
if "gotmap" not in sys.modules:
|
|
82
|
+
pkg = importlib.import_module("gotmap")
|
|
83
|
+
else:
|
|
84
|
+
pkg = sys.modules["gotmap"]
|
|
85
|
+
# force-load submodules so GOTMap is available
|
|
86
|
+
for fname in os.listdir(pkg_dir):
|
|
87
|
+
if fname.endswith(".py") and not fname.startswith("_"):
|
|
88
|
+
modname = "gotmap." + fname[:-3]
|
|
89
|
+
if modname not in sys.modules:
|
|
90
|
+
try:
|
|
91
|
+
importlib.import_module(modname)
|
|
92
|
+
except Exception:
|
|
93
|
+
pass
|
|
94
|
+
if hasattr(pkg, "GOTMap"):
|
|
95
|
+
_GOTMAP_LOADED["GOTMap"] = pkg.GOTMap
|
|
96
|
+
return pkg.GOTMap
|
|
97
|
+
finally:
|
|
98
|
+
# remove the injected path to avoid further pollution
|
|
99
|
+
try:
|
|
100
|
+
sys.path.remove(os.path.dirname(pkg_dir))
|
|
101
|
+
except ValueError:
|
|
102
|
+
pass
|
|
103
|
+
|
|
104
|
+
raise ImportError("gotmap package was found but GOTMap could not be imported.")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def run_gotmap_nn(ref, query, use_rep="X_pca", label_key=None,
|
|
108
|
+
k=30, p=3, reg=0.1, backend="batched",
|
|
109
|
+
hidden_dim=128, epochs=200, **kwargs):
|
|
110
|
+
# torch is only needed for the NN mapper; fail fast with a clear hint
|
|
111
|
+
try:
|
|
112
|
+
import torch # noqa: F401
|
|
113
|
+
except ImportError:
|
|
114
|
+
raise ImportError(
|
|
115
|
+
"gotmap_nn requires PyTorch. Install it with "
|
|
116
|
+
"`pip install torch` (CPU build is sufficient) to use this method."
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
GOTMap = _import_gotmap()
|
|
120
|
+
|
|
121
|
+
# gotmap>=0.2.0 uses the scanpy-consistent name 'use_rep'
|
|
122
|
+
mapper = GOTMap(ref_adata=ref, use_rep=use_rep, k=k, p=p, reg=reg, backend=backend)
|
|
123
|
+
mapper.fit(query)
|
|
124
|
+
|
|
125
|
+
projected = mapper.transform(query, method="nn", hidden_dim=hidden_dim, epochs=epochs)
|
|
126
|
+
out = {
|
|
127
|
+
"query_mapped": projected,
|
|
128
|
+
"method": "gotmap_nn",
|
|
129
|
+
"info": {"anchors": len(mapper.anchors_) if mapper.anchors_ is not None else 0},
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if label_key and label_key in ref.obs.columns:
|
|
133
|
+
labelled = mapper.transfer_labels(query, label_key=label_key)
|
|
134
|
+
key_p = f"{label_key}_predicted"
|
|
135
|
+
key_c = f"{label_key}_confidence"
|
|
136
|
+
if key_p in labelled.obs.columns:
|
|
137
|
+
out["predicted_labels"] = labelled.obs[key_p]
|
|
138
|
+
if key_c in labelled.obs.columns:
|
|
139
|
+
out["confidence"] = labelled.obs[key_c].to_numpy(float)
|
|
140
|
+
|
|
141
|
+
return out
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def run_gotmap_knn(ref, query, use_rep="X_pca", label_key=None,
|
|
145
|
+
k=30, p=3, reg=0.1, backend="batched",
|
|
146
|
+
n_neighbors=15, **kwargs):
|
|
147
|
+
GOTMap = _import_gotmap()
|
|
148
|
+
|
|
149
|
+
mapper = GOTMap(ref_adata=ref, use_rep=use_rep, k=k, p=p, reg=reg, backend=backend)
|
|
150
|
+
mapper.fit(query)
|
|
151
|
+
projected = mapper.transform(query, method="knn", n_neighbors=n_neighbors)
|
|
152
|
+
# mapper.transform(method='knn') returns an AnnData with obsm['X_gotmap']
|
|
153
|
+
projected.obsm["X_mapped"] = projected.obsm["X_gotmap"]
|
|
154
|
+
|
|
155
|
+
out = {"query_mapped": projected, "method": "gotmap_knn"}
|
|
156
|
+
|
|
157
|
+
if label_key and label_key in ref.obs.columns:
|
|
158
|
+
# use the mapper's transfer_labels (accepts label_key, not use_rep)
|
|
159
|
+
labelled = mapper.transfer_labels(query, label_key=label_key, k=k)
|
|
160
|
+
key_p = f"{label_key}_predicted"
|
|
161
|
+
key_c = f"{label_key}_confidence"
|
|
162
|
+
if key_p in labelled.obs.columns:
|
|
163
|
+
out["predicted_labels"] = labelled.obs[key_p]
|
|
164
|
+
if key_c in labelled.obs.columns:
|
|
165
|
+
out["confidence"] = labelled.obs[key_c].to_numpy(float)
|
|
166
|
+
|
|
167
|
+
return out
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# ---------------------------------------------------------------------------
|
|
171
|
+
# scArches (scVI)
|
|
172
|
+
# ---------------------------------------------------------------------------
|
|
173
|
+
|
|
174
|
+
def run_scarches(ref, query, use_rep="X_pca", label_key=None,
|
|
175
|
+
model_dir=None, max_epochs=100, n_latent=10, **kwargs):
|
|
176
|
+
"""Map query into a pretrained scVI latent space via scArches."""
|
|
177
|
+
import scvi
|
|
178
|
+
|
|
179
|
+
ref_work = ref.copy()
|
|
180
|
+
if "batch" not in ref_work.obs.columns:
|
|
181
|
+
ref_work.obs["batch"] = "reference"
|
|
182
|
+
|
|
183
|
+
if model_dir is None:
|
|
184
|
+
# 无预训练模型时,用 reference 现场训练一个(首次运行)
|
|
185
|
+
model_dir = kwargs.get("scvi_model_dir", "./_scarches_tmp_model")
|
|
186
|
+
|
|
187
|
+
import os
|
|
188
|
+
t0 = time.time()
|
|
189
|
+
if os.path.isdir(model_dir) and any("model.pt" in f or f.endswith(".pt")
|
|
190
|
+
for r, d, files in os.walk(model_dir) for f in files):
|
|
191
|
+
model = scvi.model.SCVI.load(model_dir, adata=ref_work)
|
|
192
|
+
else:
|
|
193
|
+
scvi.model.SCVI.setup_anndata(ref_work, batch_key="batch")
|
|
194
|
+
model = scvi.model.SCVI(ref_work, n_latent=n_latent)
|
|
195
|
+
model.train(max_epochs=max_epochs)
|
|
196
|
+
model.save(model_dir, overwrite=True)
|
|
197
|
+
|
|
198
|
+
query_work = query.copy()
|
|
199
|
+
query_work.obs["batch"] = "query"
|
|
200
|
+
query_model = model.load_query_data(query_work, inplace=False)
|
|
201
|
+
query_model.train(max_epochs=max_epochs)
|
|
202
|
+
|
|
203
|
+
latent = query_model.get_latent(query_work)
|
|
204
|
+
mapped = query.copy()
|
|
205
|
+
mapped.obsm["X_mapped"] = latent
|
|
206
|
+
|
|
207
|
+
out = {"query_mapped": mapped, "method": "scArches",
|
|
208
|
+
"info": {"scvi_train_seconds": time.time() - t0}}
|
|
209
|
+
|
|
210
|
+
if label_key and label_key in ref.obs.columns:
|
|
211
|
+
knn = KNeighborsClassifier(n_neighbors=15, weights="distance")
|
|
212
|
+
knn.fit(ref_work.obsm.get("X_scVI", ref_work.obsm[use_rep]),
|
|
213
|
+
ref_work.obs[label_key])
|
|
214
|
+
out["predicted_labels"] = pd.Series(
|
|
215
|
+
knn.predict(latent), index=query.obs_names)
|
|
216
|
+
|
|
217
|
+
return out
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
# ---------------------------------------------------------------------------
|
|
221
|
+
# Harmony project (rpy2-based, with pure-Python fallback)
|
|
222
|
+
# ---------------------------------------------------------------------------
|
|
223
|
+
|
|
224
|
+
def run_harmony_project(ref, query, use_rep="X_pca", label_key=None, **kwargs):
|
|
225
|
+
try:
|
|
226
|
+
import harmonypy # noqa: F401
|
|
227
|
+
from rpy2.robjects import numpy2ri, pandas2ri
|
|
228
|
+
from rpy2.robjects.packages import importr
|
|
229
|
+
|
|
230
|
+
numpy2ri.activate()
|
|
231
|
+
pandas2ri.activate()
|
|
232
|
+
harmony = importr("harmony")
|
|
233
|
+
|
|
234
|
+
batch = np.array(["ref"] * ref.n_obs + ["query"] * query.n_obs)
|
|
235
|
+
mat = np.vstack([ref.obsm[use_rep], query.obsm[use_rep]]).T
|
|
236
|
+
|
|
237
|
+
res = harmony.run_harmony(mat, batch, "batch")
|
|
238
|
+
# harmony object: first element is the corrected matrix (genes x cells)
|
|
239
|
+
corrected = np.asarray(res[0]).T
|
|
240
|
+
mapped = query.copy()
|
|
241
|
+
mapped.obsm["X_mapped"] = corrected[ref.n_obs:]
|
|
242
|
+
return {"query_mapped": mapped, "method": "harmony_project"}
|
|
243
|
+
except Exception as e:
|
|
244
|
+
raise ImportError(
|
|
245
|
+
"Harmony project requires R package 'harmony' + rpy2. "
|
|
246
|
+
f"Original error: {e}"
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
# ---------------------------------------------------------------------------
|
|
251
|
+
# BBKNN align (symmetric, then extract query -> ref mapping)
|
|
252
|
+
# ---------------------------------------------------------------------------
|
|
253
|
+
|
|
254
|
+
def run_bbknn_align(ref, query, use_rep="X_pca", label_key=None, **kwargs):
|
|
255
|
+
import bbknn
|
|
256
|
+
|
|
257
|
+
combined = ref.copy()
|
|
258
|
+
combined.obs["_batch_bench"] = "ref"
|
|
259
|
+
q = query.copy()
|
|
260
|
+
q.obs["_batch_bench"] = "query"
|
|
261
|
+
|
|
262
|
+
together = combined.concatenate(q, batch_key="_batch_bench", index_unique=None)
|
|
263
|
+
sc.pp.pca(together, n_comps=kwargs.get("n_comps", 20))
|
|
264
|
+
bbknn.bbknn(together, batch_key="_batch_bench", use_rep="X_pca")
|
|
265
|
+
|
|
266
|
+
mapped = query.copy()
|
|
267
|
+
# 用 PCA 坐标作为映射空间(bbknn 主要输出图,这里给一个可比坐标)
|
|
268
|
+
start = combined.n_obs
|
|
269
|
+
mapped.obsm["X_mapped"] = together.obsm["X_pca"][start:].copy()
|
|
270
|
+
|
|
271
|
+
out = {"query_mapped": mapped, "method": "BBKNN_align"}
|
|
272
|
+
if label_key and label_key in ref.obs.columns:
|
|
273
|
+
knn = KNeighborsClassifier(n_neighbors=15, weights="distance")
|
|
274
|
+
knn.fit(ref.obsm[use_rep], ref.obs[label_key])
|
|
275
|
+
out["predicted_labels"] = pd.Series(
|
|
276
|
+
knn.predict(mapped.obsm["X_mapped"]), index=query.obs_names)
|
|
277
|
+
return out
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
# ---------------------------------------------------------------------------
|
|
281
|
+
# Direct kNN baseline
|
|
282
|
+
# ---------------------------------------------------------------------------
|
|
283
|
+
|
|
284
|
+
def run_direct_knn(ref, query, use_rep="X_pca", label_key=None,
|
|
285
|
+
n_neighbors=15, **kwargs):
|
|
286
|
+
"""Baseline: directly kNN-query reference, no batch correction."""
|
|
287
|
+
nbrs = NearestNeighbors(n_neighbors=n_neighbors, metric="cosine").fit(ref.obsm[use_rep])
|
|
288
|
+
dist, idx = nbrs.kneighbors(query.obsm[use_rep])
|
|
289
|
+
weights = 1.0 / (dist + 1e-8)
|
|
290
|
+
weights = weights / weights.sum(axis=1, keepdims=True)
|
|
291
|
+
|
|
292
|
+
mapped_coords = (weights[:, :, None] * ref.obsm[use_rep][idx]).sum(axis=1)
|
|
293
|
+
mapped = query.copy()
|
|
294
|
+
mapped.obsm["X_mapped"] = mapped_coords
|
|
295
|
+
|
|
296
|
+
out = {"query_mapped": mapped, "method": "direct_knn"}
|
|
297
|
+
if label_key and label_key in ref.obs.columns:
|
|
298
|
+
labs = ref.obs[label_key].to_numpy()
|
|
299
|
+
pred = labs[idx[np.arange(len(query)), weights.argmax(axis=1)]]
|
|
300
|
+
out["predicted_labels"] = pd.Series(pred, index=query.obs_names)
|
|
301
|
+
return out
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
# ---------------------------------------------------------------------------
|
|
305
|
+
# Registry
|
|
306
|
+
# ---------------------------------------------------------------------------
|
|
307
|
+
|
|
308
|
+
METHODS = {
|
|
309
|
+
"gotmap_nn": run_gotmap_nn,
|
|
310
|
+
"gotmap_knn": run_gotmap_knn,
|
|
311
|
+
"scArches": run_scarches,
|
|
312
|
+
"harmony_project": run_harmony_project,
|
|
313
|
+
"BBKNN_align": run_bbknn_align,
|
|
314
|
+
"direct_knn": run_direct_knn,
|
|
315
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Benchmark metrics for query-to-reference mapping quality."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from sklearn.metrics import accuracy_score, f1_score, silhouette_score
|
|
7
|
+
from sklearn.neighbors import NearestNeighbors
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def compute_knn_preservation(orig_query, mapped_query, ref_coords, k=15, metric="cosine"):
|
|
11
|
+
"""
|
|
12
|
+
kNN preservation: Jaccard overlap between the kNN a query cell has
|
|
13
|
+
in *mapped* space (among reference cells) and the kNN it had in its
|
|
14
|
+
*original* (uncorrected) space (also among reference cells).
|
|
15
|
+
|
|
16
|
+
A value of 1.0 means the mapping perfectly preserves the reference
|
|
17
|
+
neighbourhood structure of every query cell.
|
|
18
|
+
"""
|
|
19
|
+
nbrs = NearestNeighbors(n_neighbors=k, metric=metric).fit(ref_coords)
|
|
20
|
+
nn_mapped = nbrs.kneighbors(mapped_query, return_distance=False)
|
|
21
|
+
nn_orig = nbrs.kneighbors(orig_query, return_distance=False)
|
|
22
|
+
|
|
23
|
+
overlaps = []
|
|
24
|
+
for i in range(len(mapped_query)):
|
|
25
|
+
s1 = set(nn_mapped[i])
|
|
26
|
+
s2 = set(nn_orig[i])
|
|
27
|
+
denom = len(s1 | s2)
|
|
28
|
+
overlaps.append(len(s1 & s2) / denom if denom > 0 else 0.0)
|
|
29
|
+
return float(np.mean(overlaps))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def compute_mapping_confidence(query_mapped, ref_coords, k=15, metric="cosine"):
|
|
33
|
+
"""
|
|
34
|
+
Mean cosine similarity of each mapped query cell to its kNN in reference.
|
|
35
|
+
Higher = more certain placement.
|
|
36
|
+
"""
|
|
37
|
+
nbrs = NearestNeighbors(n_neighbors=k, metric=metric).fit(ref_coords)
|
|
38
|
+
dist, _ = nbrs.kneighbors(query_mapped)
|
|
39
|
+
sim = np.clip(1.0 - dist, -1.0, 1.0)
|
|
40
|
+
return float(sim.mean()), float((sim.mean(axis=1) > 0.5).mean())
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def evaluate_mapping(ref, query, result, use_rep="X_pca", label_key=None,
|
|
44
|
+
k=15, verbose=False):
|
|
45
|
+
"""
|
|
46
|
+
Compute the full metric suite for one method result.
|
|
47
|
+
|
|
48
|
+
Parameters
|
|
49
|
+
----------
|
|
50
|
+
ref : AnnData
|
|
51
|
+
Reference atlas.
|
|
52
|
+
query : AnnData
|
|
53
|
+
Query data (same n_obs order as result['query_mapped']).
|
|
54
|
+
result : dict
|
|
55
|
+
Output of a run_* function from methods.py.
|
|
56
|
+
use_rep : str
|
|
57
|
+
obsm key holding the *original* query representation.
|
|
58
|
+
label_key : str or None
|
|
59
|
+
If set, label-transfer metrics are computed.
|
|
60
|
+
"""
|
|
61
|
+
mapped = result["query_mapped"]
|
|
62
|
+
mapped_coords = mapped.obsm["X_mapped"]
|
|
63
|
+
ref_coords = ref.obsm[use_rep]
|
|
64
|
+
orig_query = query.obsm[use_rep][mapped.obs_names.argsort()]
|
|
65
|
+
|
|
66
|
+
metrics = {}
|
|
67
|
+
|
|
68
|
+
# 1. Label transfer
|
|
69
|
+
if label_key and label_key in ref.obs.columns and "predicted_labels" in result:
|
|
70
|
+
y_true = query.obs[label_key].reindex(mapped.obs_names).to_numpy(str)
|
|
71
|
+
y_pred = np.asarray(result["predicted_labels"]).astype(str)
|
|
72
|
+
if len(y_true) == len(y_pred):
|
|
73
|
+
metrics["label_transfer_accuracy"] = float(accuracy_score(y_true, y_pred))
|
|
74
|
+
metrics["weighted_f1"] = float(f1_score(y_true, y_pred,
|
|
75
|
+
average="weighted", zero_division=0))
|
|
76
|
+
else:
|
|
77
|
+
metrics["label_transfer_accuracy"] = np.nan
|
|
78
|
+
metrics["weighted_f1"] = np.nan
|
|
79
|
+
else:
|
|
80
|
+
metrics["label_transfer_accuracy"] = np.nan
|
|
81
|
+
metrics["weighted_f1"] = np.nan
|
|
82
|
+
|
|
83
|
+
# 2. kNN preservation
|
|
84
|
+
try:
|
|
85
|
+
metrics["knn_preservation"] = compute_knn_preservation(
|
|
86
|
+
query.obsm[use_rep], mapped_coords, ref_coords, k=k)
|
|
87
|
+
except Exception as e:
|
|
88
|
+
if verbose:
|
|
89
|
+
print(f" kNN preservation failed: {e}")
|
|
90
|
+
metrics["knn_preservation"] = np.nan
|
|
91
|
+
|
|
92
|
+
# 3. Cell-type silhouette in mapped space
|
|
93
|
+
if label_key and label_key in query.obs.columns:
|
|
94
|
+
labs = query.obs[label_key].reindex(mapped.obs_names).to_numpy(str)
|
|
95
|
+
if len(np.unique(labs)) > 1 and len(labs) <= 20000:
|
|
96
|
+
sample = min(2000, len(labs))
|
|
97
|
+
idx = np.random.RandomState(42).choice(len(labs), sample, replace=False)
|
|
98
|
+
try:
|
|
99
|
+
metrics["cell_type_silhouette"] = float(
|
|
100
|
+
silhouette_score(mapped_coords[idx], labs[idx]))
|
|
101
|
+
except Exception:
|
|
102
|
+
metrics["cell_type_silhouette"] = np.nan
|
|
103
|
+
else:
|
|
104
|
+
metrics["cell_type_silhouette"] = np.nan
|
|
105
|
+
else:
|
|
106
|
+
metrics["cell_type_silhouette"] = np.nan
|
|
107
|
+
|
|
108
|
+
# 4. Confidence
|
|
109
|
+
if "confidence" in result:
|
|
110
|
+
conf = np.asarray(result["confidence"], dtype=float)
|
|
111
|
+
metrics["mean_confidence"] = float(np.nanmean(conf))
|
|
112
|
+
metrics["mapping_rate"] = float(np.nanmean(conf > 0.5))
|
|
113
|
+
else:
|
|
114
|
+
mean_c, rate = compute_mapping_confidence(mapped_coords, ref_coords, k=k)
|
|
115
|
+
metrics["mean_confidence"] = mean_c
|
|
116
|
+
metrics["mapping_rate"] = rate
|
|
117
|
+
|
|
118
|
+
if verbose:
|
|
119
|
+
print(" metrics:", {k_: round(v_, 3) if isinstance(v_, float) else v_
|
|
120
|
+
for k_, v_ in metrics.items()})
|
|
121
|
+
|
|
122
|
+
return metrics
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""End-to-end benchmark pipeline."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
import traceback
|
|
7
|
+
import warnings
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
import scanpy as sc
|
|
11
|
+
import matplotlib
|
|
12
|
+
matplotlib.use("Agg")
|
|
13
|
+
import matplotlib.pyplot as plt
|
|
14
|
+
import seaborn as sns
|
|
15
|
+
|
|
16
|
+
from .methods import METHODS
|
|
17
|
+
from .metrics import evaluate_mapping
|
|
18
|
+
|
|
19
|
+
warnings.filterwarnings("ignore")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class BenchmarkPipeline:
|
|
23
|
+
"""
|
|
24
|
+
Benchmark query-to-reference mapping methods.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
ref_adata : AnnData
|
|
29
|
+
Reference atlas (already preprocessed, with obsm[use_rep]).
|
|
30
|
+
query_adata : AnnData
|
|
31
|
+
Query data (same genes / representation convention as ref).
|
|
32
|
+
label_key : str or None
|
|
33
|
+
Column in .obs holding the cell-type / ground-truth labels.
|
|
34
|
+
use_rep : str, default='X_pca'
|
|
35
|
+
Key in .obsm used for neighbour searches and as the "original" space.
|
|
36
|
+
methods : list[str] or None
|
|
37
|
+
Subset of METHODS keys to run. None = all available.
|
|
38
|
+
method_kwargs : dict
|
|
39
|
+
Per-method kwargs, e.g. {'gotmap_nn': {'epochs': 300}}.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(self, ref_adata, query_adata, label_key=None, use_rep="X_pca",
|
|
43
|
+
methods=None, method_kwargs=None):
|
|
44
|
+
self.ref_adata = ref_adata.copy()
|
|
45
|
+
self.query_adata = query_adata.copy()
|
|
46
|
+
self.label_key = label_key
|
|
47
|
+
self.use_rep = use_rep
|
|
48
|
+
|
|
49
|
+
all_methods = list(METHODS.keys())
|
|
50
|
+
self.methods = methods if methods is not None else all_methods
|
|
51
|
+
self.method_kwargs = method_kwargs or {}
|
|
52
|
+
|
|
53
|
+
self.results_ = {}
|
|
54
|
+
self.metrics_ = pd.DataFrame()
|
|
55
|
+
self.errors_ = {}
|
|
56
|
+
|
|
57
|
+
# ------------------------------------------------------------------
|
|
58
|
+
def run(self, k_neighbors=15, verbose=True):
|
|
59
|
+
for method in self.methods:
|
|
60
|
+
if method not in METHODS:
|
|
61
|
+
self.errors_[method] = f"unknown method (available: {list(METHODS)})"
|
|
62
|
+
continue
|
|
63
|
+
|
|
64
|
+
if verbose:
|
|
65
|
+
print(f"\n{'='*60}\n[benchmark] running: {method}\n{'='*60}")
|
|
66
|
+
|
|
67
|
+
t0 = time.time()
|
|
68
|
+
try:
|
|
69
|
+
result = METHODS[method](
|
|
70
|
+
self.ref_adata, self.query_adata,
|
|
71
|
+
use_rep=self.use_rep, label_key=self.label_key,
|
|
72
|
+
**self.method_kwargs.get(method, {}))
|
|
73
|
+
except Exception as e:
|
|
74
|
+
msg = f"{type(e).__name__}: {e}"
|
|
75
|
+
self.errors_[method] = msg
|
|
76
|
+
if verbose:
|
|
77
|
+
print(f" [skip] {method} failed -> {msg}")
|
|
78
|
+
traceback.print_exc()
|
|
79
|
+
continue
|
|
80
|
+
|
|
81
|
+
elapsed = time.time() - t0
|
|
82
|
+
metrics = evaluate_mapping(
|
|
83
|
+
self.ref_adata, self.query_adata, result,
|
|
84
|
+
use_rep=self.use_rep, label_key=self.label_key,
|
|
85
|
+
k=k_neighbors, verbose=verbose)
|
|
86
|
+
|
|
87
|
+
self.results_[method] = {**result, "time": elapsed, "metrics": metrics}
|
|
88
|
+
if verbose:
|
|
89
|
+
line = ", ".join(
|
|
90
|
+
f"{k}={round(v,3) if isinstance(v, float) else v}"
|
|
91
|
+
for k, v in metrics.items())
|
|
92
|
+
print(f" [done] {method} in {elapsed:.1f}s -> {line}")
|
|
93
|
+
|
|
94
|
+
self._summarize()
|
|
95
|
+
return self
|
|
96
|
+
|
|
97
|
+
# ------------------------------------------------------------------
|
|
98
|
+
def _summarize(self):
|
|
99
|
+
rows = []
|
|
100
|
+
for method, res in self.results_.items():
|
|
101
|
+
rows.append({"method": method, **res["metrics"],
|
|
102
|
+
"runtime_seconds": res["time"]})
|
|
103
|
+
self.metrics_ = pd.DataFrame(rows)
|
|
104
|
+
|
|
105
|
+
if self.errors_:
|
|
106
|
+
err_rows = [{"method": m, "error": e} for m, e in self.errors_.items()]
|
|
107
|
+
self.error_df_ = pd.DataFrame(err_rows)
|
|
108
|
+
else:
|
|
109
|
+
self.error_df_ = pd.DataFrame(columns=["method", "error"])
|
|
110
|
+
|
|
111
|
+
# ------------------------------------------------------------------
|
|
112
|
+
def save_results(self, path="benchmark_metrics.csv"):
|
|
113
|
+
if self.metrics_.empty:
|
|
114
|
+
raise RuntimeError("No results yet. Call .run() first.")
|
|
115
|
+
self.metrics_.to_csv(path, index=False)
|
|
116
|
+
if not self.error_df_.empty:
|
|
117
|
+
base, ext = path.rsplit(".", 1)
|
|
118
|
+
self.error_df_.to_csv(f"{base}_errors.{ext}", index=False)
|
|
119
|
+
print(f"[benchmark] saved metrics -> {path}")
|
|
120
|
+
if not self.error_df_.empty:
|
|
121
|
+
print(f"[benchmark] {len(self.error_df_)} method(s) failed; see errors file.")
|
|
122
|
+
|
|
123
|
+
# ------------------------------------------------------------------
|
|
124
|
+
def plot(self, save="benchmark_plot.png", figsize=(16, 9)):
|
|
125
|
+
if self.metrics_.empty:
|
|
126
|
+
raise RuntimeError("No results yet. Call .run() first.")
|
|
127
|
+
|
|
128
|
+
cols = ["label_transfer_accuracy", "weighted_f1", "knn_preservation",
|
|
129
|
+
"cell_type_silhouette", "mean_confidence", "mapping_rate",
|
|
130
|
+
"runtime_seconds"]
|
|
131
|
+
cols = [c for c in cols if c in self.metrics_.columns]
|
|
132
|
+
plot_df = self.metrics_.set_index("method")[cols]
|
|
133
|
+
|
|
134
|
+
n = len(cols)
|
|
135
|
+
fig, axes = plt.subplots(2, (n + 1) // 2, figsize=figsize)
|
|
136
|
+
axes = axes.ravel()
|
|
137
|
+
|
|
138
|
+
for i, c in enumerate(cols):
|
|
139
|
+
ax = axes[i]
|
|
140
|
+
order = plot_df[c].sort_values(ascending=False)
|
|
141
|
+
colors = sns.color_palette("viridis", len(order))
|
|
142
|
+
ax.barh(range(len(order)), order.values, color=colors)
|
|
143
|
+
ax.set_yticks(range(len(order)))
|
|
144
|
+
ax.set_yticklabels(order.index)
|
|
145
|
+
ax.invert_yaxis()
|
|
146
|
+
ax.set_title(c.replace("_", " "))
|
|
147
|
+
ax.set_xlim(0, max(1.0, order.max() * 1.15) if c != "runtime_seconds" else None)
|
|
148
|
+
|
|
149
|
+
for j in range(n, len(axes)):
|
|
150
|
+
axes[j].axis("off")
|
|
151
|
+
|
|
152
|
+
fig.suptitle("gotmap benchmark: query-to-reference mapping", fontsize=16)
|
|
153
|
+
fig.tight_layout(rect=[0, 0, 1, 0.97])
|
|
154
|
+
fig.savefig(save, dpi=150, bbox_inches="tight")
|
|
155
|
+
print(f"[benchmark] plot saved -> {save}")
|
|
156
|
+
return fig
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def run_benchmark(ref, query, label_key=None, use_rep="X_pca", methods=None,
|
|
160
|
+
method_kwargs=None, k_neighbors=15, outdir=".", verbose=True):
|
|
161
|
+
"""Convenience one-shot function."""
|
|
162
|
+
import os
|
|
163
|
+
os.makedirs(outdir, exist_ok=True)
|
|
164
|
+
|
|
165
|
+
bench = BenchmarkPipeline(ref, query, label_key=label_key, use_rep=use_rep,
|
|
166
|
+
methods=methods, method_kwargs=method_kwargs)
|
|
167
|
+
bench.run(k_neighbors=k_neighbors, verbose=verbose)
|
|
168
|
+
if not bench.metrics_.empty:
|
|
169
|
+
bench.save_results(os.path.join(outdir, "benchmark_metrics.csv"))
|
|
170
|
+
bench.plot(save=os.path.join(outdir, "benchmark_plot.png"))
|
|
171
|
+
return bench
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gotmap-benchmark
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Benchmark pipeline for gotmap (query-to-reference atlas mapping) vs baselines.
|
|
5
|
+
Author: gotmap developers
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ZengFLab/gotmap-benchmark
|
|
8
|
+
Requires-Python: >=3.9
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
Requires-Dist: numpy>=1.21
|
|
11
|
+
Requires-Dist: scipy>=1.8
|
|
12
|
+
Requires-Dist: pandas>=1.3
|
|
13
|
+
Requires-Dist: scikit-learn>=1.0
|
|
14
|
+
Requires-Dist: scanpy>=1.9
|
|
15
|
+
Requires-Dist: gotmap>=0.2.0
|
|
16
|
+
Requires-Dist: grassmann-ot>=0.3.1
|
|
17
|
+
Requires-Dist: matplotlib>=3.5
|
|
18
|
+
Requires-Dist: seaborn>=0.11
|
|
19
|
+
Provides-Extra: all
|
|
20
|
+
Requires-Dist: scvi-tools>=0.20; extra == "all"
|
|
21
|
+
Requires-Dist: harmonypy>=0.0.6; extra == "all"
|
|
22
|
+
Requires-Dist: bbknn>=1.5; extra == "all"
|
|
23
|
+
Requires-Dist: rpy2>=3.5; extra == "all"
|
|
24
|
+
|
|
25
|
+
# gotmap-benchmark
|
|
26
|
+
|
|
27
|
+
**Benchmark pipeline for gotmap** — compare query-to-reference atlas mapping methods
|
|
28
|
+
on real and synthetic single-cell data and get quantitative results + plots out of the box.
|
|
29
|
+
|
|
30
|
+
> Positioning: this benchmark evaluates **query → reference mapping quality**, not
|
|
31
|
+
> symmetric batch mixing. It is designed for the gotmap use case (map a query onto a
|
|
32
|
+
> fixed atlas), not to compete with integration-tool comparisons.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install gotmap grassmann-ot scanpy
|
|
40
|
+
# optional, for more baselines:
|
|
41
|
+
pip install scvi-tools harmonypy bbknn
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Quick start (Python)
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
import scanpy as sc
|
|
50
|
+
from gotmap_benchmark import BenchmarkPipeline
|
|
51
|
+
|
|
52
|
+
adata = sc.datasets.pbmc68k_reduced()
|
|
53
|
+
if "X_pca" not in adata.obsm:
|
|
54
|
+
sc.pp.normalize_total(adata); sc.pp.log1p(adata); sc.pp.pca(adata, n_comps=30)
|
|
55
|
+
|
|
56
|
+
label_key = "bulk_labels" if "bulk_labels" in adata.obs.columns else "louvain"
|
|
57
|
+
if label_key not in adata.obs.columns:
|
|
58
|
+
sc.tl.louvain(adata, resolution=0.8, key_added="louvain"); label_key = "louvain"
|
|
59
|
+
|
|
60
|
+
ref, query = sc.pp.train_test_split(adata, test_size=0.3, random_state=42,
|
|
61
|
+
stratify=label_key)
|
|
62
|
+
|
|
63
|
+
bench = BenchmarkPipeline(ref, query, label_key=label_key, use_rep="X_pca",
|
|
64
|
+
methods=["gotmap_nn", "gotmap_knn", "direct_knn"])
|
|
65
|
+
bench.run()
|
|
66
|
+
bench.save_results("benchmark_metrics.csv")
|
|
67
|
+
bench.plot(save="benchmark_plot.png")
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Quick start (CLI)
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
gotmap-benchmark --ref ref.h5ad --query query.h5ad --label cell_type \
|
|
74
|
+
--use-rep X_pca --methods gotmap_nn direct_knn \
|
|
75
|
+
--output ./results
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## Included methods
|
|
81
|
+
|
|
82
|
+
| Key | Method | Notes |
|
|
83
|
+
|-----|--------|-------|
|
|
84
|
+
| `gotmap_nn` | gotmap, neural-network mapping | uses anchors + gamma |
|
|
85
|
+
| `gotmap_knn` | gotmap, kNN mapping | non-parametric |
|
|
86
|
+
| `scArches` | scVI + scArches | needs scvi-tools |
|
|
87
|
+
| `harmony_project` | Harmony project | needs R harmony + rpy2 |
|
|
88
|
+
| `BBKNN_align` | BBKNN | symmetric, then extracts query |
|
|
89
|
+
| `direct_knn` | kNN (no correction) | **baseline** |
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## Metrics reported
|
|
94
|
+
|
|
95
|
+
- **Label Transfer Accuracy** / **Weighted F1** — ground-truth label recovery
|
|
96
|
+
- **kNN Preservation (kNP)** — overlap of reference-neighbourhoods before/after mapping
|
|
97
|
+
- **Cell-type Silhouette** — separation of types in mapped space
|
|
98
|
+
- **Mean Confidence** / **Mapping Rate** — placement certainty
|
|
99
|
+
- **Runtime (s)**
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## Output
|
|
104
|
+
|
|
105
|
+
- `benchmark_metrics.csv` — one row per method
|
|
106
|
+
- `benchmark_plot.png` — horizontal bar chart per metric
|
|
107
|
+
- `bench.results_` — full per-method outputs (mapped AnnData, labels, confidence)
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## Project layout
|
|
112
|
+
|
|
113
|
+
```
|
|
114
|
+
gotmap-benchmark/
|
|
115
|
+
├── pyproject.toml
|
|
116
|
+
├── README.md
|
|
117
|
+
├── gotmap_benchmark/
|
|
118
|
+
│ ├── __init__.py
|
|
119
|
+
│ ├── cli.py # command-line entry point
|
|
120
|
+
│ ├── pipeline.py # BenchmarkPipeline + plotting
|
|
121
|
+
│ ├── methods.py # method wrappers
|
|
122
|
+
│ ├── metrics.py # evaluation metrics
|
|
123
|
+
│ └── datasets.py # dataset loaders + ref/query splitting
|
|
124
|
+
└── examples/
|
|
125
|
+
└── run_pbmc_benchmark.py
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
License: MIT
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
gotmap_benchmark/__init__.py
|
|
4
|
+
gotmap_benchmark/cli.py
|
|
5
|
+
gotmap_benchmark/datasets.py
|
|
6
|
+
gotmap_benchmark/methods.py
|
|
7
|
+
gotmap_benchmark/metrics.py
|
|
8
|
+
gotmap_benchmark/pipeline.py
|
|
9
|
+
gotmap_benchmark.egg-info/PKG-INFO
|
|
10
|
+
gotmap_benchmark.egg-info/SOURCES.txt
|
|
11
|
+
gotmap_benchmark.egg-info/dependency_links.txt
|
|
12
|
+
gotmap_benchmark.egg-info/entry_points.txt
|
|
13
|
+
gotmap_benchmark.egg-info/requires.txt
|
|
14
|
+
gotmap_benchmark.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
gotmap_benchmark
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=45", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "gotmap-benchmark"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Benchmark pipeline for gotmap (query-to-reference atlas mapping) vs baselines."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [{name = "gotmap developers"}]
|
|
13
|
+
dependencies = [
|
|
14
|
+
"numpy>=1.21",
|
|
15
|
+
"scipy>=1.8",
|
|
16
|
+
"pandas>=1.3",
|
|
17
|
+
"scikit-learn>=1.0",
|
|
18
|
+
"scanpy>=1.9",
|
|
19
|
+
"gotmap>=0.2.0",
|
|
20
|
+
"grassmann-ot>=0.3.1",
|
|
21
|
+
"matplotlib>=3.5",
|
|
22
|
+
"seaborn>=0.11",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
all = [
|
|
27
|
+
"scvi-tools>=0.20",
|
|
28
|
+
"harmonypy>=0.0.6",
|
|
29
|
+
"bbknn>=1.5",
|
|
30
|
+
"rpy2>=3.5",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.scripts]
|
|
34
|
+
gotmap-benchmark = "gotmap_benchmark.cli:main"
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://github.com/ZengFLab/gotmap-benchmark"
|