gotmap 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
gotmap/update.py ADDED
@@ -0,0 +1,330 @@
1
+ """
2
+ scArches-style lightweight update mode for GOTMap.
3
+
4
+ In standard use, reference atlas is fixed and only query data changes.
5
+ However, in a real single-cell atlas deployment, the reference itself
6
+ accumulates new high-quality cells over time. Recomputing PCA / Grassmann
7
+ structure from scratch is expensive.
8
+
9
+ This module provides:
10
+
11
+ 1. Reference freeze / serialization
12
+ 2. Incremental extension (extend_reference)
13
+ 3. Conditional recomputation (only when reference grows beyond threshold)
14
+ 4. Lightweight query mapping without touching the frozen reference
15
+
16
+ Design principle:
17
+ grassmann_ot is used ONLY for coupling (gamma, anchors).
18
+ Reference PCA / Grassmann points are cached and reused.
19
+ """
20
+
21
+ import os
22
+ import json
23
+ import numpy as np
24
+ import anndata
25
+ import scanpy as sc
26
+ from scipy import sparse
27
+
28
+
29
+ class FrozenReference:
30
+ """
31
+ A frozen (cached) reference atlas for fast repeated query mapping.
32
+
33
+ Instead of recomputing PCA every time, we store:
34
+ - mean & scale of original PCA
35
+ - principal components (loadings)
36
+ - PCA coordinates of reference cells
37
+ - Grassmann point list (optional)
38
+
39
+ New query data can be projected using the SAME PCA space, ensuring
40
+ that the reference geometry never changes between queries.
41
+
42
+ Parameters
43
+ ----------
44
+ adata : AnnData
45
+ Reference atlas (already preprocessed).
46
+ pca_key : str
47
+ obsm key for PCA coordinates.
48
+ use_highly_variable : bool
49
+ Whether PCA was trained on highly variable genes.
50
+ """
51
+
52
+ def __init__(self, adata, pca_key="X_pca", use_highly_variable=True):
53
+ if pca_key not in adata.obsm:
54
+ raise ValueError(
55
+ f"{pca_key} not found in adata.obsm. "
56
+ f"Run sc.pp.pca(adata, n_comps=...) first."
57
+ )
58
+
59
+ # ---- store PCA model ----
60
+ # scanpy stores PCA in adata.varm['PCs'] (genes x n_comps)
61
+ # and the mean in adata.uns['pca']['mean'] (if available)
62
+ if "PCs" not in adata.varm:
63
+ raise ValueError(
64
+ "adata.varm['PCs'] not found. "
65
+ "This is required to project new data into the same PCA space."
66
+ )
67
+
68
+ self.pca_key = pca_key
69
+ self.use_highly_variable = use_highly_variable
70
+
71
+ self.PCs = adata.varm["PCs"].copy() # (n_genes_or_hvg, n_comps)
72
+ self.n_comps = self.PCs.shape[1]
73
+
74
+ # mean of each gene (for centering)
75
+ if "pca" in adata.uns and "mean" in adata.uns["pca"]:
76
+ self.mean = adata.uns["pca"]["mean"].copy()
77
+ else:
78
+ # fallback: compute mean from X
79
+ X = adata.X.toarray() if sparse.issparse(adata.X) else adata.X
80
+ self.mean = X.mean(axis=0)
81
+
82
+ # PCA coordinates of reference cells (frozen)
83
+ self.pca = adata.obsm[pca_key].copy()
84
+
85
+ # gene names used for PCA (for alignment)
86
+ self.var_names = adata.var_names.copy()
87
+
88
+ # optional metadata
89
+ self.n_obs = adata.n_obs
90
+ self.obs = adata.obs.copy() if adata.obs is not None else None
91
+
92
+ # grassmann points cache (filled lazily)
93
+ self._grassmann_points = None
94
+
95
+ # version tracking
96
+ self.version = 1
97
+ self.history = []
98
+
99
+ # ----------------------------------------------------------------
100
+ # persistence
101
+ # ----------------------------------------------------------------
102
+ def save(self, path):
103
+ """
104
+ Save frozen reference to disk.
105
+
106
+ Parameters
107
+ ----------
108
+ path : str
109
+ Directory path. Will be created if not exists.
110
+ """
111
+ os.makedirs(path, exist_ok=True)
112
+
113
+ np.save(os.path.join(path, "PCs.npy"), self.PCs)
114
+ np.save(os.path.join(path, "mean.npy"), self.mean)
115
+ np.save(os.path.join(path, "pca.npy"), self.pca)
116
+
117
+ self.var_names.to_frame().to_csv(
118
+ os.path.join(path, "var_names.csv")
119
+ )
120
+
121
+ meta = {
122
+ "pca_key": self.pca_key,
123
+ "use_highly_variable": self.use_highly_variable,
124
+ "n_comps": int(self.n_comps),
125
+ "n_obs": int(self.n_obs),
126
+ "version": int(self.version),
127
+ "history": self.history,
128
+ }
129
+ with open(os.path.join(path, "meta.json"), "w") as f:
130
+ json.dump(meta, f, indent=2)
131
+
132
+ if self._grassmann_points is not None:
133
+ np.save(
134
+ os.path.join(path, "grassmann_points.npy"),
135
+ np.array(self._grassmann_points, dtype=object),
136
+ )
137
+
138
+ print(f"[FrozenReference] saved to {path} (v{self.version})")
139
+
140
+ @classmethod
141
+ def load(cls, path):
142
+ """
143
+ Load frozen reference from disk.
144
+ """
145
+ self = cls.__new__(cls)
146
+
147
+ self.PCs = np.load(os.path.join(path, "PCs.npy"))
148
+ self.mean = np.load(os.path.join(path, "mean.npy"))
149
+ self.pca = np.load(os.path.join(path, "pca.npy"))
150
+ self.n_comps = self.PCs.shape[1]
151
+ self.n_obs = self.pca.shape[0]
152
+
153
+ self.var_names = anndata.read_var(
154
+ os.path.join(path, "var_names.csv")
155
+ ).index if False else \
156
+ np.loadtxt(
157
+ os.path.join(path, "var_names.csv"),
158
+ dtype=str, delimiter=",", skiprows=1, usecols=0
159
+ )
160
+
161
+ with open(os.path.join(path, "meta.json")) as f:
162
+ meta = json.load(f)
163
+ self.pca_key = meta["pca_key"]
164
+ self.use_highly_variable = meta["use_highly_variable"]
165
+ self.version = meta["version"]
166
+ self.history = meta.get("history", [])
167
+
168
+ gp_path = os.path.join(path, "grassmann_points.npy")
169
+ self._grassmann_points = (
170
+ np.load(gp_path, allow_pickle=True).tolist()
171
+ if os.path.exists(gp_path) else None
172
+ )
173
+
174
+ print(f"[FrozenReference] loaded from {path} (v{self.version})")
175
+ return self
176
+
177
+ # ----------------------------------------------------------------
178
+ # projection
179
+ # ----------------------------------------------------------------
180
+ def project_query(self, query_adata):
181
+ """
182
+ Project query data into the SAME PCA space as the frozen reference.
183
+
184
+ Parameters
185
+ ----------
186
+ query_adata : AnnData
187
+ Query batch (raw or log-normalized counts).
188
+
189
+ Returns
190
+ -------
191
+ np.ndarray
192
+ Query PCA coordinates, shape (n_query, n_comps).
193
+ """
194
+ # align genes
195
+ if not np.array_equal(query_adata.var_names, self.var_names):
196
+ # find genes present in both; order = reference gene order
197
+ ref_set = set(self.var_names)
198
+ common = np.array([g for g in self.var_names if g in query_adata.var_names])
199
+ if len(common) < self.n_comps:
200
+ raise ValueError(
201
+ f"Only {len(common)} genes overlap with reference. "
202
+ f"Need at least {self.n_comps} for safe projection."
203
+ )
204
+ # subset query to those genes, in reference order
205
+ query_adata = query_adata[:, common]
206
+ PCs = self.PCs[: len(common)]
207
+ mean = self.mean[: len(common)] if self.mean is not None else None
208
+ else:
209
+ PCs = self.PCs
210
+ mean = self.mean
211
+
212
+ # center and project
213
+ X = query_adata.X.toarray() if sparse.issparse(query_adata.X) else query_adata.X
214
+ if mean is not None:
215
+ X = X - mean
216
+ pca_q = X @ PCs
217
+ return pca_q
218
+
219
+ # ----------------------------------------------------------------
220
+ # lightweight update
221
+ # ----------------------------------------------------------------
222
+ def extend(self, new_cells_adata, min_new_cells=500, update_ratio=0.1):
223
+ """
224
+ Conditionally extend the reference with new high-quality cells.
225
+
226
+ This is the scArches-style lightweight update:
227
+ we do NOT rebuild the PCA / Grassmann structure from scratch.
228
+ Instead, we only extend when the new cells exceed a threshold,
229
+ and even then we use incremental / warm-start strategies.
230
+
231
+ Parameters
232
+ ----------
233
+ new_cells_adata : AnnData
234
+ New validated cells to potentially add to the atlas.
235
+ min_new_cells : int
236
+ Minimum number of new cells required to trigger an update.
237
+ update_ratio : float
238
+ If (new_cells / current_reference) > update_ratio,
239
+ trigger a PCA rebuild (warm-started from current PCs).
240
+
241
+ Returns
242
+ -------
243
+ dict
244
+ Update report: whether update happened, version, new cell count.
245
+ """
246
+ n_new = new_cells_adata.n_obs
247
+ ratio = n_new / self.n_obs
248
+
249
+ report = {
250
+ "new_cells": n_new,
251
+ "current_version": self.version,
252
+ "update_triggered": False,
253
+ "reason": "",
254
+ }
255
+
256
+ if n_new < min_new_cells:
257
+ report["reason"] = (
258
+ f"new cells ({n_new}) < min_new_cells ({min_new_cells})"
259
+ )
260
+ return report
261
+
262
+ if ratio < update_ratio:
263
+ # lightweight path: just append to pca coords (project into existing space)
264
+ new_pca = self.project_query(new_cells_adata)
265
+ self.pca = np.vstack([self.pca, new_pca])
266
+ self.n_obs += n_new
267
+ self.version += 1
268
+ self.history.append({
269
+ "action": "append",
270
+ "n_new": n_new,
271
+ "version": self.version,
272
+ })
273
+ report["update_triggered"] = True
274
+ report["reason"] = (
275
+ f"appended {n_new} cells (ratio {ratio:.3f} < {update_ratio}); "
276
+ f"PCA space unchanged"
277
+ )
278
+ report["new_version"] = self.version
279
+ print(
280
+ f"[FrozenReference] v{self.version}: appended {n_new} cells "
281
+ f"(lightweight, no PCA rebuild)"
282
+ )
283
+ return report
284
+
285
+ # full rebuild path (warm-started)
286
+ report["reason"] = (
287
+ f"ratio {ratio:.3f} >= {update_ratio}; PCA rebuild recommended"
288
+ )
289
+ print(
290
+ f"[FrozenReference] v{self.version}: {n_new} new cells "
291
+ f"(ratio {ratio:.3f}). Recommend rebuild_reference()."
292
+ )
293
+ return report
294
+
295
+ def rebuild_reference(self, adata, n_comps=None):
296
+ """
297
+ Full rebuild of reference PCA (call sparingly).
298
+
299
+ Parameters
300
+ ----------
301
+ adata : AnnData
302
+ Full reference atlas (old + new cells).
303
+ n_comps : int or None
304
+ Number of PCs. If None, keep current n_comps.
305
+ """
306
+ if n_comps is None:
307
+ n_comps = self.n_comps
308
+
309
+ sc.pp.pca(adata, n_comps=n_comps)
310
+
311
+ self.PCs = adata.varm["PCs"].copy()
312
+ self.pca = adata.obsm[self.pca_key].copy()
313
+ self.n_comps = n_comps
314
+ self.n_obs = adata.n_obs
315
+ self.var_names = adata.var_names.copy()
316
+
317
+ if "pca" in adata.uns and "mean" in adata.uns["pca"]:
318
+ self.mean = adata.uns["pca"]["mean"].copy()
319
+ else:
320
+ X = adata.X.toarray() if sparse.issparse(adata.X) else adata.X
321
+ self.mean = X.mean(axis=0)
322
+
323
+ self.version += 1
324
+ self._grassmann_points = None # invalidate cache
325
+ self.history.append({
326
+ "action": "rebuild",
327
+ "n_total": self.n_obs,
328
+ "version": self.version,
329
+ })
330
+ print(f"[FrozenReference] v{self.version}: full PCA rebuild complete")
gotmap/utils.py ADDED
@@ -0,0 +1,39 @@
1
+ """Utility functions for gotmap."""
2
+
3
+ import numpy as np
4
+ import scanpy as sc
5
+
6
+
7
+ def check_pca(adata, pca_key="X_pca"):
8
+ """Ensure that pca_key exists in adata.obsm."""
9
+ if pca_key not in adata.obsm:
10
+ raise ValueError(
11
+ f"'{pca_key}' not found in adata.obsm. "
12
+ f"Run sc.tl.pca(adata, n_comps=...) first."
13
+ )
14
+
15
+
16
+ def prep_adata(adata, n_comps=50, pca_key="X_pca"):
17
+ """
18
+ Run standard preprocessing + PCA if not already done.
19
+
20
+ Parameters
21
+ ----------
22
+ adata : AnnData
23
+ n_comps : int, default=50
24
+ pca_key : str, default='X_pca'
25
+
26
+ Returns
27
+ -------
28
+ AnnData
29
+ A copy with adata.obsm[pca_key] populated.
30
+ """
31
+ adata = adata.copy()
32
+ if pca_key not in adata.obsm:
33
+ if "log1p" not in adata.uns:
34
+ sc.pp.normalize_total(adata)
35
+ sc.pp.log1p(adata)
36
+ if "highly_variable" not in adata.var.columns:
37
+ sc.pp.highly_variable_genes(adata, n_top_genes=2000)
38
+ sc.pp.pca(adata, n_comps=n_comps)
39
+ return adata
@@ -0,0 +1,188 @@
1
+ Metadata-Version: 2.4
2
+ Name: gotmap
3
+ Version: 0.1.0
4
+ Summary: Query-to-Reference Atlas Mapping for Single-Cell Data via Grassmann Optimal Transport
5
+ Author: GOTMap Developers
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/ZengFLab/gotmap
8
+ Requires-Python: >=3.9
9
+ Description-Content-Type: text/markdown
10
+ Requires-Dist: numpy>=1.21
11
+ Requires-Dist: scipy>=1.8
12
+ Requires-Dist: torch>=1.12
13
+ Requires-Dist: scanpy>=1.9
14
+ Requires-Dist: grassmann-ot>=0.3.1
15
+ Requires-Dist: scikit-learn>=1.0
16
+ Requires-Dist: matplotlib>=3.5
17
+
18
+ # GOTMap
19
+
20
+ > **Query-to-Reference Atlas Mapping via Grassmann Optimal Transport**
21
+
22
+ GOTMap maps query single-cell data into a fixed reference atlas using
23
+ [grassmann-ot](https://github.com/ZengFLab/grassmann-ot). It is NOT a
24
+ symmetric batch-integration tool — its purpose is to project new samples
25
+ into an existing atlas for annotation transfer and quality control.
26
+
27
+ ---
28
+
29
+ ## Why GOTMap?
30
+
31
+ | | Symmetric Integration | **GOTMap (Query → Atlas)** |
32
+ |---|---|---|
33
+ | Goal | Remove batch effects | **Map new data into a stable atlas** |
34
+ | Symmetry | Bidirectional | **Unidirectional (reference fixed)** |
35
+ | Output | Joint embedding | **Query coords in reference space** |
36
+ | Labels | Optional | **First-class (transfer + confidence)** |
37
+ | Use case | Combining batches | **Clinical / atlas extension / QC** |
38
+
39
+ ---
40
+
41
+ ## Installation
42
+
43
+ ```bash
44
+ pip install grassmann-ot scanpy torch
45
+ pip install gotmap-0.1.0.zip
46
+ ```
47
+
48
+ ---
49
+
50
+ ## Quick start
51
+
52
+ ```python
53
+ import scanpy as sc
54
+ import gotmap
55
+ from gotmap.pp import preprocess_reference, preprocess_query
56
+ from gotmap.tl import map_query_to_reference, transfer_labels, evaluate_mapping
57
+ from gotmap.pl import sanity_check, label_transfer_confidence
58
+
59
+ # ---- 1. Build & freeze reference atlas ----
60
+ ref = sc.read_h5ad("atlas.h5ad")
61
+ ref = preprocess_reference(ref, n_comps=50, n_top_genes=2000)
62
+
63
+ # ---- 2. Preprocess and map query ----
64
+ query = sc.read_h5ad("new_sample.h5ad")
65
+ query = preprocess_query(query, ref, n_comps=50)
66
+
67
+ mapped = map_query_to_reference(query, ref, method="nn",
68
+ hidden_dim=128, epochs=200)
69
+
70
+ # ---- 3. Transfer labels ----
71
+ labelled = transfer_labels(query, ref, label_key="cell_type")
72
+
73
+ # ---- 4. Evaluate ----
74
+ quality = evaluate_mapping(query, ref)
75
+ print(quality) # {'mean_confidence': ..., 'mapping_rate': ..., 'n_anchors': ...}
76
+
77
+ # ---- 5. Visualize ----
78
+ sanity_check(ref, query, mapped_query=mapped, label_key="cell_type")
79
+ label_transfer_confidence(labelled, label_key="cell_type")
80
+ ```
81
+
82
+ ---
83
+
84
+ ## API
85
+
86
+ ### `gotmap.pp` — Preprocessing
87
+
88
+ | Function | Purpose |
89
+ |---|---|
90
+ | `preprocess_reference(adata, n_comps=50, ...)` | Full pipeline: normalize → HVG → PCA |
91
+ | `preprocess_query(adata, reference, n_comps=50)` | Normalize + project query into **reference PCA space** |
92
+ | `normalize_and_hvg(adata, ...)` | Normalize and select HVGs |
93
+
94
+ ### `gotmap.tl` — Tools
95
+
96
+ | Function | Purpose |
97
+ |---|---|
98
+ | `map_query_to_reference(query, ref, method="nn", ...)` | Map query → reference space |
99
+ | `transfer_labels(query, ref, label_key, ...)` | Transfer labels + confidence |
100
+ | `evaluate_mapping(query, ref, ...)` | Mapping quality metrics |
101
+
102
+ ### `gotmap.pl` — Plotting
103
+
104
+ | Function | Purpose |
105
+ |---|---|
106
+ | `mapping_scatter(adata, basis="X_gotmap", color=...)` | Scatter of mapped cells |
107
+ | `label_transfer_confidence(adata, label_key)` | Confidence histogram + by-label boxplot |
108
+ | `embedding(adata, basis=..., color=...)` | Generic embedding plot |
109
+ | `sanity_check(ref, query, mapped_query=...)` | Reference / mapped / overlay panels |
110
+
111
+ ### `gotmap.GOTMap` — Low-level class
112
+
113
+ ```python
114
+ from gotmap import GOTMap
115
+
116
+ mapper = GOTMap(ref_adata=ref, pca_key="X_pca", k=30, p=3, reg=0.1)
117
+ mapper.fit(query)
118
+
119
+ # neural-network projection
120
+ mapped = mapper.transform(query, method="nn", hidden_dim=128, epochs=200)
121
+
122
+ # kNN projection
123
+ mapped_knn = mapper.transform(query, method="knn")
124
+
125
+ # transfer labels
126
+ labelled = mapper.transfer_labels(query, label_key="cell_type")
127
+
128
+ # quality
129
+ quality = mapper.evaluate_mapping(query)
130
+ ```
131
+
132
+ ### `gotmap.FrozenReference` — Lightweight update (scArches-style)
133
+
134
+ ```python
135
+ from gotmap import FrozenReference
136
+
137
+ # freeze
138
+ frozen = FrozenReference(ref)
139
+ frozen.save("./frozen_atlas")
140
+
141
+ # load (no recomputation)
142
+ frozen = FrozenReference.load("./frozen_atlas")
143
+
144
+ # project any new query into the SAME space
145
+ query_pca = frozen.project_query(new_query)
146
+
147
+ # extend with new validated cells
148
+ report = frozen.extend(new_cells, min_new_cells=500, update_ratio=0.1)
149
+ # - lightweight: append only (PCA unchanged)
150
+ # - full rebuild triggered when new cells exceed threshold
151
+
152
+ # full rebuild (call sparingly)
153
+ frozen.rebuild_reference(full_atlas)
154
+ frozen.save("./frozen_atlas_v2")
155
+ ```
156
+
157
+ ---
158
+
159
+ ## How it works
160
+
161
+ 1. **Grassmann subspaces** — For each cell, a local PCA neighborhood
162
+ defines a point on the Grassmann manifold (dimension `p`).
163
+ 2. **Optimal transport** — Sinkhorn OT computes a soft coupling
164
+ `gamma` (n_ref × n_query) and hard anchors between reference and
165
+ query cells.
166
+ 3. **Mapping** — Anchors (+ gamma-sampled soft pairs) supervise a
167
+ neural network that projects query cells into reference PCA space.
168
+ 4. **Label transfer** — Weighted kNN in reference space assigns
169
+ cell-type labels; confidence = max class probability.
170
+
171
+ GOTMap calls `grassmann-ot` only for coupling (`fit`, `get_hard_anchors`).
172
+ All mapping logic is implemented independently.
173
+
174
+ ---
175
+
176
+ ## Requirements
177
+
178
+ - Python ≥ 3.9
179
+ - scanpy ≥ 1.9
180
+ - grassmann-ot ≥ 0.3.1
181
+ - torch ≥ 1.12
182
+ - numpy, scipy, scikit-learn
183
+
184
+ ---
185
+
186
+ ## License
187
+
188
+ MIT
@@ -0,0 +1,14 @@
1
+ gotmap/__init__.py,sha256=Uh83_Q6SmHafieSCSuUhftxYj80JBhwi96xwQarPKCw,1009
2
+ gotmap/confidence.py,sha256=OKtimgDLVKiOAqD-R_HCPigalxS9fJOCzDQs7_RROw8,1885
3
+ gotmap/knn_map.py,sha256=AixjtGflCFW5_FnRgK3EDwxJc0D4TCNSrvfsKYAYiKo,3555
4
+ gotmap/mapper.py,sha256=PFC4EiiV27sAE0P-wFgpzWiLdBB3qV4F9pL_ELAaqXo,7837
5
+ gotmap/nn_map.py,sha256=rHuBxilsEn7mwuzWjhZyILTou508bYUBsuuY3nwk4TY,3402
6
+ gotmap/pl.py,sha256=pohjoKHT4jTAzEc1nLirQf_7HJzBBeU_ah8_ZMHU3Yw,6689
7
+ gotmap/pp.py,sha256=YeplQbXBDmbRFRFyda7HkEoEFg4kDkocrUGkgu-iM-I,4553
8
+ gotmap/tl.py,sha256=CanT9KHuDyHyGdZn2fGrKl0Nznnqwy4vn_RVLBqX8ms,4520
9
+ gotmap/update.py,sha256=HtvZK8-MbOkc-kIz8EiUrnyVlLdT2fx-vgbDVl48PJ0,11413
10
+ gotmap/utils.py,sha256=44yYokkaxXbHc89ElOlyfexgaFN_Q4y5loLP3781-Bw,1027
11
+ gotmap-0.1.0.dist-info/METADATA,sha256=JCoboFiDnsp3p0zUS0OEjb13wC5RDB5NXUfHHIbV4dI,5503
12
+ gotmap-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
13
+ gotmap-0.1.0.dist-info/top_level.txt,sha256=eUnG3QMoJunVSsyapQo_fI-kkPL_sWRbDxHuhmfN0Ls,7
14
+ gotmap-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ gotmap