spatial-aim 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aim/__init__.py +47 -0
- aim/__main__.py +6 -0
- aim/adata_schema.py +80 -0
- aim/aggregation.py +118 -0
- aim/aim_config.py +76 -0
- aim/analysis/__init__.py +13 -0
- aim/analysis/analysis.py +72 -0
- aim/analysis/biology.py +205 -0
- aim/analysis/confidence.py +49 -0
- aim/analysis/ksweep.py +184 -0
- aim/analysis/loading.py +70 -0
- aim/analysis/onehot.py +41 -0
- aim/analysis/reconstruction.py +131 -0
- aim/analysis/topology.py +142 -0
- aim/analysis/utils.py +16 -0
- aim/cli.py +371 -0
- aim/clustering.py +242 -0
- aim/data/__init__.py +0 -0
- aim/data/prepare/__init__.py +0 -0
- aim/data/prepare/build_imaging_pairs.py +397 -0
- aim/data/prepare/build_xenium_breast_pair.py +310 -0
- aim/data/prepare/convertTsvToCsv.py +39 -0
- aim/data/prepare/convert_csv_to_h5ad.py +217 -0
- aim/data/prepare/createOnlyHVGDataset.py +69 -0
- aim/data/prepare/createSmallerDataset.py +102 -0
- aim/data/prepare/normalize_gene_names.py +56 -0
- aim/data/prepare/perturb_sc_noise.py +107 -0
- aim/data/prepare/register_st_patches.py +180 -0
- aim/data/prepare/split_st_into_islands.py +551 -0
- aim/data/prepare/split_st_into_patches.py +240 -0
- aim/data/prepare/subset_st_hvg.py +43 -0
- aim/data/validate/__init__.py +35 -0
- aim/data/validate/common.py +141 -0
- aim/data/validate/runner.py +335 -0
- aim/data/validate/validate_pair.py +53 -0
- aim/data/validate/validate_sc.py +103 -0
- aim/data/validate/validate_st.py +74 -0
- aim/gui/__init__.py +8 -0
- aim/gui/__main__.py +77 -0
- aim/gui/app.py +1784 -0
- aim/gui/compute.py +106 -0
- aim/gui/data_access.py +292 -0
- aim/gui/render.py +1679 -0
- aim/gui/scaffold.py +129 -0
- aim/gui/widgets.py +675 -0
- aim/io.py +196 -0
- aim/mapping/__init__.py +14 -0
- aim/mapping/base.py +67 -0
- aim/mapping/confidence.py +51 -0
- aim/mapping/nearest_centroid.py +33 -0
- aim/mapping/reference.py +156 -0
- aim/mapping/wann.py +208 -0
- aim/metrics/__init__.py +26 -0
- aim/metrics/biology.py +69 -0
- aim/metrics/cossim.py +127 -0
- aim/metrics/kselection.py +272 -0
- aim/metrics/onehot.py +54 -0
- aim/metrics/reconstruction.py +73 -0
- aim/metrics/topology.py +62 -0
- aim/reference_aligners/__init__.py +0 -0
- aim/reference_aligners/environment_dot.yml +23 -0
- aim/reference_aligners/environment_tacco.yml +28 -0
- aim/reference_aligners/environment_tangram.yml +16 -0
- aim/reference_aligners/registry.py +350 -0
- aim/reference_aligners/run_dot.R +172 -0
- aim/reference_aligners/run_dot.py +116 -0
- aim/reference_aligners/run_tacco.py +143 -0
- aim/reference_aligners/run_tangram.py +210 -0
- aim/sweep.py +261 -0
- aim/tree.py +86 -0
- spatial_aim-1.0.0.dist-info/METADATA +1005 -0
- spatial_aim-1.0.0.dist-info/RECORD +76 -0
- spatial_aim-1.0.0.dist-info/WHEEL +5 -0
- spatial_aim-1.0.0.dist-info/entry_points.txt +2 -0
- spatial_aim-1.0.0.dist-info/licenses/LICENSE +674 -0
- spatial_aim-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Ingest the 10x Genomics / Janesick et al. 2023 human breast cancer dataset into the
|
|
3
|
+
database as a new scRNA <-> ST pair (dataset id 11).
|
|
4
|
+
|
|
5
|
+
Sources (as downloaded, see README "Datensaetze"):
|
|
6
|
+
scRNA reference scFFPE-seq (Chromium Flex), GSM7782698 raw feature-barcode matrix
|
|
7
|
+
+ cell-type annotations from the scFFPE-Seq sheet of
|
|
8
|
+
GSE243275_Barcode_Cell_Type_Matrices.xlsx
|
|
9
|
+
ST (high-res) Xenium In Situ (313-plex), <outs>/cell_feature_matrix.h5
|
|
10
|
+
+ cell centroids from <outs>/cells.parquet
|
|
11
|
+
+ supervised Xenium labels from the "Xenium R1 Fig1-5 (supervised)" sheet
|
|
12
|
+
|
|
13
|
+
Output (matching the database h5ad contract, see CLAUDE.md):
|
|
14
|
+
scRNA/11_human-breast-cancer.h5ad X=raw counts (annotated cells x genes),
|
|
15
|
+
obs['cellType'], var uppercase + unique
|
|
16
|
+
ST/11_human-breast-cancer.h5ad X=raw counts (313 panel), obsm['spatial'],
|
|
17
|
+
obs['cellType'] (Xenium ground truth)
|
|
18
|
+
and appends one row to scRNA/index.csv, ST/index.csv and pairs.csv (originals backed
|
|
19
|
+
up to *.bak). ExperimentMatch = same_paper.
|
|
20
|
+
|
|
21
|
+
Usage:
|
|
22
|
+
python -m data_preparation.build_xenium_breast_pair \
|
|
23
|
+
--xenium-outs "C:/Users/zi69hebi/Downloads/Xenium_FFPE_Human_Breast_Cancer_Rep1_outs/outs" \
|
|
24
|
+
--scffpe-h5 "C:/Users/zi69hebi/Downloads/GSE243275/GSM7782698_count_raw_feature_bc_matrix.h5" \
|
|
25
|
+
--annotation-xlsx "C:/Users/zi69hebi/Downloads/GSE243275_Barcode_Cell_Type_Matrices.xlsx" \
|
|
26
|
+
[--data-root "C:/Users/zi69hebi/Dev/10_Alignment/Data/01_Datasets"] \
|
|
27
|
+
[--dry-run]
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
import argparse
|
|
31
|
+
import csv
|
|
32
|
+
import logging
|
|
33
|
+
import shutil
|
|
34
|
+
import sys
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
|
|
37
|
+
import anndata as ad
|
|
38
|
+
import numpy as np
|
|
39
|
+
import pandas as pd
|
|
40
|
+
import scanpy as sc
|
|
41
|
+
import scipy.sparse as sp
|
|
42
|
+
|
|
43
|
+
logger = logging.getLogger(__name__)
|
|
44
|
+
|
|
45
|
+
DATASET_ID = "11"
|
|
46
|
+
NAME = "11_human-breast-cancer"
|
|
47
|
+
TISSUE = "Human breast cancer"
|
|
48
|
+
QUELLE = "https://www.nature.com/articles/s41467-023-43458-x"
|
|
49
|
+
AUTOR = "Janesick et al"
|
|
50
|
+
FEATURED_IN = "DOT"
|
|
51
|
+
SC_TECH = "Chromium Flex (scFFPE-seq)"
|
|
52
|
+
ST_TECH = "Xenium"
|
|
53
|
+
SC_LINK = "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE243275"
|
|
54
|
+
ST_LINK = (
|
|
55
|
+
"https://www.10xgenomics.com/products/xenium-in-situ/preview-dataset-human-breast"
|
|
56
|
+
)
|
|
57
|
+
EXPERIMENT_MATCH = "same_paper"
|
|
58
|
+
|
|
59
|
+
SCFFPE_SHEET = "scFFPE-Seq"
|
|
60
|
+
XENIUM_SHEET = "Xenium R1 Fig1-5 (supervised)"
|
|
61
|
+
|
|
62
|
+
DEFAULT_DATA_ROOT = Path(r"/01_Datasets")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _uppercase_and_collapse(adata: ad.AnnData) -> ad.AnnData:
|
|
66
|
+
"""Uppercase var_names, then sum counts of any genes that collide into one column.
|
|
67
|
+
|
|
68
|
+
Keeps var_names unique + uppercase so `aim validate` passes and shared-gene
|
|
69
|
+
intersection with the ST panel is exact.
|
|
70
|
+
"""
|
|
71
|
+
names = pd.Index([g.upper() for g in adata.var_names])
|
|
72
|
+
if names.is_unique:
|
|
73
|
+
adata.var_names = names
|
|
74
|
+
return adata
|
|
75
|
+
|
|
76
|
+
uniq = names.unique()
|
|
77
|
+
col_of = {g: i for i, g in enumerate(uniq)}
|
|
78
|
+
cols = np.fromiter((col_of[g] for g in names), dtype=np.int64, count=len(names))
|
|
79
|
+
# (n_old x n_new) 0/1 grouping matrix; X @ G sums duplicate columns
|
|
80
|
+
group = sp.csr_matrix(
|
|
81
|
+
(np.ones(len(names), dtype=np.float32), (np.arange(len(names)), cols)),
|
|
82
|
+
shape=(len(names), len(uniq)),
|
|
83
|
+
)
|
|
84
|
+
x_new = sp.csr_matrix(adata.X).dot(group)
|
|
85
|
+
n_collapsed = len(names) - len(uniq)
|
|
86
|
+
logger.info(
|
|
87
|
+
" collapsed %d duplicate gene symbol(s) by summing counts", n_collapsed
|
|
88
|
+
)
|
|
89
|
+
out = ad.AnnData(
|
|
90
|
+
X=x_new.astype(np.float32),
|
|
91
|
+
obs=adata.obs.copy(),
|
|
92
|
+
var=pd.DataFrame(index=uniq),
|
|
93
|
+
)
|
|
94
|
+
return out
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def build_scrna(scffpe_h5: Path, annotation_xlsx: Path) -> ad.AnnData:
|
|
98
|
+
logger.info("scRNA: reading raw matrix %s", scffpe_h5.name)
|
|
99
|
+
adata = sc.read_10x_h5(scffpe_h5)
|
|
100
|
+
logger.info(" raw matrix: %d barcodes x %d genes", adata.n_obs, adata.n_vars)
|
|
101
|
+
|
|
102
|
+
ann = pd.read_excel(annotation_xlsx, sheet_name=SCFFPE_SHEET)
|
|
103
|
+
ann = ann.set_index("Barcode")["Annotation"]
|
|
104
|
+
logger.info(
|
|
105
|
+
" annotation sheet '%s': %d cells, %d types",
|
|
106
|
+
SCFFPE_SHEET,
|
|
107
|
+
len(ann),
|
|
108
|
+
ann.nunique(),
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
keep = adata.obs_names.intersection(ann.index)
|
|
112
|
+
missing = len(ann) - len(keep)
|
|
113
|
+
if missing:
|
|
114
|
+
logger.warning(" %d annotated barcode(s) not found in raw matrix", missing)
|
|
115
|
+
adata = adata[keep].copy()
|
|
116
|
+
adata.obs["cellType"] = ann.reindex(adata.obs_names).astype(str).values
|
|
117
|
+
logger.info(" subset to %d annotated cells", adata.n_obs)
|
|
118
|
+
|
|
119
|
+
# raw integer counts as float32 sparse
|
|
120
|
+
adata.X = sp.csr_matrix(adata.X).astype(np.float32)
|
|
121
|
+
adata = _uppercase_and_collapse(adata)
|
|
122
|
+
adata.obs_names = [str(b) for b in adata.obs_names]
|
|
123
|
+
adata.obs_names_make_unique()
|
|
124
|
+
logger.info(" final scRNA: %d cells x %d genes", adata.n_obs, adata.n_vars)
|
|
125
|
+
return adata
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def build_st(xenium_outs: Path, annotation_xlsx: Path) -> ad.AnnData:
|
|
129
|
+
h5 = xenium_outs / "cell_feature_matrix.h5"
|
|
130
|
+
logger.info("ST: reading Xenium matrix %s", h5.name)
|
|
131
|
+
adata = sc.read_10x_h5(h5, gex_only=False)
|
|
132
|
+
if "feature_types" in adata.var:
|
|
133
|
+
is_gene = adata.var["feature_types"].astype(str) == "Gene Expression"
|
|
134
|
+
adata = adata[:, is_gene.values].copy()
|
|
135
|
+
logger.info(" kept %d Gene Expression features", adata.n_vars)
|
|
136
|
+
|
|
137
|
+
cells = pd.read_parquet(xenium_outs / "cells.parquet")
|
|
138
|
+
cells["cell_id"] = cells["cell_id"].astype(str)
|
|
139
|
+
cells = cells.set_index("cell_id")
|
|
140
|
+
coords = cells.reindex(adata.obs_names)[["x_centroid", "y_centroid"]]
|
|
141
|
+
if coords.isnull().any().any():
|
|
142
|
+
raise RuntimeError("ST: some cell barcodes have no centroid in cells.parquet")
|
|
143
|
+
adata.obsm["spatial"] = coords.to_numpy(dtype=np.float32)
|
|
144
|
+
|
|
145
|
+
# Xenium supervised ground-truth labels (bonus obs column; not required by schema)
|
|
146
|
+
xann = pd.read_excel(annotation_xlsx, sheet_name=XENIUM_SHEET)
|
|
147
|
+
xann = xann.set_index(xann.columns[0])[xann.columns[1]]
|
|
148
|
+
xann.index = xann.index.astype(str)
|
|
149
|
+
adata.obs["cellType"] = xann.reindex(adata.obs_names).astype(str).values
|
|
150
|
+
|
|
151
|
+
adata.X = sp.csr_matrix(adata.X).astype(np.float32)
|
|
152
|
+
adata = _uppercase_and_collapse(adata)
|
|
153
|
+
adata.obs_names = [str(b) for b in adata.obs_names]
|
|
154
|
+
adata.obs_names_make_unique()
|
|
155
|
+
logger.info(" final ST: %d cells x %d genes", adata.n_obs, adata.n_vars)
|
|
156
|
+
return adata
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _append_row(csv_path: Path, row: dict, dry_run: bool) -> None:
|
|
160
|
+
with open(csv_path, newline="") as f:
|
|
161
|
+
reader = csv.DictReader(f)
|
|
162
|
+
fields = list(reader.fieldnames or [])
|
|
163
|
+
rows = list(reader)
|
|
164
|
+
|
|
165
|
+
name = row.get("Name")
|
|
166
|
+
sc_n, st_n = row.get("scName"), row.get("stName")
|
|
167
|
+
if name and "Name" in fields:
|
|
168
|
+
duplicate = any(r.get("Name") == str(name) for r in rows)
|
|
169
|
+
elif sc_n and st_n:
|
|
170
|
+
duplicate = any(
|
|
171
|
+
r.get("scName") == str(sc_n) and r.get("stName") == str(st_n) for r in rows
|
|
172
|
+
)
|
|
173
|
+
else:
|
|
174
|
+
duplicate = False
|
|
175
|
+
if duplicate:
|
|
176
|
+
logger.warning(
|
|
177
|
+
" %s already contains this entry - not appending again", csv_path.name
|
|
178
|
+
)
|
|
179
|
+
return
|
|
180
|
+
|
|
181
|
+
full = {k: str(row.get(k, "")) for k in fields}
|
|
182
|
+
unknown = set(row) - set(fields)
|
|
183
|
+
if unknown:
|
|
184
|
+
logger.warning(
|
|
185
|
+
" %s: ignoring columns not in header: %s", csv_path.name, unknown
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
if dry_run:
|
|
189
|
+
logger.info(" [dry-run] would append to %s: %s", csv_path.name, full)
|
|
190
|
+
return
|
|
191
|
+
|
|
192
|
+
shutil.copyfile(csv_path, csv_path.with_suffix(".csv.bak"))
|
|
193
|
+
# Guard against a source file that lacks a trailing newline, which would
|
|
194
|
+
# otherwise glue the new row onto the last existing row.
|
|
195
|
+
with open(csv_path, "rb") as f:
|
|
196
|
+
needs_newline = f.seek(0, 2) > 0 and (f.seek(-1, 2), f.read(1))[1] not in (
|
|
197
|
+
b"\n",
|
|
198
|
+
b"\r",
|
|
199
|
+
)
|
|
200
|
+
with open(csv_path, "a", newline="") as f:
|
|
201
|
+
if needs_newline:
|
|
202
|
+
f.write("\r\n")
|
|
203
|
+
writer = csv.DictWriter(f, fieldnames=fields)
|
|
204
|
+
writer.writerow(full)
|
|
205
|
+
logger.info(
|
|
206
|
+
" appended row to %s (backup: %s)",
|
|
207
|
+
csv_path.name,
|
|
208
|
+
csv_path.with_suffix(".csv.bak").name,
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def main() -> None:
|
|
213
|
+
logging.basicConfig(
|
|
214
|
+
stream=sys.stdout,
|
|
215
|
+
level=logging.INFO,
|
|
216
|
+
format="%(asctime)s %(levelname)s %(message)s",
|
|
217
|
+
)
|
|
218
|
+
p = argparse.ArgumentParser(
|
|
219
|
+
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
|
220
|
+
)
|
|
221
|
+
p.add_argument("--xenium-outs", type=Path, required=True)
|
|
222
|
+
p.add_argument("--scffpe-h5", type=Path, required=True)
|
|
223
|
+
p.add_argument("--annotation-xlsx", type=Path, required=True)
|
|
224
|
+
p.add_argument("--data-root", type=Path, default=DEFAULT_DATA_ROOT)
|
|
225
|
+
p.add_argument("--dry-run", action="store_true")
|
|
226
|
+
args = p.parse_args()
|
|
227
|
+
|
|
228
|
+
sc_dir = args.data_root / "scRNA"
|
|
229
|
+
st_dir = args.data_root / "ST"
|
|
230
|
+
sc_out = sc_dir / f"{NAME}.h5ad"
|
|
231
|
+
st_out = st_dir / f"{NAME}.h5ad"
|
|
232
|
+
|
|
233
|
+
adata_sc = build_scrna(args.scffpe_h5, args.annotation_xlsx)
|
|
234
|
+
adata_st = build_st(args.xenium_outs, args.annotation_xlsx)
|
|
235
|
+
|
|
236
|
+
shared = adata_sc.var_names.intersection(adata_st.var_names)
|
|
237
|
+
logger.info("shared genes (sc-int-st): %d", len(shared))
|
|
238
|
+
n_types = int(adata_sc.obs["cellType"].nunique())
|
|
239
|
+
|
|
240
|
+
if not args.dry_run:
|
|
241
|
+
adata_sc.write_h5ad(sc_out)
|
|
242
|
+
logger.info("wrote %s", sc_out)
|
|
243
|
+
adata_st.write_h5ad(st_out)
|
|
244
|
+
logger.info("wrote %s", st_out)
|
|
245
|
+
else:
|
|
246
|
+
logger.info("[dry-run] would write %s and %s", sc_out, st_out)
|
|
247
|
+
|
|
248
|
+
_append_row(
|
|
249
|
+
sc_dir / "index.csv",
|
|
250
|
+
{
|
|
251
|
+
"Name": NAME,
|
|
252
|
+
"Tissue": TISSUE,
|
|
253
|
+
"Quelle": QUELLE,
|
|
254
|
+
"Autor": AUTOR,
|
|
255
|
+
"Tech": SC_TECH,
|
|
256
|
+
"FeaturedIn": FEATURED_IN,
|
|
257
|
+
"CellCount": adata_sc.n_obs,
|
|
258
|
+
"GeneCount": adata_sc.n_vars,
|
|
259
|
+
"CellTypeKey0": "cellType",
|
|
260
|
+
"NumberCellTypes0": n_types,
|
|
261
|
+
"Links": SC_LINK,
|
|
262
|
+
},
|
|
263
|
+
args.dry_run,
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
_append_row(
|
|
267
|
+
st_dir / "index.csv",
|
|
268
|
+
{
|
|
269
|
+
"Name": NAME,
|
|
270
|
+
"Tissue": TISSUE,
|
|
271
|
+
"Quelle": QUELLE,
|
|
272
|
+
"Autor": AUTOR,
|
|
273
|
+
"Tech": ST_TECH,
|
|
274
|
+
"FeaturedIn": FEATURED_IN,
|
|
275
|
+
"SpotCount": adata_st.n_obs,
|
|
276
|
+
"GeneCount": adata_st.n_vars,
|
|
277
|
+
"Links": ST_LINK,
|
|
278
|
+
},
|
|
279
|
+
args.dry_run,
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
pairs_path = args.data_root / "pairs.csv"
|
|
283
|
+
with open(pairs_path, newline="") as f:
|
|
284
|
+
existing = list(csv.DictReader(f))
|
|
285
|
+
next_id = (
|
|
286
|
+
max(
|
|
287
|
+
(int(r["PairID"]) for r in existing if r.get("PairID", "").isdigit()),
|
|
288
|
+
default=-1,
|
|
289
|
+
)
|
|
290
|
+
+ 1
|
|
291
|
+
)
|
|
292
|
+
_append_row(
|
|
293
|
+
pairs_path,
|
|
294
|
+
{
|
|
295
|
+
"PairID": next_id,
|
|
296
|
+
"scName": NAME,
|
|
297
|
+
"stName": NAME,
|
|
298
|
+
"NumberSharedGenes": len(shared),
|
|
299
|
+
"ExperimentMatch": EXPERIMENT_MATCH,
|
|
300
|
+
},
|
|
301
|
+
args.dry_run,
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
logger.info(
|
|
305
|
+
"Done. Next: aim validate --pairs_csv %s", Path(args.data_root) / "pairs.csv"
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
if __name__ == "__main__":
|
|
310
|
+
main()
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Converts a TSV file to a CSV file by replacing tab delimiters with commas.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def tsv_to_csv(input_path: str, output_path: str) -> None:
|
|
11
|
+
"""
|
|
12
|
+
Read a tab-separated file and write it as a comma-separated file.
|
|
13
|
+
|
|
14
|
+
Args:
|
|
15
|
+
input_path: Path to the input .tsv file.
|
|
16
|
+
output_path: Path where the output .csv file will be written.
|
|
17
|
+
"""
|
|
18
|
+
with open(input_path, newline="", encoding="utf-8") as fin:
|
|
19
|
+
reader = csv.reader(fin, delimiter="\t")
|
|
20
|
+
with open(output_path, "w", newline="", encoding="utf-8") as fout:
|
|
21
|
+
writer = csv.writer(fout) # default delimiter is comma
|
|
22
|
+
for row in reader:
|
|
23
|
+
writer.writerow(row)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
if __name__ == "__main__":
|
|
27
|
+
|
|
28
|
+
input_path = "/MPA_DATA/00_Preparing_Datasets/04_ColorectalCancer/CytoSPACE_example_colon_cancer_merscope/HumanColonCancerPatient2_ST_expressions_cytospace.tsv"
|
|
29
|
+
|
|
30
|
+
base, _ = os.path.splitext(os.path.basename(input_path))
|
|
31
|
+
out_dir = os.path.dirname(input_path) or "."
|
|
32
|
+
output_path = os.path.join(out_dir, f"{base}.csv")
|
|
33
|
+
|
|
34
|
+
try:
|
|
35
|
+
tsv_to_csv(input_path, output_path)
|
|
36
|
+
except Exception as e:
|
|
37
|
+
print(f"Error converting file: {e}", file=sys.stderr)
|
|
38
|
+
|
|
39
|
+
print(f"Successfully converted: {output_path}")
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Convert a dataset from the legacy 6-CSV format to two h5ad files.
|
|
3
|
+
|
|
4
|
+
Legacy format (per dataset folder):
|
|
5
|
+
scData_Cells.csv — cell metadata (cellID, cellType, cellTypeMinor, ...)
|
|
6
|
+
scData_Genes.csv — gene IDs (one per row, convenience only)
|
|
7
|
+
scData_GEP.csv — expression matrix (rows: genes, cols: cells)
|
|
8
|
+
stData_Spots.csv — spot metadata (spotID, cArray0, cArray1)
|
|
9
|
+
stData_Genes.csv — gene IDs (one per row, convenience only)
|
|
10
|
+
stData_GEP.csv — expression matrix (rows: genes, cols: spots)
|
|
11
|
+
|
|
12
|
+
New format:
|
|
13
|
+
sc.h5ad — AnnData (C x G): X = expression, obs = cell metadata, var = gene IDs
|
|
14
|
+
st.h5ad — AnnData (S x G): X = expression, obs = spot metadata,
|
|
15
|
+
var = gene IDs, obsm["spatial"] = (S, 2) float array
|
|
16
|
+
|
|
17
|
+
Usage:
|
|
18
|
+
# Convert a single dataset folder:
|
|
19
|
+
python -m src.data_preparation.convert_csv_to_h5ad -d <dataset_folder>
|
|
20
|
+
|
|
21
|
+
# Convert every subdirectory under a root folder:
|
|
22
|
+
python -m src.data_preparation.convert_csv_to_h5ad -d <root_folder> --all
|
|
23
|
+
|
|
24
|
+
# Also compute PCA + UMAP embeddings for cellxgene compatibility:
|
|
25
|
+
python -m src.data_preparation.convert_csv_to_h5ad -d <dataset_folder> --compute-embeddings
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
import argparse
|
|
29
|
+
import sys
|
|
30
|
+
import logging
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
|
|
33
|
+
import numpy as np
|
|
34
|
+
import pandas as pd
|
|
35
|
+
import anndata as ad
|
|
36
|
+
|
|
37
|
+
logger = logging.getLogger(__name__)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _compute_embeddings(adata_sc: ad.AnnData) -> None:
|
|
41
|
+
"""
|
|
42
|
+
Compute PCA and UMAP embeddings on a normalized copy and store them in adata_sc.
|
|
43
|
+
Operates on a copy to avoid modifying the raw counts in adata_sc.X.
|
|
44
|
+
Stores results as obsm["X_pca"] and obsm["X_umap"].
|
|
45
|
+
"""
|
|
46
|
+
import scanpy as sc
|
|
47
|
+
|
|
48
|
+
logger.info(" Computing embeddings (normalize → log1p → PCA → UMAP)...")
|
|
49
|
+
tmp = adata_sc.copy()
|
|
50
|
+
sc.pp.normalize_total(tmp)
|
|
51
|
+
sc.pp.log1p(tmp)
|
|
52
|
+
n_pcs = min(50, tmp.n_obs - 1, tmp.n_vars - 1)
|
|
53
|
+
sc.pp.pca(tmp, n_comps=n_pcs)
|
|
54
|
+
sc.pp.neighbors(tmp, n_pcs=n_pcs)
|
|
55
|
+
sc.tl.umap(tmp)
|
|
56
|
+
adata_sc.obsm["X_pca"] = tmp.obsm["X_pca"]
|
|
57
|
+
adata_sc.obsm["X_umap"] = tmp.obsm["X_umap"]
|
|
58
|
+
logger.info(" Embeddings stored (X_pca, X_umap)")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def convert_dataset(
|
|
62
|
+
folder: Path, overwrite: bool = False, compute_embeddings: bool = False
|
|
63
|
+
) -> None:
|
|
64
|
+
"""
|
|
65
|
+
Convert one dataset folder from CSV to h5ad.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
folder: Path to the dataset folder containing the 6 CSV files.
|
|
69
|
+
overwrite: If False, skip folders where sc.h5ad and st.h5ad already exist.
|
|
70
|
+
compute_embeddings: If True, compute PCA + UMAP and store in sc.h5ad obsm.
|
|
71
|
+
"""
|
|
72
|
+
sc_out = folder / "sc.h5ad"
|
|
73
|
+
st_out = folder / "st.h5ad"
|
|
74
|
+
|
|
75
|
+
if not overwrite and sc_out.exists() and st_out.exists():
|
|
76
|
+
logger.info("Skipping %s (h5ad files already exist)", folder.name)
|
|
77
|
+
return
|
|
78
|
+
|
|
79
|
+
logger.info("Converting %s ...", folder.name)
|
|
80
|
+
|
|
81
|
+
# --- scRNA ---
|
|
82
|
+
sc_gep_path = folder / "scData_GEP.csv"
|
|
83
|
+
sc_cells_path = folder / "scData_Cells.csv"
|
|
84
|
+
|
|
85
|
+
if not sc_gep_path.exists():
|
|
86
|
+
raise FileNotFoundError(f"Missing scData_GEP.csv in {folder}")
|
|
87
|
+
if not sc_cells_path.exists():
|
|
88
|
+
raise FileNotFoundError(f"Missing scData_Cells.csv in {folder}")
|
|
89
|
+
|
|
90
|
+
# GEP is G x C on disk; transpose to C x G
|
|
91
|
+
sc_gep_df = pd.read_csv(sc_gep_path, index_col=0) # G x C
|
|
92
|
+
sc_cells_df = pd.read_csv(sc_cells_path)
|
|
93
|
+
|
|
94
|
+
# Build obs: use cellID as index (if column exists), else use GEP column names
|
|
95
|
+
if "cellID" in sc_cells_df.columns:
|
|
96
|
+
sc_cells_df = sc_cells_df.set_index("cellID")
|
|
97
|
+
else:
|
|
98
|
+
sc_cells_df.index = sc_gep_df.columns
|
|
99
|
+
|
|
100
|
+
adata_sc = ad.AnnData(
|
|
101
|
+
X=sc_gep_df.T.values.astype(np.float32),
|
|
102
|
+
obs=sc_cells_df.reindex(sc_gep_df.columns),
|
|
103
|
+
var=pd.DataFrame(index=sc_gep_df.index),
|
|
104
|
+
)
|
|
105
|
+
adata_sc.obs_names = list(sc_gep_df.columns)
|
|
106
|
+
adata_sc.var_names = list(sc_gep_df.index)
|
|
107
|
+
|
|
108
|
+
if compute_embeddings:
|
|
109
|
+
_compute_embeddings(adata_sc)
|
|
110
|
+
|
|
111
|
+
adata_sc.write_h5ad(sc_out)
|
|
112
|
+
logger.info(" Written %s", sc_out)
|
|
113
|
+
|
|
114
|
+
# --- Spatial ---
|
|
115
|
+
st_gep_path = folder / "stData_GEP.csv"
|
|
116
|
+
st_spots_path = folder / "stData_Spots.csv"
|
|
117
|
+
|
|
118
|
+
if not st_gep_path.exists():
|
|
119
|
+
raise FileNotFoundError(f"Missing stData_GEP.csv in {folder}")
|
|
120
|
+
if not st_spots_path.exists():
|
|
121
|
+
raise FileNotFoundError(f"Missing stData_Spots.csv in {folder}")
|
|
122
|
+
|
|
123
|
+
# GEP is G x S on disk; transpose to S x G
|
|
124
|
+
st_gep_df = pd.read_csv(st_gep_path, index_col=0) # G x S
|
|
125
|
+
st_spots_df = pd.read_csv(st_spots_path)
|
|
126
|
+
|
|
127
|
+
# Build obs: use spotID as index if available
|
|
128
|
+
if "spotID" in st_spots_df.columns:
|
|
129
|
+
st_spots_df = st_spots_df.set_index("spotID")
|
|
130
|
+
else:
|
|
131
|
+
st_spots_df.index = st_gep_df.columns
|
|
132
|
+
|
|
133
|
+
coords = st_spots_df[["cArray0", "cArray1"]].values.astype(np.float32)
|
|
134
|
+
obs_meta = st_spots_df.drop(columns=["cArray0", "cArray1"], errors="ignore")
|
|
135
|
+
|
|
136
|
+
adata_st = ad.AnnData(
|
|
137
|
+
X=st_gep_df.T.values.astype(np.float32),
|
|
138
|
+
obs=obs_meta.reindex(st_gep_df.columns),
|
|
139
|
+
var=pd.DataFrame(index=st_gep_df.index),
|
|
140
|
+
)
|
|
141
|
+
adata_st.obs_names = list(st_gep_df.columns)
|
|
142
|
+
adata_st.var_names = list(st_gep_df.index)
|
|
143
|
+
adata_st.obsm["spatial"] = coords
|
|
144
|
+
|
|
145
|
+
adata_st.write_h5ad(st_out)
|
|
146
|
+
logger.info(" Written %s", st_out)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def main() -> None:
|
|
150
|
+
logging.basicConfig(
|
|
151
|
+
stream=sys.stdout,
|
|
152
|
+
level=logging.INFO,
|
|
153
|
+
format="%(asctime)s - %(levelname)s - %(message)s",
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
parser = argparse.ArgumentParser(
|
|
157
|
+
description="Convert dataset(s) from 6-CSV format to sc.h5ad + st.h5ad"
|
|
158
|
+
)
|
|
159
|
+
parser.add_argument(
|
|
160
|
+
"-d",
|
|
161
|
+
"--dataset",
|
|
162
|
+
type=Path,
|
|
163
|
+
required=True,
|
|
164
|
+
help="Path to a single dataset folder, or (with --all) the root folder containing dataset subfolders",
|
|
165
|
+
)
|
|
166
|
+
parser.add_argument(
|
|
167
|
+
"--all",
|
|
168
|
+
action="store_true",
|
|
169
|
+
help="Convert every immediate subdirectory of -d",
|
|
170
|
+
)
|
|
171
|
+
parser.add_argument(
|
|
172
|
+
"--overwrite",
|
|
173
|
+
action="store_true",
|
|
174
|
+
help="Overwrite existing h5ad files (default: skip if both already exist)",
|
|
175
|
+
)
|
|
176
|
+
parser.add_argument(
|
|
177
|
+
"--compute-embeddings",
|
|
178
|
+
action="store_true",
|
|
179
|
+
help="Compute PCA + UMAP embeddings and store in sc.h5ad (required for cellxgene)",
|
|
180
|
+
)
|
|
181
|
+
args = parser.parse_args()
|
|
182
|
+
|
|
183
|
+
if args.all:
|
|
184
|
+
root = args.dataset
|
|
185
|
+
if not root.is_dir():
|
|
186
|
+
print(f"ERROR: {root} is not a directory", file=sys.stderr)
|
|
187
|
+
sys.exit(1)
|
|
188
|
+
folders = sorted(p for p in root.iterdir() if p.is_dir())
|
|
189
|
+
if not folders:
|
|
190
|
+
print(f"No subdirectories found under {root}", file=sys.stderr)
|
|
191
|
+
sys.exit(1)
|
|
192
|
+
else:
|
|
193
|
+
folders = [args.dataset]
|
|
194
|
+
|
|
195
|
+
errors = []
|
|
196
|
+
for folder in folders:
|
|
197
|
+
try:
|
|
198
|
+
convert_dataset(
|
|
199
|
+
folder,
|
|
200
|
+
overwrite=args.overwrite,
|
|
201
|
+
compute_embeddings=args.compute_embeddings,
|
|
202
|
+
)
|
|
203
|
+
except Exception as e:
|
|
204
|
+
logger.error("Failed to convert %s: %s", folder.name, e)
|
|
205
|
+
errors.append((folder.name, e))
|
|
206
|
+
|
|
207
|
+
if errors:
|
|
208
|
+
print(f"\n{len(errors)} dataset(s) failed:", file=sys.stderr)
|
|
209
|
+
for name, exc in errors:
|
|
210
|
+
print(f" {name}: {exc}", file=sys.stderr)
|
|
211
|
+
sys.exit(1)
|
|
212
|
+
else:
|
|
213
|
+
print("\nAll conversions completed successfully.")
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
if __name__ == "__main__":
|
|
217
|
+
main()
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Creates a derived dataset containing only the top N highly variable genes (HVGs)
|
|
2
|
+
# from the scRNA-seq data. ST data is copied unchanged. Shared genes that are
|
|
3
|
+
# not among the top HVGs are appended to the selection to preserve cross-modal overlap.
|
|
4
|
+
# Reads from sc.h5ad + st.h5ad and writes sc.h5ad + st.h5ad.
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
import anndata as ad
|
|
9
|
+
import scanpy as sc
|
|
10
|
+
|
|
11
|
+
if __name__ == "__main__":
|
|
12
|
+
|
|
13
|
+
dataset_from = "/Users/domi/Dev/MPA_Workspace/MPA_DATA/01_Datasets/03_MouseSSP"
|
|
14
|
+
dataset_to = (
|
|
15
|
+
"/Users/domi/Dev/MPA_Workspace/MPA_DATA/01_Datasets/03_MouseSSP_HVG_2000"
|
|
16
|
+
)
|
|
17
|
+
N = 2000
|
|
18
|
+
|
|
19
|
+
""" Load datasets """
|
|
20
|
+
|
|
21
|
+
scData = ad.read_h5ad(os.path.join(dataset_from, "sc.h5ad")) # C x G
|
|
22
|
+
print("scRNA Data loaded")
|
|
23
|
+
stData = ad.read_h5ad(os.path.join(dataset_from, "st.h5ad")) # S x G
|
|
24
|
+
print("ST Data loaded")
|
|
25
|
+
|
|
26
|
+
""" Select top N HVGs, keeping all shared genes """
|
|
27
|
+
|
|
28
|
+
# Shared genes between scRNA and ST
|
|
29
|
+
sharedGenes = scData.var_names.intersection(stData.var_names)
|
|
30
|
+
|
|
31
|
+
# Compute HVGs on a normalized log-transformed copy
|
|
32
|
+
scDataLog = scData.copy()
|
|
33
|
+
sc.pp.normalize_total(scDataLog)
|
|
34
|
+
sc.pp.log1p(scDataLog)
|
|
35
|
+
sc.pp.highly_variable_genes(scDataLog, n_top_genes=N, inplace=True)
|
|
36
|
+
|
|
37
|
+
assert "highly_variable" in scDataLog.var.columns
|
|
38
|
+
hvgs = list(scDataLog.var_names[scDataLog.var["highly_variable"].values])
|
|
39
|
+
|
|
40
|
+
if len(hvgs) == 0:
|
|
41
|
+
raise RuntimeError("No HVGs selected — check input data and N value")
|
|
42
|
+
|
|
43
|
+
# Append shared genes that were not selected as HVGs
|
|
44
|
+
for sharedGene in sharedGenes:
|
|
45
|
+
if sharedGene not in hvgs:
|
|
46
|
+
print(
|
|
47
|
+
f"Warning: Shared gene {sharedGene} not selected as HVG - added manually"
|
|
48
|
+
)
|
|
49
|
+
hvgs.append(sharedGene)
|
|
50
|
+
print(
|
|
51
|
+
f"Appended {len(hvgs) - N} shared genes to HVG list. Total genes selected:",
|
|
52
|
+
len(hvgs),
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
""" Create and save HVG-filtered scRNA dataset """
|
|
56
|
+
|
|
57
|
+
scData_HVG = scData[:, hvgs].copy()
|
|
58
|
+
|
|
59
|
+
os.makedirs(dataset_to, exist_ok=True)
|
|
60
|
+
scData_HVG.write_h5ad(os.path.join(dataset_to, "sc.h5ad"))
|
|
61
|
+
print(f"Saved sc.h5ad: {scData_HVG.n_obs} cells x {scData_HVG.n_vars} genes")
|
|
62
|
+
|
|
63
|
+
""" Copy ST dataset unchanged """
|
|
64
|
+
|
|
65
|
+
shutil.copy(
|
|
66
|
+
os.path.join(dataset_from, "st.h5ad"),
|
|
67
|
+
os.path.join(dataset_to, "st.h5ad"),
|
|
68
|
+
)
|
|
69
|
+
print("Copied st.h5ad unchanged")
|