spatial-aim 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. aim/__init__.py +47 -0
  2. aim/__main__.py +6 -0
  3. aim/adata_schema.py +80 -0
  4. aim/aggregation.py +118 -0
  5. aim/aim_config.py +76 -0
  6. aim/analysis/__init__.py +13 -0
  7. aim/analysis/analysis.py +72 -0
  8. aim/analysis/biology.py +205 -0
  9. aim/analysis/confidence.py +49 -0
  10. aim/analysis/ksweep.py +184 -0
  11. aim/analysis/loading.py +70 -0
  12. aim/analysis/onehot.py +41 -0
  13. aim/analysis/reconstruction.py +131 -0
  14. aim/analysis/topology.py +142 -0
  15. aim/analysis/utils.py +16 -0
  16. aim/cli.py +371 -0
  17. aim/clustering.py +242 -0
  18. aim/data/__init__.py +0 -0
  19. aim/data/prepare/__init__.py +0 -0
  20. aim/data/prepare/build_imaging_pairs.py +397 -0
  21. aim/data/prepare/build_xenium_breast_pair.py +310 -0
  22. aim/data/prepare/convertTsvToCsv.py +39 -0
  23. aim/data/prepare/convert_csv_to_h5ad.py +217 -0
  24. aim/data/prepare/createOnlyHVGDataset.py +69 -0
  25. aim/data/prepare/createSmallerDataset.py +102 -0
  26. aim/data/prepare/normalize_gene_names.py +56 -0
  27. aim/data/prepare/perturb_sc_noise.py +107 -0
  28. aim/data/prepare/register_st_patches.py +180 -0
  29. aim/data/prepare/split_st_into_islands.py +551 -0
  30. aim/data/prepare/split_st_into_patches.py +240 -0
  31. aim/data/prepare/subset_st_hvg.py +43 -0
  32. aim/data/validate/__init__.py +35 -0
  33. aim/data/validate/common.py +141 -0
  34. aim/data/validate/runner.py +335 -0
  35. aim/data/validate/validate_pair.py +53 -0
  36. aim/data/validate/validate_sc.py +103 -0
  37. aim/data/validate/validate_st.py +74 -0
  38. aim/gui/__init__.py +8 -0
  39. aim/gui/__main__.py +77 -0
  40. aim/gui/app.py +1784 -0
  41. aim/gui/compute.py +106 -0
  42. aim/gui/data_access.py +292 -0
  43. aim/gui/render.py +1679 -0
  44. aim/gui/scaffold.py +129 -0
  45. aim/gui/widgets.py +675 -0
  46. aim/io.py +196 -0
  47. aim/mapping/__init__.py +14 -0
  48. aim/mapping/base.py +67 -0
  49. aim/mapping/confidence.py +51 -0
  50. aim/mapping/nearest_centroid.py +33 -0
  51. aim/mapping/reference.py +156 -0
  52. aim/mapping/wann.py +208 -0
  53. aim/metrics/__init__.py +26 -0
  54. aim/metrics/biology.py +69 -0
  55. aim/metrics/cossim.py +127 -0
  56. aim/metrics/kselection.py +272 -0
  57. aim/metrics/onehot.py +54 -0
  58. aim/metrics/reconstruction.py +73 -0
  59. aim/metrics/topology.py +62 -0
  60. aim/reference_aligners/__init__.py +0 -0
  61. aim/reference_aligners/environment_dot.yml +23 -0
  62. aim/reference_aligners/environment_tacco.yml +28 -0
  63. aim/reference_aligners/environment_tangram.yml +16 -0
  64. aim/reference_aligners/registry.py +350 -0
  65. aim/reference_aligners/run_dot.R +172 -0
  66. aim/reference_aligners/run_dot.py +116 -0
  67. aim/reference_aligners/run_tacco.py +143 -0
  68. aim/reference_aligners/run_tangram.py +210 -0
  69. aim/sweep.py +261 -0
  70. aim/tree.py +86 -0
  71. spatial_aim-1.0.0.dist-info/METADATA +1005 -0
  72. spatial_aim-1.0.0.dist-info/RECORD +76 -0
  73. spatial_aim-1.0.0.dist-info/WHEEL +5 -0
  74. spatial_aim-1.0.0.dist-info/entry_points.txt +2 -0
  75. spatial_aim-1.0.0.dist-info/licenses/LICENSE +674 -0
  76. spatial_aim-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,310 @@
1
+ """
2
+ Ingest the 10x Genomics / Janesick et al. 2023 human breast cancer dataset into the
3
+ database as a new scRNA <-> ST pair (dataset id 11).
4
+
5
+ Sources (as downloaded, see README "Datensaetze"):
6
+ scRNA reference scFFPE-seq (Chromium Flex), GSM7782698 raw feature-barcode matrix
7
+ + cell-type annotations from the scFFPE-Seq sheet of
8
+ GSE243275_Barcode_Cell_Type_Matrices.xlsx
9
+ ST (high-res) Xenium In Situ (313-plex), <outs>/cell_feature_matrix.h5
10
+ + cell centroids from <outs>/cells.parquet
11
+ + supervised Xenium labels from the "Xenium R1 Fig1-5 (supervised)" sheet
12
+
13
+ Output (matching the database h5ad contract, see CLAUDE.md):
14
+ scRNA/11_human-breast-cancer.h5ad X=raw counts (annotated cells x genes),
15
+ obs['cellType'], var uppercase + unique
16
+ ST/11_human-breast-cancer.h5ad X=raw counts (313 panel), obsm['spatial'],
17
+ obs['cellType'] (Xenium ground truth)
18
+ and appends one row to scRNA/index.csv, ST/index.csv and pairs.csv (originals backed
19
+ up to *.bak). ExperimentMatch = same_paper.
20
+
21
+ Usage:
22
+ python -m data_preparation.build_xenium_breast_pair \
23
+ --xenium-outs "C:/Users/zi69hebi/Downloads/Xenium_FFPE_Human_Breast_Cancer_Rep1_outs/outs" \
24
+ --scffpe-h5 "C:/Users/zi69hebi/Downloads/GSE243275/GSM7782698_count_raw_feature_bc_matrix.h5" \
25
+ --annotation-xlsx "C:/Users/zi69hebi/Downloads/GSE243275_Barcode_Cell_Type_Matrices.xlsx" \
26
+ [--data-root "C:/Users/zi69hebi/Dev/10_Alignment/Data/01_Datasets"] \
27
+ [--dry-run]
28
+ """
29
+
30
+ import argparse
31
+ import csv
32
+ import logging
33
+ import shutil
34
+ import sys
35
+ from pathlib import Path
36
+
37
+ import anndata as ad
38
+ import numpy as np
39
+ import pandas as pd
40
+ import scanpy as sc
41
+ import scipy.sparse as sp
42
+
43
+ logger = logging.getLogger(__name__)
44
+
45
+ DATASET_ID = "11"
46
+ NAME = "11_human-breast-cancer"
47
+ TISSUE = "Human breast cancer"
48
+ QUELLE = "https://www.nature.com/articles/s41467-023-43458-x"
49
+ AUTOR = "Janesick et al"
50
+ FEATURED_IN = "DOT"
51
+ SC_TECH = "Chromium Flex (scFFPE-seq)"
52
+ ST_TECH = "Xenium"
53
+ SC_LINK = "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE243275"
54
+ ST_LINK = (
55
+ "https://www.10xgenomics.com/products/xenium-in-situ/preview-dataset-human-breast"
56
+ )
57
+ EXPERIMENT_MATCH = "same_paper"
58
+
59
+ SCFFPE_SHEET = "scFFPE-Seq"
60
+ XENIUM_SHEET = "Xenium R1 Fig1-5 (supervised)"
61
+
62
+ DEFAULT_DATA_ROOT = Path(r"/01_Datasets")
63
+
64
+
65
+ def _uppercase_and_collapse(adata: ad.AnnData) -> ad.AnnData:
66
+ """Uppercase var_names, then sum counts of any genes that collide into one column.
67
+
68
+ Keeps var_names unique + uppercase so `aim validate` passes and shared-gene
69
+ intersection with the ST panel is exact.
70
+ """
71
+ names = pd.Index([g.upper() for g in adata.var_names])
72
+ if names.is_unique:
73
+ adata.var_names = names
74
+ return adata
75
+
76
+ uniq = names.unique()
77
+ col_of = {g: i for i, g in enumerate(uniq)}
78
+ cols = np.fromiter((col_of[g] for g in names), dtype=np.int64, count=len(names))
79
+ # (n_old x n_new) 0/1 grouping matrix; X @ G sums duplicate columns
80
+ group = sp.csr_matrix(
81
+ (np.ones(len(names), dtype=np.float32), (np.arange(len(names)), cols)),
82
+ shape=(len(names), len(uniq)),
83
+ )
84
+ x_new = sp.csr_matrix(adata.X).dot(group)
85
+ n_collapsed = len(names) - len(uniq)
86
+ logger.info(
87
+ " collapsed %d duplicate gene symbol(s) by summing counts", n_collapsed
88
+ )
89
+ out = ad.AnnData(
90
+ X=x_new.astype(np.float32),
91
+ obs=adata.obs.copy(),
92
+ var=pd.DataFrame(index=uniq),
93
+ )
94
+ return out
95
+
96
+
97
+ def build_scrna(scffpe_h5: Path, annotation_xlsx: Path) -> ad.AnnData:
98
+ logger.info("scRNA: reading raw matrix %s", scffpe_h5.name)
99
+ adata = sc.read_10x_h5(scffpe_h5)
100
+ logger.info(" raw matrix: %d barcodes x %d genes", adata.n_obs, adata.n_vars)
101
+
102
+ ann = pd.read_excel(annotation_xlsx, sheet_name=SCFFPE_SHEET)
103
+ ann = ann.set_index("Barcode")["Annotation"]
104
+ logger.info(
105
+ " annotation sheet '%s': %d cells, %d types",
106
+ SCFFPE_SHEET,
107
+ len(ann),
108
+ ann.nunique(),
109
+ )
110
+
111
+ keep = adata.obs_names.intersection(ann.index)
112
+ missing = len(ann) - len(keep)
113
+ if missing:
114
+ logger.warning(" %d annotated barcode(s) not found in raw matrix", missing)
115
+ adata = adata[keep].copy()
116
+ adata.obs["cellType"] = ann.reindex(adata.obs_names).astype(str).values
117
+ logger.info(" subset to %d annotated cells", adata.n_obs)
118
+
119
+ # raw integer counts as float32 sparse
120
+ adata.X = sp.csr_matrix(adata.X).astype(np.float32)
121
+ adata = _uppercase_and_collapse(adata)
122
+ adata.obs_names = [str(b) for b in adata.obs_names]
123
+ adata.obs_names_make_unique()
124
+ logger.info(" final scRNA: %d cells x %d genes", adata.n_obs, adata.n_vars)
125
+ return adata
126
+
127
+
128
+ def build_st(xenium_outs: Path, annotation_xlsx: Path) -> ad.AnnData:
129
+ h5 = xenium_outs / "cell_feature_matrix.h5"
130
+ logger.info("ST: reading Xenium matrix %s", h5.name)
131
+ adata = sc.read_10x_h5(h5, gex_only=False)
132
+ if "feature_types" in adata.var:
133
+ is_gene = adata.var["feature_types"].astype(str) == "Gene Expression"
134
+ adata = adata[:, is_gene.values].copy()
135
+ logger.info(" kept %d Gene Expression features", adata.n_vars)
136
+
137
+ cells = pd.read_parquet(xenium_outs / "cells.parquet")
138
+ cells["cell_id"] = cells["cell_id"].astype(str)
139
+ cells = cells.set_index("cell_id")
140
+ coords = cells.reindex(adata.obs_names)[["x_centroid", "y_centroid"]]
141
+ if coords.isnull().any().any():
142
+ raise RuntimeError("ST: some cell barcodes have no centroid in cells.parquet")
143
+ adata.obsm["spatial"] = coords.to_numpy(dtype=np.float32)
144
+
145
+ # Xenium supervised ground-truth labels (bonus obs column; not required by schema)
146
+ xann = pd.read_excel(annotation_xlsx, sheet_name=XENIUM_SHEET)
147
+ xann = xann.set_index(xann.columns[0])[xann.columns[1]]
148
+ xann.index = xann.index.astype(str)
149
+ adata.obs["cellType"] = xann.reindex(adata.obs_names).astype(str).values
150
+
151
+ adata.X = sp.csr_matrix(adata.X).astype(np.float32)
152
+ adata = _uppercase_and_collapse(adata)
153
+ adata.obs_names = [str(b) for b in adata.obs_names]
154
+ adata.obs_names_make_unique()
155
+ logger.info(" final ST: %d cells x %d genes", adata.n_obs, adata.n_vars)
156
+ return adata
157
+
158
+
159
+ def _append_row(csv_path: Path, row: dict, dry_run: bool) -> None:
160
+ with open(csv_path, newline="") as f:
161
+ reader = csv.DictReader(f)
162
+ fields = list(reader.fieldnames or [])
163
+ rows = list(reader)
164
+
165
+ name = row.get("Name")
166
+ sc_n, st_n = row.get("scName"), row.get("stName")
167
+ if name and "Name" in fields:
168
+ duplicate = any(r.get("Name") == str(name) for r in rows)
169
+ elif sc_n and st_n:
170
+ duplicate = any(
171
+ r.get("scName") == str(sc_n) and r.get("stName") == str(st_n) for r in rows
172
+ )
173
+ else:
174
+ duplicate = False
175
+ if duplicate:
176
+ logger.warning(
177
+ " %s already contains this entry - not appending again", csv_path.name
178
+ )
179
+ return
180
+
181
+ full = {k: str(row.get(k, "")) for k in fields}
182
+ unknown = set(row) - set(fields)
183
+ if unknown:
184
+ logger.warning(
185
+ " %s: ignoring columns not in header: %s", csv_path.name, unknown
186
+ )
187
+
188
+ if dry_run:
189
+ logger.info(" [dry-run] would append to %s: %s", csv_path.name, full)
190
+ return
191
+
192
+ shutil.copyfile(csv_path, csv_path.with_suffix(".csv.bak"))
193
+ # Guard against a source file that lacks a trailing newline, which would
194
+ # otherwise glue the new row onto the last existing row.
195
+ with open(csv_path, "rb") as f:
196
+ needs_newline = f.seek(0, 2) > 0 and (f.seek(-1, 2), f.read(1))[1] not in (
197
+ b"\n",
198
+ b"\r",
199
+ )
200
+ with open(csv_path, "a", newline="") as f:
201
+ if needs_newline:
202
+ f.write("\r\n")
203
+ writer = csv.DictWriter(f, fieldnames=fields)
204
+ writer.writerow(full)
205
+ logger.info(
206
+ " appended row to %s (backup: %s)",
207
+ csv_path.name,
208
+ csv_path.with_suffix(".csv.bak").name,
209
+ )
210
+
211
+
212
+ def main() -> None:
213
+ logging.basicConfig(
214
+ stream=sys.stdout,
215
+ level=logging.INFO,
216
+ format="%(asctime)s %(levelname)s %(message)s",
217
+ )
218
+ p = argparse.ArgumentParser(
219
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
220
+ )
221
+ p.add_argument("--xenium-outs", type=Path, required=True)
222
+ p.add_argument("--scffpe-h5", type=Path, required=True)
223
+ p.add_argument("--annotation-xlsx", type=Path, required=True)
224
+ p.add_argument("--data-root", type=Path, default=DEFAULT_DATA_ROOT)
225
+ p.add_argument("--dry-run", action="store_true")
226
+ args = p.parse_args()
227
+
228
+ sc_dir = args.data_root / "scRNA"
229
+ st_dir = args.data_root / "ST"
230
+ sc_out = sc_dir / f"{NAME}.h5ad"
231
+ st_out = st_dir / f"{NAME}.h5ad"
232
+
233
+ adata_sc = build_scrna(args.scffpe_h5, args.annotation_xlsx)
234
+ adata_st = build_st(args.xenium_outs, args.annotation_xlsx)
235
+
236
+ shared = adata_sc.var_names.intersection(adata_st.var_names)
237
+ logger.info("shared genes (sc-int-st): %d", len(shared))
238
+ n_types = int(adata_sc.obs["cellType"].nunique())
239
+
240
+ if not args.dry_run:
241
+ adata_sc.write_h5ad(sc_out)
242
+ logger.info("wrote %s", sc_out)
243
+ adata_st.write_h5ad(st_out)
244
+ logger.info("wrote %s", st_out)
245
+ else:
246
+ logger.info("[dry-run] would write %s and %s", sc_out, st_out)
247
+
248
+ _append_row(
249
+ sc_dir / "index.csv",
250
+ {
251
+ "Name": NAME,
252
+ "Tissue": TISSUE,
253
+ "Quelle": QUELLE,
254
+ "Autor": AUTOR,
255
+ "Tech": SC_TECH,
256
+ "FeaturedIn": FEATURED_IN,
257
+ "CellCount": adata_sc.n_obs,
258
+ "GeneCount": adata_sc.n_vars,
259
+ "CellTypeKey0": "cellType",
260
+ "NumberCellTypes0": n_types,
261
+ "Links": SC_LINK,
262
+ },
263
+ args.dry_run,
264
+ )
265
+
266
+ _append_row(
267
+ st_dir / "index.csv",
268
+ {
269
+ "Name": NAME,
270
+ "Tissue": TISSUE,
271
+ "Quelle": QUELLE,
272
+ "Autor": AUTOR,
273
+ "Tech": ST_TECH,
274
+ "FeaturedIn": FEATURED_IN,
275
+ "SpotCount": adata_st.n_obs,
276
+ "GeneCount": adata_st.n_vars,
277
+ "Links": ST_LINK,
278
+ },
279
+ args.dry_run,
280
+ )
281
+
282
+ pairs_path = args.data_root / "pairs.csv"
283
+ with open(pairs_path, newline="") as f:
284
+ existing = list(csv.DictReader(f))
285
+ next_id = (
286
+ max(
287
+ (int(r["PairID"]) for r in existing if r.get("PairID", "").isdigit()),
288
+ default=-1,
289
+ )
290
+ + 1
291
+ )
292
+ _append_row(
293
+ pairs_path,
294
+ {
295
+ "PairID": next_id,
296
+ "scName": NAME,
297
+ "stName": NAME,
298
+ "NumberSharedGenes": len(shared),
299
+ "ExperimentMatch": EXPERIMENT_MATCH,
300
+ },
301
+ args.dry_run,
302
+ )
303
+
304
+ logger.info(
305
+ "Done. Next: aim validate --pairs_csv %s", Path(args.data_root) / "pairs.csv"
306
+ )
307
+
308
+
309
+ if __name__ == "__main__":
310
+ main()
@@ -0,0 +1,39 @@
1
+ """
2
+ Converts a TSV file to a CSV file by replacing tab delimiters with commas.
3
+ """
4
+
5
+ import csv
6
+ import os
7
+ import sys
8
+
9
+
10
+ def tsv_to_csv(input_path: str, output_path: str) -> None:
11
+ """
12
+ Read a tab-separated file and write it as a comma-separated file.
13
+
14
+ Args:
15
+ input_path: Path to the input .tsv file.
16
+ output_path: Path where the output .csv file will be written.
17
+ """
18
+ with open(input_path, newline="", encoding="utf-8") as fin:
19
+ reader = csv.reader(fin, delimiter="\t")
20
+ with open(output_path, "w", newline="", encoding="utf-8") as fout:
21
+ writer = csv.writer(fout) # default delimiter is comma
22
+ for row in reader:
23
+ writer.writerow(row)
24
+
25
+
26
+ if __name__ == "__main__":
27
+
28
+ input_path = "/MPA_DATA/00_Preparing_Datasets/04_ColorectalCancer/CytoSPACE_example_colon_cancer_merscope/HumanColonCancerPatient2_ST_expressions_cytospace.tsv"
29
+
30
+ base, _ = os.path.splitext(os.path.basename(input_path))
31
+ out_dir = os.path.dirname(input_path) or "."
32
+ output_path = os.path.join(out_dir, f"{base}.csv")
33
+
34
+ try:
35
+ tsv_to_csv(input_path, output_path)
36
+ except Exception as e:
37
+ print(f"Error converting file: {e}", file=sys.stderr)
38
+
39
+ print(f"Successfully converted: {output_path}")
@@ -0,0 +1,217 @@
1
+ """
2
+ Convert a dataset from the legacy 6-CSV format to two h5ad files.
3
+
4
+ Legacy format (per dataset folder):
5
+ scData_Cells.csv — cell metadata (cellID, cellType, cellTypeMinor, ...)
6
+ scData_Genes.csv — gene IDs (one per row, convenience only)
7
+ scData_GEP.csv — expression matrix (rows: genes, cols: cells)
8
+ stData_Spots.csv — spot metadata (spotID, cArray0, cArray1)
9
+ stData_Genes.csv — gene IDs (one per row, convenience only)
10
+ stData_GEP.csv — expression matrix (rows: genes, cols: spots)
11
+
12
+ New format:
13
+ sc.h5ad — AnnData (C x G): X = expression, obs = cell metadata, var = gene IDs
14
+ st.h5ad — AnnData (S x G): X = expression, obs = spot metadata,
15
+ var = gene IDs, obsm["spatial"] = (S, 2) float array
16
+
17
+ Usage:
18
+ # Convert a single dataset folder:
19
+ python -m src.data_preparation.convert_csv_to_h5ad -d <dataset_folder>
20
+
21
+ # Convert every subdirectory under a root folder:
22
+ python -m src.data_preparation.convert_csv_to_h5ad -d <root_folder> --all
23
+
24
+ # Also compute PCA + UMAP embeddings for cellxgene compatibility:
25
+ python -m src.data_preparation.convert_csv_to_h5ad -d <dataset_folder> --compute-embeddings
26
+ """
27
+
28
+ import argparse
29
+ import sys
30
+ import logging
31
+ from pathlib import Path
32
+
33
+ import numpy as np
34
+ import pandas as pd
35
+ import anndata as ad
36
+
37
+ logger = logging.getLogger(__name__)
38
+
39
+
40
+ def _compute_embeddings(adata_sc: ad.AnnData) -> None:
41
+ """
42
+ Compute PCA and UMAP embeddings on a normalized copy and store them in adata_sc.
43
+ Operates on a copy to avoid modifying the raw counts in adata_sc.X.
44
+ Stores results as obsm["X_pca"] and obsm["X_umap"].
45
+ """
46
+ import scanpy as sc
47
+
48
+ logger.info(" Computing embeddings (normalize → log1p → PCA → UMAP)...")
49
+ tmp = adata_sc.copy()
50
+ sc.pp.normalize_total(tmp)
51
+ sc.pp.log1p(tmp)
52
+ n_pcs = min(50, tmp.n_obs - 1, tmp.n_vars - 1)
53
+ sc.pp.pca(tmp, n_comps=n_pcs)
54
+ sc.pp.neighbors(tmp, n_pcs=n_pcs)
55
+ sc.tl.umap(tmp)
56
+ adata_sc.obsm["X_pca"] = tmp.obsm["X_pca"]
57
+ adata_sc.obsm["X_umap"] = tmp.obsm["X_umap"]
58
+ logger.info(" Embeddings stored (X_pca, X_umap)")
59
+
60
+
61
+ def convert_dataset(
62
+ folder: Path, overwrite: bool = False, compute_embeddings: bool = False
63
+ ) -> None:
64
+ """
65
+ Convert one dataset folder from CSV to h5ad.
66
+
67
+ Args:
68
+ folder: Path to the dataset folder containing the 6 CSV files.
69
+ overwrite: If False, skip folders where sc.h5ad and st.h5ad already exist.
70
+ compute_embeddings: If True, compute PCA + UMAP and store in sc.h5ad obsm.
71
+ """
72
+ sc_out = folder / "sc.h5ad"
73
+ st_out = folder / "st.h5ad"
74
+
75
+ if not overwrite and sc_out.exists() and st_out.exists():
76
+ logger.info("Skipping %s (h5ad files already exist)", folder.name)
77
+ return
78
+
79
+ logger.info("Converting %s ...", folder.name)
80
+
81
+ # --- scRNA ---
82
+ sc_gep_path = folder / "scData_GEP.csv"
83
+ sc_cells_path = folder / "scData_Cells.csv"
84
+
85
+ if not sc_gep_path.exists():
86
+ raise FileNotFoundError(f"Missing scData_GEP.csv in {folder}")
87
+ if not sc_cells_path.exists():
88
+ raise FileNotFoundError(f"Missing scData_Cells.csv in {folder}")
89
+
90
+ # GEP is G x C on disk; transpose to C x G
91
+ sc_gep_df = pd.read_csv(sc_gep_path, index_col=0) # G x C
92
+ sc_cells_df = pd.read_csv(sc_cells_path)
93
+
94
+ # Build obs: use cellID as index (if column exists), else use GEP column names
95
+ if "cellID" in sc_cells_df.columns:
96
+ sc_cells_df = sc_cells_df.set_index("cellID")
97
+ else:
98
+ sc_cells_df.index = sc_gep_df.columns
99
+
100
+ adata_sc = ad.AnnData(
101
+ X=sc_gep_df.T.values.astype(np.float32),
102
+ obs=sc_cells_df.reindex(sc_gep_df.columns),
103
+ var=pd.DataFrame(index=sc_gep_df.index),
104
+ )
105
+ adata_sc.obs_names = list(sc_gep_df.columns)
106
+ adata_sc.var_names = list(sc_gep_df.index)
107
+
108
+ if compute_embeddings:
109
+ _compute_embeddings(adata_sc)
110
+
111
+ adata_sc.write_h5ad(sc_out)
112
+ logger.info(" Written %s", sc_out)
113
+
114
+ # --- Spatial ---
115
+ st_gep_path = folder / "stData_GEP.csv"
116
+ st_spots_path = folder / "stData_Spots.csv"
117
+
118
+ if not st_gep_path.exists():
119
+ raise FileNotFoundError(f"Missing stData_GEP.csv in {folder}")
120
+ if not st_spots_path.exists():
121
+ raise FileNotFoundError(f"Missing stData_Spots.csv in {folder}")
122
+
123
+ # GEP is G x S on disk; transpose to S x G
124
+ st_gep_df = pd.read_csv(st_gep_path, index_col=0) # G x S
125
+ st_spots_df = pd.read_csv(st_spots_path)
126
+
127
+ # Build obs: use spotID as index if available
128
+ if "spotID" in st_spots_df.columns:
129
+ st_spots_df = st_spots_df.set_index("spotID")
130
+ else:
131
+ st_spots_df.index = st_gep_df.columns
132
+
133
+ coords = st_spots_df[["cArray0", "cArray1"]].values.astype(np.float32)
134
+ obs_meta = st_spots_df.drop(columns=["cArray0", "cArray1"], errors="ignore")
135
+
136
+ adata_st = ad.AnnData(
137
+ X=st_gep_df.T.values.astype(np.float32),
138
+ obs=obs_meta.reindex(st_gep_df.columns),
139
+ var=pd.DataFrame(index=st_gep_df.index),
140
+ )
141
+ adata_st.obs_names = list(st_gep_df.columns)
142
+ adata_st.var_names = list(st_gep_df.index)
143
+ adata_st.obsm["spatial"] = coords
144
+
145
+ adata_st.write_h5ad(st_out)
146
+ logger.info(" Written %s", st_out)
147
+
148
+
149
+ def main() -> None:
150
+ logging.basicConfig(
151
+ stream=sys.stdout,
152
+ level=logging.INFO,
153
+ format="%(asctime)s - %(levelname)s - %(message)s",
154
+ )
155
+
156
+ parser = argparse.ArgumentParser(
157
+ description="Convert dataset(s) from 6-CSV format to sc.h5ad + st.h5ad"
158
+ )
159
+ parser.add_argument(
160
+ "-d",
161
+ "--dataset",
162
+ type=Path,
163
+ required=True,
164
+ help="Path to a single dataset folder, or (with --all) the root folder containing dataset subfolders",
165
+ )
166
+ parser.add_argument(
167
+ "--all",
168
+ action="store_true",
169
+ help="Convert every immediate subdirectory of -d",
170
+ )
171
+ parser.add_argument(
172
+ "--overwrite",
173
+ action="store_true",
174
+ help="Overwrite existing h5ad files (default: skip if both already exist)",
175
+ )
176
+ parser.add_argument(
177
+ "--compute-embeddings",
178
+ action="store_true",
179
+ help="Compute PCA + UMAP embeddings and store in sc.h5ad (required for cellxgene)",
180
+ )
181
+ args = parser.parse_args()
182
+
183
+ if args.all:
184
+ root = args.dataset
185
+ if not root.is_dir():
186
+ print(f"ERROR: {root} is not a directory", file=sys.stderr)
187
+ sys.exit(1)
188
+ folders = sorted(p for p in root.iterdir() if p.is_dir())
189
+ if not folders:
190
+ print(f"No subdirectories found under {root}", file=sys.stderr)
191
+ sys.exit(1)
192
+ else:
193
+ folders = [args.dataset]
194
+
195
+ errors = []
196
+ for folder in folders:
197
+ try:
198
+ convert_dataset(
199
+ folder,
200
+ overwrite=args.overwrite,
201
+ compute_embeddings=args.compute_embeddings,
202
+ )
203
+ except Exception as e:
204
+ logger.error("Failed to convert %s: %s", folder.name, e)
205
+ errors.append((folder.name, e))
206
+
207
+ if errors:
208
+ print(f"\n{len(errors)} dataset(s) failed:", file=sys.stderr)
209
+ for name, exc in errors:
210
+ print(f" {name}: {exc}", file=sys.stderr)
211
+ sys.exit(1)
212
+ else:
213
+ print("\nAll conversions completed successfully.")
214
+
215
+
216
+ if __name__ == "__main__":
217
+ main()
@@ -0,0 +1,69 @@
1
+ # Creates a derived dataset containing only the top N highly variable genes (HVGs)
2
+ # from the scRNA-seq data. ST data is copied unchanged. Shared genes that are
3
+ # not among the top HVGs are appended to the selection to preserve cross-modal overlap.
4
+ # Reads from sc.h5ad + st.h5ad and writes sc.h5ad + st.h5ad.
5
+
6
+ import os
7
+ import shutil
8
+ import anndata as ad
9
+ import scanpy as sc
10
+
11
+ if __name__ == "__main__":
12
+
13
+ dataset_from = "/Users/domi/Dev/MPA_Workspace/MPA_DATA/01_Datasets/03_MouseSSP"
14
+ dataset_to = (
15
+ "/Users/domi/Dev/MPA_Workspace/MPA_DATA/01_Datasets/03_MouseSSP_HVG_2000"
16
+ )
17
+ N = 2000
18
+
19
+ """ Load datasets """
20
+
21
+ scData = ad.read_h5ad(os.path.join(dataset_from, "sc.h5ad")) # C x G
22
+ print("scRNA Data loaded")
23
+ stData = ad.read_h5ad(os.path.join(dataset_from, "st.h5ad")) # S x G
24
+ print("ST Data loaded")
25
+
26
+ """ Select top N HVGs, keeping all shared genes """
27
+
28
+ # Shared genes between scRNA and ST
29
+ sharedGenes = scData.var_names.intersection(stData.var_names)
30
+
31
+ # Compute HVGs on a normalized log-transformed copy
32
+ scDataLog = scData.copy()
33
+ sc.pp.normalize_total(scDataLog)
34
+ sc.pp.log1p(scDataLog)
35
+ sc.pp.highly_variable_genes(scDataLog, n_top_genes=N, inplace=True)
36
+
37
+ assert "highly_variable" in scDataLog.var.columns
38
+ hvgs = list(scDataLog.var_names[scDataLog.var["highly_variable"].values])
39
+
40
+ if len(hvgs) == 0:
41
+ raise RuntimeError("No HVGs selected — check input data and N value")
42
+
43
+ # Append shared genes that were not selected as HVGs
44
+ for sharedGene in sharedGenes:
45
+ if sharedGene not in hvgs:
46
+ print(
47
+ f"Warning: Shared gene {sharedGene} not selected as HVG - added manually"
48
+ )
49
+ hvgs.append(sharedGene)
50
+ print(
51
+ f"Appended {len(hvgs) - N} shared genes to HVG list. Total genes selected:",
52
+ len(hvgs),
53
+ )
54
+
55
+ """ Create and save HVG-filtered scRNA dataset """
56
+
57
+ scData_HVG = scData[:, hvgs].copy()
58
+
59
+ os.makedirs(dataset_to, exist_ok=True)
60
+ scData_HVG.write_h5ad(os.path.join(dataset_to, "sc.h5ad"))
61
+ print(f"Saved sc.h5ad: {scData_HVG.n_obs} cells x {scData_HVG.n_vars} genes")
62
+
63
+ """ Copy ST dataset unchanged """
64
+
65
+ shutil.copy(
66
+ os.path.join(dataset_from, "st.h5ad"),
67
+ os.path.join(dataset_to, "st.h5ad"),
68
+ )
69
+ print("Copied st.h5ad unchanged")