Spacelab 0.28.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. spacelab/__init__.py +6 -0
  2. spacelab/__main__.py +3 -0
  3. spacelab/cli.py +97 -0
  4. spacelab/hlod/__init__.py +8 -0
  5. spacelab/hlod/__main__.py +5 -0
  6. spacelab/hlod/build.py +349 -0
  7. spacelab/hlod/cells.py +82 -0
  8. spacelab/hlod/cli.py +90 -0
  9. spacelab/hlod/hashing.py +38 -0
  10. spacelab/hlod/hilbert.py +91 -0
  11. spacelab/hlod/read.py +150 -0
  12. spacelab/hlod/verify.py +142 -0
  13. spacelab/prep_spatialdata.py +700 -0
  14. spacelab/server.py +130 -0
  15. spacelab/static/AI_lens_black.png +0 -0
  16. spacelab/static/ai-new-logo.png +0 -0
  17. spacelab/static/aifi_logo.png +0 -0
  18. spacelab/static/assets/AllenInstitutePlusText-Bold-DxLtQ4oE.woff2 +0 -0
  19. spacelab/static/assets/AllenInstitutePlusText-BoldItalic-mxzHQhQO.woff2 +0 -0
  20. spacelab/static/assets/AllenInstitutePlusText-Italic-CaKmxpsW.woff2 +0 -0
  21. spacelab/static/assets/AllenInstitutePlusText-Light-DUThXl1m.woff2 +0 -0
  22. spacelab/static/assets/AllenInstitutePlusText-LightItalic-CxfKknzG.woff2 +0 -0
  23. spacelab/static/assets/AllenInstitutePlusText-Regular-B0ryviHQ.woff2 +0 -0
  24. spacelab/static/assets/AllenInstitutePlusText-Semibold-3PoiDF1d.woff2 +0 -0
  25. spacelab/static/assets/AllenInstitutePlusText-SemiboldItalic-uXaVqnFo.woff2 +0 -0
  26. spacelab/static/assets/blosc-BKQbfHyB.js +18 -0
  27. spacelab/static/assets/blosc-j6ygecpn.js +18 -0
  28. spacelab/static/assets/chunk-INHXZS53-D3tQiqtZ.js +2 -0
  29. spacelab/static/assets/chunk-INHXZS53-QjxmivJp.js +2 -0
  30. spacelab/static/assets/deflate-BRVR_fNk.js +2 -0
  31. spacelab/static/assets/hlodTranscripts.worker-CjhNLk56.js +2 -0
  32. spacelab/static/assets/index-BT-mDgg2.js +5025 -0
  33. spacelab/static/assets/index-CNIqsj6m.css +1 -0
  34. spacelab/static/assets/jpeg-JUDfHHDo.js +2 -0
  35. spacelab/static/assets/lerc-CXQdKTps.js +2 -0
  36. spacelab/static/assets/lz4-C2MQfzIA.js +16 -0
  37. spacelab/static/assets/lz4-rzy_Siro.js +16 -0
  38. spacelab/static/assets/lzw-DBCCrYEo.js +2 -0
  39. spacelab/static/assets/packbits-LrtZ2mhj.js +2 -0
  40. spacelab/static/assets/pako.esm-C69tvH9t.js +2 -0
  41. spacelab/static/assets/polygons.worker-ClOPdWlW.js +2 -0
  42. spacelab/static/assets/raw-lPmjnySZ.js +2 -0
  43. spacelab/static/assets/spatialdataExpression.worker-D4W_OHS3.js +4 -0
  44. spacelab/static/assets/spatialdataShapes.worker-DQV1zeTt.js +974 -0
  45. spacelab/static/assets/spatialdataTranscripts.worker-DH9L-MwZ.js +974 -0
  46. spacelab/static/assets/transcripts.worker-CtxyO4n3.js +2 -0
  47. spacelab/static/assets/umap.worker-BPHuoLPV.js +2 -0
  48. spacelab/static/assets/webimage-xg0Pp5Jv.js +2 -0
  49. spacelab/static/assets/zstd-Cp7BEDDH.js +16 -0
  50. spacelab/static/assets/zstd-WlDjm0Vu.js +16 -0
  51. spacelab/static/blurred-bg.jpg +0 -0
  52. spacelab/static/favicon.ico +0 -0
  53. spacelab/static/index.html +22 -0
  54. spacelab/static/logo192.png +0 -0
  55. spacelab/static/logo512.png +0 -0
  56. spacelab/static/manifest.json +15 -0
  57. spacelab/static/robots.txt +3 -0
  58. spacelab-0.28.0.dist-info/METADATA +30 -0
  59. spacelab-0.28.0.dist-info/RECORD +62 -0
  60. spacelab-0.28.0.dist-info/WHEEL +4 -0
  61. spacelab-0.28.0.dist-info/entry_points.txt +2 -0
  62. spacelab-0.28.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,700 @@
1
+ # /// script
2
+ # requires-python = ">=3.10"
3
+ # dependencies = ["zarr>=3", "numcodecs", "numpy", "pillow", "pyarrow", "shapely>=2"]
4
+ # ///
5
+ """Make a SpatialData store fast to open in the app.
6
+
7
+ The app reads SpatialData stores (manifest `dataSourceType: "spatialData"`)
8
+ without conversion, but cell outlines (GeoParquet WKB) and transcripts (one
9
+ unsorted table of every detection) are slow to read in a browser. This adds
10
+ sidecars next to the SpatialData files, in the same layouts the app's own
11
+ format uses, and leaves every SpatialData file untouched:
12
+
13
+ images/<image>/ checked: must be a multiscale pyramid viv tiles
14
+ images/thumbnail.png landing-page card preview
15
+ shapes/<shapes>/shapes.zarr polygon_vertices + polygon_offset attr (zarr v3 group)
16
+ points/<points>/points.zarr Hilbert LOD store (spacelab.hlod), streamed by viewport
17
+ tables/<table>/X checked: must be dense or CSC (reported, never rewritten)
18
+ ../tissue-set-manifest.json default entry for this store, added if its key is absent
19
+
20
+ uv run data/prep_spatialdata.py <name>.zarr # one store
21
+ uv run data/prep_spatialdata.py <directory> # every SpatialData .zarr in it
22
+
23
+ Sidecar coordinates stay in each element's own units (µm for Xenium), as the
24
+ standard format's do; the app maps them to image pixels with the element's
25
+ SpatialData transform. Cells are written in the order the app indexes them
26
+ (table order when the table joins the shapes; see src/spatialdata/cells.ts).
27
+
28
+ Standalone (not part of prep_xenium_data.py) because SpatialData writes zarr v3
29
+ and the prep environment pins zarr<3. shapes.zarr and points.zarr are zarr v3; points.zarr
30
+ uses sharding.
31
+ """
32
+
33
+ import argparse
34
+ import json
35
+ import math
36
+ import os
37
+ import shutil
38
+ import sys
39
+
40
+ import numpy as np
41
+ import pyarrow as pa
42
+ import pyarrow.parquet as pq
43
+ import shapely
44
+ import zarr
45
+ from PIL import Image
46
+ from zarr.codecs import BloscCodec
47
+
48
+
49
+ class PrepError(Exception):
50
+ pass
51
+
52
+
53
+ # --- shared helpers ---
54
+
55
+ def _ome(attrs) -> dict:
56
+ """NGFF 0.5 (zarr v3) nests image metadata under `ome`; 0.4 has it top-level."""
57
+ return attrs.get("ome", attrs)
58
+
59
+
60
+ def _replace_dir(tmp: str, final: str):
61
+ """Swap a finished sidecar in, so a failed run never leaves a half-written
62
+ one where the app would read it."""
63
+ if os.path.exists(final):
64
+ shutil.rmtree(final)
65
+ os.rename(tmp, final)
66
+
67
+
68
+ # --- 1. image: a pyramid viv can tile ---
69
+
70
+ # zarrita's codec registry (node_modules/zarrita/dist/src/codecs.js).
71
+ ZARRITA_CODECS = {
72
+ "blosc", "lz4", "zstd", "gzip", "zlib", "transpose", "bytes", "crc32c", "sharding_indexed",
73
+ }
74
+ # viv's DTYPE_LOOKUP, in numpy names.
75
+ VIV_DTYPES = {"uint8", "uint16", "uint32", "int8", "int16", "int32", "float32", "float64"}
76
+ VIV_AXES = {"t", "c", "z", "y", "x"}
77
+ # viv's tile edge is level 0's chunk edge; bigger tiles still render, but every
78
+ # fetch moves a whole tile (4096² uint16 is 32 MB per channel).
79
+ LARGE_TILE = 1024
80
+
81
+
82
+ def _codec_names(arr) -> list[str]:
83
+ meta = arr.metadata.to_dict()
84
+ if meta.get("zarr_format") == 2:
85
+ comp = meta.get("compressor")
86
+ return [comp["id"]] if comp else []
87
+ names = []
88
+ for codec in meta.get("codecs", []):
89
+ names.append(codec["name"])
90
+ if codec["name"] == "sharding_indexed":
91
+ names += [c["name"] for c in codec["configuration"].get("codecs", [])]
92
+ return names
93
+
94
+
95
+ def check_viv_image(image) -> dict:
96
+ """Raise PrepError unless `image` is a multiscale OME-Zarr that viv's
97
+ loadOmeZarr opens and MultiscaleImageLayer tiles at more than one level.
98
+ Mirrors @vivjs/loaders: datasets from multiscales[0], labels from axes, tile
99
+ size = prevPowerOf2(min(level-0 y/x chunk)), and a last dim of 3 or 4
100
+ read as interleaved RGB."""
101
+ ms_list = _ome(dict(image.attrs)).get("multiscales")
102
+ if not ms_list:
103
+ raise PrepError("no `multiscales` metadata: not an OME-Zarr image")
104
+ ms = ms_list[0]
105
+ paths = [d["path"] for d in ms.get("datasets", [])]
106
+ if len(paths) < 2:
107
+ raise PrepError(f"{len(paths)} resolution level(s): viv needs a pyramid to tile zoom levels")
108
+
109
+ axes = [a if isinstance(a, str) else a.get("name") for a in ms.get("axes", [])]
110
+ if not axes:
111
+ raise PrepError("multiscales has no `axes`")
112
+ if len(set(axes)) != len(axes) or not set(axes) <= VIV_AXES:
113
+ raise PrepError(f"axes {axes} are not unique names from {sorted(VIV_AXES)}")
114
+ if axes[-2:] != ["y", "x"]:
115
+ raise PrepError(f"axes {axes} must end in y, x")
116
+
117
+ levels = []
118
+ for p in paths:
119
+ if p not in image:
120
+ raise PrepError(f"level '{p}' listed in multiscales is missing")
121
+ arr = image[p]
122
+ if arr.ndim != len(axes):
123
+ raise PrepError(f"level '{p}' has {arr.ndim} dims for {len(axes)} axes")
124
+ if str(arr.dtype) not in VIV_DTYPES:
125
+ raise PrepError(f"level '{p}' dtype {arr.dtype} is not one viv renders")
126
+ bad = [c for c in _codec_names(arr) if c not in ZARRITA_CODECS]
127
+ if bad:
128
+ raise PrepError(f"level '{p}' uses codecs {bad} the browser reader can't decode")
129
+ if arr.shape[-1] in (3, 4):
130
+ raise PrepError(f"level '{p}' is {arr.shape[-1]} px wide; viv would read it as RGB")
131
+ levels.append(arr)
132
+
133
+ for k in range(1, len(levels)):
134
+ for dim, name in ((-2, "height"), (-1, "width")):
135
+ prev, cur = levels[k - 1].shape[dim], levels[k].shape[dim]
136
+ if abs(cur - prev / 2) > 1:
137
+ raise PrepError(
138
+ f"level '{paths[k]}' {name} {cur} is not half of '{paths[k - 1]}' ({prev}); "
139
+ "viv assumes each level downsamples by 2"
140
+ )
141
+ for i, name in enumerate(axes[:-2]):
142
+ if levels[k].shape[i] != levels[0].shape[i]:
143
+ raise PrepError(f"level '{paths[k]}' changes the size of non-spatial axis '{name}'")
144
+
145
+ chunk_y, chunk_x = levels[0].chunks[-2:]
146
+ tile = 2 ** math.floor(math.log2(min(chunk_y, chunk_x)))
147
+ warnings = []
148
+ if tile > LARGE_TILE:
149
+ warnings.append(
150
+ f"level-0 chunks are {chunk_y}x{chunk_x}, so viv fetches {tile}px tiles; "
151
+ f"rechunking to <= {LARGE_TILE} would make zooming faster"
152
+ )
153
+ if chunk_y != chunk_x or chunk_y != tile:
154
+ warnings.append(
155
+ f"level-0 chunks {chunk_y}x{chunk_x} aren't square powers of 2; each {tile}px tile "
156
+ "read spans several chunks"
157
+ )
158
+ return {
159
+ "levels": [{"path": p, "shape": list(a.shape)} for p, a in zip(paths, levels)],
160
+ "axes": axes,
161
+ "dtype": str(levels[0].dtype),
162
+ "tile_size": tile,
163
+ "warnings": warnings,
164
+ }
165
+
166
+
167
+ # --- 2. thumbnail ---
168
+
169
+ THUMB_OUT_DIM = 512
170
+ THUMB_MIN_DIM = 512
171
+ THUMB_MAX_DIM = 2048
172
+ THUMB_MAX_CHANNELS = 4
173
+ THUMB_LO_PCT = 25.0
174
+ THUMB_HI_PCT = 99.9
175
+ THUMB_TARGET_LUM = 0.22
176
+ THUMB_GAMMA_RANGE = (0.7, 2.2)
177
+ THUMB_MIN_REL_SIGNAL = 0.02
178
+ # CHANNEL_DEFAULT_COLORS in src/config/constants.ts
179
+ THUMB_TINTS = np.array(
180
+ [[0, 26, 255], [0, 255, 0], [255, 0, 255], [255, 255, 0], [0, 255, 255], [255, 0, 0]],
181
+ dtype=np.float32,
182
+ ) / 255.0
183
+
184
+
185
+ def _pick_thumb_level(image) -> str:
186
+ """Smallest pyramid level still >= THUMB_MIN_DIM across, skipping any larger
187
+ than THUMB_MAX_DIM so a shallow pyramid can't make us decompress a whole slide."""
188
+ paths = [d["path"] for d in _ome(dict(image.attrs))["multiscales"][0]["datasets"]]
189
+ levels = [(p, max(image[p].shape[-2:])) for p in paths]
190
+ usable = [lv for lv in levels if lv[1] <= THUMB_MAX_DIM] or [min(levels, key=lambda lv: lv[1])]
191
+ big_enough = [lv for lv in usable if lv[1] >= THUMB_MIN_DIM]
192
+ if big_enough:
193
+ return min(big_enough, key=lambda lv: lv[1])[0]
194
+ return max(usable, key=lambda lv: lv[1])[0]
195
+
196
+
197
+ def render_thumbnail(image) -> Image.Image:
198
+ """Follows prep_xenium_data._render_thumbnail: tinted channels max-composited
199
+ rather than summed (summing saturates co-stained tissue to white), a
200
+ per-channel percentile stretch, and an auto-gamma that puts median tissue
201
+ brightness at a fixed target. Tints are the app's default channel colors, so
202
+ the card matches what the viewer opens with."""
203
+ arr = image[_pick_thumb_level(image)]
204
+ planes = np.asarray(arr[...]).reshape(-1, *arr.shape[-2:]) # [c, y, x], any leading axes
205
+ layers = []
206
+ for i in range(min(planes.shape[0], THUMB_MAX_CHANNELS)):
207
+ plane = planes[i].astype(np.float32)
208
+ lo = float(np.percentile(plane, THUMB_LO_PCT))
209
+ hi = float(np.percentile(plane, THUMB_HI_PCT))
210
+ if hi <= lo:
211
+ hi = lo + 1.0
212
+ layers.append((plane, lo, hi, THUMB_TINTS[i % len(THUMB_TINTS)]))
213
+
214
+ # A blank channel's noise floor, stretched to full range, washes the frame in
215
+ # its tint; drop channels whose range is negligible next to the widest.
216
+ if len(layers) > 1:
217
+ widest = max(hi - lo for _, lo, hi, _ in layers)
218
+ layers = [lay for lay in layers if lay[2] - lay[1] >= THUMB_MIN_REL_SIGNAL * widest] or layers
219
+
220
+ h, w = planes.shape[-2:]
221
+ rgb = np.zeros((h, w, 3), dtype=np.float32)
222
+ for plane, lo, hi, tint in layers:
223
+ norm = np.clip((plane - lo) / (hi - lo), 0.0, 1.0)
224
+ np.maximum(rgb, norm[:, :, None] * tint[None, None, :], out=rgb)
225
+
226
+ signal = rgb.max(axis=2)
227
+ signal = signal[signal > 0.02]
228
+ if signal.size:
229
+ median = float(np.median(signal))
230
+ if 0.01 < median < 0.99:
231
+ rgb **= float(np.clip(np.log(THUMB_TARGET_LUM) / np.log(median), *THUMB_GAMMA_RANGE))
232
+
233
+ img = Image.fromarray(np.clip(rgb * 255.0, 0, 255).astype(np.uint8))
234
+ if max(img.size) > THUMB_OUT_DIM:
235
+ scale = THUMB_OUT_DIM / max(img.size)
236
+ img = img.resize((max(1, int(img.width * scale)), max(1, int(img.height * scale))),
237
+ Image.LANCZOS)
238
+ return img
239
+
240
+
241
+ # --- 3. shapes -> shapes.zarr ---
242
+
243
+ # Same cap as MAX_RING_VERTICES in src/spatialdata/wkbPolygons.ts: every cell
244
+ # pays for the longest ring, so dense outlines are resampled down to this.
245
+ MAX_RING_VERTICES = 64
246
+ POLYGON_COMPRESSOR = BloscCodec(cname="lz4", clevel=5, shuffle="shuffle")
247
+ POLYGON_CHUNK = 259606 # floats, as the standard prep writes cells/uns/polygon_vertices
248
+
249
+
250
+ def _pandas_index_column(schema: pa.Schema) -> str | None:
251
+ meta = json.loads((schema.metadata or {}).get(b"pandas", b"{}"))
252
+ cols = meta.get("index_columns") or []
253
+ return cols[0] if cols and isinstance(cols[0], str) else None
254
+
255
+
256
+ def _as_str(values) -> np.ndarray:
257
+ # zarr 3 hands back numpy StringDType, which won't cast to fixed-width str.
258
+ return np.array([str(v) for v in values], dtype=object)
259
+
260
+
261
+ def _read_string_column(obs, name: str) -> np.ndarray:
262
+ """An anndata obs column as strings: a plain string array, a categorical group
263
+ of categories + codes, or a nullable-string-array group of values + mask
264
+ (newer anndata). Missing entries become ""."""
265
+ node = obs[name]
266
+ if isinstance(node, zarr.Group):
267
+ if node.attrs.get("encoding-type") == "nullable-string-array":
268
+ values = _as_str(node["values"][...])
269
+ values[np.asarray(node["mask"][...], dtype=bool)] = ""
270
+ return values
271
+ cats = np.append(_as_str(node["categories"][...]), "")
272
+ return cats[np.asarray(node["codes"][...])] # code -1 (missing) -> ""
273
+ return _as_str(node[...])
274
+
275
+
276
+ def _column_length(node) -> int:
277
+ """Rows of an anndata column: an array, or a group encoding (categorical:
278
+ codes; nullable-*: values)."""
279
+ if isinstance(node, zarr.Array):
280
+ return node.shape[0]
281
+ for child in ("codes", "values"):
282
+ if child in node:
283
+ return node[child].shape[0]
284
+ raise PrepError(f"unsupported anndata column encoding {node.attrs.get('encoding-type')!r}")
285
+
286
+
287
+ def _cell_order(root, shapes_index_col, shapes_ids, n_shapes, table):
288
+ """Destination slot for every shape row and the cell count, under the same
289
+ rules as resolveCellsLayout in src/spatialdata/cells.ts: join on the shapes'
290
+ index column when the table has it, by row order when only the lengths
291
+ match, else unjoined (cells are the shapes). Returns (dest | None, n_cells,
292
+ description, join); dest[i] = -1 drops shape i, None keeps shapes order. `join`
293
+ is the CellJoin the app would derive, recorded so it needn't read the parquet."""
294
+ if not table:
295
+ return None, n_shapes, "shapes order (no table)", {"mode": "none"}
296
+ tbl = root["tables"][table]
297
+ region = tbl.attrs.get("region")
298
+ if isinstance(region, list) and len(region) > 1:
299
+ return (None, n_shapes, f"shapes order (table '{table}' annotates {len(region)} regions)",
300
+ {"mode": "none"})
301
+ obs = tbl["obs"]
302
+ index_name = obs.attrs.get("_index", "_index")
303
+ n_obs = _column_length(obs[index_name]) if index_name in obs else 0
304
+ if n_obs == 0:
305
+ return None, n_shapes, "shapes order (empty table)", {"mode": "none"}
306
+ columns = obs.attrs.get("column-order", [])
307
+ if shapes_index_col and shapes_index_col in columns:
308
+ table_ids = _read_string_column(obs, shapes_index_col)
309
+ if n_shapes == n_obs and np.array_equal(shapes_ids, table_ids):
310
+ return (None, n_obs, f"table order (= shapes order, by '{shapes_index_col}')",
311
+ {"mode": "column", "column": shapes_index_col})
312
+ row_of = {cid: r for r, cid in enumerate(table_ids)}
313
+ dest = np.fromiter((row_of.get(cid, -1) for cid in shapes_ids), np.int64, n_shapes)
314
+ return (dest, n_obs, f"table order (joined on '{shapes_index_col}')",
315
+ {"mode": "column", "column": shapes_index_col})
316
+ if n_obs == n_shapes:
317
+ return (None, n_obs, "table order (assumed = shapes order: no shared id column)",
318
+ {"mode": "order"})
319
+ return None, n_shapes, f"shapes order (table '{table}' doesn't match the shapes)", {"mode": "none"}
320
+
321
+
322
+ def _ring_coords(geoms: np.ndarray):
323
+ """Outer ring of each Polygon, or of a MultiPolygon's first polygon (as
324
+ outerRing in wkbPolygons.ts). Returns xy [n_pts, 2] and per-ring start /
325
+ length; empty or unsupported geometry gets length 0."""
326
+ types = shapely.get_type_id(geoms)
327
+ polys = np.where(types == 6, shapely.get_geometry(geoms, 0), geoms)
328
+ polys = np.where(shapely.get_type_id(polys) == 3, polys, None)
329
+ rings = shapely.get_exterior_ring(polys)
330
+ xy, ring_of = shapely.get_coordinates(rings, return_index=True)
331
+ lengths = np.bincount(ring_of, minlength=len(geoms))
332
+ starts = np.zeros(len(geoms), dtype=np.int64)
333
+ np.cumsum(lengths[:-1], out=starts[1:])
334
+ return xy, starts, lengths
335
+
336
+
337
+ def flatten_rings(xy, starts, lengths, dest, n_cells):
338
+ """The app's fixed-stride layout: every ring closed (shapely rings already
339
+ are) and padded to the longest by repeating its closing vertex, or evenly
340
+ resampled when longer than MAX_RING_VERTICES — writeRing in wkbPolygons.ts.
341
+ Cells nobody writes stay NaN, which the polygon worker skips."""
342
+ n_verts = int(min(max(lengths.max(initial=0), 4), MAX_RING_VERTICES))
343
+ out = np.full((n_cells, n_verts, 2), np.nan, dtype=np.float32)
344
+ slots = np.arange(len(lengths)) if dest is None else dest
345
+ keep = (lengths > 0) & (slots >= 0) & (slots < n_cells)
346
+ i = np.arange(n_verts)
347
+ for lo in range(0, len(lengths), 100_000):
348
+ sel = np.nonzero(keep[lo:lo + 100_000])[0] + lo
349
+ if sel.size == 0:
350
+ continue
351
+ ln = lengths[sel][:, None]
352
+ # Math.round(i * (len - 1) / (nVerts - 1)) for long rings, JS rounding.
353
+ resampled = np.floor(i[None, :] * (ln - 1) / (n_verts - 1) + 0.5).astype(np.int64)
354
+ src = np.where(ln <= n_verts, np.minimum(i[None, :], ln - 1), resampled)
355
+ out[slots[sel]] = xy[starts[sel][:, None] + src]
356
+ n_missing = int(n_cells - np.unique(slots[keep]).size)
357
+ return out.reshape(-1), n_verts * 2, n_missing
358
+
359
+
360
+ def write_shapes_group(path: str, vertices: np.ndarray, attrs: dict) -> None:
361
+ """The shapes.zarr sidecar: a zarr v3 group whose zarr.json carries `attrs`
362
+ (polygon_offset among them) and whose one array, `polygon_vertices`, is the
363
+ flat float32 [x0, y0, x1, y1, ...] of every cell's ring."""
364
+ if os.path.exists(path):
365
+ shutil.rmtree(path)
366
+ g = zarr.open_group(path, mode="w", zarr_format=3, attributes=attrs)
367
+ g.create_array("polygon_vertices", data=vertices.astype(np.float32, copy=False),
368
+ chunks=(POLYGON_CHUNK,), compressors=POLYGON_COMPRESSOR)
369
+
370
+
371
+ def write_shapes_zarr(root, store_path: str, shapes: str, table: str | None) -> dict:
372
+ parquet = os.path.join(store_path, "shapes", shapes, "shapes.parquet")
373
+ if not os.path.exists(parquet):
374
+ raise PrepError(f"shapes/{shapes}/shapes.parquet not found")
375
+ pf = pq.ParquetFile(parquet) if os.path.isfile(parquet) else None
376
+ schema = pf.schema_arrow if pf else pq.read_schema(next(
377
+ os.path.join(parquet, f) for f in sorted(os.listdir(parquet)) if f.endswith(".parquet")))
378
+ geo = json.loads((schema.metadata or {}).get(b"geo", b"{}"))
379
+ geom_col = geo.get("primary_column", "geometry")
380
+ index_col = _pandas_index_column(schema)
381
+ cols = [geom_col] + ([index_col] if index_col else [])
382
+ tbl = pq.read_table(parquet, columns=cols)
383
+
384
+ geoms = shapely.from_wkb(tbl.column(geom_col).to_numpy(zero_copy_only=False))
385
+ ids = _as_str(tbl.column(index_col).to_pylist()) if index_col else None
386
+ dest, n_cells, order, join = _cell_order(root, index_col, ids, len(geoms), table)
387
+ xy, starts, lengths = _ring_coords(geoms)
388
+ vertices, polygon_offset, n_missing = flatten_rings(xy, starts, lengths, dest, n_cells)
389
+
390
+ final = os.path.join(store_path, "shapes", shapes, "shapes.zarr")
391
+ tmp = final + ".tmp"
392
+ write_shapes_group(tmp, vertices, {
393
+ "polygon_offset": polygon_offset,
394
+ "number_cells": n_cells,
395
+ "cell_order": order,
396
+ "source_element": f"shapes/{shapes}",
397
+ "table": table if join["mode"] != "none" else None,
398
+ # What resolveCellsLayout would otherwise read from the parquet footer, so
399
+ # the app can lay cells out without touching shapes.parquet at all.
400
+ "join": join,
401
+ "n_shapes": len(geoms),
402
+ "shapes_index_column": index_col,
403
+ "spatial_units": "element",
404
+ })
405
+ _replace_dir(tmp, final)
406
+ return {"path": final, "n_cells": n_cells, "n_shapes": len(geoms),
407
+ "polygon_offset": polygon_offset, "cells_without_outline": n_missing,
408
+ "cell_order": order}
409
+
410
+
411
+ # --- 4. points -> points.zarr ---
412
+
413
+ # The Hilbert LOD store (spacelab.hlod; format in py/src/spacelab/hlod/build.py):
414
+ # every transcript once, sorted by (gene, level, Hilbert index), which the app
415
+ # streams by viewport instead of reading whole genes. Co-sorted extras, when the
416
+ # points have them.
417
+ POINTS_COLUMNS = ("qv", "cell_id")
418
+
419
+
420
+ def _load_hlod():
421
+ """spacelab.hlod.build, from the bundled package (this file sits in spacelab/
422
+ when installed) or from the repo (data/ next to py/src/spacelab)."""
423
+ import importlib.util
424
+
425
+ here = os.path.dirname(os.path.abspath(__file__))
426
+ for pkg in (os.path.join(here, "hlod"),
427
+ os.path.join(here, os.pardir, "py", "src", "spacelab", "hlod")):
428
+ init = os.path.join(pkg, "__init__.py")
429
+ if os.path.isfile(init):
430
+ spec = importlib.util.spec_from_file_location(
431
+ "spacelab_hlod", init, submodule_search_locations=[pkg])
432
+ mod = importlib.util.module_from_spec(spec)
433
+ sys.modules["spacelab_hlod"] = mod
434
+ spec.loader.exec_module(mod)
435
+ return importlib.import_module("spacelab_hlod.build")
436
+ raise PrepError("spacelab.hlod not found next to prep_spatialdata.py (broken install?)")
437
+
438
+
439
+ def write_points_zarr(root, store_path: str, points: str) -> dict:
440
+ parquet = os.path.join(store_path, "points", points, "points.parquet")
441
+ if not os.path.exists(parquet):
442
+ raise PrepError(f"points/{points}/points.parquet not found")
443
+ sd_attrs = dict(root["points"][points].attrs).get("spatialdata_attrs", {})
444
+ feature = sd_attrs.get("feature_key")
445
+ schema = pq.read_schema(parquet if os.path.isfile(parquet) else next(
446
+ os.path.join(parquet, f) for f in sorted(os.listdir(parquet)) if f.endswith(".parquet")))
447
+ names_in_file = set(schema.names)
448
+ if not feature or feature not in names_in_file:
449
+ raise PrepError(f"points/{points} has no feature column (feature_key={feature!r})")
450
+ hlod = _load_hlod()
451
+ final = os.path.join(store_path, "points", points, "points.zarr")
452
+ try:
453
+ summary = hlod.build(
454
+ parquet, final, gene_col=feature, x_col="x", y_col="y",
455
+ id_col="transcript_id" if "transcript_id" in names_in_file else None,
456
+ columns=[c for c in POINTS_COLUMNS if c in names_in_file],
457
+ extra_attrs={"source_element": f"points/{points}"}, log=lambda _: None)
458
+ except hlod.BuildError as e:
459
+ raise PrepError(f"points/{points}: {e}") from None
460
+ return {"path": final, "n_genes": summary["genes"], "n_rnas": summary["rows"],
461
+ "Dmax": summary["Dmax"]}
462
+
463
+
464
+ # --- 5. expression matrix: dense or CSC ---
465
+
466
+ def check_expression_matrix(root, table: str | None) -> dict:
467
+ """The app reads a gene's per-cell column from tables/<table>/X; dense and
468
+ CSC store columns contiguously, CSR would mean reading the whole matrix.
469
+ Only reports — rewriting X is left to the user (e.g. in anndata:
470
+ `adata.X = adata.X.tocsc()` and write the table back)."""
471
+ if not table:
472
+ return {"table": None, "encoding": None, "ok": False, "message": "store has no table"}
473
+ tbl = root["tables"][table]
474
+ if "X" not in tbl:
475
+ return {"table": table, "encoding": None, "ok": False, "message": f"tables/{table} has no X"}
476
+ x = tbl["X"]
477
+ encoding = "dense" if isinstance(x, zarr.Array) else x.attrs.get("encoding-type")
478
+ ok = encoding in ("dense", "csc_matrix")
479
+ report = {"table": table, "encoding": encoding, "ok": ok}
480
+ if not ok:
481
+ report["message"] = (
482
+ f"tables/{table}/X is {encoding}; the app reads expression only from a dense or CSC "
483
+ "X, so Counts and Scores stay unavailable until X is rewritten as one"
484
+ )
485
+ return report
486
+
487
+
488
+ # --- 5. tissue-set-manifest.json ---
489
+
490
+ MANIFEST_NAME = "tissue-set-manifest.json"
491
+ # Same 1%/99% start/end and ~66% bounds as prep_xenium_data's channel windows.
492
+ CHANNEL_START_PCT, CHANNEL_END_PCT, CHANNEL_BOUND_FRAC = 1.0, 99.0, 0.66
493
+ CHANNEL_HEX = ["%02X%02X%02X" % tuple(int(v) for v in (t * 255).round()) for t in THUMB_TINTS]
494
+
495
+
496
+ def format_cell_count(n: int) -> str:
497
+ """The card's cell count, as prep_xenium_data.py writes it (xenium_utils, which
498
+ this standalone script can't import): 950, 170k, 162.2k, 1.2M."""
499
+ if n < 1000:
500
+ return str(n)
501
+ if n < 1_000_000:
502
+ return f"{int(n / 100) / 10.0:g}k"
503
+ return f"{int(n / 100000) / 10.0:g}M"
504
+
505
+
506
+ def count_cells(root, table: str | None) -> int | None:
507
+ """Cells as the table counts them: the rows of its obs index, else of X (a
508
+ sparse X records its `shape` in the group attrs, a dense X is the array
509
+ itself). None without a table, so the caller can fall back to the outlines."""
510
+ if not table:
511
+ return None
512
+ tbl = root["tables"][table]
513
+ if "obs" in tbl:
514
+ obs = tbl["obs"]
515
+ index = obs.attrs.get("_index", "_index")
516
+ if index in obs:
517
+ return _column_length(obs[index])
518
+ if "X" not in tbl:
519
+ return None
520
+ x = tbl["X"]
521
+ shape = x.shape if isinstance(x, zarr.Array) else x.attrs.get("shape")
522
+ return int(shape[0]) if shape else None
523
+
524
+
525
+ def default_channels(image) -> list[dict]:
526
+ """Manifest `channels` for the image: labels from omero when present, the
527
+ app's default tints, and contrast windows from the thumbnail level. The
528
+ first THUMB_MAX_CHANNELS start active, as in the thumbnail."""
529
+ arr = image[_pick_thumb_level(image)]
530
+ planes = np.asarray(arr[...]).reshape(-1, *arr.shape[-2:])
531
+ omero = _ome(dict(image.attrs)).get("omero", {}).get("channels", [])
532
+ channels = []
533
+ for i, plane in enumerate(planes):
534
+ label = omero[i].get("label") if i < len(omero) else None
535
+ lo, hi = float(plane.min()), float(plane.max())
536
+ start = float(np.percentile(plane, CHANNEL_START_PCT))
537
+ end = float(np.percentile(plane, CHANNEL_END_PCT))
538
+ if end <= start:
539
+ end = start + 1.0
540
+ channels.append({
541
+ "label": str(label) if label else f"Channel {i}",
542
+ "color": CHANNEL_HEX[i % len(CHANNEL_HEX)],
543
+ "active": i < THUMB_MAX_CHANNELS,
544
+ "window": {"min": int(lo * CHANNEL_BOUND_FRAC), "max": int(hi * CHANNEL_BOUND_FRAC),
545
+ "start": start, "end": end},
546
+ })
547
+ return channels
548
+
549
+
550
+ def ask_overwrite(key: str) -> bool:
551
+ """Y/n prompt (Enter = yes). No answer available (piped stdin) means keep the entry."""
552
+ try:
553
+ answer = input(f"'{key}' is already in {MANIFEST_NAME}. Overwrite it? [Y/n] ")
554
+ except EOFError:
555
+ return False
556
+ return answer.strip().lower() in ("", "y", "yes")
557
+
558
+
559
+ def write_manifest(store_path: str, root, image, table: str | None, overwrite=ask_overwrite,
560
+ n_shapes: int | None = None) -> dict:
561
+ """Add a default entry for this store to tissue-set-manifest.json in the
562
+ store's parent directory, keyed by the store name minus .zarr (the app opens
563
+ `<key>.zarr`). An existing entry may carry hand edits, so `overwrite(key)`
564
+ decides whether to replace it."""
565
+ store_path = os.path.abspath(store_path)
566
+ key = os.path.basename(store_path)
567
+ key = key[: -len(".zarr")] if key.endswith(".zarr") else key
568
+ path = os.path.join(os.path.dirname(store_path), MANIFEST_NAME)
569
+ manifest = {}
570
+ if os.path.exists(path):
571
+ with open(path) as f:
572
+ manifest = json.load(f)
573
+ if key in manifest and not overwrite(key):
574
+ return {"path": path, "key": key, "written": False}
575
+ # The card counts obs rows (cells QC kept), else the outlines.
576
+ n_cells = count_cells(root, table)
577
+ if n_cells is None:
578
+ n_cells = n_shapes
579
+ manifest[key] = {
580
+ "dataSourceType": "spatialData",
581
+ "majorLabel": key,
582
+ "description": "",
583
+ **({"nCells": format_cell_count(n_cells)} if n_cells is not None else {}),
584
+ "filterableMetadata": {"dataType": "SpatialData"},
585
+ "sampleInfo": [{"label": "Data type", "value": "SpatialData"}],
586
+ "channels": default_channels(image),
587
+ }
588
+ tmp = path + ".tmp"
589
+ with open(tmp, "w") as f:
590
+ json.dump(manifest, f, indent=2)
591
+ os.replace(tmp, path)
592
+ return {"path": path, "key": key, "written": True}
593
+
594
+
595
+ # --- entry point ---
596
+
597
+ def _pick(root, kind: str, requested: str | None) -> str | None:
598
+ """The element the app reads: the manifest's choice, else the first in
599
+ sorted order (pickElement in src/spatialdata/store.ts)."""
600
+ names = sorted(root[kind].group_keys()) if kind in root else []
601
+ if requested:
602
+ if requested not in names:
603
+ raise PrepError(f"no {kind}/{requested} (has: {', '.join(names) or 'none'})")
604
+ return requested
605
+ return names[0] if names else None
606
+
607
+
608
+ def prep_spatialdata(store_path: str, image: str | None = "morphology_focus",
609
+ shapes: str | None = None, points: str | None = None,
610
+ table: str | None = None, overwrite=ask_overwrite) -> dict:
611
+ """Check the image, write the thumbnail and the shapes/points sidecars, and
612
+ report whether X is in a layout the app reads. Element names default as the app picks them, so pass the same names as the
613
+ manifest's `spatialDataElements` when it sets any. Raises PrepError (and
614
+ writes nothing) if the image won't tile in viv."""
615
+ root = zarr.open_group(store_path, mode="r")
616
+ image = _pick(root, "images", image)
617
+ if not image:
618
+ raise PrepError("store has no images")
619
+ img = root["images"][image]
620
+ try:
621
+ report = {"image": {"name": image, **check_viv_image(img)}}
622
+ except PrepError as e:
623
+ raise PrepError(f"images/{image}: {e}") from None
624
+
625
+ thumb = os.path.join(store_path, "images", "thumbnail.png")
626
+ render_thumbnail(img).save(thumb)
627
+ report["thumbnail"] = thumb
628
+
629
+ table = _pick(root, "tables", table)
630
+ shapes = _pick(root, "shapes", shapes)
631
+ if shapes:
632
+ report["shapes"] = write_shapes_zarr(root, store_path, shapes, table)
633
+ points = _pick(root, "points", points)
634
+ if points:
635
+ report["points"] = write_points_zarr(root, store_path, points)
636
+ report["expression"] = check_expression_matrix(root, table)
637
+ n_shapes = report["shapes"]["n_shapes"] if "shapes" in report else None
638
+ report["manifest"] = write_manifest(store_path, root, img, table, overwrite, n_shapes)
639
+ return report
640
+
641
+
642
+ def is_spatialdata_store(path: str) -> bool:
643
+ return os.path.isdir(path) and any(
644
+ os.path.isdir(os.path.join(path, kind)) for kind in ("images", "shapes", "points", "tables")
645
+ )
646
+
647
+
648
+ def find_stores(directory: str) -> list[str]:
649
+ return [
650
+ p for p in (os.path.join(directory, n) for n in sorted(os.listdir(directory)))
651
+ if p.endswith(".zarr") and is_spatialdata_store(p)
652
+ ]
653
+
654
+
655
+ def prep_one(store: str, args) -> bool:
656
+ try:
657
+ report = prep_spatialdata(store, args.image, args.shapes, args.points, args.table)
658
+ except PrepError as e:
659
+ print(f"error: {store}: {e}", file=sys.stderr)
660
+ return False
661
+ for w in report["image"].pop("warnings"):
662
+ print(f"warning: images/{report['image']['name']}: {w}", file=sys.stderr)
663
+ if not report["expression"]["ok"]:
664
+ print(f"notice: {report['expression']['message']}", file=sys.stderr)
665
+ entry = "added to" if report["manifest"]["written"] else "left unchanged in"
666
+ print(f"✓ {os.path.basename(store)} prepped successfully; manifest entry {entry} {MANIFEST_NAME}")
667
+ return True
668
+
669
+
670
+ def main(argv=None) -> int:
671
+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
672
+ ap.add_argument("store", help="a <name>.zarr SpatialData store, or a directory of them "
673
+ "(each is prepped in turn and added to the directory's tissue-set-manifest.json)")
674
+ ap.add_argument("--image", default="morphology_focus", help="images/<name> (default: %(default)s)")
675
+ ap.add_argument("--shapes", help="shapes/<name> for cells (default: first in sorted order)")
676
+ ap.add_argument("--points", help="points/<name> for transcripts (default: first in sorted order)")
677
+ ap.add_argument("--table", help="tables/<name> annotating the cells (default: first)")
678
+ args = ap.parse_args(argv)
679
+
680
+ if is_spatialdata_store(args.store):
681
+ return 0 if prep_one(args.store, args) else 1
682
+ if not os.path.isdir(args.store):
683
+ print(f"error: {args.store} is not a directory", file=sys.stderr)
684
+ return 1
685
+ stores = find_stores(args.store)
686
+ if not stores:
687
+ print(f"error: no SpatialData .zarr stores found in {args.store}", file=sys.stderr)
688
+ return 1
689
+ failed = []
690
+ for n, store in enumerate(stores, 1):
691
+ print(f"[{n}/{len(stores)}] {os.path.basename(store)}", file=sys.stderr)
692
+ if not prep_one(store, args):
693
+ failed.append(os.path.basename(store))
694
+ if failed:
695
+ print(f"failed: {', '.join(failed)}", file=sys.stderr)
696
+ return 1 if failed else 0
697
+
698
+
699
+ if __name__ == "__main__":
700
+ sys.exit(main())