mlarray 0.0.52__tar.gz → 0.0.54__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. {mlarray-0.0.52 → mlarray-0.0.54}/PKG-INFO +15 -1
  2. {mlarray-0.0.52 → mlarray-0.0.54}/README.md +14 -0
  3. mlarray-0.0.54/assets/banner.png +0 -0
  4. mlarray-0.0.54/bench/benchmark_formats.py +527 -0
  5. {mlarray-0.0.52 → mlarray-0.0.54}/docs/api.md +2 -0
  6. mlarray-0.0.54/docs/cli.md +59 -0
  7. {mlarray-0.0.52 → mlarray-0.0.54}/docs/schema.md +30 -0
  8. {mlarray-0.0.52 → mlarray-0.0.54}/docs/usage.md +30 -0
  9. {mlarray-0.0.52 → mlarray-0.0.54}/mlarray/__init__.py +4 -1
  10. mlarray-0.0.54/mlarray/cli.py +226 -0
  11. {mlarray-0.0.52 → mlarray-0.0.54}/mlarray/meta.py +49 -9
  12. {mlarray-0.0.52 → mlarray-0.0.54}/mlarray/mlarray.py +26 -14
  13. {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/PKG-INFO +15 -1
  14. {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/SOURCES.txt +3 -1
  15. mlarray-0.0.54/mlarray.egg-info/scm_file_list.json +52 -0
  16. mlarray-0.0.54/mlarray.egg-info/scm_version.json +8 -0
  17. mlarray-0.0.54/tests/test_cli.py +420 -0
  18. {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_meta_safety.py +32 -0
  19. {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_metadata.py +40 -1
  20. mlarray-0.0.52/assets/banner.png +0 -0
  21. mlarray-0.0.52/assets/banner.png~ +0 -0
  22. mlarray-0.0.52/docs/cli.md +0 -34
  23. mlarray-0.0.52/mlarray/cli.py +0 -133
  24. mlarray-0.0.52/tests/test_cli.py +0 -139
  25. {mlarray-0.0.52 → mlarray-0.0.54}/.github/workflows/workflow.yml +0 -0
  26. {mlarray-0.0.52 → mlarray-0.0.54}/.gitignore +0 -0
  27. {mlarray-0.0.52 → mlarray-0.0.54}/LICENSE +0 -0
  28. {mlarray-0.0.52 → mlarray-0.0.54}/MANIFEST.in +0 -0
  29. {mlarray-0.0.52 → mlarray-0.0.54}/bench/.gitignore +0 -0
  30. {mlarray-0.0.52 → mlarray-0.0.54}/bench/README.md +0 -0
  31. {mlarray-0.0.52 → mlarray-0.0.54}/bench/bench_convert_nii_to_mla_random_read.py +0 -0
  32. {mlarray-0.0.52 → mlarray-0.0.54}/bench/bench_io_blosc2_layouts.py +0 -0
  33. {mlarray-0.0.52 → mlarray-0.0.54}/bench/helper/print_mla_layouts.py +0 -0
  34. {mlarray-0.0.52 → mlarray-0.0.54}/docs/index.md +0 -0
  35. {mlarray-0.0.52 → mlarray-0.0.54}/docs/optimization.md +0 -0
  36. {mlarray-0.0.52 → mlarray-0.0.54}/docs/why.md +0 -0
  37. {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_asarray.py +0 -0
  38. {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_bboxes_only.py +0 -0
  39. {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_channel.py +0 -0
  40. {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_compress_decompress.py +0 -0
  41. {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_compressed_vs_uncompressed.py +0 -0
  42. {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_in_memory_constructors.py +0 -0
  43. {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_metadata_only.py +0 -0
  44. {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_non_spatial.py +0 -0
  45. {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_open.py +0 -0
  46. {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_save_load.py +0 -0
  47. {mlarray-0.0.52 → mlarray-0.0.54}/mkdocs.yml +0 -0
  48. {mlarray-0.0.52 → mlarray-0.0.54}/mlarray/blosc2_layout_strategies.py +0 -0
  49. {mlarray-0.0.52 → mlarray-0.0.54}/mlarray/utils.py +0 -0
  50. {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/dependency_links.txt +0 -0
  51. {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/entry_points.txt +0 -0
  52. {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/requires.txt +0 -0
  53. {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/top_level.txt +0 -0
  54. {mlarray-0.0.52 → mlarray-0.0.54}/pyproject.toml +0 -0
  55. {mlarray-0.0.52 → mlarray-0.0.54}/setup.cfg +0 -0
  56. {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_asarray.py +0 -0
  57. {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_bboxes.py +0 -0
  58. {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_compress_decompress.py +0 -0
  59. {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_constructors.py +0 -0
  60. {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_create.py +0 -0
  61. {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_open.py +0 -0
  62. {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_optimization.py +0 -0
  63. {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_usage.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mlarray
3
- Version: 0.0.52
3
+ Version: 0.0.54
4
4
  Summary: Array format specialized for Machine Learning with Blosc2 backend and standardized metadata.
5
5
  Author-email: Karol Gotkowski <karol.gotkowski@dkfz.de>
6
6
  License: MIT
@@ -245,6 +245,20 @@ When converting from MLArray to NIfTI/NRRD, only `meta.source` is copied into
245
245
  the output header. Spatial metadata (`spacing`, `origin`, `direction`) is set
246
246
  explicitly from `meta.spatial`.
247
247
 
248
+ `meta.spatial.coord_system` is propagated conservatively. For NRRD input, the
249
+ CLI reads explicit NRRD space metadata when available and maps
250
+ `right-anterior-superior` to `RAS` and `left-posterior-superior` to `LPS`,
251
+ while preserving other explicit NRRD space strings verbatim. For NIfTI input,
252
+ the CLI sets `coord_system` to `LPS` based on the current
253
+ `MedVol -> SimpleITK/ITK` import path, where the imported geometry is
254
+ represented in the ITK physical-space convention. MedVol's reindexing to NumPy
255
+ layout does not change that world-space convention. On NRRD output, `RAS` and
256
+ `LPS` are written back as NRRD `space` metadata when possible; arbitrary custom
257
+ `coord_system` strings are not emitted as NRRD `space` declarations unless they
258
+ already match a supported explicit NRRD space string. NIfTI output preserves
259
+ geometry, but does not export `coord_system` as an explicit NIfTI metadata
260
+ field.
261
+
248
262
  ```bash
249
263
  mlarray_convert sample.nii.gz output.mla
250
264
  mlarray_convert sample.mla output.nii.gz
@@ -211,6 +211,20 @@ When converting from MLArray to NIfTI/NRRD, only `meta.source` is copied into
211
211
  the output header. Spatial metadata (`spacing`, `origin`, `direction`) is set
212
212
  explicitly from `meta.spatial`.
213
213
 
214
+ `meta.spatial.coord_system` is propagated conservatively. For NRRD input, the
215
+ CLI reads explicit NRRD space metadata when available and maps
216
+ `right-anterior-superior` to `RAS` and `left-posterior-superior` to `LPS`,
217
+ while preserving other explicit NRRD space strings verbatim. For NIfTI input,
218
+ the CLI sets `coord_system` to `LPS` based on the current
219
+ `MedVol -> SimpleITK/ITK` import path, where the imported geometry is
220
+ represented in the ITK physical-space convention. MedVol's reindexing to NumPy
221
+ layout does not change that world-space convention. On NRRD output, `RAS` and
222
+ `LPS` are written back as NRRD `space` metadata when possible; arbitrary custom
223
+ `coord_system` strings are not emitted as NRRD `space` declarations unless they
224
+ already match a supported explicit NRRD space string. NIfTI output preserves
225
+ geometry, but does not export `coord_system` as an explicit NIfTI metadata
226
+ field.
227
+
214
228
  ```bash
215
229
  mlarray_convert sample.nii.gz output.mla
216
230
  mlarray_convert sample.mla output.nii.gz
Binary file
@@ -0,0 +1,527 @@
1
+ #!/usr/bin/env python
2
+ """
3
+ Benchmark random patch read throughput across array formats.
4
+
5
+ Stage 1: Convert a NIfTI/NRRD dataset to npy, npz, zarr, hdf5, blosc2, mlarray.
6
+ Stage 2: Benchmark N random patch reads (open + read + close) per format.
7
+
8
+ The dataset_dir must contain a 'source' subdirectory with NIfTI/NRRD files.
9
+ Converted formats are written as siblings of 'source' inside dataset_dir.
10
+
11
+ Usage:
12
+ python bench/benchmark_formats.py /path/to/dataset_dir \\
13
+ --n_reads 1000 \\
14
+ --patch_size 64 64 8 \\
15
+ --seed 42
16
+
17
+ # Skip conversion if already done:
18
+ python bench/benchmark_formats.py /path/to/dataset_dir --skip_convert
19
+
20
+ # Benchmark only specific formats:
21
+ python bench/benchmark_formats.py /path/to/dataset_dir \\
22
+ --formats mlarray blosc2 hdf5
23
+ """
24
+
25
+ import argparse
26
+ import json
27
+ import math
28
+ import os
29
+ import random
30
+ import shutil
31
+ import sys
32
+ import time
33
+ from functools import partial
34
+ from pathlib import Path
35
+
36
+ import blosc2
37
+ import h5py
38
+ import nibabel as nib
39
+ import nrrd
40
+ import numpy as np
41
+ import zarr
42
+ from rich.console import Console
43
+ from rich.table import Table
44
+ from rich import box
45
+ from tqdm import tqdm
46
+ from tqdmp import tqdmp
47
+
48
+ from mlarray import MLArray
49
+
50
+ ALL_FORMATS = ["npy", "npz", "zarr", "hdf5", "blosc2", "mlarray"]
51
+ DEFAULT_FORMATS = ["npy", "zarr", "hdf5", "blosc2", "mlarray"]
52
+
53
+
54
+ # ---------------------------------------------------------------------------
55
+ # Source file loading
56
+ # ---------------------------------------------------------------------------
57
+
58
+ _NRRD_TYPE_TO_NUMPY = {
59
+ "signed char": "int8", "int8": "int8", "int8_t": "int8",
60
+ "uchar": "uint8", "unsigned char": "uint8", "uint8": "uint8", "uint8_t": "uint8",
61
+ "short": "int16", "short int": "int16", "signed short": "int16", "int16": "int16", "int16_t": "int16",
62
+ "ushort": "uint16", "unsigned short": "uint16", "uint16": "uint16", "uint16_t": "uint16",
63
+ "int": "int32", "signed int": "int32", "int32": "int32", "int32_t": "int32",
64
+ "uint": "uint32", "unsigned int": "uint32", "uint32": "uint32", "uint32_t": "uint32",
65
+ "longlong": "int64", "long long": "int64", "int64": "int64", "int64_t": "int64",
66
+ "ulonglong": "uint64", "unsigned long long": "uint64", "uint64": "uint64", "uint64_t": "uint64",
67
+ "float": "float32",
68
+ "double": "float64",
69
+ }
70
+
71
+
72
+ def _nbytes_from_header(filepath: Path) -> int:
73
+ """Return uncompressed byte count by reading only the file header."""
74
+ if str(filepath).endswith(".nrrd"):
75
+ with open(filepath, "rb") as fh:
76
+ header = nrrd.read_header(fh)
77
+ shape = tuple(int(x) for x in header["sizes"])
78
+ dtype = np.dtype(_NRRD_TYPE_TO_NUMPY[header["type"]])
79
+ else:
80
+ img = nib.load(str(filepath))
81
+ shape = img.shape
82
+ dtype = np.dtype(img.header.get_data_dtype())
83
+ return int(np.prod(shape)) * dtype.itemsize
84
+
85
+
86
+ def load_source_array(filepath: Path) -> np.ndarray:
87
+ """Load a NIfTI or NRRD file as a contiguous numpy array."""
88
+ if str(filepath).endswith(".nrrd"):
89
+ arr, _ = nrrd.read(str(filepath))
90
+ else:
91
+ arr = np.asarray(nib.load(str(filepath)).dataobj)
92
+ if not arr.flags["C_CONTIGUOUS"]:
93
+ arr = np.ascontiguousarray(arr)
94
+ return arr
95
+
96
+
97
+ def discover_source_files(source_dir: Path) -> list[Path]:
98
+ files = []
99
+ for ext in ("**/*.nii.gz", "**/*.nii", "**/*.nrrd"):
100
+ files.extend(source_dir.glob(ext))
101
+ return sorted(files)
102
+
103
+
104
+ def flat_stem(src: Path, source_dir: Path) -> str:
105
+ """Unique flat filename stem: relative path with separators replaced by '__'."""
106
+ rel = src.relative_to(source_dir)
107
+ parts = list(rel.parts)
108
+ # Strip all NIfTI/NRRD suffixes from the last part
109
+ name = parts[-1]
110
+ for suffix in (".nii.gz", ".nii", ".nrrd"):
111
+ if name.endswith(suffix):
112
+ name = name[: -len(suffix)]
113
+ break
114
+ parts[-1] = name
115
+ return "__".join(parts)
116
+
117
+
118
+ # ---------------------------------------------------------------------------
119
+ # Stage 1: Conversion
120
+ # ---------------------------------------------------------------------------
121
+
122
+ def convert_npy(arr: np.ndarray, out_path: Path) -> None:
123
+ np.save(out_path, arr)
124
+
125
+
126
+ def convert_npz(arr: np.ndarray, out_path: Path) -> None:
127
+ np.savez_compressed(out_path, data=arr)
128
+
129
+
130
+ def convert_zarr(arr: np.ndarray, out_path: Path) -> None:
131
+ """Zarr v3 with auto chunk/codec selection (BytesCodec + ZstdCodec at level 0)."""
132
+ if out_path.exists():
133
+ shutil.rmtree(out_path)
134
+ z = zarr.open(str(out_path), mode="w", shape=arr.shape, dtype=arr.dtype)
135
+ z[:] = arr
136
+
137
+
138
+ def convert_hdf5(arr: np.ndarray, out_path: Path) -> None:
139
+ """HDF5 with h5py auto chunking and gzip compression (level 4)."""
140
+ with h5py.File(out_path, "w") as f:
141
+ f.create_dataset("data", data=arr, chunks=True, compression="gzip", compression_opts=4)
142
+
143
+
144
+ def convert_blosc2(arr: np.ndarray, out_path: Path) -> None:
145
+ """Blosc2 NDArray with fully default chunk/block/codec settings."""
146
+ if out_path.exists():
147
+ out_path.unlink()
148
+ blosc2.asarray(arr, urlpath=str(out_path), mode="w")
149
+
150
+
151
+ def convert_mlarray(arr: np.ndarray, out_path: Path) -> None:
152
+ """MLArray with patch-size optimized chunk/block sizes via comp_blosc2_params."""
153
+ if out_path.exists():
154
+ out_path.unlink()
155
+ mla = MLArray.asarray(arr)
156
+ mla.save(out_path)
157
+
158
+
159
+ CONVERTERS = {
160
+ "npy": (convert_npy, ".npy"),
161
+ "npz": (convert_npz, ".npz"),
162
+ "zarr": (convert_zarr, ".zarr"),
163
+ "hdf5": (convert_hdf5, ".h5"),
164
+ "blosc2": (convert_blosc2, ".b2nd"),
165
+ "mlarray": (convert_mlarray, ".mla"),
166
+ }
167
+
168
+
169
+ def _convert_one(src: Path, output_dir: Path, source_dir: Path, formats: list[str]) -> None:
170
+ """Convert a single source file to all requested formats. Top-level for pickling."""
171
+ arr = load_source_array(src)
172
+ stem = flat_stem(src, source_dir)
173
+ for fmt in formats:
174
+ converter, suffix = CONVERTERS[fmt]
175
+ out_path = output_dir / fmt / (stem + suffix)
176
+ if out_path.exists():
177
+ continue
178
+ converter(arr, out_path)
179
+
180
+
181
+ def run_conversion(
182
+ source_files: list[Path],
183
+ source_dir: Path,
184
+ output_dir: Path,
185
+ formats: list[str],
186
+ num_workers: int,
187
+ ) -> None:
188
+ for fmt in formats:
189
+ (output_dir / fmt).mkdir(parents=True, exist_ok=True)
190
+
191
+ fn = partial(_convert_one, output_dir=output_dir, source_dir=source_dir, formats=formats)
192
+ tqdmp(fn, source_files, num_workers, desc=" Converting")
193
+
194
+
195
+ # ---------------------------------------------------------------------------
196
+ # Stage 2: Benchmark readers
197
+ # ---------------------------------------------------------------------------
198
+
199
+ def slices_for(starts: tuple, patch_size: tuple) -> tuple:
200
+ return tuple(slice(s, s + p) for s, p in zip(starts, patch_size))
201
+
202
+
203
+ def random_starts(shape: tuple, patch_size: tuple, rng: random.Random) -> tuple:
204
+ """Return per-axis start indices such that patch fits within shape."""
205
+ return tuple(
206
+ rng.randint(0, max(sh - ps, 0))
207
+ for sh, ps in zip(shape, patch_size)
208
+ )
209
+
210
+
211
+ def effective_patch(shape: tuple, patch_size: tuple) -> tuple:
212
+ """Clamp patch size to image dimensions."""
213
+ return tuple(min(ps, sh) for ps, sh in zip(patch_size, shape))
214
+
215
+
216
+ def read_patch_npy(filepath: Path, sl: tuple) -> np.ndarray:
217
+ arr = np.load(filepath, mmap_mode="r")
218
+ return arr[sl].copy()
219
+
220
+
221
+ def read_patch_npz(filepath: Path, sl: tuple) -> np.ndarray:
222
+ data = np.load(filepath)["data"]
223
+ return data[sl].copy()
224
+
225
+
226
+ def read_patch_zarr(filepath: Path, sl: tuple) -> np.ndarray:
227
+ z = zarr.open(str(filepath), mode="r")
228
+ return np.asarray(z[sl])
229
+
230
+
231
+ def read_patch_hdf5(filepath: Path, sl: tuple) -> np.ndarray:
232
+ with h5py.File(filepath, "r") as f:
233
+ return f["data"][sl]
234
+
235
+
236
+ def read_patch_blosc2(filepath: Path, sl: tuple) -> np.ndarray:
237
+ arr = blosc2.open(str(filepath), mode="r", mmap_mode="r", dparams={"nthreads": 1})
238
+ patch = arr[sl]
239
+ del arr
240
+ return patch
241
+
242
+
243
+ def read_patch_mlarray(filepath: Path, sl: tuple) -> np.ndarray:
244
+ arr = MLArray.open(str(filepath))
245
+ patch = arr[sl]
246
+ arr.close()
247
+ return patch
248
+
249
+
250
+ READERS = {
251
+ "npy": read_patch_npy,
252
+ "npz": read_patch_npz,
253
+ "zarr": read_patch_zarr,
254
+ "hdf5": read_patch_hdf5,
255
+ "blosc2": read_patch_blosc2,
256
+ "mlarray": read_patch_mlarray,
257
+ }
258
+
259
+
260
+ def collect_files(output_dir: Path, fmt: str) -> list[Path]:
261
+ _, suffix = CONVERTERS[fmt]
262
+ fmt_dir = output_dir / fmt
263
+ return sorted(fmt_dir.glob(f"*{suffix}"))
264
+
265
+
266
+ def get_shape(filepath: Path, fmt: str) -> tuple:
267
+ if fmt == "npy":
268
+ arr = np.load(filepath, mmap_mode="r")
269
+ return arr.shape
270
+ if fmt == "npz":
271
+ data = np.load(filepath)
272
+ return data["data"].shape
273
+ if fmt == "zarr":
274
+ z = zarr.open(str(filepath), mode="r")
275
+ return z.shape
276
+ if fmt == "hdf5":
277
+ with h5py.File(filepath, "r") as f:
278
+ return f["data"].shape
279
+ if fmt == "blosc2":
280
+ arr = blosc2.open(str(filepath), mode="r", mmap_mode="r")
281
+ shape = arr.shape
282
+ del arr
283
+ return shape
284
+ if fmt == "mlarray":
285
+ arr = MLArray.open(str(filepath))
286
+ shape = arr.shape
287
+ arr.close()
288
+ return shape
289
+ raise ValueError(fmt)
290
+
291
+
292
+ def disk_size_mb(path: Path) -> float:
293
+ if path.is_dir():
294
+ return sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) / 1e6
295
+ return path.stat().st_size / 1e6
296
+
297
+
298
+ def total_disk_mb(output_dir: Path, fmt: str) -> float:
299
+ fmt_dir = output_dir / fmt
300
+ return sum(disk_size_mb(p) for p in fmt_dir.iterdir())
301
+
302
+
303
+ def run_benchmark(
304
+ output_dir: Path,
305
+ formats: list[str],
306
+ n_reads: int,
307
+ patch_size: tuple,
308
+ rng: random.Random,
309
+ ) -> dict:
310
+ results = {}
311
+
312
+ for fmt in formats:
313
+ reader = READERS[fmt]
314
+ files = collect_files(output_dir, fmt)
315
+ if not files:
316
+ print(f" [{fmt}] No converted files found, skipping.")
317
+ continue
318
+
319
+ print(f" [{fmt}] Loading shapes ...", end=" ", flush=True)
320
+ file_shapes = [(f, get_shape(f, fmt)) for f in files]
321
+ print("done")
322
+
323
+ times_ms = []
324
+ patch_bytes = None
325
+
326
+ with tqdm(total=n_reads, desc=f" [{fmt}]", unit="read", leave=True, file=sys.stdout) as pbar:
327
+ for _ in range(n_reads):
328
+ filepath, shape = rng.choice(file_shapes)
329
+ ep = effective_patch(shape, patch_size)
330
+ starts = random_starts(shape, ep, rng)
331
+ sl = slices_for(starts, ep)
332
+
333
+ if patch_bytes is None:
334
+ patch_bytes = math.prod(ep) * 4 # float32
335
+
336
+ t0 = time.perf_counter()
337
+ _ = reader(filepath, sl)
338
+ t1 = time.perf_counter()
339
+ times_ms.append((t1 - t0) * 1000.0)
340
+ pbar.update(1)
341
+
342
+ times_arr = np.array(times_ms)
343
+ throughput_mbs = (patch_bytes / 1e6) / (np.mean(times_arr) / 1000.0)
344
+
345
+ results[fmt] = {
346
+ "n_reads": n_reads,
347
+ "patch_bytes": patch_bytes,
348
+ "disk_mb": round(total_disk_mb(output_dir, fmt), 2),
349
+ "mean_ms": round(float(np.mean(times_arr)), 3),
350
+ "median_ms": round(float(np.median(times_arr)), 3),
351
+ "std_ms": round(float(np.std(times_arr)), 3),
352
+ "min_ms": round(float(np.min(times_arr)), 3),
353
+ "max_ms": round(float(np.max(times_arr)), 3),
354
+ "p95_ms": round(float(np.percentile(times_arr, 95)), 3),
355
+ "throughput_mbs": round(throughput_mbs, 2),
356
+ }
357
+
358
+ return results
359
+
360
+
361
+ # ---------------------------------------------------------------------------
362
+ # Output formatting
363
+ # ---------------------------------------------------------------------------
364
+
365
+ def print_results_table(results: dict, source_disk_mb: float) -> None:
366
+ best_mean = min(r["mean_ms"] for r in results.values())
367
+ best_mb = min(r["disk_mb"] for r in results.values())
368
+ best_tput = max(r["throughput_mbs"] for r in results.values())
369
+
370
+ table = Table(
371
+ box=box.ROUNDED,
372
+ show_header=True,
373
+ header_style="bold cyan",
374
+ title="[bold]Random Patch Read Benchmark[/bold]",
375
+ title_style="bold white",
376
+ caption=(
377
+ f"Ratio = {source_disk_mb:.1f} MB uncompressed / format disk size • "
378
+ "Times include open + decompress + close per read • single process"
379
+ ),
380
+ caption_style="dim",
381
+ )
382
+
383
+ table.add_column("Format", style="bold", justify="left", no_wrap=True)
384
+ table.add_column("Disk (MB)", justify="right")
385
+ table.add_column("Ratio", justify="right")
386
+ table.add_column("Mean (ms)", justify="right")
387
+ table.add_column("Median (ms)", justify="right")
388
+ table.add_column("Std (ms)", justify="right")
389
+ table.add_column("P95 (ms)", justify="right")
390
+ table.add_column("Max (ms)", justify="right")
391
+ table.add_column("MB/s", style="bold", justify="right")
392
+
393
+ for fmt, r in results.items():
394
+ ratio = source_disk_mb / r["disk_mb"] if r["disk_mb"] > 0 else float("nan")
395
+
396
+ disk_str = f"[green]{r['disk_mb']:.1f}[/green]" if r["disk_mb"] == best_mb else f"{r['disk_mb']:.1f}"
397
+ mean_str = f"[green]{r['mean_ms']:.2f}[/green]" if r["mean_ms"] == best_mean else f"{r['mean_ms']:.2f}"
398
+ tput_str = f"[green]{r['throughput_mbs']:.1f}[/green]" if r["throughput_mbs"] == best_tput else f"{r['throughput_mbs']:.1f}"
399
+
400
+ table.add_row(
401
+ fmt,
402
+ disk_str,
403
+ f"{ratio:.2f}×",
404
+ mean_str,
405
+ f"{r['median_ms']:.2f}",
406
+ f"{r['std_ms']:.2f}",
407
+ f"{r['p95_ms']:.2f}",
408
+ f"{r['max_ms']:.2f}",
409
+ tput_str,
410
+ )
411
+
412
+ console = Console()
413
+ console.print()
414
+ console.print(table)
415
+
416
+
417
+ # ---------------------------------------------------------------------------
418
+ # Main
419
+ # ---------------------------------------------------------------------------
420
+
421
+ def parse_args() -> argparse.Namespace:
422
+ parser = argparse.ArgumentParser(
423
+ description="Benchmark random patch read throughput across array formats.",
424
+ formatter_class=argparse.RawDescriptionHelpFormatter,
425
+ epilog=__doc__,
426
+ )
427
+ parser.add_argument(
428
+ "dataset_dir",
429
+ type=Path,
430
+ help="Dataset root directory. Must contain a 'source' subdirectory with NIfTI/NRRD files. Converted formats are written as siblings of 'source'.",
431
+ )
432
+ parser.add_argument("--n_reads", type=int, default=1000, help="Number of random patch reads per format. Default: 1000.")
433
+ parser.add_argument(
434
+ "--patch_size",
435
+ type=int,
436
+ nargs="+",
437
+ default=[64, 64, 8],
438
+ help="Patch size per axis (must match array ndim). Default: 64 64 8.",
439
+ )
440
+ parser.add_argument(
441
+ "--formats",
442
+ nargs="+",
443
+ choices=ALL_FORMATS,
444
+ default=DEFAULT_FORMATS,
445
+ help=f"Formats to benchmark. Default: {', '.join(DEFAULT_FORMATS)} (npz excluded by default due to slow full-decompress reads).",
446
+ )
447
+ parser.add_argument("--max_images", type=int, default=None, help="Maximum number of images to use. The full list is shuffled then sliced. Default: all images.")
448
+ parser.add_argument("--skip_convert", action="store_true", help="Skip Stage 1 (conversion).")
449
+ parser.add_argument("--num_workers", type=int, default=12, help="Parallel workers for Stage 1 conversion. Default: all CPU cores.")
450
+ parser.add_argument("--seed", type=int, default=42, help="Random seed. Default: 42.")
451
+ return parser.parse_args()
452
+
453
+
454
+ def main() -> None:
455
+ args = parse_args()
456
+ dataset_dir = args.dataset_dir.resolve()
457
+ source_dir = dataset_dir / "source"
458
+ if not source_dir.is_dir():
459
+ raise RuntimeError(f"Expected a 'source' subdirectory in {dataset_dir}")
460
+ output_dir = dataset_dir
461
+ patch_size = tuple(args.patch_size)
462
+ rng = random.Random(args.seed)
463
+
464
+ print(f"Dataset dir: {dataset_dir}")
465
+ print(f"Source dir : {source_dir}")
466
+ print(f"Formats : {args.formats}")
467
+ print(f"Patch size : {patch_size}")
468
+ print(f"N reads : {args.n_reads}")
469
+ print(f"Max images : {args.max_images if args.max_images is not None else 'all'}")
470
+ print(f"Num workers: {args.num_workers}")
471
+ print(f"Seed : {args.seed}")
472
+
473
+ all_source_files = discover_source_files(source_dir)
474
+ if not all_source_files:
475
+ raise RuntimeError(f"No NIfTI/NRRD files found under {source_dir}")
476
+ print(f"\nFound {len(all_source_files)} source file(s).")
477
+
478
+ rng.shuffle(all_source_files)
479
+ source_files = all_source_files[: args.max_images] if args.max_images is not None else all_source_files
480
+ if len(source_files) < len(all_source_files):
481
+ print(f"Using {len(source_files)} image(s) (--max_images={args.max_images}).")
482
+
483
+ source_disk_mb = sum(_nbytes_from_header(f) for f in source_files) / 1e6
484
+ print(f"Total uncompressed data: {source_disk_mb:.1f} MB")
485
+
486
+ # ------------------------------------------------------------------
487
+ # Stage 1: Conversion
488
+ # ------------------------------------------------------------------
489
+ if not args.skip_convert:
490
+ print("\n=== Stage 1: Converting ===")
491
+ run_conversion(source_files, source_dir, output_dir, args.formats, args.num_workers)
492
+ print("Conversion complete.")
493
+ else:
494
+ print("\nStage 1 skipped (--skip_convert).")
495
+
496
+ # ------------------------------------------------------------------
497
+ # Stage 2: Benchmark
498
+ # ------------------------------------------------------------------
499
+ print("\n=== Stage 2: Benchmarking ===")
500
+ print(
501
+ "Note: results reflect OS page cache state. Files will warm in cache during the run."
502
+ )
503
+ results = run_benchmark(output_dir, args.formats, args.n_reads, patch_size, rng)
504
+
505
+ # ------------------------------------------------------------------
506
+ # Results
507
+ # ------------------------------------------------------------------
508
+ print_results_table(results, source_disk_mb)
509
+
510
+ results_path = output_dir / "results.json"
511
+ payload = {
512
+ "dataset_dir": str(dataset_dir),
513
+ "source_dir": str(source_dir),
514
+ "n_images": len(source_files),
515
+ "n_reads": args.n_reads,
516
+ "patch_size": list(patch_size),
517
+ "seed": args.seed,
518
+ "source_disk_mb": round(source_disk_mb, 2),
519
+ "formats": results,
520
+ }
521
+ results_path.parent.mkdir(parents=True, exist_ok=True)
522
+ results_path.write_text(json.dumps(payload, indent=2))
523
+ print(f"\nResults saved to {results_path}")
524
+
525
+
526
+ if __name__ == "__main__":
527
+ main()
@@ -20,6 +20,8 @@
20
20
 
21
21
  ::: mlarray.meta.AxisLabelEnum
22
22
 
23
+ ::: mlarray.meta.CoordSystemEnum
24
+
23
25
  ::: mlarray.meta.MetaStatistics
24
26
 
25
27
  ::: mlarray.meta.MetaBbox
@@ -0,0 +1,59 @@
1
+ # CLI
2
+
3
+ MLArray includes a small command-line interface for common tasks such as **inspecting file headers** and **converting between MLArray and existing image formats**. This is especially useful when you want to quickly verify metadata, debug a dataset, or batch-convert files without writing Python code.
4
+
5
+ The CLI currently focuses on core workflows (header inspection and conversion). Support for converting a wider range of image formats will be added over time.
6
+
7
+ ---
8
+
9
+ ## `mlarray_header`
10
+
11
+ Print the metadata header from a `.mla` file.
12
+
13
+ This command is useful for quickly checking spatial metadata, stored schemas, and other file-level information without loading the full array into memory.
14
+
15
+ ```bash
16
+ mlarray_header sample.mla
17
+ ```
18
+
19
+ ---
20
+
21
+ ## `mlarray_convert`
22
+
23
+ Convert between MLArray and NIfTI/NRRD files.
24
+
25
+ This provides an easy way to move medical imaging data into or out of an MLArray-based workflow.
26
+
27
+ When converting from NIfTI/NRRD to MLArray, the source header is copied into `meta.source`.
28
+
29
+ When converting from NIfTI/NRRD to MLArray, `meta.spatial.coord_system` is
30
+ propagated conservatively:
31
+
32
+ - For NRRD input, the converter looks for explicit NRRD space metadata in the
33
+ header (`space`, falling back to `NRRD_space` from the SimpleITK path). It
34
+ maps `right-anterior-superior` to `RAS`, `left-posterior-superior` to `LPS`,
35
+ and preserves other explicit NRRD space strings verbatim. If no explicit
36
+ space metadata is available, `coord_system` is left unset.
37
+ - For NIfTI input, the converter sets `coord_system` to `LPS`. This is based on
38
+ the current `MedVol -> SimpleITK/ITK` import path: the imported geometry is
39
+ represented in the ITK physical-space convention, and MedVol only reindexes
40
+ axes to match NumPy array layout. It is not a statement that raw NIfTI
41
+ qform/sform metadata is preserved verbatim.
42
+
43
+ When converting from MLArray to NIfTI/NRRD, only `meta.source` is copied into
44
+ the output header. Spatial metadata (`spacing`, `origin`, and `direction`) is
45
+ written explicitly from `meta.spatial`.
46
+
47
+ - For NRRD output, `coord_system="RAS"` and `coord_system="LPS"` are written as
48
+ the corresponding NRRD `space` strings. A small set of already-valid NRRD
49
+ space strings is also preserved. Unsupported arbitrary `coord_system` strings
50
+ are not written as NRRD `space` metadata.
51
+ - For NIfTI output, geometry is preserved through the existing affine-related
52
+ path, but `coord_system` is not exported as an explicit NIfTI metadata field
53
+ because the current conversion path does not expose a reliable direct
54
+ representation for it.
55
+
56
+ ```bash
57
+ mlarray_convert sample.nii.gz output.mla
58
+ mlarray_convert sample.mla output.nrrd
59
+ ```
@@ -67,6 +67,7 @@ This section stores the information needed to interpret the array in physical sp
67
67
  | direction | Optional[List[List[float]]] | Direction matrix, shape `[ndims][ndims]`. |
68
68
  | shape | Optional[List[int]] | Full array shape, length = spatial + non-spatial axes. |
69
69
  | axis_labels | Optional[List[str,AxisLabel]] | Per-axis labels or roles, length = full array `ndims`. |
70
+ | coord_system | Optional[Union[str,CoordSystem]] | World/anatomical coordinate convention that `spacing`, `origin`, `direction`, or `affine` map into. |
70
71
  | axis_units | Optional[List[str]] | Per-axis units, length = full array `ndims`. |
71
72
 
72
73
  ---
@@ -87,6 +88,35 @@ Axis labels describe the semantic role of each axis. They may be provided as str
87
88
  | continuous | Continuous-valued axis (non-spatial). |
88
89
  | components | Component axis (e.g., vector components). |
89
90
 
91
+ `axis_labels` defines which array axes correspond to spatial axes such as
92
+ `spatial_x`, `spatial_y`, and `spatial_z`. It does not define what world-space
93
+ X/Y/Z mean.
94
+
95
+ ---
96
+
97
+ #### CoordSystem
98
+
99
+ `coord_system` describes the world/anatomical coordinate convention that the
100
+ spatial metadata maps into. It applies to both geometry styles:
101
+ `spacing`/`origin`/`direction` and `affine`.
102
+
103
+ Built-in standard values are:
104
+
105
+ | value | description |
106
+ | ------- | ----------- |
107
+ | RAS | Right-Anterior-Superior world convention. |
108
+ | LPS | Left-Posterior-Superior world convention. |
109
+ | unknown | Coordinate system is unknown. |
110
+ | other | Coordinate system exists but falls outside the built-in vocabulary. |
111
+
112
+ Arbitrary strings are also allowed and are preserved exactly as provided. This
113
+ makes `coord_system` an extensible standard vocabulary rather than a closed
114
+ enum.
115
+
116
+ `coord_system` does not determine array axis order. `axis_labels` tells you
117
+ which array axes are spatial axes; `coord_system` tells you what world
118
+ X/Y/Z mean in those spatial coordinates.
119
+
90
120
  ---
91
121
 
92
122
  ### stats