mlarray 0.0.52__tar.gz → 0.0.54__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mlarray-0.0.52 → mlarray-0.0.54}/PKG-INFO +15 -1
- {mlarray-0.0.52 → mlarray-0.0.54}/README.md +14 -0
- mlarray-0.0.54/assets/banner.png +0 -0
- mlarray-0.0.54/bench/benchmark_formats.py +527 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/docs/api.md +2 -0
- mlarray-0.0.54/docs/cli.md +59 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/docs/schema.md +30 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/docs/usage.md +30 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/mlarray/__init__.py +4 -1
- mlarray-0.0.54/mlarray/cli.py +226 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/mlarray/meta.py +49 -9
- {mlarray-0.0.52 → mlarray-0.0.54}/mlarray/mlarray.py +26 -14
- {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/PKG-INFO +15 -1
- {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/SOURCES.txt +3 -1
- mlarray-0.0.54/mlarray.egg-info/scm_file_list.json +52 -0
- mlarray-0.0.54/mlarray.egg-info/scm_version.json +8 -0
- mlarray-0.0.54/tests/test_cli.py +420 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_meta_safety.py +32 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_metadata.py +40 -1
- mlarray-0.0.52/assets/banner.png +0 -0
- mlarray-0.0.52/assets/banner.png~ +0 -0
- mlarray-0.0.52/docs/cli.md +0 -34
- mlarray-0.0.52/mlarray/cli.py +0 -133
- mlarray-0.0.52/tests/test_cli.py +0 -139
- {mlarray-0.0.52 → mlarray-0.0.54}/.github/workflows/workflow.yml +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/.gitignore +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/LICENSE +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/MANIFEST.in +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/bench/.gitignore +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/bench/README.md +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/bench/bench_convert_nii_to_mla_random_read.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/bench/bench_io_blosc2_layouts.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/bench/helper/print_mla_layouts.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/docs/index.md +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/docs/optimization.md +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/docs/why.md +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_asarray.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_bboxes_only.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_channel.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_compress_decompress.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_compressed_vs_uncompressed.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_in_memory_constructors.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_metadata_only.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_non_spatial.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_open.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/examples/example_save_load.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/mkdocs.yml +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/mlarray/blosc2_layout_strategies.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/mlarray/utils.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/dependency_links.txt +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/entry_points.txt +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/requires.txt +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/mlarray.egg-info/top_level.txt +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/pyproject.toml +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/setup.cfg +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_asarray.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_bboxes.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_compress_decompress.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_constructors.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_create.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_open.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_optimization.py +0 -0
- {mlarray-0.0.52 → mlarray-0.0.54}/tests/test_usage.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mlarray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.54
|
|
4
4
|
Summary: Array format specialized for Machine Learning with Blosc2 backend and standardized metadata.
|
|
5
5
|
Author-email: Karol Gotkowski <karol.gotkowski@dkfz.de>
|
|
6
6
|
License: MIT
|
|
@@ -245,6 +245,20 @@ When converting from MLArray to NIfTI/NRRD, only `meta.source` is copied into
|
|
|
245
245
|
the output header. Spatial metadata (`spacing`, `origin`, `direction`) is set
|
|
246
246
|
explicitly from `meta.spatial`.
|
|
247
247
|
|
|
248
|
+
`meta.spatial.coord_system` is propagated conservatively. For NRRD input, the
|
|
249
|
+
CLI reads explicit NRRD space metadata when available and maps
|
|
250
|
+
`right-anterior-superior` to `RAS` and `left-posterior-superior` to `LPS`,
|
|
251
|
+
while preserving other explicit NRRD space strings verbatim. For NIfTI input,
|
|
252
|
+
the CLI sets `coord_system` to `LPS` based on the current
|
|
253
|
+
`MedVol -> SimpleITK/ITK` import path, where the imported geometry is
|
|
254
|
+
represented in the ITK physical-space convention. MedVol's reindexing to NumPy
|
|
255
|
+
layout does not change that world-space convention. On NRRD output, `RAS` and
|
|
256
|
+
`LPS` are written back as NRRD `space` metadata when possible; arbitrary custom
|
|
257
|
+
`coord_system` strings are not emitted as NRRD `space` declarations unless they
|
|
258
|
+
already match a supported explicit NRRD space string. NIfTI output preserves
|
|
259
|
+
geometry, but does not export `coord_system` as an explicit NIfTI metadata
|
|
260
|
+
field.
|
|
261
|
+
|
|
248
262
|
```bash
|
|
249
263
|
mlarray_convert sample.nii.gz output.mla
|
|
250
264
|
mlarray_convert sample.mla output.nii.gz
|
|
@@ -211,6 +211,20 @@ When converting from MLArray to NIfTI/NRRD, only `meta.source` is copied into
|
|
|
211
211
|
the output header. Spatial metadata (`spacing`, `origin`, `direction`) is set
|
|
212
212
|
explicitly from `meta.spatial`.
|
|
213
213
|
|
|
214
|
+
`meta.spatial.coord_system` is propagated conservatively. For NRRD input, the
|
|
215
|
+
CLI reads explicit NRRD space metadata when available and maps
|
|
216
|
+
`right-anterior-superior` to `RAS` and `left-posterior-superior` to `LPS`,
|
|
217
|
+
while preserving other explicit NRRD space strings verbatim. For NIfTI input,
|
|
218
|
+
the CLI sets `coord_system` to `LPS` based on the current
|
|
219
|
+
`MedVol -> SimpleITK/ITK` import path, where the imported geometry is
|
|
220
|
+
represented in the ITK physical-space convention. MedVol's reindexing to NumPy
|
|
221
|
+
layout does not change that world-space convention. On NRRD output, `RAS` and
|
|
222
|
+
`LPS` are written back as NRRD `space` metadata when possible; arbitrary custom
|
|
223
|
+
`coord_system` strings are not emitted as NRRD `space` declarations unless they
|
|
224
|
+
already match a supported explicit NRRD space string. NIfTI output preserves
|
|
225
|
+
geometry, but does not export `coord_system` as an explicit NIfTI metadata
|
|
226
|
+
field.
|
|
227
|
+
|
|
214
228
|
```bash
|
|
215
229
|
mlarray_convert sample.nii.gz output.mla
|
|
216
230
|
mlarray_convert sample.mla output.nii.gz
|
|
Binary file
|
|
@@ -0,0 +1,527 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
"""
|
|
3
|
+
Benchmark random patch read throughput across array formats.
|
|
4
|
+
|
|
5
|
+
Stage 1: Convert a NIfTI/NRRD dataset to npy, npz, zarr, hdf5, blosc2, mlarray.
|
|
6
|
+
Stage 2: Benchmark N random patch reads (open + read + close) per format.
|
|
7
|
+
|
|
8
|
+
The dataset_dir must contain a 'source' subdirectory with NIfTI/NRRD files.
|
|
9
|
+
Converted formats are written as siblings of 'source' inside dataset_dir.
|
|
10
|
+
|
|
11
|
+
Usage:
|
|
12
|
+
python bench/benchmark_formats.py /path/to/dataset_dir \\
|
|
13
|
+
--n_reads 1000 \\
|
|
14
|
+
--patch_size 64 64 8 \\
|
|
15
|
+
--seed 42
|
|
16
|
+
|
|
17
|
+
# Skip conversion if already done:
|
|
18
|
+
python bench/benchmark_formats.py /path/to/dataset_dir --skip_convert
|
|
19
|
+
|
|
20
|
+
# Benchmark only specific formats:
|
|
21
|
+
python bench/benchmark_formats.py /path/to/dataset_dir \\
|
|
22
|
+
--formats mlarray blosc2 hdf5
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
import argparse
|
|
26
|
+
import json
|
|
27
|
+
import math
|
|
28
|
+
import os
|
|
29
|
+
import random
|
|
30
|
+
import shutil
|
|
31
|
+
import sys
|
|
32
|
+
import time
|
|
33
|
+
from functools import partial
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
|
|
36
|
+
import blosc2
|
|
37
|
+
import h5py
|
|
38
|
+
import nibabel as nib
|
|
39
|
+
import nrrd
|
|
40
|
+
import numpy as np
|
|
41
|
+
import zarr
|
|
42
|
+
from rich.console import Console
|
|
43
|
+
from rich.table import Table
|
|
44
|
+
from rich import box
|
|
45
|
+
from tqdm import tqdm
|
|
46
|
+
from tqdmp import tqdmp
|
|
47
|
+
|
|
48
|
+
from mlarray import MLArray
|
|
49
|
+
|
|
50
|
+
ALL_FORMATS = ["npy", "npz", "zarr", "hdf5", "blosc2", "mlarray"]
|
|
51
|
+
DEFAULT_FORMATS = ["npy", "zarr", "hdf5", "blosc2", "mlarray"]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# ---------------------------------------------------------------------------
|
|
55
|
+
# Source file loading
|
|
56
|
+
# ---------------------------------------------------------------------------
|
|
57
|
+
|
|
58
|
+
_NRRD_TYPE_TO_NUMPY = {
|
|
59
|
+
"signed char": "int8", "int8": "int8", "int8_t": "int8",
|
|
60
|
+
"uchar": "uint8", "unsigned char": "uint8", "uint8": "uint8", "uint8_t": "uint8",
|
|
61
|
+
"short": "int16", "short int": "int16", "signed short": "int16", "int16": "int16", "int16_t": "int16",
|
|
62
|
+
"ushort": "uint16", "unsigned short": "uint16", "uint16": "uint16", "uint16_t": "uint16",
|
|
63
|
+
"int": "int32", "signed int": "int32", "int32": "int32", "int32_t": "int32",
|
|
64
|
+
"uint": "uint32", "unsigned int": "uint32", "uint32": "uint32", "uint32_t": "uint32",
|
|
65
|
+
"longlong": "int64", "long long": "int64", "int64": "int64", "int64_t": "int64",
|
|
66
|
+
"ulonglong": "uint64", "unsigned long long": "uint64", "uint64": "uint64", "uint64_t": "uint64",
|
|
67
|
+
"float": "float32",
|
|
68
|
+
"double": "float64",
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _nbytes_from_header(filepath: Path) -> int:
|
|
73
|
+
"""Return uncompressed byte count by reading only the file header."""
|
|
74
|
+
if str(filepath).endswith(".nrrd"):
|
|
75
|
+
with open(filepath, "rb") as fh:
|
|
76
|
+
header = nrrd.read_header(fh)
|
|
77
|
+
shape = tuple(int(x) for x in header["sizes"])
|
|
78
|
+
dtype = np.dtype(_NRRD_TYPE_TO_NUMPY[header["type"]])
|
|
79
|
+
else:
|
|
80
|
+
img = nib.load(str(filepath))
|
|
81
|
+
shape = img.shape
|
|
82
|
+
dtype = np.dtype(img.header.get_data_dtype())
|
|
83
|
+
return int(np.prod(shape)) * dtype.itemsize
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def load_source_array(filepath: Path) -> np.ndarray:
|
|
87
|
+
"""Load a NIfTI or NRRD file as a contiguous numpy array."""
|
|
88
|
+
if str(filepath).endswith(".nrrd"):
|
|
89
|
+
arr, _ = nrrd.read(str(filepath))
|
|
90
|
+
else:
|
|
91
|
+
arr = np.asarray(nib.load(str(filepath)).dataobj)
|
|
92
|
+
if not arr.flags["C_CONTIGUOUS"]:
|
|
93
|
+
arr = np.ascontiguousarray(arr)
|
|
94
|
+
return arr
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def discover_source_files(source_dir: Path) -> list[Path]:
|
|
98
|
+
files = []
|
|
99
|
+
for ext in ("**/*.nii.gz", "**/*.nii", "**/*.nrrd"):
|
|
100
|
+
files.extend(source_dir.glob(ext))
|
|
101
|
+
return sorted(files)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def flat_stem(src: Path, source_dir: Path) -> str:
|
|
105
|
+
"""Unique flat filename stem: relative path with separators replaced by '__'."""
|
|
106
|
+
rel = src.relative_to(source_dir)
|
|
107
|
+
parts = list(rel.parts)
|
|
108
|
+
# Strip all NIfTI/NRRD suffixes from the last part
|
|
109
|
+
name = parts[-1]
|
|
110
|
+
for suffix in (".nii.gz", ".nii", ".nrrd"):
|
|
111
|
+
if name.endswith(suffix):
|
|
112
|
+
name = name[: -len(suffix)]
|
|
113
|
+
break
|
|
114
|
+
parts[-1] = name
|
|
115
|
+
return "__".join(parts)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# ---------------------------------------------------------------------------
|
|
119
|
+
# Stage 1: Conversion
|
|
120
|
+
# ---------------------------------------------------------------------------
|
|
121
|
+
|
|
122
|
+
def convert_npy(arr: np.ndarray, out_path: Path) -> None:
|
|
123
|
+
np.save(out_path, arr)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def convert_npz(arr: np.ndarray, out_path: Path) -> None:
|
|
127
|
+
np.savez_compressed(out_path, data=arr)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def convert_zarr(arr: np.ndarray, out_path: Path) -> None:
|
|
131
|
+
"""Zarr v3 with auto chunk/codec selection (BytesCodec + ZstdCodec at level 0)."""
|
|
132
|
+
if out_path.exists():
|
|
133
|
+
shutil.rmtree(out_path)
|
|
134
|
+
z = zarr.open(str(out_path), mode="w", shape=arr.shape, dtype=arr.dtype)
|
|
135
|
+
z[:] = arr
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def convert_hdf5(arr: np.ndarray, out_path: Path) -> None:
|
|
139
|
+
"""HDF5 with h5py auto chunking and gzip compression (level 4)."""
|
|
140
|
+
with h5py.File(out_path, "w") as f:
|
|
141
|
+
f.create_dataset("data", data=arr, chunks=True, compression="gzip", compression_opts=4)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def convert_blosc2(arr: np.ndarray, out_path: Path) -> None:
|
|
145
|
+
"""Blosc2 NDArray with fully default chunk/block/codec settings."""
|
|
146
|
+
if out_path.exists():
|
|
147
|
+
out_path.unlink()
|
|
148
|
+
blosc2.asarray(arr, urlpath=str(out_path), mode="w")
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def convert_mlarray(arr: np.ndarray, out_path: Path) -> None:
|
|
152
|
+
"""MLArray with patch-size optimized chunk/block sizes via comp_blosc2_params."""
|
|
153
|
+
if out_path.exists():
|
|
154
|
+
out_path.unlink()
|
|
155
|
+
mla = MLArray.asarray(arr)
|
|
156
|
+
mla.save(out_path)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
CONVERTERS = {
|
|
160
|
+
"npy": (convert_npy, ".npy"),
|
|
161
|
+
"npz": (convert_npz, ".npz"),
|
|
162
|
+
"zarr": (convert_zarr, ".zarr"),
|
|
163
|
+
"hdf5": (convert_hdf5, ".h5"),
|
|
164
|
+
"blosc2": (convert_blosc2, ".b2nd"),
|
|
165
|
+
"mlarray": (convert_mlarray, ".mla"),
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _convert_one(src: Path, output_dir: Path, source_dir: Path, formats: list[str]) -> None:
|
|
170
|
+
"""Convert a single source file to all requested formats. Top-level for pickling."""
|
|
171
|
+
arr = load_source_array(src)
|
|
172
|
+
stem = flat_stem(src, source_dir)
|
|
173
|
+
for fmt in formats:
|
|
174
|
+
converter, suffix = CONVERTERS[fmt]
|
|
175
|
+
out_path = output_dir / fmt / (stem + suffix)
|
|
176
|
+
if out_path.exists():
|
|
177
|
+
continue
|
|
178
|
+
converter(arr, out_path)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def run_conversion(
|
|
182
|
+
source_files: list[Path],
|
|
183
|
+
source_dir: Path,
|
|
184
|
+
output_dir: Path,
|
|
185
|
+
formats: list[str],
|
|
186
|
+
num_workers: int,
|
|
187
|
+
) -> None:
|
|
188
|
+
for fmt in formats:
|
|
189
|
+
(output_dir / fmt).mkdir(parents=True, exist_ok=True)
|
|
190
|
+
|
|
191
|
+
fn = partial(_convert_one, output_dir=output_dir, source_dir=source_dir, formats=formats)
|
|
192
|
+
tqdmp(fn, source_files, num_workers, desc=" Converting")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
# ---------------------------------------------------------------------------
|
|
196
|
+
# Stage 2: Benchmark readers
|
|
197
|
+
# ---------------------------------------------------------------------------
|
|
198
|
+
|
|
199
|
+
def slices_for(starts: tuple, patch_size: tuple) -> tuple:
|
|
200
|
+
return tuple(slice(s, s + p) for s, p in zip(starts, patch_size))
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def random_starts(shape: tuple, patch_size: tuple, rng: random.Random) -> tuple:
|
|
204
|
+
"""Return per-axis start indices such that patch fits within shape."""
|
|
205
|
+
return tuple(
|
|
206
|
+
rng.randint(0, max(sh - ps, 0))
|
|
207
|
+
for sh, ps in zip(shape, patch_size)
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def effective_patch(shape: tuple, patch_size: tuple) -> tuple:
|
|
212
|
+
"""Clamp patch size to image dimensions."""
|
|
213
|
+
return tuple(min(ps, sh) for ps, sh in zip(patch_size, shape))
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def read_patch_npy(filepath: Path, sl: tuple) -> np.ndarray:
|
|
217
|
+
arr = np.load(filepath, mmap_mode="r")
|
|
218
|
+
return arr[sl].copy()
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def read_patch_npz(filepath: Path, sl: tuple) -> np.ndarray:
|
|
222
|
+
data = np.load(filepath)["data"]
|
|
223
|
+
return data[sl].copy()
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def read_patch_zarr(filepath: Path, sl: tuple) -> np.ndarray:
|
|
227
|
+
z = zarr.open(str(filepath), mode="r")
|
|
228
|
+
return np.asarray(z[sl])
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def read_patch_hdf5(filepath: Path, sl: tuple) -> np.ndarray:
|
|
232
|
+
with h5py.File(filepath, "r") as f:
|
|
233
|
+
return f["data"][sl]
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def read_patch_blosc2(filepath: Path, sl: tuple) -> np.ndarray:
|
|
237
|
+
arr = blosc2.open(str(filepath), mode="r", mmap_mode="r", dparams={"nthreads": 1})
|
|
238
|
+
patch = arr[sl]
|
|
239
|
+
del arr
|
|
240
|
+
return patch
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def read_patch_mlarray(filepath: Path, sl: tuple) -> np.ndarray:
|
|
244
|
+
arr = MLArray.open(str(filepath))
|
|
245
|
+
patch = arr[sl]
|
|
246
|
+
arr.close()
|
|
247
|
+
return patch
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
READERS = {
|
|
251
|
+
"npy": read_patch_npy,
|
|
252
|
+
"npz": read_patch_npz,
|
|
253
|
+
"zarr": read_patch_zarr,
|
|
254
|
+
"hdf5": read_patch_hdf5,
|
|
255
|
+
"blosc2": read_patch_blosc2,
|
|
256
|
+
"mlarray": read_patch_mlarray,
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def collect_files(output_dir: Path, fmt: str) -> list[Path]:
|
|
261
|
+
_, suffix = CONVERTERS[fmt]
|
|
262
|
+
fmt_dir = output_dir / fmt
|
|
263
|
+
return sorted(fmt_dir.glob(f"*{suffix}"))
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def get_shape(filepath: Path, fmt: str) -> tuple:
|
|
267
|
+
if fmt == "npy":
|
|
268
|
+
arr = np.load(filepath, mmap_mode="r")
|
|
269
|
+
return arr.shape
|
|
270
|
+
if fmt == "npz":
|
|
271
|
+
data = np.load(filepath)
|
|
272
|
+
return data["data"].shape
|
|
273
|
+
if fmt == "zarr":
|
|
274
|
+
z = zarr.open(str(filepath), mode="r")
|
|
275
|
+
return z.shape
|
|
276
|
+
if fmt == "hdf5":
|
|
277
|
+
with h5py.File(filepath, "r") as f:
|
|
278
|
+
return f["data"].shape
|
|
279
|
+
if fmt == "blosc2":
|
|
280
|
+
arr = blosc2.open(str(filepath), mode="r", mmap_mode="r")
|
|
281
|
+
shape = arr.shape
|
|
282
|
+
del arr
|
|
283
|
+
return shape
|
|
284
|
+
if fmt == "mlarray":
|
|
285
|
+
arr = MLArray.open(str(filepath))
|
|
286
|
+
shape = arr.shape
|
|
287
|
+
arr.close()
|
|
288
|
+
return shape
|
|
289
|
+
raise ValueError(fmt)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def disk_size_mb(path: Path) -> float:
|
|
293
|
+
if path.is_dir():
|
|
294
|
+
return sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) / 1e6
|
|
295
|
+
return path.stat().st_size / 1e6
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def total_disk_mb(output_dir: Path, fmt: str) -> float:
|
|
299
|
+
fmt_dir = output_dir / fmt
|
|
300
|
+
return sum(disk_size_mb(p) for p in fmt_dir.iterdir())
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def run_benchmark(
|
|
304
|
+
output_dir: Path,
|
|
305
|
+
formats: list[str],
|
|
306
|
+
n_reads: int,
|
|
307
|
+
patch_size: tuple,
|
|
308
|
+
rng: random.Random,
|
|
309
|
+
) -> dict:
|
|
310
|
+
results = {}
|
|
311
|
+
|
|
312
|
+
for fmt in formats:
|
|
313
|
+
reader = READERS[fmt]
|
|
314
|
+
files = collect_files(output_dir, fmt)
|
|
315
|
+
if not files:
|
|
316
|
+
print(f" [{fmt}] No converted files found, skipping.")
|
|
317
|
+
continue
|
|
318
|
+
|
|
319
|
+
print(f" [{fmt}] Loading shapes ...", end=" ", flush=True)
|
|
320
|
+
file_shapes = [(f, get_shape(f, fmt)) for f in files]
|
|
321
|
+
print("done")
|
|
322
|
+
|
|
323
|
+
times_ms = []
|
|
324
|
+
patch_bytes = None
|
|
325
|
+
|
|
326
|
+
with tqdm(total=n_reads, desc=f" [{fmt}]", unit="read", leave=True, file=sys.stdout) as pbar:
|
|
327
|
+
for _ in range(n_reads):
|
|
328
|
+
filepath, shape = rng.choice(file_shapes)
|
|
329
|
+
ep = effective_patch(shape, patch_size)
|
|
330
|
+
starts = random_starts(shape, ep, rng)
|
|
331
|
+
sl = slices_for(starts, ep)
|
|
332
|
+
|
|
333
|
+
if patch_bytes is None:
|
|
334
|
+
patch_bytes = math.prod(ep) * 4 # float32
|
|
335
|
+
|
|
336
|
+
t0 = time.perf_counter()
|
|
337
|
+
_ = reader(filepath, sl)
|
|
338
|
+
t1 = time.perf_counter()
|
|
339
|
+
times_ms.append((t1 - t0) * 1000.0)
|
|
340
|
+
pbar.update(1)
|
|
341
|
+
|
|
342
|
+
times_arr = np.array(times_ms)
|
|
343
|
+
throughput_mbs = (patch_bytes / 1e6) / (np.mean(times_arr) / 1000.0)
|
|
344
|
+
|
|
345
|
+
results[fmt] = {
|
|
346
|
+
"n_reads": n_reads,
|
|
347
|
+
"patch_bytes": patch_bytes,
|
|
348
|
+
"disk_mb": round(total_disk_mb(output_dir, fmt), 2),
|
|
349
|
+
"mean_ms": round(float(np.mean(times_arr)), 3),
|
|
350
|
+
"median_ms": round(float(np.median(times_arr)), 3),
|
|
351
|
+
"std_ms": round(float(np.std(times_arr)), 3),
|
|
352
|
+
"min_ms": round(float(np.min(times_arr)), 3),
|
|
353
|
+
"max_ms": round(float(np.max(times_arr)), 3),
|
|
354
|
+
"p95_ms": round(float(np.percentile(times_arr, 95)), 3),
|
|
355
|
+
"throughput_mbs": round(throughput_mbs, 2),
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
return results
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
# ---------------------------------------------------------------------------
|
|
362
|
+
# Output formatting
|
|
363
|
+
# ---------------------------------------------------------------------------
|
|
364
|
+
|
|
365
|
+
def print_results_table(results: dict, source_disk_mb: float) -> None:
|
|
366
|
+
best_mean = min(r["mean_ms"] for r in results.values())
|
|
367
|
+
best_mb = min(r["disk_mb"] for r in results.values())
|
|
368
|
+
best_tput = max(r["throughput_mbs"] for r in results.values())
|
|
369
|
+
|
|
370
|
+
table = Table(
|
|
371
|
+
box=box.ROUNDED,
|
|
372
|
+
show_header=True,
|
|
373
|
+
header_style="bold cyan",
|
|
374
|
+
title="[bold]Random Patch Read Benchmark[/bold]",
|
|
375
|
+
title_style="bold white",
|
|
376
|
+
caption=(
|
|
377
|
+
f"Ratio = {source_disk_mb:.1f} MB uncompressed / format disk size • "
|
|
378
|
+
"Times include open + decompress + close per read • single process"
|
|
379
|
+
),
|
|
380
|
+
caption_style="dim",
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
table.add_column("Format", style="bold", justify="left", no_wrap=True)
|
|
384
|
+
table.add_column("Disk (MB)", justify="right")
|
|
385
|
+
table.add_column("Ratio", justify="right")
|
|
386
|
+
table.add_column("Mean (ms)", justify="right")
|
|
387
|
+
table.add_column("Median (ms)", justify="right")
|
|
388
|
+
table.add_column("Std (ms)", justify="right")
|
|
389
|
+
table.add_column("P95 (ms)", justify="right")
|
|
390
|
+
table.add_column("Max (ms)", justify="right")
|
|
391
|
+
table.add_column("MB/s", style="bold", justify="right")
|
|
392
|
+
|
|
393
|
+
for fmt, r in results.items():
|
|
394
|
+
ratio = source_disk_mb / r["disk_mb"] if r["disk_mb"] > 0 else float("nan")
|
|
395
|
+
|
|
396
|
+
disk_str = f"[green]{r['disk_mb']:.1f}[/green]" if r["disk_mb"] == best_mb else f"{r['disk_mb']:.1f}"
|
|
397
|
+
mean_str = f"[green]{r['mean_ms']:.2f}[/green]" if r["mean_ms"] == best_mean else f"{r['mean_ms']:.2f}"
|
|
398
|
+
tput_str = f"[green]{r['throughput_mbs']:.1f}[/green]" if r["throughput_mbs"] == best_tput else f"{r['throughput_mbs']:.1f}"
|
|
399
|
+
|
|
400
|
+
table.add_row(
|
|
401
|
+
fmt,
|
|
402
|
+
disk_str,
|
|
403
|
+
f"{ratio:.2f}×",
|
|
404
|
+
mean_str,
|
|
405
|
+
f"{r['median_ms']:.2f}",
|
|
406
|
+
f"{r['std_ms']:.2f}",
|
|
407
|
+
f"{r['p95_ms']:.2f}",
|
|
408
|
+
f"{r['max_ms']:.2f}",
|
|
409
|
+
tput_str,
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
console = Console()
|
|
413
|
+
console.print()
|
|
414
|
+
console.print(table)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
# ---------------------------------------------------------------------------
|
|
418
|
+
# Main
|
|
419
|
+
# ---------------------------------------------------------------------------
|
|
420
|
+
|
|
421
|
+
def parse_args() -> argparse.Namespace:
|
|
422
|
+
parser = argparse.ArgumentParser(
|
|
423
|
+
description="Benchmark random patch read throughput across array formats.",
|
|
424
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
425
|
+
epilog=__doc__,
|
|
426
|
+
)
|
|
427
|
+
parser.add_argument(
|
|
428
|
+
"dataset_dir",
|
|
429
|
+
type=Path,
|
|
430
|
+
help="Dataset root directory. Must contain a 'source' subdirectory with NIfTI/NRRD files. Converted formats are written as siblings of 'source'.",
|
|
431
|
+
)
|
|
432
|
+
parser.add_argument("--n_reads", type=int, default=1000, help="Number of random patch reads per format. Default: 1000.")
|
|
433
|
+
parser.add_argument(
|
|
434
|
+
"--patch_size",
|
|
435
|
+
type=int,
|
|
436
|
+
nargs="+",
|
|
437
|
+
default=[64, 64, 8],
|
|
438
|
+
help="Patch size per axis (must match array ndim). Default: 64 64 8.",
|
|
439
|
+
)
|
|
440
|
+
parser.add_argument(
|
|
441
|
+
"--formats",
|
|
442
|
+
nargs="+",
|
|
443
|
+
choices=ALL_FORMATS,
|
|
444
|
+
default=DEFAULT_FORMATS,
|
|
445
|
+
help=f"Formats to benchmark. Default: {', '.join(DEFAULT_FORMATS)} (npz excluded by default due to slow full-decompress reads).",
|
|
446
|
+
)
|
|
447
|
+
parser.add_argument("--max_images", type=int, default=None, help="Maximum number of images to use. The full list is shuffled then sliced. Default: all images.")
|
|
448
|
+
parser.add_argument("--skip_convert", action="store_true", help="Skip Stage 1 (conversion).")
|
|
449
|
+
parser.add_argument("--num_workers", type=int, default=12, help="Parallel workers for Stage 1 conversion. Default: all CPU cores.")
|
|
450
|
+
parser.add_argument("--seed", type=int, default=42, help="Random seed. Default: 42.")
|
|
451
|
+
return parser.parse_args()
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def main() -> None:
|
|
455
|
+
args = parse_args()
|
|
456
|
+
dataset_dir = args.dataset_dir.resolve()
|
|
457
|
+
source_dir = dataset_dir / "source"
|
|
458
|
+
if not source_dir.is_dir():
|
|
459
|
+
raise RuntimeError(f"Expected a 'source' subdirectory in {dataset_dir}")
|
|
460
|
+
output_dir = dataset_dir
|
|
461
|
+
patch_size = tuple(args.patch_size)
|
|
462
|
+
rng = random.Random(args.seed)
|
|
463
|
+
|
|
464
|
+
print(f"Dataset dir: {dataset_dir}")
|
|
465
|
+
print(f"Source dir : {source_dir}")
|
|
466
|
+
print(f"Formats : {args.formats}")
|
|
467
|
+
print(f"Patch size : {patch_size}")
|
|
468
|
+
print(f"N reads : {args.n_reads}")
|
|
469
|
+
print(f"Max images : {args.max_images if args.max_images is not None else 'all'}")
|
|
470
|
+
print(f"Num workers: {args.num_workers}")
|
|
471
|
+
print(f"Seed : {args.seed}")
|
|
472
|
+
|
|
473
|
+
all_source_files = discover_source_files(source_dir)
|
|
474
|
+
if not all_source_files:
|
|
475
|
+
raise RuntimeError(f"No NIfTI/NRRD files found under {source_dir}")
|
|
476
|
+
print(f"\nFound {len(all_source_files)} source file(s).")
|
|
477
|
+
|
|
478
|
+
rng.shuffle(all_source_files)
|
|
479
|
+
source_files = all_source_files[: args.max_images] if args.max_images is not None else all_source_files
|
|
480
|
+
if len(source_files) < len(all_source_files):
|
|
481
|
+
print(f"Using {len(source_files)} image(s) (--max_images={args.max_images}).")
|
|
482
|
+
|
|
483
|
+
source_disk_mb = sum(_nbytes_from_header(f) for f in source_files) / 1e6
|
|
484
|
+
print(f"Total uncompressed data: {source_disk_mb:.1f} MB")
|
|
485
|
+
|
|
486
|
+
# ------------------------------------------------------------------
|
|
487
|
+
# Stage 1: Conversion
|
|
488
|
+
# ------------------------------------------------------------------
|
|
489
|
+
if not args.skip_convert:
|
|
490
|
+
print("\n=== Stage 1: Converting ===")
|
|
491
|
+
run_conversion(source_files, source_dir, output_dir, args.formats, args.num_workers)
|
|
492
|
+
print("Conversion complete.")
|
|
493
|
+
else:
|
|
494
|
+
print("\nStage 1 skipped (--skip_convert).")
|
|
495
|
+
|
|
496
|
+
# ------------------------------------------------------------------
|
|
497
|
+
# Stage 2: Benchmark
|
|
498
|
+
# ------------------------------------------------------------------
|
|
499
|
+
print("\n=== Stage 2: Benchmarking ===")
|
|
500
|
+
print(
|
|
501
|
+
"Note: results reflect OS page cache state. Files will warm in cache during the run."
|
|
502
|
+
)
|
|
503
|
+
results = run_benchmark(output_dir, args.formats, args.n_reads, patch_size, rng)
|
|
504
|
+
|
|
505
|
+
# ------------------------------------------------------------------
|
|
506
|
+
# Results
|
|
507
|
+
# ------------------------------------------------------------------
|
|
508
|
+
print_results_table(results, source_disk_mb)
|
|
509
|
+
|
|
510
|
+
results_path = output_dir / "results.json"
|
|
511
|
+
payload = {
|
|
512
|
+
"dataset_dir": str(dataset_dir),
|
|
513
|
+
"source_dir": str(source_dir),
|
|
514
|
+
"n_images": len(source_files),
|
|
515
|
+
"n_reads": args.n_reads,
|
|
516
|
+
"patch_size": list(patch_size),
|
|
517
|
+
"seed": args.seed,
|
|
518
|
+
"source_disk_mb": round(source_disk_mb, 2),
|
|
519
|
+
"formats": results,
|
|
520
|
+
}
|
|
521
|
+
results_path.parent.mkdir(parents=True, exist_ok=True)
|
|
522
|
+
results_path.write_text(json.dumps(payload, indent=2))
|
|
523
|
+
print(f"\nResults saved to {results_path}")
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
if __name__ == "__main__":
|
|
527
|
+
main()
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# CLI
|
|
2
|
+
|
|
3
|
+
MLArray includes a small command-line interface for common tasks such as **inspecting file headers** and **converting between MLArray and existing image formats**. This is especially useful when you want to quickly verify metadata, debug a dataset, or batch-convert files without writing Python code.
|
|
4
|
+
|
|
5
|
+
The CLI currently focuses on core workflows (header inspection and conversion). Support for converting a wider range of image formats will be added over time.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## `mlarray_header`
|
|
10
|
+
|
|
11
|
+
Print the metadata header from a `.mla` file.
|
|
12
|
+
|
|
13
|
+
This command is useful for quickly checking spatial metadata, stored schemas, and other file-level information without loading the full array into memory.
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
mlarray_header sample.mla
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## `mlarray_convert`
|
|
22
|
+
|
|
23
|
+
Convert between MLArray and NIfTI/NRRD files.
|
|
24
|
+
|
|
25
|
+
This provides an easy way to move medical imaging data into or out of an MLArray-based workflow.
|
|
26
|
+
|
|
27
|
+
When converting from NIfTI/NRRD to MLArray, the source header is copied into `meta.source`.
|
|
28
|
+
|
|
29
|
+
When converting from NIfTI/NRRD to MLArray, `meta.spatial.coord_system` is
|
|
30
|
+
propagated conservatively:
|
|
31
|
+
|
|
32
|
+
- For NRRD input, the converter looks for explicit NRRD space metadata in the
|
|
33
|
+
header (`space`, falling back to `NRRD_space` from the SimpleITK path). It
|
|
34
|
+
maps `right-anterior-superior` to `RAS`, `left-posterior-superior` to `LPS`,
|
|
35
|
+
and preserves other explicit NRRD space strings verbatim. If no explicit
|
|
36
|
+
space metadata is available, `coord_system` is left unset.
|
|
37
|
+
- For NIfTI input, the converter sets `coord_system` to `LPS`. This is based on
|
|
38
|
+
the current `MedVol -> SimpleITK/ITK` import path: the imported geometry is
|
|
39
|
+
represented in the ITK physical-space convention, and MedVol only reindexes
|
|
40
|
+
axes to match NumPy array layout. It is not a statement that raw NIfTI
|
|
41
|
+
qform/sform metadata is preserved verbatim.
|
|
42
|
+
|
|
43
|
+
When converting from MLArray to NIfTI/NRRD, only `meta.source` is copied into
|
|
44
|
+
the output header. Spatial metadata (`spacing`, `origin`, and `direction`) is
|
|
45
|
+
written explicitly from `meta.spatial`.
|
|
46
|
+
|
|
47
|
+
- For NRRD output, `coord_system="RAS"` and `coord_system="LPS"` are written as
|
|
48
|
+
the corresponding NRRD `space` strings. A small set of already-valid NRRD
|
|
49
|
+
space strings is also preserved. Unsupported arbitrary `coord_system` strings
|
|
50
|
+
are not written as NRRD `space` metadata.
|
|
51
|
+
- For NIfTI output, geometry is preserved through the existing affine-related
|
|
52
|
+
path, but `coord_system` is not exported as an explicit NIfTI metadata field
|
|
53
|
+
because the current conversion path does not expose a reliable direct
|
|
54
|
+
representation for it.
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
mlarray_convert sample.nii.gz output.mla
|
|
58
|
+
mlarray_convert sample.mla output.nrrd
|
|
59
|
+
```
|
|
@@ -67,6 +67,7 @@ This section stores the information needed to interpret the array in physical sp
|
|
|
67
67
|
| direction | Optional[List[List[float]]] | Direction matrix, shape `[ndims][ndims]`. |
|
|
68
68
|
| shape | Optional[List[int]] | Full array shape, length = spatial + non-spatial axes. |
|
|
69
69
|
| axis_labels | Optional[List[str,AxisLabel]] | Per-axis labels or roles, length = full array `ndims`. |
|
|
70
|
+
| coord_system | Optional[Union[str,CoordSystem]] | World/anatomical coordinate convention that `spacing`, `origin`, `direction`, or `affine` map into. |
|
|
70
71
|
| axis_units | Optional[List[str]] | Per-axis units, length = full array `ndims`. |
|
|
71
72
|
|
|
72
73
|
---
|
|
@@ -87,6 +88,35 @@ Axis labels describe the semantic role of each axis. They may be provided as str
|
|
|
87
88
|
| continuous | Continuous-valued axis (non-spatial). |
|
|
88
89
|
| components | Component axis (e.g., vector components). |
|
|
89
90
|
|
|
91
|
+
`axis_labels` defines which array axes correspond to spatial axes such as
|
|
92
|
+
`spatial_x`, `spatial_y`, and `spatial_z`. It does not define what world-space
|
|
93
|
+
X/Y/Z mean.
|
|
94
|
+
|
|
95
|
+
---
|
|
96
|
+
|
|
97
|
+
#### CoordSystem
|
|
98
|
+
|
|
99
|
+
`coord_system` describes the world/anatomical coordinate convention that the
|
|
100
|
+
spatial metadata maps into. It applies to both geometry styles:
|
|
101
|
+
`spacing`/`origin`/`direction` and `affine`.
|
|
102
|
+
|
|
103
|
+
Built-in standard values are:
|
|
104
|
+
|
|
105
|
+
| value | description |
|
|
106
|
+
| ------- | ----------- |
|
|
107
|
+
| RAS | Right-Anterior-Superior world convention. |
|
|
108
|
+
| LPS | Left-Posterior-Superior world convention. |
|
|
109
|
+
| unknown | Coordinate system is unknown. |
|
|
110
|
+
| other | Coordinate system exists but falls outside the built-in vocabulary. |
|
|
111
|
+
|
|
112
|
+
Arbitrary strings are also allowed and are preserved exactly as provided. This
|
|
113
|
+
makes `coord_system` an extensible standard vocabulary rather than a closed
|
|
114
|
+
enum.
|
|
115
|
+
|
|
116
|
+
`coord_system` does not determine array axis order. `axis_labels` tells you
|
|
117
|
+
which array axes are spatial axes; `coord_system` tells you what world
|
|
118
|
+
X/Y/Z mean in those spatial coordinates.
|
|
119
|
+
|
|
90
120
|
---
|
|
91
121
|
|
|
92
122
|
### stats
|