mlarray 0.0.32__tar.gz → 0.0.39__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mlarray-0.0.32 → mlarray-0.0.39}/PKG-INFO +7 -7
- {mlarray-0.0.32 → mlarray-0.0.39}/README.md +6 -6
- {mlarray-0.0.32 → mlarray-0.0.39}/docs/api.md +3 -1
- {mlarray-0.0.32 → mlarray-0.0.39}/docs/optimization.md +2 -4
- {mlarray-0.0.32 → mlarray-0.0.39}/docs/schema.md +36 -15
- {mlarray-0.0.32 → mlarray-0.0.39}/docs/usage.md +44 -7
- {mlarray-0.0.32 → mlarray-0.0.39}/docs/why.md +2 -2
- {mlarray-0.0.32 → mlarray-0.0.39}/examples/example_channel.py +4 -4
- {mlarray-0.0.32 → mlarray-0.0.39}/examples/example_metadata_only.py +3 -3
- mlarray-0.0.39/examples/example_non_spatial.py +42 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/examples/example_open.py +3 -3
- {mlarray-0.0.32 → mlarray-0.0.39}/examples/example_save_load.py +3 -4
- {mlarray-0.0.32 → mlarray-0.0.39}/mlarray/__init__.py +6 -3
- {mlarray-0.0.32 → mlarray-0.0.39}/mlarray/meta.py +226 -64
- {mlarray-0.0.32 → mlarray-0.0.39}/mlarray/mlarray.py +58 -52
- {mlarray-0.0.32 → mlarray-0.0.39}/mlarray.egg-info/PKG-INFO +7 -7
- {mlarray-0.0.32 → mlarray-0.0.39}/mlarray.egg-info/SOURCES.txt +2 -0
- mlarray-0.0.39/tests/test_bboxes.py +67 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/tests/test_metadata.py +9 -9
- {mlarray-0.0.32 → mlarray-0.0.39}/tests/test_usage.py +6 -6
- {mlarray-0.0.32 → mlarray-0.0.39}/.github/workflows/workflow.yml +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/.gitignore +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/LICENSE +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/MANIFEST.in +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/assets/banner.png +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/assets/banner.png~ +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/docs/cli.md +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/docs/index.md +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/mkdocs.yml +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/mlarray/cli.py +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/mlarray/utils.py +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/mlarray.egg-info/dependency_links.txt +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/mlarray.egg-info/entry_points.txt +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/mlarray.egg-info/requires.txt +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/mlarray.egg-info/top_level.txt +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/pyproject.toml +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/setup.cfg +0 -0
- {mlarray-0.0.32 → mlarray-0.0.39}/tests/test_optimization.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mlarray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.39
|
|
4
4
|
Summary: Array format specialized for Machine Learning with Blosc2 backend and standardized metadata.
|
|
5
5
|
Author-email: Karol Gotkowski <karol.gotkowski@dkfz.de>
|
|
6
6
|
License: MIT
|
|
@@ -118,15 +118,15 @@ image = MLArray(
|
|
|
118
118
|
|
|
119
119
|
print(image.spacing) # [1.0, 1.0, 1.5]
|
|
120
120
|
print(image.origin) # [10.0, 10.0, 30.0]
|
|
121
|
-
print(image.meta.
|
|
121
|
+
print(image.meta.source) # {"patient_id": "123", "modality": "CT"}
|
|
122
122
|
|
|
123
123
|
image.spacing[1] = 5.3
|
|
124
|
-
image.meta.
|
|
124
|
+
image.meta.source["study_id"] = "study-001"
|
|
125
125
|
image.save("with-metadata.mla")
|
|
126
126
|
|
|
127
127
|
# Open memory-mapped
|
|
128
128
|
image = MLArray.open("with-metadata.mla", mmap='r+')
|
|
129
|
-
image.meta.
|
|
129
|
+
image.meta.source["study_id"] = "new-study" # Modify metadata
|
|
130
130
|
image.close() # Close and save metadata, only necessary to save modified metadata
|
|
131
131
|
```
|
|
132
132
|
|
|
@@ -157,13 +157,13 @@ from mlarray import MLArray, Meta
|
|
|
157
157
|
array = np.random.random((64, 128, 128))
|
|
158
158
|
image = MLArray(
|
|
159
159
|
array,
|
|
160
|
-
meta=Meta(
|
|
160
|
+
meta=Meta(source={"patient_id": "123", "modality": "CT"}, is_seg=True), # Add metadata in a pre-defined format
|
|
161
161
|
)
|
|
162
162
|
|
|
163
|
-
print(image.meta.
|
|
163
|
+
print(image.meta.source) # {"patient_id": "123", "modality": "CT"}
|
|
164
164
|
print(image.meta.is_seg) # True
|
|
165
165
|
|
|
166
|
-
image.meta.
|
|
166
|
+
image.meta.source["study_id"] = "study-001"
|
|
167
167
|
image.meta.is_seg = False
|
|
168
168
|
image.save("with-metadata.mla")
|
|
169
169
|
```
|
|
@@ -84,15 +84,15 @@ image = MLArray(
|
|
|
84
84
|
|
|
85
85
|
print(image.spacing) # [1.0, 1.0, 1.5]
|
|
86
86
|
print(image.origin) # [10.0, 10.0, 30.0]
|
|
87
|
-
print(image.meta.
|
|
87
|
+
print(image.meta.source) # {"patient_id": "123", "modality": "CT"}
|
|
88
88
|
|
|
89
89
|
image.spacing[1] = 5.3
|
|
90
|
-
image.meta.
|
|
90
|
+
image.meta.source["study_id"] = "study-001"
|
|
91
91
|
image.save("with-metadata.mla")
|
|
92
92
|
|
|
93
93
|
# Open memory-mapped
|
|
94
94
|
image = MLArray.open("with-metadata.mla", mmap='r+')
|
|
95
|
-
image.meta.
|
|
95
|
+
image.meta.source["study_id"] = "new-study" # Modify metadata
|
|
96
96
|
image.close() # Close and save metadata, only necessary to save modified metadata
|
|
97
97
|
```
|
|
98
98
|
|
|
@@ -123,13 +123,13 @@ from mlarray import MLArray, Meta
|
|
|
123
123
|
array = np.random.random((64, 128, 128))
|
|
124
124
|
image = MLArray(
|
|
125
125
|
array,
|
|
126
|
-
meta=Meta(
|
|
126
|
+
meta=Meta(source={"patient_id": "123", "modality": "CT"}, is_seg=True), # Add metadata in a pre-defined format
|
|
127
127
|
)
|
|
128
128
|
|
|
129
|
-
print(image.meta.
|
|
129
|
+
print(image.meta.source) # {"patient_id": "123", "modality": "CT"}
|
|
130
130
|
print(image.meta.is_seg) # True
|
|
131
131
|
|
|
132
|
-
image.meta.
|
|
132
|
+
image.meta.source["study_id"] = "study-001"
|
|
133
133
|
image.meta.is_seg = False
|
|
134
134
|
image.save("with-metadata.mla")
|
|
135
135
|
```
|
|
@@ -12,12 +12,14 @@
|
|
|
12
12
|
|
|
13
13
|
::: mlarray.meta.Meta
|
|
14
14
|
|
|
15
|
-
::: mlarray.meta.
|
|
15
|
+
::: mlarray.meta.MetaSource
|
|
16
16
|
|
|
17
17
|
::: mlarray.meta.MetaExtra
|
|
18
18
|
|
|
19
19
|
::: mlarray.meta.MetaSpatial
|
|
20
20
|
|
|
21
|
+
::: mlarray.meta.AxisLabelEnum
|
|
22
|
+
|
|
21
23
|
::: mlarray.meta.MetaStatistics
|
|
22
24
|
|
|
23
25
|
::: mlarray.meta.MetaBbox
|
|
@@ -25,7 +25,7 @@ Instead of requiring users to be storage experts, MLArray introduces a **patch s
|
|
|
25
25
|
* element size (bytes per pixel),
|
|
26
26
|
* CPU cache sizes (L1 / L3 per core),
|
|
27
27
|
* your patch size (2D or 3D),
|
|
28
|
-
*
|
|
28
|
+
* non-spatial axes layout (via `axis_labels`),
|
|
29
29
|
* and then chooses block/chunk sizes that aim to keep decompression and reads cache-friendly.
|
|
30
30
|
|
|
31
31
|
Practically: this means *reading a training patch should tend to require as few chunk/block touches as possible*, while keeping the decompressed working set aligned with CPU caches.
|
|
@@ -201,8 +201,6 @@ When to use:
|
|
|
201
201
|
|
|
202
202
|
## Notes and practical tips
|
|
203
203
|
|
|
204
|
-
* **Patch optimization is currently implemented for 2D and 3D images** (
|
|
204
|
+
* **Patch optimization is currently implemented for 2D and 3D images** (with at most one further non-spatial axis). If your data falls outside that, you can still set `chunk_size`/`block_size` manually or let Blosc2 decide.
|
|
205
205
|
* The best patch size to use is usually the **patch size your dataloader requests most often** (training patch, not necessarily inference tile size).
|
|
206
206
|
* If you’re unsure: start with the default (`patch_size='default'`) and only tune if profiling shows I/O bottlenecks.
|
|
207
|
-
|
|
208
|
-
If you want, I can also help you add a short “How to pick patch_size” subsection tailored to typical pipelines (nnU-Net, 2D slice training, multi-channel inputs).
|
|
@@ -13,7 +13,7 @@ All fields in the schema are **JSON-serializable** unless otherwise noted. Field
|
|
|
13
13
|
|
|
14
14
|
### Representation notes
|
|
15
15
|
|
|
16
|
-
Many namespaces are implemented as **single-key dataclasses** (subclasses of `SingleKeyBaseMeta`). In Python, these behave like their wrapped value (e.g., `meta.is_seg` is a `bool`, `meta.
|
|
16
|
+
Many namespaces are implemented as **single-key dataclasses** (subclasses of `SingleKeyBaseMeta`). In Python, these behave like their wrapped value (e.g., `meta.is_seg` is a `bool`, `meta.source` is a `dict`). When serialized via `Meta.to_mapping()`, they appear as a one-field object keyed by their internal field name (e.g., `is_seg: {"is_seg": true}` or `source: {"data": {...}}`). `Meta.from_mapping()` accepts either the one-field object or the raw value and will coerce it to the correct class.
|
|
17
17
|
|
|
18
18
|
---
|
|
19
19
|
|
|
@@ -23,16 +23,16 @@ Top-level metadata container.
|
|
|
23
23
|
|
|
24
24
|
### Overview
|
|
25
25
|
|
|
26
|
-
`Meta` is the root object that groups all metadata into well-defined namespaces. Some namespaces are **standardized** (e.g., `spatial`, `stats`), while others are intentionally **free-form** (`
|
|
26
|
+
`Meta` is the root object that groups all metadata into well-defined namespaces. Some namespaces are **standardized** (e.g., `spatial`, `stats`), while others are intentionally **free-form** (`source`, `extra`) to support arbitrary metadata and long-term extensibility. Several entries are single-key dataclasses that wrap a primitive value while still allowing schema-aware validation.
|
|
27
27
|
|
|
28
28
|
---
|
|
29
29
|
|
|
30
|
-
###
|
|
30
|
+
### source
|
|
31
31
|
|
|
32
32
|
* **Description:** Arbitrary JSON-serializable dictionary for metadata from the original image source.
|
|
33
33
|
Stores information from image sources such as DICOM, NIfTI, NRRD,
|
|
34
34
|
or other imaging formats.
|
|
35
|
-
* **Dataclass:** `
|
|
35
|
+
* **Dataclass:** `MetaSource` (single-key wrapper).
|
|
36
36
|
|
|
37
37
|
| field | type | description |
|
|
38
38
|
| ----- | --------------------- | -------------------------------------------- |
|
|
@@ -58,15 +58,34 @@ Top-level metadata container.
|
|
|
58
58
|
* **Description:** Spatial metadata for the image.
|
|
59
59
|
* **Dataclass:** `MetaSpatial`.
|
|
60
60
|
|
|
61
|
-
This section stores the information needed to interpret the array in physical space (e.g., voxel spacing, coordinate origin, and orientation). It also optionally captures array shape and
|
|
61
|
+
This section stores the information needed to interpret the array in physical space (e.g., voxel spacing, coordinate origin, and orientation). It also optionally captures array shape and axes layout to make downstream consumers more robust.
|
|
62
62
|
|
|
63
63
|
| field | type | description |
|
|
64
64
|
| ------------ | --------------------------- | ---------------------------------------------------------------------------------------- |
|
|
65
65
|
| spacing | Optional[List[float]] | Voxel spacing per spatial axis, length = `ndims`. |
|
|
66
66
|
| origin | Optional[List[float]] | Origin per spatial axis, length = `ndims`. |
|
|
67
67
|
| direction | Optional[List[List[float]]] | Direction matrix, shape `[ndims][ndims]`. |
|
|
68
|
-
| shape | Optional[List[int]] |
|
|
69
|
-
|
|
|
68
|
+
| shape | Optional[List[int]] | Full array shape, length = spatial + non-spatial axes. |
|
|
69
|
+
| axis_labels | Optional[List[str,AxisLabel]] | Per-axis labels or roles, length = full array `ndims`. |
|
|
70
|
+
| axis_units | Optional[List[str]] | Per-axis units, length = full array `ndims`. |
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
#### AxisLabel
|
|
75
|
+
|
|
76
|
+
Axis labels describe the semantic role of each axis. They may be provided as strings or enum values.
|
|
77
|
+
|
|
78
|
+
| value | description |
|
|
79
|
+
| ------------- | -------------------------------------------------------- |
|
|
80
|
+
| spatial | Generic spatial axis (used when no axis-specific label). |
|
|
81
|
+
| spatial_x | Spatial axis representing X. |
|
|
82
|
+
| spatial_y | Spatial axis representing Y. |
|
|
83
|
+
| spatial_z | Spatial axis representing Z. |
|
|
84
|
+
| non_spatial | Generic non-spatial axis. |
|
|
85
|
+
| channel | Channel axis (e.g., color channels or feature maps). |
|
|
86
|
+
| temporal | Time axis. |
|
|
87
|
+
| continuous | Continuous-valued axis (non-spatial). |
|
|
88
|
+
| components | Component axis (e.g., vector components). |
|
|
70
89
|
|
|
71
90
|
---
|
|
72
91
|
|
|
@@ -97,15 +116,17 @@ This section stores precomputed global statistics for the array, which can be us
|
|
|
97
116
|
### bbox
|
|
98
117
|
|
|
99
118
|
* **Description:** Bounding boxes for objects/regions in the image.
|
|
100
|
-
* **Dataclass:** `MetaBbox
|
|
119
|
+
* **Dataclass:** `MetaBbox`.
|
|
101
120
|
* **Structure:** List of bboxes, each bbox is a list with length equal to image `ndims`,
|
|
102
121
|
and each entry is `[min, max]`.
|
|
103
122
|
|
|
104
123
|
Bounding boxes are stored in a normalized, axis-aligned representation that works across dimensionalities (2D, 3D, …). This is especially useful for detection-style workflows, ROI cropping, dataset summaries, and interactive visualization.
|
|
105
124
|
|
|
106
|
-
| field | type
|
|
107
|
-
| ------ |
|
|
108
|
-
| bboxes | Optional[List[List[List[int]]]]
|
|
125
|
+
| field | type | description |
|
|
126
|
+
| ------ | ----------------------------------------------- | --------------------------------------------------------------------------- |
|
|
127
|
+
| bboxes | Optional[List[List[List[Union[int, float]]]]] | Bounding boxes shaped `[num_bboxes][ndims][2]` (min/max), ints or floats. |
|
|
128
|
+
| scores | Optional[List[Union[int, float]]] | Optional confidence scores aligned with `bboxes`. |
|
|
129
|
+
| labels | Optional[List[Union[str, int, float]]] | Optional labels aligned with `bboxes`. |
|
|
109
130
|
|
|
110
131
|
---
|
|
111
132
|
|
|
@@ -129,9 +150,9 @@ This section records how the array was laid out on disk (chunking, blocking, pat
|
|
|
129
150
|
|
|
130
151
|
| field | type | description |
|
|
131
152
|
| ---------- | --------------------- | ---------------------------------------------------------------------- |
|
|
132
|
-
| chunk_size | Optional[List[float]] | Chunk size per axis, length = full array `ndims` (including
|
|
133
|
-
| block_size | Optional[List[float]] | Block size per axis, length = full array `ndims` (including
|
|
134
|
-
| patch_size | Optional[List[float]] | Patch size per spatial axis, length = `ndims` (
|
|
153
|
+
| chunk_size | Optional[List[float]] | Chunk size per axis, length = full array `ndims` (including non-spatial axes). |
|
|
154
|
+
| block_size | Optional[List[float]] | Block size per axis, length = full array `ndims` (including non-spatial axes). |
|
|
155
|
+
| patch_size | Optional[List[float]] | Patch size per spatial axis, length = `ndims` (non-spatial axes excluded). |
|
|
135
156
|
|
|
136
157
|
---
|
|
137
158
|
|
|
@@ -154,7 +175,7 @@ This section records how the array was laid out on disk (chunking, blocking, pat
|
|
|
154
175
|
|
|
155
176
|
| field | type | description |
|
|
156
177
|
| ----------------- | -------------- | --------------------------------------------- |
|
|
157
|
-
| image_meta_format | Optional[str] | Identifier for the
|
|
178
|
+
| image_meta_format | Optional[str] | Identifier for the source metadata format. |
|
|
158
179
|
|
|
159
180
|
---
|
|
160
181
|
|
|
@@ -51,7 +51,7 @@ image[...] = array # Modify image in memory and disk
|
|
|
51
51
|
|
|
52
52
|
## Metadata inspection and manipulation
|
|
53
53
|
|
|
54
|
-
MLArray provides first-class support for common image metadata (spacing, origin, direction), and also lets you attach arbitrary metadata from the
|
|
54
|
+
MLArray provides first-class support for common image metadata (spacing, origin, direction), and also lets you attach arbitrary metadata from the source image source via `meta=...` (e.g., raw DICOM fields, acquisition parameters, dataset identifiers).
|
|
55
55
|
|
|
56
56
|
```python
|
|
57
57
|
import numpy as np
|
|
@@ -68,15 +68,15 @@ image = MLArray(
|
|
|
68
68
|
|
|
69
69
|
print(image.spacing) # [1.0, 1.0, 1.5]
|
|
70
70
|
print(image.origin) # [10.0, 10.0, 30.0]
|
|
71
|
-
print(image.meta.
|
|
71
|
+
print(image.meta.source) # {"patient_id": "123", "modality": "CT"}
|
|
72
72
|
|
|
73
73
|
image.spacing[1] = 5.3
|
|
74
|
-
image.meta.
|
|
74
|
+
image.meta.source["study_id"] = "study-001"
|
|
75
75
|
image.save("with-metadata.mla")
|
|
76
76
|
|
|
77
77
|
# Open memory-mapped
|
|
78
78
|
image = MLArray.open("with-metadata.mla", mmap='r+')
|
|
79
|
-
image.meta.
|
|
79
|
+
image.meta.source["study_id"] = "new-study" # Modify metadata
|
|
80
80
|
image.close() # Close and save metadata, only necessary to save modified metadata
|
|
81
81
|
```
|
|
82
82
|
|
|
@@ -115,19 +115,56 @@ from mlarray import MLArray, Meta
|
|
|
115
115
|
array = np.random.random((64, 128, 128))
|
|
116
116
|
image = MLArray(
|
|
117
117
|
array,
|
|
118
|
-
meta=Meta(
|
|
118
|
+
meta=Meta(source={"patient_id": "123", "modality": "CT"}, is_seg=True), # Add metadata in a pre-defined format
|
|
119
119
|
)
|
|
120
120
|
|
|
121
|
-
print(image.meta.
|
|
121
|
+
print(image.meta.source) # {"patient_id": "123", "modality": "CT"}
|
|
122
122
|
print(image.meta.is_seg) # True
|
|
123
123
|
|
|
124
|
-
image.meta.
|
|
124
|
+
image.meta.source["study_id"] = "study-001"
|
|
125
125
|
image.meta.is_seg = False
|
|
126
126
|
image.save("with-metadata.mla")
|
|
127
127
|
```
|
|
128
128
|
|
|
129
129
|
---
|
|
130
130
|
|
|
131
|
+
## Non-spatial data usage
|
|
132
|
+
|
|
133
|
+
Use `axis_labels` to mark which axes are spatial and which are non-spatial
|
|
134
|
+
(channels, temporal, components, etc.). Spatial metadata (`spacing`, `origin`,
|
|
135
|
+
`direction`) is specified only for the spatial axes, while the full array shape
|
|
136
|
+
includes both spatial and non-spatial axes.
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
import numpy as np
|
|
140
|
+
from mlarray import MLArray, MetaSpatial
|
|
141
|
+
|
|
142
|
+
# Example shape: (time, z, y, x, channels)
|
|
143
|
+
array = np.random.random((2, 6, 4, 4, 3, 2))
|
|
144
|
+
|
|
145
|
+
axis_labels = [
|
|
146
|
+
MetaSpatial.AxisLabel.temporal,
|
|
147
|
+
MetaSpatial.AxisLabel.spatial_z,
|
|
148
|
+
MetaSpatial.AxisLabel.spatial_y,
|
|
149
|
+
"spatial_x", # Possible to pass predefined labels as strings as well
|
|
150
|
+
MetaSpatial.AxisLabel.channel,
|
|
151
|
+
"some-other-type" # Possible to pass arbitrary strings as well
|
|
152
|
+
]
|
|
153
|
+
|
|
154
|
+
image = MLArray(
|
|
155
|
+
array,
|
|
156
|
+
spacing=(2.5, 0.7, 0.7), # spatial axes only (z, y, x)
|
|
157
|
+
origin=(0.0, 0.0, 0.0),
|
|
158
|
+
axis_labels=axis_labels,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# Optional per-axis units (length = full array ndims)
|
|
162
|
+
image.meta.spatial.axis_units = ["s", "mm", "mm", "mm", ""]
|
|
163
|
+
image.save("time-series.mla", patch_size=None)
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
|
|
131
168
|
## Patch size variants
|
|
132
169
|
|
|
133
170
|
MLArray stores arrays in a chunked layout to enable efficient partial reads. You can control how data is chunked using `patch_size` (recommended in most cases), or manually specify chunk and block sizes when you need full control.
|
|
@@ -13,8 +13,8 @@ MLArray is designed to bridge that gap: a machine-learning-friendly array format
|
|
|
13
13
|
* **A standardized, extensible metadata schema**
|
|
14
14
|
MLArray defines a metadata schema that balances *standardization* and *flexibility*: software that supports MLArray has a consistent way to access relevant metadata, while users can still attach arbitrary custom metadata when needed.
|
|
15
15
|
|
|
16
|
-
* **Preserve
|
|
17
|
-
Users can convert images from arbitrary formats to MLArray while preserving the
|
|
16
|
+
* **Preserve source metadata across conversions**
|
|
17
|
+
Users can convert images from arbitrary formats to MLArray while preserving the source metadata *in a structured and reproducible way*. Tools that integrate MLArray can still access metadata according to the source format’s conventions, which makes MLArray a practical alternative for ML pipelines without breaking downstream analysis or visualization workflows.
|
|
18
18
|
|
|
19
19
|
* **Machine learning–specific metadata support**
|
|
20
20
|
In addition to format-preserving metadata, MLArray includes a dedicated schema for machine-learning-relevant information, and it also supports storing dynamic metadata outside predefined schemas.
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import os
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from mlarray import MLArray, Meta
|
|
4
|
+
from mlarray import MLArray, Meta, MetaBbox, MetaSpatial
|
|
5
5
|
import json
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
if __name__ == '__main__':
|
|
9
9
|
print("Creating array...")
|
|
10
10
|
array = np.random.random((32, 64, 64, 3))
|
|
11
|
-
|
|
11
|
+
axis_labels = (MetaSpatial.AxisLabel.spatial_z, MetaSpatial.AxisLabel.spatial_y, MetaSpatial.AxisLabel.spatial_x, MetaSpatial.AxisLabel.channel)
|
|
12
12
|
spacing = np.array((2, 2.5, 4))
|
|
13
13
|
origin = (1, 1, 1)
|
|
14
14
|
direction = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
|
|
15
|
-
|
|
15
|
+
source_meta = {"tmp1": "This is an image", "tmp2": 5, "tmp3": {"test1": 16.4587, "test2": [1, 2, 3, 4, 5, 6]}}
|
|
16
16
|
bboxes = [[[0, 1], [0, 1], [0, 1]]]
|
|
17
17
|
filepath = "tmp.mla"
|
|
18
18
|
|
|
@@ -20,7 +20,7 @@ if __name__ == '__main__':
|
|
|
20
20
|
os.remove(filepath)
|
|
21
21
|
|
|
22
22
|
print("Initializing image...")
|
|
23
|
-
image = MLArray(array, spacing=spacing, origin=origin, direction=direction,
|
|
23
|
+
image = MLArray(array, spacing=spacing, origin=origin, direction=direction, axis_labels=axis_labels, meta=Meta(source=source_meta, bbox=MetaBbox(bboxes)))
|
|
24
24
|
print("Saving image...")
|
|
25
25
|
image.save(filepath)
|
|
26
26
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import os
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from mlarray import MLArray, Meta
|
|
4
|
+
from mlarray import MLArray, Meta, MetaBbox
|
|
5
5
|
import json
|
|
6
6
|
|
|
7
7
|
|
|
@@ -11,7 +11,7 @@ if __name__ == '__main__':
|
|
|
11
11
|
spacing = np.array((2, 2.5, 4))
|
|
12
12
|
origin = (1, 1, 1)
|
|
13
13
|
direction = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
|
|
14
|
-
|
|
14
|
+
source_meta = {"tmp1": "This is an image", "tmp2": 5, "tmp3": {"test1": 16.4587, "test2": [1, 2, 3, 4, 5, 6]}}
|
|
15
15
|
bboxes = [[[0, 1], [0, 1], [0, 1]]]
|
|
16
16
|
filepath = "tmp.mla"
|
|
17
17
|
|
|
@@ -19,7 +19,7 @@ if __name__ == '__main__':
|
|
|
19
19
|
os.remove(filepath)
|
|
20
20
|
|
|
21
21
|
print("Initializing image...")
|
|
22
|
-
image = MLArray(spacing=spacing, origin=origin, direction=direction, meta=Meta(
|
|
22
|
+
image = MLArray(spacing=spacing, origin=origin, direction=direction, meta=Meta(source=source_meta, bbox=MetaBbox(bboxes)))
|
|
23
23
|
print("Saving image...")
|
|
24
24
|
image.save(filepath)
|
|
25
25
|
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from mlarray import MLArray, Meta, MetaBbox, MetaSpatial
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
if __name__ == '__main__':
|
|
9
|
+
print("Creating array...")
|
|
10
|
+
array = np.random.random((2, 6, 4, 4, 3, 2))
|
|
11
|
+
axis_labels = [
|
|
12
|
+
MetaSpatial.AxisLabel.temporal,
|
|
13
|
+
MetaSpatial.AxisLabel.spatial_z,
|
|
14
|
+
MetaSpatial.AxisLabel.spatial_y,
|
|
15
|
+
"spatial_x", # Possible to pass predefined labels as strings as well
|
|
16
|
+
MetaSpatial.AxisLabel.channel,
|
|
17
|
+
"some-other-type" # Possible to pass arbitrary strings as well
|
|
18
|
+
]
|
|
19
|
+
spacing = np.array((2, 2.5, 4))
|
|
20
|
+
origin = (1, 1, 1)
|
|
21
|
+
direction = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
|
|
22
|
+
source_meta = {"tmp1": "This is an image", "tmp2": 5, "tmp3": {"test1": 16.4587, "test2": [1, 2, 3, 4, 5, 6]}}
|
|
23
|
+
bboxes = [[[0, 1], [0, 1], [0, 1]]]
|
|
24
|
+
filepath = "tmp.mla"
|
|
25
|
+
|
|
26
|
+
if Path(filepath).is_file():
|
|
27
|
+
os.remove(filepath)
|
|
28
|
+
|
|
29
|
+
print("Initializing image...")
|
|
30
|
+
image = MLArray(array, spacing=spacing, origin=origin, direction=direction, axis_labels=axis_labels, meta=Meta(source=source_meta, bbox=MetaBbox(bboxes)))
|
|
31
|
+
image.meta.spatial.axis_units = ["s", "mm", "mm", "mm", ""]
|
|
32
|
+
print("Saving image...")
|
|
33
|
+
image.save(filepath, patch_size=None)
|
|
34
|
+
|
|
35
|
+
print("Loading image...")
|
|
36
|
+
image = MLArray(filepath)
|
|
37
|
+
print(json.dumps(image.meta.to_mapping(), indent=2, sort_keys=True))
|
|
38
|
+
print("Image mean value: ", np.mean(image.to_numpy()))
|
|
39
|
+
print("Some array data: \n", image[:2, :2, 0])
|
|
40
|
+
|
|
41
|
+
if Path(filepath).is_file():
|
|
42
|
+
os.remove(filepath)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import os
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from mlarray import MLArray, Meta, MetaSpatial
|
|
4
|
+
from mlarray import MLArray, Meta, MetaSpatial, MetaBbox
|
|
5
5
|
import json
|
|
6
6
|
|
|
7
7
|
|
|
@@ -11,7 +11,7 @@ if __name__ == '__main__':
|
|
|
11
11
|
spacing = np.array((2, 2.5, 4))
|
|
12
12
|
origin = (1, 1, 1)
|
|
13
13
|
direction = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
|
|
14
|
-
|
|
14
|
+
source_meta = {"tmp1": "This is an image", "tmp2": 5, "tmp3": {"test1": 16.4587, "test2": [1, 2, 3, 4, 5, 6]}}
|
|
15
15
|
bboxes = [[[0, 1], [0, 1], [0, 1]]]
|
|
16
16
|
filepath = "tmp.mla"
|
|
17
17
|
|
|
@@ -22,7 +22,7 @@ if __name__ == '__main__':
|
|
|
22
22
|
image = MLArray.open(filepath, shape=array.shape, dtype=array.dtype, mmap='w+')
|
|
23
23
|
print("Saving image...")
|
|
24
24
|
image[...] = array
|
|
25
|
-
image.meta.copy_from(Meta(
|
|
25
|
+
image.meta.copy_from(Meta(source=source_meta, spatial=MetaSpatial(spacing=spacing, origin=origin, direction=direction), bbox=MetaBbox(bboxes)))
|
|
26
26
|
image.meta.is_seg = True
|
|
27
27
|
image.close()
|
|
28
28
|
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import os
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from mlarray import MLArray, Meta
|
|
5
|
-
from mlarray.meta import MetaOriginal, MetaIsSeg
|
|
4
|
+
from mlarray import MLArray, Meta, MetaBbox
|
|
6
5
|
import json
|
|
7
6
|
|
|
8
7
|
|
|
@@ -12,7 +11,7 @@ if __name__ == '__main__':
|
|
|
12
11
|
spacing = np.array((2, 2.5, 4))
|
|
13
12
|
origin = (1, 1, 1)
|
|
14
13
|
direction = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
|
|
15
|
-
|
|
14
|
+
source_meta = {"tmp1": "This is an image", "tmp2": 5, "tmp3": {"test1": 16.4587, "test2": [1, 2, 3, 4, 5, 6]}}
|
|
16
15
|
bboxes = [[[0, 1], [0, 1], [0, 1]]]
|
|
17
16
|
filepath = "tmp.mla"
|
|
18
17
|
|
|
@@ -20,7 +19,7 @@ if __name__ == '__main__':
|
|
|
20
19
|
os.remove(filepath)
|
|
21
20
|
|
|
22
21
|
print("Initializing image...")
|
|
23
|
-
image = MLArray(array, spacing=spacing, origin=origin, direction=direction, meta=Meta(
|
|
22
|
+
image = MLArray(array, spacing=spacing, origin=origin, direction=direction, meta=Meta(source=source_meta, bbox=MetaBbox(bboxes), is_seg=True))
|
|
24
23
|
print("Saving image...")
|
|
25
24
|
image.save(filepath)
|
|
26
25
|
|
|
@@ -11,12 +11,13 @@ if TYPE_CHECKING:
|
|
|
11
11
|
MetaBlosc2,
|
|
12
12
|
MetaExtra,
|
|
13
13
|
MetaHasArray,
|
|
14
|
-
|
|
14
|
+
MetaSource,
|
|
15
15
|
MetaImageFormat,
|
|
16
16
|
MetaIsSeg,
|
|
17
17
|
MetaSpatial,
|
|
18
18
|
MetaStatistics,
|
|
19
19
|
MetaVersion,
|
|
20
|
+
AxisLabel
|
|
20
21
|
)
|
|
21
22
|
from mlarray.utils import is_serializable
|
|
22
23
|
from mlarray.cli import cli_print_header, cli_convert_to_mlarray
|
|
@@ -30,12 +31,13 @@ __all__ = [
|
|
|
30
31
|
"MetaBlosc2",
|
|
31
32
|
"MetaExtra",
|
|
32
33
|
"MetaHasArray",
|
|
33
|
-
"
|
|
34
|
+
"MetaSource",
|
|
34
35
|
"MetaImageFormat",
|
|
35
36
|
"MetaIsSeg",
|
|
36
37
|
"MetaSpatial",
|
|
37
38
|
"MetaStatistics",
|
|
38
39
|
"MetaVersion",
|
|
40
|
+
"AxisLabel",
|
|
39
41
|
"is_serializable",
|
|
40
42
|
"cli_print_header",
|
|
41
43
|
"cli_convert_to_mlarray",
|
|
@@ -55,12 +57,13 @@ _LAZY_ATTRS = {
|
|
|
55
57
|
"MetaBlosc2": ("mlarray.meta", "MetaBlosc2"),
|
|
56
58
|
"MetaExtra": ("mlarray.meta", "MetaExtra"),
|
|
57
59
|
"MetaHasArray": ("mlarray.meta", "MetaHasArray"),
|
|
58
|
-
"
|
|
60
|
+
"MetaSource": ("mlarray.meta", "MetaSource"),
|
|
59
61
|
"MetaImageFormat": ("mlarray.meta", "MetaImageFormat"),
|
|
60
62
|
"MetaIsSeg": ("mlarray.meta", "MetaIsSeg"),
|
|
61
63
|
"MetaSpatial": ("mlarray.meta", "MetaSpatial"),
|
|
62
64
|
"MetaStatistics": ("mlarray.meta", "MetaStatistics"),
|
|
63
65
|
"MetaVersion": ("mlarray.meta", "MetaVersion"),
|
|
66
|
+
"AxisLabel": ("mlarray.meta", "AxisLabel"),
|
|
64
67
|
"is_serializable": ("mlarray.utils", "is_serializable"),
|
|
65
68
|
"cli_print_header": ("mlarray.cli", "cli_print_header"),
|
|
66
69
|
"cli_convert_to_mlarray": ("mlarray.cli", "cli_convert_to_mlarray"),
|