mlarray 0.0.26__tar.gz → 0.0.27__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {mlarray-0.0.26 → mlarray-0.0.27}/PKG-INFO +5 -5
  2. {mlarray-0.0.26 → mlarray-0.0.27}/README.md +4 -4
  3. {mlarray-0.0.26 → mlarray-0.0.27}/docs/optimization.md +3 -3
  4. {mlarray-0.0.26 → mlarray-0.0.27}/docs/usage.md +4 -4
  5. {mlarray-0.0.26 → mlarray-0.0.27}/examples/example_open.py +4 -5
  6. {mlarray-0.0.26 → mlarray-0.0.27}/examples/example_save_load.py +1 -0
  7. {mlarray-0.0.26 → mlarray-0.0.27}/mlarray/mlarray.py +94 -10
  8. {mlarray-0.0.26 → mlarray-0.0.27}/mlarray.egg-info/PKG-INFO +5 -5
  9. {mlarray-0.0.26 → mlarray-0.0.27}/tests/test_metadata.py +4 -4
  10. {mlarray-0.0.26 → mlarray-0.0.27}/tests/test_optimization.py +3 -3
  11. {mlarray-0.0.26 → mlarray-0.0.27}/tests/test_usage.py +1 -2
  12. {mlarray-0.0.26 → mlarray-0.0.27}/.github/workflows/workflow.yml +0 -0
  13. {mlarray-0.0.26 → mlarray-0.0.27}/.gitignore +0 -0
  14. {mlarray-0.0.26 → mlarray-0.0.27}/LICENSE +0 -0
  15. {mlarray-0.0.26 → mlarray-0.0.27}/MANIFEST.in +0 -0
  16. {mlarray-0.0.26 → mlarray-0.0.27}/assets/banner.png +0 -0
  17. {mlarray-0.0.26 → mlarray-0.0.27}/assets/banner.png~ +0 -0
  18. {mlarray-0.0.26 → mlarray-0.0.27}/docs/api.md +0 -0
  19. {mlarray-0.0.26 → mlarray-0.0.27}/docs/cli.md +0 -0
  20. {mlarray-0.0.26 → mlarray-0.0.27}/docs/index.md +0 -0
  21. {mlarray-0.0.26 → mlarray-0.0.27}/docs/schema.md +0 -0
  22. {mlarray-0.0.26 → mlarray-0.0.27}/docs/why.md +0 -0
  23. {mlarray-0.0.26 → mlarray-0.0.27}/examples/example_channel.py +0 -0
  24. {mlarray-0.0.26 → mlarray-0.0.27}/examples/example_metadata_only.py +0 -0
  25. {mlarray-0.0.26 → mlarray-0.0.27}/mkdocs.yml +0 -0
  26. {mlarray-0.0.26 → mlarray-0.0.27}/mlarray/__init__.py +0 -0
  27. {mlarray-0.0.26 → mlarray-0.0.27}/mlarray/cli.py +0 -0
  28. {mlarray-0.0.26 → mlarray-0.0.27}/mlarray/meta.py +0 -0
  29. {mlarray-0.0.26 → mlarray-0.0.27}/mlarray/utils.py +0 -0
  30. {mlarray-0.0.26 → mlarray-0.0.27}/mlarray.egg-info/SOURCES.txt +0 -0
  31. {mlarray-0.0.26 → mlarray-0.0.27}/mlarray.egg-info/dependency_links.txt +0 -0
  32. {mlarray-0.0.26 → mlarray-0.0.27}/mlarray.egg-info/entry_points.txt +0 -0
  33. {mlarray-0.0.26 → mlarray-0.0.27}/mlarray.egg-info/requires.txt +0 -0
  34. {mlarray-0.0.26 → mlarray-0.0.27}/mlarray.egg-info/top_level.txt +0 -0
  35. {mlarray-0.0.26 → mlarray-0.0.27}/pyproject.toml +0 -0
  36. {mlarray-0.0.26 → mlarray-0.0.27}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mlarray
3
- Version: 0.0.26
3
+ Version: 0.0.27
4
4
  Summary: Array format specialized for Machine Learning with Blosc2 backend and standardized metadata.
5
5
  Author-email: Karol Gotkowski <karol.gotkowski@dkfz.de>
6
6
  License: MIT
@@ -88,16 +88,16 @@ from mlarray import MLArray
88
88
  import numpy as np
89
89
 
90
90
  # read-only, partial access (default)
91
- image = MLArray().open("sample.mla", mmap='r')
91
+ image = MLArray.open("sample.mla", mmap='r')
92
92
  crop = image[10:20, 50:60] # Read crop
93
93
 
94
94
  # read/write, partial access
95
- image = MLArray().open("sample.mla", mmap='r+')
95
+ image = MLArray.open("sample.mla", mmap='r+')
96
96
  image[10:20, 50:60] *= 5 # Modify crop in memory and disk
97
97
 
98
98
  # read/write, partial access, create/overwrite
99
99
  array = np.random.random((128, 256, 256))
100
- image = MLArray().open("sample.mla", shape=array.shape, dtype=array.dtype, mmap='w+')
100
+ image = MLArray.open("sample.mla", shape=array.shape, dtype=array.dtype, mmap='w+')
101
101
  image[...] = array # Modify image in memory and disk
102
102
  ```
103
103
 
@@ -125,7 +125,7 @@ image.meta.image["study_id"] = "study-001"
125
125
  image.save("with-metadata.mla")
126
126
 
127
127
  # Open memory-mapped
128
- image = MLArray().open("with-metadata.mla", mmap='r+')
128
+ image = MLArray.open("with-metadata.mla", mmap='r+')
129
129
  image.meta.image["study_id"] = "new-study" # Modify metadata
130
130
  image.close() # Close and save metadata, only necessary to save modified metadata
131
131
  ```
@@ -54,16 +54,16 @@ from mlarray import MLArray
54
54
  import numpy as np
55
55
 
56
56
  # read-only, partial access (default)
57
- image = MLArray().open("sample.mla", mmap='r')
57
+ image = MLArray.open("sample.mla", mmap='r')
58
58
  crop = image[10:20, 50:60] # Read crop
59
59
 
60
60
  # read/write, partial access
61
- image = MLArray().open("sample.mla", mmap='r+')
61
+ image = MLArray.open("sample.mla", mmap='r+')
62
62
  image[10:20, 50:60] *= 5 # Modify crop in memory and disk
63
63
 
64
64
  # read/write, partial access, create/overwrite
65
65
  array = np.random.random((128, 256, 256))
66
- image = MLArray().open("sample.mla", shape=array.shape, dtype=array.dtype, mmap='w+')
66
+ image = MLArray.open("sample.mla", shape=array.shape, dtype=array.dtype, mmap='w+')
67
67
  image[...] = array # Modify image in memory and disk
68
68
  ```
69
69
 
@@ -91,7 +91,7 @@ image.meta.image["study_id"] = "study-001"
91
91
  image.save("with-metadata.mla")
92
92
 
93
93
  # Open memory-mapped
94
- image = MLArray().open("with-metadata.mla", mmap='r+')
94
+ image = MLArray.open("with-metadata.mla", mmap='r+')
95
95
  image.meta.image["study_id"] = "new-study" # Modify metadata
96
96
  image.close() # Close and save metadata, only necessary to save modified metadata
97
97
  ```
@@ -96,7 +96,7 @@ For large files, you typically want **mmap reads** so random patches don’t req
96
96
  from mlarray import MLArray
97
97
 
98
98
  # read-only mmap: fast random access without loading the full volume
99
- image = MLArray().open("patch-non-iso.mla", mmap='r')
99
+ image = MLArray.open("patch-non-iso.mla", mmap='r')
100
100
 
101
101
  patch = image[10:20, 50:60] # Read a crop/patch (partial read)
102
102
  ```
@@ -119,7 +119,7 @@ You can modify regions in-place with `mmap='r+'`. This is useful for workflows l
119
119
  ```python
120
120
  from mlarray import MLArray
121
121
 
122
- image = MLArray().open("patch-non-iso.mla", mmap='r+')
122
+ image = MLArray.open("patch-non-iso.mla", mmap='r+')
123
123
  image[10:20, 50:60] *= 5 # Modify crop in memory and on disk
124
124
  image.close()
125
125
  ```
@@ -137,7 +137,7 @@ from mlarray import MLArray
137
137
  shape = (128, 256, 256)
138
138
  dtype = np.float32
139
139
 
140
- image = MLArray().open(
140
+ image = MLArray.open(
141
141
  "streamed-write.mla",
142
142
  shape=shape,
143
143
  dtype=dtype,
@@ -34,16 +34,16 @@ from mlarray import MLArray
34
34
  import numpy as np
35
35
 
36
36
  # read-only, partial access (default)
37
- image = MLArray().open("sample.mla", mmap='r')
37
+ image = MLArray.open("sample.mla", mmap='r')
38
38
  crop = image[10:20, 50:60] # Read crop
39
39
 
40
40
  # read/write, partial access
41
- image = MLArray().open("sample.mla", mmap='r+')
41
+ image = MLArray.open("sample.mla", mmap='r+')
42
42
  image[10:20, 50:60] *= 5 # Modify crop in memory and disk
43
43
 
44
44
  # read/write, partial access, create/overwrite
45
45
  array = np.random.random((128, 256, 256))
46
- image = MLArray().open("sample.mla", shape=array.shape, dtype=array.dtype, mmap='w+')
46
+ image = MLArray.open("sample.mla", shape=array.shape, dtype=array.dtype, mmap='w+')
47
47
  image[...] = array # Modify image in memory and disk
48
48
  ```
49
49
 
@@ -75,7 +75,7 @@ image.meta.image["study_id"] = "study-001"
75
75
  image.save("with-metadata.mla")
76
76
 
77
77
  # Open memory-mapped
78
- image = MLArray().open("with-metadata.mla", mmap='r+')
78
+ image = MLArray.open("with-metadata.mla", mmap='r+')
79
79
  image.meta.image["study_id"] = "new-study" # Modify metadata
80
80
  image.close() # Close and save metadata, only necessary to save modified metadata
81
81
  ```
@@ -1,7 +1,7 @@
1
1
  import numpy as np
2
2
  import os
3
3
  from pathlib import Path
4
- from mlarray import MLArray, Meta
4
+ from mlarray import MLArray, Meta, MetaSpatial
5
5
  import json
6
6
 
7
7
 
@@ -19,16 +19,15 @@ if __name__ == '__main__':
19
19
  os.remove(filepath)
20
20
 
21
21
  print("Initializing image...")
22
- image = MLArray(spacing=spacing, origin=origin, direction=direction, meta=Meta(image=image_meta, bbox=bboxes))
23
- image.open(filepath, shape=array.shape, dtype=array.dtype, mmap='w+')
22
+ image = MLArray.open(filepath, shape=array.shape, dtype=array.dtype, mmap='w+')
24
23
  print("Saving image...")
25
24
  image[...] = array
25
+ image.meta = Meta(image=image_meta, spatial=MetaSpatial(spacing=spacing, origin=origin, direction=direction), bbox=bboxes)
26
26
  image.meta.is_seg = True
27
27
  image.close()
28
28
 
29
29
  print("Loading image...")
30
- image = MLArray()
31
- image.open(filepath)
30
+ image = MLArray.open(filepath)
32
31
  print(json.dumps(image.meta.to_dict(), indent=2, sort_keys=True))
33
32
  print("Image mean value: ", np.mean(image.to_numpy()))
34
33
  print("Some array data: \n", image[:2, :2, 0])
@@ -25,6 +25,7 @@ if __name__ == '__main__':
25
25
 
26
26
  print("Loading image...")
27
27
  image = MLArray(filepath)
28
+ image = MLArray.load(filepath) # Just testing both load methods
28
29
  print(json.dumps(image.meta.to_dict(), indent=2, sort_keys=True))
29
30
  print("Image mean value: ", np.mean(image.to_numpy()))
30
31
  print("Some array data: \n", image[:2, :2, 0])
@@ -55,22 +55,22 @@ class MLArray:
55
55
  set via arguments.
56
56
  """
57
57
  self.filepath = None
58
- self.support_metadata = None
58
+ self.support_metadata = None
59
59
  self.mmap = None
60
60
  self.meta = None
61
61
  if isinstance(array, (str, Path)) and (spacing is not None or origin is not None or direction is not None or meta is not None or channel_axis is not None or copy is not None):
62
62
  raise ("Spacing, origin, direction, meta, channel_axis or copy cannot be set when array is a filepath.")
63
63
  if isinstance(array, (str, Path)):
64
- self.load(array, num_threads)
64
+ self._load(array, num_threads)
65
65
  else:
66
66
  self._store = array
67
- self._validate_and_add_meta(meta, spacing, origin, direction, channel_axis)
68
-
69
- if copy is not None:
70
- self.meta.copy_from(copy.meta)
67
+ self._validate_and_add_meta(meta, spacing, origin, direction, channel_axis)
68
+ if copy is not None:
69
+ self.meta.copy_from(copy.meta)
71
70
 
71
+ @classmethod
72
72
  def open(
73
- self,
73
+ cls,
74
74
  filepath: Union[str, Path],
75
75
  shape: Optional[Union[List, Tuple, np.ndarray]] = None,
76
76
  dtype: Optional[np.dtype] = None,
@@ -89,6 +89,66 @@ class MLArray:
89
89
  files. When creating a new file, both ``shape`` and ``dtype`` must be
90
90
  provided.
91
91
 
92
+ WARNING:
93
+ MLArray supports both ".b2nd" and ".mla" files. The MLArray
94
+ format standard and standardized metadata are honored only for
95
+ ".mla". For ".b2nd", metadata is ignored when loading.
96
+
97
+ Args:
98
+ filepath (Union[str, Path]): Target file path. Must end with
99
+ ".b2nd" or ".mla".
100
+ shape (Optional[Union[List, Tuple, np.ndarray]]): Shape of the array
101
+ to create. If provided, a new file is created. Length must match
102
+ the full array dimensionality (including channels if present).
103
+ dtype (Optional[np.dtype]): Numpy dtype for a newly created array.
104
+ channel_axis (Optional[int]): Axis index for channels in the array.
105
+ Used for patch/chunk/block calculations.
106
+ mmap (str): Blosc2 mmap mode. One of "r", "r+", "w+", "c".
107
+ patch_size (Optional[Union[int, List, Tuple]]): Patch size hint for
108
+ chunk/block optimization. Provide an int for isotropic sizes or
109
+ a list/tuple with length equal to the number of spatial
110
+ dimensions. Use "default" to use the default patch size of 192.
111
+ chunk_size (Optional[Union[int, List, Tuple]]): Explicit chunk size.
112
+ Provide an int or tuple/list with length equal to the array
113
+ dimensions. Ignored when ``patch_size`` is provided.
114
+ block_size (Optional[Union[int, List, Tuple]]): Explicit block size.
115
+ Provide an int or tuple/list with length equal to the array
116
+ dimensions. Ignored when ``patch_size`` is provided.
117
+ num_threads (int): Number of threads for Blosc2 operations.
118
+ cparams (Optional[Dict]): Blosc2 compression parameters.
119
+ dparams (Optional[Dict]): Blosc2 decompression parameters.
120
+
121
+ Returns:
122
+ MLArray: The current instance (for chaining).
123
+
124
+ Raises:
125
+ RuntimeError: If the file extension is invalid, if shape/dtype are
126
+ inconsistent, or if mmap mode is invalid for creation.
127
+ """
128
+ class_instance = cls()
129
+ class_instance._open(filepath, shape, dtype, channel_axis, mmap, patch_size, chunk_size, block_size, num_threads, cparams, dparams)
130
+ return class_instance
131
+
132
+ def _open(
133
+ self,
134
+ filepath: Union[str, Path],
135
+ shape: Optional[Union[List, Tuple, np.ndarray]] = None,
136
+ dtype: Optional[np.dtype] = None,
137
+ channel_axis: Optional[int] = None,
138
+ mmap: str = 'r',
139
+ patch_size: Optional[Union[int, List, Tuple]] = 'default', # 'default' means that the default of 192 is used. However, if set to 'default', the patch_size will be skipped if self.patch_size is set from a previously loaded MLArray image. In that case the self.patch_size is used.
140
+ chunk_size: Optional[Union[int, List, Tuple]]= None,
141
+ block_size: Optional[Union[int, List, Tuple]] = None,
142
+ num_threads: int = 1,
143
+ cparams: Optional[Dict] = None,
144
+ dparams: Optional[Dict] = None
145
+ ):
146
+ """Internal open method. Open an existing Blosc2 file or create a new one with memory mapping.
147
+
148
+ This method supports both MLArray (".mla") and plain Blosc2 (".b2nd")
149
+ files. When creating a new file, both ``shape`` and ``dtype`` must be
150
+ provided.
151
+
92
152
  WARNING:
93
153
  MLArray supports both ".b2nd" and ".mla" files. The MLArray
94
154
  format standard and standardized metadata are honored only for
@@ -164,7 +224,6 @@ class MLArray:
164
224
  self.meta._blosc2.block_size = list(self._store.blocks)
165
225
  self.mmap = mmap
166
226
  self._write_metadata()
167
- return self
168
227
 
169
228
  def close(self):
170
229
  """Flush metadata and close the underlying store.
@@ -177,14 +236,39 @@ class MLArray:
177
236
  self.support_metadata = None
178
237
  self.mmap = None
179
238
  self.meta = None
180
-
239
+
240
+ @classmethod
181
241
  def load(
182
- self,
242
+ cls,
183
243
  filepath: Union[str, Path],
184
244
  num_threads: int = 1,
185
245
  ):
186
246
  """Loads a Blosc2-compressed file. Both MLArray ('.mla') and Blosc2 ('.b2nd') files are supported.
187
247
 
248
+ WARNING:
249
+ MLArray supports both ".b2nd" and ".mla" files. The MLArray
250
+ format standard and standardized metadata are honored only for
251
+ ".mla". For ".b2nd", metadata is ignored when loading.
252
+
253
+ Args:
254
+ filepath (Union[str, Path]): Path to the Blosc2 file to be loaded.
255
+ The filepath needs to have the extension ".b2nd" or ".mla".
256
+ num_threads (int): Number of threads to use for loading the file.
257
+
258
+ Raises:
259
+ RuntimeError: If the file extension is not ".b2nd" or ".mla".
260
+ """
261
+ class_instance = cls()
262
+ class_instance._load(filepath, num_threads)
263
+ return class_instance
264
+
265
+ def _load(
266
+ self,
267
+ filepath: Union[str, Path],
268
+ num_threads: int = 1,
269
+ ):
270
+ """Internal MLArray load method. Loads a Blosc2-compressed file. Both MLArray ('.mla') and Blosc2 ('.b2nd') files are supported.
271
+
188
272
  WARNING:
189
273
  MLArray supports both ".b2nd" and ".mla" files. The MLArray
190
274
  format standard and standardized metadata are honored only for
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mlarray
3
- Version: 0.0.26
3
+ Version: 0.0.27
4
4
  Summary: Array format specialized for Machine Learning with Blosc2 backend and standardized metadata.
5
5
  Author-email: Karol Gotkowski <karol.gotkowski@dkfz.de>
6
6
  License: MIT
@@ -88,16 +88,16 @@ from mlarray import MLArray
88
88
  import numpy as np
89
89
 
90
90
  # read-only, partial access (default)
91
- image = MLArray().open("sample.mla", mmap='r')
91
+ image = MLArray.open("sample.mla", mmap='r')
92
92
  crop = image[10:20, 50:60] # Read crop
93
93
 
94
94
  # read/write, partial access
95
- image = MLArray().open("sample.mla", mmap='r+')
95
+ image = MLArray.open("sample.mla", mmap='r+')
96
96
  image[10:20, 50:60] *= 5 # Modify crop in memory and disk
97
97
 
98
98
  # read/write, partial access, create/overwrite
99
99
  array = np.random.random((128, 256, 256))
100
- image = MLArray().open("sample.mla", shape=array.shape, dtype=array.dtype, mmap='w+')
100
+ image = MLArray.open("sample.mla", shape=array.shape, dtype=array.dtype, mmap='w+')
101
101
  image[...] = array # Modify image in memory and disk
102
102
  ```
103
103
 
@@ -125,7 +125,7 @@ image.meta.image["study_id"] = "study-001"
125
125
  image.save("with-metadata.mla")
126
126
 
127
127
  # Open memory-mapped
128
- image = MLArray().open("with-metadata.mla", mmap='r+')
128
+ image = MLArray.open("with-metadata.mla", mmap='r+')
129
129
  image.meta.image["study_id"] = "new-study" # Modify metadata
130
130
  image.close() # Close and save metadata, only necessary to save modified metadata
131
131
  ```
@@ -61,7 +61,7 @@ class TestMetadataStorage(unittest.TestCase):
61
61
  image = MLArray(array, meta={"a": 1})
62
62
  image.save(path)
63
63
 
64
- opened = MLArray().open(path, mmap="r")
64
+ opened = MLArray.open(path, mmap="r")
65
65
  opened.meta.image["a"] = 2
66
66
  opened.close()
67
67
 
@@ -75,7 +75,7 @@ class TestMetadataStorage(unittest.TestCase):
75
75
  image = MLArray(array, meta={"a": 1})
76
76
  image.save(path)
77
77
 
78
- opened = MLArray().open(path, mmap="r+")
78
+ opened = MLArray.open(path, mmap="r+")
79
79
  opened.meta.image["a"] = 2
80
80
  opened.close()
81
81
 
@@ -89,7 +89,7 @@ class TestMetadataStorage(unittest.TestCase):
89
89
  image = MLArray(array, meta={"a": 1})
90
90
  image.save(path)
91
91
 
92
- opened = MLArray().open(path, mmap="c")
92
+ opened = MLArray.open(path, mmap="c")
93
93
  opened.meta.image["a"] = 3
94
94
  opened.close()
95
95
 
@@ -102,7 +102,7 @@ class TestMetadataStorage(unittest.TestCase):
102
102
  shape = (8, 16, 16)
103
103
  dtype = np.float32
104
104
 
105
- opened = MLArray().open(path, shape=shape, dtype=dtype, mmap="w+")
105
+ opened = MLArray.open(path, shape=shape, dtype=dtype, mmap="w+")
106
106
  opened.meta.extra["created"] = True
107
107
  opened.close()
108
108
 
@@ -43,7 +43,7 @@ class TestOptimizationExamples(unittest.TestCase):
43
43
 
44
44
  MLArray(array).save(path, patch_size=(8, 16, 16))
45
45
 
46
- image = MLArray().open(path, mmap="r")
46
+ image = MLArray.open(path, mmap="r")
47
47
  patch = image[10:20, 5:15, 7:17]
48
48
 
49
49
  self.assertEqual(patch.shape, (10, 10, 10))
@@ -55,7 +55,7 @@ class TestOptimizationExamples(unittest.TestCase):
55
55
 
56
56
  MLArray(array).save(path, patch_size=(8, 16, 16))
57
57
 
58
- image = MLArray().open(path, mmap="r+")
58
+ image = MLArray.open(path, mmap="r+")
59
59
  image[0:2, 0:2, 0:2] *= 0.0
60
60
  image.close()
61
61
 
@@ -68,7 +68,7 @@ class TestOptimizationExamples(unittest.TestCase):
68
68
  dtype = np.float32
69
69
  path = Path(tmpdir) / "streamed-write.mla"
70
70
 
71
- image = MLArray().open(
71
+ image = MLArray.open(
72
72
  path,
73
73
  shape=shape,
74
74
  dtype=dtype,
@@ -31,8 +31,7 @@ class TestUsage(unittest.TestCase):
31
31
  path = Path(tmpdir) / "sample.mla"
32
32
  MLArray(array).save(path)
33
33
 
34
- loaded = MLArray(array)
35
- loaded.open(path, mmap="r")
34
+ loaded = MLArray.open(path, mmap="r")
36
35
  self.assertFalse(isinstance(loaded._store, np.ndarray))
37
36
 
38
37
  def test_loading_and_saving(self):
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes