large-image-source-zarr 1.35.8.dev2__tar.gz → 1.35.8.dev12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/PKG-INFO +4 -4
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/large_image_source_zarr/__init__.py +115 -29
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/large_image_source_zarr.egg-info/PKG-INFO +4 -4
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/large_image_source_zarr.egg-info/requires.txt +3 -3
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/setup.py +2 -1
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/LICENSE +0 -0
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/README.rst +0 -0
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/large_image_source_zarr/girder_source.py +0 -0
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/large_image_source_zarr.egg-info/SOURCES.txt +0 -0
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/large_image_source_zarr.egg-info/dependency_links.txt +0 -0
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/large_image_source_zarr.egg-info/entry_points.txt +0 -0
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/large_image_source_zarr.egg-info/scm_file_list.json +0 -0
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/large_image_source_zarr.egg-info/top_level.txt +0 -0
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/pyproject.toml +0 -0
- {large_image_source_zarr-1.35.8.dev2 → large_image_source_zarr-1.35.8.dev12}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: large-image-source-zarr
|
|
3
|
-
Version: 1.35.8.
|
|
3
|
+
Version: 1.35.8.dev12
|
|
4
4
|
Summary: A OME Zarr tilesource for large_image.
|
|
5
5
|
Home-page: https://github.com/girder/large_image
|
|
6
6
|
Author: Kitware, Inc.
|
|
@@ -18,13 +18,13 @@ Classifier: Programming Language :: Python :: 3.15
|
|
|
18
18
|
Requires-Python: >=3.10
|
|
19
19
|
Description-Content-Type: text/x-rst
|
|
20
20
|
License-File: LICENSE
|
|
21
|
-
Requires-Dist: large-image>=1.35.8.
|
|
21
|
+
Requires-Dist: large-image>=1.35.8.dev12
|
|
22
22
|
Requires-Dist: zarr<3
|
|
23
23
|
Requires-Dist: numcodecs<0.16
|
|
24
24
|
Requires-Dist: imagecodecs-numcodecs!=2024.9.22,!=2026.8.16
|
|
25
|
-
Requires-Dist: filelock
|
|
25
|
+
Requires-Dist: filelock>=3.13
|
|
26
26
|
Provides-Extra: girder
|
|
27
|
-
Requires-Dist: girder-large-image>=1.35.8.
|
|
27
|
+
Requires-Dist: girder-large-image>=1.35.8.dev12; extra == "girder"
|
|
28
28
|
Provides-Extra: all
|
|
29
29
|
Requires-Dist: large-image-converter; extra == "all"
|
|
30
30
|
Dynamic: author
|
|
@@ -3,6 +3,7 @@ import importlib.metadata
|
|
|
3
3
|
import json
|
|
4
4
|
import math
|
|
5
5
|
import os
|
|
6
|
+
import re
|
|
6
7
|
import shutil
|
|
7
8
|
import tempfile
|
|
8
9
|
import threading
|
|
@@ -27,6 +28,8 @@ with contextlib.suppress(importlib.metadata.PackageNotFoundError):
|
|
|
27
28
|
|
|
28
29
|
|
|
29
30
|
zarr = None
|
|
31
|
+
# Not part of the zarr data; used for cross-process safety
|
|
32
|
+
_internalFiles = ('.zarr_flock', '.zarr_stamp')
|
|
30
33
|
|
|
31
34
|
|
|
32
35
|
def _lazyImport():
|
|
@@ -46,6 +49,19 @@ def _lazyImport():
|
|
|
46
49
|
raise TileSourceError(msg)
|
|
47
50
|
|
|
48
51
|
|
|
52
|
+
def _copyStore(src, dst):
|
|
53
|
+
"""
|
|
54
|
+
Copy the zarr data from one store to another, excluding our internal
|
|
55
|
+
files.
|
|
56
|
+
|
|
57
|
+
:param src: the source store.
|
|
58
|
+
:param dst: the destination store.
|
|
59
|
+
"""
|
|
60
|
+
zarr.copy_store(
|
|
61
|
+
src, dst, if_exists='replace',
|
|
62
|
+
excludes=[f'^{re.escape(name)}$' for name in _internalFiles])
|
|
63
|
+
|
|
64
|
+
|
|
49
65
|
class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
50
66
|
"""
|
|
51
67
|
Provides tile access to files that the zarr library can read.
|
|
@@ -162,14 +178,18 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
162
178
|
self._projection = None
|
|
163
179
|
self._gcps = None
|
|
164
180
|
if not self._created:
|
|
165
|
-
with contextlib.suppress(Exception):
|
|
166
|
-
|
|
181
|
+
with self._threadLock, self._processLock(), contextlib.suppress(Exception):
|
|
182
|
+
# Always read the existing data; this reopens the group while
|
|
183
|
+
# holding the lock
|
|
184
|
+
self._lastStamp = False
|
|
185
|
+
self._syncFromOtherProcesses()
|
|
167
186
|
|
|
168
187
|
def _processLock(self):
|
|
169
188
|
"""
|
|
170
189
|
This is a function to avoid pickling issues.
|
|
171
190
|
"""
|
|
172
|
-
return filelock.FileLock(
|
|
191
|
+
return filelock.FileLock(
|
|
192
|
+
os.path.join(str(self._tempdir), '.zarr_flock'), is_singleton=True)
|
|
173
193
|
|
|
174
194
|
def __del__(self):
|
|
175
195
|
if not hasattr(self, '_derivedSource'):
|
|
@@ -401,12 +421,15 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
401
421
|
if frame_values.size > 0:
|
|
402
422
|
self._frameValues = frame_values
|
|
403
423
|
|
|
404
|
-
def _validateZarr(self):
|
|
424
|
+
def _validateZarr(self, writeMetadata=True):
|
|
405
425
|
"""
|
|
406
426
|
Validate that we can read tiles from the zarr parent group in
|
|
407
427
|
self._zarr. Set up the appropriate class variables.
|
|
428
|
+
|
|
429
|
+
:param writeMetadata: if True and this is an editable image, write
|
|
430
|
+
this process's metadata before reading it back.
|
|
408
431
|
"""
|
|
409
|
-
if self._editable and hasattr(self, '_axes'):
|
|
432
|
+
if writeMetadata and self._editable and hasattr(self, '_axes'):
|
|
410
433
|
self._writeInternalMetadata()
|
|
411
434
|
found = self._scanZarrGroup(self._zarr)
|
|
412
435
|
if found['best'] is None:
|
|
@@ -728,9 +751,6 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
728
751
|
``level`` is a reserved word and not permitted for an axis name.
|
|
729
752
|
"""
|
|
730
753
|
self._checkEditable()
|
|
731
|
-
with contextlib.suppress(TileSourceError):
|
|
732
|
-
# read any info written by other processes
|
|
733
|
-
self._validateZarr()
|
|
734
754
|
updateMetadata = False
|
|
735
755
|
store_path = str(kwargs.pop('level', 0))
|
|
736
756
|
placement = {
|
|
@@ -742,9 +762,10 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
742
762
|
k.replace('_value', ''): v for k, v in kwargs.items()
|
|
743
763
|
if k not in placement
|
|
744
764
|
}
|
|
745
|
-
tile, mask, placement, axes = self._validateNewTile(tile, mask, placement, axes)
|
|
746
765
|
|
|
747
|
-
with self._threadLock
|
|
766
|
+
with self._threadLock, self._processLock():
|
|
767
|
+
self._syncFromOtherProcesses()
|
|
768
|
+
tile, mask, placement, axes = self._validateNewTile(tile, mask, placement, axes)
|
|
748
769
|
old_axes = self._axes if hasattr(self, '_axes') else {}
|
|
749
770
|
self._axes = {k: i for i, k in enumerate(axes)}
|
|
750
771
|
new_axes = {k: i for k, i in self._axes.items() if k not in old_axes}
|
|
@@ -767,12 +788,13 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
767
788
|
updateMetadata = True
|
|
768
789
|
self._updateFrameValues(frame_values, placement, axes, new_axes, new_dims)
|
|
769
790
|
|
|
770
|
-
current_arrays =
|
|
791
|
+
current_arrays = self._currentArrays()
|
|
771
792
|
if store_path == '0':
|
|
772
793
|
# if writing to base data, invalidate generated levels
|
|
773
|
-
for path in current_arrays:
|
|
794
|
+
for path in list(current_arrays):
|
|
774
795
|
if path != store_path:
|
|
775
796
|
self._zarr_store.rmdir(path)
|
|
797
|
+
self._arrays = None
|
|
776
798
|
chunking = None
|
|
777
799
|
if store_path not in current_arrays:
|
|
778
800
|
chunking = tuple([
|
|
@@ -790,6 +812,7 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
790
812
|
path=store_path,
|
|
791
813
|
write_empty_chunks=False,
|
|
792
814
|
)
|
|
815
|
+
self._arrays = None
|
|
793
816
|
chunking = None
|
|
794
817
|
else:
|
|
795
818
|
arr = current_arrays[store_path]
|
|
@@ -818,6 +841,7 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
818
841
|
store=self._zarr_store,
|
|
819
842
|
path=store_path,
|
|
820
843
|
)
|
|
844
|
+
self._arrays = None
|
|
821
845
|
|
|
822
846
|
# If base data changed, update large_image attributes
|
|
823
847
|
if store_path == '0':
|
|
@@ -835,8 +859,9 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
835
859
|
self.levels = int(max(1, math.ceil(math.log(max(
|
|
836
860
|
self.sizeX / self.tileWidth, self.sizeY / self.tileHeight)) / math.log(2)) + 1))
|
|
837
861
|
updateMetadata = True
|
|
838
|
-
|
|
839
|
-
|
|
862
|
+
self._markChanged()
|
|
863
|
+
if updateMetadata:
|
|
864
|
+
self._writeInternalMetadata()
|
|
840
865
|
|
|
841
866
|
def addAssociatedImage(self, image, imageKey=None):
|
|
842
867
|
"""
|
|
@@ -846,7 +871,8 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
846
871
|
with an image. The numpy array can have 2 or 3 dimensions.
|
|
847
872
|
"""
|
|
848
873
|
data, _ = _imageToNumpy(image)
|
|
849
|
-
with self._threadLock
|
|
874
|
+
with self._threadLock, self._processLock():
|
|
875
|
+
self._syncFromOtherProcesses()
|
|
850
876
|
if imageKey is None:
|
|
851
877
|
# Each associated image should be in its own group
|
|
852
878
|
num_existing = len(self.getAssociatedImagesList())
|
|
@@ -858,6 +884,7 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
858
884
|
path=f'{imageKey}/image',
|
|
859
885
|
)
|
|
860
886
|
self._associatedImages[imageKey] = (group, arr)
|
|
887
|
+
self._markChanged()
|
|
861
888
|
|
|
862
889
|
def _getAxisInternalMetadata(self, axis_name):
|
|
863
890
|
"""
|
|
@@ -896,15 +923,64 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
896
923
|
axis_metadata['unit'] = unit
|
|
897
924
|
return axis_metadata
|
|
898
925
|
|
|
926
|
+
def _readStamp(self):
|
|
927
|
+
"""
|
|
928
|
+
Read the stamp that is changed every time the image is modified.
|
|
929
|
+
|
|
930
|
+
:returns: the stamp or None if the image has not been modified.
|
|
931
|
+
"""
|
|
932
|
+
with contextlib.suppress(OSError):
|
|
933
|
+
return (self._tempdir / '.zarr_stamp').read_text()
|
|
934
|
+
return None
|
|
935
|
+
|
|
936
|
+
def _syncFromOtherProcesses(self):
|
|
937
|
+
"""
|
|
938
|
+
If another process has modified the image since this process last
|
|
939
|
+
did, read what it wrote. Every modification updates the stamp, so
|
|
940
|
+
this is cheap when nothing has changed. This must be called while
|
|
941
|
+
holding the process lock.
|
|
942
|
+
"""
|
|
943
|
+
stamp = self._readStamp()
|
|
944
|
+
if stamp == getattr(self, '_lastStamp', None):
|
|
945
|
+
return
|
|
946
|
+
self._lastStamp = stamp
|
|
947
|
+
self._arrays = None
|
|
948
|
+
# zarr caches group attributes in memory, so reopen the group
|
|
949
|
+
self._zarr = zarr.open_group(self._zarr_store, mode='a')
|
|
950
|
+
with contextlib.suppress(TileSourceError):
|
|
951
|
+
# Don't write our metadata first; that would replace what the
|
|
952
|
+
# other process wrote
|
|
953
|
+
self._validateZarr(writeMetadata=False)
|
|
954
|
+
|
|
955
|
+
def _markChanged(self):
|
|
956
|
+
"""
|
|
957
|
+
Record that this process modified the image. This must be called
|
|
958
|
+
while holding the process lock.
|
|
959
|
+
"""
|
|
960
|
+
self._lastStamp = str(uuid.uuid4())
|
|
961
|
+
(self._tempdir / '.zarr_stamp').write_text(self._lastStamp)
|
|
962
|
+
|
|
963
|
+
def _currentArrays(self):
|
|
964
|
+
"""
|
|
965
|
+
Get the arrays in the root group. Listing them is slow, so this is cached
|
|
966
|
+
until this or another process changes them.
|
|
967
|
+
|
|
968
|
+
:returns: a dictionary of array names and zarr arrays.
|
|
969
|
+
"""
|
|
970
|
+
if getattr(self, '_arrays', None) is None:
|
|
971
|
+
self._arrays = dict(self._zarr.arrays())
|
|
972
|
+
return self._arrays
|
|
973
|
+
|
|
899
974
|
def _writeInternalMetadata(self):
|
|
900
975
|
"""
|
|
901
976
|
Write metadata to Zarr attributes.
|
|
902
977
|
Metadata structure adheres to OME schema from https://ngff.openmicroscopy.org/latest/
|
|
903
978
|
"""
|
|
904
979
|
self._checkEditable()
|
|
905
|
-
with self._threadLock
|
|
980
|
+
with self._threadLock, self._processLock():
|
|
981
|
+
self._syncFromOtherProcesses()
|
|
906
982
|
name = str(self._tempdir.name).split('/')[-1]
|
|
907
|
-
arrays =
|
|
983
|
+
arrays = self._currentArrays()
|
|
908
984
|
datasets = []
|
|
909
985
|
axes = []
|
|
910
986
|
channels = []
|
|
@@ -965,7 +1041,7 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
965
1041
|
channel_metadata['color'] = self._channelColors[c]
|
|
966
1042
|
channels.append(channel_metadata)
|
|
967
1043
|
# Guidelines from https://ngff.openmicroscopy.org/latest/
|
|
968
|
-
|
|
1044
|
+
attrs = {
|
|
969
1045
|
'multiscales': [{
|
|
970
1046
|
'version': '0.5',
|
|
971
1047
|
'name': name,
|
|
@@ -986,7 +1062,13 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
986
1062
|
'rdefs': rdefs,
|
|
987
1063
|
},
|
|
988
1064
|
'bioformats2raw.layout': 3,
|
|
989
|
-
}
|
|
1065
|
+
}
|
|
1066
|
+
# Writing attributes is slow and makes other processes reread
|
|
1067
|
+
# them, so only do so if they changed
|
|
1068
|
+
current = self._zarr.attrs.asdict()
|
|
1069
|
+
if any(current.get(k) != v for k, v in attrs.items()):
|
|
1070
|
+
self._zarr.attrs.update(attrs)
|
|
1071
|
+
self._markChanged()
|
|
990
1072
|
|
|
991
1073
|
@property
|
|
992
1074
|
def crop(self):
|
|
@@ -1417,16 +1499,20 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
|
|
|
1417
1499
|
source._generateDownsampledLevels(resample)
|
|
1418
1500
|
source._writeInternalMetadata() # rewrite with new level datasets
|
|
1419
1501
|
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1502
|
+
# Don't let other processes change the data while we copy it
|
|
1503
|
+
with source._threadLock, source._processLock():
|
|
1504
|
+
if suffix == '.zarr':
|
|
1505
|
+
shutil.copytree(
|
|
1506
|
+
str(source._tempdir), path,
|
|
1507
|
+
ignore=shutil.ignore_patterns(*_internalFiles))
|
|
1508
|
+
elif suffix in ['.db', '.sqlite']:
|
|
1509
|
+
sqlite_store = zarr.SQLiteStore(path)
|
|
1510
|
+
_copyStore(source._zarr_store, sqlite_store)
|
|
1511
|
+
sqlite_store.close()
|
|
1512
|
+
elif suffix == '.zip':
|
|
1513
|
+
zip_store = zarr.ZipStore(path)
|
|
1514
|
+
_copyStore(source._zarr_store, zip_store)
|
|
1515
|
+
zip_store.close()
|
|
1430
1516
|
|
|
1431
1517
|
else:
|
|
1432
1518
|
from large_image_converter import convert
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: large-image-source-zarr
|
|
3
|
-
Version: 1.35.8.
|
|
3
|
+
Version: 1.35.8.dev12
|
|
4
4
|
Summary: A OME Zarr tilesource for large_image.
|
|
5
5
|
Home-page: https://github.com/girder/large_image
|
|
6
6
|
Author: Kitware, Inc.
|
|
@@ -18,13 +18,13 @@ Classifier: Programming Language :: Python :: 3.15
|
|
|
18
18
|
Requires-Python: >=3.10
|
|
19
19
|
Description-Content-Type: text/x-rst
|
|
20
20
|
License-File: LICENSE
|
|
21
|
-
Requires-Dist: large-image>=1.35.8.
|
|
21
|
+
Requires-Dist: large-image>=1.35.8.dev12
|
|
22
22
|
Requires-Dist: zarr<3
|
|
23
23
|
Requires-Dist: numcodecs<0.16
|
|
24
24
|
Requires-Dist: imagecodecs-numcodecs!=2024.9.22,!=2026.8.16
|
|
25
|
-
Requires-Dist: filelock
|
|
25
|
+
Requires-Dist: filelock>=3.13
|
|
26
26
|
Provides-Extra: girder
|
|
27
|
-
Requires-Dist: girder-large-image>=1.35.8.
|
|
27
|
+
Requires-Dist: girder-large-image>=1.35.8.dev12; extra == "girder"
|
|
28
28
|
Provides-Extra: all
|
|
29
29
|
Requires-Dist: large-image-converter; extra == "all"
|
|
30
30
|
Dynamic: author
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
large-image>=1.35.8.
|
|
1
|
+
large-image>=1.35.8.dev12
|
|
2
2
|
zarr<3
|
|
3
3
|
numcodecs<0.16
|
|
4
4
|
imagecodecs-numcodecs!=2024.9.22,!=2026.8.16
|
|
5
|
-
filelock
|
|
5
|
+
filelock>=3.13
|
|
6
6
|
|
|
7
7
|
[all]
|
|
8
8
|
large-image-converter
|
|
9
9
|
|
|
10
10
|
[girder]
|
|
11
|
-
girder-large-image>=1.35.8.
|
|
11
|
+
girder-large-image>=1.35.8.dev12
|
|
@@ -42,7 +42,8 @@ setup(
|
|
|
42
42
|
'numcodecs<0.16',
|
|
43
43
|
# Without imagecodecs-numcodecs, some jpeg encoded data cannot be read
|
|
44
44
|
'imagecodecs-numcodecs!=2024.9.22,!=2026.8.16',
|
|
45
|
-
|
|
45
|
+
# Needs is_singleton
|
|
46
|
+
'filelock>=3.13',
|
|
46
47
|
],
|
|
47
48
|
extras_require={
|
|
48
49
|
'girder': f'girder-large-image{limit_version}',
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|