large-image-source-zarr 1.35.8.dev4__tar.gz → 1.35.8.dev14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (15) hide show
  1. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/PKG-INFO +4 -4
  2. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/large_image_source_zarr/__init__.py +115 -29
  3. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/large_image_source_zarr.egg-info/PKG-INFO +4 -4
  4. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/large_image_source_zarr.egg-info/requires.txt +3 -3
  5. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/setup.py +2 -1
  6. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/LICENSE +0 -0
  7. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/README.rst +0 -0
  8. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/large_image_source_zarr/girder_source.py +0 -0
  9. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/large_image_source_zarr.egg-info/SOURCES.txt +0 -0
  10. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/large_image_source_zarr.egg-info/dependency_links.txt +0 -0
  11. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/large_image_source_zarr.egg-info/entry_points.txt +0 -0
  12. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/large_image_source_zarr.egg-info/scm_file_list.json +0 -0
  13. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/large_image_source_zarr.egg-info/top_level.txt +0 -0
  14. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/pyproject.toml +0 -0
  15. {large_image_source_zarr-1.35.8.dev4 → large_image_source_zarr-1.35.8.dev14}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: large-image-source-zarr
3
- Version: 1.35.8.dev4
3
+ Version: 1.35.8.dev14
4
4
  Summary: A OME Zarr tilesource for large_image.
5
5
  Home-page: https://github.com/girder/large_image
6
6
  Author: Kitware, Inc.
@@ -18,13 +18,13 @@ Classifier: Programming Language :: Python :: 3.15
18
18
  Requires-Python: >=3.10
19
19
  Description-Content-Type: text/x-rst
20
20
  License-File: LICENSE
21
- Requires-Dist: large-image>=1.35.8.dev4
21
+ Requires-Dist: large-image>=1.35.8.dev14
22
22
  Requires-Dist: zarr<3
23
23
  Requires-Dist: numcodecs<0.16
24
24
  Requires-Dist: imagecodecs-numcodecs!=2024.9.22,!=2026.8.16
25
- Requires-Dist: filelock
25
+ Requires-Dist: filelock>=3.13
26
26
  Provides-Extra: girder
27
- Requires-Dist: girder-large-image>=1.35.8.dev4; extra == "girder"
27
+ Requires-Dist: girder-large-image>=1.35.8.dev14; extra == "girder"
28
28
  Provides-Extra: all
29
29
  Requires-Dist: large-image-converter; extra == "all"
30
30
  Dynamic: author
@@ -3,6 +3,7 @@ import importlib.metadata
3
3
  import json
4
4
  import math
5
5
  import os
6
+ import re
6
7
  import shutil
7
8
  import tempfile
8
9
  import threading
@@ -27,6 +28,8 @@ with contextlib.suppress(importlib.metadata.PackageNotFoundError):
27
28
 
28
29
 
29
30
  zarr = None
31
+ # Not part of the zarr data; used for cross-process safety
32
+ _internalFiles = ('.zarr_flock', '.zarr_stamp')
30
33
 
31
34
 
32
35
  def _lazyImport():
@@ -46,6 +49,19 @@ def _lazyImport():
46
49
  raise TileSourceError(msg)
47
50
 
48
51
 
52
+ def _copyStore(src, dst):
53
+ """
54
+ Copy the zarr data from one store to another, excluding our internal
55
+ files.
56
+
57
+ :param src: the source store.
58
+ :param dst: the destination store.
59
+ """
60
+ zarr.copy_store(
61
+ src, dst, if_exists='replace',
62
+ excludes=[f'^{re.escape(name)}$' for name in _internalFiles])
63
+
64
+
49
65
  class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
50
66
  """
51
67
  Provides tile access to files that the zarr library can read.
@@ -162,14 +178,18 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
162
178
  self._projection = None
163
179
  self._gcps = None
164
180
  if not self._created:
165
- with contextlib.suppress(Exception):
166
- self._validateZarr()
181
+ with self._threadLock, self._processLock(), contextlib.suppress(Exception):
182
+ # Always read the existing data; this reopens the group while
183
+ # holding the lock
184
+ self._lastStamp = False
185
+ self._syncFromOtherProcesses()
167
186
 
168
187
  def _processLock(self):
169
188
  """
170
189
  This is a function to avoid pickling issues.
171
190
  """
172
- return filelock.FileLock(os.path.join(str(self._tempdir), '.zarr_flock'))
191
+ return filelock.FileLock(
192
+ os.path.join(str(self._tempdir), '.zarr_flock'), is_singleton=True)
173
193
 
174
194
  def __del__(self):
175
195
  if not hasattr(self, '_derivedSource'):
@@ -401,12 +421,15 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
401
421
  if frame_values.size > 0:
402
422
  self._frameValues = frame_values
403
423
 
404
- def _validateZarr(self):
424
+ def _validateZarr(self, writeMetadata=True):
405
425
  """
406
426
  Validate that we can read tiles from the zarr parent group in
407
427
  self._zarr. Set up the appropriate class variables.
428
+
429
+ :param writeMetadata: if True and this is an editable image, write
430
+ this process's metadata before reading it back.
408
431
  """
409
- if self._editable and hasattr(self, '_axes'):
432
+ if writeMetadata and self._editable and hasattr(self, '_axes'):
410
433
  self._writeInternalMetadata()
411
434
  found = self._scanZarrGroup(self._zarr)
412
435
  if found['best'] is None:
@@ -728,9 +751,6 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
728
751
  ``level`` is a reserved word and not permitted for an axis name.
729
752
  """
730
753
  self._checkEditable()
731
- with contextlib.suppress(TileSourceError):
732
- # read any info written by other processes
733
- self._validateZarr()
734
754
  updateMetadata = False
735
755
  store_path = str(kwargs.pop('level', 0))
736
756
  placement = {
@@ -742,9 +762,10 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
742
762
  k.replace('_value', ''): v for k, v in kwargs.items()
743
763
  if k not in placement
744
764
  }
745
- tile, mask, placement, axes = self._validateNewTile(tile, mask, placement, axes)
746
765
 
747
- with self._threadLock and self._processLock():
766
+ with self._threadLock, self._processLock():
767
+ self._syncFromOtherProcesses()
768
+ tile, mask, placement, axes = self._validateNewTile(tile, mask, placement, axes)
748
769
  old_axes = self._axes if hasattr(self, '_axes') else {}
749
770
  self._axes = {k: i for i, k in enumerate(axes)}
750
771
  new_axes = {k: i for k, i in self._axes.items() if k not in old_axes}
@@ -767,12 +788,13 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
767
788
  updateMetadata = True
768
789
  self._updateFrameValues(frame_values, placement, axes, new_axes, new_dims)
769
790
 
770
- current_arrays = dict(self._zarr.arrays())
791
+ current_arrays = self._currentArrays()
771
792
  if store_path == '0':
772
793
  # if writing to base data, invalidate generated levels
773
- for path in current_arrays:
794
+ for path in list(current_arrays):
774
795
  if path != store_path:
775
796
  self._zarr_store.rmdir(path)
797
+ self._arrays = None
776
798
  chunking = None
777
799
  if store_path not in current_arrays:
778
800
  chunking = tuple([
@@ -790,6 +812,7 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
790
812
  path=store_path,
791
813
  write_empty_chunks=False,
792
814
  )
815
+ self._arrays = None
793
816
  chunking = None
794
817
  else:
795
818
  arr = current_arrays[store_path]
@@ -818,6 +841,7 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
818
841
  store=self._zarr_store,
819
842
  path=store_path,
820
843
  )
844
+ self._arrays = None
821
845
 
822
846
  # If base data changed, update large_image attributes
823
847
  if store_path == '0':
@@ -835,8 +859,9 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
835
859
  self.levels = int(max(1, math.ceil(math.log(max(
836
860
  self.sizeX / self.tileWidth, self.sizeY / self.tileHeight)) / math.log(2)) + 1))
837
861
  updateMetadata = True
838
- if updateMetadata:
839
- self._writeInternalMetadata()
862
+ self._markChanged()
863
+ if updateMetadata:
864
+ self._writeInternalMetadata()
840
865
 
841
866
  def addAssociatedImage(self, image, imageKey=None):
842
867
  """
@@ -846,7 +871,8 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
846
871
  with an image. The numpy array can have 2 or 3 dimensions.
847
872
  """
848
873
  data, _ = _imageToNumpy(image)
849
- with self._threadLock and self._processLock():
874
+ with self._threadLock, self._processLock():
875
+ self._syncFromOtherProcesses()
850
876
  if imageKey is None:
851
877
  # Each associated image should be in its own group
852
878
  num_existing = len(self.getAssociatedImagesList())
@@ -858,6 +884,7 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
858
884
  path=f'{imageKey}/image',
859
885
  )
860
886
  self._associatedImages[imageKey] = (group, arr)
887
+ self._markChanged()
861
888
 
862
889
  def _getAxisInternalMetadata(self, axis_name):
863
890
  """
@@ -896,15 +923,64 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
896
923
  axis_metadata['unit'] = unit
897
924
  return axis_metadata
898
925
 
926
+ def _readStamp(self):
927
+ """
928
+ Read the stamp that is changed every time the image is modified.
929
+
930
+ :returns: the stamp or None if the image has not been modified.
931
+ """
932
+ with contextlib.suppress(OSError):
933
+ return (self._tempdir / '.zarr_stamp').read_text()
934
+ return None
935
+
936
+ def _syncFromOtherProcesses(self):
937
+ """
938
+ If another process has modified the image since this process last
939
+ did, read what it wrote. Every modification updates the stamp, so
940
+ this is cheap when nothing has changed. This must be called while
941
+ holding the process lock.
942
+ """
943
+ stamp = self._readStamp()
944
+ if stamp == getattr(self, '_lastStamp', None):
945
+ return
946
+ self._lastStamp = stamp
947
+ self._arrays = None
948
+ # zarr caches group attributes in memory, so reopen the group
949
+ self._zarr = zarr.open_group(self._zarr_store, mode='a')
950
+ with contextlib.suppress(TileSourceError):
951
+ # Don't write our metadata first; that would replace what the
952
+ # other process wrote
953
+ self._validateZarr(writeMetadata=False)
954
+
955
+ def _markChanged(self):
956
+ """
957
+ Record that this process modified the image. This must be called
958
+ while holding the process lock.
959
+ """
960
+ self._lastStamp = str(uuid.uuid4())
961
+ (self._tempdir / '.zarr_stamp').write_text(self._lastStamp)
962
+
963
+ def _currentArrays(self):
964
+ """
965
+ Get the arrays in the root group. Listing them is slow, so this is cached
966
+ until this or another process changes them.
967
+
968
+ :returns: a dictionary of array names and zarr arrays.
969
+ """
970
+ if getattr(self, '_arrays', None) is None:
971
+ self._arrays = dict(self._zarr.arrays())
972
+ return self._arrays
973
+
899
974
  def _writeInternalMetadata(self):
900
975
  """
901
976
  Write metadata to Zarr attributes.
902
977
  Metadata structure adheres to OME schema from https://ngff.openmicroscopy.org/latest/
903
978
  """
904
979
  self._checkEditable()
905
- with self._threadLock and self._processLock():
980
+ with self._threadLock, self._processLock():
981
+ self._syncFromOtherProcesses()
906
982
  name = str(self._tempdir.name).split('/')[-1]
907
- arrays = dict(self._zarr.arrays())
983
+ arrays = self._currentArrays()
908
984
  datasets = []
909
985
  axes = []
910
986
  channels = []
@@ -965,7 +1041,7 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
965
1041
  channel_metadata['color'] = self._channelColors[c]
966
1042
  channels.append(channel_metadata)
967
1043
  # Guidelines from https://ngff.openmicroscopy.org/latest/
968
- self._zarr.attrs.update({
1044
+ attrs = {
969
1045
  'multiscales': [{
970
1046
  'version': '0.5',
971
1047
  'name': name,
@@ -986,7 +1062,13 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
986
1062
  'rdefs': rdefs,
987
1063
  },
988
1064
  'bioformats2raw.layout': 3,
989
- })
1065
+ }
1066
+ # Writing attributes is slow and makes other processes reread
1067
+ # them, so only do so if they changed
1068
+ current = self._zarr.attrs.asdict()
1069
+ if any(current.get(k) != v for k, v in attrs.items()):
1070
+ self._zarr.attrs.update(attrs)
1071
+ self._markChanged()
990
1072
 
991
1073
  @property
992
1074
  def crop(self):
@@ -1417,16 +1499,20 @@ class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
1417
1499
  source._generateDownsampledLevels(resample)
1418
1500
  source._writeInternalMetadata() # rewrite with new level datasets
1419
1501
 
1420
- if suffix == '.zarr':
1421
- shutil.copytree(str(source._tempdir), path)
1422
- elif suffix in ['.db', '.sqlite']:
1423
- sqlite_store = zarr.SQLiteStore(path)
1424
- zarr.copy_store(source._zarr_store, sqlite_store, if_exists='replace')
1425
- sqlite_store.close()
1426
- elif suffix == '.zip':
1427
- zip_store = zarr.ZipStore(path)
1428
- zarr.copy_store(source._zarr_store, zip_store, if_exists='replace')
1429
- zip_store.close()
1502
+ # Don't let other processes change the data while we copy it
1503
+ with source._threadLock, source._processLock():
1504
+ if suffix == '.zarr':
1505
+ shutil.copytree(
1506
+ str(source._tempdir), path,
1507
+ ignore=shutil.ignore_patterns(*_internalFiles))
1508
+ elif suffix in ['.db', '.sqlite']:
1509
+ sqlite_store = zarr.SQLiteStore(path)
1510
+ _copyStore(source._zarr_store, sqlite_store)
1511
+ sqlite_store.close()
1512
+ elif suffix == '.zip':
1513
+ zip_store = zarr.ZipStore(path)
1514
+ _copyStore(source._zarr_store, zip_store)
1515
+ zip_store.close()
1430
1516
 
1431
1517
  else:
1432
1518
  from large_image_converter import convert
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: large-image-source-zarr
3
- Version: 1.35.8.dev4
3
+ Version: 1.35.8.dev14
4
4
  Summary: A OME Zarr tilesource for large_image.
5
5
  Home-page: https://github.com/girder/large_image
6
6
  Author: Kitware, Inc.
@@ -18,13 +18,13 @@ Classifier: Programming Language :: Python :: 3.15
18
18
  Requires-Python: >=3.10
19
19
  Description-Content-Type: text/x-rst
20
20
  License-File: LICENSE
21
- Requires-Dist: large-image>=1.35.8.dev4
21
+ Requires-Dist: large-image>=1.35.8.dev14
22
22
  Requires-Dist: zarr<3
23
23
  Requires-Dist: numcodecs<0.16
24
24
  Requires-Dist: imagecodecs-numcodecs!=2024.9.22,!=2026.8.16
25
- Requires-Dist: filelock
25
+ Requires-Dist: filelock>=3.13
26
26
  Provides-Extra: girder
27
- Requires-Dist: girder-large-image>=1.35.8.dev4; extra == "girder"
27
+ Requires-Dist: girder-large-image>=1.35.8.dev14; extra == "girder"
28
28
  Provides-Extra: all
29
29
  Requires-Dist: large-image-converter; extra == "all"
30
30
  Dynamic: author
@@ -1,11 +1,11 @@
1
- large-image>=1.35.8.dev4
1
+ large-image>=1.35.8.dev14
2
2
  zarr<3
3
3
  numcodecs<0.16
4
4
  imagecodecs-numcodecs!=2024.9.22,!=2026.8.16
5
- filelock
5
+ filelock>=3.13
6
6
 
7
7
  [all]
8
8
  large-image-converter
9
9
 
10
10
  [girder]
11
- girder-large-image>=1.35.8.dev4
11
+ girder-large-image>=1.35.8.dev14
@@ -42,7 +42,8 @@ setup(
42
42
  'numcodecs<0.16',
43
43
  # Without imagecodecs-numcodecs, some jpeg encoded data cannot be read
44
44
  'imagecodecs-numcodecs!=2024.9.22,!=2026.8.16',
45
- 'filelock',
45
+ # Needs is_singleton
46
+ 'filelock>=3.13',
46
47
  ],
47
48
  extras_require={
48
49
  'girder': f'girder-large-image{limit_version}',