funlib.persistence 0.8.0__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. funlib_persistence-0.9.0/PKG-INFO +59 -0
  2. funlib_persistence-0.9.0/README.md +34 -0
  3. funlib_persistence-0.9.0/pyproject.toml +94 -0
  4. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/__init__.py +1 -1
  5. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/array.py +37 -32
  6. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/datasets.py +42 -36
  7. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/freezable.py +2 -2
  8. funlib_persistence-0.9.0/src/funlib/persistence/arrays/lazy_ops.py +6 -0
  9. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/metadata.py +55 -33
  10. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/ome_datasets.py +7 -3
  11. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/__init__.py +1 -0
  12. funlib_persistence-0.9.0/src/funlib/persistence/graphs/csr_graph_database.py +1070 -0
  13. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/graph_database.py +15 -15
  14. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/pgsql_graph_database.py +18 -15
  15. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/sql_graph_database.py +51 -53
  16. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/sqlite_graph_database.py +14 -12
  17. funlib_persistence-0.9.0/src/funlib.persistence.egg-info/PKG-INFO +59 -0
  18. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/SOURCES.txt +2 -0
  19. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/requires.txt +0 -10
  20. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_array.py +92 -13
  21. funlib_persistence-0.9.0/tests/test_config.py +102 -0
  22. funlib_persistence-0.9.0/tests/test_csr_graph_database.py +244 -0
  23. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_datasets.py +337 -472
  24. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_graph.py +261 -304
  25. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_metadata.py +53 -1
  26. funlib_persistence-0.8.0/PKG-INFO +0 -46
  27. funlib_persistence-0.8.0/README.md +0 -15
  28. funlib_persistence-0.8.0/pyproject.toml +0 -59
  29. funlib_persistence-0.8.0/src/funlib/persistence/arrays/lazy_ops.py +0 -6
  30. funlib_persistence-0.8.0/src/funlib.persistence.egg-info/PKG-INFO +0 -46
  31. funlib_persistence-0.8.0/tests/test_config.py +0 -84
  32. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/LICENSE +0 -0
  33. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/setup.cfg +0 -0
  34. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/__init__.py +0 -0
  35. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/utils.py +0 -0
  36. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/py.typed +0 -0
  37. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/types.py +0 -0
  38. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/dependency_links.txt +0 -0
  39. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/top_level.txt +0 -0
  40. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_pgsql_graph_database.py +0 -0
  41. {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_slices.py +0 -0
@@ -0,0 +1,59 @@
1
+ Metadata-Version: 2.4
2
+ Name: funlib.persistence
3
+ Version: 0.9.0
4
+ Summary: Interfaces for data (arrays and graphs) and storage formats (databases and file formats)
5
+ Author-email: William Patton <pattonw@hhmi.org>, Jan Funke <funkej@hhmi.org>
6
+ License: MIT
7
+ Project-URL: Documentation, https://funkelab.github.io/funlib.persistence
8
+ Project-URL: Source, https://github.com/funkelab/funlib.persistence
9
+ Project-URL: Issues, https://github.com/funkelab/funlib.persistence/issues
10
+ Classifier: Programming Language :: Python :: 3
11
+ Requires-Python: >=3.11
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: zarr<4,>=3
15
+ Requires-Dist: iohub>=0.3.0a5
16
+ Requires-Dist: funlib.geometry>=0.3.0
17
+ Requires-Dist: networkx>=3.0.0
18
+ Requires-Dist: pymongo>=4.0.0
19
+ Requires-Dist: numpy>=2.0.0
20
+ Requires-Dist: pydantic>=2.0.0
21
+ Requires-Dist: dask>=2024.0.0
22
+ Requires-Dist: toml>=0.10.0
23
+ Requires-Dist: psycopg2-binary>=2.9.11
24
+ Dynamic: license-file
25
+
26
+ [![tests](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml/badge.svg)](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml)
27
+ [![lint](https://github.com/funkelab/funlib.persistence/actions/workflows/lint.yaml/badge.svg)](https://github.com/funkelab/funlib.persistence/actions/workflows/lint.yaml)
28
+ [![docs](https://github.com/funkelab/funlib.persistence/actions/workflows/docs.yaml/badge.svg)](https://funkelab.github.io/funlib.persistence)
29
+ [![publish](https://github.com/funkelab/funlib.persistence/actions/workflows/publish.yaml/badge.svg)](https://pypi.org/project/funlib.persistence/)
30
+
31
+ # funlib.persistence
32
+ Interfaces for data (arrays and graphs) and storage formats (databases and file formats)
33
+
34
+ **Documentation:** https://funkelab.github.io/funlib.persistence
35
+
36
+ ## Install
37
+
38
+ ```bash
39
+ pip install funlib.persistence # or: uv add funlib.persistence
40
+ ```
41
+
42
+ ## Contributing
43
+
44
+ The commands below are the ones CI runs, so a green local run means a green PR.
45
+
46
+ ```bash
47
+ git clone https://github.com/funkelab/funlib.persistence
48
+ cd funlib.persistence
49
+ uv sync # Python, dependencies, dev tools (from uv.lock)
50
+ uv run pytest # tests with coverage
51
+ uv run pytest --no-cov tests/<file>.py # one file, without the coverage floor
52
+ uv run ruff check . && uv run ruff format --check . # lint and formatting
53
+ uv run ty check src tests # types
54
+ uv run --group docs sphinx-build -W docs/source docs/build/html # docs
55
+ ```
56
+
57
+ The graph-database tests need a PostgreSQL server (CI starts one as a service, `PGUSER=postgres`); they are skipped when none is reachable.
58
+
59
+ `uv.lock` pins the development environment; the `lock-upgrade` workflow refreshes it monthly and opens a pull request.
@@ -0,0 +1,34 @@
1
+ [![tests](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml/badge.svg)](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml)
2
+ [![lint](https://github.com/funkelab/funlib.persistence/actions/workflows/lint.yaml/badge.svg)](https://github.com/funkelab/funlib.persistence/actions/workflows/lint.yaml)
3
+ [![docs](https://github.com/funkelab/funlib.persistence/actions/workflows/docs.yaml/badge.svg)](https://funkelab.github.io/funlib.persistence)
4
+ [![publish](https://github.com/funkelab/funlib.persistence/actions/workflows/publish.yaml/badge.svg)](https://pypi.org/project/funlib.persistence/)
5
+
6
+ # funlib.persistence
7
+ Interfaces for data (arrays and graphs) and storage formats (databases and file formats)
8
+
9
+ **Documentation:** https://funkelab.github.io/funlib.persistence
10
+
11
+ ## Install
12
+
13
+ ```bash
14
+ pip install funlib.persistence # or: uv add funlib.persistence
15
+ ```
16
+
17
+ ## Contributing
18
+
19
+ The commands below are the ones CI runs, so a green local run means a green PR.
20
+
21
+ ```bash
22
+ git clone https://github.com/funkelab/funlib.persistence
23
+ cd funlib.persistence
24
+ uv sync # Python, dependencies, dev tools (from uv.lock)
25
+ uv run pytest # tests with coverage
26
+ uv run pytest --no-cov tests/<file>.py # one file, without the coverage floor
27
+ uv run ruff check . && uv run ruff format --check . # lint and formatting
28
+ uv run ty check src tests # types
29
+ uv run --group docs sphinx-build -W docs/source docs/build/html # docs
30
+ ```
31
+
32
+ The graph-database tests need a PostgreSQL server (CI starts one as a service, `PGUSER=postgres`); they are skipped when none is reachable.
33
+
34
+ `uv.lock` pins the development environment; the `lock-upgrade` workflow refreshes it monthly and opens a pull request.
@@ -0,0 +1,94 @@
1
+ [build-system]
2
+ build-backend = "setuptools.build_meta"
3
+ requires = ["setuptools", "wheel"]
4
+
5
+ [project]
6
+ name = "funlib.persistence"
7
+ description = "Interfaces for data (arrays and graphs) and storage formats (databases and file formats)"
8
+ license = { text = "MIT" }
9
+ readme = "README.md"
10
+ authors = [
11
+ { name = "William Patton", email = "pattonw@hhmi.org" },
12
+ { name = "Jan Funke", email = "funkej@hhmi.org" },
13
+ ]
14
+ dynamic = ['version']
15
+
16
+ requires-python = ">=3.11"
17
+ classifiers = ["Programming Language :: Python :: 3"]
18
+ keywords = []
19
+
20
+ dependencies = [
21
+ "zarr>=3,<4",
22
+ "iohub>=0.3.0a5",
23
+ "funlib.geometry>=0.3.0",
24
+ "networkx>=3.0.0",
25
+ "pymongo>=4.0.0",
26
+ "numpy>=2.0.0",
27
+ "pydantic>=2.0.0",
28
+ "dask>=2024.0.0",
29
+ "toml>=0.10.0",
30
+ "psycopg2-binary>=2.9.11",
31
+ ]
32
+
33
+ [tool.setuptools.dynamic]
34
+ version = { attr = "funlib.persistence.__version__" }
35
+
36
+ [project.urls]
37
+ Documentation = "https://funkelab.github.io/funlib.persistence"
38
+ Source = "https://github.com/funkelab/funlib.persistence"
39
+ Issues = "https://github.com/funkelab/funlib.persistence/issues"
40
+
41
+ [dependency-groups]
42
+ # Installed by `uv sync` (PEP 735); not part of the published package.
43
+ # ruff and ty versions come from uv.lock; `uv lock --upgrade` moves them.
44
+ dev = [
45
+ "coverage>=7.7.1",
46
+ "pytest>=8.3.5",
47
+ "pytest-cov>=6",
48
+ "pytest-mock>=3.14.0",
49
+ "ruff>=0.17.3",
50
+ "ty>=0.0.96",
51
+ "types-psycopg2", # psycopg2 ships no inline types; ty reads this stub package
52
+ ]
53
+ docs = [
54
+ "sphinx>=8",
55
+ "sphinx-autodoc-typehints>=2",
56
+ "autodoc_pydantic>=2",
57
+ "myst-parser>=4",
58
+ "pydata-sphinx-theme>=0.16",
59
+ ]
60
+
61
+ [tool.setuptools.package-data]
62
+ "funlib.persistence" = ["py.typed"]
63
+
64
+ [tool.ty.src]
65
+ include = ["src", "tests", "benchmarks"]
66
+ exclude = [] # only ever shrinks
67
+
68
+ [tool.ruff]
69
+ line-length = 88
70
+ extend-exclude = ["docs/build", "*.md"]
71
+
72
+ [tool.ruff.lint]
73
+ select = ["E", "F", "W", "I", "UP", "B"]
74
+ ignore = ["E501"] # the formatter owns line length
75
+
76
+ [tool.ruff.lint.per-file-ignores]
77
+ "tutorials/**/*.py" = ["E402"] # jupytext cells import mid-file
78
+ "examples/**/*.py" = ["E402"]
79
+ "__init__.py" = ["F401"] # re-exports
80
+
81
+ [tool.pytest.ini_options]
82
+ testpaths = ["tests"]
83
+ addopts = "--cov --cov-report=term-missing"
84
+
85
+ [tool.coverage.run]
86
+ branch = true
87
+ source = ["funlib.persistence"]
88
+
89
+ [tool.coverage.report]
90
+ # Floor measured from the first branch-enabled run; raise it when measured
91
+ # coverage is two or more points ahead, never lower it without saying why.
92
+ fail_under = 75
93
+ show_missing = true
94
+ skip_covered = true
@@ -1,4 +1,4 @@
1
1
  from .arrays import Array, open_ds, prepare_ds, open_ome_ds, prepare_ome_ds # noqa
2
2
 
3
- __version__ = "0.8.0"
3
+ __version__ = "0.9.0"
4
4
  __version_info__ = tuple(int(i) for i in __version__.split("."))
@@ -1,7 +1,7 @@
1
1
  import logging
2
2
  from collections.abc import Sequence
3
3
  from functools import reduce
4
- from typing import Any, Optional, Union
4
+ from typing import Any
5
5
 
6
6
  import dask.array as da
7
7
  import numpy as np
@@ -93,15 +93,15 @@ class Array(Freezable):
93
93
  def __init__(
94
94
  self,
95
95
  data,
96
- offset: Optional[Sequence[int]] = None,
97
- voxel_size: Optional[Sequence[int]] = None,
98
- axis_names: Optional[Sequence[str]] = None,
99
- units: Optional[Sequence[str]] = None,
100
- types: Optional[Sequence[str]] = None,
101
- chunks: Optional[Union[int, Sequence[int], str]] = "auto",
102
- lazy_op: Optional[LazyOp] = None,
96
+ offset: Sequence[int] | None = None,
97
+ voxel_size: Sequence[int] | None = None,
98
+ axis_names: Sequence[str] | None = None,
99
+ units: Sequence[str] | None = None,
100
+ types: Sequence[str] | None = None,
101
+ chunks: int | Sequence[int] | str | None = "auto",
102
+ lazy_op: LazyOp | None = None,
103
103
  strict_metadata: bool = False,
104
- scheduler: Optional[str] = None,
104
+ scheduler: str | None = None,
105
105
  ):
106
106
  self.scheduler = scheduler
107
107
  if not isinstance(data, da.Array):
@@ -149,7 +149,7 @@ class Array(Freezable):
149
149
  return self._attrs
150
150
 
151
151
  @property
152
- def chunk_shape(self) -> Optional[Coordinate]:
152
+ def chunk_shape(self) -> Coordinate | None:
153
153
  """The write-safe chunk shape: the shard shape if sharded, else the chunk
154
154
  shape (an ``open_ds(chunks=...)`` override is reported as given). ``None``
155
155
  for a view that is not writeable or that lazy operations moved off the
@@ -160,14 +160,14 @@ class Array(Freezable):
160
160
  return shards if shards is not None else self._dask_chunk_shape()
161
161
 
162
162
  @property
163
- def inner_chunk_shape(self) -> Optional[Coordinate]:
163
+ def inner_chunk_shape(self) -> Coordinate | None:
164
164
  """Chunk shape for efficient reads. Can be smaller than the write-safe
165
165
  :attr:`chunk_shape` in the case of sharding. ``None`` for a view that
166
166
  lazy operations moved off the stored chunk grid."""
167
167
  return self._dask_chunk_shape() if self._on_chunk_grid else None
168
168
 
169
169
  @property
170
- def shard_shape(self) -> Optional[Coordinate]:
170
+ def shard_shape(self) -> Coordinate | None:
171
171
  """The shard shape in the case of sharded datasets. ``None`` for non
172
172
  sharded or unwritable datasets, or off the stored chunk grid. See
173
173
  https://zarr.readthedocs.io/en/stable/user-guide/performance.html#sharding
@@ -176,7 +176,9 @@ class Array(Freezable):
176
176
  if shards is None or not self.is_writeable or not self._on_chunk_grid:
177
177
  return None
178
178
  return Coordinate(
179
- s for s, uncollapsed in zip(shards, self._uncollapsed_dims) if uncollapsed
179
+ s
180
+ for s, uncollapsed in zip(shards, self._uncollapsed_dims, strict=False)
181
+ if uncollapsed
180
182
  )
181
183
 
182
184
  def _dask_chunk_shape(self) -> Coordinate:
@@ -196,9 +198,13 @@ class Array(Freezable):
196
198
  unit = self.chunk_shape
197
199
  if unit is None:
198
200
  return False
199
- physical = [c for c, t in zip(unit, self.types) if t in ["space", "time"]]
201
+ physical = [
202
+ c
203
+ for c, t in zip(unit, self.types, strict=False)
204
+ if t in ["space", "time"]
205
+ ]
200
206
  begin = (lazy_op.offset - self.offset) / self.voxel_size
201
- return all(b % c == 0 for b, c in zip(begin, physical))
207
+ return all(b % c == 0 for b, c in zip(begin, physical, strict=False))
202
208
  if not isinstance(lazy_op, tuple):
203
209
  lazy_op = (lazy_op,)
204
210
  for a in lazy_op:
@@ -220,7 +226,9 @@ class Array(Freezable):
220
226
  if physical:
221
227
  return [
222
228
  x
223
- for x, t in zip(self._uncollapsed_dims, self._metadata.types)
229
+ for x, t in zip(
230
+ self._uncollapsed_dims, self._metadata.types, strict=False
231
+ )
224
232
  if t in ["space", "time"]
225
233
  ]
226
234
  else:
@@ -336,7 +344,7 @@ class Array(Freezable):
336
344
  # previous in-place walk re-counted uncollapsed dims after each collapse,
337
345
  # so a tuple with two or more ints collapsed the wrong dimensions.
338
346
  dims = [i for i, uc in enumerate(self._uncollapsed_dims) if uc]
339
- for a, i in zip(lazy_op, dims):
347
+ for a, i in zip(lazy_op, dims, strict=False):
340
348
  if isinstance(a, int):
341
349
  self._uncollapsed_dims[i] = False
342
350
  self.data = self.data[lazy_op]
@@ -353,7 +361,7 @@ class Array(Freezable):
353
361
  elif callable(lazy_op):
354
362
  self.data = lazy_op(self.data)
355
363
  else:
356
- raise Exception(
364
+ raise TypeError(
357
365
  f"LazyOp {lazy_op} is not a supported lazy_op. "
358
366
  f"Supported lazy_ops are: {LazyOp}"
359
367
  )
@@ -384,15 +392,15 @@ class Array(Freezable):
384
392
 
385
393
  Args:
386
394
 
387
- key (`class:Roi` or `class:Coordinate` or any numpy compatible key):
395
+ key (:class:`Roi` or :class:`Coordinate` or any numpy compatible key):
388
396
 
389
397
  The `Roi`, `Coordinate`, or numpy compatible key indicating the data
390
398
  you want to read.
391
399
 
392
400
  Returns:
393
401
 
394
- A numpy array containing the data requested by the key. If using a `class:Roi`
395
- or `class:Coordinate`, the data will be returned in world units. If using
402
+ A numpy array containing the data requested by the key. If using a :class:`Roi`
403
+ or :class:`Coordinate`, the data will be returned in world units. If using
396
404
  a numpy compatible key, the data will be sliced as a regular numpy array.
397
405
  """
398
406
 
@@ -401,8 +409,7 @@ class Array(Freezable):
401
409
 
402
410
  if not self.roi.contains(roi):
403
411
  raise IndexError(
404
- "Requested roi %s is not contained in this array %s."
405
- % (roi, self.roi)
412
+ f"Requested roi {roi} is not contained in this array {self.roi}."
406
413
  )
407
414
 
408
415
  return self.data[self.__slices(roi, use_lazy_slices=False)].compute(
@@ -426,7 +433,7 @@ class Array(Freezable):
426
433
 
427
434
  Args:
428
435
 
429
- key (`class:Roi` or any numpy compatible key):
436
+ key (:class:`Roi` or any numpy compatible key):
430
437
 
431
438
  The region to write to. Can be a `Roi` for world-unit indexing,
432
439
  or any numpy-compatible key (e.g. ``np.s_[:]``, a slice, a tuple
@@ -444,8 +451,7 @@ class Array(Freezable):
444
451
 
445
452
  if not self.roi.contains(roi):
446
453
  raise IndexError(
447
- "Requested roi %s is not contained in this array %s."
448
- % (roi, self.roi)
454
+ f"Requested roi {roi} is not contained in this array {self.roi}."
449
455
  )
450
456
 
451
457
  region_slices = self.__slices(roi)
@@ -481,7 +487,7 @@ class Array(Freezable):
481
487
 
482
488
  Args:
483
489
 
484
- roi (`class:Roi`, optional):
490
+ roi (:class:`Roi`, optional):
485
491
 
486
492
  If given, copy only the data represented by this ROI. This is
487
493
  equivalent to::
@@ -527,9 +533,8 @@ class Array(Freezable):
527
533
  assert begin_align_with_chunks and (
528
534
  shape_align_with_chunks or end_of_array
529
535
  ), (
530
- "ROI %s (in voxels: %s) does not align with chunks of "
531
- "size %s (mismatch in dimension %d)"
532
- % (roi, voxel_roi, chunk_shape, d)
536
+ f"ROI {roi} (in voxels: {voxel_roi}) does not align with "
537
+ f"chunks of size {chunk_shape} (mismatch in dimension {d})"
533
538
  )
534
539
 
535
540
  roi_slices = tuple(
@@ -556,7 +561,7 @@ class Array(Freezable):
556
561
  elif isinstance(lazy_op, list) and all([isinstance(a, int) for a in lazy_op]):
557
562
  return True
558
563
  elif isinstance(lazy_op, tuple) and all(
559
- [self._is_slice(a, writeable) for a in lazy_op] # type: ignore[arg-type] # ty can't narrow parameterized tuple iteration
564
+ [self._is_slice(a, writeable) for a in lazy_op] # ty: ignore[invalid-argument-type] # tuple elements are not narrowed
560
565
  ):
561
566
  return True
562
567
  elif (
@@ -622,7 +627,7 @@ class Array(Freezable):
622
627
 
623
628
  def to_world_space(
624
629
  self, pixel_loc: Roi | Coordinate | Sequence[int | float] | np.ndarray
625
- ) -> Roi | Coordinate:
630
+ ) -> Roi | Coordinate | np.ndarray:
626
631
  """Convert a point or roi from pixel space in this array to the world
627
632
  coordinate system defined by this array's roi and voxel size.
628
633
  Works on Coordiantes and Rois, as well as sequences of ints or floats.
@@ -1,7 +1,7 @@
1
1
  import inspect
2
2
  import logging
3
3
  from collections.abc import Sequence
4
- from typing import Any, Literal, Optional, Union
4
+ from typing import Any, Literal
5
5
 
6
6
  import numpy as np
7
7
  import zarr
@@ -15,24 +15,19 @@ from .metadata import MetaDataFormat, get_default_metadata_format
15
15
  logger = logging.getLogger(__name__)
16
16
 
17
17
  DEFAULT_INNER_CHUNK = 64
18
- """The chunk inside each shard when `prepare_ds(sharding=True)` is given none, per axis
19
- (the whole axis where the shard is smaller)."""
18
+ """The target chunk inside each shard when `prepare_ds(sharding=True)` is given none, per
19
+ axis: the largest divisor of the shard's extent that is at most this (the whole axis where
20
+ the shard is smaller)."""
20
21
 
21
22
 
22
23
  def _default_inner_chunk(chunk_shape: Sequence[int]) -> tuple[int, ...]:
23
24
  inner = []
24
25
  for c in chunk_shape:
25
26
  c = int(c)
26
- if c <= DEFAULT_INNER_CHUNK:
27
- inner.append(c)
28
- elif c % DEFAULT_INNER_CHUNK == 0:
29
- inner.append(DEFAULT_INNER_CHUNK)
30
- else:
31
- raise ValueError(
32
- f"chunk_shape {tuple(int(x) for x in chunk_shape)}: axis extent {c} is "
33
- f"larger than the default inner chunk {DEFAULT_INNER_CHUNK} and not a "
34
- "multiple of it; pass inner_chunk_shape explicitly"
35
- )
27
+ i = max(d for d in range(1, min(c, DEFAULT_INNER_CHUNK) + 1) if c % d == 0)
28
+ # an extent with no divisor near the target (a prime, twice a prime) keeps the
29
+ # whole axis as one inner chunk rather than splitting into slivers
30
+ inner.append(i if i >= DEFAULT_INNER_CHUNK // 4 else c)
36
31
  return tuple(inner)
37
32
 
38
33
 
@@ -55,15 +50,15 @@ def _location_kwargs(kwargs: dict) -> dict:
55
50
  def open_ds(
56
51
  store,
57
52
  mode: Literal["r", "r+", "a", "w", "w-"] = "r",
58
- metadata_format: Optional[MetaDataFormat] = None,
59
- offset: Optional[Sequence[int]] = None,
60
- voxel_size: Optional[Sequence[int]] = None,
61
- axis_names: Optional[Sequence[str]] = None,
62
- units: Optional[Sequence[str]] = None,
63
- types: Optional[Sequence[str]] = None,
64
- chunks: Optional[Union[int, Sequence[int], str]] = "strict",
53
+ metadata_format: MetaDataFormat | None = None,
54
+ offset: Sequence[int] | None = None,
55
+ voxel_size: Sequence[int] | None = None,
56
+ axis_names: Sequence[str] | None = None,
57
+ units: Sequence[str] | None = None,
58
+ types: Sequence[str] | None = None,
59
+ chunks: int | Sequence[int] | str | None = "strict",
65
60
  strict_metadata: bool = False,
66
- scheduler: Optional[str] = None,
61
+ scheduler: str | None = None,
67
62
  **kwargs,
68
63
  ) -> Array:
69
64
  """
@@ -171,11 +166,11 @@ def open_ds(
171
166
 
172
167
 
173
168
  def _validate_layout(
174
- chunk_shape: Optional[Coordinate],
169
+ chunk_shape: Coordinate | None,
175
170
  sharding: bool,
176
- inner_chunk_shape: Optional[Coordinate],
171
+ inner_chunk_shape: Coordinate | None,
177
172
  kwargs: dict,
178
- ) -> Optional[tuple[int, ...]]:
173
+ ) -> tuple[int, ...] | None:
179
174
  """Check the chunk arguments against each other. Returns the chunk inside each
180
175
  shard when sharding, else ``None``."""
181
176
  if "shard_shape" in kwargs:
@@ -211,7 +206,7 @@ def _validate_layout(
211
206
  f"inner_chunk_shape {inner} and chunk_shape {tuple(chunk_shape)} differ "
212
207
  "in dimensions"
213
208
  )
214
- if any(int(c) % i != 0 for c, i in zip(chunk_shape, inner)):
209
+ if any(int(c) % i != 0 for c, i in zip(chunk_shape, inner, strict=False)):
215
210
  raise ValueError(
216
211
  f"chunk_shape {tuple(chunk_shape)} (the shard) must be a whole number of "
217
212
  f"inner_chunk_shape {inner} per axis"
@@ -232,7 +227,7 @@ def _normalise_sub_location(kwargs: dict) -> None:
232
227
  kwargs["path"] = name
233
228
 
234
229
 
235
- def _find_existing(store, kwargs: dict) -> tuple[Optional[Array], bool]:
230
+ def _find_existing(store, kwargs: dict) -> tuple[Array | None, bool]:
236
231
  """The array at the location, or ``None``; and whether a stray placeholder (the
237
232
  empty, attr-less group an interrupted sharded ``w-`` create leaves) is there
238
233
  instead."""
@@ -347,7 +342,7 @@ def _create_sharded(
347
342
  )
348
343
 
349
344
 
350
- def _write_metadata(ds: zarr.Array, metadata, custom_metadata: Optional[dict]) -> None:
345
+ def _write_metadata(ds: zarr.Array, metadata, custom_metadata: dict | None) -> None:
351
346
  fmt = get_default_metadata_format()
352
347
  attrs = {
353
348
  fmt.axis_names_attr: metadata.axis_names,
@@ -365,18 +360,18 @@ def _write_metadata(ds: zarr.Array, metadata, custom_metadata: Optional[dict]) -
365
360
  def prepare_ds(
366
361
  store,
367
362
  shape: Sequence[int],
368
- offset: Optional[Coordinate] = None,
369
- voxel_size: Optional[Coordinate] = None,
370
- axis_names: Optional[Sequence[str]] = None,
371
- units: Optional[Sequence[str]] = None,
372
- types: Optional[Sequence[str]] = None,
373
- chunk_shape: Optional[Coordinate] = None,
363
+ offset: Coordinate | None = None,
364
+ voxel_size: Coordinate | None = None,
365
+ axis_names: Sequence[str] | None = None,
366
+ units: Sequence[str] | None = None,
367
+ types: Sequence[str] | None = None,
368
+ chunk_shape: Coordinate | None = None,
374
369
  dtype: DTypeLike = np.float32,
375
370
  mode: Literal["r", "r+", "a", "w", "w-"] = "a",
376
371
  custom_metadata: dict[str, Any] | None = None,
377
372
  sharding: bool = False,
378
- inner_chunk_shape: Optional[Coordinate] = None,
379
- scheduler: Optional[str] = None,
373
+ inner_chunk_shape: Coordinate | None = None,
374
+ scheduler: str | None = None,
380
375
  **kwargs,
381
376
  ) -> Array:
382
377
  """Prepare a Zarr or N5 dataset.
@@ -453,7 +448,9 @@ def prepare_ds(
453
448
  inner_chunk_shape:
454
449
 
455
450
  The efficient-read chunk shape. Can be less than the write-safe chunk shape in
456
- the case of sharding; defaults to `DEFAULT_INNER_CHUNK` (64) per axis there.
451
+ the case of sharding; defaults per axis to the largest divisor of `chunk_shape`
452
+ that is at most `DEFAULT_INNER_CHUNK` (64), or the whole axis if that divisor
453
+ is below 16.
457
454
  Unsharded it can only restate `chunk_shape`. Given against an existing dataset
458
455
  it must match that dataset's chunks.
459
456
 
@@ -476,6 +473,15 @@ def prepare_ds(
476
473
  Returns:
477
474
 
478
475
  A :class:`Array` pointing to the newly created dataset.
476
+
477
+ Raises:
478
+
479
+ A combination of `chunk_shape`, `sharding` and `inner_chunk_shape` that cannot
480
+ be created raises ``ValueError``, and a keyword argument the create does not
481
+ take raises ``TypeError``. With `sharding=True` this also covers a value
482
+ `zarr.create_array` refuses. All of these are refused before the location is
483
+ touched: nothing is created at `store` and nothing already there is changed,
484
+ so the corrected call can be retried in any mode.
479
485
  """
480
486
 
481
487
  metadata_format = get_default_metadata_format()
@@ -1,9 +1,9 @@
1
- class Freezable(object):
1
+ class Freezable:
2
2
  __isfrozen = False
3
3
 
4
4
  def __setattr__(self, key, value):
5
5
  if self.__isfrozen and not hasattr(self, key):
6
- raise TypeError("%r is frozen, you can't add attributes to it" % self)
6
+ raise TypeError(f"{self!r} is frozen, you can't add attributes to it")
7
7
  object.__setattr__(self, key, value)
8
8
 
9
9
  def freeze(self):
@@ -0,0 +1,6 @@
1
+ from collections.abc import Callable
2
+
3
+ import numpy as np
4
+ from funlib.geometry import Roi
5
+
6
+ LazyOp = slice | int | tuple[int | slice | list[int] | np.ndarray, ...] | Callable | Roi