funlib.persistence 0.8.0__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- funlib_persistence-0.9.0/PKG-INFO +59 -0
- funlib_persistence-0.9.0/README.md +34 -0
- funlib_persistence-0.9.0/pyproject.toml +94 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/__init__.py +1 -1
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/array.py +37 -32
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/datasets.py +42 -36
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/freezable.py +2 -2
- funlib_persistence-0.9.0/src/funlib/persistence/arrays/lazy_ops.py +6 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/metadata.py +55 -33
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/ome_datasets.py +7 -3
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/__init__.py +1 -0
- funlib_persistence-0.9.0/src/funlib/persistence/graphs/csr_graph_database.py +1070 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/graph_database.py +15 -15
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/pgsql_graph_database.py +18 -15
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/sql_graph_database.py +51 -53
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/sqlite_graph_database.py +14 -12
- funlib_persistence-0.9.0/src/funlib.persistence.egg-info/PKG-INFO +59 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/SOURCES.txt +2 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/requires.txt +0 -10
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_array.py +92 -13
- funlib_persistence-0.9.0/tests/test_config.py +102 -0
- funlib_persistence-0.9.0/tests/test_csr_graph_database.py +244 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_datasets.py +337 -472
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_graph.py +261 -304
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_metadata.py +53 -1
- funlib_persistence-0.8.0/PKG-INFO +0 -46
- funlib_persistence-0.8.0/README.md +0 -15
- funlib_persistence-0.8.0/pyproject.toml +0 -59
- funlib_persistence-0.8.0/src/funlib/persistence/arrays/lazy_ops.py +0 -6
- funlib_persistence-0.8.0/src/funlib.persistence.egg-info/PKG-INFO +0 -46
- funlib_persistence-0.8.0/tests/test_config.py +0 -84
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/LICENSE +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/setup.cfg +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/__init__.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/utils.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/py.typed +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/types.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/dependency_links.txt +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/top_level.txt +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_pgsql_graph_database.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/tests/test_slices.py +0 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: funlib.persistence
|
|
3
|
+
Version: 0.9.0
|
|
4
|
+
Summary: Interfaces for data (arrays and graphs) and storage formats (databases and file formats)
|
|
5
|
+
Author-email: William Patton <pattonw@hhmi.org>, Jan Funke <funkej@hhmi.org>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Documentation, https://funkelab.github.io/funlib.persistence
|
|
8
|
+
Project-URL: Source, https://github.com/funkelab/funlib.persistence
|
|
9
|
+
Project-URL: Issues, https://github.com/funkelab/funlib.persistence/issues
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Requires-Python: >=3.11
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: zarr<4,>=3
|
|
15
|
+
Requires-Dist: iohub>=0.3.0a5
|
|
16
|
+
Requires-Dist: funlib.geometry>=0.3.0
|
|
17
|
+
Requires-Dist: networkx>=3.0.0
|
|
18
|
+
Requires-Dist: pymongo>=4.0.0
|
|
19
|
+
Requires-Dist: numpy>=2.0.0
|
|
20
|
+
Requires-Dist: pydantic>=2.0.0
|
|
21
|
+
Requires-Dist: dask>=2024.0.0
|
|
22
|
+
Requires-Dist: toml>=0.10.0
|
|
23
|
+
Requires-Dist: psycopg2-binary>=2.9.11
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
[](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml)
|
|
27
|
+
[](https://github.com/funkelab/funlib.persistence/actions/workflows/lint.yaml)
|
|
28
|
+
[](https://funkelab.github.io/funlib.persistence)
|
|
29
|
+
[](https://pypi.org/project/funlib.persistence/)
|
|
30
|
+
|
|
31
|
+
# funlib.persistence
|
|
32
|
+
Interfaces for data (arrays and graphs) and storage formats (databases and file formats)
|
|
33
|
+
|
|
34
|
+
**Documentation:** https://funkelab.github.io/funlib.persistence
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install funlib.persistence # or: uv add funlib.persistence
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Contributing
|
|
43
|
+
|
|
44
|
+
The commands below are the ones CI runs, so a green local run means a green PR.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
git clone https://github.com/funkelab/funlib.persistence
|
|
48
|
+
cd funlib.persistence
|
|
49
|
+
uv sync # Python, dependencies, dev tools (from uv.lock)
|
|
50
|
+
uv run pytest # tests with coverage
|
|
51
|
+
uv run pytest --no-cov tests/<file>.py # one file, without the coverage floor
|
|
52
|
+
uv run ruff check . && uv run ruff format --check . # lint and formatting
|
|
53
|
+
uv run ty check src tests # types
|
|
54
|
+
uv run --group docs sphinx-build -W docs/source docs/build/html # docs
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
The graph-database tests need a PostgreSQL server (CI starts one as a service, `PGUSER=postgres`); they are skipped when none is reachable.
|
|
58
|
+
|
|
59
|
+
`uv.lock` pins the development environment; the `lock-upgrade` workflow refreshes it monthly and opens a pull request.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml)
|
|
2
|
+
[](https://github.com/funkelab/funlib.persistence/actions/workflows/lint.yaml)
|
|
3
|
+
[](https://funkelab.github.io/funlib.persistence)
|
|
4
|
+
[](https://pypi.org/project/funlib.persistence/)
|
|
5
|
+
|
|
6
|
+
# funlib.persistence
|
|
7
|
+
Interfaces for data (arrays and graphs) and storage formats (databases and file formats)
|
|
8
|
+
|
|
9
|
+
**Documentation:** https://funkelab.github.io/funlib.persistence
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install funlib.persistence # or: uv add funlib.persistence
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Contributing
|
|
18
|
+
|
|
19
|
+
The commands below are the ones CI runs, so a green local run means a green PR.
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
git clone https://github.com/funkelab/funlib.persistence
|
|
23
|
+
cd funlib.persistence
|
|
24
|
+
uv sync # Python, dependencies, dev tools (from uv.lock)
|
|
25
|
+
uv run pytest # tests with coverage
|
|
26
|
+
uv run pytest --no-cov tests/<file>.py # one file, without the coverage floor
|
|
27
|
+
uv run ruff check . && uv run ruff format --check . # lint and formatting
|
|
28
|
+
uv run ty check src tests # types
|
|
29
|
+
uv run --group docs sphinx-build -W docs/source docs/build/html # docs
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The graph-database tests need a PostgreSQL server (CI starts one as a service, `PGUSER=postgres`); they are skipped when none is reachable.
|
|
33
|
+
|
|
34
|
+
`uv.lock` pins the development environment; the `lock-upgrade` workflow refreshes it monthly and opens a pull request.
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
build-backend = "setuptools.build_meta"
|
|
3
|
+
requires = ["setuptools", "wheel"]
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "funlib.persistence"
|
|
7
|
+
description = "Interfaces for data (arrays and graphs) and storage formats (databases and file formats)"
|
|
8
|
+
license = { text = "MIT" }
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [
|
|
11
|
+
{ name = "William Patton", email = "pattonw@hhmi.org" },
|
|
12
|
+
{ name = "Jan Funke", email = "funkej@hhmi.org" },
|
|
13
|
+
]
|
|
14
|
+
dynamic = ['version']
|
|
15
|
+
|
|
16
|
+
requires-python = ">=3.11"
|
|
17
|
+
classifiers = ["Programming Language :: Python :: 3"]
|
|
18
|
+
keywords = []
|
|
19
|
+
|
|
20
|
+
dependencies = [
|
|
21
|
+
"zarr>=3,<4",
|
|
22
|
+
"iohub>=0.3.0a5",
|
|
23
|
+
"funlib.geometry>=0.3.0",
|
|
24
|
+
"networkx>=3.0.0",
|
|
25
|
+
"pymongo>=4.0.0",
|
|
26
|
+
"numpy>=2.0.0",
|
|
27
|
+
"pydantic>=2.0.0",
|
|
28
|
+
"dask>=2024.0.0",
|
|
29
|
+
"toml>=0.10.0",
|
|
30
|
+
"psycopg2-binary>=2.9.11",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.dynamic]
|
|
34
|
+
version = { attr = "funlib.persistence.__version__" }
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Documentation = "https://funkelab.github.io/funlib.persistence"
|
|
38
|
+
Source = "https://github.com/funkelab/funlib.persistence"
|
|
39
|
+
Issues = "https://github.com/funkelab/funlib.persistence/issues"
|
|
40
|
+
|
|
41
|
+
[dependency-groups]
|
|
42
|
+
# Installed by `uv sync` (PEP 735); not part of the published package.
|
|
43
|
+
# ruff and ty versions come from uv.lock; `uv lock --upgrade` moves them.
|
|
44
|
+
dev = [
|
|
45
|
+
"coverage>=7.7.1",
|
|
46
|
+
"pytest>=8.3.5",
|
|
47
|
+
"pytest-cov>=6",
|
|
48
|
+
"pytest-mock>=3.14.0",
|
|
49
|
+
"ruff>=0.17.3",
|
|
50
|
+
"ty>=0.0.96",
|
|
51
|
+
"types-psycopg2", # psycopg2 ships no inline types; ty reads this stub package
|
|
52
|
+
]
|
|
53
|
+
docs = [
|
|
54
|
+
"sphinx>=8",
|
|
55
|
+
"sphinx-autodoc-typehints>=2",
|
|
56
|
+
"autodoc_pydantic>=2",
|
|
57
|
+
"myst-parser>=4",
|
|
58
|
+
"pydata-sphinx-theme>=0.16",
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
[tool.setuptools.package-data]
|
|
62
|
+
"funlib.persistence" = ["py.typed"]
|
|
63
|
+
|
|
64
|
+
[tool.ty.src]
|
|
65
|
+
include = ["src", "tests", "benchmarks"]
|
|
66
|
+
exclude = [] # only ever shrinks
|
|
67
|
+
|
|
68
|
+
[tool.ruff]
|
|
69
|
+
line-length = 88
|
|
70
|
+
extend-exclude = ["docs/build", "*.md"]
|
|
71
|
+
|
|
72
|
+
[tool.ruff.lint]
|
|
73
|
+
select = ["E", "F", "W", "I", "UP", "B"]
|
|
74
|
+
ignore = ["E501"] # the formatter owns line length
|
|
75
|
+
|
|
76
|
+
[tool.ruff.lint.per-file-ignores]
|
|
77
|
+
"tutorials/**/*.py" = ["E402"] # jupytext cells import mid-file
|
|
78
|
+
"examples/**/*.py" = ["E402"]
|
|
79
|
+
"__init__.py" = ["F401"] # re-exports
|
|
80
|
+
|
|
81
|
+
[tool.pytest.ini_options]
|
|
82
|
+
testpaths = ["tests"]
|
|
83
|
+
addopts = "--cov --cov-report=term-missing"
|
|
84
|
+
|
|
85
|
+
[tool.coverage.run]
|
|
86
|
+
branch = true
|
|
87
|
+
source = ["funlib.persistence"]
|
|
88
|
+
|
|
89
|
+
[tool.coverage.report]
|
|
90
|
+
# Floor measured from the first branch-enabled run; raise it when measured
|
|
91
|
+
# coverage is two or more points ahead, never lower it without saying why.
|
|
92
|
+
fail_under = 75
|
|
93
|
+
show_missing = true
|
|
94
|
+
skip_covered = true
|
{funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/array.py
RENAMED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from collections.abc import Sequence
|
|
3
3
|
from functools import reduce
|
|
4
|
-
from typing import Any
|
|
4
|
+
from typing import Any
|
|
5
5
|
|
|
6
6
|
import dask.array as da
|
|
7
7
|
import numpy as np
|
|
@@ -93,15 +93,15 @@ class Array(Freezable):
|
|
|
93
93
|
def __init__(
|
|
94
94
|
self,
|
|
95
95
|
data,
|
|
96
|
-
offset:
|
|
97
|
-
voxel_size:
|
|
98
|
-
axis_names:
|
|
99
|
-
units:
|
|
100
|
-
types:
|
|
101
|
-
chunks:
|
|
102
|
-
lazy_op:
|
|
96
|
+
offset: Sequence[int] | None = None,
|
|
97
|
+
voxel_size: Sequence[int] | None = None,
|
|
98
|
+
axis_names: Sequence[str] | None = None,
|
|
99
|
+
units: Sequence[str] | None = None,
|
|
100
|
+
types: Sequence[str] | None = None,
|
|
101
|
+
chunks: int | Sequence[int] | str | None = "auto",
|
|
102
|
+
lazy_op: LazyOp | None = None,
|
|
103
103
|
strict_metadata: bool = False,
|
|
104
|
-
scheduler:
|
|
104
|
+
scheduler: str | None = None,
|
|
105
105
|
):
|
|
106
106
|
self.scheduler = scheduler
|
|
107
107
|
if not isinstance(data, da.Array):
|
|
@@ -149,7 +149,7 @@ class Array(Freezable):
|
|
|
149
149
|
return self._attrs
|
|
150
150
|
|
|
151
151
|
@property
|
|
152
|
-
def chunk_shape(self) ->
|
|
152
|
+
def chunk_shape(self) -> Coordinate | None:
|
|
153
153
|
"""The write-safe chunk shape: the shard shape if sharded, else the chunk
|
|
154
154
|
shape (an ``open_ds(chunks=...)`` override is reported as given). ``None``
|
|
155
155
|
for a view that is not writeable or that lazy operations moved off the
|
|
@@ -160,14 +160,14 @@ class Array(Freezable):
|
|
|
160
160
|
return shards if shards is not None else self._dask_chunk_shape()
|
|
161
161
|
|
|
162
162
|
@property
|
|
163
|
-
def inner_chunk_shape(self) ->
|
|
163
|
+
def inner_chunk_shape(self) -> Coordinate | None:
|
|
164
164
|
"""Chunk shape for efficient reads. Can be smaller than the write-safe
|
|
165
165
|
:attr:`chunk_shape` in the case of sharding. ``None`` for a view that
|
|
166
166
|
lazy operations moved off the stored chunk grid."""
|
|
167
167
|
return self._dask_chunk_shape() if self._on_chunk_grid else None
|
|
168
168
|
|
|
169
169
|
@property
|
|
170
|
-
def shard_shape(self) ->
|
|
170
|
+
def shard_shape(self) -> Coordinate | None:
|
|
171
171
|
"""The shard shape in the case of sharded datasets. ``None`` for non
|
|
172
172
|
sharded or unwritable datasets, or off the stored chunk grid. See
|
|
173
173
|
https://zarr.readthedocs.io/en/stable/user-guide/performance.html#sharding
|
|
@@ -176,7 +176,9 @@ class Array(Freezable):
|
|
|
176
176
|
if shards is None or not self.is_writeable or not self._on_chunk_grid:
|
|
177
177
|
return None
|
|
178
178
|
return Coordinate(
|
|
179
|
-
s
|
|
179
|
+
s
|
|
180
|
+
for s, uncollapsed in zip(shards, self._uncollapsed_dims, strict=False)
|
|
181
|
+
if uncollapsed
|
|
180
182
|
)
|
|
181
183
|
|
|
182
184
|
def _dask_chunk_shape(self) -> Coordinate:
|
|
@@ -196,9 +198,13 @@ class Array(Freezable):
|
|
|
196
198
|
unit = self.chunk_shape
|
|
197
199
|
if unit is None:
|
|
198
200
|
return False
|
|
199
|
-
physical = [
|
|
201
|
+
physical = [
|
|
202
|
+
c
|
|
203
|
+
for c, t in zip(unit, self.types, strict=False)
|
|
204
|
+
if t in ["space", "time"]
|
|
205
|
+
]
|
|
200
206
|
begin = (lazy_op.offset - self.offset) / self.voxel_size
|
|
201
|
-
return all(b % c == 0 for b, c in zip(begin, physical))
|
|
207
|
+
return all(b % c == 0 for b, c in zip(begin, physical, strict=False))
|
|
202
208
|
if not isinstance(lazy_op, tuple):
|
|
203
209
|
lazy_op = (lazy_op,)
|
|
204
210
|
for a in lazy_op:
|
|
@@ -220,7 +226,9 @@ class Array(Freezable):
|
|
|
220
226
|
if physical:
|
|
221
227
|
return [
|
|
222
228
|
x
|
|
223
|
-
for x, t in zip(
|
|
229
|
+
for x, t in zip(
|
|
230
|
+
self._uncollapsed_dims, self._metadata.types, strict=False
|
|
231
|
+
)
|
|
224
232
|
if t in ["space", "time"]
|
|
225
233
|
]
|
|
226
234
|
else:
|
|
@@ -336,7 +344,7 @@ class Array(Freezable):
|
|
|
336
344
|
# previous in-place walk re-counted uncollapsed dims after each collapse,
|
|
337
345
|
# so a tuple with two or more ints collapsed the wrong dimensions.
|
|
338
346
|
dims = [i for i, uc in enumerate(self._uncollapsed_dims) if uc]
|
|
339
|
-
for a, i in zip(lazy_op, dims):
|
|
347
|
+
for a, i in zip(lazy_op, dims, strict=False):
|
|
340
348
|
if isinstance(a, int):
|
|
341
349
|
self._uncollapsed_dims[i] = False
|
|
342
350
|
self.data = self.data[lazy_op]
|
|
@@ -353,7 +361,7 @@ class Array(Freezable):
|
|
|
353
361
|
elif callable(lazy_op):
|
|
354
362
|
self.data = lazy_op(self.data)
|
|
355
363
|
else:
|
|
356
|
-
raise
|
|
364
|
+
raise TypeError(
|
|
357
365
|
f"LazyOp {lazy_op} is not a supported lazy_op. "
|
|
358
366
|
f"Supported lazy_ops are: {LazyOp}"
|
|
359
367
|
)
|
|
@@ -384,15 +392,15 @@ class Array(Freezable):
|
|
|
384
392
|
|
|
385
393
|
Args:
|
|
386
394
|
|
|
387
|
-
key (
|
|
395
|
+
key (:class:`Roi` or :class:`Coordinate` or any numpy compatible key):
|
|
388
396
|
|
|
389
397
|
The `Roi`, `Coordinate`, or numpy compatible key indicating the data
|
|
390
398
|
you want to read.
|
|
391
399
|
|
|
392
400
|
Returns:
|
|
393
401
|
|
|
394
|
-
A numpy array containing the data requested by the key. If using a
|
|
395
|
-
or
|
|
402
|
+
A numpy array containing the data requested by the key. If using a :class:`Roi`
|
|
403
|
+
or :class:`Coordinate`, the data will be returned in world units. If using
|
|
396
404
|
a numpy compatible key, the data will be sliced as a regular numpy array.
|
|
397
405
|
"""
|
|
398
406
|
|
|
@@ -401,8 +409,7 @@ class Array(Freezable):
|
|
|
401
409
|
|
|
402
410
|
if not self.roi.contains(roi):
|
|
403
411
|
raise IndexError(
|
|
404
|
-
"Requested roi
|
|
405
|
-
% (roi, self.roi)
|
|
412
|
+
f"Requested roi {roi} is not contained in this array {self.roi}."
|
|
406
413
|
)
|
|
407
414
|
|
|
408
415
|
return self.data[self.__slices(roi, use_lazy_slices=False)].compute(
|
|
@@ -426,7 +433,7 @@ class Array(Freezable):
|
|
|
426
433
|
|
|
427
434
|
Args:
|
|
428
435
|
|
|
429
|
-
key (
|
|
436
|
+
key (:class:`Roi` or any numpy compatible key):
|
|
430
437
|
|
|
431
438
|
The region to write to. Can be a `Roi` for world-unit indexing,
|
|
432
439
|
or any numpy-compatible key (e.g. ``np.s_[:]``, a slice, a tuple
|
|
@@ -444,8 +451,7 @@ class Array(Freezable):
|
|
|
444
451
|
|
|
445
452
|
if not self.roi.contains(roi):
|
|
446
453
|
raise IndexError(
|
|
447
|
-
"Requested roi
|
|
448
|
-
% (roi, self.roi)
|
|
454
|
+
f"Requested roi {roi} is not contained in this array {self.roi}."
|
|
449
455
|
)
|
|
450
456
|
|
|
451
457
|
region_slices = self.__slices(roi)
|
|
@@ -481,7 +487,7 @@ class Array(Freezable):
|
|
|
481
487
|
|
|
482
488
|
Args:
|
|
483
489
|
|
|
484
|
-
roi (
|
|
490
|
+
roi (:class:`Roi`, optional):
|
|
485
491
|
|
|
486
492
|
If given, copy only the data represented by this ROI. This is
|
|
487
493
|
equivalent to::
|
|
@@ -527,9 +533,8 @@ class Array(Freezable):
|
|
|
527
533
|
assert begin_align_with_chunks and (
|
|
528
534
|
shape_align_with_chunks or end_of_array
|
|
529
535
|
), (
|
|
530
|
-
"ROI
|
|
531
|
-
"size
|
|
532
|
-
% (roi, voxel_roi, chunk_shape, d)
|
|
536
|
+
f"ROI {roi} (in voxels: {voxel_roi}) does not align with "
|
|
537
|
+
f"chunks of size {chunk_shape} (mismatch in dimension {d})"
|
|
533
538
|
)
|
|
534
539
|
|
|
535
540
|
roi_slices = tuple(
|
|
@@ -556,7 +561,7 @@ class Array(Freezable):
|
|
|
556
561
|
elif isinstance(lazy_op, list) and all([isinstance(a, int) for a in lazy_op]):
|
|
557
562
|
return True
|
|
558
563
|
elif isinstance(lazy_op, tuple) and all(
|
|
559
|
-
[self._is_slice(a, writeable) for a in lazy_op] #
|
|
564
|
+
[self._is_slice(a, writeable) for a in lazy_op] # ty: ignore[invalid-argument-type] # tuple elements are not narrowed
|
|
560
565
|
):
|
|
561
566
|
return True
|
|
562
567
|
elif (
|
|
@@ -622,7 +627,7 @@ class Array(Freezable):
|
|
|
622
627
|
|
|
623
628
|
def to_world_space(
|
|
624
629
|
self, pixel_loc: Roi | Coordinate | Sequence[int | float] | np.ndarray
|
|
625
|
-
) -> Roi | Coordinate:
|
|
630
|
+
) -> Roi | Coordinate | np.ndarray:
|
|
626
631
|
"""Convert a point or roi from pixel space in this array to the world
|
|
627
632
|
coordinate system defined by this array's roi and voxel size.
|
|
628
633
|
Works on Coordiantes and Rois, as well as sequences of ints or floats.
|
{funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/datasets.py
RENAMED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
import logging
|
|
3
3
|
from collections.abc import Sequence
|
|
4
|
-
from typing import Any, Literal
|
|
4
|
+
from typing import Any, Literal
|
|
5
5
|
|
|
6
6
|
import numpy as np
|
|
7
7
|
import zarr
|
|
@@ -15,24 +15,19 @@ from .metadata import MetaDataFormat, get_default_metadata_format
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
|
|
17
17
|
DEFAULT_INNER_CHUNK = 64
|
|
18
|
-
"""The chunk inside each shard when `prepare_ds(sharding=True)` is given none, per
|
|
19
|
-
|
|
18
|
+
"""The target chunk inside each shard when `prepare_ds(sharding=True)` is given none, per
|
|
19
|
+
axis: the largest divisor of the shard's extent that is at most this (the whole axis where
|
|
20
|
+
the shard is smaller)."""
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
def _default_inner_chunk(chunk_shape: Sequence[int]) -> tuple[int, ...]:
|
|
23
24
|
inner = []
|
|
24
25
|
for c in chunk_shape:
|
|
25
26
|
c = int(c)
|
|
26
|
-
if c
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
else:
|
|
31
|
-
raise ValueError(
|
|
32
|
-
f"chunk_shape {tuple(int(x) for x in chunk_shape)}: axis extent {c} is "
|
|
33
|
-
f"larger than the default inner chunk {DEFAULT_INNER_CHUNK} and not a "
|
|
34
|
-
"multiple of it; pass inner_chunk_shape explicitly"
|
|
35
|
-
)
|
|
27
|
+
i = max(d for d in range(1, min(c, DEFAULT_INNER_CHUNK) + 1) if c % d == 0)
|
|
28
|
+
# an extent with no divisor near the target (a prime, twice a prime) keeps the
|
|
29
|
+
# whole axis as one inner chunk rather than splitting into slivers
|
|
30
|
+
inner.append(i if i >= DEFAULT_INNER_CHUNK // 4 else c)
|
|
36
31
|
return tuple(inner)
|
|
37
32
|
|
|
38
33
|
|
|
@@ -55,15 +50,15 @@ def _location_kwargs(kwargs: dict) -> dict:
|
|
|
55
50
|
def open_ds(
|
|
56
51
|
store,
|
|
57
52
|
mode: Literal["r", "r+", "a", "w", "w-"] = "r",
|
|
58
|
-
metadata_format:
|
|
59
|
-
offset:
|
|
60
|
-
voxel_size:
|
|
61
|
-
axis_names:
|
|
62
|
-
units:
|
|
63
|
-
types:
|
|
64
|
-
chunks:
|
|
53
|
+
metadata_format: MetaDataFormat | None = None,
|
|
54
|
+
offset: Sequence[int] | None = None,
|
|
55
|
+
voxel_size: Sequence[int] | None = None,
|
|
56
|
+
axis_names: Sequence[str] | None = None,
|
|
57
|
+
units: Sequence[str] | None = None,
|
|
58
|
+
types: Sequence[str] | None = None,
|
|
59
|
+
chunks: int | Sequence[int] | str | None = "strict",
|
|
65
60
|
strict_metadata: bool = False,
|
|
66
|
-
scheduler:
|
|
61
|
+
scheduler: str | None = None,
|
|
67
62
|
**kwargs,
|
|
68
63
|
) -> Array:
|
|
69
64
|
"""
|
|
@@ -171,11 +166,11 @@ def open_ds(
|
|
|
171
166
|
|
|
172
167
|
|
|
173
168
|
def _validate_layout(
|
|
174
|
-
chunk_shape:
|
|
169
|
+
chunk_shape: Coordinate | None,
|
|
175
170
|
sharding: bool,
|
|
176
|
-
inner_chunk_shape:
|
|
171
|
+
inner_chunk_shape: Coordinate | None,
|
|
177
172
|
kwargs: dict,
|
|
178
|
-
) ->
|
|
173
|
+
) -> tuple[int, ...] | None:
|
|
179
174
|
"""Check the chunk arguments against each other. Returns the chunk inside each
|
|
180
175
|
shard when sharding, else ``None``."""
|
|
181
176
|
if "shard_shape" in kwargs:
|
|
@@ -211,7 +206,7 @@ def _validate_layout(
|
|
|
211
206
|
f"inner_chunk_shape {inner} and chunk_shape {tuple(chunk_shape)} differ "
|
|
212
207
|
"in dimensions"
|
|
213
208
|
)
|
|
214
|
-
if any(int(c) % i != 0 for c, i in zip(chunk_shape, inner)):
|
|
209
|
+
if any(int(c) % i != 0 for c, i in zip(chunk_shape, inner, strict=False)):
|
|
215
210
|
raise ValueError(
|
|
216
211
|
f"chunk_shape {tuple(chunk_shape)} (the shard) must be a whole number of "
|
|
217
212
|
f"inner_chunk_shape {inner} per axis"
|
|
@@ -232,7 +227,7 @@ def _normalise_sub_location(kwargs: dict) -> None:
|
|
|
232
227
|
kwargs["path"] = name
|
|
233
228
|
|
|
234
229
|
|
|
235
|
-
def _find_existing(store, kwargs: dict) -> tuple[
|
|
230
|
+
def _find_existing(store, kwargs: dict) -> tuple[Array | None, bool]:
|
|
236
231
|
"""The array at the location, or ``None``; and whether a stray placeholder (the
|
|
237
232
|
empty, attr-less group an interrupted sharded ``w-`` create leaves) is there
|
|
238
233
|
instead."""
|
|
@@ -347,7 +342,7 @@ def _create_sharded(
|
|
|
347
342
|
)
|
|
348
343
|
|
|
349
344
|
|
|
350
|
-
def _write_metadata(ds: zarr.Array, metadata, custom_metadata:
|
|
345
|
+
def _write_metadata(ds: zarr.Array, metadata, custom_metadata: dict | None) -> None:
|
|
351
346
|
fmt = get_default_metadata_format()
|
|
352
347
|
attrs = {
|
|
353
348
|
fmt.axis_names_attr: metadata.axis_names,
|
|
@@ -365,18 +360,18 @@ def _write_metadata(ds: zarr.Array, metadata, custom_metadata: Optional[dict]) -
|
|
|
365
360
|
def prepare_ds(
|
|
366
361
|
store,
|
|
367
362
|
shape: Sequence[int],
|
|
368
|
-
offset:
|
|
369
|
-
voxel_size:
|
|
370
|
-
axis_names:
|
|
371
|
-
units:
|
|
372
|
-
types:
|
|
373
|
-
chunk_shape:
|
|
363
|
+
offset: Coordinate | None = None,
|
|
364
|
+
voxel_size: Coordinate | None = None,
|
|
365
|
+
axis_names: Sequence[str] | None = None,
|
|
366
|
+
units: Sequence[str] | None = None,
|
|
367
|
+
types: Sequence[str] | None = None,
|
|
368
|
+
chunk_shape: Coordinate | None = None,
|
|
374
369
|
dtype: DTypeLike = np.float32,
|
|
375
370
|
mode: Literal["r", "r+", "a", "w", "w-"] = "a",
|
|
376
371
|
custom_metadata: dict[str, Any] | None = None,
|
|
377
372
|
sharding: bool = False,
|
|
378
|
-
inner_chunk_shape:
|
|
379
|
-
scheduler:
|
|
373
|
+
inner_chunk_shape: Coordinate | None = None,
|
|
374
|
+
scheduler: str | None = None,
|
|
380
375
|
**kwargs,
|
|
381
376
|
) -> Array:
|
|
382
377
|
"""Prepare a Zarr or N5 dataset.
|
|
@@ -453,7 +448,9 @@ def prepare_ds(
|
|
|
453
448
|
inner_chunk_shape:
|
|
454
449
|
|
|
455
450
|
The efficient-read chunk shape. Can be less than the write-safe chunk shape in
|
|
456
|
-
the case of sharding; defaults to
|
|
451
|
+
the case of sharding; defaults per axis to the largest divisor of `chunk_shape`
|
|
452
|
+
that is at most `DEFAULT_INNER_CHUNK` (64), or the whole axis if that divisor
|
|
453
|
+
is below 16.
|
|
457
454
|
Unsharded it can only restate `chunk_shape`. Given against an existing dataset
|
|
458
455
|
it must match that dataset's chunks.
|
|
459
456
|
|
|
@@ -476,6 +473,15 @@ def prepare_ds(
|
|
|
476
473
|
Returns:
|
|
477
474
|
|
|
478
475
|
A :class:`Array` pointing to the newly created dataset.
|
|
476
|
+
|
|
477
|
+
Raises:
|
|
478
|
+
|
|
479
|
+
A combination of `chunk_shape`, `sharding` and `inner_chunk_shape` that cannot
|
|
480
|
+
be created raises ``ValueError``, and a keyword argument the create does not
|
|
481
|
+
take raises ``TypeError``. With `sharding=True` this also covers a value
|
|
482
|
+
`zarr.create_array` refuses. All of these are refused before the location is
|
|
483
|
+
touched: nothing is created at `store` and nothing already there is changed,
|
|
484
|
+
so the corrected call can be retried in any mode.
|
|
479
485
|
"""
|
|
480
486
|
|
|
481
487
|
metadata_format = get_default_metadata_format()
|
{funlib_persistence-0.8.0 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/freezable.py
RENAMED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
class Freezable
|
|
1
|
+
class Freezable:
|
|
2
2
|
__isfrozen = False
|
|
3
3
|
|
|
4
4
|
def __setattr__(self, key, value):
|
|
5
5
|
if self.__isfrozen and not hasattr(self, key):
|
|
6
|
-
raise TypeError("
|
|
6
|
+
raise TypeError(f"{self!r} is frozen, you can't add attributes to it")
|
|
7
7
|
object.__setattr__(self, key, value)
|
|
8
8
|
|
|
9
9
|
def freeze(self):
|