funlib.persistence 0.8.1__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- funlib_persistence-0.9.0/PKG-INFO +59 -0
- funlib_persistence-0.9.0/README.md +34 -0
- funlib_persistence-0.9.0/pyproject.toml +94 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/__init__.py +1 -1
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/array.py +37 -32
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/datasets.py +32 -23
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/freezable.py +2 -2
- funlib_persistence-0.9.0/src/funlib/persistence/arrays/lazy_ops.py +6 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/metadata.py +55 -33
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/ome_datasets.py +7 -3
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/__init__.py +1 -0
- funlib_persistence-0.9.0/src/funlib/persistence/graphs/csr_graph_database.py +1070 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/graph_database.py +15 -15
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/pgsql_graph_database.py +18 -15
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/sql_graph_database.py +51 -53
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/graphs/sqlite_graph_database.py +14 -12
- funlib_persistence-0.9.0/src/funlib.persistence.egg-info/PKG-INFO +59 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/SOURCES.txt +2 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/requires.txt +0 -10
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/tests/test_array.py +92 -13
- funlib_persistence-0.9.0/tests/test_config.py +102 -0
- funlib_persistence-0.9.0/tests/test_csr_graph_database.py +244 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/tests/test_datasets.py +335 -476
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/tests/test_graph.py +261 -304
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/tests/test_metadata.py +53 -1
- funlib_persistence-0.8.1/PKG-INFO +0 -46
- funlib_persistence-0.8.1/README.md +0 -15
- funlib_persistence-0.8.1/pyproject.toml +0 -59
- funlib_persistence-0.8.1/src/funlib/persistence/arrays/lazy_ops.py +0 -6
- funlib_persistence-0.8.1/src/funlib.persistence.egg-info/PKG-INFO +0 -46
- funlib_persistence-0.8.1/tests/test_config.py +0 -84
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/LICENSE +0 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/setup.cfg +0 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/__init__.py +0 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/utils.py +0 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/py.typed +0 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/types.py +0 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/dependency_links.txt +0 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib.persistence.egg-info/top_level.txt +0 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/tests/test_pgsql_graph_database.py +0 -0
- {funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/tests/test_slices.py +0 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: funlib.persistence
|
|
3
|
+
Version: 0.9.0
|
|
4
|
+
Summary: Interfaces for data (arrays and graphs) and storage formats (databases and file formats)
|
|
5
|
+
Author-email: William Patton <pattonw@hhmi.org>, Jan Funke <funkej@hhmi.org>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Documentation, https://funkelab.github.io/funlib.persistence
|
|
8
|
+
Project-URL: Source, https://github.com/funkelab/funlib.persistence
|
|
9
|
+
Project-URL: Issues, https://github.com/funkelab/funlib.persistence/issues
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Requires-Python: >=3.11
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: zarr<4,>=3
|
|
15
|
+
Requires-Dist: iohub>=0.3.0a5
|
|
16
|
+
Requires-Dist: funlib.geometry>=0.3.0
|
|
17
|
+
Requires-Dist: networkx>=3.0.0
|
|
18
|
+
Requires-Dist: pymongo>=4.0.0
|
|
19
|
+
Requires-Dist: numpy>=2.0.0
|
|
20
|
+
Requires-Dist: pydantic>=2.0.0
|
|
21
|
+
Requires-Dist: dask>=2024.0.0
|
|
22
|
+
Requires-Dist: toml>=0.10.0
|
|
23
|
+
Requires-Dist: psycopg2-binary>=2.9.11
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
[](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml)
|
|
27
|
+
[](https://github.com/funkelab/funlib.persistence/actions/workflows/lint.yaml)
|
|
28
|
+
[](https://funkelab.github.io/funlib.persistence)
|
|
29
|
+
[](https://pypi.org/project/funlib.persistence/)
|
|
30
|
+
|
|
31
|
+
# funlib.persistence
|
|
32
|
+
Interfaces for data (arrays and graphs) and storage formats (databases and file formats)
|
|
33
|
+
|
|
34
|
+
**Documentation:** https://funkelab.github.io/funlib.persistence
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install funlib.persistence # or: uv add funlib.persistence
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Contributing
|
|
43
|
+
|
|
44
|
+
The commands below are the ones CI runs, so a green local run means a green PR.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
git clone https://github.com/funkelab/funlib.persistence
|
|
48
|
+
cd funlib.persistence
|
|
49
|
+
uv sync # Python, dependencies, dev tools (from uv.lock)
|
|
50
|
+
uv run pytest # tests with coverage
|
|
51
|
+
uv run pytest --no-cov tests/<file>.py # one file, without the coverage floor
|
|
52
|
+
uv run ruff check . && uv run ruff format --check . # lint and formatting
|
|
53
|
+
uv run ty check src tests # types
|
|
54
|
+
uv run --group docs sphinx-build -W docs/source docs/build/html # docs
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
The graph-database tests need a PostgreSQL server (CI starts one as a service, `PGUSER=postgres`); they are skipped when none is reachable.
|
|
58
|
+
|
|
59
|
+
`uv.lock` pins the development environment; the `lock-upgrade` workflow refreshes it monthly and opens a pull request.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml)
|
|
2
|
+
[](https://github.com/funkelab/funlib.persistence/actions/workflows/lint.yaml)
|
|
3
|
+
[](https://funkelab.github.io/funlib.persistence)
|
|
4
|
+
[](https://pypi.org/project/funlib.persistence/)
|
|
5
|
+
|
|
6
|
+
# funlib.persistence
|
|
7
|
+
Interfaces for data (arrays and graphs) and storage formats (databases and file formats)
|
|
8
|
+
|
|
9
|
+
**Documentation:** https://funkelab.github.io/funlib.persistence
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install funlib.persistence # or: uv add funlib.persistence
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Contributing
|
|
18
|
+
|
|
19
|
+
The commands below are the ones CI runs, so a green local run means a green PR.
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
git clone https://github.com/funkelab/funlib.persistence
|
|
23
|
+
cd funlib.persistence
|
|
24
|
+
uv sync # Python, dependencies, dev tools (from uv.lock)
|
|
25
|
+
uv run pytest # tests with coverage
|
|
26
|
+
uv run pytest --no-cov tests/<file>.py # one file, without the coverage floor
|
|
27
|
+
uv run ruff check . && uv run ruff format --check . # lint and formatting
|
|
28
|
+
uv run ty check src tests # types
|
|
29
|
+
uv run --group docs sphinx-build -W docs/source docs/build/html # docs
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The graph-database tests need a PostgreSQL server (CI starts one as a service, `PGUSER=postgres`); they are skipped when none is reachable.
|
|
33
|
+
|
|
34
|
+
`uv.lock` pins the development environment; the `lock-upgrade` workflow refreshes it monthly and opens a pull request.
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
build-backend = "setuptools.build_meta"
|
|
3
|
+
requires = ["setuptools", "wheel"]
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "funlib.persistence"
|
|
7
|
+
description = "Interfaces for data (arrays and graphs) and storage formats (databases and file formats)"
|
|
8
|
+
license = { text = "MIT" }
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [
|
|
11
|
+
{ name = "William Patton", email = "pattonw@hhmi.org" },
|
|
12
|
+
{ name = "Jan Funke", email = "funkej@hhmi.org" },
|
|
13
|
+
]
|
|
14
|
+
dynamic = ['version']
|
|
15
|
+
|
|
16
|
+
requires-python = ">=3.11"
|
|
17
|
+
classifiers = ["Programming Language :: Python :: 3"]
|
|
18
|
+
keywords = []
|
|
19
|
+
|
|
20
|
+
dependencies = [
|
|
21
|
+
"zarr>=3,<4",
|
|
22
|
+
"iohub>=0.3.0a5",
|
|
23
|
+
"funlib.geometry>=0.3.0",
|
|
24
|
+
"networkx>=3.0.0",
|
|
25
|
+
"pymongo>=4.0.0",
|
|
26
|
+
"numpy>=2.0.0",
|
|
27
|
+
"pydantic>=2.0.0",
|
|
28
|
+
"dask>=2024.0.0",
|
|
29
|
+
"toml>=0.10.0",
|
|
30
|
+
"psycopg2-binary>=2.9.11",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.dynamic]
|
|
34
|
+
version = { attr = "funlib.persistence.__version__" }
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Documentation = "https://funkelab.github.io/funlib.persistence"
|
|
38
|
+
Source = "https://github.com/funkelab/funlib.persistence"
|
|
39
|
+
Issues = "https://github.com/funkelab/funlib.persistence/issues"
|
|
40
|
+
|
|
41
|
+
[dependency-groups]
|
|
42
|
+
# Installed by `uv sync` (PEP 735); not part of the published package.
|
|
43
|
+
# ruff and ty versions come from uv.lock; `uv lock --upgrade` moves them.
|
|
44
|
+
dev = [
|
|
45
|
+
"coverage>=7.7.1",
|
|
46
|
+
"pytest>=8.3.5",
|
|
47
|
+
"pytest-cov>=6",
|
|
48
|
+
"pytest-mock>=3.14.0",
|
|
49
|
+
"ruff>=0.17.3",
|
|
50
|
+
"ty>=0.0.96",
|
|
51
|
+
"types-psycopg2", # psycopg2 ships no inline types; ty reads this stub package
|
|
52
|
+
]
|
|
53
|
+
docs = [
|
|
54
|
+
"sphinx>=8",
|
|
55
|
+
"sphinx-autodoc-typehints>=2",
|
|
56
|
+
"autodoc_pydantic>=2",
|
|
57
|
+
"myst-parser>=4",
|
|
58
|
+
"pydata-sphinx-theme>=0.16",
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
[tool.setuptools.package-data]
|
|
62
|
+
"funlib.persistence" = ["py.typed"]
|
|
63
|
+
|
|
64
|
+
[tool.ty.src]
|
|
65
|
+
include = ["src", "tests", "benchmarks"]
|
|
66
|
+
exclude = [] # only ever shrinks
|
|
67
|
+
|
|
68
|
+
[tool.ruff]
|
|
69
|
+
line-length = 88
|
|
70
|
+
extend-exclude = ["docs/build", "*.md"]
|
|
71
|
+
|
|
72
|
+
[tool.ruff.lint]
|
|
73
|
+
select = ["E", "F", "W", "I", "UP", "B"]
|
|
74
|
+
ignore = ["E501"] # the formatter owns line length
|
|
75
|
+
|
|
76
|
+
[tool.ruff.lint.per-file-ignores]
|
|
77
|
+
"tutorials/**/*.py" = ["E402"] # jupytext cells import mid-file
|
|
78
|
+
"examples/**/*.py" = ["E402"]
|
|
79
|
+
"__init__.py" = ["F401"] # re-exports
|
|
80
|
+
|
|
81
|
+
[tool.pytest.ini_options]
|
|
82
|
+
testpaths = ["tests"]
|
|
83
|
+
addopts = "--cov --cov-report=term-missing"
|
|
84
|
+
|
|
85
|
+
[tool.coverage.run]
|
|
86
|
+
branch = true
|
|
87
|
+
source = ["funlib.persistence"]
|
|
88
|
+
|
|
89
|
+
[tool.coverage.report]
|
|
90
|
+
# Floor measured from the first branch-enabled run; raise it when measured
|
|
91
|
+
# coverage is two or more points ahead, never lower it without saying why.
|
|
92
|
+
fail_under = 75
|
|
93
|
+
show_missing = true
|
|
94
|
+
skip_covered = true
|
{funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/array.py
RENAMED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from collections.abc import Sequence
|
|
3
3
|
from functools import reduce
|
|
4
|
-
from typing import Any
|
|
4
|
+
from typing import Any
|
|
5
5
|
|
|
6
6
|
import dask.array as da
|
|
7
7
|
import numpy as np
|
|
@@ -93,15 +93,15 @@ class Array(Freezable):
|
|
|
93
93
|
def __init__(
|
|
94
94
|
self,
|
|
95
95
|
data,
|
|
96
|
-
offset:
|
|
97
|
-
voxel_size:
|
|
98
|
-
axis_names:
|
|
99
|
-
units:
|
|
100
|
-
types:
|
|
101
|
-
chunks:
|
|
102
|
-
lazy_op:
|
|
96
|
+
offset: Sequence[int] | None = None,
|
|
97
|
+
voxel_size: Sequence[int] | None = None,
|
|
98
|
+
axis_names: Sequence[str] | None = None,
|
|
99
|
+
units: Sequence[str] | None = None,
|
|
100
|
+
types: Sequence[str] | None = None,
|
|
101
|
+
chunks: int | Sequence[int] | str | None = "auto",
|
|
102
|
+
lazy_op: LazyOp | None = None,
|
|
103
103
|
strict_metadata: bool = False,
|
|
104
|
-
scheduler:
|
|
104
|
+
scheduler: str | None = None,
|
|
105
105
|
):
|
|
106
106
|
self.scheduler = scheduler
|
|
107
107
|
if not isinstance(data, da.Array):
|
|
@@ -149,7 +149,7 @@ class Array(Freezable):
|
|
|
149
149
|
return self._attrs
|
|
150
150
|
|
|
151
151
|
@property
|
|
152
|
-
def chunk_shape(self) ->
|
|
152
|
+
def chunk_shape(self) -> Coordinate | None:
|
|
153
153
|
"""The write-safe chunk shape: the shard shape if sharded, else the chunk
|
|
154
154
|
shape (an ``open_ds(chunks=...)`` override is reported as given). ``None``
|
|
155
155
|
for a view that is not writeable or that lazy operations moved off the
|
|
@@ -160,14 +160,14 @@ class Array(Freezable):
|
|
|
160
160
|
return shards if shards is not None else self._dask_chunk_shape()
|
|
161
161
|
|
|
162
162
|
@property
|
|
163
|
-
def inner_chunk_shape(self) ->
|
|
163
|
+
def inner_chunk_shape(self) -> Coordinate | None:
|
|
164
164
|
"""Chunk shape for efficient reads. Can be smaller than the write-safe
|
|
165
165
|
:attr:`chunk_shape` in the case of sharding. ``None`` for a view that
|
|
166
166
|
lazy operations moved off the stored chunk grid."""
|
|
167
167
|
return self._dask_chunk_shape() if self._on_chunk_grid else None
|
|
168
168
|
|
|
169
169
|
@property
|
|
170
|
-
def shard_shape(self) ->
|
|
170
|
+
def shard_shape(self) -> Coordinate | None:
|
|
171
171
|
"""The shard shape in the case of sharded datasets. ``None`` for non
|
|
172
172
|
sharded or unwritable datasets, or off the stored chunk grid. See
|
|
173
173
|
https://zarr.readthedocs.io/en/stable/user-guide/performance.html#sharding
|
|
@@ -176,7 +176,9 @@ class Array(Freezable):
|
|
|
176
176
|
if shards is None or not self.is_writeable or not self._on_chunk_grid:
|
|
177
177
|
return None
|
|
178
178
|
return Coordinate(
|
|
179
|
-
s
|
|
179
|
+
s
|
|
180
|
+
for s, uncollapsed in zip(shards, self._uncollapsed_dims, strict=False)
|
|
181
|
+
if uncollapsed
|
|
180
182
|
)
|
|
181
183
|
|
|
182
184
|
def _dask_chunk_shape(self) -> Coordinate:
|
|
@@ -196,9 +198,13 @@ class Array(Freezable):
|
|
|
196
198
|
unit = self.chunk_shape
|
|
197
199
|
if unit is None:
|
|
198
200
|
return False
|
|
199
|
-
physical = [
|
|
201
|
+
physical = [
|
|
202
|
+
c
|
|
203
|
+
for c, t in zip(unit, self.types, strict=False)
|
|
204
|
+
if t in ["space", "time"]
|
|
205
|
+
]
|
|
200
206
|
begin = (lazy_op.offset - self.offset) / self.voxel_size
|
|
201
|
-
return all(b % c == 0 for b, c in zip(begin, physical))
|
|
207
|
+
return all(b % c == 0 for b, c in zip(begin, physical, strict=False))
|
|
202
208
|
if not isinstance(lazy_op, tuple):
|
|
203
209
|
lazy_op = (lazy_op,)
|
|
204
210
|
for a in lazy_op:
|
|
@@ -220,7 +226,9 @@ class Array(Freezable):
|
|
|
220
226
|
if physical:
|
|
221
227
|
return [
|
|
222
228
|
x
|
|
223
|
-
for x, t in zip(
|
|
229
|
+
for x, t in zip(
|
|
230
|
+
self._uncollapsed_dims, self._metadata.types, strict=False
|
|
231
|
+
)
|
|
224
232
|
if t in ["space", "time"]
|
|
225
233
|
]
|
|
226
234
|
else:
|
|
@@ -336,7 +344,7 @@ class Array(Freezable):
|
|
|
336
344
|
# previous in-place walk re-counted uncollapsed dims after each collapse,
|
|
337
345
|
# so a tuple with two or more ints collapsed the wrong dimensions.
|
|
338
346
|
dims = [i for i, uc in enumerate(self._uncollapsed_dims) if uc]
|
|
339
|
-
for a, i in zip(lazy_op, dims):
|
|
347
|
+
for a, i in zip(lazy_op, dims, strict=False):
|
|
340
348
|
if isinstance(a, int):
|
|
341
349
|
self._uncollapsed_dims[i] = False
|
|
342
350
|
self.data = self.data[lazy_op]
|
|
@@ -353,7 +361,7 @@ class Array(Freezable):
|
|
|
353
361
|
elif callable(lazy_op):
|
|
354
362
|
self.data = lazy_op(self.data)
|
|
355
363
|
else:
|
|
356
|
-
raise
|
|
364
|
+
raise TypeError(
|
|
357
365
|
f"LazyOp {lazy_op} is not a supported lazy_op. "
|
|
358
366
|
f"Supported lazy_ops are: {LazyOp}"
|
|
359
367
|
)
|
|
@@ -384,15 +392,15 @@ class Array(Freezable):
|
|
|
384
392
|
|
|
385
393
|
Args:
|
|
386
394
|
|
|
387
|
-
key (
|
|
395
|
+
key (:class:`Roi` or :class:`Coordinate` or any numpy compatible key):
|
|
388
396
|
|
|
389
397
|
The `Roi`, `Coordinate`, or numpy compatible key indicating the data
|
|
390
398
|
you want to read.
|
|
391
399
|
|
|
392
400
|
Returns:
|
|
393
401
|
|
|
394
|
-
A numpy array containing the data requested by the key. If using a
|
|
395
|
-
or
|
|
402
|
+
A numpy array containing the data requested by the key. If using a :class:`Roi`
|
|
403
|
+
or :class:`Coordinate`, the data will be returned in world units. If using
|
|
396
404
|
a numpy compatible key, the data will be sliced as a regular numpy array.
|
|
397
405
|
"""
|
|
398
406
|
|
|
@@ -401,8 +409,7 @@ class Array(Freezable):
|
|
|
401
409
|
|
|
402
410
|
if not self.roi.contains(roi):
|
|
403
411
|
raise IndexError(
|
|
404
|
-
"Requested roi
|
|
405
|
-
% (roi, self.roi)
|
|
412
|
+
f"Requested roi {roi} is not contained in this array {self.roi}."
|
|
406
413
|
)
|
|
407
414
|
|
|
408
415
|
return self.data[self.__slices(roi, use_lazy_slices=False)].compute(
|
|
@@ -426,7 +433,7 @@ class Array(Freezable):
|
|
|
426
433
|
|
|
427
434
|
Args:
|
|
428
435
|
|
|
429
|
-
key (
|
|
436
|
+
key (:class:`Roi` or any numpy compatible key):
|
|
430
437
|
|
|
431
438
|
The region to write to. Can be a `Roi` for world-unit indexing,
|
|
432
439
|
or any numpy-compatible key (e.g. ``np.s_[:]``, a slice, a tuple
|
|
@@ -444,8 +451,7 @@ class Array(Freezable):
|
|
|
444
451
|
|
|
445
452
|
if not self.roi.contains(roi):
|
|
446
453
|
raise IndexError(
|
|
447
|
-
"Requested roi
|
|
448
|
-
% (roi, self.roi)
|
|
454
|
+
f"Requested roi {roi} is not contained in this array {self.roi}."
|
|
449
455
|
)
|
|
450
456
|
|
|
451
457
|
region_slices = self.__slices(roi)
|
|
@@ -481,7 +487,7 @@ class Array(Freezable):
|
|
|
481
487
|
|
|
482
488
|
Args:
|
|
483
489
|
|
|
484
|
-
roi (
|
|
490
|
+
roi (:class:`Roi`, optional):
|
|
485
491
|
|
|
486
492
|
If given, copy only the data represented by this ROI. This is
|
|
487
493
|
equivalent to::
|
|
@@ -527,9 +533,8 @@ class Array(Freezable):
|
|
|
527
533
|
assert begin_align_with_chunks and (
|
|
528
534
|
shape_align_with_chunks or end_of_array
|
|
529
535
|
), (
|
|
530
|
-
"ROI
|
|
531
|
-
"size
|
|
532
|
-
% (roi, voxel_roi, chunk_shape, d)
|
|
536
|
+
f"ROI {roi} (in voxels: {voxel_roi}) does not align with "
|
|
537
|
+
f"chunks of size {chunk_shape} (mismatch in dimension {d})"
|
|
533
538
|
)
|
|
534
539
|
|
|
535
540
|
roi_slices = tuple(
|
|
@@ -556,7 +561,7 @@ class Array(Freezable):
|
|
|
556
561
|
elif isinstance(lazy_op, list) and all([isinstance(a, int) for a in lazy_op]):
|
|
557
562
|
return True
|
|
558
563
|
elif isinstance(lazy_op, tuple) and all(
|
|
559
|
-
[self._is_slice(a, writeable) for a in lazy_op] #
|
|
564
|
+
[self._is_slice(a, writeable) for a in lazy_op] # ty: ignore[invalid-argument-type] # tuple elements are not narrowed
|
|
560
565
|
):
|
|
561
566
|
return True
|
|
562
567
|
elif (
|
|
@@ -622,7 +627,7 @@ class Array(Freezable):
|
|
|
622
627
|
|
|
623
628
|
def to_world_space(
|
|
624
629
|
self, pixel_loc: Roi | Coordinate | Sequence[int | float] | np.ndarray
|
|
625
|
-
) -> Roi | Coordinate:
|
|
630
|
+
) -> Roi | Coordinate | np.ndarray:
|
|
626
631
|
"""Convert a point or roi from pixel space in this array to the world
|
|
627
632
|
coordinate system defined by this array's roi and voxel size.
|
|
628
633
|
Works on Coordiantes and Rois, as well as sequences of ints or floats.
|
{funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/datasets.py
RENAMED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
import logging
|
|
3
3
|
from collections.abc import Sequence
|
|
4
|
-
from typing import Any, Literal
|
|
4
|
+
from typing import Any, Literal
|
|
5
5
|
|
|
6
6
|
import numpy as np
|
|
7
7
|
import zarr
|
|
@@ -50,15 +50,15 @@ def _location_kwargs(kwargs: dict) -> dict:
|
|
|
50
50
|
def open_ds(
|
|
51
51
|
store,
|
|
52
52
|
mode: Literal["r", "r+", "a", "w", "w-"] = "r",
|
|
53
|
-
metadata_format:
|
|
54
|
-
offset:
|
|
55
|
-
voxel_size:
|
|
56
|
-
axis_names:
|
|
57
|
-
units:
|
|
58
|
-
types:
|
|
59
|
-
chunks:
|
|
53
|
+
metadata_format: MetaDataFormat | None = None,
|
|
54
|
+
offset: Sequence[int] | None = None,
|
|
55
|
+
voxel_size: Sequence[int] | None = None,
|
|
56
|
+
axis_names: Sequence[str] | None = None,
|
|
57
|
+
units: Sequence[str] | None = None,
|
|
58
|
+
types: Sequence[str] | None = None,
|
|
59
|
+
chunks: int | Sequence[int] | str | None = "strict",
|
|
60
60
|
strict_metadata: bool = False,
|
|
61
|
-
scheduler:
|
|
61
|
+
scheduler: str | None = None,
|
|
62
62
|
**kwargs,
|
|
63
63
|
) -> Array:
|
|
64
64
|
"""
|
|
@@ -166,11 +166,11 @@ def open_ds(
|
|
|
166
166
|
|
|
167
167
|
|
|
168
168
|
def _validate_layout(
|
|
169
|
-
chunk_shape:
|
|
169
|
+
chunk_shape: Coordinate | None,
|
|
170
170
|
sharding: bool,
|
|
171
|
-
inner_chunk_shape:
|
|
171
|
+
inner_chunk_shape: Coordinate | None,
|
|
172
172
|
kwargs: dict,
|
|
173
|
-
) ->
|
|
173
|
+
) -> tuple[int, ...] | None:
|
|
174
174
|
"""Check the chunk arguments against each other. Returns the chunk inside each
|
|
175
175
|
shard when sharding, else ``None``."""
|
|
176
176
|
if "shard_shape" in kwargs:
|
|
@@ -206,7 +206,7 @@ def _validate_layout(
|
|
|
206
206
|
f"inner_chunk_shape {inner} and chunk_shape {tuple(chunk_shape)} differ "
|
|
207
207
|
"in dimensions"
|
|
208
208
|
)
|
|
209
|
-
if any(int(c) % i != 0 for c, i in zip(chunk_shape, inner)):
|
|
209
|
+
if any(int(c) % i != 0 for c, i in zip(chunk_shape, inner, strict=False)):
|
|
210
210
|
raise ValueError(
|
|
211
211
|
f"chunk_shape {tuple(chunk_shape)} (the shard) must be a whole number of "
|
|
212
212
|
f"inner_chunk_shape {inner} per axis"
|
|
@@ -227,7 +227,7 @@ def _normalise_sub_location(kwargs: dict) -> None:
|
|
|
227
227
|
kwargs["path"] = name
|
|
228
228
|
|
|
229
229
|
|
|
230
|
-
def _find_existing(store, kwargs: dict) -> tuple[
|
|
230
|
+
def _find_existing(store, kwargs: dict) -> tuple[Array | None, bool]:
|
|
231
231
|
"""The array at the location, or ``None``; and whether a stray placeholder (the
|
|
232
232
|
empty, attr-less group an interrupted sharded ``w-`` create leaves) is there
|
|
233
233
|
instead."""
|
|
@@ -342,7 +342,7 @@ def _create_sharded(
|
|
|
342
342
|
)
|
|
343
343
|
|
|
344
344
|
|
|
345
|
-
def _write_metadata(ds: zarr.Array, metadata, custom_metadata:
|
|
345
|
+
def _write_metadata(ds: zarr.Array, metadata, custom_metadata: dict | None) -> None:
|
|
346
346
|
fmt = get_default_metadata_format()
|
|
347
347
|
attrs = {
|
|
348
348
|
fmt.axis_names_attr: metadata.axis_names,
|
|
@@ -360,18 +360,18 @@ def _write_metadata(ds: zarr.Array, metadata, custom_metadata: Optional[dict]) -
|
|
|
360
360
|
def prepare_ds(
|
|
361
361
|
store,
|
|
362
362
|
shape: Sequence[int],
|
|
363
|
-
offset:
|
|
364
|
-
voxel_size:
|
|
365
|
-
axis_names:
|
|
366
|
-
units:
|
|
367
|
-
types:
|
|
368
|
-
chunk_shape:
|
|
363
|
+
offset: Coordinate | None = None,
|
|
364
|
+
voxel_size: Coordinate | None = None,
|
|
365
|
+
axis_names: Sequence[str] | None = None,
|
|
366
|
+
units: Sequence[str] | None = None,
|
|
367
|
+
types: Sequence[str] | None = None,
|
|
368
|
+
chunk_shape: Coordinate | None = None,
|
|
369
369
|
dtype: DTypeLike = np.float32,
|
|
370
370
|
mode: Literal["r", "r+", "a", "w", "w-"] = "a",
|
|
371
371
|
custom_metadata: dict[str, Any] | None = None,
|
|
372
372
|
sharding: bool = False,
|
|
373
|
-
inner_chunk_shape:
|
|
374
|
-
scheduler:
|
|
373
|
+
inner_chunk_shape: Coordinate | None = None,
|
|
374
|
+
scheduler: str | None = None,
|
|
375
375
|
**kwargs,
|
|
376
376
|
) -> Array:
|
|
377
377
|
"""Prepare a Zarr or N5 dataset.
|
|
@@ -473,6 +473,15 @@ def prepare_ds(
|
|
|
473
473
|
Returns:
|
|
474
474
|
|
|
475
475
|
A :class:`Array` pointing to the newly created dataset.
|
|
476
|
+
|
|
477
|
+
Raises:
|
|
478
|
+
|
|
479
|
+
A combination of `chunk_shape`, `sharding` and `inner_chunk_shape` that cannot
|
|
480
|
+
be created raises ``ValueError``, and a keyword argument the create does not
|
|
481
|
+
take raises ``TypeError``. With `sharding=True` this also covers a value
|
|
482
|
+
`zarr.create_array` refuses. All of these are refused before the location is
|
|
483
|
+
touched: nothing is created at `store` and nothing already there is changed,
|
|
484
|
+
so the corrected call can be retried in any mode.
|
|
476
485
|
"""
|
|
477
486
|
|
|
478
487
|
metadata_format = get_default_metadata_format()
|
{funlib_persistence-0.8.1 → funlib_persistence-0.9.0}/src/funlib/persistence/arrays/freezable.py
RENAMED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
class Freezable
|
|
1
|
+
class Freezable:
|
|
2
2
|
__isfrozen = False
|
|
3
3
|
|
|
4
4
|
def __setattr__(self, key, value):
|
|
5
5
|
if self.__isfrozen and not hasattr(self, key):
|
|
6
|
-
raise TypeError("
|
|
6
|
+
raise TypeError(f"{self!r} is frozen, you can't add attributes to it")
|
|
7
7
|
object.__setattr__(self, key, value)
|
|
8
8
|
|
|
9
9
|
def freeze(self):
|