funlib.persistence 0.8.0__tar.gz → 0.8.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/PKG-INFO +1 -1
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/__init__.py +1 -1
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/datasets.py +10 -13
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib.persistence.egg-info/PKG-INFO +1 -1
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/tests/test_datasets.py +12 -6
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/LICENSE +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/README.md +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/pyproject.toml +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/setup.cfg +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/__init__.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/array.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/freezable.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/lazy_ops.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/metadata.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/ome_datasets.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/utils.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/graphs/__init__.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/graphs/graph_database.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/graphs/pgsql_graph_database.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/graphs/sql_graph_database.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/graphs/sqlite_graph_database.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/py.typed +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/types.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib.persistence.egg-info/SOURCES.txt +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib.persistence.egg-info/dependency_links.txt +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib.persistence.egg-info/requires.txt +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib.persistence.egg-info/top_level.txt +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/tests/test_array.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/tests/test_config.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/tests/test_graph.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/tests/test_metadata.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/tests/test_pgsql_graph_database.py +0 -0
- {funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/tests/test_slices.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: funlib.persistence
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.1
|
|
4
4
|
Summary: Interfaces for data (arrays and graphs) and storage formats (databases and file formats)
|
|
5
5
|
Author-email: William Patton <pattonw@hhmi.org>, Jan Funke <funkej@hhmi.org>
|
|
6
6
|
License: MIT
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/datasets.py
RENAMED
|
@@ -15,24 +15,19 @@ from .metadata import MetaDataFormat, get_default_metadata_format
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
|
|
17
17
|
DEFAULT_INNER_CHUNK = 64
|
|
18
|
-
"""The chunk inside each shard when `prepare_ds(sharding=True)` is given none, per
|
|
19
|
-
|
|
18
|
+
"""The target chunk inside each shard when `prepare_ds(sharding=True)` is given none, per
|
|
19
|
+
axis: the largest divisor of the shard's extent that is at most this (the whole axis where
|
|
20
|
+
the shard is smaller)."""
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
def _default_inner_chunk(chunk_shape: Sequence[int]) -> tuple[int, ...]:
|
|
23
24
|
inner = []
|
|
24
25
|
for c in chunk_shape:
|
|
25
26
|
c = int(c)
|
|
26
|
-
if c
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
else:
|
|
31
|
-
raise ValueError(
|
|
32
|
-
f"chunk_shape {tuple(int(x) for x in chunk_shape)}: axis extent {c} is "
|
|
33
|
-
f"larger than the default inner chunk {DEFAULT_INNER_CHUNK} and not a "
|
|
34
|
-
"multiple of it; pass inner_chunk_shape explicitly"
|
|
35
|
-
)
|
|
27
|
+
i = max(d for d in range(1, min(c, DEFAULT_INNER_CHUNK) + 1) if c % d == 0)
|
|
28
|
+
# an extent with no divisor near the target (a prime, twice a prime) keeps the
|
|
29
|
+
# whole axis as one inner chunk rather than splitting into slivers
|
|
30
|
+
inner.append(i if i >= DEFAULT_INNER_CHUNK // 4 else c)
|
|
36
31
|
return tuple(inner)
|
|
37
32
|
|
|
38
33
|
|
|
@@ -453,7 +448,9 @@ def prepare_ds(
|
|
|
453
448
|
inner_chunk_shape:
|
|
454
449
|
|
|
455
450
|
The efficient-read chunk shape. Can be less than the write-safe chunk shape in
|
|
456
|
-
the case of sharding; defaults to
|
|
451
|
+
the case of sharding; defaults per axis to the largest divisor of `chunk_shape`
|
|
452
|
+
that is at most `DEFAULT_INNER_CHUNK` (64), or the whole axis if that divisor
|
|
453
|
+
is below 16.
|
|
457
454
|
Unsharded it can only restate `chunk_shape`. Given against an existing dataset
|
|
458
455
|
it must match that dataset's chunks.
|
|
459
456
|
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib.persistence.egg-info/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: funlib.persistence
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.1
|
|
4
4
|
Summary: Interfaces for data (arrays and graphs) and storage formats (databases and file formats)
|
|
5
5
|
Author-email: William Patton <pattonw@hhmi.org>, Jan Funke <funkej@hhmi.org>
|
|
6
6
|
License: MIT
|
|
@@ -390,8 +390,8 @@ def test_unsharded_is_reused_by_a_sharding_caller_with_the_same_write_unit(tmp_p
|
|
|
390
390
|
|
|
391
391
|
|
|
392
392
|
def test_inner_chunk_defaults(tmp_path):
|
|
393
|
-
"""
|
|
394
|
-
|
|
393
|
+
"""The largest divisor of each shard axis that is at most 64; the whole axis where the
|
|
394
|
+
shard is smaller, or where no divisor comes near 64."""
|
|
395
395
|
a = prepare_ds(
|
|
396
396
|
tmp_path / "a.zarr",
|
|
397
397
|
(2, 256, 256),
|
|
@@ -411,15 +411,21 @@ def test_inner_chunk_defaults(tmp_path):
|
|
|
411
411
|
mode="w",
|
|
412
412
|
)
|
|
413
413
|
assert b.inner_chunk_shape == Coordinate(2, 32, 32)
|
|
414
|
-
|
|
415
|
-
|
|
414
|
+
for shard, inner in [
|
|
415
|
+
((3, 100, 96, 200), (3, 50, 48, 50)),
|
|
416
|
+
((1, 128, 1000, 4096), (1, 64, 50, 64)),
|
|
417
|
+
((2, 127, 134, 70), (2, 127, 134, 35)),
|
|
418
|
+
]:
|
|
419
|
+
c = prepare_ds(
|
|
416
420
|
tmp_path / "c.zarr",
|
|
417
|
-
|
|
418
|
-
chunk_shape=Coordinate(
|
|
421
|
+
shard,
|
|
422
|
+
chunk_shape=Coordinate(shard),
|
|
419
423
|
sharding=True,
|
|
420
424
|
dtype=np.uint8,
|
|
421
425
|
mode="w",
|
|
422
426
|
)
|
|
427
|
+
assert c.inner_chunk_shape == Coordinate(inner)
|
|
428
|
+
assert c.chunk_shape == Coordinate(shard)
|
|
423
429
|
|
|
424
430
|
|
|
425
431
|
def test_sharding_option_refusals(tmp_path):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/__init__.py
RENAMED
|
File without changes
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/array.py
RENAMED
|
File without changes
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/freezable.py
RENAMED
|
File without changes
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/lazy_ops.py
RENAMED
|
File without changes
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/metadata.py
RENAMED
|
File without changes
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/ome_datasets.py
RENAMED
|
File without changes
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/arrays/utils.py
RENAMED
|
File without changes
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib/persistence/graphs/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib.persistence.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib.persistence.egg-info/requires.txt
RENAMED
|
File without changes
|
{funlib_persistence-0.8.0 → funlib_persistence-0.8.1}/src/funlib.persistence.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|