worldarray 0.5.0__tar.gz → 0.7.0__tar.gz

{worldarray-0.5.0 → worldarray-0.7.0}/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## 0.7.0 (2026-10-23)
+
+Opens time out. No placement change and no unit table change.
+
+- Every metadata document read or written and every store opened is timed out and
+  retried under `Retry`, as data reads are (Behaviour 6): by default 5 s times the attempt,
+  ten attempts. A server that accepts a connection and never answers held `wa.open` and
+  `wa.metadata` forever before; it now raises `TimeoutError` naming the URL and the
+  attempts (cellmap-flow, 2026-10-23). The policy is the call's `retries`, which
+  `wa.metadata`, `wa.detect` and `wa.open_tensorstore` now take as `wa.open`,
+  `wa.open_pyramid` and `wa.create` already did. Listings and the open that creates an
+  array are not timed out.
+
+## 0.6.0 (2026-10-23)
+
+One new argument, at chunkmirage's request. No placement change and no unit table change;
+nothing changes for a caller that does not pass it.
+
+- `wa.shared_context(cache_bytes, threads=)`: `None` (the default) limits the decoding and
+  file I/O pools to `thread_budget()` as in 0.5, a positive number limits them to that many,
+  and `0` writes no limit, so tensorstore sizes them itself (every CPU on the node). Each
+  `(cache_bytes, threads)` is one watched context for the process, so a server can share
+  one cache pool across every store it opens at the thread count it chose, without building
+  and watching its own context. A pickled handle on such a context comes back on the same
+  one; `threads=None` still lets the other process size by its own job.
+- Documented what a pool of one thread costs (spec Behaviour 7, `capped_context_spec`):
+  nothing on one-chunk blocks; on 8-chunk blocks, half the speed of 4 in a two-slot job
+  but only 7 to 19% slower in a one-slot one (c-elegans on /nrs), so a worker per slot
+  loses little at the default.
+
 ## 0.5.0 (2026-10-23)
 
 Two bugs that hung or crashed cellmap-analyze 0.6 on /nrs (salamander's and c-elegans'

{worldarray-0.5.0 → worldarray-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: worldarray
-Version: 0.5.0
+Version: 0.7.0
 Summary: Read zarr, N5 and Neuroglancer precomputed volumes in world coordinates through tensorstore: correct voxel placement, pyramid levels, padded region reads, resampling. numpy + tensorstore only.
 Project-URL: Homepage, https://janeliascicomp.github.io/worldarray/
 Project-URL: Documentation, https://janeliascicomp.github.io/worldarray/
@@ -205,11 +205,13 @@ wa.open(path, *, level=None, voxel_size=None, level_mode="floor", on_mismatch="e
         metadata_conflicts="error", base_voxel_size=None, base_units=None, anchor=None,
         transform=None, strict_units=True) -> WorldArray
 wa.open_pyramid(path, ...) -> Pyramid          # .names .levels .by_name() .select() .placements() .metas(): no opens needed
-wa.metadata(path, level=None, units=None, open=True) -> ArrayMeta   # open=False: from documents alone
-wa.detect(path, missing_ok=False) -> Node      # format, array or group, raw attrs
-wa.open_tensorstore(path, level=None, context=None)
-wa.shared_context(cache_bytes=0) ; wa.capped_context(cache_bytes=0, concurrency=1, network=True) ; wa.Retry(...)
-wa.thread_budget()                              # the shared context's thread limit: OMP_NUM_THREADS, the job's slots, or CPUs
+wa.metadata(path, level=None, units=None, open=True, retries=None) -> ArrayMeta   # open=False: from documents alone
+wa.detect(path, missing_ok=False, retries=None) -> Node      # format, array or group, raw attrs
+wa.open_tensorstore(path, level=None, context=None, retries=None)
+                    # retries= times out and retries document reads and the open, as it does data reads
+wa.shared_context(cache_bytes=0, threads=None) ; wa.capped_context(cache_bytes=0, concurrency=1, network=True) ; wa.Retry(...)
+wa.thread_budget()                              # the shared context's default thread limit: OMP_NUM_THREADS, the job's slots, or CPUs;
+                                                #   threads=N limits it to N, threads=0 leaves tensorstore's own pool sizes
 wa.list_chunk_keys(path, separator=".")        # chunk indices from keys alone, no metadata needed
 wa.Box ; wa.units.TABLE
 

{worldarray-0.5.0 → worldarray-0.7.0}/README.md

@@ -146,11 +146,13 @@ wa.open(path, *, level=None, voxel_size=None, level_mode="floor", on_mismatch="e
         metadata_conflicts="error", base_voxel_size=None, base_units=None, anchor=None,
         transform=None, strict_units=True) -> WorldArray
 wa.open_pyramid(path, ...) -> Pyramid          # .names .levels .by_name() .select() .placements() .metas(): no opens needed
-wa.metadata(path, level=None, units=None, open=True) -> ArrayMeta   # open=False: from documents alone
-wa.detect(path, missing_ok=False) -> Node      # format, array or group, raw attrs
-wa.open_tensorstore(path, level=None, context=None)
-wa.shared_context(cache_bytes=0) ; wa.capped_context(cache_bytes=0, concurrency=1, network=True) ; wa.Retry(...)
-wa.thread_budget()                              # the shared context's thread limit: OMP_NUM_THREADS, the job's slots, or CPUs
+wa.metadata(path, level=None, units=None, open=True, retries=None) -> ArrayMeta   # open=False: from documents alone
+wa.detect(path, missing_ok=False, retries=None) -> Node      # format, array or group, raw attrs
+wa.open_tensorstore(path, level=None, context=None, retries=None)
+                    # retries= times out and retries document reads and the open, as it does data reads
+wa.shared_context(cache_bytes=0, threads=None) ; wa.capped_context(cache_bytes=0, concurrency=1, network=True) ; wa.Retry(...)
+wa.thread_budget()                              # the shared context's default thread limit: OMP_NUM_THREADS, the job's slots, or CPUs;
+                                                #   threads=N limits it to N, threads=0 leaves tensorstore's own pool sizes
 wa.list_chunk_keys(path, separator=".")        # chunk indices from keys alone, no metadata needed
 wa.Box ; wa.units.TABLE
 

{worldarray-0.5.0 → worldarray-0.7.0}/docs/spec.md

@@ -177,15 +177,16 @@ pyr.voxel_sizes() ; pyr.translations()   #   where it lists the levels (OME-NGFF
                     #   choosing or placing a level opens nothing, whether the pyramid was opened by its
                     #   group path or by one of its level paths; else by opening each level
 
-wa.metadata(path, level=None, units=None, open=True) -> ArrayMeta   # open=False: from the documents alone
-wa.detect(path, missing_ok=False) -> Node | None   # format, kind (array|group), path, docs, scale_key; .attrs (raw user
+wa.metadata(path, level=None, units=None, open=True, retries=None) -> ArrayMeta   # open=False: from the documents alone
+wa.detect(path, missing_ok=False, retries=None) -> Node | None   # format, kind (array|group), path, docs, scale_key; .attrs (raw user
                                                    #   attributes), .multiscales (OME 0.5's `ome` unwrapped)
-wa.open_tensorstore(path, level=None, context=None) -> tensorstore.TensorStore   # reads no world metadata, so an
+wa.open_tensorstore(path, level=None, context=None, retries=None) -> tensorstore.TensorStore   # reads no world metadata, so an
                                                    #   array with odd placement attributes (a coordinate array) still opens
 wa.corner_to_centre(corner, voxel_size, axes=None) ; wa.default_axes(ndim) ; wa.index_range(begin, end, voxel_size, translation)
 wa.units.TABLE      # the unit tables, public: {"length": {...metres per unit}, "time": {...seconds per unit}}
 wa.select_level(levels, voxel_size, mode) ; wa.Box ; wa.Retry(max_attempts=10, timeout=None, base_delay=1.0)
-wa.shared_context(cache_bytes=0) ; wa.capped_context(cache_bytes=0, concurrency=1, network=True) ; wa.capped_context_spec(...)
+wa.shared_context(cache_bytes=0, threads=None) ; wa.capped_context(cache_bytes=0, concurrency=1, network=True) ; wa.capped_context_spec(...)
+                    # threads: None = thread_budget(), N = that many, 0 = tensorstore's own pool sizes (0.6)
                     # network=False caps decoding and local file I/O only, leaving tensorstore's request pools alone
 wa.list_chunk_keys(path, separator=".", ndim=None)   # chunk indices present under a path, from its keys alone: no
                     #   metadata document needed, so a directory of chunk files with no .zarray lists too
@@ -229,9 +230,11 @@ returned one would raise on inputs cellmap-analyze reads every day. A consumer t
 in integer Rois builds them from `corner`, `voxel_size` and `shape` in its own subclass,
 where it also chooses the integer scale factor that makes the corner whole.
 
-`shared_context(cache_bytes)` is one `tensorstore.Context` per budget for the process, so
-the levels of a pyramid and repeat opens share one cache pool; the default budget is 0. Its
-decoding and file I/O pools are sized by `thread_budget()` (0.5, Behaviour 7).
+`shared_context(cache_bytes, threads=None)` is one `tensorstore.Context` per budget and
+thread limit for the process, so the levels of a pyramid and repeat opens share one cache
+pool; the default budget is 0. Its decoding and file I/O pools are sized by
+`thread_budget()` (0.5), or by `threads` when given, `0` leaving tensorstore's own sizes
+(0.6, Behaviour 7).
 `capped_context` is mesh-n-bone's worker-process context (every tensorstore concurrency
 resource limited, cache off); it returns a **new** context per call, as those workers
 inline the spec into every open today, and `capped_context_spec` exports the dict for
@@ -518,7 +521,8 @@ this document and that one differ, fix one of them.
    one probe key counts as "missing" when another probe key was read under the same way,
    and only when every probe key is refused is the next way tried (see Open question 4).
    Metadata reads retry for about five seconds in total (`max_retries 5, initial_delay
-   0.2s, max_delay 2s`); data reads keep tensorstore's defaults plus `Retry`.
+   0.2s, max_delay 2s`); data reads keep tensorstore's defaults plus `Retry`, and each
+   document read and store open is also timed out under `Retry` (0.7, Behaviour 6).
 
 6. **Writing is reading inverted, in one format.** The package writes OME-NGFF zarr only:
    zarr v2 with OME 0.4 (`.zgroup`, and `.zattrs` holding `multiscales`, each entry with
@@ -646,9 +650,21 @@ this document and that one differ, fix one of them.
    still land after its cancel, with the same bytes the next attempt writes. Each retry is
    logged as a warning naming the error's first line, or its type when the message is
    empty (tensorstore's timeout is `TimeoutError()`); when every attempt times out the
-   `TimeoutError` names the operation, the path and the attempts. Metadata documents are not
-   retried this way (Convention 5). The policy is a per-handle attribute and a per-read and
-   per-write argument, never only constructor state.
+   `TimeoutError` names the operation, the path and the attempts. The policy is a
+   per-handle attribute and a per-read and per-write argument, never only constructor
+   state.
+
+   Documents and opens (0.7): every metadata document read or written and every store
+   opened is timed out and retried the same way, as an operation of no bytes (by default
+   5 s times the attempt, ten attempts), with tensorstore's own request retries
+   (Convention 5) running inside each attempt. A server that accepts a connection and
+   never answers used to hold `wa.open` and `wa.metadata` forever: cellmap-flow's picker
+   kept a 5 s probe of its own in front of them (2026-10-23). The policy is the `retries`
+   of the public call doing the reading: `open`, `open_pyramid`, `create`, `metadata`,
+   `detect` and `open_tensorstore`, which all take it; a document read later outside such
+   a call (a zarr v2 `.zattrs` read on demand, a pyramid placing its levels) waits under
+   `Retry()`. Listings are not timed out, their size being unknown, and neither is the
+   open that creates an array, which a second attempt would find already made.
 
    The watchdog (0.5): tensorstore up to at least 0.1.85 can strand a queued task. A pool
    worker idle for 20 ms exits, and a task queued as it exits gets no thread while the pool
@@ -684,13 +700,27 @@ this document and that one differ, fix one of them.
    task (a 256^3 block write took 2.1 s against 1.3 s uncapped) but not a job of many
    workers, whose throughput its cores bound either way (four writers: 1.8 s against
    1.9 s). A process that wants more threads than its job has slots, a server under
-   `bsub -n 1` say, asks for more slots or passes its own context (decided 2026-10-23,
-   over chunkmirage's objection, once the watchdog had removed the hang risk).
+   `bsub -n 1` say, asks for more slots or says so: `shared_context(cache_bytes,
+   threads=N)` limits both pools to `N`, and `threads=0` writes no limit, leaving
+   tensorstore's own sizes (decided 2026-10-23, over chunkmirage's objection, once the
+   watchdog had removed the hang risk; the argument is 0.6, at chunkmirage's request, so
+   its server keeps one watched context per cache budget for every store it opens
+   instead of building its own). Each `(cache_bytes, threads)` is one context for the
+   process, made on first use and watched; `threads=None` and the number
+   `thread_budget()` returns are two contexts, as a pickle carries `None` for the other
+   process to size by its own job and a number as that number.
    `capped_context` sets `data_copy_concurrency`, `file_io_concurrency`,
    `gcs_request_concurrency`, `s3_request_concurrency` and `http_request_concurrency` to
    `concurrency` and the cache to `cache_bytes`, and returns a new context each call, so two
    handles never share one capped pool unless the caller passes the same object twice; a
-   `concurrency` of 1 is safe under Behaviour 6's watchdog. Both are plain tensorstore contexts; a caller may pass its own, at construction or through
+   `concurrency` of 1 is safe under Behaviour 6's watchdog, but a read or write spanning
+   several chunks then handles them one at a time. On /nrs, from one process alternating
+   limits op by op (c-elegans, 2026-10-23): a one-chunk block (128^3 uint64) read in 18 to
+   19 ms and wrote in 51 ms at both 1 and 4. An 8-chunk block (256^3, what a downsampling
+   step reads) read in 157 ms against 73 and wrote in 312 ms against 158 in a two-slot
+   job, but in a one-slot job read in 147 ms against 119 and wrote in 325 ms against 302
+   (and that job still used 1.3 cores, LSF not pinning it). Most of the gain is a core the
+   worker was not given, so a worker per slot, as a dask job runs, loses little at 1. Both are plain tensorstore contexts; a caller may pass its own, at construction or through
    `reopen(context=)` when a long-lived worker changes its concurrency (cellmap-analyze
    reopens tensorstore on every read today for that reason).
 
@@ -769,8 +799,8 @@ Behaviours 10 to 16 are 0.2; 17 to 19 are 0.3.
     unchanged, and a handle pickles by its arguments and its context's spec, so each worker
     process opens its own cheaply (a local path goes resolved against the working directory
     at open, so a worker that changes directory before unpickling, as cellmap-analyze's dask
-    workers do, opens the same store; `reopen` uses it too), on the shared context when the handle had one, else on
-    a context rebuilt once per process from the same spec, so a capped context stays
+    workers do, opens the same store; `reopen` uses it too), on the shared context when the handle had one (the same
+    `threads`, and the same cache budget), else on a context rebuilt once per process from the same spec, so a capped context stays
     capped (mesh-n-bone's per-process cache and thread caps).
 
 12. **Group attributes belong to everyone.** The package rewrites only the `multiscales`
@@ -1000,7 +1030,10 @@ nothing resolves) cancelled and retried, the warning naming `TimeoutError`, and
 after the last attempt; a capped context and a caller's context passed to an open both
 kicked by the watchdog, a dead context no longer watched, and a kick landing on a capped
 context's own pools; `thread_budget` from each variable, a bad value skipped, and the
-shared context's limits. Opt-in (`WORLDARRAY_STRESS=1`, about a minute,
+shared context's limits. 0.6: `shared_context(threads=)` at `None`, `N` and `0`, one
+context per pair, a negative count refused, and a pickled handle coming back on the same
+shared context. 0.7: `metadata`, `open` and `detect` against a server
+that accepts and never answers raising `TimeoutError` after the attempts. Opt-in (`WORLDARRAY_STRESS=1`, about a minute,
 `tests/test_lost_wakeup.py`): worker processes aim tasks at the moment a one-thread pool's
 worker exits idle, so a plain context strands some for good (13 of about 7,000 on
 tensorstore 0.1.78; 40 to 44 of 12,000 to 15,000 with 16 workers on 0.1.75 and 0.1.85)
@@ -1141,13 +1174,20 @@ over funlib geometry, by this spec's own assignment.
    a real bucket with listing disabled. One network test before 0.1.0 may change the rule.
 4. **The network test** against a non-integer-voxel OpenOrganelle volume (Tests) is not yet
    written; it needs the dataset chosen from the catalog.
-5. **Legacy attributes on new outputs.** cellmap-analyze writes funlib `voxel_size` and
-   `offset` (holding the centre) on every output and nothing in the five repos reads them;
-   dacapo's checkout has nine `open_ds` users, and cellmap-data and fibsem-tools were not
-   checked. Does any live consumer of cellmap-analyze or cellmap-flow outputs read funlib
-   attributes? If so, should the package offer `funlib_attrs(voxel_size, translation, axes,
-   units)`, writing `offset` as the corner, funlib's meaning, for `create(attrs=)`, or do
-   the repos keep writing their own?
+5. **Legacy attributes on new outputs.** Resolved 2026-10-23: nobody needs them, so there
+   is no `funlib_attrs`. cellmap-analyze stopped writing funlib `voxel_size` and `offset`
+   (which held the centre) in 0.6.0, whose outputs carry only the OME documents `create`
+   writes. cellmap-flow writes them through funlib's own `prepare_ds`, `offset` as the
+   corner beside an agreeing OME 0.4 translation, and is their only reader. dacapo reads
+   funlib attributes through `open_ds` (`ome_metadata=False` by default), but nothing feeds
+   it these outputs, and on a zarr v3 one funlib 0.7 silently reads voxel size 1 and
+   offset 0. cellmap-data and fibsem-tools read OME only, and both pin `zarr<3`. funlib
+   0.6 and 0.7 also require integer offsets and voxel sizes, so a float helper here would
+   either refuse placements or write numbers funlib truncates; a repo that wants the keys
+   passes them through `create(attrs=)`. Found on the way, outside this package:
+   cellmap-data (main 356c3ac, `image.py`) takes an OME translation as voxel 0's corner,
+   half a voxel off on every OME-correct store, and funlib's `open_ome_ds` puts the
+   translation straight into its corner `offset`.
 6. **The block iterator's shape** (Behaviour 17). Resolved 2026-10-22: both sessions
    confirmed it. mesh-n-bone asked that `box` may reach past the array and that `clip`
    clips read boxes only, and derives its per-level boxes from one grid itself;
@@ -1178,7 +1218,7 @@ over funlib geometry, by this spec's own assignment.
    unlisted has its placement nowhere until it is listed, so a `create(listed=False)`
    handle, and its pickles, would carry the placement from its arguments alone. Decide
    whether that is worth having, or whether an unlisted level should describe itself on
-   the array (open question 5's funlib attributes, or an OME-shaped array attribute).
+   the array (an OME-shaped array attribute; funlib keys are out, open question 5).
 12. **Per-open object-store credentials.** cellmap-flow's MinIO writers open with a
    per-session key, secret and endpoint while the same process reads public S3
    anonymously. tensorstore 0.1.85's `aws_credentials` takes no inline key and secret, and

{worldarray-0.5.0 → worldarray-0.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "worldarray"
-version = "0.5.0"
+version = "0.7.0"
 description = "Read zarr, N5 and Neuroglancer precomputed volumes in world coordinates through tensorstore: correct voxel placement, pyramid levels, padded region reads, resampling. numpy + tensorstore only."
 readme = "README.md"
 license = { file = "LICENSE" }

{worldarray-0.5.0 → worldarray-0.7.0}/src/worldarray/__init__.py

@@ -28,7 +28,7 @@ from .write import (
     parse_chunk_key,
 )
 
-__version__ = "0.5.0"
+__version__ = "0.7.0"
 
 __all__ = [
     "ArrayMeta",

{worldarray-0.5.0 → worldarray-0.7.0}/src/worldarray/array.py

@@ -44,6 +44,7 @@ from .store import (
     normalize_path,
     open_array,
     precomputed_scale_index,
+    retried_documents,
 )
 
 logger = logging.getLogger(__name__)
@@ -68,6 +69,7 @@ class WorldArray:
     ``voxel_size`` picks the level; a group with neither raises, listing its levels).
     """
 
+    @retried_documents
     def __init__(
         self,
         path: str,
@@ -1002,6 +1004,7 @@ class Pyramid:
     own metadata where it lists the levels' scales (OME-NGFF, COSEM N5, precomputed), and
     opens the levels only when it does not."""
 
+    @retried_documents
     def __init__(
         self,
         path: str,
@@ -1201,6 +1204,7 @@ class Pyramid:
 # Functions
 
 
+@retried_documents
 def create(
     path: str,
     shape: Sequence[int] | None = None,
@@ -1345,14 +1349,21 @@ def open_pyramid(path: str, **kwargs) -> Pyramid:
     return Pyramid(path, **kwargs)
 
 
+@retried_documents
 def metadata(
-    path: str, *, level: str | int | None = None, units: Units = None, open: bool = True
+    path: str,
+    *,
+    level: str | int | None = None,
+    units: Units = None,
+    open: bool = True,
+    retries: Retry | None = None,
 ) -> ArrayMeta:
     """The ``ArrayMeta`` of an array (or of ``level`` under a group), in native units unless
     ``units`` says otherwise. ``open=False`` reads it from the documents alone, with no
-    tensorstore opened."""
+    tensorstore opened. ``retries`` times out and retries each document read and the open
+    (Behaviour 6)."""
     if open:
-        vol = WorldArray(path, level=level, units=units)
+        vol = WorldArray(path, level=level, units=units, retries=retries)
         return vol.meta if units is None else vol.grid
     node = detect(path)
     if level == "first":
@@ -1386,13 +1397,18 @@ def metadata(
     return meta.in_units(space=space, time=time)
 
 
+@retried_documents
 def open_tensorstore(
-    path: str, *, level: str | int | None = None, context: ts.Context | None = None
+    path: str,
+    *,
+    level: str | int | None = None,
+    context: ts.Context | None = None,
+    retries: Retry | None = None,
 ) -> ts.TensorStore:
     """The raw tensorstore of an array or level: C order, origin 0, a precomputed single
-    channel dropped. The access fallbacks and shared context apply. No world metadata is
-    read, so an array whose placement attributes are odd (a coordinate array, a field) still
-    opens."""
+    channel dropped. The access fallbacks and shared context apply, and ``retries`` to each
+    document read and the open (Behaviour 6). No world metadata is read, so an array whose
+    placement attributes are odd (a coordinate array, a field) still opens."""
     context = context if context is not None else shared_context(0)
     node = detect(path)
     if level == "first":

{worldarray-0.5.0 → worldarray-0.7.0}/src/worldarray/context.py

@@ -1,12 +1,14 @@
 """Tensorstore contexts: one shared per cache budget, or a capped one per call.
 
-``shared_context(cache_bytes)`` is one ``tensorstore.Context`` per budget for the whole
-process, so the levels of a pyramid and repeat opens of an array draw on one cache pool.
-The default budget is 0: tensorstore's decoded-chunk cache is off, because every consumer
-either reads each region once or caches decoded chunks itself. Its decoding and file I/O
-pools are sized by ``thread_budget()``, from the variables a cluster job sets, where
-tensorstore alone would size them to every CPU on the node. Every context here, and every
-one a store is opened on, is watched for tensorstore's lost wakeup (``watch``).
+``shared_context(cache_bytes, threads=)`` is one ``tensorstore.Context`` per budget and
+thread limit for the whole process, so the levels of a pyramid and repeat opens of an
+array draw on one cache pool. The default budget is 0: tensorstore's decoded-chunk cache
+is off, because every consumer either reads each region once or caches decoded chunks
+itself. By default its decoding and file I/O pools are sized by ``thread_budget()``, from
+the variables a cluster job sets, where tensorstore alone would size them to every CPU on
+the node; ``threads=0`` leaves them at tensorstore's sizes (chunkmirage's server). Every
+context here, and every one a store is opened on, is watched for tensorstore's lost wakeup
+(``watch``).
 
 ``capped_context`` is mesh-n-bone's worker-process context: every tensorstore concurrency
 resource limited, so dozens of worker processes on one host do not multiply into thousands
@@ -26,7 +28,7 @@ from typing import Any
 
 import tensorstore as ts
 
-_shared: dict[int, ts.Context] = {}
+_shared: dict[tuple[int, int | None], ts.Context] = {}
 _rebuilt: dict[str, ts.Context] = {}
 _lock = threading.Lock()
 
@@ -136,20 +138,23 @@ def thread_budget() -> int:
         return os.cpu_count() or 1
 
 
-def shared_context(cache_bytes: int = 0) -> ts.Context:
+def shared_context(cache_bytes: int = 0, *, threads: int | None = None) -> ts.Context:
+    """The process's one context for ``cache_bytes`` and ``threads``, made on first use and
+    watched. ``threads`` limits its decoding and file I/O pools: ``None`` to
+    ``thread_budget()``, a positive number to that many, ``0`` not at all (tensorstore's own
+    sizes, every CPU on the node)."""
+    key = (int(cache_bytes), None if threads is None else int(threads))
+    if key[1] is not None and key[1] < 0:
+        raise ValueError(f"threads must be None, 0 or positive, not {threads!r}")
     with _lock:
-        ctx = _shared.get(int(cache_bytes))
+        ctx = _shared.get(key)
         if ctx is None:
-            limit = {"limit": thread_budget()}
-            ctx = ts.Context(
-                {
-                    "cache_pool": {"total_bytes_limit": int(cache_bytes)},
-                    "data_copy_concurrency": limit,
-                    "file_io_concurrency": limit,
-                }
-            )
-            watch(ctx)
-            _shared[int(cache_bytes)] = ctx
+            spec: dict = {"cache_pool": {"total_bytes_limit": key[0]}}
+            limit = thread_budget() if threads is None else key[1]
+            if limit:
+                spec["data_copy_concurrency"] = spec["file_io_concurrency"] = {"limit": limit}
+            ctx = watch(ts.Context(spec))
+            _shared[key] = ctx
         return ctx
 
 
@@ -157,7 +162,9 @@ def capped_context_spec(cache_bytes: int = 0, concurrency: int = 1, network: boo
     """The spec dict behind ``capped_context``, for callers building their own opens.
     ``network=False`` caps only decoding and local file I/O, leaving tensorstore's request
     pools alone (cellmap-analyze's choice for remote reads). A ``concurrency`` of 1 is safe:
-    the context is watched (see ``KICK_EVERY``)."""
+    the context is watched (see ``KICK_EVERY``). It is not free: a read or write spanning
+    several chunks then decodes and stores them one at a time, half the speed of 4 on
+    8-chunk blocks in a two-slot job, 7 to 19% slower in a one-slot one (spec, Behaviour 7)."""
     limit = {"limit": int(concurrency)}
     spec = {
         "data_copy_concurrency": limit,
@@ -179,12 +186,14 @@ def capped_context(cache_bytes: int = 0, concurrency: int = 1, network: bool = T
 
 def recipe_of(context: ts.Context) -> tuple[str, Any]:
     """How another process rebuilds ``context``: ``("shared", cache_bytes)`` for a shared
-    context, else ``("spec", json)`` from the context's own spec, so a capped context stays
-    capped across a pickle (Behaviour 11, mesh-n-bone's workers)."""
+    context at the default limit (the other process takes its own ``thread_budget()``),
+    ``("shared", (cache_bytes, threads))`` for one with ``threads`` given, else
+    ``("spec", json)`` from the context's own spec, so a capped context stays capped across
+    a pickle (Behaviour 11, mesh-n-bone's workers)."""
     with _lock:
-        for cache_bytes, ctx in _shared.items():
+        for (cache_bytes, threads), ctx in _shared.items():
             if ctx is context:
-                return ("shared", cache_bytes)
+                return ("shared", cache_bytes if threads is None else (cache_bytes, threads))
     spec = context.spec
     return ("spec", json.dumps(spec.to_json() if spec is not None else {}, sort_keys=True))
 
@@ -196,7 +205,8 @@ def from_recipe(recipe: tuple[str, Any] | None) -> ts.Context | None:
         return None
     kind, value = recipe
     if kind == "shared":
-        return shared_context(int(value))
+        cache_bytes, threads = value if isinstance(value, (tuple, list)) else (value, None)
+        return shared_context(int(cache_bytes), threads=threads)
     with _lock:
         ctx = _rebuilt.get(value)
         if ctx is None:

{worldarray-0.5.0 → worldarray-0.7.0}/src/worldarray/store.py

@@ -8,6 +8,8 @@ one in, with credentials only when a public read is refused (spec, Conventions 5
 
 from __future__ import annotations
 
+import contextlib
+import contextvars
 import functools
 import json
 import logging
@@ -21,6 +23,7 @@ from typing import Any, TypeVar
 import tensorstore as ts
 
 from .context import watch
+from .retry import Retry, with_retries
 
 logger = logging.getLogger(__name__)
 T = TypeVar("T")
@@ -33,6 +36,11 @@ PROBE_KEYS = ("zarr.json", ".zarray", ".zgroup", "info", "attributes.json")
 _SCHEME = re.compile(r"^(s3|gs|http|https|file|memory)://")
 _REMOTE = ("s3://", "gs://", "http://", "https://")
 
+# the Retry that documents and opens wait under: the policy of the public call they serve
+_documents_retry: contextvars.ContextVar[Retry | None] = contextvars.ContextVar(
+    "worldarray_documents_retry", default=None
+)
+
 
 # ---------------------------------------------------------------------------
 # Paths
@@ -189,6 +197,47 @@ def with_access(specs: Sequence[dict], attempt: Callable[[dict], T]) -> T:
     return result
 
 
+@contextlib.contextmanager
+def documents_retried(retry: Retry | None):
+    """Metadata documents read or written, and stores opened, inside this block are timed
+    out and retried per ``retry`` as reads of no bytes; ``None`` keeps the enclosing
+    policy, else ``Retry()`` (spec, Behaviour 6)."""
+    token = _documents_retry.set(retry) if retry is not None else None
+    try:
+        yield
+    finally:
+        if token is not None:
+            _documents_retry.reset(token)
+
+
+def retried_documents(fn: Callable[..., T]) -> Callable[..., T]:
+    """``fn`` run under ``documents_retried(retries)``, ``retries`` being its keyword."""
+
+    @functools.wraps(fn)
+    def run(*args, **kwargs):
+        with documents_retried(kwargs.get("retries")):
+            return fn(*args, **kwargs)
+
+    return run
+
+
+def _where(spec: dict, key: str = "") -> str:
+    if spec["driver"] == "file":
+        return spec["path"] + key
+    if spec["driver"] == "http":
+        return spec["base_url"].rstrip("/") + "/" + spec.get("path", "").lstrip("/") + key
+    return f"{spec['driver']}://{spec.get('bucket', '')}/{spec.get('path', '')}{key}"
+
+
+def _wait(start: Callable[[], Any], spec: dict, what: str, verb: str) -> Any:
+    """``start().result()`` for a document or an open: a server that accepts and never
+    answers would otherwise hold it forever (tensorstore's own request retries run inside
+    each attempt)."""
+    retry = _documents_retry.get() or Retry()
+    remote = spec["driver"] in ("http", "gcs", "s3")
+    return with_retries(start, 0, retry, remote=remote, what=what, verb=verb)
+
+
 def metadata_spec(spec: dict) -> dict:
     """``spec`` with its requests' retries capped, so a mistyped host fails in seconds."""
     if spec["driver"] not in ("s3", "gcs", "http"):
@@ -201,14 +250,15 @@ def _metadata_kvstore(spec_json: str, gcs_url: str | None) -> ts.KvStore:
     """One opened kvstore per spec, and per ``TENSORSTORE_GCS_HTTP_URL``: tensorstore's gcs
     driver reads that variable when the store is opened, so a cached store would keep
     talking to the host the variable named then."""
-    return ts.KvStore.open(metadata_spec(json.loads(spec_json))).result()
+    spec = json.loads(spec_json)
+    return _wait(lambda: ts.KvStore.open(metadata_spec(spec)), spec, _where(spec), "open")
 
 
 def _read_key(spec: dict, key: str) -> bytes | None:
     kv = _metadata_kvstore(
         json.dumps(spec, sort_keys=True), os.environ.get("TENSORSTORE_GCS_HTTP_URL")
     )
-    result = kv.read(key).result()
+    result = _wait(lambda: kv.read(key), spec, _where(spec, key), "read")
     return None if result.state == "missing" else bytes(result.value)
 
 
@@ -216,7 +266,7 @@ def _write_key(spec: dict, key: str, data: bytes) -> None:
     kv = _metadata_kvstore(
         json.dumps(spec, sort_keys=True), os.environ.get("TENSORSTORE_GCS_HTTP_URL")
     )
-    kv.write(key, data).result()
+    _wait(lambda: kv.write(key, data), spec, _where(spec, key), "write")
 
 
 class Location:
@@ -323,12 +373,14 @@ def _decode(docs: dict[str, bytes | None]) -> dict[str, Any]:
     return {k: json.loads(v) for k, v in docs.items() if v is not None}
 
 
-def detect(path: str, *, missing_ok: bool = False) -> Node | None:
+@retried_documents
+def detect(path: str, *, missing_ok: bool = False, retries: Retry | None = None) -> Node | None:
     """Probe ``zarr.json``, ``.zarray``, ``.zgroup``, ``info``, ``attributes.json`` in that
     order; failing all, probe the parent for ``info`` and take the last path segment as a
     precomputed scale key, then probe ``s0`` for a group that has no document of its own.
     The ``.n5`` or ``.zarr`` suffix is never consulted. Nothing found raises
-    ``FileNotFoundError``, or returns None with ``missing_ok``."""
+    ``FileNotFoundError``, or returns None with ``missing_ok``. ``retries`` times out and
+    retries each document read (Behaviour 6)."""
     path = normalize_path(path)
     docs = _decode(Location(path).read_many(PROBE_KEYS))
     if "zarr.json" in docs:
@@ -440,7 +492,12 @@ def open_array(
             spec["assume_metadata"] = True
         if node.format == "precomputed" and scale_index is not None:
             spec["scale_index"] = scale_index
-        return ts.open(spec, read=True, write=write, context=watch(context)).result()
+        return _wait(
+            lambda: ts.open(spec, read=True, write=write, context=watch(context)),
+            kvstore,
+            node.path,
+            "open",
+        )
 
     store = with_access(kvstore_specs(node.path), attempt)
     if node.format in ("n5", "precomputed"):

{worldarray-0.5.0 → worldarray-0.7.0}/tests/test_access.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 import gc
 import logging
 import os
+import socket
+import threading
 import time
 import weakref
 
@@ -301,3 +303,58 @@ def test_gce_guard_is_left_alone_on_a_google_machine(monkeypatch):
     monkeypatch.setattr(store, "_on_google_cloud", lambda: False)
     store.kvstore_specs("gs://bucket/vol")
     assert os.environ["GCE_METADATA_ROOT"] == "metadata.google.internal.invalid"
+
+
+def test_a_shared_context_takes_the_callers_thread_count(monkeypatch):
+    monkeypatch.setenv("OMP_NUM_THREADS", "1")
+    monkeypatch.setattr(wa.context, "_shared", {})
+    default = wa.shared_context(7)
+    assert default.spec.to_json()["data_copy_concurrency"] == {"limit": 1}
+    four = wa.shared_context(7, threads=4).spec.to_json()
+    assert four["data_copy_concurrency"] == four["file_io_concurrency"] == {"limit": 4}
+    machine = wa.shared_context(7, threads=0)
+    assert machine is wa.shared_context(7, threads=0) and machine is not default
+    spec = machine.spec.to_json()
+    assert "data_copy_concurrency" not in spec and "file_io_concurrency" not in spec
+    assert spec["cache_pool"] == {"total_bytes_limit": 7}
+    assert machine in wa.context._watched
+    assert wa.context.recipe_of(machine) == ("shared", (7, 0))
+    assert wa.context.from_recipe(("shared", (7, 0))) is machine
+    assert wa.context.from_recipe(("shared", 7)) is default
+    with pytest.raises(ValueError):
+        wa.shared_context(threads=-1)
+
+
+@pytest.fixture
+def silent_server():
+    """An http address whose server accepts every connection and never answers."""
+    srv = socket.socket()
+    srv.bind(("127.0.0.1", 0))
+    srv.listen(16)
+    held = []
+
+    def accept():
+        while True:
+            try:
+                held.append(srv.accept()[0])
+            except OSError:
+                return
+
+    threading.Thread(target=accept, daemon=True).start()
+    yield f"http://127.0.0.1:{srv.getsockname()[1]}"
+    srv.close()
+    for c in held:
+        c.close()
+
+
+def test_a_server_that_never_answers_times_out_the_open(silent_server):
+    quick = wa.Retry(timeout=0.3, max_attempts=2, base_delay=0.0)
+    for open_it in (
+        lambda url: wa.metadata(url, open=False, retries=quick),
+        lambda url: wa.open(url, retries=quick),
+        lambda url: wa.detect(url, retries=quick),
+    ):
+        start = time.monotonic()
+        with pytest.raises(TimeoutError, match="127.0.0.1.*2 attempts"):
+            open_it(f"{silent_server}/vol.zarr/s0")
+        assert time.monotonic() - start < 10

{worldarray-0.5.0 → worldarray-0.7.0}/tests/test_write.py

@@ -826,6 +826,8 @@ def test_a_pickled_handle_keeps_its_context(tmp_path):
     )  # one rebuilt context per spec
     shared = pickle.loads(pickle.dumps(wa.open(path)))
     assert shared.context is wa.shared_context(0)
+    machine = wa.shared_context(0, threads=0)
+    assert pickle.loads(pickle.dumps(wa.open(path, context=machine))).context is machine
 
 
 def test_codecs_of_a_path_and_the_fill_write_that_removes_a_key(tmp_path):

{worldarray-0.5.0 → worldarray-0.7.0}/uv.lock

@@ -1247,7 +1247,7 @@ wheels = [
 
 [[package]]
 name = "worldarray"
-version = "0.5.0"
+version = "0.7.0"
 source = { editable = "." }
 dependencies = [
     { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },