arrayview 0.47.4__tar.gz → 0.47.5__tar.gz

{arrayview-0.47.4 → arrayview-0.47.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: arrayview
-Version: 0.47.4
+Version: 0.47.5
 Summary: Fast multi-dimensional array viewer
 Project-URL: Home, https://github.com/oscarvanderheide/arrayview
 Project-URL: Source, https://github.com/oscarvanderheide/arrayview

{arrayview-0.47.4 → arrayview-0.47.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "arrayview"
-version = "0.47.4"
+version = "0.47.5"
 description = "Fast multi-dimensional array viewer"
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.12"

{arrayview-0.47.4 → arrayview-0.47.5}/src/arrayview/_io.py

@@ -3,11 +3,14 @@
 from __future__ import annotations
 
 import fnmatch
+import functools
 import gzip
 import glob
 import os
 import re
+import sys
 import threading
+import zlib
 from concurrent.futures import ThreadPoolExecutor
 from collections import OrderedDict
 
@@ -114,6 +117,23 @@ def list_npz_keys(filepath):
     return keys
 
 
+# MATLAB class -> dtype scipy.io.loadmat returns for it (complex is not
+# visible in the header; the picker only shows this when several arrays exist).
+_MAT_CLASS_DTYPES = {
+    "double": "float64",
+    "single": "float32",
+    "int8": "int8",
+    "uint8": "uint8",
+    "int16": "int16",
+    "uint16": "uint16",
+    "int32": "int32",
+    "uint32": "uint32",
+    "int64": "int64",
+    "uint64": "uint64",
+    "logical": "uint8",
+}
+
+
 def list_mat_keys(filepath):
     """Return [{key, shape, dtype}] for each array in a .mat file.
 
@@ -123,6 +143,17 @@ def list_mat_keys(filepath):
     try:
         import scipy.io
 
+        # The headers alone settle the common single-array file, whose array
+        # is loaded right after this anyway; reading it here too doubled the
+        # wait on network shares.
+        numeric = [
+            (name, shape, cls)
+            for name, shape, cls in scipy.io.whosmat(filepath)
+            if cls in _MAT_CLASS_DTYPES
+        ]
+        if len(numeric) == 1:
+            name, shape, cls = numeric[0]
+            return [{"key": name, "shape": list(shape), "dtype": _MAT_CLASS_DTYPES[cls]}]
         mat = scipy.io.loadmat(filepath)
         keys = []
         for k, v in mat.items():
@@ -435,12 +466,100 @@ def _load_nifti_with_meta(filepath):
     random-access, but it is no longer silently reordered.
     """
     img, meta = _nifti_header_with_meta(filepath, native=True)
-    arr = (
-        img.dataobj
-        if filepath.endswith(".nii")
-        else _nifti_display_array(img.dataobj)
-    )
-    return arr, meta
+    if filepath.endswith(".nii"):
+        return img.dataobj, meta
+    source = img
+    if _nifti_voxel_bytes(img) <= npy_eager_limit():
+        raw = read_gzip_file_parallel(filepath)
+        if raw is not None:
+            try:
+                source = type(img).from_bytes(raw)
+            except Exception:
+                source = img
+    return _nifti_display_array(source.dataobj), meta
+
+
+def _nifti_voxel_bytes(img) -> int:
+    try:
+        return int(np.prod(img.shape)) * np.dtype(img.get_data_dtype()).itemsize
+    except Exception:
+        return 1 << 62
+
+
+_GZIP_READ_CHUNK_BYTES = 8 << 20
+_GZIP_READ_THREADS = 8
+
+
+def read_gzip_file_parallel(filepath):
+    """Decompress a whole .gz file, reading it with several reads in flight.
+
+    A gzip stream can only be decoded front to back, so the decode is one
+    thread; but the bytes can arrive out of order. Reading 8 MB chunks in
+    parallel and decoding each as soon as its turn comes overlaps the two, so
+    on an SMB share the wait is close to the parallel read alone (~1.3 s for a
+    110 MB file) instead of a single slow reader followed by the decode.
+    Handles multi-member files. Returns None on any read or decode error so
+    the caller can fall back to its ordinary reader.
+    """
+    try:
+        size = os.path.getsize(filepath)
+        fd = os.open(filepath, os.O_RDONLY)
+    except OSError:
+        return None
+    buf = bytearray(size)
+    view = memoryview(buf)
+    starts = list(range(0, size, _GZIP_READ_CHUNK_BYTES))
+    ready = [threading.Event() for _ in starts]
+    failed = []
+
+    def _read(i):
+        start = starts[i]
+        stop = min(start + _GZIP_READ_CHUNK_BYTES, size)
+        pos = start
+        try:
+            while pos < stop:
+                n = os.preadv(fd, [view[pos:stop]], pos)
+                if n <= 0:
+                    raise OSError("unexpected end of file")
+                pos += n
+        except Exception as exc:  # noqa: BLE001 - reported to the decoder
+            failed.append(exc)
+        finally:
+            ready[i].set()
+
+    out = []
+    try:
+        with ThreadPoolExecutor(
+            max_workers=max(1, min(_GZIP_READ_THREADS, len(starts))),
+            thread_name_prefix="arrayview-gzip-read",
+        ) as pool:
+            for i in range(len(starts)):
+                pool.submit(_read, i)
+            decoder = zlib.decompressobj(wbits=31)
+            fed = False
+            for i, start in enumerate(starts):
+                ready[i].wait()
+                if failed:
+                    return None
+                chunk = view[start:min(start + _GZIP_READ_CHUNK_BYTES, size)]
+                while chunk:
+                    fed = True
+                    out.append(decoder.decompress(chunk))
+                    if not decoder.eof:
+                        break
+                    # End of one gzip member: any rest starts the next one.
+                    chunk = decoder.unused_data
+                    decoder = zlib.decompressobj(wbits=31)
+                    fed = False
+            if fed and not decoder.eof:
+                return None  # truncated: let the ordinary reader report it
+            if not out:
+                return None
+    except (zlib.error, OSError):
+        return None
+    finally:
+        os.close(fd)
+    return b"".join(out)
 
 
 # ---------------------------------------------------------------------------
@@ -1601,41 +1720,220 @@ def mmap_first_frame_is_cheap(arr, total_bytes) -> bool:
     return runs <= max_runs and faulted <= fraction * total_bytes
 
 
-def _read_npy_reporting_progress(mapped, progress):
-    """Copy a memmapped .npy into RAM in chunks, saying how far along it is.
+# Fetching the first frame's pages one by one beats reading the whole file
+# only on a local disk: on an SMB share the same 65k small reads of a 1.6 GB
+# file took 11 s against 0.7 s locally.
+_FIRST_FRAME_PREFETCH_MAX_PAGES = 1 << 17
+_FIRST_FRAME_PREFETCH_THREADS = 16
+
+
+def _on_known_local_disk(filepath) -> bool:
+    if not sys.platform.startswith("linux"):
+        return False
+    try:
+        from ._source_safety import _KNOWN_LOCAL_STAGE_FILESYSTEMS, mount_for_path
+
+        mount = mount_for_path(os.path.realpath(filepath))
+    except Exception:
+        return False
+    return mount is not None and mount.filesystem in _KNOWN_LOCAL_STAGE_FILESYSTEMS
 
-    For a large array this read is the whole wait between a click and the
-    first frame — 9 s for a 1.6 GB file on a cold cache — and ``np.load``
-    reports nothing while it runs.  Copying in pieces costs the same reads
-    and makes the wait explainable.
 
-    The flat view follows the array's own memory order, so the copy stays
-    sequential whether the file is C- or Fortran-ordered.  A .npy memmap is
-    always contiguous in one of the two, so both reshapes are views.
+def _viewer_start_index(shape, dims):
+    """The slice the viewer opens on; mirrors its ``_initialIndices``."""
+    ref = max(shape[dims[0]], shape[dims[1]])
+    return tuple(
+        size // 2 if (i in dims or size >= 0.25 * ref) else 0
+        for i, size in enumerate(shape)
+    )
+
+
+def prefetch_first_frame_pages(mapped, filepath) -> bool:
+    """Pull just the pages of the viewer's first frame into memory, in parallel.
+
+    When the first frame is one element per page across the whole file (a
+    C-order array viewed along its leading axes), reading the file first made
+    a 1.6 GB array wait 3-4 s cold for its picture. Fetching only the pages the
+    frame touches, many at a time, takes ~0.7 s; the memmap then renders the
+    frame from memory while the full read continues in the background.
+
+    Returns False, having read nothing, when this would not pay off: not a
+    known local disk, the viewer would start elsewhere, or too many pages.
     """
-    order = "F" if mapped.flags.f_contiguous and not mapped.flags.c_contiguous else "C"
-    out = np.empty(mapped.shape, dtype=mapped.dtype, order=order)
-    src = mapped.reshape(-1, order="A")
-    dst = out.reshape(-1, order="A")
-    total = int(src.shape[0])
-    itemsize = int(mapped.dtype.itemsize)
-    # ~200 updates: smooth enough to read as motion, coarse enough that the
-    # per-chunk overhead stays lost in the read itself.
-    step = max(1, total // 200)
-    done = 0
-    while done < total:
-        end = min(done + step, total)
-        dst[done:end] = src[done:end]
-        done = end
+    from ._session import _default_start_dims_for_data, _viewer_start_dims_for_shape
+
+    shape = tuple(int(s) for s in mapped.shape)
+    if len(shape) < 3 or _default_start_dims_for_data(mapped) is not None:
+        return False
+    dims = _viewer_start_dims_for_shape(shape)
+    if dims is None:
+        return False
+    return prefetch_slice_pages(mapped, filepath, dims, _viewer_start_index(shape, dims))
+
+
+@functools.lru_cache(maxsize=64)
+def _on_known_local_disk_cached(filepath) -> bool:
+    return _on_known_local_disk(filepath)
+
+
+def prefetch_slice_pages(mapped, filepath, dims, index) -> bool:
+    """Fetch the pages of one 2-D slice of a memmap with parallel reads.
+
+    Faulting such a slice in through the map is the slow way: readahead drags
+    in the pages between the wanted elements (6 s for one slice of a 1.6 GB
+    C-order file), and without readahead each page is its own wait (7.7 s).
+    Many small reads in flight take ~0.7 s on a local disk. Only worth it when
+    the slice is scattered one element per page; returns False otherwise, or
+    off a known local disk, having read nothing.
+    """
+    shape = tuple(int(s) for s in mapped.shape)
+    strides = tuple(int(s) for s in mapped.strides)
+    runs, _faulted = first_frame_fault_cost(
+        shape, mapped.dtype.itemsize, strides, dims, mapped.nbytes
+    )
+    if runs <= 1024 or not _on_known_local_disk_cached(str(filepath)):
+        return False
+    base = int(mapped.offset) + sum(
+        int(index[i]) * strides[i] for i in range(len(shape)) if i not in dims
+    )
+    rows = np.arange(shape[dims[0]], dtype=np.int64) * strides[dims[0]]
+    cols = np.arange(shape[dims[1]], dtype=np.int64) * strides[dims[1]]
+    starts = (base + rows[:, None] + cols[None, :]).ravel()
+    ends = starts + (mapped.dtype.itemsize - 1)
+    pages = np.unique(np.concatenate((starts, ends)) // _MMAP_PAGE_BYTES)
+    if pages.size > _FIRST_FRAME_PREFETCH_MAX_PAGES:
+        return False
+
+    try:
+        fd = os.open(filepath, os.O_RDONLY)
+    except OSError:
+        return False
+    try:
+        # Only the pages asked for: readahead around each would multiply the I/O.
         try:
-            progress(done * itemsize, total * itemsize)
-        except Exception:
+            os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_RANDOM)
+        except (AttributeError, OSError):
             pass
+        offsets = (pages * _MMAP_PAGE_BYTES).tolist()
+        n = _FIRST_FRAME_PREFETCH_THREADS
+
+        def _warm(part):
+            for offset in part:
+                os.pread(fd, _MMAP_PAGE_BYTES, offset)
+
+        with ThreadPoolExecutor(max_workers=n, thread_name_prefix="arrayview-slice-prefetch") as pool:
+            for _ in pool.map(_warm, [offsets[k::n] for k in range(n)]):
+                pass
+    except OSError:
+        return False
+    finally:
+        os.close(fd)
+    return True
+
+
+def _h5_contiguous_memmap(filepath, ds):
+    """Map an HDF5 dataset straight from the file, or None if it can't be.
+
+    MATLAB v7.3 writes large arrays as one uncompressed contiguous block, so
+    the same first-frame shortcut as .npy applies: the viewer can draw a slice
+    without reading the whole file first. Chunked, compressed or filtered
+    datasets are not one block and return None.
+    """
+    try:
+        if ds.chunks is not None or ds.compression is not None:
+            return None
+        offset = ds.id.get_offset()
+        if offset is None:
+            return None
+        dtype = ds.dtype
+        if dtype.names:
+            # MATLAB complex: {real, imag} of one float type, packed, which is
+            # byte-for-byte numpy's complex layout.
+            if set(dtype.names) != {"real", "imag"}:
+                return None
+            real_t, real_off = dtype.fields["real"][:2]
+            imag_t, imag_off = dtype.fields["imag"][:2]
+            # Only single precision: the eager path downcasts complex to
+            # complex64, and the mapped array must come out the same.
+            if (
+                real_t != imag_t
+                or real_t.kind != "f"
+                or real_t.itemsize != 4
+                or real_off != 0
+                or imag_off != real_t.itemsize
+                or dtype.itemsize != 2 * real_t.itemsize
+            ):
+                return None
+            dtype = np.dtype(f"{real_t.byteorder}c{2 * real_t.itemsize}")
+        elif dtype.kind not in "biuf":
+            return None
+        if offset + int(np.prod(ds.shape)) * dtype.itemsize > os.path.getsize(filepath):
+            return None
+        return np.memmap(filepath, dtype=dtype, mode="r", offset=offset, shape=ds.shape)
+    except Exception:
+        return None
+
+
+_NPY_READ_CHUNK_BYTES = 8 << 20
+_NPY_READ_THREADS = 8
+
+
+def read_npy_into_ram(mapped, progress=None):
+    """Read a memmapped .npy into RAM with several reads in flight.
+
+    For a large array this read is the whole wait between a click and the
+    first frame. One sequential reader leaves a network share mostly idle:
+    on SMB a 220 MB file took 5.5 s that way and ~3 s with eight readers.
+    Local disks are not slowed by it.
+
+    The reads land directly in the result's memory, in the file's own byte
+    order: a .npy body is contiguous in either C or Fortran order, so the
+    result is allocated in that order and filled byte for byte.  *progress*,
+    if given, is called with ``(done_bytes, total_bytes)`` as chunks land.
+    """
+    order = "F" if mapped.flags.f_contiguous and not mapped.flags.c_contiguous else "C"
+    out = np.empty(mapped.shape, dtype=mapped.dtype, order=order)
+    total = int(out.nbytes)
+    if total == 0:
+        return out
+    dst = memoryview(out.reshape(-1, order="A").view(np.uint8))
+    offset = int(mapped.offset)
+    lock = threading.Lock()
+    done = [0]
+
+    fd = os.open(mapped.filename, os.O_RDONLY)
+    try:
+        def _read(start):
+            stop = min(start + _NPY_READ_CHUNK_BYTES, total)
+            pos = start
+            while pos < stop:
+                n = os.preadv(fd, [dst[pos:stop]], offset + pos)
+                if n <= 0:
+                    raise OSError(f"unexpected end of file reading {mapped.filename!r}")
+                pos += n
+            if progress is not None:
+                with lock:
+                    done[0] += stop - start
+                    finished = done[0]
+                try:
+                    progress(finished, total)
+                except Exception:
+                    pass
+
+        starts = range(0, total, _NPY_READ_CHUNK_BYTES)
+        with ThreadPoolExecutor(
+            max_workers=min(_NPY_READ_THREADS, len(starts)),
+            thread_name_prefix="arrayview-npy-read",
+        ) as pool:
+            for _ in pool.map(_read, starts):
+                pass
+    finally:
+        os.close(fd)
     return out
 
 
 def load_data_with_meta(filepath, key=None, *, load="lazy", stack="auto", select=None,
-                        progress=None):
+                        progress=None, fast_first_frame=False):
     """Like load_data but also returns spatial metadata for NIfTI files.
 
     Returns (array, meta_or_None). meta is None for non-NIfTI formats.
@@ -1661,10 +1959,12 @@ def load_data_with_meta(filepath, key=None, *, load="lazy", stack="auto", select
         return _load_file_series(filepath, load=load, stack=stack)
     if filepath.endswith(".nii") or filepath.endswith(".nii.gz"):
         return _load_nifti_with_meta(filepath)
-    return load_data(filepath, key=key, progress=progress), None
+    return load_data(
+        filepath, key=key, progress=progress, fast_first_frame=fast_first_frame
+    ), None
 
 
-def load_data(filepath, key=None, *, progress=None):
+def load_data(filepath, key=None, *, progress=None, fast_first_frame=False):
     if filepath.lower().endswith(".dcm"):
         from ._dicom import load_dicom_series
 
@@ -1685,16 +1985,19 @@ def load_data(filepath, key=None, *, progress=None):
         # read runs behind it (see mmap_first_frame_is_cheap) or the viewer shows
         # a spinner until the read completes.
         size = os.path.getsize(filepath)
-        if size >= npy_eager_limit():
-            return np.load(filepath, mmap_mode="r")
         mapped = np.load(filepath, mmap_mode="r")
+        if size >= npy_eager_limit():
+            if fast_first_frame:
+                prefetch_first_frame_pages(mapped, filepath)
+            return mapped
         if mmap_first_frame_is_cheap(mapped, size):
             # Session creation upgrades this to an in-RAM array in the background.
             return mapped
-        if progress is not None:
-            return _read_npy_reporting_progress(mapped, progress)
-        del mapped
-        return np.load(filepath)
+        if fast_first_frame and prefetch_first_frame_pages(mapped, filepath):
+            # Same as above: the first frame's pages are already in memory, and
+            # the full read runs behind the viewer.
+            return mapped
+        return read_npy_into_ram(mapped, progress)
     elif filepath.endswith(".npz"):
         npz = np.load(filepath)
         try:
@@ -1766,10 +2069,13 @@ def load_data(filepath, key=None, *, progress=None):
 
             f = h5py.File(filepath, "r")
             arrays = {k: f[k] for k in f.keys() if _is_viewable_h5_dataset(f[k])}
-            if key is not None:
-                return _fix_mat_complex(np.array(arrays[key]))
-            if len(arrays) == 1:
-                return _fix_mat_complex(np.array(next(iter(arrays.values()))))
+            if key is not None or len(arrays) == 1:
+                ds = arrays[key] if key is not None else next(iter(arrays.values()))
+                mapped = _h5_contiguous_memmap(filepath, ds)
+                if mapped is not None and mmap_first_frame_is_cheap(mapped, mapped.nbytes):
+                    # Session creation upgrades this to an in-RAM array in the background.
+                    return mapped
+                return _fix_mat_complex(np.array(ds))
             raise ValueError(
                 f".mat (v7.3) file contains multiple datasets: {list(arrays.keys())}. "
                 "Select one in the viewer or pass a key."

{arrayview-0.47.4 → arrayview-0.47.5}/src/arrayview/_launcher.py

@@ -4853,6 +4853,7 @@ def _serve_daemon(
                     key=_load_key,
                     load=collection_load,
                     stack=collection_stack,
+                    fast_first_frame=True,
                 )
                 dir_overlay_items = None
                 collection_spatial_ndim = None

{arrayview-0.47.4 → arrayview-0.47.5}/src/arrayview/_render.py

@@ -207,7 +207,17 @@ def extract_slice(session, dim_x, dim_y, idx_list, progress=None):
     # already served from cache.
     if progress is None:
         progress = getattr(session, 'progress_sink', None)
-    extracted = _extract_with_progress(session.data, slicer, progress)
+    data = session.data
+    if isinstance(data, np.memmap) and data.filename:
+        # A slice scattered one element per page reads far faster as many
+        # parallel small reads than by faulting through the map (local disks).
+        from ._io import prefetch_slice_pages
+
+        try:
+            prefetch_slice_pages(data, data.filename, (dim_x, dim_y), idx_list)
+        except Exception:
+            pass
+    extracted = _extract_with_progress(data, slicer, progress)
     if dim_x < dim_y:
         extracted = extracted.T
     # scipy.io.loadmat returns complex arrays as structured dtypes; convert here.

{arrayview-0.47.4 → arrayview-0.47.5}/src/arrayview/_routes_loading.py

@@ -341,6 +341,7 @@ def register_loading_routes(app, *, notify_shells, setup_rgb) -> None:
                         filepath,
                         select=body.get("select"),
                         progress=_report_read_progress,
+                        fast_first_frame=True,
                     )
                     session = Session(data, filepath=filepath, name=name)
                     session.sid = sid

{arrayview-0.47.4 → arrayview-0.47.5}/src/arrayview/_routes_query.py

@@ -247,6 +247,7 @@ def register_query_routes(app, *, get_session_or_404, pil_image, pil_imageops) -
                         request_id=request_id,
                         navigation_attempt=journal.get("navigation_attempt"),
                         phase=phase,
+                        client_ms=body.get("client_ms"),
                     )
                 except Exception:
                     pass

{arrayview-0.47.4 → arrayview-0.47.5}/src/arrayview/_session.py

@@ -743,12 +743,9 @@ def upgrade_memmap_in_background(session) -> bool:
         return False
 
     if is_memmap:
-        if not filepath.endswith(".npy"):
-            return False
-        try:
-            nbytes = os.path.getsize(filepath)
-        except OSError:
+        if not filepath.endswith((".npy", ".mat")):
             return False
+        nbytes = int(data.nbytes)
     else:
         nbytes = _nifti_proxy_nbytes(data)
     if nbytes >= npy_eager_limit():
@@ -759,8 +756,14 @@ def upgrade_memmap_in_background(session) -> bool:
 
     def _upgrade():
         try:
-            if is_memmap:
-                loaded = np.load(filepath)
+            if is_memmap and filepath.endswith(".npy"):
+                from ._io import read_npy_into_ram
+
+                loaded = read_npy_into_ram(data)
+            elif is_memmap:
+                # A block mapped out of a larger file (.mat v7.3): copy it in
+                # file order, which is one sequential read.
+                loaded = np.array(data, copy=True, order="K")
             else:
                 from ._io import _nifti_display_array