ewoksdata 0.6.0__tar.gz → 1.0.0__tar.gz

ewoksdata-1.0.0/PKG-INFO

@@ -0,0 +1,67 @@
+Metadata-Version: 2.2
+Name: ewoksdata
+Version: 1.0.0
+Summary: Ewoks tasks for data access
+Author-email: ESRF <dau-pydev@esrf.fr>
+License: # MIT License
+        
+        **Copyright (c) 2021 European Synchrotron Radiation Facility**
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of
+        this software and associated documentation files (the "Software"), to deal in
+        the Software without restriction, including without limitation the rights to
+        use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+        the Software, and to permit persons to whom the Software is furnished to do so,
+        subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+        FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+        COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+        IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+        CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+        
+Project-URL: Homepage, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
+Project-URL: Documentation, https://ewoksdata.readthedocs.io
+Project-URL: Repository, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
+Project-URL: Issues, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/issues
+Project-URL: Changelog, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/-/blob/main/CHANGELOG.md
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: h5py>=2.8
+Requires-Dist: silx>=1.1.0
+Requires-Dist: hdf5plugin
+Requires-Dist: blissdata>=0.3.4
+Requires-Dist: redis
+Requires-Dist: redis!=5.3.0b1,!=5.3.0b3
+Requires-Dist: packaging
+Requires-Dist: beartype!=0.20.0rc0
+Provides-Extra: online
+Requires-Dist: blissdata[tango]; extra == "online"
+Provides-Extra: test
+Requires-Dist: ewoksdata[online]; extra == "test"
+Requires-Dist: pytest>=7; extra == "test"
+Provides-Extra: dev
+Requires-Dist: ewoksdata[test]; extra == "dev"
+Requires-Dist: black>=25; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
+Provides-Extra: doc
+Requires-Dist: ewoksdata[test]; extra == "doc"
+Requires-Dist: sphinx>=4.5; extra == "doc"
+Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
+Requires-Dist: pydata-sphinx-theme<0.15; extra == "doc"
+
+# ewoksdata
+
+Data access utilities for [ewoks](https://ewoks.readthedocs.io/) workflows.
+
+## Documentation
+
+https://ewoksdata.readthedocs.io

ewoksdata-1.0.0/pyproject.toml

@@ -0,0 +1,65 @@
+[build-system]
+requires = [
+    "setuptools>=46.4"
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ewoksdata"
+version = "1.0.0"
+authors = [{name = "ESRF", email = "dau-pydev@esrf.fr"}]
+description = "Ewoks tasks for data access"
+readme = {file = "README.md", content-type = "text/markdown"}
+license = {file = "LICENSE.md"}
+classifiers = [
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+]
+requires-python = ">=3.8"
+dependencies = [
+    "h5py >=2.8",
+    "silx >=1.1.0",
+    "hdf5plugin",
+    "blissdata >=0.3.4",
+    "redis", # blissdata<1
+    "redis !=5.3.0b1,!=5.3.0b3",
+    "packaging",
+    "beartype !=0.20.0rc0" # https://github.com/beartype/beartype/issues/483
+]
+
+[project.urls]
+Homepage = "https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/"
+Documentation = "https://ewoksdata.readthedocs.io"
+Repository = "https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/"
+Issues = "https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/issues"
+Changelog = "https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/-/blob/main/CHANGELOG.md"
+
+[project.optional-dependencies]
+online = [
+    "blissdata[tango]",
+]
+test = [
+    "ewoksdata[online]",
+    "pytest >=7",
+]
+dev = [
+    "ewoksdata[test]",
+    "black >=25",
+    "flake8 >=4",
+]
+doc = [
+    "ewoksdata[test]",
+    "sphinx >=4.5",
+    "sphinx-autodoc-typehints >=1.16",
+    "pydata-sphinx-theme < 0.15",
+]
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.coverage.run]
+omit = ['*/tests/*']

ewoksdata-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

{ewoksdata-0.6.0 → ewoksdata-1.0.0}/src/ewoksdata/data/bliss.py

@@ -12,6 +12,7 @@ from silx.utils import retry as retrymod
 from silx.io.utils import get_data as silx_get_data
 
 from .blissdata import iter_bliss_scan_data_from_memory  # noqa F401
+from .blissdata import iter_bliss_scan_data_from_memory_slice  # noqa F401
 from .blissdata import last_lima_image  # noqa F401
 from .blissdata import dynamic_hdf5
 

{ewoksdata-0.6.0 → ewoksdata-1.0.0}/src/ewoksdata/data/blissdata/__init__.py

@@ -1,9 +1,4 @@
-import sys
-
-if sys.version_info >= (3, 8):
-    from importlib.metadata import version
-else:
-    from importlib_metadata import version
+from importlib.metadata import version
 from packaging.specifiers import SpecifierSet
 
 

{ewoksdata-0.6.0 → ewoksdata-1.0.0}/src/ewoksdata/data/hdf5/config.py

@@ -1,5 +1,4 @@
-"""HDF5 configuration of optimal data storage (IO speed, compression, ...)
-"""
+"""HDF5 configuration of optimal data storage (IO speed, compression, ...)"""
 
 from typing import Optional
 from numbers import Integral

{ewoksdata-0.6.0 → ewoksdata-1.0.0}/src/ewoksdata/data/hdf5/dataset_writer.py

@@ -17,6 +17,7 @@ class _DatasetWriterBase:
         attrs: Optional[dict] = None,
         flush_period: Optional[float] = None,
     ) -> None:
+        self._file = parent.file
         self._parent = parent
         self._name = name
         self._attrs = attrs
@@ -53,6 +54,10 @@ class _DatasetWriterBase:
             return False
         return (time.time() - self._last_flush) >= self._flush_period
 
+    def _flush_hdf5(self) -> None:
+        """Explicit HDF5 flush for non-locking readers."""
+        self._file.flush()
+
 
 class DatasetWriter(_DatasetWriterBase):
     """Append arrays of the same shape to a new HDF5 dataset in a sequential manner.
@@ -84,11 +89,12 @@ class DatasetWriter(_DatasetWriterBase):
     ) -> None:
         super().__init__(parent, name, attrs=attrs, flush_period=flush_period)
         self._npoints = npoints
-        self._buffer: List[ArrayLike] = list()
-        self._npoints_added: int = 0
         self._chunked: bool = False
-        self._nchunk: int = 0
-        self._nflushed: int = 0
+        self._npoints_added: int = 0
+
+        self._buffer: List[ArrayLike] = list()
+        self._chunk_size: int = 0
+        self._flushed_size: int = 0
 
     def _create_dataset(self, first_data_point: numpy.ndarray) -> h5py.Dataset:
         scan_shape = (self._npoints,)
@@ -111,7 +117,7 @@ class DatasetWriter(_DatasetWriterBase):
             options["maxshape"] = max_shape
         if options["chunks"]:
             self._chunked = True
-            self._nchunk = options["chunks"][0]
+            self._chunk_size = options["chunks"][0]
         dset = self._parent.create_dataset(self._name, **options)
         if self._attrs:
             dset.attrs.update(self._attrs)
@@ -139,30 +145,35 @@ class DatasetWriter(_DatasetWriterBase):
 
     def flush_buffer(self, align: bool = False) -> bool:
         # Determine how many points to flush
-        nbuffer = len(self._buffer)
+        chunk_size = len(self._buffer)
+
         if self._flush_time_expired():
-            nflush = nbuffer
+            flush_size = chunk_size
         elif align and self._chunked:
-            n = nbuffer + (self._nflushed % self._nchunk)
-            nflush = n // self._nchunk * self._nchunk
-            nflush = min(nflush, nbuffer)
+            n = chunk_size + (self._flushed_size % self._chunk_size)
+            flush_size = n // self._chunk_size * self._chunk_size
+            flush_size = min(flush_size, chunk_size)
         else:
-            nflush = nbuffer
-        if nflush == 0:
+            flush_size = chunk_size
+
+        if flush_size == 0:
             return False
 
         # Enlarge the dataset when needed
         nalloc = self._dataset.shape[0]
-        istart = self._nflushed
-        nflushed = istart + nflush
-        if self._chunked and nflushed > nalloc:
-            self._dataset.resize(nflushed, axis=0)
+        istart = self._flushed_size
+        flushed_size = istart + flush_size
+        if self._chunked and flushed_size > nalloc:
+            self._dataset.resize(flushed_size, axis=0)
+
+        # Copy data from buffer to HDF5
+        self._dataset[istart : istart + flush_size] = self._buffer[:flush_size]
 
-        # Move data from memory to HDF5
-        self._dataset[istart : istart + nflush] = self._buffer[:nflush]
-        self._buffer = self._buffer[nflush:]
-        self._nflushed = nflushed
+        # Remove copied data from buffer
+        self._buffer = self._buffer[flush_size:]
+        self._flushed_size = flushed_size
 
+        self._flush_hdf5()
         self._last_flush = time.time()
         return True
 
@@ -204,11 +215,12 @@ class StackDatasetWriter(_DatasetWriterBase):
     ) -> None:
         super().__init__(parent, name, attrs=attrs, flush_period=flush_period)
         self._npoints = npoints
+        self._chunked: bool = False
         self._nstack = nstack
+
         self._buffers: List[List[ArrayLike]] = list()
-        self._chunked: bool = False
-        self._nchunk: ArrayLike = numpy.zeros(2, dtype=int)
-        self._nflushed: ArrayLike = numpy.array(list(), dtype=int)
+        self._chunk_size: ArrayLike = numpy.zeros(2, dtype=int)
+        self._flushed_size_dim1: ArrayLike = numpy.array(list(), dtype=int)
 
     def _create_dataset(
         self, first_data_point: numpy.ndarray, stack_index: int
@@ -233,7 +245,7 @@ class StackDatasetWriter(_DatasetWriterBase):
             options["maxshape"] = max_shape
         if options["chunks"]:
             self._chunked = True
-            self._nchunk = numpy.array(options["chunks"][:2], dtype=int)
+            self._chunk_size = numpy.array(options["chunks"][:2], dtype=int)
         dset = self._parent.create_dataset(self._name, **options)
         if self._attrs:
             dset.attrs.update(self._attrs)
@@ -243,7 +255,7 @@ class StackDatasetWriter(_DatasetWriterBase):
         # Add stack buffers when needed
         for _ in range(max(stack_index - len(self._buffers) + 1, 0)):
             self._buffers.append(list())
-            self._nflushed = numpy.append(self._nflushed, 0)
+            self._flushed_size_dim1 = numpy.append(self._flushed_size_dim1, 0)
         return self._buffers[stack_index]
 
     def add_point(self, data: ArrayLike, stack_index: int) -> bool:
@@ -264,60 +276,79 @@ class StackDatasetWriter(_DatasetWriterBase):
 
     def flush_buffer(self, align: bool = False) -> bool:
         # Determine how many points to flush for each buffer in the stack
-        nbuffer = numpy.array([len(buffer) for buffer in self._buffers])
-        nchunk_dim0, nchunk_dim1 = self._nchunk[:2]
+        chunk_sizes = numpy.array([len(buffer) for buffer in self._buffers])
+        flushed_size_dim1 = self._flushed_size_dim1
+        size_dim0 = len(chunk_sizes)
+        assert size_dim0 == len(
+            flushed_size_dim1
+        ), "Number of buffers and number of flushed dim1 points must be the same"
+
+        chunk_size_dim0, chunk_size_dim1 = self._chunk_size[:2]
+        if chunk_size_dim0 == 0:
+            chunk_size_dim0 = size_dim0
+
         if self._flush_time_expired():
-            nflush = nbuffer
+            flush_sizes = chunk_sizes
         elif align and self._chunked:
-            n = nbuffer + (self._nflushed % nchunk_dim1)
-            nflush = n // nchunk_dim1 * nchunk_dim1
-            nflush = numpy.minimum(nflush, nbuffer)
-            for i0_chunk0 in range(0, len(nbuffer), nchunk_dim0):
-                nflush[i0_chunk0 : i0_chunk0 + nchunk_dim0] = min(
-                    nflush[i0_chunk0 : i0_chunk0 + nchunk_dim0]
-                )
+            size_dim0 = size_dim0 // chunk_size_dim0 * chunk_size_dim0
+            chunk_sizes = chunk_sizes[:size_dim0]
+            flushed_size_dim1 = flushed_size_dim1[:size_dim0]
+            if size_dim0:
+                n1 = chunk_sizes + (flushed_size_dim1 % chunk_size_dim1)
+                flush_sizes = n1 // chunk_size_dim1 * chunk_size_dim1
+                flush_sizes = numpy.minimum(flush_sizes, chunk_sizes)
+                for i0_chunk0 in range(0, size_dim0, chunk_size_dim0):
+                    flush_sizes[i0_chunk0 : i0_chunk0 + chunk_size_dim0] = min(
+                        flush_sizes[i0_chunk0 : i0_chunk0 + chunk_size_dim0]
+                    )
+            else:
+                flush_sizes = list()
         else:
-            nflush = nbuffer
-        if not any(nflush):
+            flush_sizes = chunk_sizes
+
+        if not any(flush_sizes):
             return False
 
         # Enlarge the dataset when needed
         nalloc = self._dataset.shape[:2]
-        istart = self._nflushed
-        nflushed = istart + nflush
-        nalloc_new = numpy.array([len(nbuffer), max(nflushed)])
+        istart_dim1 = flushed_size_dim1
+        flushed_size_dim1 = istart_dim1 + flush_sizes
+        nalloc_new = numpy.array([size_dim0, max(flushed_size_dim1)])
         if self._chunked and any(nalloc_new > nalloc):
             for axis, n in enumerate(nalloc_new):
                 self._dataset.resize(n, axis=axis)
 
-        # Move data from memory to HDF5
-        if nchunk_dim0 == 0:
-            nchunk_dim0 = len(nbuffer)
-        for i0_chunk0 in range(0, len(nbuffer), nchunk_dim0):
-            idx_dim0 = slice(i0_chunk0, i0_chunk0 + nchunk_dim0)
-            nflush_dim1 = nflush[idx_dim0]
-            istart0_dim1 = istart[idx_dim0]
+        # Copy data from buffer to HDF5
+        for i0_chunk0 in range(0, size_dim0, chunk_size_dim0):
+            idx_dim0 = slice(i0_chunk0, i0_chunk0 + chunk_size_dim0)
             buffers = self._buffers[idx_dim0]
-            if all(nflush_dim1 == nflush_dim1[0]) and all(
-                istart0_dim1 == istart0_dim1[0]
-            ):
-                data = [buffer[: nflush_dim1[0]] for buffer in buffers]
-                idx_dim1 = slice(istart0_dim1[0], istart0_dim1[0] + nflush_dim1[0])
+
+            flush_sizes_dim1 = flush_sizes[idx_dim0]
+            non_ragged_buffers = len(set(flush_sizes_dim1)) == 1
+
+            istart0_dim1 = istart_dim1[idx_dim0]
+            non_ragged_destination = len(set(istart0_dim1)) == 1
+
+            if non_ragged_destination and non_ragged_buffers:
+                data = [buffer[: flush_sizes_dim1[0]] for buffer in buffers]
+                idx_dim1 = slice(istart0_dim1[0], istart0_dim1[0] + flush_sizes_dim1[0])
                 self._dataset[idx_dim0, idx_dim1] = data
             else:
-                for buffer, i_dim0, istart_dim1, n_dim1 in zip(
+                for buffer, i_dim0, istart_dim1, i_flush_size_dim1 in zip(
                     buffers,
-                    range(i0_chunk0, i0_chunk0 + nchunk_dim0),
+                    range(i0_chunk0, i0_chunk0 + chunk_size_dim0),
                     istart0_dim1,
-                    nflush_dim1,
+                    flush_sizes_dim1,
                 ):
-                    self._dataset[i_dim0, istart_dim1 : istart_dim1 + n_dim1, ...] = (
-                        buffer[:n_dim1]
-                    )
-        self._buffers = [
-            buffer[n_dim1:] for buffer, n_dim1 in zip(self._buffers, nflush)
-        ]
-        self._nflushed = nflushed
+                    self._dataset[
+                        i_dim0, istart_dim1 : istart_dim1 + i_flush_size_dim1, ...
+                    ] = buffer[:i_flush_size_dim1]
+
+        # Remove copied data from buffer
+        for i0 in range(size_dim0):
+            self._buffers[i0] = self._buffers[i0][flush_sizes[i0] :]
+            self._flushed_size_dim1[i0] = flushed_size_dim1[i0]
 
+        self._flush_hdf5()
         self._last_flush = time.time()
         return True

{ewoksdata-0.6.0 → ewoksdata-1.0.0}/src/ewoksdata/tests/conftest.py

@@ -1,5 +1,6 @@
 import numpy
 import pytest
+
 from . import data
 
 

{ewoksdata-0.6.0 → ewoksdata-1.0.0}/src/ewoksdata/tests/data/bliss_scans.py

@@ -1,7 +1,7 @@
 import os
 from datetime import datetime
 from pathlib import Path
-from typing import Tuple
+from typing import Optional, Tuple
 import numpy
 import h5py
 
@@ -19,6 +19,8 @@ def save_bliss_scan(
     sequence: str = "add",
     lima_names: Tuple[str, ...] = ("p3",),
     counter_names: Tuple[str, ...] = ("diode1", "diode2"),
+    positioner_names: Tuple[str, ...] = ("diffty", "diffrz"),
+    title: Optional[str] = None,
 ) -> Path:
     dataset_filename = tmpdir / "sample_dataset.h5"
     lima_dirname = tmpdir / f"scan{scannr:04d}"
@@ -28,12 +30,18 @@ def save_bliss_scan(
         root.attrs["creator"] = "bliss"
         scan = root.create_group(f"{scannr}.{subscannr}")
         scan.attrs["NX_class"] = "NXentry"
+        if title is not None:
+            scan["title"] = title
 
         instrument = scan.create_group("instrument")
         instrument.attrs["NX_class"] = "NXinstrument"
         measurement = scan.create_group("measurement")
         measurement.attrs["NX_class"] = "NXcollection"
 
+        if positioner_names:
+            positioners = instrument.create_group("positioners")
+            positioners.attrs["NX_class"] = "NXcollection"
+
         for name in counter_names:
             _save_counter(scan, name, npoints)
         for name in lima_names:
@@ -47,6 +55,8 @@ def save_bliss_scan(
                 scannr,
                 sequence,
             )
+        for name in positioner_names:
+            _save_positioner(scan, name, npoints)
 
         scan["end_time"] = datetime.now().astimezone().isoformat()
     return dataset_filename
@@ -100,7 +110,7 @@ def _save_lima(
             sshape = (i1 - i0,) + lima_shape[1:]
             vsource = h5py.VirtualSource(spath, sdset, shape=sshape, dtype=lima_dtype)
 
-            layout[i0:i1] = vsource
+            layout[i0:i1] = vsource[:]
     detector = _add_detector(scan, name)
     detector.create_virtual_dataset("data", layout, fillvalue=numpy.nan)
 
@@ -110,3 +120,17 @@ def _add_detector(scan: h5py.Group, name: str) -> h5py.Group:
     grp.attrs["NX_class"] = "NXdetector"
     scan["measurement"][name] = h5py.SoftLink(f"{grp.name}/data")
     return grp
+
+
+def _save_positioner(scan: h5py.Group, name: str, npoints: int) -> None:
+    positioner = _add_positioner(scan, name)
+    positioner["value"] = numpy.arange(npoints)
+
+
+def _add_positioner(scan: h5py.Group, name: str) -> h5py.Group:
+    instrument = scan["instrument"]
+    grp = instrument.create_group(name)
+    grp.attrs["NX_class"] = "Nxpositioner"
+    scan["measurement"][name] = h5py.SoftLink(f"{grp.name}/value")
+    instrument["positioners"][name] = h5py.SoftLink(f"{grp.name}/value")
+    return grp

ewoksdata-1.0.0/src/ewoksdata/tests/test_dataset_writer.py

@@ -0,0 +1,186 @@
+import time
+import itertools
+import multiprocessing
+
+import h5py
+import numpy
+import pytest
+from silx.io import h5py_utils
+
+
+from ..data.hdf5 import dataset_writer
+
+
+@pytest.mark.parametrize("npoints", (1, 3, 1000))
+@pytest.mark.parametrize("flush_period", (None, 0.1))
+@pytest.mark.parametrize("known_npoints", (True, False))
+def test_dataset_writer(tmpdir, npoints, flush_period, known_npoints):
+    expected = list()
+    filename = str(tmpdir / "test.h5")
+    if flush_period is None:
+        sleep_time = None
+    else:
+        sleep_time = flush_period + 0.1
+    isleep = npoints // 3
+
+    kwargs = {"flush_period": flush_period}
+    if known_npoints:
+        kwargs["npoints"] = npoints
+
+    with h5py.File(filename, mode="w") as f:
+        with dataset_writer.DatasetWriter(f, "data", **kwargs) as writer:
+            for ipoint in range(npoints):
+                data = numpy.random.random((10, 20))
+                writer.add_point(data)
+                expected.append(data)
+                if sleep_time and ipoint == isleep:
+                    time.sleep(sleep_time)
+
+    with h5py.File(filename, mode="r") as f:
+        data = f["data"][()]
+    numpy.testing.assert_allclose(data, expected)
+
+
+@pytest.mark.parametrize("nstack", (1, 4))
+@pytest.mark.parametrize("npoints", (1, 3, 1000))
+@pytest.mark.parametrize("flush_period", (None, 0.1))
+@pytest.mark.parametrize("known_npoints", (True, False))
+@pytest.mark.parametrize("known_nstack", (True, False))
+@pytest.mark.parametrize("append_stacks_in_parallel", (True, False))
+def test_stack_dataset_writer(
+    tmpdir,
+    nstack,
+    npoints,
+    flush_period,
+    known_npoints,
+    known_nstack,
+    append_stacks_in_parallel,
+):
+    expected = [list() for _ in range(nstack)]
+    filename = str(tmpdir / "test.h5")
+    if flush_period is None:
+        sleep_time = None
+    else:
+        sleep_time = flush_period + 0.1
+    isleep = (nstack * npoints) // 3
+
+    kwargs = {"flush_period": flush_period}
+    if known_npoints:
+        kwargs["npoints"] = npoints
+    if known_nstack:
+        kwargs["nstack"] = nstack
+
+    if append_stacks_in_parallel:
+        itpoints = itertools.product(range(npoints), range(nstack))
+    else:
+        itpoints = itertools.product(range(nstack), range(npoints))
+
+    with h5py.File(filename, mode="w") as f:
+        with dataset_writer.StackDatasetWriter(f, "data", **kwargs) as writer:
+            for tpl in itpoints:
+                if append_stacks_in_parallel:
+                    ipoint, istack = tpl
+                else:
+                    istack, ipoint = tpl
+                data = numpy.random.random((10, 20))
+                writer.add_point(data, istack)
+                expected[istack].append(data)
+                if sleep_time and (ipoint * nstack + istack) == isleep:
+                    time.sleep(sleep_time)
+
+    with h5py.File(filename, mode="r") as f:
+        data = f["data"][()]
+    numpy.testing.assert_allclose(data, expected)
+
+
+def test_concurrent_reader(tmpdir):
+    npoints = 50
+    hdf5_filename = str(tmpdir / "test.h5")
+    read_timestamps = str(tmpdir / "read_timestamps.log")
+    write_timestamps = str(tmpdir / "write_timestamps.log")
+    dataset_name = "data"
+
+    # Run read loop in a sub-process
+    start_event = multiprocessing.Event()
+    reader_process = multiprocessing.Process(
+        target=_read_hdf5_file,
+        args=(hdf5_filename, dataset_name, npoints, read_timestamps, start_event),
+    )
+    reader_process.start()
+
+    assert start_event.wait(timeout=10)
+
+    # Run write loop
+    try:
+        _write_hdf5_file(hdf5_filename, dataset_name, npoints, write_timestamps)
+    except Exception:
+        reader_process.terminate()
+        raise
+
+    # Wait for read loop to finish
+    try:
+        reader_process.join(timeout=10)
+        if reader_process.exitcode is None:
+            raise TimeoutError("Reader process did not terminate within the timeout.")
+        elif reader_process.exitcode != 0:
+            raise RuntimeError(
+                f"Reader process terminated with exit code {reader_process.exitcode}."
+            )
+    except Exception:
+        reader_process.terminate()
+        raise
+
+    # Check that reading and writing happened concurrently
+    with open(read_timestamps, "r") as eventfile:
+        read_events = [int(line.strip()) for line in eventfile.readlines()]
+    with open(write_timestamps, "r") as eventfile:
+        write_events = [int(line.strip()) for line in eventfile.readlines()]
+
+    assert len(read_events) == npoints
+    assert len(write_events) == npoints
+
+    all_events = sorted(read_events + write_events)
+    assert (
+        read_events != all_events[: len(read_events)]
+    ), "reads and writes are not interleaved"
+
+
+class _DatasetWriter(dataset_writer.DatasetWriter):
+    """Testing concurrent reading and writing always ends up
+    being flaky. So we introduce a sleep after flushing
+    to give the reader plenty of time to fetch new data.
+    """
+
+    def _flush_hdf5(self) -> None:
+        print("flush")
+        super()._flush_hdf5()
+        time.sleep(0.2)
+
+
+def _write_hdf5_file(hdf5_filename, dataset_name, npoints, event_filename):
+    data = numpy.zeros((500, 500), dtype=numpy.int64)
+    with open(event_filename, "a") as eventfile:
+        with h5py.File(hdf5_filename, mode="w") as h5file:
+            with _DatasetWriter(h5file, dataset_name) as writer:
+                for i in range(npoints):
+                    writer.add_point(data)
+                    eventfile.write(f"{time.perf_counter_ns()}\n")
+                    print(f"write {i}")
+
+
+def _read_hdf5_file(hdf5_filename, dataset_name, npoints, event_filename, start_event):
+    i = 0
+    with open(event_filename, "a") as eventfile:
+        start_event.set()
+        while i < npoints:
+            for _ in _iter_data(hdf5_filename, dataset_name):
+                eventfile.write(f"{time.perf_counter_ns()}\n")
+                print(f"read {i}")
+                i += 1
+
+
+@h5py_utils.retry()
+def _iter_data(hdf5_filename, dataset_name, start_index=0):
+    with h5py_utils.File(hdf5_filename) as h5file:
+        for _ in h5file[dataset_name][start_index:], start_index:
+            yield None

ewoksdata-1.0.0/src/ewoksdata.egg-info/PKG-INFO

@@ -0,0 +1,67 @@
+Metadata-Version: 2.2
+Name: ewoksdata
+Version: 1.0.0
+Summary: Ewoks tasks for data access
+Author-email: ESRF <dau-pydev@esrf.fr>
+License: # MIT License
+        
+        **Copyright (c) 2021 European Synchrotron Radiation Facility**
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of
+        this software and associated documentation files (the "Software"), to deal in
+        the Software without restriction, including without limitation the rights to
+        use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+        the Software, and to permit persons to whom the Software is furnished to do so,
+        subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+        FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+        COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+        IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+        CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+        
+Project-URL: Homepage, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
+Project-URL: Documentation, https://ewoksdata.readthedocs.io
+Project-URL: Repository, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
+Project-URL: Issues, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/issues
+Project-URL: Changelog, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/-/blob/main/CHANGELOG.md
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: h5py>=2.8
+Requires-Dist: silx>=1.1.0
+Requires-Dist: hdf5plugin
+Requires-Dist: blissdata>=0.3.4
+Requires-Dist: redis
+Requires-Dist: redis!=5.3.0b1,!=5.3.0b3
+Requires-Dist: packaging
+Requires-Dist: beartype!=0.20.0rc0
+Provides-Extra: online
+Requires-Dist: blissdata[tango]; extra == "online"
+Provides-Extra: test
+Requires-Dist: ewoksdata[online]; extra == "test"
+Requires-Dist: pytest>=7; extra == "test"
+Provides-Extra: dev
+Requires-Dist: ewoksdata[test]; extra == "dev"
+Requires-Dist: black>=25; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
+Provides-Extra: doc
+Requires-Dist: ewoksdata[test]; extra == "doc"
+Requires-Dist: sphinx>=4.5; extra == "doc"
+Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
+Requires-Dist: pydata-sphinx-theme<0.15; extra == "doc"
+
+# ewoksdata
+
+Data access utilities for [ewoks](https://ewoks.readthedocs.io/) workflows.
+
+## Documentation
+
+https://ewoksdata.readthedocs.io

{ewoksdata-0.6.0 → ewoksdata-1.0.0}/src/ewoksdata.egg-info/SOURCES.txt

@@ -1,8 +1,6 @@
 LICENSE.md
 README.md
 pyproject.toml
-setup.cfg
-setup.py
 src/ewoksdata/__init__.py
 src/ewoksdata.egg-info/PKG-INFO
 src/ewoksdata.egg-info/SOURCES.txt

{ewoksdata-0.6.0 → ewoksdata-1.0.0}/src/ewoksdata.egg-info/requires.txt

@@ -3,24 +3,17 @@ silx>=1.1.0
 hdf5plugin
 blissdata>=0.3.4
 redis
+redis!=5.3.0b1,!=5.3.0b3
 packaging
-
-[:python_version < "3.8"]
-blissdata<1.0.1
-importlib_metadata
-
-[:python_version >= "3.8"]
-blissdata
+beartype!=0.20.0rc0
 
 [dev]
-blissdata[tango]
-pytest>=7
-black>=22
+ewoksdata[test]
+black>=25
 flake8>=4
 
 [doc]
-blissdata[tango]
-pytest>=7
+ewoksdata[test]
 sphinx>=4.5
 sphinx-autodoc-typehints>=1.16
 pydata-sphinx-theme<0.15
@@ -29,5 +22,5 @@ pydata-sphinx-theme<0.15
 blissdata[tango]
 
 [test]
-blissdata[tango]
+ewoksdata[online]
 pytest>=7

ewoksdata-0.6.0/PKG-INFO

@@ -1,50 +0,0 @@
-Metadata-Version: 2.1
-Name: ewoksdata
-Version: 0.6.0
-Summary: Ewoks tasks for data access
-Home-page: https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
-Author: ESRF
-Author-email: wout.de_nolf@esrf.fr
-License: MIT
-Project-URL: Source, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
-Project-URL: Documentation, https://ewoksdata.readthedocs.io
-Project-URL: Tracker, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/issues/
-Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.7
-Description-Content-Type: text/markdown
-License-File: LICENSE.md
-Requires-Dist: h5py>=2.8
-Requires-Dist: silx>=1.1.0
-Requires-Dist: hdf5plugin
-Requires-Dist: blissdata>=0.3.4
-Requires-Dist: blissdata<1.0.1; python_version < "3.8"
-Requires-Dist: blissdata; python_version >= "3.8"
-Requires-Dist: redis
-Requires-Dist: packaging
-Requires-Dist: importlib_metadata; python_version < "3.8"
-Provides-Extra: online
-Requires-Dist: blissdata[tango]; extra == "online"
-Provides-Extra: test
-Requires-Dist: blissdata[tango]; extra == "test"
-Requires-Dist: pytest>=7; extra == "test"
-Provides-Extra: dev
-Requires-Dist: blissdata[tango]; extra == "dev"
-Requires-Dist: pytest>=7; extra == "dev"
-Requires-Dist: black>=22; extra == "dev"
-Requires-Dist: flake8>=4; extra == "dev"
-Provides-Extra: doc
-Requires-Dist: blissdata[tango]; extra == "doc"
-Requires-Dist: pytest>=7; extra == "doc"
-Requires-Dist: sphinx>=4.5; extra == "doc"
-Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
-Requires-Dist: pydata-sphinx-theme<0.15; extra == "doc"
-
-# ewoksdata
-
-Data access utilities for [ewoks](https://ewoks.readthedocs.io/) workflows.
-
-## Documentation
-
-https://ewoksdata.readthedocs.io

ewoksdata-0.6.0/pyproject.toml

@@ -1,5 +0,0 @@
-[build-system]
-requires = [
-    "setuptools>=46.4"
-]
-build-backend = "setuptools.build_meta"

ewoksdata-0.6.0/setup.cfg

@@ -1,69 +0,0 @@
-[metadata]
-name = ewoksdata
-version = attr: ewoksdata.__version__
-author = ESRF
-author_email = wout.de_nolf@esrf.fr
-description = Ewoks tasks for data access
-long_description = file: README.md
-long_description_content_type = text/markdown
-license = MIT
-url = https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
-project_urls = 
-	Source = https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
-	Documentation = https://ewoksdata.readthedocs.io
-	Tracker = https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/issues/
-classifiers = 
-	Intended Audience :: Science/Research
-	License :: OSI Approved :: MIT License
-	Programming Language :: Python :: 3
-
-[options]
-package_dir = 
-	=src
-packages = find:
-python_requires = >=3.7
-install_requires = 
-	h5py >=2.8
-	silx >=1.1.0
-	hdf5plugin
-	blissdata >=0.3.4
-	blissdata <1.0.1; python_version < "3.8"
-	blissdata; python_version >= "3.8"
-	redis # blissdata<1
-	packaging
-	importlib_metadata; python_version < "3.8"
-
-[options.packages.find]
-where = src
-
-[options.extras_require]
-online = 
-	blissdata[tango]
-test = 
-	%(online)s
-	pytest >=7
-dev = 
-	%(test)s
-	black >=22
-	flake8 >=4
-doc = 
-	%(test)s
-	sphinx >=4.5
-	sphinx-autodoc-typehints >=1.16
-	pydata-sphinx-theme < 0.15
-
-[flake8]
-ignore = E501, E203, W503
-max-line-length = 88
-exclude = 
-	.eggs
-
-[coverage:run]
-omit = 
-	setup.py
-	*/tests/*
-
-[egg_info]
-tag_build = 
-tag_date = 0
-

ewoksdata-0.6.0/setup.py

@@ -1,4 +0,0 @@
-import setuptools
-
-if __name__ == "__main__":
-    setuptools.setup()

ewoksdata-0.6.0/src/ewoksdata/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.6.0"

ewoksdata-0.6.0/src/ewoksdata/tests/test_dataset_writer.py

@@ -1,89 +0,0 @@
-import time
-import itertools
-
-import h5py
-import numpy
-import pytest
-from ..data.hdf5 import dataset_writer
-
-
-@pytest.mark.parametrize("npoints", (1, 3, 1000))
-@pytest.mark.parametrize("flush_period", (None, 0.1))
-@pytest.mark.parametrize("known_npoints", (True, False))
-def test_dataset_writer(tmpdir, npoints, flush_period, known_npoints):
-    expected = list()
-    filename = str(tmpdir / "test.h5")
-    if flush_period is None:
-        sleep_time = None
-    else:
-        sleep_time = flush_period + 0.1
-    isleep = npoints // 3
-
-    kwargs = {"flush_period": flush_period}
-    if known_npoints:
-        kwargs["npoints"] = npoints
-
-    with h5py.File(filename, mode="w") as f:
-        with dataset_writer.DatasetWriter(f, "data", **kwargs) as writer:
-            for ipoint in range(npoints):
-                data = numpy.random.random((10, 20))
-                writer.add_point(data)
-                expected.append(data)
-                if sleep_time and ipoint == isleep:
-                    time.sleep(sleep_time)
-
-    with h5py.File(filename, mode="r") as f:
-        data = f["data"][()]
-    numpy.testing.assert_allclose(data, expected)
-
-
-@pytest.mark.parametrize("nstack", (1, 4))
-@pytest.mark.parametrize("npoints", (1, 3, 1000))
-@pytest.mark.parametrize("flush_period", (None, 0.1))
-@pytest.mark.parametrize("known_npoints", (True, False))
-@pytest.mark.parametrize("known_nstack", (True, False))
-@pytest.mark.parametrize("append_stacks_in_parallel", (True, False))
-def test_stack_dataset_writer(
-    tmpdir,
-    nstack,
-    npoints,
-    flush_period,
-    known_npoints,
-    known_nstack,
-    append_stacks_in_parallel,
-):
-    expected = [list() for _ in range(nstack)]
-    filename = str(tmpdir / "test.h5")
-    if flush_period is None:
-        sleep_time = None
-    else:
-        sleep_time = flush_period + 0.1
-    isleep = (nstack * npoints) // 3
-
-    kwargs = {"flush_period": flush_period}
-    if known_npoints:
-        kwargs["npoints"] = npoints
-    if known_nstack:
-        kwargs["nstack"] = nstack
-
-    if append_stacks_in_parallel:
-        itpoints = itertools.product(range(npoints), range(nstack))
-    else:
-        itpoints = itertools.product(range(nstack), range(npoints))
-
-    with h5py.File(filename, mode="w") as f:
-        with dataset_writer.StackDatasetWriter(f, "data", **kwargs) as writer:
-            for tpl in itpoints:
-                if append_stacks_in_parallel:
-                    ipoint, istack = tpl
-                else:
-                    istack, ipoint = tpl
-                data = numpy.random.random((10, 20))
-                writer.add_point(data, istack)
-                expected[istack].append(data)
-                if sleep_time and (ipoint * nstack + istack) == isleep:
-                    time.sleep(sleep_time)
-
-    with h5py.File(filename, mode="r") as f:
-        data = f["data"][()]
-    numpy.testing.assert_allclose(data, expected)

ewoksdata-0.6.0/src/ewoksdata.egg-info/PKG-INFO

@@ -1,50 +0,0 @@
-Metadata-Version: 2.1
-Name: ewoksdata
-Version: 0.6.0
-Summary: Ewoks tasks for data access
-Home-page: https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
-Author: ESRF
-Author-email: wout.de_nolf@esrf.fr
-License: MIT
-Project-URL: Source, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
-Project-URL: Documentation, https://ewoksdata.readthedocs.io
-Project-URL: Tracker, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/issues/
-Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.7
-Description-Content-Type: text/markdown
-License-File: LICENSE.md
-Requires-Dist: h5py>=2.8
-Requires-Dist: silx>=1.1.0
-Requires-Dist: hdf5plugin
-Requires-Dist: blissdata>=0.3.4
-Requires-Dist: blissdata<1.0.1; python_version < "3.8"
-Requires-Dist: blissdata; python_version >= "3.8"
-Requires-Dist: redis
-Requires-Dist: packaging
-Requires-Dist: importlib_metadata; python_version < "3.8"
-Provides-Extra: online
-Requires-Dist: blissdata[tango]; extra == "online"
-Provides-Extra: test
-Requires-Dist: blissdata[tango]; extra == "test"
-Requires-Dist: pytest>=7; extra == "test"
-Provides-Extra: dev
-Requires-Dist: blissdata[tango]; extra == "dev"
-Requires-Dist: pytest>=7; extra == "dev"
-Requires-Dist: black>=22; extra == "dev"
-Requires-Dist: flake8>=4; extra == "dev"
-Provides-Extra: doc
-Requires-Dist: blissdata[tango]; extra == "doc"
-Requires-Dist: pytest>=7; extra == "doc"
-Requires-Dist: sphinx>=4.5; extra == "doc"
-Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
-Requires-Dist: pydata-sphinx-theme<0.15; extra == "doc"
-
-# ewoksdata
-
-Data access utilities for [ewoks](https://ewoks.readthedocs.io/) workflows.
-
-## Documentation
-
-https://ewoksdata.readthedocs.io