ewoksdata 2.0.2__tar.gz → 2.1.0__tar.gz

{ewoksdata-2.0.2/src/ewoksdata.egg-info → ewoksdata-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ewoksdata
-Version: 2.0.2
+Version: 2.1.0
 Summary: Ewoks tasks for data access
 Author-email: ESRF <dau-pydev@esrf.fr>
 License: # MIT License

{ewoksdata-2.0.2 → ewoksdata-2.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ewoksdata"
-version = "2.0.2"
+version = "2.1.0"
 authors = [{ name = "ESRF", email = "dau-pydev@esrf.fr" }]
 description = "Ewoks tasks for data access"
 readme = { file = "README.md", content-type = "text/markdown" }

{ewoksdata-2.0.2 → ewoksdata-2.1.0}/src/ewoksdata/data/bliss.py

@@ -35,29 +35,56 @@ from .blissdata import iter_bliss_scan_data_from_memory
 from .blissdata import iter_bliss_scan_data_from_memory_slice  # noqa: F401
 from .blissdata import last_lima_image  # noqa: F401
 from .contextiterator import contextiterator
+from .utils import validate_and_init_names
 
 logger = logging.getLogger(__name__)
 
 
 def get_data(
     data: Union[str, DataUrl, ArrayLike, Number],
+    retry_timeout: Optional[float] = None,
+    retry_period: Optional[float] = None,
     **hdf5_options: Any,
 ) -> Union[numpy.ndarray, Number, Sequence]:
     """
-    Return a NumPy array or a scalar from *data*.
-
-    :param data: Either a data-URL, an array-like object or a scalar number.
-    :param **hdf5_options: Additional options forwarded to :class:`silx.io.h5py_utils.File`.
-    :return: The requested data.
+    Retrieve numerical data from a data source and return it as a NumPy array or scalar.
+
+    This function accepts various data representations, including data URLs,
+    array-like objects, and scalar numbers. The data source may not be
+    immediately available so a retry mechanism is used.
+
+    :param data: The input data source. May be:
+        - A string or :class:`DataUrl` referencing a dataset (e.g., HDF5),
+        - An array-like object convertible to a NumPy array,
+        - A scalar numerical value.
+    :param retry_timeout: Maximum time in seconds to wait for the data to become
+        available. If ``None`` (default), no timeout is applied.
+    :param retry_period: Interval in seconds between availability checks when
+        retrying. If ``None`` (default), a period of 0.1 seconds is used.
+    :param hdf5_options: Additional keyword arguments forwarded to
+        :class:`silx.io.h5py_utils.File` when opening HDF5 resources.
+
+    :returns: The resolved data. Array-like inputs are returned as NumPy arrays,
+        while scalar inputs are returned as numbers.
+
+    :raises TimeoutError: If the data does not become available within ``retry_timeout``.
+    :raises ValueError: If the input type is unsupported or the data cannot be resolved.
     """
     if isinstance(data, (str, DataUrl)):
         data_url = url.as_dataurl(data)
         filename, h5path, idx = url.h5dataset_url_parse(data_url)
 
-        _wait_for_file(filename, **hdf5_options)
+        _wait_for_file(filename, retry_timeout=retry_timeout, retry_period=retry_period)
 
-        if h5py.is_hdf5(filename):
-            return _get_hdf5_data(filename, h5path, idx=idx, **hdf5_options)
+        if _is_hdf5_filename(filename):
+            return _get_hdf5_data(
+                filename,
+                h5path,
+                idx=idx,
+                retry_timeout=retry_timeout,
+                retry_period=retry_period,
+                **hdf5_options,
+            )
 
         if not data_url.scheme():
             for scheme in ("fabio", "silx"):
@@ -97,6 +124,11 @@ def _wait_for_file(
         time.sleep(retry_period)
 
 
+def _is_hdf5_filename(filename: str) -> bool:
+    _, ext = os.path.splitext(filename)
+    return ext in (".h5", ".hdf5", ".nx", ".nxs", ".nexus")
+
+
 def get_image(
     data: Union[str, DataUrl, ArrayLike, Number], **hdf5_options: Any
 ) -> numpy.ndarray:
@@ -167,8 +199,7 @@ def iter_bliss_scan_data_from_hdf5(
     """
     if not subscan:
         subscan = 1
-    lima_names = lima_names or []
-    counter_names = counter_names or []
+    lima_names, counter_names = validate_and_init_names(lima_names, counter_names)
 
     with dynamic_hdf5.File(
         filename,

{ewoksdata-2.0.2 → ewoksdata-2.1.0}/src/ewoksdata/data/blissdata/blissdatav0.py

@@ -11,6 +11,8 @@ from blissdata.data.node import get_node
 from numpy.typing import ArrayLike
 from silx.utils import retry as retrymod
 
+from ..utils import validate_and_init_names
+
 logger = logging.getLogger(__name__)
 
 
@@ -24,10 +26,7 @@ def iter_bliss_scan_data_from_memory(
 ) -> Generator[Dict[str, Any], None, None]:
     if not buffer_in_memory:
         raise NotImplementedError("Requires blissdata>=2.0.0rc1")
-    if lima_names is None:
-        lima_names = list()
-    if counter_names is None:
-        counter_names = list()
+    lima_names, counter_names = validate_and_init_names(lima_names, counter_names)
 
     scan_node = _get_node(
         db_name, "scan", retry_timeout=retry_timeout, retry_period=retry_period

{ewoksdata-2.0.2 → ewoksdata-2.1.0}/src/ewoksdata/data/blissdata/blissdatav1.py

@@ -23,6 +23,8 @@ from blissdata.redis_engine.store import DataStore
 from blissdata.redis_engine.stream import StreamingClient
 from numpy.typing import ArrayLike
 
+from ..utils import validate_and_init_names
+
 try:
     from blissdata.stream import LimaStream
 except ImportError:
@@ -49,10 +51,7 @@ def iter_bliss_scan_data_from_memory(
 ) -> Generator[Dict[str, Any], None, None]:
     if not buffer_in_memory:
         raise NotImplementedError("Requires blissdata>=2.0.0rc1")
-    if lima_names is None:
-        lima_names = list()
-    if counter_names is None:
-        counter_names = list()
+    lima_names, counter_names = validate_and_init_names(lima_names, counter_names)
 
     data_store = _get_data_store()
     scan = data_store.load_scan(scan_key, scan_cls=Scan)

{ewoksdata-2.0.2 → ewoksdata-2.1.0}/src/ewoksdata/data/blissdata/blissdatav2.py

@@ -18,6 +18,8 @@ from blissdata.redis_engine.store import DataStore
 from numpy.typing import ArrayLike
 from packaging.version import Version
 
+from ..utils import validate_and_init_names
+
 _BLISSDATA_VERSION = Version(version("blissdata"))
 if _BLISSDATA_VERSION >= Version("2.3.0"):
     from blissdata.exceptions import EndOfStream
@@ -50,10 +52,7 @@ def iter_bliss_scan_data_from_memory(
     retry_period: Optional[float] = None,
     buffer_in_memory: bool = False,
 ) -> Generator[Dict[str, Any], None, None]:
-    if lima_names is None:
-        lima_names = list()
-    if counter_names is None:
-        counter_names = list()
+    lima_names, counter_names = validate_and_init_names(lima_names, counter_names)
 
     streams = _get_streams(scan_key, lima_names, counter_names)
     if not streams:

ewoksdata-2.1.0/src/ewoksdata/data/hdf5/dataset_writer/__init__.py

@@ -0,0 +1,4 @@
+from ._non_sequential import IndexedDatasetWriter  # noqa F401
+from ._non_sequential_stack import IndexedStackDatasetWriter  # noqa F401
+from ._sequential import DatasetWriter  # noqa F401
+from ._sequential_stack import StackDatasetWriter  # noqa F401

ewoksdata-2.1.0/src/ewoksdata/data/hdf5/dataset_writer/_base.py

@@ -0,0 +1,55 @@
+import time
+from typing import Optional
+
+import h5py
+import numpy
+
+
+class DatasetWriterBase:
+    def __init__(
+        self,
+        parent: h5py.Group,
+        name: str,
+        attrs: Optional[dict] = None,
+        flush_period: Optional[float] = None,
+    ) -> None:
+        self._file = parent.file
+        self._parent = parent
+        self._name = name
+        self._attrs = attrs
+        self._dataset_name = f"{parent.name}/{name}"
+        self._dataset = None
+        self._flush_period = flush_period
+        self._last_flush = None
+
+    @property
+    def dataset_name(self) -> str:
+        return self._dataset_name
+
+    def __enter__(self) -> "DatasetWriterBase":
+        return self
+
+    def __exit__(self, *args) -> None:
+        self.flush_buffer()
+
+    @property
+    def dataset(self) -> Optional[h5py.Dataset]:
+        return self._dataset
+
+    def _create_dataset(self, first_data_point: numpy.ndarray) -> h5py.Dataset:
+        raise NotImplementedError
+
+    def flush_buffer(self, align: bool = False) -> bool:
+        raise NotImplementedError
+
+    def _flush_time_expired(self) -> bool:
+        if self._flush_period is None:
+            return False
+        if self._last_flush is None:
+            self._last_flush = time.time()
+            return False
+        return (time.time() - self._last_flush) >= self._flush_period
+
+    def _flush_hdf5(self) -> None:
+        """Explicit HDF5 flush for non-locking readers."""
+        self._file.flush()

ewoksdata-2.1.0/src/ewoksdata/data/hdf5/dataset_writer/_non_sequential.py

@@ -0,0 +1,139 @@
+import logging
+import time
+from typing import List
+from typing import Optional
+
+import h5py
+import numpy
+from numpy.typing import ArrayLike
+
+from ..config import guess_dataset_config
+from ..types import StrictPositiveIntegral
+from ._base import DatasetWriterBase
+
+logger = logging.getLogger(__name__)
+
+
+class IndexedDatasetWriter(DatasetWriterBase):
+    """Append arrays of the same shape to a new HDF5 dataset in a potentially non-sequential manner.
+
+    Instead of creating a dataset with the :code:`h5py` API
+
+    .. code-block::python
+
+        h5group["mydataset"] = [[1,2,3], [4,5,6], [7,8,9]]
+
+    it can be done like this
+
+    .. code-block::python
+
+        with IndexedDatasetWriter(h5group, "mydataset") as writer:
+            writer.add_point(1, [4,5,6])
+            writer.add_points([0, 2], [[1,2,3], [7,8,9]])
+
+    Chunk size determination, compression and flushing is handled.
+    """
+
+    def __init__(
+        self,
+        parent: h5py.Group,
+        name: str,
+        npoints: Optional[StrictPositiveIntegral] = None,
+        attrs: Optional[dict] = None,
+        flush_period: Optional[float] = None,
+        overwrite: bool = False,
+    ) -> None:
+        super().__init__(parent, name, attrs=attrs, flush_period=flush_period)
+        self._npoints = npoints
+        self._overwrite = overwrite
+        self._npoints_added: int = 0
+
+        self._buffer: List[tuple[int, ArrayLike]] = []
+
+    def _create_dataset(self, first_data_point: numpy.ndarray) -> h5py.Dataset:
+        scan_shape = (self._npoints,)
+        detector_shape = first_data_point.shape
+        dtype = first_data_point.dtype
+
+        if self._npoints is None:
+            shape = (1,) + detector_shape
+            max_shape = scan_shape + detector_shape
+        else:
+            shape = scan_shape + detector_shape
+            max_shape = None
+
+        options = guess_dataset_config(
+            scan_shape, detector_shape, dtype=dtype, max_shape=max_shape
+        )
+        options.update(
+            shape=shape,
+            dtype=dtype,
+            fillvalue=numpy.nan,  # converts to 0 for integers
+        )
+        if max_shape:
+            options["maxshape"] = max_shape
+
+        if self._overwrite and self._name in self._parent:
+            del self._parent[self._name]
+
+        dset = self._parent.create_dataset(self._name, **options)
+        if self._attrs:
+            dset.attrs.update(self._attrs)
+        return dset
+
+    @property
+    def npoints_added(self) -> int:
+        return self._npoints_added
+
+    def add_point(
+        self,
+        data: ArrayLike,
+        index: int,
+    ) -> bool:
+        """Append one array to the dataset."""
+        if self._dataset is None:
+            self._dataset = self._create_dataset(numpy.asarray(data))
+
+        self._buffer.append((index, data))
+        self._npoints_added += 1
+        return self.flush_buffer()
+
+    def add_points(self, data: ArrayLike, indices: ArrayLike) -> bool:
+        """Append several arrays at once to the dataset."""
+        if self._dataset is None:
+            self._dataset = self._create_dataset(numpy.asarray(data[0]))
+
+        self._buffer.extend(zip(indices, data))
+        self._npoints_added += len(data)
+        return self.flush_buffer()
+
+    def flush_buffer(self, align: bool = False) -> bool:
+        if not self._buffer:
+            return False
+
+        if align:
+            logger.debug(
+                "Chunk-aligned flushing is only supported by DatasetWriter. "
+                "Only flush when `flush_period` has passed since the last flush."
+            )
+            if self._flush_time_expired():
+                return False
+
+        indices, values = zip(*self._buffer)
+        indices = numpy.asarray(indices)
+        values = numpy.asarray(values)
+
+        # Enlarge the dataset when needed
+        max_index = int(indices.max())
+        if max_index >= self._dataset.shape[0]:
+            self._dataset.resize(max_index + 1, axis=0)
+
+        # Copy data from buffer to HDF5
+        self._dataset[indices] = values
+
+        # Remove copied data from buffer
+        self._buffer.clear()
+
+        self._flush_hdf5()
+        self._last_flush = time.time()
+        return True

ewoksdata-2.1.0/src/ewoksdata/data/hdf5/dataset_writer/_non_sequential_stack.py

@@ -0,0 +1,158 @@
+import logging
+import time
+from typing import List
+from typing import Optional
+
+import h5py
+import numpy
+from numpy.typing import ArrayLike
+
+from ..config import guess_dataset_config
+from ..types import StrictPositiveIntegral
+from ._base import DatasetWriterBase
+
+logger = logging.getLogger(__name__)
+
+
+class IndexedStackDatasetWriter(DatasetWriterBase):
+    """Append arrays of the same shape to each item of a new HDF5 dataset
+    in a potentially non-sequential manner per item. So each item of the HDF5 dataset is a
+    stack to which we can append data in a sequential manner.
+
+    Instead of creating a dataset with the :code:`h5py` API
+
+    .. code-block::python
+
+        stack0 = [[1,2,3], [4,5,6], [7,8,9]]
+        stack1 = [[10,11,12], [13,14,15], [16,17,18]]
+        h5group["mydataset"] = [stack0, stack1]
+
+    it can be done like this
+
+    .. code-block::python
+
+        with IndexedStackDatasetWriter(h5group, "mydataset") as writer:
+            writer.add_point([4,5,6], 0, 1)
+            writer.add_point([13,14,15], 1, 1)
+            writer.add_points([[10,11,12], [16,17,18]], [1, 1], [0, 2])
+            writer.add_points([[1,2,3], [7,8,9]], [0, 0], [0, 2])
+
+    Chunk size determination, compression and flushing is handled.
+    """
+
+    def __init__(
+        self,
+        parent: h5py.Group,
+        name: str,
+        npoints: Optional[StrictPositiveIntegral] = None,
+        nstack: Optional[StrictPositiveIntegral] = None,
+        attrs: Optional[dict] = None,
+        flush_period: Optional[float] = None,
+    ) -> None:
+        super().__init__(parent, name, attrs=attrs, flush_period=flush_period)
+        self._npoints = npoints
+        self._nstack = nstack
+        self._buffer: List[tuple[int, int, ArrayLike]] = []
+
+    def _create_dataset(
+        self, first_data_point: ArrayLike, stack_index: int
+    ) -> h5py.Dataset:
+        scan_shape = (self._nstack, self._npoints)
+        first_data_point = numpy.asarray(first_data_point)
+        detector_shape = first_data_point.shape
+        dtype = first_data_point.dtype
+
+        if self._nstack is None or self._npoints is None:
+            shape = (1, 1) + detector_shape
+            max_shape = scan_shape + detector_shape
+        else:
+            shape = scan_shape + detector_shape
+            max_shape = None
+
+        options = guess_dataset_config(
+            scan_shape, detector_shape, dtype=dtype, max_shape=max_shape
+        )
+        options.update(
+            shape=shape,
+            dtype=dtype,
+            fillvalue=numpy.nan,  # converts to 0 for integers
+        )
+        if max_shape:
+            options["maxshape"] = max_shape
+
+        dset = self._parent.create_dataset(self._name, **options)
+        if self._attrs:
+            dset.attrs.update(self._attrs)
+        return dset
+
+    def add_point(self, data: ArrayLike, stack_index: int, point_index: int) -> bool:
+        """Append one array to one stack of the dataset."""
+        if self._dataset is None:
+            self._dataset = self._create_dataset(data, stack_index)
+
+        self._buffer.append((stack_index, point_index, data))
+        return self.flush_buffer()
+
+    def add_points(
+        self,
+        data: ArrayLike,
+        stack_indices: ArrayLike,
+        point_indices: ArrayLike,
+    ) -> bool:
+        """Append several arrays at once to one stack of the dataset."""
+        if not (len(stack_indices) == len(point_indices) == len(data)):
+            raise ValueError(
+                "stack_indices, point_indices, and data must have same length"
+            )
+
+        if self._dataset is None:
+            self._dataset = self._create_dataset(data[0], stack_indices[0])
+
+        self._buffer.extend(
+            zip(
+                map(int, stack_indices),
+                map(int, point_indices),
+                data,
+            )
+        )
+        return self.flush_buffer()
+
+    def flush_buffer(self, align: bool = False) -> bool:
+        if not self._buffer:
+            return False
+
+        if align:
+            logger.debug(
+                "Chunk-aligned flushing is only supported by StackDatasetWriter. "
+                "Only flush when `flush_period` has passed since the last flush."
+            )
+            if self._flush_time_expired():
+                return False
+
+        stack_indices, point_indices, values = zip(*self._buffer)
+        stack_indices = numpy.asarray(stack_indices)
+        point_indices = numpy.asarray(point_indices)
+        values = numpy.asarray(values)
+
+        # Enlarge the dataset when needed
+        max_stack = int(stack_indices.max())
+        max_point = int(point_indices.max())
+
+        shape0, shape1 = self._dataset.shape[:2]
+
+        if max_stack >= shape0:
+            self._dataset.resize(max_stack + 1, axis=0)
+
+        if max_point >= shape1:
+            self._dataset.resize(max_point + 1, axis=1)
+
+        # Copy data from buffer to HDF5
+        for i0, i1, value in zip(stack_indices, point_indices, values):
+            self._dataset[i0, i1] = value
+
+        # Remove copied data from buffer
+        self._buffer.clear()
+
+        self._flush_hdf5()
+        self._last_flush = time.time()
+        return True

ewoksdata-2.1.0/src/ewoksdata/data/hdf5/dataset_writer/_sequential.py

@@ -0,0 +1,143 @@
+import time
+from typing import List
+from typing import Optional
+
+import h5py
+import numpy
+from numpy.typing import ArrayLike
+
+from ..config import guess_dataset_config
+from ..types import StrictPositiveIntegral
+from ._base import DatasetWriterBase
+
+
+class DatasetWriter(DatasetWriterBase):
+    """Append arrays of the same shape to a new HDF5 dataset in a sequential manner.
+
+    Instead of creating a dataset with the :code:`h5py` API
+
+    .. code-block::python
+
+        h5group["mydataset"] = [[1,2,3], [4,5,6], [7,8,9]]
+
+    it can be done like this
+
+    .. code-block::python
+
+        with DatasetWriter(h5group, "mydataset") as writer:
+            writer.add_point([1,2,3])
+            writer.add_points([[4,5,6], [7,8,9]])
+
+    Chunk size determination, chunk-aligned writing, compression and flushing is handled.
+    """
+
+    def __init__(
+        self,
+        parent: h5py.Group,
+        name: str,
+        npoints: Optional[StrictPositiveIntegral] = None,
+        attrs: Optional[dict] = None,
+        flush_period: Optional[float] = None,
+        overwrite: bool = False,
+    ) -> None:
+        super().__init__(parent, name, attrs=attrs, flush_period=flush_period)
+        self._npoints = npoints
+        self._overwrite = overwrite
+        self._chunked: bool = False
+        self._npoints_added: int = 0
+
+        self._buffer: List[ArrayLike] = list()
+        self._chunk_size: int = 0
+        self._flushed_size: int = 0
+
+    def _create_dataset(self, first_data_point: numpy.ndarray) -> h5py.Dataset:
+        scan_shape = (self._npoints,)
+        detector_shape = first_data_point.shape
+        dtype = first_data_point.dtype
+
+        if self._npoints is None:
+            shape = (1,) + detector_shape
+            max_shape = scan_shape + detector_shape
+        else:
+            shape = scan_shape + detector_shape
+            max_shape = None
+
+        options = guess_dataset_config(
+            scan_shape, detector_shape, dtype=dtype, max_shape=max_shape
+        )
+        options.update(
+            shape=shape,
+            dtype=dtype,
+            fillvalue=numpy.nan,  # converts to 0 for integers
+        )
+
+        if max_shape:
+            options["maxshape"] = max_shape
+
+        if options["chunks"]:
+            self._chunked = True
+            self._chunk_size = options["chunks"][0]
+
+        if self._overwrite and self._name in self._parent:
+            del self._parent[self._name]
+
+        dset = self._parent.create_dataset(self._name, **options)
+        if self._attrs:
+            dset.attrs.update(self._attrs)
+        return dset
+
+    @property
+    def npoints_added(self) -> int:
+        return self._npoints_added
+
+    def add_point(self, data: ArrayLike) -> bool:
+        """Append one array to the dataset."""
+        if self._dataset is None:
+            self._dataset = self._create_dataset(data)
+
+        self._buffer.append(data)
+        self._npoints_added += 1
+        return self.flush_buffer(align=True)
+
+    def add_points(self, data: ArrayLike) -> bool:
+        """Append several arrays at once to the dataset."""
+        if self._dataset is None:
+            self._dataset = self._create_dataset(data[0])
+
+        self._buffer.extend(data)
+        self._npoints_added += len(data)
+        return self.flush_buffer(align=True)
+
+    def flush_buffer(self, align: bool = False) -> bool:
+        # Determine how many points to flush
+        chunk_size = len(self._buffer)
+
+        if self._flush_time_expired():
+            flush_size = chunk_size
+        elif align and self._chunked:
+            n = chunk_size + (self._flushed_size % self._chunk_size)
+            flush_size = n // self._chunk_size * self._chunk_size
+            flush_size = min(flush_size, chunk_size)
+        else:
+            flush_size = chunk_size
+
+        if flush_size == 0:
+            return False
+
+        # Enlarge the dataset when needed
+        nalloc = self._dataset.shape[0]
+        istart = self._flushed_size
+        flushed_size = istart + flush_size
+        if self._chunked and flushed_size > nalloc:
+            self._dataset.resize(flushed_size, axis=0)
+
+        # Copy data from buffer to HDF5
+        self._dataset[istart : istart + flush_size] = self._buffer[:flush_size]
+
+        # Remove copied data from buffer
+        self._buffer = self._buffer[flush_size:]
+        self._flushed_size = flushed_size
+
+        self._flush_hdf5()
+        self._last_flush = time.time()
+        return True

ewoksdata-2.0.2/src/ewoksdata/data/hdf5/dataset_writer.py → ewoksdata-2.1.0/src/ewoksdata/data/hdf5/dataset_writer/_sequential_stack.py

@@ -6,184 +6,12 @@ import h5py
 import numpy
 from numpy.typing import ArrayLike
 
-from .config import guess_dataset_config
-from .types import StrictPositiveIntegral
+from ..config import guess_dataset_config
+from ..types import StrictPositiveIntegral
+from ._base import DatasetWriterBase
 
 
-class _DatasetWriterBase:
-    def __init__(
-        self,
-        parent: h5py.Group,
-        name: str,
-        attrs: Optional[dict] = None,
-        flush_period: Optional[float] = None,
-    ) -> None:
-        self._file = parent.file
-        self._parent = parent
-        self._name = name
-        self._attrs = attrs
-        self._dataset_name = f"{parent.name}/{name}"
-        self._dataset = None
-        self._flush_period = flush_period
-        self._last_flush = None
-
-    @property
-    def dataset_name(self) -> str:
-        return self._dataset_name
-
-    def __enter__(self) -> "_DatasetWriterBase":
-        return self
-
-    def __exit__(self, *args) -> None:
-        self.flush_buffer()
-
-    @property
-    def dataset(self) -> Optional[h5py.Dataset]:
-        return self._dataset
-
-    def _create_dataset(self, first_data_point: numpy.ndarray) -> h5py.Dataset:
-        raise NotImplementedError
-
-    def flush_buffer(self, align: bool = False) -> bool:
-        raise NotImplementedError
-
-    def _flush_time_expired(self) -> bool:
-        if self._flush_period is None:
-            return False
-        if self._last_flush is None:
-            self._last_flush = time.time()
-            return False
-        return (time.time() - self._last_flush) >= self._flush_period
-
-    def _flush_hdf5(self) -> None:
-        """Explicit HDF5 flush for non-locking readers."""
-        self._file.flush()
-
-
-class DatasetWriter(_DatasetWriterBase):
-    """Append arrays of the same shape to a new HDF5 dataset in a sequential manner.
-
-    Instead of creating a dataset with the :code:`h5py` API
-
-    .. code-block::python
-
-        h5group["mydataset"] = [[1,2,3], [4,5,6], [7,8,9]]
-
-    it can be done like this
-
-    .. code-block::python
-
-        with DatasetWriter(h5group, "mydataset") as writer:
-            writer.add_point([1,2,3])
-            writer.add_points([[4,5,6], [7,8,9]])
-
-    Chunk size determination, chunk-aligned writing, compression and flushing is handled.
-    """
-
-    def __init__(
-        self,
-        parent: h5py.Group,
-        name: str,
-        npoints: Optional[StrictPositiveIntegral] = None,
-        attrs: Optional[dict] = None,
-        flush_period: Optional[float] = None,
-        overwrite: bool = False,
-    ) -> None:
-        super().__init__(parent, name, attrs=attrs, flush_period=flush_period)
-        self._npoints = npoints
-        self._overwrite = overwrite
-        self._chunked: bool = False
-        self._npoints_added: int = 0
-
-        self._buffer: List[ArrayLike] = list()
-        self._chunk_size: int = 0
-        self._flushed_size: int = 0
-
-    def _create_dataset(self, first_data_point: numpy.ndarray) -> h5py.Dataset:
-        scan_shape = (self._npoints,)
-        detector_shape = first_data_point.shape
-        dtype = first_data_point.dtype
-        if self._npoints is None:
-            max_shape = scan_shape + detector_shape
-            shape = (1,) + first_data_point.shape
-        else:
-            max_shape = None
-            shape = scan_shape + first_data_point.shape
-
-        options = guess_dataset_config(
-            scan_shape, detector_shape, dtype=dtype, max_shape=max_shape
-        )
-        options["shape"] = shape
-        options["dtype"] = dtype
-        options["fillvalue"] = numpy.nan  # converts to 0 for integers
-        if max_shape:
-            options["maxshape"] = max_shape
-        if options["chunks"]:
-            self._chunked = True
-            self._chunk_size = options["chunks"][0]
-        if self._overwrite and self._name in self._parent:
-            del self._parent[self._name]
-        dset = self._parent.create_dataset(self._name, **options)
-        if self._attrs:
-            dset.attrs.update(self._attrs)
-        return dset
-
-    @property
-    def npoints_added(self) -> int:
-        return self._npoints_added
-
-    def add_point(self, data: ArrayLike) -> bool:
-        """Append one array to the dataset."""
-        if self._dataset is None:
-            self._dataset = self._create_dataset(data)
-        self._buffer.append(data)
-        self._npoints_added += 1
-        return self.flush_buffer(align=True)
-
-    def add_points(self, data: ArrayLike) -> bool:
-        """Append several arrays at once to the dataset."""
-        if self._dataset is None:
-            self._dataset = self._create_dataset(data[0])
-        self._buffer.extend(data)
-        self._npoints_added += len(data)
-        return self.flush_buffer(align=True)
-
-    def flush_buffer(self, align: bool = False) -> bool:
-        # Determine how many points to flush
-        chunk_size = len(self._buffer)
-
-        if self._flush_time_expired():
-            flush_size = chunk_size
-        elif align and self._chunked:
-            n = chunk_size + (self._flushed_size % self._chunk_size)
-            flush_size = n // self._chunk_size * self._chunk_size
-            flush_size = min(flush_size, chunk_size)
-        else:
-            flush_size = chunk_size
-
-        if flush_size == 0:
-            return False
-
-        # Enlarge the dataset when needed
-        nalloc = self._dataset.shape[0]
-        istart = self._flushed_size
-        flushed_size = istart + flush_size
-        if self._chunked and flushed_size > nalloc:
-            self._dataset.resize(flushed_size, axis=0)
-
-        # Copy data from buffer to HDF5
-        self._dataset[istart : istart + flush_size] = self._buffer[:flush_size]
-
-        # Remove copied data from buffer
-        self._buffer = self._buffer[flush_size:]
-        self._flushed_size = flushed_size
-
-        self._flush_hdf5()
-        self._last_flush = time.time()
-        return True
-
-
-class StackDatasetWriter(_DatasetWriterBase):
+class StackDatasetWriter(DatasetWriterBase):
     """Append arrays of the same shape to each item of a new HDF5 dataset
     in a sequential manner per item. So each item of the HDF5 dataset is a
     stack to which we can append data in a sequential manner.
@@ -228,45 +56,45 @@ class StackDatasetWriter(_DatasetWriterBase):
         self._flushed_size_dim1: ArrayLike = numpy.array(list(), dtype=int)
 
     def _create_dataset(
-        self, first_data_point: numpy.ndarray, stack_index: int
+        self, first_data_point: ArrayLike, stack_index: int
     ) -> h5py.Dataset:
         scan_shape = (self._nstack, self._npoints)
+        first_data_point = numpy.asarray(first_data_point)
         detector_shape = first_data_point.shape
         dtype = first_data_point.dtype
-        if self._npoints is None or self._nstack is None:
+
+        if self._nstack is None or self._npoints is None:
+            shape = (stack_index + 1, 1) + detector_shape
             max_shape = scan_shape + detector_shape
-            shape = (stack_index + 1, 1) + first_data_point.shape
         else:
+            shape = scan_shape + detector_shape
             max_shape = None
-            shape = scan_shape + first_data_point.shape
 
         options = guess_dataset_config(
             scan_shape, detector_shape, dtype=dtype, max_shape=max_shape
         )
-        options["shape"] = shape
-        options["dtype"] = dtype
-        options["fillvalue"] = numpy.nan  # converts to 0 for integers
+        options.update(
+            shape=shape,
+            dtype=dtype,
+            fillvalue=numpy.nan,  # converts to 0 for integers
+        )
         if max_shape:
             options["maxshape"] = max_shape
+
         if options["chunks"]:
             self._chunked = True
             self._chunk_size = numpy.array(options["chunks"][:2], dtype=int)
+
         dset = self._parent.create_dataset(self._name, **options)
         if self._attrs:
             dset.attrs.update(self._attrs)
         return dset
 
-    def _get_buffer(self, stack_index: int) -> List[ArrayLike]:
-        # Add stack buffers when needed
-        for _ in range(max(stack_index - len(self._buffers) + 1, 0)):
-            self._buffers.append(list())
-            self._flushed_size_dim1 = numpy.append(self._flushed_size_dim1, 0)
-        return self._buffers[stack_index]
-
     def add_point(self, data: ArrayLike, stack_index: int) -> bool:
         """Append one array to one stack of the dataset."""
         if self._dataset is None:
             self._dataset = self._create_dataset(data, stack_index)
+
         buffer = self._get_buffer(stack_index)
         buffer.append(data)
         return self.flush_buffer(align=True)
@@ -275,6 +103,7 @@ class StackDatasetWriter(_DatasetWriterBase):
         """Append several arrays at once to one stack of the dataset."""
         if self._dataset is None:
             self._dataset = self._create_dataset(data[0], stack_index)
+
         buffer = self._get_buffer(stack_index)
         buffer.extend(data)
         return self.flush_buffer(align=True)
@@ -357,3 +186,10 @@ class StackDatasetWriter(_DatasetWriterBase):
         self._flush_hdf5()
         self._last_flush = time.time()
         return True
+
+    def _get_buffer(self, stack_index: int) -> List[ArrayLike]:
+        # Add stack buffers when needed
+        for _ in range(max(stack_index - len(self._buffers) + 1, 0)):
+            self._buffers.append(list())
+            self._flushed_size_dim1 = numpy.append(self._flushed_size_dim1, 0)
+        return self._buffers[stack_index]

{ewoksdata-2.0.2 → ewoksdata-2.1.0}/src/ewoksdata/data/utils.py

@@ -1,5 +1,8 @@
 from numbers import Number
+from typing import List
 from typing import Mapping
+from typing import Optional
+from typing import Tuple
 
 import numpy
 
@@ -28,3 +31,23 @@ def data_from_storage(data, remove_numpy=True):
         }
     else:
         return data
+
+
+def _validate_names(names: Optional[List[str]], name_type: str) -> None:
+    if names is None:
+        return
+    for n in names:
+        if not isinstance(n, str) or not n.strip():
+            raise ValueError(f"Invalid {name_type} name: {n!r}")
+
+
+def validate_and_init_names(
+    lima_names: Optional[List[str]],
+    counter_names: Optional[List[str]],
+) -> Tuple[List[str], List[str]]:
+    _validate_names(lima_names, "LIMA")
+    _validate_names(counter_names, "Counter")
+    return (
+        lima_names if lima_names is not None else [],
+        counter_names if counter_names is not None else [],
+    )

{ewoksdata-2.0.2 → ewoksdata-2.1.0}/src/ewoksdata/tests/test_data_bliss.py

@@ -31,6 +31,10 @@ def test_get_data_hdf5(tmpdir):
     with pytest.raises(TimeoutError):
         _ = bliss.get_data(f"{filename_non_existing}?path=/img", retry_timeout=0)
 
+    filename_non_existing = str(tmpdir / "data_non_existing.edf")
+    with pytest.raises(TimeoutError):
+        _ = bliss.get_data(filename_non_existing, retry_timeout=0)
+
 
 @pytest.mark.parametrize("lima_names", [(), ("p3",), ("p3", "p4")])
 @pytest.mark.parametrize("counter_names", [(), ("diode1",), ("diode1", "diode2")])
@@ -58,6 +62,27 @@ def test_iter_bliss_data(lima_names, counter_names, bliss_scan):
         assert index is None
 
 
+@pytest.mark.parametrize(
+    "lima_names,counter_names",
+    [
+        ([""], []),
+        ([" "], []),
+        ([], [""]),
+        ([], [" "]),
+    ],
+)
+def test_iter_bliss_data_invalid_names(bliss_scan, lima_names, counter_names):
+    filename, _, scan_nb = bliss_scan
+    with bliss.iter_bliss_data(
+        filename, scan_nb, lima_names=lima_names, counter_names=counter_names
+    ) as bliss_data_iterator:
+        with pytest.raises(ValueError) as exc_info:
+            next(bliss_data_iterator)
+    assert "Invalid LIMA name" in str(exc_info.value) or "Invalid Counter name" in str(
+        exc_info.value
+    )
+
+
 def test_closure_of_iter_bliss_data(bliss_scan):
     filename, npoints, scan_nb = bliss_scan
 

{ewoksdata-2.0.2 → ewoksdata-2.1.0}/src/ewoksdata/tests/test_dataset_writer.py

@@ -13,9 +13,9 @@ from ..data.hdf5 import dataset_writer
 @pytest.mark.parametrize("npoints", (1, 3, 1000))
 @pytest.mark.parametrize("flush_period", (None, 0.1))
 @pytest.mark.parametrize("known_npoints", (True, False))
-def test_dataset_writer(tmpdir, npoints, flush_period, known_npoints):
+def test_dataset_writer(tmp_path, npoints, flush_period, known_npoints):
     expected = list()
-    filename = str(tmpdir / "test.h5")
+    filename = str(tmp_path / "test.h5")
     if flush_period is None:
         sleep_time = None
     else:
@@ -47,7 +47,7 @@ def test_dataset_writer(tmpdir, npoints, flush_period, known_npoints):
 @pytest.mark.parametrize("known_nstack", (True, False))
 @pytest.mark.parametrize("append_stacks_in_parallel", (True, False))
 def test_stack_dataset_writer(
-    tmpdir,
+    tmp_path,
     nstack,
     npoints,
     flush_period,
@@ -56,7 +56,7 @@ def test_stack_dataset_writer(
     append_stacks_in_parallel,
 ):
     expected = [list() for _ in range(nstack)]
-    filename = str(tmpdir / "test.h5")
+    filename = str(tmp_path / "test.h5")
     if flush_period is None:
         sleep_time = None
     else:
@@ -92,11 +92,101 @@ def test_stack_dataset_writer(
     numpy.testing.assert_allclose(data, expected)
 
 
-def test_concurrent_reader(tmpdir):
+@pytest.mark.parametrize("npoints", (1, 3, 1000))
+@pytest.mark.parametrize("flush_period", (None, 0.1))
+@pytest.mark.parametrize("known_npoints", (True, False))
+def test_indexed_dataset_writer(tmp_path, npoints, flush_period, known_npoints):
+    filename = str(tmp_path / "test.h5")
+
+    if flush_period is None:
+        sleep_time = None
+    else:
+        sleep_time = flush_period + 0.1
+
+    isleep = npoints // 3
+
+    kwargs = {"flush_period": flush_period}
+    if known_npoints:
+        kwargs["npoints"] = npoints
+
+    # Generate data and shuffled indices
+    indices = numpy.arange(npoints)
+    numpy.random.shuffle(indices)
+
+    expected = [None] * npoints
+    data_points = [numpy.random.random((10, 20)) for _ in range(npoints)]
+    for idx, data in zip(indices, data_points):
+        expected[idx] = data
+
+    with h5py.File(filename, mode="w") as f:
+        with dataset_writer.IndexedDatasetWriter(f, "data", **kwargs) as writer:
+            for i, (idx, data) in enumerate(zip(indices, data_points)):
+                writer.add_point(data, idx)
+                if sleep_time and i == isleep:
+                    time.sleep(sleep_time)
+
+    with h5py.File(filename, mode="r") as f:
+        data = f["data"][()]
+
+    numpy.testing.assert_allclose(data, expected)
+
+
+@pytest.mark.parametrize("nstack", (1, 4))
+@pytest.mark.parametrize("npoints", (1, 3, 100))
+@pytest.mark.parametrize("flush_period", (None, 0.1))
+@pytest.mark.parametrize("known_npoints", (True, False))
+@pytest.mark.parametrize("known_nstack", (True, False))
+def test_indexed_stack_dataset_writer(
+    tmp_path,
+    nstack,
+    npoints,
+    flush_period,
+    known_npoints,
+    known_nstack,
+):
+    filename = str(tmp_path / "test.h5")
+
+    if flush_period is None:
+        sleep_time = None
+    else:
+        sleep_time = flush_period + 0.1
+
+    isleep = (nstack * npoints) // 3
+
+    kwargs = {"flush_period": flush_period}
+    if known_npoints:
+        kwargs["npoints"] = npoints
+    if known_nstack:
+        kwargs["nstack"] = nstack
+
+    # Generate data and shuffled indices
+    indices = list(itertools.product(range(nstack), range(npoints)))
+    numpy.random.shuffle(indices)
+
+    expected = [[None for _ in range(npoints)] for _ in range(nstack)]
+    values = [numpy.random.random((10, 20)) for _ in indices]
+
+    for (istack, ipoint), value in zip(indices, values):
+        expected[istack][ipoint] = value
+
+    with h5py.File(filename, mode="w") as f:
+        with dataset_writer.IndexedStackDatasetWriter(f, "data", **kwargs) as writer:
+            for i, ((istack, ipoint), value) in enumerate(zip(indices, values)):
+                writer.add_point(value, istack, ipoint)
+                if sleep_time and i == isleep:
+                    time.sleep(sleep_time)
+
+    with h5py.File(filename, mode="r") as f:
+        data = f["data"][()]
+
+    numpy.testing.assert_allclose(data, expected)
+
+
+def test_concurrent_reader(tmp_path):
     npoints = 50
-    hdf5_filename = str(tmpdir / "test.h5")
-    read_timestamps = str(tmpdir / "read_timestamps.log")
-    write_timestamps = str(tmpdir / "write_timestamps.log")
+    hdf5_filename = str(tmp_path / "test.h5")
+    read_timestamps = str(tmp_path / "read_timestamps.log")
+    write_timestamps = str(tmp_path / "write_timestamps.log")
     dataset_name = "data"
 
     # Run read loop in a sub-process

{ewoksdata-2.0.2 → ewoksdata-2.1.0/src/ewoksdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ewoksdata
-Version: 2.0.2
+Version: 2.1.0
 Summary: Ewoks tasks for data access
 Author-email: ESRF <dau-pydev@esrf.fr>
 License: # MIT License

{ewoksdata-2.0.2 → ewoksdata-2.1.0}/src/ewoksdata.egg-info/SOURCES.txt

@@ -19,8 +19,13 @@ src/ewoksdata/data/blissdata/blissdatav1.py
 src/ewoksdata/data/blissdata/blissdatav2.py
 src/ewoksdata/data/hdf5/__init__.py
 src/ewoksdata/data/hdf5/config.py
-src/ewoksdata/data/hdf5/dataset_writer.py
 src/ewoksdata/data/hdf5/types.py
+src/ewoksdata/data/hdf5/dataset_writer/__init__.py
+src/ewoksdata/data/hdf5/dataset_writer/_base.py
+src/ewoksdata/data/hdf5/dataset_writer/_non_sequential.py
+src/ewoksdata/data/hdf5/dataset_writer/_non_sequential_stack.py
+src/ewoksdata/data/hdf5/dataset_writer/_sequential.py
+src/ewoksdata/data/hdf5/dataset_writer/_sequential_stack.py
 src/ewoksdata/tests/__init__.py
 src/ewoksdata/tests/conftest.py
 src/ewoksdata/tests/test_contextiterator.py