ewoksdata 1.3.0__tar.gz → 2.0.0__tar.gz

{ewoksdata-1.3.0/src/ewoksdata.egg-info → ewoksdata-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ewoksdata
-Version: 1.3.0
+Version: 2.0.0
 Summary: Ewoks tasks for data access
 Author-email: ESRF <dau-pydev@esrf.fr>
 License: # MIT License
@@ -53,6 +53,7 @@ Provides-Extra: dev
 Requires-Dist: ewoksdata[test]; extra == "dev"
 Requires-Dist: black>=25; extra == "dev"
 Requires-Dist: flake8>=4; extra == "dev"
+Requires-Dist: isort; extra == "dev"
 Provides-Extra: doc
 Requires-Dist: ewoksdata[test]; extra == "doc"
 Requires-Dist: sphinx>=4.5; extra == "doc"

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ewoksdata"
-version = "1.3.0"
+version = "2.0.0"
 authors = [{ name = "ESRF", email = "dau-pydev@esrf.fr" }]
 description = "Ewoks tasks for data access"
 readme = { file = "README.md", content-type = "text/markdown" }
@@ -35,9 +35,19 @@ Issues = "https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/issues"
 Changelog = "https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/-/blob/main/CHANGELOG.md"
 
 [project.optional-dependencies]
-online = ["blissdata[tango]"]
-test = ["ewoksdata[online]", "pytest >=7"]
-dev = ["ewoksdata[test]", "black >=25", "flake8 >=4"]
+online = [
+    "blissdata[tango]"
+]
+test = [
+    "ewoksdata[online]",
+    "pytest >=7",
+]
+dev = [
+    "ewoksdata[test]",
+    "black >=25",
+    "flake8 >=4",
+    "isort",
+]
 doc = [
     "ewoksdata[test]",
     "sphinx >=4.5",
@@ -53,3 +63,7 @@ where = ["src"]
 
 [tool.coverage.run]
 omit = ['*/tests/*']
+
+[tool.isort]
+profile = "black"
+force_single_line = "True"

ewoksdata-2.0.0/src/ewoksdata/data/bliss.py

@@ -0,0 +1,286 @@
+"""Utilities for reading Bliss scan data from HDF5/Nexus files
+or from memory (Redis or device servers).
+"""
+
+import logging
+import sys
+from numbers import Integral
+from numbers import Number
+from typing import Any
+from typing import Dict
+from typing import Iterator
+from typing import List
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+from typing import Union
+
+import h5py
+import hdf5plugin  # noqa: F401 - needed for transparent compression handling
+import numpy
+from numpy.typing import ArrayLike
+from silx.io import h5py_utils
+from silx.io.url import DataUrl
+from silx.io.utils import get_data as silx_get_data
+from silx.utils import retry as retrymod
+
+from . import hdf5
+from . import nexus
+from . import url
+from .blissdata import BUFFER_IN_MEMORY_DEFAULT
+from .blissdata import dynamic_hdf5
+from .blissdata import iter_bliss_scan_data_from_memory
+from .blissdata import iter_bliss_scan_data_from_memory_slice  # noqa: F401
+from .blissdata import last_lima_image  # noqa: F401
+from .contextiterator import contextiterator
+
+logger = logging.getLogger(__name__)
+
+
+def get_data(
+    data: Union[str, DataUrl, ArrayLike, Number],
+    **hdf5_options: Any,
+) -> Union[numpy.ndarray, Number, Sequence]:
+    """
+    Return a NumPy array or a scalar from *data*.
+
+    :param data: Either a data-URL, an array-like object or a scalar number.
+    :param **hdf5_options: Additional options forwarded to :class:`silx.io.h5py_utils.File`.
+    :return: The requested data.
+    """
+    if isinstance(data, (str, DataUrl)):
+        data_url = url.as_dataurl(data)
+        filename, h5path, idx = url.h5dataset_url_parse(data_url)
+
+        if h5py.is_hdf5(filename):
+            return _get_hdf5_data(filename, h5path, idx=idx, **hdf5_options)
+
+        if not data_url.scheme():
+            for scheme in ("fabio", "silx"):
+                if sys.platform == "win32":
+                    new_data_url = url.as_dataurl(f"{scheme}:///{data}")
+                else:
+                    new_data_url = url.as_dataurl(f"{scheme}://{data}")
+                if new_data_url.is_valid():
+                    data_url = new_data_url
+                    break
+
+        return silx_get_data(data_url)
+
+    if isinstance(data, (Sequence, Number, numpy.ndarray)):
+        return data
+
+    raise TypeError(f"Unsupported data type: {type(data)!r}")
+
+
+def get_image(
+    data: Union[str, DataUrl, ArrayLike, Number], **hdf5_options: Any
+) -> numpy.ndarray:
+    """
+    Return a NumPy array or a scalar from *data*.
+
+    :param data: Either a data-URL, an array-like object or a scalar number.
+    :param retry_timeout: Timeout for the retry mechanism.
+    :param retry_period: Interval between retries.
+    :param **hdf5_options: Additional options forwarded to :class:`silx.io.h5py_utils.File`.
+    :return: A 2-D image.
+    """
+    data = get_data(data, **hdf5_options)
+    return numpy.atleast_2d(numpy.squeeze(data))
+
+
+@h5py_utils.retry()
+def _get_hdf5_data(
+    filename: str,
+    h5path: str,
+    idx: Optional[Tuple[int, ...]] = None,
+    **hdf5_options: Any,
+) -> numpy.ndarray:
+    """
+    Read a slice of a dataset inside an HDF5 / Nexus file.
+
+    :param filename: Path to the HDF5 / Nexus file.
+    :param h5path: Internal path to the dataset (e.g. ``/entry/data``).
+    :param idx: Optional index or slice tuple used to read a subset of the dataset.
+    :param **hdf5_options: Additional options forwarded to :class:`silx.io.h5py_utils.File`.
+    :return: The requested slice.
+    """
+    with hdf5.h5context(filename, h5path, **hdf5_options) as dset:
+        if _is_bliss_file(dset):
+            # ``end_time`` is added by the Bliss writer when the scan is
+            # finished.  If it is missing we wait for the file to be
+            # completed.
+            if "end_time" not in nexus.get_nxentry(dset):
+                raise retrymod.RetryError("Bliss file not yet complete")
+        if idx is None:
+            idx = tuple()
+        return dset[idx]
+
+
+@contextiterator
+def iter_bliss_scan_data_from_hdf5(
+    filename: str,
+    scan_nr: Integral,
+    subscan: Optional[Integral] = None,
+    lima_names: Optional[List[str]] = None,
+    counter_names: Optional[List[str]] = None,
+    retry_timeout: Optional[float] = None,
+    retry_period: Optional[float] = None,
+    **hdf5_options: Any,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Iterate over the data of a single Bliss scan stored in an HDF5 file.
+
+    :param filename: Bliss dataset filename (HDF5 / Nexus).
+    :param scan_nr: Scan number inside the file.
+    :param lima_names: List of LIMA detector names.
+    :param counter_names: List of non-LIMA counter names.
+    :param subscan: Sub-scan number (e.g. ``10.2`` -> ``scan_nr=10`` and ``subscan=2``).
+    :param retry_timeout: Timeout for the retry mechanism.
+    :param retry_period: Interval between retries.
+    :param **hdf5_options: Additional options forwarded to :class:`blissdata.h5api.dynamic_hdf5.File`.
+    :yields: Mapping ``detector_name -> value`` for the each scan point.
+    """
+    if not subscan:
+        subscan = 1
+    lima_names = lima_names or []
+    counter_names = counter_names or []
+
+    with dynamic_hdf5.File(
+        filename,
+        lima_names=lima_names,
+        retry_timeout=retry_timeout,
+        retry_period=retry_period,
+        **hdf5_options,
+    ) as root:
+        scan = root[f"{scan_nr}.{subscan}"]
+        # assert _is_bliss_file(scan), "Not a Bliss dataset file"
+        measurement = scan["measurement"]
+        instrument = scan["instrument"]
+
+        datasets: Dict[str, Any] = {name: measurement[name] for name in counter_names}
+        for name in lima_names:
+            datasets[name] = instrument[f"{name}/data"]
+
+        names = list(datasets.keys())
+        for values in zip(*datasets.values()):
+            yield dict(zip(names, values))
+
+
+@contextiterator
+def iter_bliss_data(
+    filename: str,
+    scan_nr: Integral,
+    subscan: Optional[Integral] = None,
+    scan_key: Optional[str] = None,
+    lima_names: Optional[List[str]] = None,
+    counter_names: Optional[List[str]] = None,
+    start_index: Optional[int] = None,
+    retry_timeout: Optional[float] = None,
+    retry_period: Optional[float] = None,
+    buffer_in_memory: bool = BUFFER_IN_MEMORY_DEFAULT,
+    **options: Any,
+) -> Iterator[Tuple[int, Dict[str, Any]]]:
+    """
+    Iterate over a Bliss scan and also return the scan-point index.
+
+    :param filename: Bliss dataset filename.
+    :param scan_nr: Scan number inside the dataset.
+    :param subscan: Sub-scan number (e.g. ``10.2`` -> ``scan_nr=10`` and ``subscan=2``).
+    :param scan_key: Identifier of a scan for blissdata. Iterate scan data from blissdata-cache.
+    :param lima_names: List of LIMA detector names.
+    :param counter_names: List of non-LIMA counter names.
+    :param start_index: First point to yield (default ``0`` -> all points).
+    :param retry_timeout: Timeout for the retry mechanism.
+    :param retry_period: Interval between retries.
+    :param buffer_in_memory: Blissdata buffering in memory. Only applies when `scan_key` is provided.
+                             Use only when you have enough memory or the data processing can keep up with the scan.
+    :param **hdf5_options: Additional options forwarded to :class:`blissdata.h5api.dynamic_hdf5.File`.
+    :yields: ``(point_index, data_dict)`` where ``point_index`` is the zero-based
+             scan-point number and ``data_dict`` maps detector names to values.
+    """
+    start_index = 0 if start_index is None else start_index
+
+    for idx, data in enumerate(
+        iter_bliss_scan_data(
+            filename,
+            scan_nr,
+            subscan=subscan,
+            scan_key=scan_key,
+            lima_names=lima_names,
+            counter_names=counter_names,
+            retry_timeout=retry_timeout,
+            retry_period=retry_period,
+            buffer_in_memory=buffer_in_memory,
+            **options,
+        )
+    ):
+        if idx >= start_index:
+            yield idx, data
+
+
+@contextiterator
+def iter_bliss_scan_data(
+    filename: str,
+    scan_nr: Integral,
+    subscan: Optional[Integral] = None,
+    scan_key: Optional[str] = None,
+    lima_names: Optional[List[str]] = None,
+    counter_names: Optional[List[str]] = None,
+    retry_timeout: Optional[float] = None,
+    retry_period: Optional[float] = None,
+    buffer_in_memory: bool = BUFFER_IN_MEMORY_DEFAULT,
+    **hdf5_options: Any,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Yield the data of a Bliss scan either from an HDF5 file or from memory (Redis or device servers).
+
+    :param filename: Path to the HDF5 / Nexus file (ignored when *scan_key* is supplied).
+    :param scan_nr: Scan number inside the file.
+    :param subscan: Sub-scan number (e.g. ``10.2`` -> ``scan_nr=10`` and ``subscan=2``).
+    :param scan_key: Identifier of a scan for blissdata. Iterate scan data from blissdata-cache.
+    :param lima_names: List of LIMA detector names.
+    :param counter_names: List of non-LIMA counter names.
+    :param retry_timeout: Timeout for the retry mechanism.
+    :param retry_period: Interval between retries.
+    :param buffer_in_memory: Blissdata buffering in memory. Only applies when `scan_key` is provided.
+                             Use only when you have enough memory or the data processing can keep up with the scan.
+    :param **hdf5_options: Additional options forwarded to :class:`blissdata.h5api.dynamic_hdf5.File`.
+    :yields: Mapping ``detector_name -> value`` for each scan point.
+    """
+    if scan_key:
+        yield from iter_bliss_scan_data_from_memory(
+            scan_key,
+            lima_names=lima_names,
+            counter_names=counter_names,
+            retry_timeout=retry_timeout,
+            retry_period=retry_period,
+            buffer_in_memory=buffer_in_memory,
+        )
+    else:
+        yield from iter_bliss_scan_data_from_hdf5(
+            filename,
+            scan_nr,
+            lima_names=lima_names,
+            counter_names=counter_names,
+            subscan=subscan,
+            retry_timeout=retry_timeout,
+            retry_period=retry_period,
+            **hdf5_options,
+        )
+
+
+def _is_bliss_file(h5item: Union[h5py.Dataset, h5py.Group]) -> bool:
+    """
+    Return ``True`` if *h5item* belongs to a Bliss-generated file.
+
+    :param h5item: HDF5 dataset or group.
+    :return: ``True`` when the file attributes indicate a Bliss creator/publisher.
+    """
+    attrs = h5item.file.attrs
+    creator = attrs.get("creator", "").lower()
+    publisher = attrs.get("publisher", "").lower()
+    return creator in _BLISS_PUBLISHERS or publisher in _BLISS_PUBLISHERS
+
+
+_BLISS_PUBLISHERS = ("bliss", "blissdata")

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/data/blissdata/__init__.py

@@ -1,6 +1,7 @@
-from packaging.version import Version
 from importlib.metadata import version
 
+from packaging.version import Version
+
 
 def _not_supported_by_blissdata_version(*_, **kw):
     raise RuntimeError(f"Not supported by blissdata {_BLISSDATA_VERSION}")
@@ -14,12 +15,17 @@ if _BLISSDATA_VERSION >= Version("2.0.0rc1"):
     from .blissdatav2 import iter_bliss_scan_data_from_memory  # noqa F401
     from .blissdatav2 import iter_bliss_scan_data_from_memory_slice  # noqa F401
     from .blissdatav2 import last_lima_image  # noqa F401
+
+    BUFFER_IN_MEMORY_DEFAULT = True
 elif _BLISSDATA_VERSION >= Version("1"):
     from .blissdatav1 import iter_bliss_scan_data_from_memory  # noqa F401
     from .blissdatav1 import iter_bliss_scan_data_from_memory_slice  # noqa F401
     from .blissdatav1 import last_lima_image  # noqa F401
+
+    BUFFER_IN_MEMORY_DEFAULT = True
 else:
     from .blissdatav0 import iter_bliss_scan_data_from_memory  # noqa F401
     from .blissdatav0 import last_lima_image  # noqa F401
 
     iter_bliss_scan_data_from_memory_slice = _not_supported_by_blissdata_version
+    BUFFER_IN_MEMORY_DEFAULT = False

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/data/blissdata/blissdatav0.py

@@ -1,28 +1,42 @@
 import logging
 from numbers import Number
-from typing import List, Optional
+from typing import Any
+from typing import Dict
+from typing import Generator
+from typing import List
+from typing import Optional
 
+from blissdata.data.events.lima import ImageNotSaved
+from blissdata.data.node import get_node
 from numpy.typing import ArrayLike
 from silx.utils import retry as retrymod
-from blissdata.data.node import get_node
-from blissdata.data.events.lima import ImageNotSaved
-
 
 logger = logging.getLogger(__name__)
 
 
 def iter_bliss_scan_data_from_memory(
     db_name: str,
-    lima_names: List[str],
-    counter_names: List[str],
+    lima_names: Optional[List[str]] = None,
+    counter_names: Optional[List[str]] = None,
     retry_timeout: Optional[Number] = None,
     retry_period: Optional[Number] = None,
-):
+    buffer_in_memory: bool = True,
+) -> Generator[Dict[str, Any], None, None]:
+    if not buffer_in_memory:
+        raise NotImplementedError("Requires blissdata>=2.0.0rc1")
+    if lima_names is None:
+        lima_names = list()
+    if counter_names is None:
+        counter_names = list()
+
     scan_node = _get_node(
         db_name, "scan", retry_timeout=retry_timeout, retry_period=retry_period
     )
     indices = {name: 0 for name in lima_names + counter_names}
     buffers = {name: list() for name in lima_names + counter_names}
+    if not buffers:
+        return
+
     lima_acq_nb = dict()
     for event_type, node, event_data in scan_node.walk_events():
         if node.type == "lima":

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/data/blissdata/blissdatav1.py

@@ -1,24 +1,27 @@
+import logging
 import sys
 import time
-import logging
-from numbers import Number
 from collections import Counter
-from typing import List, Optional, Tuple
+from numbers import Number
+from typing import Any
+from typing import Dict
+from typing import Generator
+from typing import List
+from typing import Optional
+from typing import Tuple
 
 import numpy
-from numpy.typing import ArrayLike
 from blissdata.beacon.data import BeaconData
-from blissdata.redis_engine.store import DataStore
+from blissdata.lima.client import lima_client_factory
+from blissdata.redis_engine.exceptions import EndOfStream
+from blissdata.redis_engine.exceptions import IndexNoMoreThereError
+from blissdata.redis_engine.exceptions import IndexNotYetThereError
+from blissdata.redis_engine.exceptions import IndexWontBeThereError
 from blissdata.redis_engine.scan import Scan
 from blissdata.redis_engine.scan import ScanState
+from blissdata.redis_engine.store import DataStore
 from blissdata.redis_engine.stream import StreamingClient
-from blissdata.redis_engine.exceptions import (
-    EndOfStream,
-    IndexNoMoreThereError,
-    IndexNotYetThereError,
-    IndexWontBeThereError,
-)
-from blissdata.lima.client import lima_client_factory
+from numpy.typing import ArrayLike
 
 try:
     from blissdata.stream import LimaStream
@@ -38,14 +41,24 @@ def _get_data_store() -> None:
 
 def iter_bliss_scan_data_from_memory(
     scan_key: str,
-    lima_names: List[str],
-    counter_names: List[str],
+    lima_names: Optional[List[str]] = None,
+    counter_names: Optional[List[str]] = None,
     retry_timeout: Optional[Number] = None,
     retry_period: Optional[Number] = None,
-):
+    buffer_in_memory: bool = True,
+) -> Generator[Dict[str, Any], None, None]:
+    if not buffer_in_memory:
+        raise NotImplementedError("Requires blissdata>=2.0.0rc1")
+    if lima_names is None:
+        lima_names = list()
+    if counter_names is None:
+        counter_names = list()
+
     data_store = _get_data_store()
     scan = data_store.load_scan(scan_key, scan_cls=Scan)
     buffers = {name: list() for name in lima_names + counter_names}
+    if not buffers:
+        return
 
     while scan.state < ScanState.PREPARED:
         scan.update()

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/data/blissdata/blissdatav2.py

@@ -1,23 +1,36 @@
+import logging
 import sys
 import time
-import logging
 from collections import Counter
-from typing import List, Optional, Tuple
+from collections.abc import Iterator
+from importlib.metadata import version
+from typing import Any
+from typing import Dict
+from typing import Generator
+from typing import List
+from typing import Optional
+from typing import Tuple
 
 import numpy
-from numpy.typing import ArrayLike
-
 from blissdata.beacon.data import BeaconData
-from blissdata.redis_engine.store import DataStore
 from blissdata.redis_engine.scan import ScanState
-from blissdata.redis_engine.exceptions import (
-    EndOfStream,
-    IndexNoMoreThereError,
-    IndexNotYetThereError,
-    IndexWontBeThereError,
-)
-from blissdata.streams.base import CursorGroup
-
+from blissdata.redis_engine.store import DataStore
+from numpy.typing import ArrayLike
+from packaging.version import Version
+
+_BLISSDATA_VERSION = Version(version("blissdata"))
+if _BLISSDATA_VERSION >= Version("2.3.0"):
+    from blissdata.exceptions import EndOfStream
+    from blissdata.exceptions import IndexNoMoreThereError
+    from blissdata.exceptions import IndexNotYetThereError
+    from blissdata.exceptions import IndexWontBeThereError
+    from blissdata.streams import CursorGroup
+else:
+    from blissdata.redis_engine.exceptions import EndOfStream
+    from blissdata.redis_engine.exceptions import IndexNoMoreThereError
+    from blissdata.redis_engine.exceptions import IndexNotYetThereError
+    from blissdata.redis_engine.exceptions import IndexWontBeThereError
+    from blissdata.streams.base import CursorGroup
 
 logger = logging.getLogger(__name__)
 
@@ -31,51 +44,131 @@ def _get_data_store() -> None:
 
 def iter_bliss_scan_data_from_memory(
     scan_key: str,
-    lima_names: List[str],
-    counter_names: List[str],
+    lima_names: Optional[List[str]] = None,
+    counter_names: Optional[List[str]] = None,
     retry_timeout: Optional[float] = None,
     retry_period: Optional[float] = None,
-):
+    buffer_in_memory: bool = False,
+) -> Generator[Dict[str, Any], None, None]:
+    if lima_names is None:
+        lima_names = list()
+    if counter_names is None:
+        counter_names = list()
+
     streams = _get_streams(scan_key, lima_names, counter_names)
     if not streams:
         return
 
     cursor = CursorGroup(list(streams.values()))
     cursor_timeout = retry_period or 0
-    buffers = {ctr_name: list() for ctr_name in lima_names + counter_names}
-    ctr_names = {stream.name: ctr_name for ctr_name, stream in streams.items()}
+    ctr_names = lima_names + counter_names
+    data_buffers = {ctr_name: _DataBuffer() for ctr_name in ctr_names}
+    stream_to_ctr_names = {
+        stream.name: ctr_name for ctr_name, stream in streams.items()
+    }
 
     while True:
         try:
             views = cursor.read(timeout=cursor_timeout)
         except EndOfStream:
+            # No more data - stop the generator
+            progress = [
+                f"{ctr_name}: {data_buffer.progress}"
+                for ctr_name, data_buffer in data_buffers.items()
+            ]
+            logger.info("Data progress on exit:\n %s", "\n".join(progress))
+            for ctr_name, data_buffer in data_buffers.items():
+                if data_buffer.length_left:
+                    logger.warning(
+                        "Data points for counter %r: %s",
+                        ctr_name,
+                        data_buffer.progress,
+                    )
             break
 
-        views = {ctr_names[stream.name]: view for stream, view in views.items()}
-
-        nyield = min(
-            len(buffer) + len(views.get(ctr_name, ()))
-            for ctr_name, buffer in buffers.items()
-        )
-        view_positions = {ctr_name: 0 for ctr_name in buffers.keys()}
-        for _ in range(nyield):
-            data = {}
-            for ctr_name, buffer in buffers.items():
-                if buffer:
-                    data_point = buffer.pop(0)
-                else:
-                    view = views[ctr_name]
-                    start = view_positions[ctr_name]
-                    data_points = view.get_data(start=start, stop=start + 1)
-                    data_point = data_points[0]
-                    view_positions[ctr_name] += 1
-                data[ctr_name] = data_point
-            yield data
-
-        for ctr_name, view in views.items():
-            start = view_positions[ctr_name]
-            data_points = view.get_data(start=start, stop=None)
-            buffers[ctr_name].extend(data_points)
+        for stream, view in views.items():
+            ctr_name = stream_to_ctr_names[stream.name]
+            data_buffers[ctr_name].append_view(view)
+
+        if buffer_in_memory:
+            for data_buffer in data_buffers.values():
+                data_buffer.buffer_in_memory()
+
+        while True:
+            try:
+                ctr_values = [
+                    next(data_buffer) for data_buffer in data_buffers.values()
+                ]
+            except StopIteration:
+                # One of the buffers ran out of data
+                break
+
+            yield dict(zip(ctr_names, ctr_values))
+
+
+class _DataBuffer(Iterator):
+    """Data iterator with view and data buffering for a single counter."""
+
+    def __init__(self):
+        self._views = []
+        self._view_lengths = []
+        self._first_view_index = 0
+        self._buffer = []
+        self._total_length = 0
+        self._total_yielded = 0
+
+    @property
+    def length_left(self) -> int:
+        return self._total_length - self._total_yielded
+
+    @property
+    def progress(self) -> str:
+        return f"{self._total_yielded}/{self._total_length}"
+
+    def buffer_in_memory(self) -> None:
+        """Buffer and clear all view objects."""
+        if not self._views:
+            return
+        starts = [0] * len(self._views)
+        starts[0] = self._first_view_index
+        for view, start in zip(self._views, starts):
+            self._buffer.extend(view.get_data(start=start, stop=None))
+        self._views.clear()
+        self._view_lengths.clear()
+        self._first_view_index = 0
+
+    def append_view(self, view) -> None:
+        """Append view object when not empty."""
+        view_length = len(view)
+        if view_length:
+            self._views.append(view)
+            self._view_lengths.append(view_length)
+            self._total_length += view_length
+
+    def __iter__(self) -> "_DataBuffer":
+        return self
+
+    def __next__(self):
+        """Return next scan data point for this counter."""
+        if self._buffer:
+            return self._buffer.pop(0)
+
+        if len(self._views) == 0:
+            raise StopIteration
+
+        view = self._views[0]
+        start = self._first_view_index
+
+        self._total_yielded += 1
+        self._first_view_index += 1
+
+        if self._first_view_index == self._view_lengths[0]:
+            self._views.pop(0)
+            self._view_lengths.pop(0)
+            self._first_view_index = 0
+
+        data = view.get_data(start=start, stop=start + 1)
+        return data[0]
 
 
 def last_lima_image(scan_key: str, lima_name: str) -> ArrayLike:

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/data/hdf5/config.py

@@ -1,8 +1,8 @@
 """HDF5 configuration of optimal data storage (IO speed, compression, ...)"""
 
-from typing import Optional
-from numbers import Integral
 from collections.abc import Mapping
+from numbers import Integral
+from typing import Optional
 
 import numpy
 from numpy.typing import DTypeLike
@@ -12,7 +12,9 @@ try:
 except ImportError:
     hdf5plugin = None
 
-from .types import ShapeType, VarShapeType, VarH5pyShapeType
+from .types import ShapeType
+from .types import VarH5pyShapeType
+from .types import VarShapeType
 
 DEFAULT_CHUNK_NBYTES = 1 << 20
 DEFAULT_COMPRESSION_LIMIT_NBYTES = 1 << 20

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/data/hdf5/dataset_writer.py

@@ -1,12 +1,13 @@
 import time
-from typing import Optional, List
+from typing import List
+from typing import Optional
 
 import h5py
 import numpy
 from numpy.typing import ArrayLike
 
-from .types import StrictPositiveIntegral
 from .config import guess_dataset_config
+from .types import StrictPositiveIntegral
 
 
 class _DatasetWriterBase:

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/data/hdf5/types.py

@@ -1,5 +1,7 @@
 from numbers import Integral
-from typing import Optional, Tuple, NewType
+from typing import NewType
+from typing import Optional
+from typing import Tuple
 
 PositiveIntegral = NewType("PositiveIntegral", Integral)  # >= 0
 StrictPositiveIntegral = NewType("StrictPositiveIntegral", Integral)  # > 0

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/data/nexus.py

@@ -1,9 +1,14 @@
 import os
 from contextlib import contextmanager
-from typing import Union, Iterator, Optional, Tuple, Sequence
+from typing import Iterator
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+from typing import Union
 
 import h5py
 from silx.io import h5py_utils
+
 from .url import DataUrl
 from .url import as_dataurl
 

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/data/url.py

@@ -1,5 +1,7 @@
 import sys
-from typing import Tuple, Union
+from typing import Tuple
+from typing import Union
+
 from silx.io.url import DataUrl
 
 

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/data/utils.py

@@ -1,5 +1,6 @@
 from numbers import Number
 from typing import Mapping
+
 import numpy
 
 

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/tests/data/bliss_scans.py

@@ -1,10 +1,11 @@
 import os
 from datetime import datetime
 from pathlib import Path
-from typing import Optional, Tuple
-import numpy
-import h5py
+from typing import Optional
+from typing import Tuple
 
+import h5py
+import numpy
 
 __all__ = ["save_bliss_scan"]
 

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/tests/test_data_bliss.py

@@ -1,7 +1,8 @@
-import numpy
 import h5py
-from fabio.edfimage import edfimage
+import numpy
 import pytest
+from fabio.edfimage import edfimage
+
 from ..data import bliss
 
 
@@ -19,11 +20,17 @@ def test_get_data_hdf5(tmpdir):
     img1 = numpy.random.uniform(((10, 12)))
     with h5py.File(filename, "w") as f:
         f["img"] = img1
+
     img2 = bliss.get_data(f"{filename}::/img")
     numpy.testing.assert_array_equal(img1, img2)
+
     img3 = bliss.get_data(f"{filename}?path=/img")
     numpy.testing.assert_array_equal(img1, img3)
 
+    filename_non_existing = str(tmpdir / "data_non_existing.h5")
+    with pytest.raises(IOError):
+        _ = bliss.get_data(f"{filename_non_existing}?path=/img")
+
 
 @pytest.mark.parametrize("lima_names", [(), ("p3",), ("p3", "p4")])
 @pytest.mark.parametrize("counter_names", [(), ("diode1",), ("diode1", "diode2")])

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/tests/test_dataset_writer.py

@@ -1,13 +1,12 @@
-import time
 import itertools
 import multiprocessing
+import time
 
 import h5py
 import numpy
 import pytest
 from silx.io import h5py_utils
 
-
 from ..data.hdf5 import dataset_writer
 
 

{ewoksdata-1.3.0 → ewoksdata-2.0.0/src/ewoksdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ewoksdata
-Version: 1.3.0
+Version: 2.0.0
 Summary: Ewoks tasks for data access
 Author-email: ESRF <dau-pydev@esrf.fr>
 License: # MIT License
@@ -53,6 +53,7 @@ Provides-Extra: dev
 Requires-Dist: ewoksdata[test]; extra == "dev"
 Requires-Dist: black>=25; extra == "dev"
 Requires-Dist: flake8>=4; extra == "dev"
+Requires-Dist: isort; extra == "dev"
 Provides-Extra: doc
 Requires-Dist: ewoksdata[test]; extra == "doc"
 Requires-Dist: sphinx>=4.5; extra == "doc"

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata.egg-info/requires.txt

@@ -14,6 +14,7 @@ redis-om!=1.0.3b0
 ewoksdata[test]
 black>=25
 flake8>=4
+isort
 
 [doc]
 ewoksdata[test]

ewoksdata-1.3.0/src/ewoksdata/data/bliss.py

@@ -1,148 +0,0 @@
-import sys
-import logging
-from numbers import Integral, Number
-from typing import Iterator, List, Optional, Tuple, Sequence, Union
-
-import numpy
-import h5py
-import hdf5plugin  # noqa F401
-from numpy.typing import ArrayLike
-from silx.io import h5py_utils
-from silx.utils import retry as retrymod
-from silx.io.utils import get_data as silx_get_data
-
-from .blissdata import iter_bliss_scan_data_from_memory  # noqa F401
-from .blissdata import iter_bliss_scan_data_from_memory_slice  # noqa F401
-from .blissdata import last_lima_image  # noqa F401
-from .blissdata import dynamic_hdf5
-
-from .contextiterator import contextiterator
-from . import hdf5
-from . import nexus
-from . import url
-
-
-logger = logging.getLogger(__name__)
-
-
-def get_data(
-    data: Union[str, ArrayLike, Number], **options
-) -> Union[numpy.ndarray, Number]:
-    if isinstance(data, str):
-        data_url = url.as_dataurl(data)
-        filename, h5path, idx = url.h5dataset_url_parse(data_url)
-        if filename.endswith(".h5") or filename.endswith(".nx"):
-            return _get_hdf5_data(filename, h5path, idx=idx, **options)
-        if not data_url.scheme():
-            if sys.platform == "win32":
-                data_url = f"fabio:///{data}"
-            else:
-                data_url = f"fabio://{data}"
-        return silx_get_data(data_url)
-    elif isinstance(data, (Sequence, Number, numpy.ndarray)):
-        return data
-    else:
-        raise TypeError(type(data))
-
-
-def get_image(*args, **kwargs) -> numpy.ndarray:
-    data = get_data(*args, **kwargs)
-    return numpy.atleast_2d(numpy.squeeze(data))
-
-
-@h5py_utils.retry()
-def _get_hdf5_data(filename: str, h5path: str, idx=None, **options) -> numpy.ndarray:
-    with hdf5.h5context(filename, h5path, **options) as dset:
-        if _is_bliss_file(dset):
-            if "end_time" not in nexus.get_nxentry(dset):
-                raise retrymod.RetryError
-        if idx is None:
-            idx = tuple()
-        return dset[idx]
-
-
-@contextiterator
-def iter_bliss_scan_data(
-    filename: str,
-    scan_nr: Integral,
-    lima_names: Optional[List[str]] = None,
-    counter_names: Optional[List[str]] = None,
-    subscan: Optional[Integral] = None,
-    **options,
-) -> Iterator[dict]:
-    """Iterate over the data from one Bliss scan. The counters are assumed to have
-    many data values as scan points.
-
-    :param str filename: the Bliss dataset filename
-    :param Integral filename: the scan number in the dataset
-    :param list lima_names: names of lima detectors
-    :param list counter_names: names of non-lima detectors (you need to provide at least one)
-    :param Integral subscan: subscan number (for example "10.2" has `scan_nr=10` and `subscan=2`)
-    :param Number retry_timeout: timeout when it cannot access the data for `retry_timeout` seconds
-    :param Number retry_period: interval in seconds between data access retries
-    :yields dict: data
-    """
-    if not subscan:
-        subscan = 1
-    if counter_names is None:
-        counter_names = list()
-    with dynamic_hdf5.File(filename, lima_names=lima_names, **options) as root:
-        scan = root[f"{scan_nr}.{subscan}"]
-        # assert _is_bliss_file(scan), "Not a Bliss dataset file"
-        measurement = scan["measurement"]
-        instrument = scan["instrument"]
-        datasets = {name: measurement[name] for name in counter_names}
-        for name in lima_names:
-            datasets[name] = instrument[f"{name}/data"]
-        names = list(datasets.keys())
-        for values in zip(*datasets.values()):
-            yield dict(zip(names, values))
-
-
-@contextiterator
-def iter_bliss_data(
-    filename: str,
-    scan_nr: Integral,
-    lima_names: List[str],
-    counter_names: List[str],
-    subscan: Optional[Integral] = None,
-    start_index: Optional[Integral] = None,
-    **options,
-) -> Iterator[Tuple[int, dict]]:
-    """Iterate over the data from one Bliss scan. The counters are assumed to have
-    many data values as scan points.
-
-    :param str filename: the Bliss dataset filename
-    :param Integral filename: the scan number in the dataset
-    :param list lima_names: names of lima detectors
-    :param list counter_names: names of non-lima detectors (you need to provide at least one)
-    :param Integral subscan: subscan number (for example "10.2" has `scan_nr=10` and `subscan=2`)
-    :param Number retry_timeout: timeout when it cannot access the data for `retry_timeout` seconds
-    :param Number retry_period: interval in seconds between data access retries
-    :param Integral start_index: start iterating from this scan point index
-    :yields tuple: scan index, data
-    """
-    if start_index is None:
-        start_index = 0
-    for index, data in enumerate(
-        iter_bliss_scan_data(
-            filename,
-            scan_nr,
-            lima_names=lima_names,
-            counter_names=counter_names,
-            subscan=subscan,
-            **options,
-        )
-    ):
-        if index >= start_index:
-            yield index, data
-
-
-def _is_bliss_file(h5item: Union[h5py.Dataset, h5py.Group]) -> bool:
-    return (
-        h5item.file.attrs.get("creator", "").lower() in _BLISS_PUBLISHERS
-        or h5item.file.attrs.get("publisher", "").lower() in _BLISS_PUBLISHERS
-    )
-
-
-_BLISS_PUBLISHERS = ("bliss", "blissdata")

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/data/contextiterator.py

@@ -1,6 +1,6 @@
 import functools
-from typing import Iterator
 from collections.abc import Generator
+from typing import Iterator
 
 
 class ContextIterator(Generator):

{ewoksdata-1.3.0 → ewoksdata-2.0.0}/src/ewoksdata/tests/test_data_nexus.py

@@ -4,8 +4,8 @@ import pytest
 from silx.io import h5py_utils
 from silx.io.url import DataUrl
 
-from ..data import url
 from ..data import nexus
+from ..data import url
 
 
 def test_create_nexus_group(tmp_path):