ewoksdata 0.5.0__tar.gz → 0.6.0__tar.gz

{ewoksdata-0.5.0 → ewoksdata-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ewoksdata
-Version: 0.5.0
+Version: 0.6.0
 Summary: Ewoks tasks for data access
 Home-page: https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
 Author: ESRF
@@ -18,9 +18,12 @@ License-File: LICENSE.md
 Requires-Dist: h5py>=2.8
 Requires-Dist: silx>=1.1.0
 Requires-Dist: hdf5plugin
+Requires-Dist: blissdata>=0.3.4
 Requires-Dist: blissdata<1.0.1; python_version < "3.8"
 Requires-Dist: blissdata; python_version >= "3.8"
 Requires-Dist: redis
+Requires-Dist: packaging
+Requires-Dist: importlib_metadata; python_version < "3.8"
 Provides-Extra: online
 Requires-Dist: blissdata[tango]; extra == "online"
 Provides-Extra: test

{ewoksdata-0.5.0 → ewoksdata-0.6.0}/setup.cfg

@@ -26,9 +26,12 @@ install_requires =
 	h5py >=2.8
 	silx >=1.1.0
 	hdf5plugin
+	blissdata >=0.3.4
 	blissdata <1.0.1; python_version < "3.8"
 	blissdata; python_version >= "3.8"
 	redis # blissdata<1
+	packaging
+	importlib_metadata; python_version < "3.8"
 
 [options.packages.find]
 where = src

ewoksdata-0.6.0/src/ewoksdata/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.0"

ewoksdata-0.6.0/src/ewoksdata/data/blissdata/__init__.py

@@ -0,0 +1,30 @@
+import sys
+
+if sys.version_info >= (3, 8):
+    from importlib.metadata import version
+else:
+    from importlib_metadata import version
+from packaging.specifiers import SpecifierSet
+
+
+def _not_supported_by_blissdata_version(*_, **kw):
+    raise RuntimeError(f"Not supported by blissdata {_BLISSDATA_VERSION}")
+
+
+from blissdata.h5api import dynamic_hdf5  # noqa F401
+
+_BLISSDATA_VERSION = version("blissdata")
+
+if _BLISSDATA_VERSION in SpecifierSet("<1", prereleases=True):
+    from .blissdatav0 import iter_bliss_scan_data_from_memory  # noqa F401
+    from .blissdatav0 import last_lima_image  # noqa F401
+
+    iter_bliss_scan_data_from_memory_slice = _not_supported_by_blissdata_version
+elif _BLISSDATA_VERSION in SpecifierSet("<2", prereleases=True):
+    from .blissdatav1 import iter_bliss_scan_data_from_memory  # noqa F401
+    from .blissdatav1 import iter_bliss_scan_data_from_memory_slice  # noqa F401
+    from .blissdatav1 import last_lima_image  # noqa F401
+else:
+    from .blissdatav2 import iter_bliss_scan_data_from_memory  # noqa F401
+    from .blissdatav2 import iter_bliss_scan_data_from_memory_slice  # noqa F401
+    from .blissdatav2 import last_lima_image  # noqa F401

{ewoksdata-0.5.0 → ewoksdata-0.6.0}/src/ewoksdata/data/blissdata/blissdatav0.py

@@ -4,14 +4,8 @@ from typing import List, Optional
 
 from numpy.typing import ArrayLike
 from silx.utils import retry as retrymod
-
-from .exceptions import VersionError
-
-try:
-    from blissdata.data.node import get_node
-    from blissdata.data.events.lima import ImageNotSaved
-except ImportError as e:
-    raise VersionError(str(e)) from e
+from blissdata.data.node import get_node
+from blissdata.data.events.lima import ImageNotSaved
 
 
 logger = logging.getLogger(__name__)

ewoksdata-0.6.0/src/ewoksdata/data/blissdata/blissdatav1.py

@@ -0,0 +1,267 @@
+import sys
+import time
+import logging
+from numbers import Number
+from collections import Counter
+from typing import List, Optional, Tuple
+
+import numpy
+from numpy.typing import ArrayLike
+from blissdata.beacon.data import BeaconData
+from blissdata.redis_engine.store import DataStore
+from blissdata.redis_engine.scan import Scan
+from blissdata.redis_engine.scan import ScanState
+from blissdata.redis_engine.stream import StreamingClient
+from blissdata.redis_engine.exceptions import (
+    EndOfStream,
+    IndexNoMoreThereError,
+    IndexNotYetThereError,
+    IndexWontBeThereError,
+)
+from blissdata.lima.client import lima_client_factory
+
+try:
+    from blissdata.stream import LimaStream
+except ImportError:
+    from blissdata.streams.lima_stream import LimaStream
+
+
+logger = logging.getLogger(__name__)
+
+INFINITY = sys.maxsize
+
+
+def _get_data_store() -> None:
+    redis_url = BeaconData().get_redis_data_db()
+    return DataStore(redis_url)
+
+
+def iter_bliss_scan_data_from_memory(
+    scan_key: str,
+    lima_names: List[str],
+    counter_names: List[str],
+    retry_timeout: Optional[Number] = None,
+    retry_period: Optional[Number] = None,
+):
+    data_store = _get_data_store()
+    scan = data_store.load_scan(scan_key, scan_cls=Scan)
+    buffers = {name: list() for name in lima_names + counter_names}
+
+    while scan.state < ScanState.PREPARED:
+        scan.update()
+
+    lima_streams = dict()
+    lima_clients = dict()
+    counter_streams = dict()
+    for name, stream in scan.streams.items():
+        if stream.encoding["type"] == "json" and "lima" in stream.info["format"]:
+            if name.split(":")[-2] in lima_names:
+                lima_streams[name] = stream
+                lima_clients[name] = lima_client_factory(data_store, stream.info)
+        elif name.split(":")[-1] in counter_names:
+            counter_streams[name] = stream
+
+    client = StreamingClient({**lima_streams, **counter_streams})
+    client_timeout = retry_period or 0
+    lima_buffer_count = Counter()
+
+    while True:
+        try:
+            output = client.read(timeout=client_timeout)
+        except EndOfStream:
+            break
+        for stream, (_, payload) in output.items():
+            name_parts = stream.name.split(":")
+            if stream.name in lima_streams:
+                # payload is a sequence of JSON statuses
+                ctr_name = name_parts[-2]
+                last_status = payload[-1]
+                lima_client = lima_clients[stream.name]
+                lima_client.update(**last_status)
+                n_already_read = lima_buffer_count[ctr_name]
+                try:
+                    data = lima_client[n_already_read:]
+                except IndexNoMoreThereError:
+                    continue
+                buffers[ctr_name].extend(data)
+                lima_buffer_count[ctr_name] += len(data)
+            else:
+                # payload is a sequence of data points (0D, 1D, 2D)
+                ctr_name = name_parts[-1]
+                buffers[ctr_name].extend(payload)
+
+        nyield = min(len(v) for v in buffers.values())
+        if nyield:
+            for i in range(nyield):
+                yield {name: values[i] for name, values in buffers.items()}
+            buffers = {name: values[nyield:] for name, values in buffers.items()}
+
+
+def last_lima_image(channel_info: dict) -> ArrayLike:
+    """Get last lima image from memory"""
+    data_store = _get_data_store()
+    lima_client = lima_client_factory(data_store, channel_info)
+    return lima_client.get_last_live_image().array
+
+
+def _get_streams(
+    scan_key: str,
+    lima_names: List[str],
+    counter_names: List[str],
+):
+    data_store = _get_data_store()
+    scan = data_store.load_scan(scan_key, scan_cls=Scan)
+
+    while scan.state < ScanState.PREPARED:
+        scan.update()
+
+    lima_streams = dict()
+    counter_streams = dict()
+
+    for name, stream in scan.streams.items():
+        if stream.encoding["type"] == "json" and "lima" in stream.info["format"]:
+            if name.split(":")[-2] in lima_names:
+                lima_streams[name.split(":")[-2]] = LimaStream(stream)
+
+        elif name.split(":")[-1] in counter_names:
+            counter_streams[name.split(":")[-1]] = stream
+
+    return lima_streams, counter_streams
+
+
+def iter_bliss_scan_data_from_memory_slice(
+    scan_key: str,
+    lima_names: List[str],
+    counter_names: List[str],
+    slice_range: Optional[Tuple[int, int]] = None,
+    retry_timeout: Optional[Number] = None,
+    retry_period: Optional[Number] = None,
+    yield_timeout: Optional[Number] = None,
+    max_slicing_size: Optional[Number] = None,
+    verbose: Optional[bool] = False,
+):
+    """Iterates over the data from a Bliss scan, slicing the streams associated to a lima detector or a counter between specific indexes of the scan (optional)
+
+    :param str scan_key: key of the Bliss scan (e.g. "esrf:scan:XXXX")
+    :param list lima_names: names of lima detectors
+    :param list counter_names: names of non-lima detectors (you need to provide at least one)
+    :param tuple slice_range: two elements which define the limits of the iteration along the scan. If None, it iterates along the whole scan
+    :param Number retry_timeout: timeout when it cannot access the data for `retry_timeout` seconds
+    :param Number retry_period: interval in seconds between data access retries
+    :param Number yield_timeout: timeout to stop slicing the stream and yield the buffered data
+    :param Number max_slicing_size: maximum size of frames to be sliced out of the stream in one single iteration. If None, it will slice all the available data in the stream
+    :yields dict: data
+    """
+    lima_streams, counter_streams = _get_streams(scan_key, lima_names, counter_names)
+
+    all_streams = {**lima_streams, **counter_streams}
+    if not all_streams:
+        logger.warning("There is no stream to slice")
+        return
+
+    if slice_range is None:
+        slice_range = (0, INFINITY)
+
+    if retry_period is None:
+        retry_period = 1
+
+    if yield_timeout is None:
+        yield_timeout = 0.01
+
+    buffers_count = Counter({counter: slice_range[0] for counter in all_streams.keys()})
+
+    # Read and yield continuously
+    stream_on = True
+
+    incoming_buffers = {stream_name: [] for stream_name in all_streams.keys()}
+    non_yielded_buffers = {stream_name: [] for stream_name in all_streams.keys()}
+
+    restart_buffer = time.perf_counter()
+    while stream_on:
+
+        # While loop will stop unless one single stream is successfully sliced
+        stream_on = False
+
+        for stream_name, stream in all_streams.items():
+            try:
+                # Stop condition for limited slices
+                if (
+                    slice_range[1] is not INFINITY
+                    and buffers_count[stream_name] >= slice_range[1]
+                ):
+                    continue
+
+                # Test first index, (slicing between limits do not fall into Error)
+                _ = stream[buffers_count[stream_name]]
+                if max_slicing_size is None:
+                    stream_data = stream[buffers_count[stream_name] : slice_range[1]]
+                else:
+                    stream_data = stream[
+                        buffers_count[stream_name] : min(
+                            slice_range[1],
+                            buffers_count[stream_name] + max_slicing_size,
+                        )
+                    ]
+                incoming_buffers[stream_name] = stream_data
+                buffers_count[stream_name] += len(stream_data)
+                stream_on = True
+
+            except IndexNotYetThereError:
+                stream_on = True
+            except IndexWontBeThereError:
+                pass
+            except IndexNoMoreThereError:
+                pass
+            except EndOfStream:
+                pass
+            except RuntimeError:
+                pass
+
+            for stream_name in incoming_buffers.keys():
+                if len(incoming_buffers[stream_name]) > 0:
+                    if len(non_yielded_buffers[stream_name]) == 0:
+                        non_yielded_buffers[stream_name] = numpy.array(
+                            incoming_buffers[stream_name]
+                        )
+                    else:
+                        non_yielded_buffers[stream_name] = numpy.concatenate(
+                            (
+                                non_yielded_buffers[stream_name],
+                                incoming_buffers[stream_name],
+                            )
+                        )
+                    incoming_buffers[stream_name] = []
+
+        if not stream_on or ((time.perf_counter() - restart_buffer) > yield_timeout):
+
+            frames_to_yield = min(
+                [len(value) for value in non_yielded_buffers.values()]
+            )
+
+            if frames_to_yield > 0:
+                if verbose:
+                    for stream_name, stream_buffer in non_yielded_buffers.items():
+                        print(
+                            f"After slicing the stream: {stream_name} buffer contains {len(stream_buffer)} items"
+                        )
+
+                # Yield point by point
+                for index in range(frames_to_yield):
+                    yield {
+                        stream_name: stream_buffer[index]
+                        for stream_name, stream_buffer in non_yielded_buffers.items()
+                    }
+
+                # Save the non-yielded points for the next iteration
+                for stream_name in non_yielded_buffers.keys():
+                    non_yielded_buffers[stream_name] = non_yielded_buffers[stream_name][
+                        frames_to_yield:
+                    ]
+
+                if verbose:
+                    for stream_name, stream_buffer in non_yielded_buffers.items():
+                        print(
+                            f"After yielding: {stream_name} buffer contains {len(stream_buffer)} non-yielded items"
+                        )
+
+            restart_buffer = time.perf_counter()

ewoksdata-0.6.0/src/ewoksdata/data/blissdata/blissdatav2.py

@@ -0,0 +1,234 @@
+import sys
+import time
+import logging
+from collections import Counter
+from typing import List, Optional, Tuple
+
+import numpy
+from numpy.typing import ArrayLike
+
+from blissdata.beacon.data import BeaconData
+from blissdata.redis_engine.store import DataStore
+from blissdata.redis_engine.scan import ScanState
+from blissdata.redis_engine.exceptions import (
+    EndOfStream,
+    IndexNoMoreThereError,
+    IndexNotYetThereError,
+    IndexWontBeThereError,
+)
+from blissdata.streams.base import CursorGroup
+
+
+logger = logging.getLogger(__name__)
+
+INFINITY = sys.maxsize
+
+
+def _get_data_store() -> None:
+    redis_url = BeaconData().get_redis_data_db()
+    return DataStore(redis_url)
+
+
+def iter_bliss_scan_data_from_memory(
+    scan_key: str,
+    lima_names: List[str],
+    counter_names: List[str],
+    retry_timeout: Optional[float] = None,
+    retry_period: Optional[float] = None,
+):
+    streams = _get_streams(scan_key, lima_names, counter_names)
+    if not streams:
+        return
+
+    cursor = CursorGroup(list(streams.values()))
+    cursor_timeout = retry_period or 0
+    buffers = {ctr_name: list() for ctr_name in lima_names + counter_names}
+    ctr_names = {stream.name: ctr_name for ctr_name, stream in streams.items()}
+
+    while True:
+        try:
+            views = cursor.read(timeout=cursor_timeout)
+        except EndOfStream:
+            break
+
+        for stream, view in views.items():
+            ctr_name = ctr_names[stream.name]
+            buffers[ctr_name].extend(view.get_data())
+
+        nyield = min(len(v) for v in buffers.values())
+        if nyield:
+            for i in range(nyield):
+                yield {name: values[i] for name, values in buffers.items()}
+            buffers = {name: values[nyield:] for name, values in buffers.items()}
+
+
+def last_lima_image(scan_key: str, lima_name: str) -> ArrayLike:
+    """Get last lima image from memory"""
+    streams = _get_streams(scan_key, [lima_name], [])
+    stream = list(streams.values())[0]
+    return stream.get_last_live_image().array
+
+
+def _get_streams(
+    scan_key: str,
+    lima_names: List[str],
+    counter_names: List[str],
+):
+    data_store = _get_data_store()
+    scan = data_store.load_scan(scan_key)
+
+    while scan.state < ScanState.PREPARED:
+        scan.update()
+
+    streams = dict()
+
+    for name, stream in scan.streams.items():
+        if (
+            stream.event_stream.encoding["type"] == "json"
+            and "lima" in stream.info["format"]
+        ):
+            if name.split(":")[-2] in lima_names:
+                streams[name.split(":")[-2]] = stream
+
+        elif name.split(":")[-1] in counter_names:
+            streams[name.split(":")[-1]] = stream
+
+    nnames = len(lima_names) + len(counter_names)
+    nstreams = len(streams)
+    if nnames != nstreams:
+        logger.warning("asked for %d names but got %s streams", nnames, nstreams)
+        return dict()
+
+    return streams
+
+
+def iter_bliss_scan_data_from_memory_slice(
+    scan_key: str,
+    lima_names: List[str],
+    counter_names: List[str],
+    slice_range: Optional[Tuple[int, int]] = None,
+    retry_timeout: Optional[float] = None,
+    retry_period: Optional[float] = None,
+    yield_timeout: Optional[float] = None,
+    max_slicing_size: Optional[float] = None,
+    verbose: Optional[bool] = False,
+):
+    """Iterates over the data from a Bliss scan, slicing the streams associated to a lima detector or a counter between specific indexes of the scan (optional)
+
+    :param str scan_key: key of the Bliss scan (e.g. "esrf:scan:XXXX")
+    :param list lima_names: names of lima detectors
+    :param list counter_names: names of non-lima detectors (you need to provide at least one)
+    :param tuple slice_range: two elements which define the limits of the iteration along the scan. If None, it iterates along the whole scan
+    :param float retry_timeout: timeout when it cannot access the data for `retry_timeout` seconds
+    :param float retry_period: interval in seconds between data access retries
+    :param float yield_timeout: timeout to stop slicing the stream and yield the buffered data
+    :param float max_slicing_size: maximum size of frames to be sliced out of the stream in one single iteration. If None, it will slice all the available data in the stream
+    :yields dict: data
+    """
+    streams = _get_streams(scan_key, lima_names, counter_names)
+    if not streams:
+        return
+
+    if slice_range is None:
+        slice_range = (0, INFINITY)
+
+    if yield_timeout is None:
+        yield_timeout = 0.01
+
+    buffers_count = Counter({counter: slice_range[0] for counter in streams.keys()})
+
+    # Read and yield continuously
+    stream_on = True
+
+    incoming_buffers = {stream_name: [] for stream_name in streams.keys()}
+    non_yielded_buffers = {stream_name: [] for stream_name in streams.keys()}
+
+    restart_buffer = time.perf_counter()
+    while stream_on:
+
+        # While loop will stop unless one single stream is successfully sliced
+        stream_on = False
+
+        for stream_name, stream in streams.items():
+            try:
+                # Stop condition for limited slices
+                if (
+                    slice_range[1] is not INFINITY
+                    and buffers_count[stream_name] >= slice_range[1]
+                ):
+                    continue
+
+                # Test first index, (slicing between limits do not fall into Error)
+                _ = stream[buffers_count[stream_name]]
+                if max_slicing_size is None:
+                    stream_data = stream[buffers_count[stream_name] : slice_range[1]]
+                else:
+                    stream_data = stream[
+                        buffers_count[stream_name] : min(
+                            slice_range[1],
+                            buffers_count[stream_name] + max_slicing_size,
+                        )
+                    ]
+                incoming_buffers[stream_name] = stream_data
+                buffers_count[stream_name] += len(stream_data)
+                stream_on = True
+
+            except IndexNotYetThereError:
+                stream_on = True
+            except IndexWontBeThereError:
+                pass
+            except IndexNoMoreThereError:
+                pass
+            except EndOfStream:
+                pass
+            except RuntimeError:
+                pass
+
+            for stream_name in incoming_buffers.keys():
+                if len(incoming_buffers[stream_name]) > 0:
+                    if len(non_yielded_buffers[stream_name]) == 0:
+                        non_yielded_buffers[stream_name] = numpy.array(
+                            incoming_buffers[stream_name]
+                        )
+                    else:
+                        non_yielded_buffers[stream_name] = numpy.concatenate(
+                            (
+                                non_yielded_buffers[stream_name],
+                                incoming_buffers[stream_name],
+                            )
+                        )
+                    incoming_buffers[stream_name] = []
+
+        if not stream_on or ((time.perf_counter() - restart_buffer) > yield_timeout):
+
+            frames_to_yield = min(
+                [len(value) for value in non_yielded_buffers.values()]
+            )
+
+            if frames_to_yield > 0:
+                if verbose:
+                    for stream_name, stream_buffer in non_yielded_buffers.items():
+                        print(
+                            f"After slicing the stream: {stream_name} buffer contains {len(stream_buffer)} items"
+                        )
+
+                # Yield point by point
+                for index in range(frames_to_yield):
+                    yield {
+                        stream_name: stream_buffer[index]
+                        for stream_name, stream_buffer in non_yielded_buffers.items()
+                    }
+
+                # Save the non-yielded points for the next iteration
+                for stream_name in non_yielded_buffers.keys():
+                    non_yielded_buffers[stream_name] = non_yielded_buffers[stream_name][
+                        frames_to_yield:
+                    ]
+
+                if verbose:
+                    for stream_name, stream_buffer in non_yielded_buffers.items():
+                        print(
+                            f"After yielding: {stream_name} buffer contains {len(stream_buffer)} non-yielded items"
+                        )
+
+            restart_buffer = time.perf_counter()

ewoksdata-0.6.0/src/ewoksdata/data/contextiterator.py

@@ -0,0 +1,43 @@
+import functools
+from typing import Iterator
+from collections.abc import Generator
+
+
+class ContextIterator(Generator):
+    """Can wrap an iterator or generator to ensure cleanup when used as a context manager."""
+
+    def __init__(self, it: Iterator):
+        self._it = it
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return next(self._it)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_):
+        if hasattr(self._it, "close"):
+            return self._it.close()
+
+    def send(self, value):
+        if hasattr(self._it, "send"):
+            return self._it.send(value)
+
+    def throw(self, *args):
+        if hasattr(self._it, "throw"):
+            return self._it.throw(*args)
+
+
+def contextiterator(iterator):
+    """Decorator that allows to use an iterator or generator as a context manager.
+    This ensures that the iterator is closed when exiting the context manager.
+    """
+
+    @functools.wraps(iterator)
+    def wrapper(*args, **kw):
+        return ContextIterator(iterator(*args, **kw))
+
+    return wrapper

{ewoksdata-0.5.0 → ewoksdata-0.6.0}/src/ewoksdata/data/hdf5/dataset_writer.py

@@ -57,17 +57,21 @@ class _DatasetWriterBase:
 class DatasetWriter(_DatasetWriterBase):
     """Append arrays of the same shape to a new HDF5 dataset in a sequential manner.
 
+    Instead of creating a dataset with the :code:`h5py` API
+
     .. code-block::python
 
-        with DatasetWriter(parent, "mydataset") as writer:
-            writer.add_point([1,2,3])
-            writer.add_points([[4,5,6], [7,8,9]])
+        h5group["mydataset"] = [[1,2,3], [4,5,6], [7,8,9]]
 
-    This is equivalent to
+    it can be done like this
 
     .. code-block::python
 
-        parent["mydataset"] = [[1,2,3], [4,5,6], [7,8,9]]
+        with DatasetWriter(h5group, "mydataset") as writer:
+            writer.add_point([1,2,3])
+            writer.add_points([[4,5,6], [7,8,9]])
+
+    Chunk size determination, chunk-aligned writing, compression and flushing is handled.
     """
 
     def __init__(
@@ -168,21 +172,25 @@ class StackDatasetWriter(_DatasetWriterBase):
     in a sequential manner per item. So each item of the HDF5 dataset is a
     stack to which we can append data in a sequential manner.
 
+    Instead of creating a dataset with the :code:`h5py` API
+
     .. code-block::python
 
-        with DatasetWriter(parent, "mydataset") as writer:
-            writer.add_point([1,2,3], 0)
-            writer.add_point([10,11,12], 1)
-            writer.add_points([[4,5,6], [7,8,9]], 0)
-            writer.add_points([[13,14,15], [16,17,18]], 1)
+        stack0 = [[1,2,3], [4,5,6], [7,8,9]]
+        stack1 = [[10,11,12], [13,14,15], [16,17,18]]
+        h5group["mydataset"] = [stack0, stack1]
 
-    This is equivalent to
+    it can be done like this
 
     .. code-block::python
 
-        stack0 = [[1,2,3], [4,5,6], [7,8,9]]
-        stack1 = [[10,11,12], [13,14,15], [16,17,18]]
-        parent["mydataset"] = [stack0, stack1]
+        with StackDatasetWriter(h5group, "mydataset") as writer:
+            writer.add_point([1,2,3], 0)
+            writer.add_point([10,11,12], 1)
+            writer.add_points([[13,14,15], [16,17,18]], 1)
+            writer.add_points([[4,5,6], [7,8,9]], 0)
+
+    Chunk size determination, chunk-aligned writing, compression and flushing is handled.
     """
 
     def __init__(

ewoksdata-0.6.0/src/ewoksdata/tests/test_contextiterator.py

@@ -0,0 +1,61 @@
+from ..data import contextiterator
+
+
+def test_context_iterator():
+
+    exited = False
+
+    @contextiterator.contextiterator
+    def iterator():
+        nonlocal exited
+        exited = False
+        try:
+            yield from [1, 2, 3]
+        finally:
+            exited = True
+
+    # Use like a normal iterator
+    it = iterator()
+    assert next(it) == 1
+    assert not exited
+
+    # Use like a context iterator
+    with iterator() as it:
+        assert next(it) == 1
+        assert not exited
+    assert exited
+
+
+def test_context_generator():
+
+    @contextiterator.contextiterator
+    def generator():
+        nonlocal exited
+        exited = False
+        try:
+            while True:
+                x = yield
+                yield x * 2
+        finally:
+            exited = True
+
+    # Use like a normal generator
+    exited = False
+    gen = generator()
+    assert next(gen) is None  # run up to first yield
+    assert not exited
+    assert gen.send(10) == 20
+    assert next(gen) is None
+    assert gen.send(15) == 30
+    assert not exited
+
+    # Use like a context iterator
+    exited = False
+    with generator() as gen:
+        assert next(gen) is None
+        assert not exited
+        assert gen.send(10) == 20
+        assert next(gen) is None
+        assert gen.send(15) == 30
+        assert not exited
+    assert exited

{ewoksdata-0.5.0 → ewoksdata-0.6.0}/src/ewoksdata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ewoksdata
-Version: 0.5.0
+Version: 0.6.0
 Summary: Ewoks tasks for data access
 Home-page: https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
 Author: ESRF
@@ -18,9 +18,12 @@ License-File: LICENSE.md
 Requires-Dist: h5py>=2.8
 Requires-Dist: silx>=1.1.0
 Requires-Dist: hdf5plugin
+Requires-Dist: blissdata>=0.3.4
 Requires-Dist: blissdata<1.0.1; python_version < "3.8"
 Requires-Dist: blissdata; python_version >= "3.8"
 Requires-Dist: redis
+Requires-Dist: packaging
+Requires-Dist: importlib_metadata; python_version < "3.8"
 Provides-Extra: online
 Requires-Dist: blissdata[tango]; extra == "online"
 Provides-Extra: test

{ewoksdata-0.5.0 → ewoksdata-0.6.0}/src/ewoksdata.egg-info/SOURCES.txt

@@ -18,14 +18,14 @@ src/ewoksdata/data/utils.py
 src/ewoksdata/data/blissdata/__init__.py
 src/ewoksdata/data/blissdata/blissdatav0.py
 src/ewoksdata/data/blissdata/blissdatav1.py
-src/ewoksdata/data/blissdata/blissdatavid31.py
-src/ewoksdata/data/blissdata/exceptions.py
+src/ewoksdata/data/blissdata/blissdatav2.py
 src/ewoksdata/data/hdf5/__init__.py
 src/ewoksdata/data/hdf5/config.py
 src/ewoksdata/data/hdf5/dataset_writer.py
 src/ewoksdata/data/hdf5/types.py
 src/ewoksdata/tests/__init__.py
 src/ewoksdata/tests/conftest.py
+src/ewoksdata/tests/test_contextiterator.py
 src/ewoksdata/tests/test_data_bliss.py
 src/ewoksdata/tests/test_data_hdf5.py
 src/ewoksdata/tests/test_data_nexus.py

{ewoksdata-0.5.0 → ewoksdata-0.6.0}/src/ewoksdata.egg-info/requires.txt

@@ -1,10 +1,13 @@
 h5py>=2.8
 silx>=1.1.0
 hdf5plugin
+blissdata>=0.3.4
 redis
+packaging
 
 [:python_version < "3.8"]
 blissdata<1.0.1
+importlib_metadata
 
 [:python_version >= "3.8"]
 blissdata

ewoksdata-0.5.0/src/ewoksdata/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.5.0"

ewoksdata-0.5.0/src/ewoksdata/data/blissdata/__init__.py

@@ -1,36 +0,0 @@
-from .exceptions import VersionError
-
-try:
-    # blissdata >=1
-    from .blissdatav1 import iter_bliss_scan_data_from_memory
-    from .blissdatav1 import last_lima_image
-    from blissdata.h5api import dynamic_hdf5
-except VersionError as exc:
-    try:
-        # blissdata >0.3.3,<1 (unreleased, branch id31_2.0)
-        from .blissdatavid31 import iter_bliss_scan_data_from_memory
-        from .blissdatavid31 import last_lima_image
-        from blissdata.h5api import dynamic_hdf5
-    except VersionError:
-        try:
-            # blissdata <=0.3.3
-            from .blissdatav0 import iter_bliss_scan_data_from_memory
-            from .blissdatav0 import last_lima_image
-            from blissdata.h5api import dynamic_hdf5
-        except VersionError:  # noqa F841
-            _EXC = exc
-
-            def _reraise(*_, **kw):
-                raise _EXC
-
-            class BlissDataVersionNotSupported:
-                def __getattr__(self, *_):
-                    return _reraise
-
-            dynamic_hdf5 = BlissDataVersionNotSupported()
-
-            def iter_bliss_scan_data_from_memory(*_, **kw):
-                raise _EXC
-
-            def last_lima_image(*_, **kw):
-                raise _EXC

ewoksdata-0.5.0/src/ewoksdata/data/blissdata/blissdatav1.py

@@ -1,91 +0,0 @@
-from numbers import Number
-from collections import Counter
-from typing import List, Optional
-
-from numpy.typing import ArrayLike
-
-from .exceptions import VersionError
-
-try:
-    from blissdata.beacon.data import BeaconData
-    from blissdata.redis_engine.store import DataStore
-    from blissdata.redis_engine.scan import Scan
-    from blissdata.redis_engine.scan import ScanState
-    from blissdata.redis_engine.stream import StreamingClient
-    from blissdata.redis_engine.exceptions import EndOfStream
-    from blissdata.redis_engine.exceptions import IndexNoMoreThereError
-    from blissdata.lima.client import lima_client_factory
-except ImportError as e:
-    raise VersionError(str(e)) from e
-
-
-def iter_bliss_scan_data_from_memory(
-    db_name: str,
-    lima_names: List[str],
-    counter_names: List[str],
-    retry_timeout: Optional[Number] = None,
-    retry_period: Optional[Number] = None,
-):
-    data_store = _get_data_store()
-    scan = data_store.load_scan(db_name, scan_cls=Scan)
-    buffers = {name: list() for name in lima_names + counter_names}
-
-    while scan.state < ScanState.PREPARED:
-        scan.update()
-
-    lima_streams = dict()
-    lima_clients = dict()
-    counter_streams = dict()
-    for name, stream in scan.streams.items():
-        if stream.encoding["type"] == "json" and "lima" in stream.info["format"]:
-            if name.split(":")[-2] in lima_names:
-                lima_streams[name] = stream
-                lima_clients[name] = lima_client_factory(data_store, stream.info)
-        elif name.split(":")[-1] in counter_names:
-            counter_streams[name] = stream
-
-    client = StreamingClient({**lima_streams, **counter_streams})
-    lima_buffer_count = Counter()
-
-    while True:
-        try:
-            output = client.read()
-        except EndOfStream:
-            break
-        for stream, (_, payload) in output.items():
-            name_parts = stream.name.split(":")
-            if stream.name in lima_streams:
-                # payload is a sequence of JSON statuses
-                ctr_name = name_parts[-2]
-                last_status = payload[-1]
-                lima_client = lima_clients[stream.name]
-                lima_client.update(**last_status)
-                n_already_read = lima_buffer_count[ctr_name]
-                try:
-                    data = lima_client[n_already_read:]
-                except IndexNoMoreThereError:
-                    continue
-                buffers[ctr_name].extend(data)
-                lima_buffer_count[ctr_name] += len(data)
-            else:
-                # payload is a sequence of data points (0D, 1D, 2D)
-                ctr_name = name_parts[-1]
-                buffers[ctr_name].extend(payload)
-
-        nyield = min(len(v) for v in buffers.values())
-        if nyield:
-            for i in range(nyield):
-                yield {name: values[i] for name, values in buffers.items()}
-            buffers = {name: values[nyield:] for name, values in buffers.items()}
-
-
-def last_lima_image(channel_info: dict) -> ArrayLike:
-    """Get last lima image from memory"""
-    data_store = _get_data_store()
-    lima_client = lima_client_factory(data_store, channel_info)
-    return lima_client.get_last_live_image().array
-
-
-def _get_data_store() -> None:
-    redis_url = BeaconData().get_redis_data_db()
-    return DataStore(redis_url)

ewoksdata-0.5.0/src/ewoksdata/data/blissdata/blissdatavid31.py

@@ -1,105 +0,0 @@
-from numbers import Number
-from collections import Counter
-from typing import List, Optional
-
-from numpy.typing import ArrayLike
-
-from .exceptions import VersionError
-
-try:
-    from blissdata.redis_engine.scan import Scan
-    from blissdata.redis_engine.stream import StreamingClient
-    from blissdata.redis_engine.models import ScanState
-    from blissdata.redis_engine.exceptions import EndOfStream
-    from blissdata.lima.client import lima_client_factory
-    from blissdata.beacon.data import BeaconData
-    from blissdata import redis_engine
-except ImportError as e:
-    raise VersionError(str(e)) from e
-
-
-def iter_bliss_scan_data_from_memory(
-    db_name: str,
-    lima_names: List[str],
-    counter_names: List[str],
-    retry_timeout: Optional[Number] = None,
-    retry_period: Optional[Number] = None,
-):
-    _ensure_redis()
-    scan = Scan.load(db_name)
-    buffers = {name: list() for name in lima_names + counter_names}
-
-    while scan.state < ScanState.PREPARED:
-        scan.update()
-    if scan.state is ScanState.ABORTED:
-        return
-
-    lima_streams = dict()
-    lima_clients = dict()
-    lima_buffer_count = dict()
-    counter_streams = dict()
-    for name, stream in scan.streams.items():
-        if stream.encoding["type"] == "json" and "lima" in stream.info["format"]:
-            if name.split(":")[-2] in lima_names:
-                lima_streams[name] = stream
-                lima_clients[name] = lima_client_factory(stream.info)
-                lima_buffer_count[name] = 0
-        elif name.split(":")[-1] in counter_names:
-            counter_streams[name] = stream
-
-    client = StreamingClient({**lima_streams, **counter_streams})
-    lima_buffer_count = Counter()
-
-    while True:
-        try:
-            output = client.read()
-        except EndOfStream:
-            break
-        for stream, (_, payload) in output.items():
-            name_parts = stream.name.split(":")
-            if stream.name in lima_streams:
-                # payload is a sequence of JSON statuses
-                ctr_name = name_parts[-2]
-                lima_client = lima_clients[stream.name]
-                lima_client.update(**payload[-1])
-                n_already_read = lima_buffer_count[ctr_name]
-                data = lima_client[n_already_read:]
-                buffers[ctr_name].extend(data)
-                lima_buffer_count[ctr_name] += len(data)
-            else:
-                # payload is a sequence of data points (0D, 1D, 2D)
-                ctr_name = name_parts[-1]
-                buffers[ctr_name].extend(payload)
-
-        nyield = min(len(v) for v in buffers.values())
-        if nyield:
-            for i in range(nyield):
-                yield {name: values[i] for name, values in buffers.items()}
-            buffers = {name: values[nyield:] for name, values in buffers.items()}
-
-
-def last_lima_image(channel_info: dict) -> ArrayLike:
-    """Get last lima image from memory"""
-    _ensure_redis()
-    lima_client = lima_client_factory(channel_info)
-    return lima_client.get_last_live_image().array
-
-
-def _ensure_redis() -> None:
-    url = BeaconData().get_redis_data_db()
-    current_url = _current_redis_url()
-    if current_url:
-        if url == current_url:
-            return
-        raise RuntimeError("The Redis URL has changed. Restart the Nexus writer.")
-    redis_engine.set_redis_url(url)
-
-
-def _current_redis_url() -> Optional[str]:
-    if redis_engine._redis is None:
-        return
-    kwargs = redis_engine._redis.connection_pool.connection_kwargs
-    if "path" in kwargs:
-        return f"unix://{kwargs['path']}"
-    else:
-        return f"redis://{kwargs['host']}:{kwargs['port']}"

ewoksdata-0.5.0/src/ewoksdata/data/blissdata/exceptions.py

@@ -1,4 +0,0 @@
-class VersionError(ImportError):
-    """Import attempt failed from a library for which we support multiple versions with different import API's."""
-
-    pass

ewoksdata-0.5.0/src/ewoksdata/data/contextiterator.py

@@ -1,35 +0,0 @@
-from collections.abc import Generator
-
-
-class ContextIterator(Generator):
-    def __init__(self, it):
-        self._it = it
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        return next(self._it)
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *_):
-        self._it.close()
-
-    def send(self):
-        return next(self._it)
-
-    def throw(self):
-        raise StopIteration
-
-
-def contextiterator(iterator):
-    """Decorator that allows to use an iterator as a context manager.
-    This ensures that the iterator is closed when exiting the context manager.
-    """
-
-    def wrapper(*args, **kw):
-        return ContextIterator(iterator(*args, **kw))
-
-    return wrapper