ewoksdata 0.3.0__tar.gz → 0.4.1__tar.gz

{ewoksdata-0.3.0 → ewoksdata-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ewoksdata
-Version: 0.3.0
+Version: 0.4.1
 Summary: Ewoks tasks for data access
 Home-page: https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
 Author: ESRF
@@ -14,11 +14,28 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: h5py>=2.8
+Requires-Dist: silx>=1.1.0
+Requires-Dist: hdf5plugin
+Requires-Dist: blissdata<1.0.1; python_version < "3.8"
+Requires-Dist: blissdata; python_version >= "3.8"
+Requires-Dist: redis
 Provides-Extra: online
+Requires-Dist: blissdata[tango]; extra == "online"
 Provides-Extra: test
+Requires-Dist: blissdata[tango]; extra == "test"
+Requires-Dist: pytest>=7; extra == "test"
 Provides-Extra: dev
+Requires-Dist: blissdata[tango]; extra == "dev"
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: black>=22; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
 Provides-Extra: doc
-License-File: LICENSE.md
+Requires-Dist: blissdata[tango]; extra == "doc"
+Requires-Dist: pytest>=7; extra == "doc"
+Requires-Dist: sphinx>=4.5; extra == "doc"
+Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
 
 # ewoksdata
 

{ewoksdata-0.3.0 → ewoksdata-0.4.1}/setup.cfg

@@ -26,7 +26,9 @@ install_requires =
 	h5py >=2.8
 	silx >=1.1.0
 	hdf5plugin
-	blissdata
+	blissdata <1.0.1; python_version < "3.8"
+	blissdata; python_version >= "3.8"
+	redis # blissdata<1
 
 [options.packages.find]
 where = src

ewoksdata-0.4.1/src/ewoksdata/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.1"

{ewoksdata-0.3.0 → ewoksdata-0.4.1}/src/ewoksdata/data/bliss.py

@@ -10,14 +10,10 @@ from numpy.typing import ArrayLike
 from silx.io import h5py_utils
 from silx.utils import retry as retrymod
 from silx.io.utils import get_data as silx_get_data
-from blissdata.h5api import dynamic_hdf5
 
-try:
-    import gevent.queue  # noqa F401 bliss MR 5369
-    from blissdata.data.node import get_node
-    from blissdata.data.events.lima import ImageNotSaved
-except ImportError:
-    get_node = None
+from .blissdata import iter_bliss_scan_data_from_memory  # noqa F401
+from .blissdata import last_lima_image  # noqa F401
+from .blissdata import dynamic_hdf5
 
 from . import hdf5
 from . import nexus
@@ -140,100 +136,3 @@ def iter_bliss_data(
 
 def _is_bliss_file(h5item: Union[h5py.Dataset, h5py.Group]) -> bool:
     return h5item.file.attrs.get("creator", "").lower() == "bliss"
-
-
-def last_lima_image(db_name: str) -> ArrayLike:
-    """Get last lima image from memory"""
-    node = _get_node(db_name, "lima")
-    node.from_stream = True
-    dataview = node.get(-1)
-    try:
-        image = dataview.get_last_live_image()
-    except AttributeError:
-        image = None
-    if image is None or image.data is None:
-        raise RuntimeError("Cannot get last image from lima")
-    return image.data
-
-
-def iter_bliss_scan_data_from_memory(
-    db_name: str,
-    lima_names: List[str],
-    counter_names: List[str],
-    retry_timeout: Optional[Number] = None,
-    retry_period: Optional[Number] = None,
-):
-    scan_node = _get_node(
-        db_name, "scan", retry_timeout=retry_timeout, retry_period=retry_period
-    )
-    indices = {name: 0 for name in lima_names + counter_names}
-    buffers = {name: list() for name in lima_names + counter_names}
-    lima_acq_nb = dict()
-    for event_type, node, event_data in scan_node.walk_events():
-        if node.type == "lima":
-            name = node.db_name.split(":")[-2]
-            if name not in lima_names:
-                continue
-            dataview = _get_lima_dataview(
-                node,
-                indices[name],
-                retry_timeout=retry_timeout,
-                retry_period=retry_period,
-            )
-            current_lima_acq_nb = dataview.status_event.status["lima_acq_nb"]
-            first_lima_acq_nb = lima_acq_nb.setdefault(name, current_lima_acq_nb)
-            if first_lima_acq_nb != current_lima_acq_nb:
-                logger.warning("lima is already acquiring the next scan")
-                continue
-            try:
-                data = list(dataview)
-            except ImageNotSaved:
-                logger.warning(
-                    "cannot read lima data from file because images are not being saved"
-                )
-                continue
-            except Exception as e:
-                logger.warning("cannot read lima data (%s)", str(e))
-                continue
-            indices[name] += len(data)
-            buffers[name].extend(data)
-        elif node.type == "channel":
-            name = node.db_name.split(":")[-1]
-            if name not in counter_names:
-                continue
-            if event_data:
-                data = event_data.data
-            else:
-                data = node.get_as_array(indices[name], -1)
-            indices[name] += len(data)
-            buffers[name].extend(data)
-        nyield = min(len(v) for v in buffers.values())
-        if nyield:
-            for i in range(nyield):
-                yield {name: values[i] for name, values in buffers.items()}
-            buffers = {name: values[nyield:] for name, values in buffers.items()}
-        if event_type == event_type.END_SCAN:
-            break
-
-
-@retrymod.retry()
-def _get_node(db_name: str, node_type: str):
-    if get_node is None:
-        raise ModuleNotFoundError("No module named 'blissdata'")
-    node = get_node(db_name)
-    if node is None:
-        raise retrymod.RetryError(f"Redis node {db_name} does not exist")
-    if node.type != node_type:
-        raise RuntimeError(f"Not a Redis {node_type} node")
-    return node
-
-
-@retrymod.retry()
-def _get_lima_dataview(node, start_index: int):
-    dataview = node.get(start_index, -1)
-    try:
-        if dataview.status_event.proxy is None:
-            raise retrymod.RetryError("Lima proxy not known (yet)")
-    except Exception:
-        raise retrymod.RetryError("Lima proxy not known (yet)")
-    return dataview

ewoksdata-0.4.1/src/ewoksdata/data/blissdata/__init__.py

@@ -0,0 +1,36 @@
+from .exceptions import VersionError
+
+try:
+    # blissdata >=1
+    from .blissdatav1 import iter_bliss_scan_data_from_memory
+    from .blissdatav1 import last_lima_image
+    from blissdata.h5api import dynamic_hdf5
+except VersionError as exc:
+    try:
+        # blissdata >0.3.3,<1 (unreleased, branch id31_2.0)
+        from .blissdatavid31 import iter_bliss_scan_data_from_memory
+        from .blissdatavid31 import last_lima_image
+        from blissdata.h5api import dynamic_hdf5
+    except VersionError:
+        try:
+            # blissdata <=0.3.3
+            from .blissdatav0 import iter_bliss_scan_data_from_memory
+            from .blissdatav0 import last_lima_image
+            from blissdata.h5api import dynamic_hdf5
+        except VersionError:  # noqa F841
+            _EXC = exc
+
+            def _reraise(*_, **kw):
+                raise _EXC
+
+            class BlissDataVersionNotSupported:
+                def __getattr__(self, *_):
+                    return _reraise
+
+            dynamic_hdf5 = BlissDataVersionNotSupported()
+
+            def iter_bliss_scan_data_from_memory(*_, **kw):
+                raise _EXC
+
+            def last_lima_image(*_, **kw):
+                raise _EXC

ewoksdata-0.4.1/src/ewoksdata/data/blissdata/blissdatav0.py

@@ -0,0 +1,112 @@
+import logging
+from numbers import Number
+from typing import List, Optional
+
+from numpy.typing import ArrayLike
+from silx.utils import retry as retrymod
+
+from .exceptions import VersionError
+
+try:
+    from blissdata.data.node import get_node
+    from blissdata.data.events.lima import ImageNotSaved
+except ImportError as e:
+    raise VersionError(str(e)) from e
+
+
+logger = logging.getLogger(__name__)
+
+
+def iter_bliss_scan_data_from_memory(
+    db_name: str,
+    lima_names: List[str],
+    counter_names: List[str],
+    retry_timeout: Optional[Number] = None,
+    retry_period: Optional[Number] = None,
+):
+    scan_node = _get_node(
+        db_name, "scan", retry_timeout=retry_timeout, retry_period=retry_period
+    )
+    indices = {name: 0 for name in lima_names + counter_names}
+    buffers = {name: list() for name in lima_names + counter_names}
+    lima_acq_nb = dict()
+    for event_type, node, event_data in scan_node.walk_events():
+        if node.type == "lima":
+            name = node.db_name.split(":")[-2]
+            if name not in lima_names:
+                continue
+            dataview = _get_lima_dataview(
+                node,
+                indices[name],
+                retry_timeout=retry_timeout,
+                retry_period=retry_period,
+            )
+            current_lima_acq_nb = dataview.status_event.status["lima_acq_nb"]
+            first_lima_acq_nb = lima_acq_nb.setdefault(name, current_lima_acq_nb)
+            if first_lima_acq_nb != current_lima_acq_nb:
+                logger.warning("lima is already acquiring the next scan")
+                continue
+            try:
+                data = list(dataview)
+            except ImageNotSaved:
+                logger.warning(
+                    "cannot read lima data from file because images are not being saved"
+                )
+                continue
+            except Exception as e:
+                logger.warning("cannot read lima data (%s)", str(e))
+                continue
+            indices[name] += len(data)
+            buffers[name].extend(data)
+        elif node.type == "channel":
+            name = node.db_name.split(":")[-1]
+            if name not in counter_names:
+                continue
+            if event_data:
+                data = event_data.data
+            else:
+                data = node.get_as_array(indices[name], -1)
+            indices[name] += len(data)
+            buffers[name].extend(data)
+        nyield = min(len(v) for v in buffers.values())
+        if nyield:
+            for i in range(nyield):
+                yield {name: values[i] for name, values in buffers.items()}
+            buffers = {name: values[nyield:] for name, values in buffers.items()}
+        if event_type == event_type.END_SCAN:
+            break
+
+
+def last_lima_image(db_name: str) -> ArrayLike:
+    """Get last lima image from memory"""
+    node = _get_node(db_name, "lima")
+    node.from_stream = True
+    dataview = node.get(-1)
+    try:
+        image = dataview.get_last_live_image()
+    except AttributeError:
+        image = None
+    if image is None or image.data is None:
+        raise RuntimeError("Cannot get last image from lima")
+    return image.data
+
+
+@retrymod.retry()
+def _get_node(db_name: str, node_type: str):
+    node = get_node(db_name)
+    if node is None:
+        raise retrymod.RetryError(f"Redis node {db_name} does not exist")
+    if node.type != node_type:
+        raise RuntimeError(f"Not a Redis {node_type} node")
+    return node
+
+
+@retrymod.retry()
+def _get_lima_dataview(node, start_index: int):
+    dataview = node.get(start_index, -1)
+    try:
+        if dataview.status_event.proxy is None:
+            raise retrymod.RetryError("Lima proxy not known (yet)")
+    except Exception:
+        raise retrymod.RetryError("Lima proxy not known (yet)")
+    return dataview

ewoksdata-0.4.1/src/ewoksdata/data/blissdata/blissdatav1.py

@@ -0,0 +1,84 @@
+from numbers import Number
+from typing import List, Optional
+
+from numpy.typing import ArrayLike
+
+from .exceptions import VersionError
+
+try:
+    from blissdata.beacon.data import BeaconData
+    from blissdata.redis_engine.store import DataStore
+    from blissdata.redis_engine.scan import Scan
+    from blissdata.redis_engine.scan import ScanState
+    from blissdata.redis_engine.stream import StreamingClient
+    from blissdata.redis_engine.exceptions import EndOfStream
+    from blissdata.redis_engine.exceptions import IndexNoMoreThereError
+    from blissdata.lima.client import lima_client_factory
+except ImportError as e:
+    raise VersionError(str(e)) from e
+
+
+def iter_bliss_scan_data_from_memory(
+    db_name: str,
+    lima_names: List[str],
+    counter_names: List[str],
+    retry_timeout: Optional[Number] = None,
+    retry_period: Optional[Number] = None,
+):
+    data_store = _get_data_store()
+    scan = data_store.load_scan(db_name, scan_cls=Scan)
+    buffers = {name: list() for name in lima_names + counter_names}
+
+    while scan.state < ScanState.PREPARED:
+        scan.update()
+
+    lima_streams = dict()
+    lima_clients = dict()
+    counter_streams = dict()
+    for name, stream in scan.streams.items():
+        if stream.encoding["type"] == "json" and "lima" in stream.info["format"]:
+            if name.split(":")[-2] in lima_names:
+                lima_streams[name] = stream
+                lima_clients[name] = lima_client_factory(data_store, stream.info)
+        elif name.split(":")[-1] in counter_names:
+            counter_streams[name] = stream
+
+    client = StreamingClient({**lima_streams, **counter_streams})
+
+    while True:
+        try:
+            output = client.read()
+        except EndOfStream:
+            break
+        else:
+            for stream, data in output.items():
+                _, payload = data  # drop index
+                if stream.name in lima_streams:
+                    lima_client = lima_clients[stream.name]
+                    prev_len = len(lima_client)
+                    lima_client.update(**payload[-1])
+                    try:
+                        data = lima_client[prev_len - 1 :]
+                    except IndexNoMoreThereError:
+                        continue
+                    buffers[stream.name.split(":")[-2]].extend(data)
+                else:
+                    buffers[stream.name.split(":")[-1]].extend(payload)
+
+            nyield = min(len(v) for v in buffers.values())
+            if nyield:
+                for i in range(nyield):
+                    yield {name: values[i] for name, values in buffers.items()}
+                buffers = {name: values[nyield:] for name, values in buffers.items()}
+
+
+def last_lima_image(channel_info: dict) -> ArrayLike:
+    """Get last lima image from memory"""
+    data_store = _get_data_store()
+    lima_client = lima_client_factory(data_store, channel_info)
+    return lima_client.get_last_live_image().array
+
+
+def _get_data_store() -> None:
+    redis_url = BeaconData().get_redis_data_db()
+    return DataStore(redis_url)

ewoksdata-0.4.1/src/ewoksdata/data/blissdata/blissdatavid31.py

@@ -0,0 +1,97 @@
+from numbers import Number
+from typing import List, Optional
+
+from numpy.typing import ArrayLike
+
+from .exceptions import VersionError
+
+try:
+    from blissdata.redis_engine.scan import Scan
+    from blissdata.redis_engine.stream import StreamingClient
+    from blissdata.redis_engine.models import ScanState
+    from blissdata.redis_engine.exceptions import EndOfStream
+    from blissdata.lima.client import lima_client_factory
+    from blissdata.beacon.data import BeaconData
+    from blissdata import redis_engine
+except ImportError as e:
+    raise VersionError(str(e)) from e
+
+
+def iter_bliss_scan_data_from_memory(
+    db_name: str,
+    lima_names: List[str],
+    counter_names: List[str],
+    retry_timeout: Optional[Number] = None,
+    retry_period: Optional[Number] = None,
+):
+    _ensure_redis()
+    scan = Scan.load(db_name)
+    buffers = {name: list() for name in lima_names + counter_names}
+
+    while scan.state < ScanState.PREPARED:
+        scan.update()
+    if scan.state is ScanState.ABORTED:
+        return
+
+    lima_streams = dict()
+    lima_clients = dict()
+    counter_streams = dict()
+    for name, stream in scan.streams.items():
+        if stream.encoding["type"] == "json" and "lima" in stream.info["format"]:
+            if name.split(":")[-2] in lima_names:
+                lima_streams[name] = stream
+                lima_clients[name] = lima_client_factory(stream.info)
+        elif name.split(":")[-1] in counter_names:
+            counter_streams[name] = stream
+
+    client = StreamingClient({**lima_streams, **counter_streams})
+
+    while True:
+        try:
+            output = client.read()
+        except EndOfStream:
+            break
+        else:
+            for stream, data in output.items():
+                _, payload = data  # drop index
+                if stream.name in lima_streams:
+                    lima_client = lima_clients[stream.name]
+                    prev_len = len(lima_client)
+                    lima_client.update(**payload[-1])
+                    data = lima_client[prev_len - 1 :]
+                    buffers[stream.name.split(":")[-2]].extend(data)
+                else:
+                    buffers[stream.name.split(":")[-1]].extend(payload)
+
+            nyield = min(len(v) for v in buffers.values())
+            if nyield:
+                for i in range(nyield):
+                    yield {name: values[i] for name, values in buffers.items()}
+                buffers = {name: values[nyield:] for name, values in buffers.items()}
+
+
+def last_lima_image(channel_info: dict) -> ArrayLike:
+    """Get last lima image from memory"""
+    _ensure_redis()
+    lima_client = lima_client_factory(channel_info)
+    return lima_client.get_last_live_image().array
+
+
+def _ensure_redis() -> None:
+    url = BeaconData().get_redis_data_db()
+    current_url = _current_redis_url()
+    if current_url:
+        if url == current_url:
+            return
+        raise RuntimeError("The Redis URL has changed. Restart the Nexus writer.")
+    redis_engine.set_redis_url(url)
+
+
+def _current_redis_url() -> Optional[str]:
+    if redis_engine._redis is None:
+        return
+    kwargs = redis_engine._redis.connection_pool.connection_kwargs
+    if "path" in kwargs:
+        return f"unix://{kwargs['path']}"
+    else:
+        return f"redis://{kwargs['host']}:{kwargs['port']}"

ewoksdata-0.4.1/src/ewoksdata/data/blissdata/exceptions.py

@@ -0,0 +1,4 @@
+class VersionError(ImportError):
+    """Import attempt failed from a library for which we support multiple versions with different import API's."""
+
+    pass

{ewoksdata-0.3.0 → ewoksdata-0.4.1}/src/ewoksdata/data/hdf5/dataset_writer.py

@@ -262,9 +262,9 @@ class StackDatasetWriter(_DatasetWriterBase):
                     istart0_dim1,
                     nflush_dim1,
                 ):
-                    self._dataset[
-                        i_dim0, istart_dim1 : istart_dim1 + n_dim1, ...
-                    ] = buffer[:n_dim1]
+                    self._dataset[i_dim0, istart_dim1 : istart_dim1 + n_dim1, ...] = (
+                        buffer[:n_dim1]
+                    )
         self._buffers = [
             buffer[n_dim1:] for buffer, n_dim1 in zip(self._buffers, nflush)
         ]

{ewoksdata-0.3.0 → ewoksdata-0.4.1}/src/ewoksdata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ewoksdata
-Version: 0.3.0
+Version: 0.4.1
 Summary: Ewoks tasks for data access
 Home-page: https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
 Author: ESRF
@@ -14,11 +14,28 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: h5py>=2.8
+Requires-Dist: silx>=1.1.0
+Requires-Dist: hdf5plugin
+Requires-Dist: blissdata<1.0.1; python_version < "3.8"
+Requires-Dist: blissdata; python_version >= "3.8"
+Requires-Dist: redis
 Provides-Extra: online
+Requires-Dist: blissdata[tango]; extra == "online"
 Provides-Extra: test
+Requires-Dist: blissdata[tango]; extra == "test"
+Requires-Dist: pytest>=7; extra == "test"
 Provides-Extra: dev
+Requires-Dist: blissdata[tango]; extra == "dev"
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: black>=22; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
 Provides-Extra: doc
-License-File: LICENSE.md
+Requires-Dist: blissdata[tango]; extra == "doc"
+Requires-Dist: pytest>=7; extra == "doc"
+Requires-Dist: sphinx>=4.5; extra == "doc"
+Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
 
 # ewoksdata
 

{ewoksdata-0.3.0 → ewoksdata-0.4.1}/src/ewoksdata.egg-info/SOURCES.txt

@@ -14,6 +14,11 @@ src/ewoksdata/data/bliss.py
 src/ewoksdata/data/nexus.py
 src/ewoksdata/data/url.py
 src/ewoksdata/data/utils.py
+src/ewoksdata/data/blissdata/__init__.py
+src/ewoksdata/data/blissdata/blissdatav0.py
+src/ewoksdata/data/blissdata/blissdatav1.py
+src/ewoksdata/data/blissdata/blissdatavid31.py
+src/ewoksdata/data/blissdata/exceptions.py
 src/ewoksdata/data/hdf5/__init__.py
 src/ewoksdata/data/hdf5/config.py
 src/ewoksdata/data/hdf5/dataset_writer.py

{ewoksdata-0.3.0 → ewoksdata-0.4.1}/src/ewoksdata.egg-info/requires.txt

@@ -1,6 +1,12 @@
 h5py>=2.8
 silx>=1.1.0
 hdf5plugin
+redis
+
+[:python_version < "3.8"]
+blissdata<1.0.1
+
+[:python_version >= "3.8"]
 blissdata
 
 [dev]

ewoksdata-0.3.0/src/ewoksdata/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.0"