blissdata 0.3.2__tar.gz → 0.3.4__tar.gz

{blissdata-0.3.2 → blissdata-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blissdata
-Version: 0.3.2
+Version: 0.3.4
 Summary: Bliss data streaming client
 Home-page: https://gitlab.esrf.fr/bliss/bliss/-/tree/master/blissdata
 Author: BCU (ESRF)
@@ -10,11 +10,23 @@ Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGP
 Classifier: Programming Language :: Python :: 3
 Requires-Python: <3.10,>=3.7
 Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: gevent>=20.9.0
+Requires-Dist: h5py>=2.9
+Requires-Dist: numpy>=1.13
+Requires-Dist: silx>=1.1.0
 Provides-Extra: tango
+Requires-Dist: pytango>=9.3.1; extra == "tango"
 Provides-Extra: test
+Requires-Dist: pytest>=7; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
 Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Requires-Dist: black==22.3; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
 Provides-Extra: beacon
-License-File: LICENSE
+Requires-Dist: ruamel.yaml; extra == "beacon"
 
 # blissdata
 

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/__init__.py

@@ -14,4 +14,4 @@
     data
 """
 
-__version__ = "0.3.2"
+__version__ = "0.3.4"

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/beacon/_base.py

@@ -1,5 +1,6 @@
 """Base client to communicate with Beacon."""
 
+import platform
 import socket
 import struct
 import threading
@@ -28,7 +29,8 @@ class BeaconClient:
         else:
             self._address = host, port
         connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        connection.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10)
+        if platform.system() != "Windows":
+            connection.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10)
         connection.connect(self._address)
         connection.settimeout(timeout)
         self._connection = connection

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/common/utils.py

@@ -6,6 +6,7 @@
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
 
 import fnmatch
+import re
 from collections.abc import Mapping
 
 
@@ -141,3 +142,18 @@ def get_matching_names(patterns, names, strict_pattern_as_short_name=False):
         matches[pat] = matching_names
 
     return matches
+
+
+def natural_sort(words):
+    def convert(text):
+        return int(text) if text.isdigit() else text.lower()
+
+    def alphanum_key(key):
+        return [
+            convert(c)
+            for c in re.split(
+                "([0-9]+)", key.decode() if isinstance(key, bytes) else key
+            )
+        ]
+
+    return sorted(words, key=alphanum_key)

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/data/events/lima.py

@@ -244,9 +244,24 @@ class LimaImageStatusEvent(streaming_events.StreamEvent):
             images_per_acquisition = info["saving_frame_per_acquisition"]
 
         first_file_index_in_scan = info["saving_next_number"]
-        subdir_format = (
-            info["saving_prefix"] + info["saving_index_format"] + info["saving_suffix"]
-        )
+        if "saving_managed_mode" not in info:
+            raise RuntimeError(
+                "Lima info does not have 'saving_managed_mode'. Upgrade Bliss to solve it."
+            )
+        if info["saving_managed_mode"] == "HARDWARE":
+            # Assume Dectris hardware saving
+            subdir_format = (
+                info["saving_prefix"]
+                + "_data_"
+                + info["saving_index_format"]
+                + info["saving_suffix"]
+            )
+        else:
+            subdir_format = (
+                info["saving_prefix"]
+                + info["saving_index_format"]
+                + info["saving_suffix"]
+            )
         path_format = os.path.join(info["saving_directory"], subdir_format)
         file_format = info["saving_format"]
 
@@ -260,9 +275,13 @@ class LimaImageStatusEvent(streaming_events.StreamEvent):
             )
             if file_index_in_scan is None:
                 break
-            file_path = path_format % file_index_in_scan
             if file_format.lower().startswith("hdf5"):
-                if info["lima_version"] == "<1.9.1":
+                if info["saving_managed_mode"] == "HARDWARE":
+                    # Assume Dectris hardware saving
+                    path_in_file = "/entry/data/data"
+                    # Dectris starts counting from 1 instead of 0 like lima does
+                    file_index_in_scan += 1
+                elif info["lima_version"] == "<1.9.1":
                     # 'old' lima
                     path_in_file = (
                         "/entry_0000/instrument/"
@@ -277,6 +296,7 @@ class LimaImageStatusEvent(streaming_events.StreamEvent):
                     )
             else:
                 path_in_file = ""
+            file_path = path_format % file_index_in_scan
             yield file_path, path_in_file, image_index_in_file, file_format
 
     @staticmethod

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/data/expiration.py

@@ -1,7 +1,7 @@
 from typing import Optional
 from blissdata.client import get_redis_proxy
 
-_DATA_EXPIRATION_TIME = 600  # 5 minutes
+_DATA_EXPIRATION_TIME = 600  # 10 minutes
 _PARENT_EXPIRATION_TIME = 24 * 3600  # 1 day
 
 

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/data/nodes/channel.py

@@ -15,6 +15,10 @@ from blissdata.data.events import EventData, ChannelDataEvent
 CHANNEL_MAX_LEN = 2048
 
 
+class RedisDataExpiredError(Exception):
+    pass
+
+
 class _ChannelDataNodeBase(DataNode):
     _NODE_TYPE = NotImplemented
 

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/data/nodes/scan.py

@@ -6,7 +6,7 @@
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
 
 from blissdata.data.node import DataNodeContainer
-from blissdata.data.nodes.channel import ChannelDataNode
+from blissdata.data.nodes.channel import ChannelDataNode, RedisDataExpiredError
 from blissdata.data.nodes.lima import LimaImageChannelDataNode
 from blissdata.streaming_events import StreamEvent
 from blissdata.data.events import (
@@ -179,6 +179,14 @@ def get_data_from_nodes(pipeline, *nodes):
     for i, (channel_name, get_data_func) in enumerate(
         scan_channel_get_data_func.items()
     ):
+        if not result[i]:
+            raise RedisDataExpiredError("Redis data channel expired.")
+
+        if result[i][0][0] != b"1-0":  # first event is 1-0, not 0-0
+            raise RedisDataExpiredError(
+                f"Redis data channel {channel_name} have been trimmed out."
+            )
+
         yield channel_name, get_data_func(result[i])
     for channel_name, view in scan_image_get_view.items():
         yield channel_name, view

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/data/remote_node.py

@@ -199,16 +199,6 @@ class RemoteNodeWalker:
             node.node_type,
             node.name,
             node.db_name,
-            RemoteNodeWalker.pack_info(node.info),
+            node.info.get_all(),
             redis_url,
         )
-
-    @staticmethod
-    def pack_info(info):
-        info_dict = info.get_all()
-        try:
-            # ScanState objects brings Bliss dependencies with them, replace with name
-            info_dict["state"] = info["state"].name
-        except KeyError:
-            pass
-        return info_dict

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/data/scan.py

@@ -9,6 +9,7 @@
 import sys
 import numpy
 import gevent
+import gevent.event
 import typing
 import warnings
 import contextlib
@@ -22,6 +23,12 @@ from blissdata.data.events import EventType
 from blissdata.streaming import DataStreamReaderStopHandler
 
 
+def get_data(scan):
+    # keep get_data() wrapper for scripts backward compatibility until the
+    # new data API provide the same functionality (expected for Bliss 1.12)
+    return scan.get_data()
+
+
 def get_counter_names(scan):
     """
     Return a list of counter names

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/h5api/utils/bliss.py

@@ -102,7 +102,7 @@ class BlissDynamicHDF5Handler(hdf5.DynamicHDF5Handler):
             if h5item.name == hdf5.SEP:
                 return False
             scan = [s for s in h5item.name.split(hdf5.SEP) if s][0]
-            nxentry = self.file_obj.file[scan]
+            nxentry = self.file_obj[scan]
         except AttributeError:
             raise RetryError("file is closed")
         if "end_time" in nxentry:
@@ -119,10 +119,7 @@ class BlissDynamicHDF5Handler(hdf5.DynamicHDF5Handler):
             nxentry = self.file_obj[scan]
         except AttributeError:
             raise RetryError("file is closed")
-        if "end_time" in nxentry:
-            # Last dataset written to the file
-            return True
-        return self._get_writer_status(nxentry) in ("SUCCEEDED", "FAILED")
+        return "end_time" in nxentry  # Last dataset written to the file
 
     def _get_writer_status(self, nxentry: h5py.Dataset) -> Optional[str]:
         nxnote = nxentry.get("writer", None)

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/h5api/utils/hdf5.py

@@ -10,7 +10,7 @@ from . import types
 from . import lima
 from .hdf5_retry import retry_file_access
 from .hdf5_retry import ignore_retry_timeout
-from .hdf5_retry import RetryError, SoftRetryError
+from .hdf5_retry import RetryTimeoutError, RetryError, SoftRetryError
 
 
 HDF5Item = Union[h5py.Dataset, h5py.Group, h5py.File, lima.LimaDataset, lima.LimaGroup]
@@ -71,13 +71,19 @@ class DynamicHDF5Handler:
         return self._file_obj
 
     def get_item(self, name: str) -> HDF5Item:
-        return self._retry_get_item(name)
+        try:
+            return self._retry_get_item(name)
+        except RetryTimeoutError:
+            raise KeyError(name)
 
     def slice_dataset(self, name: str, idx: types.DataIndexType) -> types.DataType:
         return self._retry_slice_dataset(name, idx)
 
     def get_attr(self, name: str, key: str) -> Any:
-        return self._retry_get_attr(name, key)
+        try:
+            return self._retry_get_attr(name, key)
+        except RetryTimeoutError:
+            raise AttributeError(name)
 
     @ignore_retry_timeout
     def iter_attrs(self, name: str) -> Iterator[str]:
@@ -138,7 +144,24 @@ class DynamicHDF5Handler:
         item = self._native_items.get(name)
         if item is not None:
             return item
-        item = self.file_obj[name]
+        try:
+            item = self.file_obj[name]
+        except KeyError:
+            parent_name, _, item_name = name.rpartition(SEP)
+            if not parent_name:
+                parent_name = SEP
+            if parent_name == name:
+                raise
+            parent_item = self._get_item(parent_name)
+            if not self._is_finished(parent_item):
+                raise  # item could still be created
+            try:
+                item = parent_item[item_name]  # try one more time
+            except KeyError as e:
+                if not _key_does_not_exist(e):
+                    raise
+                # item will never be created
+                raise RetryTimeoutError(str(e)) from e
         self._native_items[name] = item
         return item
 
@@ -152,16 +175,28 @@ class DynamicHDF5Handler:
 
     def _get_attr(self, name: str, key: str) -> Any:
         item = self._get_item(name)
-        return item.attrs[key]
+        try:
+            return item.attrs[key]
+        except KeyError:
+            if not self._is_finished(item):
+                raise  # attribute could still be created
+            try:
+                return item.attrs[key]  # try one more time
+            except KeyError as e:
+                if not _key_does_not_exist(e):
+                    raise
+                # attribute will never be created
+                raise RetryTimeoutError(str(e)) from e
 
     def _iter_attrs(self, name: str, start_index: int = 0) -> Iterator[str]:
         item = self._get_item(name)
+        is_complete = self._is_initialized(item)
         if start_index == 0:
             yield from item.attrs
         else:
             yield from list(item.attrs)[start_index:]
-        if not self._is_initialized(item):
-            raise RetryError("not fully initialized")
+        if not is_complete:
+            raise RetryError("attributes could still be added")
 
     def _len_attrs(self, name: str) -> int:
         item = self._get_item(name)
@@ -173,19 +208,19 @@ class DynamicHDF5Handler:
         item = self._get_item(name)
         self._check_dataset_before_read(item)
         if self.is_group(item):
-            is_initialized = self._is_initialized(item)
+            is_complete = self._is_initialized(item)
             for key in self._iter_group(item, start_index):
-                is_initialized = False
+                is_complete = False
                 yield key
-            if not is_initialized:
-                raise RetryError("not initialized")
+            if not is_complete:
+                raise RetryError("children could still be added to the group")
         else:
-            is_finished = self._is_finished(item)
+            is_complete = self._is_finished(item)
             for data in self._iter_dataset(item, start_index):
-                is_finished = False
+                is_complete = False
                 yield data
-            if not is_finished:
-                raise RetryError("not finished")
+            if not is_complete:
+                raise RetryError("dataset could still grow")
 
     @staticmethod
     def is_group(h5item: HDF5Item):
@@ -225,3 +260,21 @@ class DynamicHDF5Handler:
 
     def _is_finished(self, h5item: HDF5Item) -> bool:
         return True
+
+
+def _key_does_not_exist(exc: KeyError) -> bool:
+    """
+    A real KeyError (meaning that the key does not exist) gives the following error message:
+
+    .. code
+
+        KeyError: "Unable to synchronously open object (object 'end_time' doesn't exist)"
+
+    An example of a KeyError caused by another reason
+
+    .. code
+
+        KeyError: 'Unable to synchronously open object (addr overflow, addr = 64392369, size = 96, eoa = 64365353)'
+
+    """
+    return "doesn't exist" in str(exc)

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/h5api/utils/hdf5_retry.py

@@ -22,7 +22,7 @@ def ignore_retry_timeout(method: Callable) -> Callable:
         @wraps(method)
         def wrapper(self, *args, **kw):
             try:
-                yield from method(self, *args, *kw)
+                yield from method(self, *args, **kw)
             except RetryTimeoutError:
                 pass
 

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/redis/caching.py

@@ -1,9 +1,10 @@
 """Client-side Redis cache with an invalidation connection.
 """
 
-import os
+import socket
 import gevent
 import gevent.event
+import gevent.select
 from functools import wraps
 from contextlib import ExitStack, contextmanager
 from collections.abc import MutableMapping
@@ -38,7 +39,7 @@ class CachingConnectionGreenlet(gevent.Greenlet):
         self._started_event = started_event
         self._tracking_redirect_id = None
         self.connection = None
-        self._close_pipe = None
+        self._rp, self._wp = None, None
         super().__init__()
 
     def stop(self):
@@ -47,11 +48,11 @@ class CachingConnectionGreenlet(gevent.Greenlet):
 
         Calling `kill` might not return the pubsub connection to the pool.
         """
-        self._send_fd(self._close_pipe, b"X")
+        self._send_fd(self._wp, b"X")
 
     def _run(self):
         with ExitStack() as stack:
-            ctx = self._setup_close_pipe()
+            ctx = self._setup_wp()
             rp, stop_callback = stack.enter_context(ctx)
 
             ctx = self._setup_invalidation_connection(stop_callback)
@@ -64,44 +65,38 @@ class CachingConnectionGreenlet(gevent.Greenlet):
                 self._started_event.set()
                 self._invalidation_loop(pubsub, rp)
 
-    @contextmanager
-    def _setup_close_pipe(self):
-        rp, wp = os.pipe()
-
-        def stop_callback(*args, **kw):
+    @staticmethod
+    def _close_fd(fd):
+        if fd is not None:
             try:
-                os.close(wp)
+                fd.close()
             except OSError:
                 # already closed
                 pass
 
-        try:
-            self._close_pipe = wp
-            yield rp, stop_callback
-        finally:
-            self._close_pipe = None
-            self._close_fd(wp)
-            self._close_fd(rp)
-
     @staticmethod
-    def _close_fd(fd):
+    def _send_fd(fd, msg):
         if fd is None:
             return
         try:
-            os.close(fd)
+            fd.send(msg)
         except OSError:
             # already closed
             pass
 
-    @staticmethod
-    def _send_fd(fd, msg):
-        if fd is None:
-            return
+    def _close_wp(self):
+        return self._close_fd(self._wp)
+
+    @contextmanager
+    def _setup_wp(self):
+        self._rp, self._wp = socket.socketpair()
+
         try:
-            os.write(fd, msg)
-        except OSError:
-            # already closed
-            pass
+            yield self._rp, self._close_wp
+        finally:
+            self._close_fd(self._rp)
+            self._close_fd(self._wp)
+            self._rp, self._wp = None, None
 
     @contextmanager
     def _setup_invalidation_connection(self, stop_callback):

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/redis/connection.py

@@ -27,6 +27,7 @@ These proxies are also greenlet-safe. You will have one cache per pool.
 
 import os
 import gevent
+import gevent.lock
 import socket
 import redis
 

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/redis/proxy.py

@@ -11,6 +11,8 @@ from collections.abc import MutableMapping
 from collections import Counter
 from contextlib import contextmanager
 import gevent
+import gevent.event
+import gevent.lock
 import redis
 import redis.client
 

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/settings.py

@@ -13,7 +13,7 @@ import logging
 import numpy
 from blissdata.client import get_redis_proxy
 from blissdata.redis.scripting import register_script, evaluate_script
-from blissdata.common.utils import Undefined
+from blissdata.common.utils import Undefined, natural_sort
 
 logger = logging.getLogger(__name__)
 
@@ -712,7 +712,7 @@ local set_size = redis.call("ZCARD", setkey)
 if set_max_score > set_size  then
     -- we need to reassign scores as some was deleted
     local new_order = 1
-    table = redis.call("ZRANGE", setkey, 0, -1)
+    local table = redis.call("ZRANGE", setkey, 0, -1)
     for k, attr in pairs(table)
     do
         --redis.call("ZADD", setkey, new_order, attr)
@@ -760,6 +760,20 @@ class OrderedHashSetting(BaseHashSetting):
             orderedhashsetting_helper_script,
         )
 
+        # check if this ordered hash is built from an existing hash setting
+        if self.connection.exists(self.name) and not self.connection.exists(
+            self._name_order
+        ):
+            if self.connection.type(self.name) == b"hash":  # noqa E721
+                # we can provide a default order (alphabetical), to ensure an ordered hash
+                # can be built on top of an existing hash ; this is to preserve
+                # settings coherency in case of type "upgrade" (like when Lima ROIs
+                # have been ordered from "normal" hashes)
+                sorted_keys = natural_sort(self.connection.hkeys(self.name))
+                self.connection.zadd(
+                    self._name_order, {v: k for k, v in enumerate(sorted_keys)}
+                )
+
     @property
     def _name_order(self):
         return self._name + ":creation_order"

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/streaming.py

@@ -7,6 +7,8 @@
 
 import sys
 import gevent
+import gevent.event
+import gevent.queue
 import uuid
 import enum
 import fnmatch

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata/tests/h5api/scanner.py

@@ -268,7 +268,7 @@ def writer_context(filename: str, scan_number: int) -> Iterator[h5py.Group]:
 
         try:
             entry.attrs["NX_class"] = "NXentry"
-            entry.attrs["start_time"] = timestamp()
+            entry["start_time"] = timestamp()
             writer = entry.create_group("writer")
             writer.attrs["NX_class"] = "NXnote"
             writer["status"] = "STARTING"
@@ -279,7 +279,7 @@ def writer_context(filename: str, scan_number: int) -> Iterator[h5py.Group]:
         else:
             writer["status"][()] = "SUCCEEDED"
         finally:
-            entry.attrs["end_time"] = timestamp()
+            entry["end_time"] = timestamp()
 
 
 @contextmanager

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blissdata
-Version: 0.3.2
+Version: 0.3.4
 Summary: Bliss data streaming client
 Home-page: https://gitlab.esrf.fr/bliss/bliss/-/tree/master/blissdata
 Author: BCU (ESRF)
@@ -10,11 +10,23 @@ Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGP
 Classifier: Programming Language :: Python :: 3
 Requires-Python: <3.10,>=3.7
 Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: gevent>=20.9.0
+Requires-Dist: h5py>=2.9
+Requires-Dist: numpy>=1.13
+Requires-Dist: silx>=1.1.0
 Provides-Extra: tango
+Requires-Dist: pytango>=9.3.1; extra == "tango"
 Provides-Extra: test
+Requires-Dist: pytest>=7; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
 Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Requires-Dist: black==22.3; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
 Provides-Extra: beacon
-License-File: LICENSE
+Requires-Dist: ruamel.yaml; extra == "beacon"
 
 # blissdata
 

{blissdata-0.3.2 → blissdata-0.3.4}/blissdata.egg-info/requires.txt

@@ -9,7 +9,7 @@ ruamel.yaml
 [dev]
 pytest>=7
 pytest-mock
-black>=22
+black==22.3
 flake8>=4
 
 [tango]

{blissdata-0.3.2 → blissdata-0.3.4}/pyproject.toml

@@ -1,6 +1,3 @@
 [build-system]
 build-backend = "setuptools.build_meta"
-requires = [
-    "setuptools>=46.4<63",
-    "wheel",
-]
+requires = ["setuptools>=46.4", "wheel"]

{blissdata-0.3.2 → blissdata-0.3.4}/setup.cfg

@@ -29,7 +29,7 @@ test =
 	pytest-mock
 dev = 
 	%(test)s
-	black >=22
+	black ==22.3
 	flake8 >=4
 beacon = 
 	ruamel.yaml