blissdata 0.3.3__tar.gz → 0.3.4__tar.gz

{blissdata-0.3.3 → blissdata-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blissdata
-Version: 0.3.3
+Version: 0.3.4
 Summary: Bliss data streaming client
 Home-page: https://gitlab.esrf.fr/bliss/bliss/-/tree/master/blissdata
 Author: BCU (ESRF)
@@ -10,11 +10,23 @@ Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGP
 Classifier: Programming Language :: Python :: 3
 Requires-Python: <3.10,>=3.7
 Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: gevent>=20.9.0
+Requires-Dist: h5py>=2.9
+Requires-Dist: numpy>=1.13
+Requires-Dist: silx>=1.1.0
 Provides-Extra: tango
+Requires-Dist: pytango>=9.3.1; extra == "tango"
 Provides-Extra: test
+Requires-Dist: pytest>=7; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
 Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Requires-Dist: black==22.3; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
 Provides-Extra: beacon
-License-File: LICENSE
+Requires-Dist: ruamel.yaml; extra == "beacon"
 
 # blissdata
 

{blissdata-0.3.3 → blissdata-0.3.4}/blissdata/__init__.py

@@ -14,4 +14,4 @@
     data
 """
 
-__version__ = "0.3.3"
+__version__ = "0.3.4"

{blissdata-0.3.3 → blissdata-0.3.4}/blissdata/beacon/_base.py

@@ -1,5 +1,6 @@
 """Base client to communicate with Beacon."""
 
+import platform
 import socket
 import struct
 import threading
@@ -28,7 +29,8 @@ class BeaconClient:
         else:
             self._address = host, port
         connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        connection.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10)
+        if platform.system() != "Windows":
+            connection.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10)
         connection.connect(self._address)
         connection.settimeout(timeout)
         self._connection = connection

{blissdata-0.3.3 → blissdata-0.3.4}/blissdata/data/events/lima.py

@@ -244,9 +244,24 @@ class LimaImageStatusEvent(streaming_events.StreamEvent):
             images_per_acquisition = info["saving_frame_per_acquisition"]
 
         first_file_index_in_scan = info["saving_next_number"]
-        subdir_format = (
-            info["saving_prefix"] + info["saving_index_format"] + info["saving_suffix"]
-        )
+        if "saving_managed_mode" not in info:
+            raise RuntimeError(
+                "Lima info does not have 'saving_managed_mode'. Upgrade Bliss to solve it."
+            )
+        if info["saving_managed_mode"] == "HARDWARE":
+            # Assume Dectris hardware saving
+            subdir_format = (
+                info["saving_prefix"]
+                + "_data_"
+                + info["saving_index_format"]
+                + info["saving_suffix"]
+            )
+        else:
+            subdir_format = (
+                info["saving_prefix"]
+                + info["saving_index_format"]
+                + info["saving_suffix"]
+            )
         path_format = os.path.join(info["saving_directory"], subdir_format)
         file_format = info["saving_format"]
 
@@ -260,9 +275,13 @@ class LimaImageStatusEvent(streaming_events.StreamEvent):
             )
             if file_index_in_scan is None:
                 break
-            file_path = path_format % file_index_in_scan
             if file_format.lower().startswith("hdf5"):
-                if info["lima_version"] == "<1.9.1":
+                if info["saving_managed_mode"] == "HARDWARE":
+                    # Assume Dectris hardware saving
+                    path_in_file = "/entry/data/data"
+                    # Dectris starts counting from 1 instead of 0 like lima does
+                    file_index_in_scan += 1
+                elif info["lima_version"] == "<1.9.1":
                     # 'old' lima
                     path_in_file = (
                         "/entry_0000/instrument/"
@@ -277,6 +296,7 @@ class LimaImageStatusEvent(streaming_events.StreamEvent):
                     )
             else:
                 path_in_file = ""
+            file_path = path_format % file_index_in_scan
             yield file_path, path_in_file, image_index_in_file, file_format
 
     @staticmethod

{blissdata-0.3.3 → blissdata-0.3.4}/blissdata/data/remote_node.py

@@ -199,16 +199,6 @@ class RemoteNodeWalker:
             node.node_type,
             node.name,
             node.db_name,
-            RemoteNodeWalker.pack_info(node.info),
+            node.info.get_all(),
             redis_url,
         )
-
-    @staticmethod
-    def pack_info(info):
-        info_dict = info.get_all()
-        try:
-            # ScanState objects brings Bliss dependencies with them, replace with name
-            info_dict["state"] = info["state"].name
-        except KeyError:
-            pass
-        return info_dict

{blissdata-0.3.3 → blissdata-0.3.4}/blissdata/h5api/utils/bliss.py

@@ -102,7 +102,7 @@ class BlissDynamicHDF5Handler(hdf5.DynamicHDF5Handler):
             if h5item.name == hdf5.SEP:
                 return False
             scan = [s for s in h5item.name.split(hdf5.SEP) if s][0]
-            nxentry = self.file_obj.file[scan]
+            nxentry = self.file_obj[scan]
         except AttributeError:
             raise RetryError("file is closed")
         if "end_time" in nxentry:
@@ -119,10 +119,7 @@ class BlissDynamicHDF5Handler(hdf5.DynamicHDF5Handler):
             nxentry = self.file_obj[scan]
         except AttributeError:
             raise RetryError("file is closed")
-        if "end_time" in nxentry:
-            # Last dataset written to the file
-            return True
-        return self._get_writer_status(nxentry) in ("SUCCEEDED", "FAILED")
+        return "end_time" in nxentry  # Last dataset written to the file
 
     def _get_writer_status(self, nxentry: h5py.Dataset) -> Optional[str]:
         nxnote = nxentry.get("writer", None)

{blissdata-0.3.3 → blissdata-0.3.4}/blissdata/h5api/utils/hdf5.py

@@ -10,7 +10,7 @@ from . import types
 from . import lima
 from .hdf5_retry import retry_file_access
 from .hdf5_retry import ignore_retry_timeout
-from .hdf5_retry import RetryError, SoftRetryError
+from .hdf5_retry import RetryTimeoutError, RetryError, SoftRetryError
 
 
 HDF5Item = Union[h5py.Dataset, h5py.Group, h5py.File, lima.LimaDataset, lima.LimaGroup]
@@ -71,13 +71,19 @@ class DynamicHDF5Handler:
         return self._file_obj
 
     def get_item(self, name: str) -> HDF5Item:
-        return self._retry_get_item(name)
+        try:
+            return self._retry_get_item(name)
+        except RetryTimeoutError:
+            raise KeyError(name)
 
     def slice_dataset(self, name: str, idx: types.DataIndexType) -> types.DataType:
         return self._retry_slice_dataset(name, idx)
 
     def get_attr(self, name: str, key: str) -> Any:
-        return self._retry_get_attr(name, key)
+        try:
+            return self._retry_get_attr(name, key)
+        except RetryTimeoutError:
+            raise AttributeError(name)
 
     @ignore_retry_timeout
     def iter_attrs(self, name: str) -> Iterator[str]:
@@ -138,7 +144,24 @@ class DynamicHDF5Handler:
         item = self._native_items.get(name)
         if item is not None:
             return item
-        item = self.file_obj[name]
+        try:
+            item = self.file_obj[name]
+        except KeyError:
+            parent_name, _, item_name = name.rpartition(SEP)
+            if not parent_name:
+                parent_name = SEP
+            if parent_name == name:
+                raise
+            parent_item = self._get_item(parent_name)
+            if not self._is_finished(parent_item):
+                raise  # item could still be created
+            try:
+                item = parent_item[item_name]  # try one more time
+            except KeyError as e:
+                if not _key_does_not_exist(e):
+                    raise
+                # item will never be created
+                raise RetryTimeoutError(str(e)) from e
         self._native_items[name] = item
         return item
 
@@ -152,16 +175,28 @@ class DynamicHDF5Handler:
 
     def _get_attr(self, name: str, key: str) -> Any:
         item = self._get_item(name)
-        return item.attrs[key]
+        try:
+            return item.attrs[key]
+        except KeyError:
+            if not self._is_finished(item):
+                raise  # attribute could still be created
+            try:
+                return item.attrs[key]  # try one more time
+            except KeyError as e:
+                if not _key_does_not_exist(e):
+                    raise
+                # attribute will never be created
+                raise RetryTimeoutError(str(e)) from e
 
     def _iter_attrs(self, name: str, start_index: int = 0) -> Iterator[str]:
         item = self._get_item(name)
+        is_complete = self._is_initialized(item)
         if start_index == 0:
             yield from item.attrs
         else:
             yield from list(item.attrs)[start_index:]
-        if not self._is_initialized(item):
-            raise RetryError("not fully initialized")
+        if not is_complete:
+            raise RetryError("attributes could still be added")
 
     def _len_attrs(self, name: str) -> int:
         item = self._get_item(name)
@@ -173,19 +208,19 @@ class DynamicHDF5Handler:
         item = self._get_item(name)
         self._check_dataset_before_read(item)
         if self.is_group(item):
-            is_initialized = self._is_initialized(item)
+            is_complete = self._is_initialized(item)
             for key in self._iter_group(item, start_index):
-                is_initialized = False
+                is_complete = False
                 yield key
-            if not is_initialized:
-                raise RetryError("not initialized")
+            if not is_complete:
+                raise RetryError("children could still be added to the group")
         else:
-            is_finished = self._is_finished(item)
+            is_complete = self._is_finished(item)
             for data in self._iter_dataset(item, start_index):
-                is_finished = False
+                is_complete = False
                 yield data
-            if not is_finished:
-                raise RetryError("not finished")
+            if not is_complete:
+                raise RetryError("dataset could still grow")
 
     @staticmethod
     def is_group(h5item: HDF5Item):
@@ -225,3 +260,21 @@ class DynamicHDF5Handler:
 
     def _is_finished(self, h5item: HDF5Item) -> bool:
         return True
+
+
+def _key_does_not_exist(exc: KeyError) -> bool:
+    """
+    A real KeyError (meaning that the key does not exist) gives the following error message:
+
+    .. code
+
+        KeyError: "Unable to synchronously open object (object 'end_time' doesn't exist)"
+
+    An example of a KeyError caused by another reason
+
+    .. code
+
+        KeyError: 'Unable to synchronously open object (addr overflow, addr = 64392369, size = 96, eoa = 64365353)'
+
+    """
+    return "doesn't exist" in str(exc)

{blissdata-0.3.3 → blissdata-0.3.4}/blissdata/redis/caching.py

@@ -1,7 +1,7 @@
 """Client-side Redis cache with an invalidation connection.
 """
 
-import os
+import socket
 import gevent
 import gevent.event
 import gevent.select
@@ -39,7 +39,7 @@ class CachingConnectionGreenlet(gevent.Greenlet):
         self._started_event = started_event
         self._tracking_redirect_id = None
         self.connection = None
-        self._close_pipe = None
+        self._rp, self._wp = None, None
         super().__init__()
 
     def stop(self):
@@ -48,11 +48,11 @@ class CachingConnectionGreenlet(gevent.Greenlet):
 
         Calling `kill` might not return the pubsub connection to the pool.
         """
-        self._send_fd(self._close_pipe, b"X")
+        self._send_fd(self._wp, b"X")
 
     def _run(self):
         with ExitStack() as stack:
-            ctx = self._setup_close_pipe()
+            ctx = self._setup_wp()
             rp, stop_callback = stack.enter_context(ctx)
 
             ctx = self._setup_invalidation_connection(stop_callback)
@@ -65,44 +65,38 @@ class CachingConnectionGreenlet(gevent.Greenlet):
                 self._started_event.set()
                 self._invalidation_loop(pubsub, rp)
 
-    @contextmanager
-    def _setup_close_pipe(self):
-        rp, wp = os.pipe()
-
-        def stop_callback(*args, **kw):
+    @staticmethod
+    def _close_fd(fd):
+        if fd is not None:
             try:
-                os.close(wp)
+                fd.close()
             except OSError:
                 # already closed
                 pass
 
-        try:
-            self._close_pipe = wp
-            yield rp, stop_callback
-        finally:
-            self._close_pipe = None
-            self._close_fd(wp)
-            self._close_fd(rp)
-
     @staticmethod
-    def _close_fd(fd):
+    def _send_fd(fd, msg):
         if fd is None:
             return
         try:
-            os.close(fd)
+            fd.send(msg)
         except OSError:
             # already closed
             pass
 
-    @staticmethod
-    def _send_fd(fd, msg):
-        if fd is None:
-            return
+    def _close_wp(self):
+        return self._close_fd(self._wp)
+
+    @contextmanager
+    def _setup_wp(self):
+        self._rp, self._wp = socket.socketpair()
+
         try:
-            os.write(fd, msg)
-        except OSError:
-            # already closed
-            pass
+            yield self._rp, self._close_wp
+        finally:
+            self._close_fd(self._rp)
+            self._close_fd(self._wp)
+            self._rp, self._wp = None, None
 
     @contextmanager
     def _setup_invalidation_connection(self, stop_callback):

{blissdata-0.3.3 → blissdata-0.3.4}/blissdata/settings.py

@@ -712,7 +712,7 @@ local set_size = redis.call("ZCARD", setkey)
 if set_max_score > set_size  then
     -- we need to reassign scores as some was deleted
     local new_order = 1
-    table = redis.call("ZRANGE", setkey, 0, -1)
+    local table = redis.call("ZRANGE", setkey, 0, -1)
     for k, attr in pairs(table)
     do
         --redis.call("ZADD", setkey, new_order, attr)
@@ -764,7 +764,7 @@ class OrderedHashSetting(BaseHashSetting):
         if self.connection.exists(self.name) and not self.connection.exists(
             self._name_order
         ):
-            if self.connection.type(self.name) == b"hash":
+            if self.connection.type(self.name) == b"hash":  # noqa E721
                 # we can provide a default order (alphabetical), to ensure an ordered hash
                 # can be built on top of an existing hash ; this is to preserve
                 # settings coherency in case of type "upgrade" (like when Lima ROIs

{blissdata-0.3.3 → blissdata-0.3.4}/blissdata/tests/h5api/scanner.py

@@ -268,7 +268,7 @@ def writer_context(filename: str, scan_number: int) -> Iterator[h5py.Group]:
 
         try:
             entry.attrs["NX_class"] = "NXentry"
-            entry.attrs["start_time"] = timestamp()
+            entry["start_time"] = timestamp()
             writer = entry.create_group("writer")
             writer.attrs["NX_class"] = "NXnote"
             writer["status"] = "STARTING"
@@ -279,7 +279,7 @@ def writer_context(filename: str, scan_number: int) -> Iterator[h5py.Group]:
         else:
             writer["status"][()] = "SUCCEEDED"
         finally:
-            entry.attrs["end_time"] = timestamp()
+            entry["end_time"] = timestamp()
 
 
 @contextmanager

{blissdata-0.3.3 → blissdata-0.3.4}/blissdata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blissdata
-Version: 0.3.3
+Version: 0.3.4
 Summary: Bliss data streaming client
 Home-page: https://gitlab.esrf.fr/bliss/bliss/-/tree/master/blissdata
 Author: BCU (ESRF)
@@ -10,11 +10,23 @@ Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGP
 Classifier: Programming Language :: Python :: 3
 Requires-Python: <3.10,>=3.7
 Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: gevent>=20.9.0
+Requires-Dist: h5py>=2.9
+Requires-Dist: numpy>=1.13
+Requires-Dist: silx>=1.1.0
 Provides-Extra: tango
+Requires-Dist: pytango>=9.3.1; extra == "tango"
 Provides-Extra: test
+Requires-Dist: pytest>=7; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
 Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Requires-Dist: black==22.3; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
 Provides-Extra: beacon
-License-File: LICENSE
+Requires-Dist: ruamel.yaml; extra == "beacon"
 
 # blissdata