blissdata 1.0.2__tar.gz → 1.0.4__tar.gz

{blissdata-1.0.2 → blissdata-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blissdata
-Version: 1.0.2
+Version: 1.0.4
 Summary: Bliss data streaming client
 Home-page: https://gitlab.esrf.fr/bliss/bliss/-/tree/master/blissdata
 Author: BCU (ESRF)
@@ -15,8 +15,8 @@ Requires-Dist: beartype
 Requires-Dist: h5py<3.10,>=2.9
 Requires-Dist: msgpack
 Requires-Dist: numpy>=1.13
-Requires-Dist: redis>=4.6.0
-Requires-Dist: redis-om
+Requires-Dist: redis<5,>=4.6.0
+Requires-Dist: redis-om<0.3
 Requires-Dist: silx>=1.1.0
 Provides-Extra: tango
 Requires-Dist: pytango>=9.3.1; extra == "tango"

{blissdata-1.0.2 → blissdata-1.0.4}/blissdata/__init__.py

@@ -4,4 +4,4 @@
 #
 # Copyright (c) Beamline Control Unit, ESRF
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
-__version__ = "1.0.2"
+__version__ = "1.0.4"

{blissdata-1.0.2 → blissdata-1.0.4}/blissdata/h5api/utils/bliss.py

@@ -102,7 +102,7 @@ class BlissDynamicHDF5Handler(hdf5.DynamicHDF5Handler):
             if h5item.name == hdf5.SEP:
                 return False
             scan = [s for s in h5item.name.split(hdf5.SEP) if s][0]
-            nxentry = self.file_obj.file[scan]
+            nxentry = self.file_obj[scan]
         except AttributeError:
             raise RetryError("file is closed")
         if "end_time" in nxentry:
@@ -119,10 +119,7 @@ class BlissDynamicHDF5Handler(hdf5.DynamicHDF5Handler):
             nxentry = self.file_obj[scan]
         except AttributeError:
             raise RetryError("file is closed")
-        if "end_time" in nxentry:
-            # Last dataset written to the file
-            return True
-        return self._get_writer_status(nxentry) in ("SUCCEEDED", "FAILED")
+        return "end_time" in nxentry  # Last dataset written to the file
 
     def _get_writer_status(self, nxentry: h5py.Dataset) -> Optional[str]:
         nxnote = nxentry.get("writer", None)

{blissdata-1.0.2 → blissdata-1.0.4}/blissdata/h5api/utils/hdf5.py

@@ -146,19 +146,22 @@ class DynamicHDF5Handler:
             return item
         try:
             item = self.file_obj[name]
-        except KeyError as e:
-            parent_name = name.rpartition(SEP)[0]
+        except KeyError:
+            parent_name, _, item_name = name.rpartition(SEP)
             if not parent_name:
                 parent_name = SEP
-            if parent_name != name:
-                try:
-                    parent_item = self._get_item(parent_name)
-                except KeyError:
-                    pass
-                else:
-                    if self._is_finished(parent_item):
-                        raise RetryTimeoutError(str(e)) from e
-            raise
+            if parent_name == name:
+                raise
+            parent_item = self._get_item(parent_name)
+            if not self._is_finished(parent_item):
+                raise  # item could still be created
+            try:
+                item = parent_item[item_name]  # try one more time
+            except KeyError as e:
+                if not _key_does_not_exist(e):
+                    raise
+                # item will never be created
+                raise RetryTimeoutError(str(e)) from e
         self._native_items[name] = item
         return item
 
@@ -174,19 +177,26 @@ class DynamicHDF5Handler:
         item = self._get_item(name)
         try:
             return item.attrs[key]
-        except KeyError as e:
-            if self._is_finished(item):
+        except KeyError:
+            if not self._is_finished(item):
+                raise  # attribute could still be created
+            try:
+                return item.attrs[key]  # try one more time
+            except KeyError as e:
+                if not _key_does_not_exist(e):
+                    raise
+                # attribute will never be created
                 raise RetryTimeoutError(str(e)) from e
-            raise
 
     def _iter_attrs(self, name: str, start_index: int = 0) -> Iterator[str]:
         item = self._get_item(name)
+        is_complete = self._is_initialized(item)
         if start_index == 0:
             yield from item.attrs
         else:
             yield from list(item.attrs)[start_index:]
-        if not self._is_initialized(item):
-            raise RetryError("not fully initialized")
+        if not is_complete:
+            raise RetryError("attributes could still be added")
 
     def _len_attrs(self, name: str) -> int:
         item = self._get_item(name)
@@ -198,19 +208,19 @@ class DynamicHDF5Handler:
         item = self._get_item(name)
         self._check_dataset_before_read(item)
         if self.is_group(item):
-            is_initialized = self._is_initialized(item)
+            is_complete = self._is_initialized(item)
             for key in self._iter_group(item, start_index):
-                is_initialized = False
+                is_complete = False
                 yield key
-            if not is_initialized:
-                raise RetryError("not initialized")
+            if not is_complete:
+                raise RetryError("children could still be added to the group")
         else:
-            is_finished = self._is_finished(item)
+            is_complete = self._is_finished(item)
             for data in self._iter_dataset(item, start_index):
-                is_finished = False
+                is_complete = False
                 yield data
-            if not is_finished:
-                raise RetryError("not finished")
+            if not is_complete:
+                raise RetryError("dataset could still grow")
 
     @staticmethod
     def is_group(h5item: HDF5Item):
@@ -250,3 +260,21 @@ class DynamicHDF5Handler:
 
     def _is_finished(self, h5item: HDF5Item) -> bool:
         return True
+
+
+def _key_does_not_exist(exc: KeyError) -> bool:
+    """
+    A real KeyError (meaning that the key does not exist) gives the following error message:
+
+    .. code
+
+        KeyError: "Unable to synchronously open object (object 'end_time' doesn't exist)"
+
+    An example of a KeyError caused by another reason
+
+    .. code
+
+        KeyError: 'Unable to synchronously open object (addr overflow, addr = 64392369, size = 96, eoa = 64365353)'
+
+    """
+    return "doesn't exist" in str(exc)

{blissdata-1.0.2 → blissdata-1.0.4}/blissdata/redis_engine/scan.py

@@ -73,6 +73,8 @@ class Scan:
         for prop_name in scan._data_store._scan_model.id.field.type_.schema()[
             "properties"
         ].keys():
+            if prop_name == "pk":
+                continue
 
             def get_id_field(self, field=prop_name):
                 return getattr(self._model.id, field, None)
@@ -109,6 +111,8 @@ class Scan:
         for prop_name in scan._data_store._scan_model.id.field.type_.schema()[
             "properties"
         ].keys():
+            if prop_name == "pk":
+                continue
 
             def get_id_field(self, field=prop_name):
                 return getattr(self._model.id, field, None)
@@ -278,10 +282,7 @@ class Scan:
         the still running writers to fail.
         """
         for stream_writer in self._writer_streams.values():
-            if not stream_writer.is_sealed():
-                stream_writer.seal()
-            else:
-                stream_writer._sink.stop()
+            stream_writer.seal()
         self._writer_streams = {}
 
     @needs_write_permission
@@ -301,15 +302,15 @@ class Scan:
     @needs_write_permission
     def stop(self):
         if self.state is ScanState.STARTED:
-            self._set_state(ScanState.STOPPED)
             self._close_stream_writers()
+            self._set_state(ScanState.STOPPED)
         else:
             raise UnauthorizeStateTransition(self.state, ScanState.STOPPED)
 
     @needs_write_permission
     def close(self):
-        self._set_state(ScanState.CLOSED)
         self._close_stream_writers()
+        self._set_state(ScanState.CLOSED)
 
     @needs_write_permission
     def _set_state(self, state):

{blissdata-1.0.2 → blissdata-1.0.4}/blissdata/redis_engine/stream.py

@@ -4,6 +4,7 @@
 #
 # Copyright (c) Beamline Control Unit, ESRF
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
+import redis
 from functools import wraps
 from dataclasses import dataclass
 from collections.abc import Mapping
@@ -270,8 +271,14 @@ class Stream:
 
     @needs_write_permission
     def seal(self):
-        self._sink.fcall("seal_stream", 1, self.key)
         self._sink.stop()
+        try:
+            return self._data_store._redis.fcall("seal_stream", 1, self.key)
+        except redis.exceptions.ResponseError as e:
+            if "is already sealed" in str(e):
+                return len(self)
+            else:
+                raise
 
     def __del__(self):
         # Thread based sinks can't be garbage collected, because their internal thread

{blissdata-1.0.2 → blissdata-1.0.4}/blissdata/tests/conftest.py

@@ -3,7 +3,7 @@ import subprocess
 from redis import Redis
 from socket import socket
 from pytest import fixture
-from pytest_redis import factories
+from pytest_redis.factories.noproc import redis_noproc
 
 from blissdata.redis_engine.store import DataStore
 
@@ -15,7 +15,7 @@ def _find_available_port():
 
 
 redis_port = _find_available_port()
-redis_db = factories.redis_noproc(host="localhost", port=redis_port, startup_timeout=1)
+redis_db = redis_noproc(host="localhost", port=redis_port, startup_timeout=1)
 
 
 def pytest_configure(config):

{blissdata-1.0.2 → blissdata-1.0.4}/blissdata/tests/h5api/scanner.py

@@ -268,7 +268,7 @@ def writer_context(filename: str, scan_number: int) -> Iterator[h5py.Group]:
 
         try:
             entry.attrs["NX_class"] = "NXentry"
-            entry.attrs["start_time"] = timestamp()
+            entry["start_time"] = timestamp()
             writer = entry.create_group("writer")
             writer.attrs["NX_class"] = "NXnote"
             writer["status"] = "STARTING"
@@ -279,7 +279,7 @@ def writer_context(filename: str, scan_number: int) -> Iterator[h5py.Group]:
         else:
             writer["status"][()] = "SUCCEEDED"
         finally:
-            entry.attrs["end_time"] = timestamp()
+            entry["end_time"] = timestamp()
 
 
 @contextmanager

{blissdata-1.0.2 → blissdata-1.0.4}/blissdata/tests/redis_engine/test_streaming.py

@@ -80,6 +80,22 @@ def test_stream_unknown_encoding(data_store):
         Stream.create(data_store, "my_stream", model, None)
 
 
+def test_multiple_calls_to_seal(data_store):
+    input = np.arange(1000)
+    rw_stream, _ = stream_pair(data_store, input.dtype)
+    rw_stream.send(input)
+
+    # make a second writing stream to test re-sealing from an unaware object
+    external_writer = Stream.create(
+        data_store, rw_stream.name, rw_stream._model, rw_stream._encoder
+    )
+
+    assert rw_stream.seal() == 1000
+    assert rw_stream.seal() == 1000
+    assert external_writer.seal() == 1000
+    assert external_writer.seal() == 1000
+
+
 def test_stream_chunk_slicing(data_store):
     data = np.arange(100)
     rw_stream, ro_stream = stream_pair(data_store, data.dtype)

{blissdata-1.0.2 → blissdata-1.0.4}/blissdata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blissdata
-Version: 1.0.2
+Version: 1.0.4
 Summary: Bliss data streaming client
 Home-page: https://gitlab.esrf.fr/bliss/bliss/-/tree/master/blissdata
 Author: BCU (ESRF)
@@ -15,8 +15,8 @@ Requires-Dist: beartype
 Requires-Dist: h5py<3.10,>=2.9
 Requires-Dist: msgpack
 Requires-Dist: numpy>=1.13
-Requires-Dist: redis>=4.6.0
-Requires-Dist: redis-om
+Requires-Dist: redis<5,>=4.6.0
+Requires-Dist: redis-om<0.3
 Requires-Dist: silx>=1.1.0
 Provides-Extra: tango
 Requires-Dist: pytango>=9.3.1; extra == "tango"

{blissdata-1.0.2 → blissdata-1.0.4}/blissdata.egg-info/requires.txt

@@ -2,8 +2,8 @@ beartype
 h5py<3.10,>=2.9
 msgpack
 numpy>=1.13
-redis>=4.6.0
-redis-om
+redis<5,>=4.6.0
+redis-om<0.3
 silx>=1.1.0
 
 [beacon]

{blissdata-1.0.2 → blissdata-1.0.4}/setup.cfg

@@ -20,8 +20,8 @@ install_requires =
 	h5py >=2.9, <3.10  # https://gitlab.esrf.fr/bliss/bliss/-/issues/3959
 	msgpack
 	numpy >=1.13
-	redis >=4.6.0
-	redis-om
+	redis >=4.6.0, <5
+	redis-om <0.3
 	silx >=1.1.0
 
 [options.extras_require]