blissdata 2.2.0__tar.gz → 2.4.0__tar.gz

{blissdata-2.2.0/src/blissdata.egg-info → blissdata-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: blissdata
-Version: 2.2.0
+Version: 2.4.0
 Summary: Bliss data streaming client
 Maintainer: BCU (ESRF)
 License-Expression: LGPL-3.0-or-later
@@ -19,15 +19,14 @@ Requires-Dist: h5py>=2.9
 Requires-Dist: msgpack
 Requires-Dist: numpy>=1.20
 Requires-Dist: numpydantic<1.6.10
-Requires-Dist: pydantic>=2.3.0
+Requires-Dist: pydantic>=2.9
 Requires-Dist: redis>=4.6.0
-Requires-Dist: redis-om<1.0,>=0.3.1
+Requires-Dist: redis-om~=1.0
 Requires-Dist: silx>=1.1.0
 Provides-Extra: tango
 Requires-Dist: pytango>=9.3.1; extra == "tango"
 Provides-Extra: lima2
-Requires-Dist: blissdata[tango]; extra == "lima2"
-Requires-Dist: lima2-client~=4.0; extra == "lima2"
+Requires-Dist: blissdata-lima2>=2.0; extra == "lima2"
 Provides-Extra: test
 Requires-Dist: gevent>=20.9.0; extra == "test"
 Requires-Dist: pytest>=7; extra == "test"
@@ -37,7 +36,7 @@ Requires-Dist: pytest-timeout; extra == "test"
 Requires-Dist: packaging; extra == "test"
 Provides-Extra: dev
 Requires-Dist: blissdata[test]; extra == "dev"
-Requires-Dist: black==22.3; extra == "dev"
+Requires-Dist: black; extra == "dev"
 Requires-Dist: flake8>=4; extra == "dev"
 Requires-Dist: flake8-pep585; extra == "dev"
 Requires-Dist: mkdocs; extra == "dev"

{blissdata-2.2.0 → blissdata-2.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "blissdata"
-version = "2.2.0"
+version = "2.4.0"
 description = "Bliss data streaming client"
 readme = "README.md"
 license = "LGPL-3.0-or-later"
@@ -19,9 +19,9 @@ dependencies = [
     "msgpack",
     "numpy>=1.20",
     "numpydantic<1.6.10",
-    "pydantic>=2.3.0", # (not a direct dep) redis-om 0.3.2 doesn't support lower versions
+    "pydantic>=2.9",
     "redis>=4.6.0",
-    "redis-om>=0.3.1,<1.0",
+    "redis-om~=1.0",
     "silx>=1.1.0",
 ]
 
@@ -32,10 +32,7 @@ Changes = "https://gitlab.esrf.fr/bliss/blissdata/-/blob/master/CHANGELOG.md"
 
 [project.optional-dependencies]
 tango = ["pytango>=9.3.1"]
-lima2 = [
-    "blissdata[tango]",
-    "lima2-client~=4.0"
-]
+lima2 = ["blissdata-lima2>=2.0"]
 test = [
     "gevent>=20.9.0",
     "pytest>=7",
@@ -46,7 +43,7 @@ test = [
 ]
 dev = [
     "blissdata[test]",
-    "black==22.3",
+    "black",
     "flake8>=4",
     "flake8-pep585",
     "mkdocs",
@@ -62,7 +59,6 @@ memory_tracker = "blissdata.redis_engine.memory_tracker:cli"
 hdf5_fallback = "blissdata.streams.hdf5_fallback"
 scan_sequence = "blissdata.streams.scan_sequence"
 lima = "blissdata.streams.lima"
-lima2 = "blissdata.streams.lima2"
 
 [build-system]
 requires = ["setuptools"]

blissdata-2.4.0/src/blissdata/__init__.py

@@ -0,0 +1,4 @@
+from .redis_engine.store import DataStore  # noqa F401
+from .redis_engine.scan import Scan, ScanState  # noqa F401
+
+__all__ = ["DataStore", "Scan", "ScanState"]

blissdata-2.4.0/src/blissdata/deprecation.py

@@ -0,0 +1,12 @@
+from warnings import warn
+
+
+def warn_deprecated(old_name, new_name, removal_version=None):
+    msg = f"'{old_name}' is deprecated, use '{new_name}' instead."
+    if removal_version:
+        msg += f" It will be removed in version {removal_version}."
+    warn(
+        msg,
+        FutureWarning,
+        stacklevel=3,
+    )

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/h5api/h5_scan_map.py

@@ -2,7 +2,7 @@ import time
 import logging
 from collections.abc import Mapping
 from pathlib import PurePosixPath as Path
-from blissdata.redis_engine.scan import ScanState
+from blissdata import ScanState
 from blissdata.h5map import HDF5ItemType, Group
 
 _logger = logging.getLogger(__name__)

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/h5api/redis_hdf5.py

@@ -7,9 +7,9 @@ from abc import ABC, abstractmethod
 from pathlib import PurePosixPath as Path
 
 import blissdata.h5map as h5m
+from blissdata import DataStore
 from blissdata.beacon.data import BeaconData
-from blissdata.redis_engine.store import DataStore
-from blissdata.redis_engine.exceptions import EndOfStream, IndexNotYetThereError
+from blissdata.exceptions import EndOfStream, IndexNotYetThereError
 from blissdata.h5api import abstract
 from blissdata.h5api.h5_scan_map import H5ScanMap
 

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/hdf5.py

@@ -251,7 +251,7 @@ class DynamicHDF5Handler:
             try:
                 return item.attrs[key]  # try one more time
             except KeyError as e:
-                if not _key_does_not_exist(e):
+                if not _attribute_does_not_exist(e):
                     raise
                 # attribute will never be created
                 raise RetryTimeoutError(str(e)) from e
@@ -366,3 +366,14 @@ def _key_does_not_exist(exc: KeyError) -> bool:
 
     """
     return "doesn't exist" in str(exc)
+
+
+def _attribute_does_not_exist(exc: KeyError) -> bool:
+    """
+    A real KeyError (meaning that the attribute does not exist) gives the following error message:
+
+    .. code
+
+        KeyError: "Unable to synchronously open attribute (can't locate attribute: 'myattr')"
+    """
+    return "can't locate attribute" in str(exc)

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/h5map/__init__.py

@@ -4,6 +4,7 @@ from .data_items import (  # noqa: F401
     InlineRaw,
     Inline,
     InlineBase64,
+    Compound,
     Stream,
 )
 from .hdf5_items import (  # noqa: F401

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/h5map/data_items.py

@@ -6,6 +6,7 @@ from pydantic import (
     Base64Bytes,
     Field,
     field_validator,
+    model_validator,
     model_serializer,
     Discriminator,
     Tag,
@@ -64,7 +65,13 @@ class InlineRaw(RootModel):
     def decode(self):
         value = self.model_dump()
         if value.dtype.kind in "UT":
-            return np.vectorize(lambda x: x.encode(), otypes="O")(value)
+            # Make strings similar to those read back from h5py:
+            #   - Convert to "object" dtype
+            #   - Tell h5py that "object" is variable length string using
+            #     metadata, so writing doesn't fail even in a compound dataset.
+            return np.vectorize(
+                lambda x: x.encode(), otypes=[np.dtype("O", metadata={"vlen": str})]
+            )(value)
         else:
             return value
 
@@ -155,14 +162,6 @@ class InlineBase64(BaseModel):
             return value
 
 
-class Stream(BaseModel):
-    type: Literal["stream"] = "stream"
-    stream: str
-
-    def decode(self):
-        raise TypeError("Stream model is just a reference, can't use decode()")
-
-
 def get_discriminator_value(v: Any) -> str:
     if isinstance(v, dict):
         return v.get("type")
@@ -171,12 +170,78 @@ def get_discriminator_value(v: Any) -> str:
     return "raw"
 
 
+class Compound(BaseModel):
+    """Composite model to combine multiple DataItems as a structured array.
+    Unlike numpy structured arrays where data is a list of heterogeneous tuples,
+    here all elements of one dtype are stored together. This way existing
+    DataItem models can be reused as columns.
+
+    Validation ensures:
+    - Column names uniqueness
+    - Shape consistency between columns
+    """
+
+    type: Literal["compound"] = "compound"
+    columns: list[
+        tuple[
+            str,
+            Annotated[
+                (
+                    Annotated[Inline, Tag("inline")]
+                    | Annotated[InlineBase64, Tag("inline_b64")]
+                    | Annotated[InlineRaw, Tag("raw")]
+                    | Annotated["Compound", Tag("compound")]
+                    # Note: Stream can't be part of a compound object
+                ),
+                Discriminator(get_discriminator_value),
+            ],
+        ]
+    ] = Field(min_length=1)
+    # Reusing DataItemType simplifies this model as it does not redefine
+    # handling of dtype, shape or base64 version of the values.
+    # Therefore the representation is the transposed of numpy one, which can be
+    # harder to read.
+
+    def decode(self):
+        decoded_columns = [(name, value.decode()) for name, value in self.columns]
+        rec = np.rec.fromarrays(
+            [value for _, value in decoded_columns],
+            names=[name for name, _ in decoded_columns],
+        )
+        return rec.view(np.ndarray)
+
+    @model_validator(mode="after")
+    @classmethod
+    def verify_names_and_shapes(cls, value):
+        """Ensure uniqueness of names and shape equality between columns"""
+        if len(value.columns) != len({name for name, _ in value.columns}):
+            raise ValueError("found columns with same name")
+
+        shape_iterator = iter((col.decode().shape for _, col in value.columns))
+        try:
+            first_shape = next(shape_iterator)
+        except StopIteration:
+            return value
+        if not all(first_shape == shape for shape in shape_iterator):
+            raise ValueError("columns have different shapes")
+        return value
+
+
+class Stream(BaseModel):
+    type: Literal["stream"] = "stream"
+    stream: str
+
+    def decode(self):
+        raise TypeError("Stream model is just a reference, can't use decode()")
+
+
 DataItemType = Annotated[
     (
         Annotated[Inline, Tag("inline")]
         | Annotated[InlineBase64, Tag("inline_b64")]
         | Annotated[Stream, Tag("stream")]
         | Annotated[InlineRaw, Tag("raw")]
+        | Annotated[Compound, Tag("compound")]
     ),
     Discriminator(get_discriminator_value),
 ]

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/lima/client.py

@@ -5,16 +5,13 @@
 # Copyright (c) Beamline Control Unit, ESRF
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
 import math
+import logging
 import numpy as np
 from packaging.version import Version
 from functools import cached_property
-import logging
 from dataclasses import dataclass
 
-from blissdata.redis_engine.exceptions import (
-    IndexNoMoreThereError,
-    IndexNotYetThereError,
-)
+from blissdata.exceptions import IndexNoMoreThereError, IndexNotYetThereError
 from . import image_utils
 
 try:

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/lima/image_utils.py

@@ -5,8 +5,8 @@
 # Copyright (c) Beamline Control Unit, ESRF
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
 
-"""Utility functions to read Lima images from file or device server.
-"""
+"""Utility functions to read Lima images from file or device server."""
+
 import struct
 import numpy
 import typing

blissdata-2.4.0/src/blissdata/redis_engine/encoding/json.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+from blissdata.deprecation import warn_deprecated
+
+warn_deprecated(__name__, "blissdata.streams.encoding.json", "3")
+
+from blissdata.streams.encoding.json import JsonStreamEncoder  # noqa F401, E402

blissdata-2.4.0/src/blissdata/redis_engine/encoding/numeric.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+from blissdata.deprecation import warn_deprecated
+
+warn_deprecated(__name__, "blissdata.streams.encoding.numeric", "3")
+
+from blissdata.streams.encoding.numeric import NumericStreamEncoder  # noqa F401, E402

blissdata-2.4.0/src/blissdata/redis_engine/event_stream.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+from blissdata.deprecation import warn_deprecated
+
+warn_deprecated(__name__, "blissdata.streams.event_stream", "3")
+
+from blissdata.streams.event_stream import EventStream  # noqa F401, E402

blissdata-2.4.0/src/blissdata/redis_engine/exceptions.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+from blissdata.deprecation import warn_deprecated
+
+warn_deprecated(__name__, "blissdata.exceptions", "3")
+
+from blissdata.exceptions import *  # noqa F401, E402

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/redis_engine/memory_tracker.py

@@ -13,11 +13,8 @@ import redis
 from datetime import timedelta
 from dataclasses import dataclass
 
-from .store import DataStore
-from .scan import Scan
-from .scan import ScanState
-from .exceptions import NoScanAvailable, ScanNotFoundError, ScanLoadError
-
+from blissdata import DataStore, Scan, ScanState
+from blissdata.exceptions import NoScanAvailable, ScanNotFoundError, ScanLoadError
 
 memory_tracking_stream = "_MEMORY_TRACKING_"
 SEAL_ID = ("-".join([str(2**64 - 1)] * 2)).encode()
@@ -37,6 +34,7 @@ class NoFreeableMemoryError(Exception):
 class TrackedScan:
     scan: Scan
     last_activity: int
+    last_streams_ids: dict[str, int]
 
     def __eq__(self, other):
         return self.scan.key == other.scan.key
@@ -95,7 +93,7 @@ class MemoryTracker:
                     scan._model.state_stream.key(), count=1
                 )
                 last_activity = int(raw[0][0].split(b"-")[0]) / 1000
-                self._tracked_scans.add(TrackedScan(scan, last_activity))
+                self._tracked_scans.add(TrackedScan(scan, last_activity, {}))
             except ScanNotFoundError:
                 # scan already deleted from Redis by user, skip it
                 pass
@@ -115,7 +113,7 @@ class MemoryTracker:
                 try:
                     scan = self._data_store.load_scan(scan_key)
                     last_activity = int(since.split("-")[0]) / 1000
-                    self._tracked_scans.add(TrackedScan(scan, last_activity))
+                    self._tracked_scans.add(TrackedScan(scan, last_activity, {}))
                     self._since = since
                     logging.info(f"Start tracking {scan.key}")
                 except ScanNotFoundError:
@@ -137,7 +135,14 @@ class MemoryTracker:
 
             if track.scan.state >= ScanState.PREPARED:
                 streams_ids = self._data_streams_ids(track.scan)
-                if streams_ids:
+                stream_activity = {
+                    key: idx
+                    for key, idx in streams_ids.items()
+                    if key not in track.last_streams_ids
+                    or track.last_streams_ids[key] != idx
+                }
+                if stream_activity:
+                    track.last_streams_ids = streams_ids
                     track.last_activity, _ = self._data_store._redis.time()
                 all_data_streams_ids.update(streams_ids)
 

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/redis_engine/scan.py

@@ -11,21 +11,19 @@ import math
 import logging
 import numpy as np
 from functools import wraps
-
 from pydantic import ValidationError
 from importlib.metadata import entry_points
 
-from .event_stream import EventStream
-from .exceptions import (
+from blissdata.streams.event_stream import EventStream
+from blissdata.exceptions import (
     UnauthorizeStateTransition,
     ScanNotFoundError,
     ScanValidationError,
     NoWritePermission,
 )
-from blissdata.streams.base import (
-    BaseStream,
+from blissdata.streams import BaseStream, StreamDefinition
+from blissdata.streams.default import (
     Stream,
-    StreamDefinition,
     BrokenStream,
     BrokenPluginStream,
     MissingPluginStream,

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/redis_engine/store.py

@@ -13,8 +13,8 @@ from redis.commands.search.query import Query
 from redis.commands.search.querystring import querystring, tags
 from redis_om import HashModel, JsonModel, Migrator
 
+from blissdata.exceptions import NoScanAvailable, ScanNotFoundError
 from .scan import Scan, scan_creation_stream, ScanState
-from .exceptions import NoScanAvailable, ScanNotFoundError
 from .identities import ESRFIdentityModel
 
 _PROTOCOL_KEY = "_PROTOCOL_VERSION_"
@@ -43,7 +43,7 @@ class DataStore:
         close_func = type(redis_pool).disconnect
         weakref.finalize(self, close_func, redis_pool)
 
-        class StreamModel(HashModel):
+        class StreamModel(HashModel, index=True):
             """Actual stream model to be embedded into ScanModel, but not exposed to user directly.
             They are wrapped into Stream objects instead."""
 
@@ -60,7 +60,7 @@ class DataStore:
             encoding: dict | str | None = None
             info: dict | str | None = None
 
-        class ScanModel(JsonModel):
+        class ScanModel(JsonModel, index=True):
             class Meta:
                 global_key_prefix = identity_model_cls._meta.global_key_prefix
                 model_key_prefix = "scan"

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/scan.py

@@ -4,4 +4,8 @@
 #
 # Copyright (c) Beamline Control Unit, ESRF
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
-from .redis_engine.scan import Scan  # noqa: F401
+from .deprecation import warn_deprecated
+
+warn_deprecated(__name__, "blissdata", "3")
+
+from . import Scan  # noqa: F401, E402

{blissdata-2.2.0 → blissdata-2.4.0}/src/blissdata/schemas/scan_info_display.py

@@ -7,7 +7,6 @@
 import typing
 import typing_extensions
 
-
 ChannelRef = str
 """Identify a channel from the `scan_info["channels"]` field"""
 

blissdata-2.2.0/src/blissdata/streams/base.py → blissdata-2.4.0/src/blissdata/streams/__init__.py

@@ -4,21 +4,21 @@
 #
 # Copyright (c) Beamline Control Unit, ESRF
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
-import numpy as np
 from abc import ABC, abstractmethod
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
-from blissdata.redis_engine.event_stream import EventStream
-from blissdata.redis_engine.exceptions import (
-    EndOfStream,
-    IndexNoMoreThereError,
-    UnknownEncodingError,
-    MissingPluginException,
-    EmptyViewException,
-)
-from blissdata.redis_engine.encoding import StreamEncoder
-from blissdata.redis_engine.encoding.numeric import NumericStreamEncoder
-from blissdata.redis_engine.encoding.json import JsonStreamEncoder
+from blissdata.exceptions import EndOfStream, EmptyViewException
+from .encoding import StreamEncoder
+from .event_stream import EventStream
+
+__all__ = [
+    "BaseStream",
+    "BaseView",
+    "Cursor",
+    "CursorGroup",
+    "EventRange",
+    "StreamDefinition",
+]
 
 
 @dataclass(slots=True)
@@ -174,151 +174,6 @@ class BaseView(ABC):
         pass
 
 
-class View(BaseView):
-    def __init__(self, event_range: EventRange):
-        self._events = event_range
-
-    @property
-    def index(self) -> int:
-        return self._events.index
-
-    def __len__(self) -> int:
-        return len(self._events)
-
-    def __iter__(self):
-        yield from self.get_data()
-
-    def get_data(self, start=None, stop=None):
-        trimmed_range = range(len(self))[start:stop]
-        offset = self._events.nb_expired
-        data_start = trimmed_range.start - offset
-        data_stop = trimmed_range.stop - offset
-        if data_start < 0:
-            raise IndexNoMoreThereError
-        else:
-            return self._events.data[data_start:data_stop]
-
-
-class Stream(BaseStream):
-    def __init__(self, event_stream):
-        super().__init__(event_stream)
-        if event_stream.encoding["type"] == "numeric":
-            self._kind = "array"
-        elif event_stream.encoding["type"] == "json":
-            self._kind = "json"
-        else:
-            raise UnknownEncodingError(
-                f"Unknow stream encoding {event_stream.encoding}"
-            )
-
-    @staticmethod
-    def make_definition(name, dtype, shape=None, info={}) -> StreamDefinition:
-        if dtype == "json":
-            if shape is None:
-                return StreamDefinition(name, info, JsonStreamEncoder())
-            else:
-                raise ValueError("JSON stream cannot have shape")
-        else:
-            if shape is None:
-                shape = ()
-            info = info.copy()
-            info["dtype"] = np.dtype(dtype).name
-            info["shape"] = shape
-            return StreamDefinition(name, info, NumericStreamEncoder(dtype, shape))
-
-    @property
-    def kind(self):
-        return self._kind
-
-    @property
-    def plugin(self):
-        None
-
-    @property
-    def dtype(self):
-        return self.event_stream.dtype
-
-    @property
-    def shape(self):
-        return self.event_stream.shape
-
-    def __len__(self):
-        return len(self._event_stream)
-
-    def __getitem__(self, key):
-        return self._event_stream[key]
-
-    def _need_last_only(self, last_only):
-        return last_only
-
-    def _build_view_from_events(self, index, events: EventRange, last_only):
-        # NOTE event_range is never empty
-        return View(events)
-
-
-class BrokenStream(BaseStream):
-    def __init__(self, event_stream, plugin_name, exception, kind):
-        self._event_stream = event_stream
-        self._plugin_name = plugin_name
-        self._exception = exception
-        self._kind = kind
-
-    @property
-    def kind(self):
-        self._kind
-
-    @staticmethod
-    def make_definition(*args, **kwargs) -> StreamDefinition:
-        raise NotImplementedError
-
-    @property
-    def plugin(self):
-        return self._plugin_name
-
-    @property
-    def dtype(self):
-        raise self._exception
-
-    @property
-    def shape(self):
-        raise self._exception
-
-    def __len__(self):
-        raise self._exception
-
-    def __getitem__(self, key):
-        raise self._exception
-
-    def _need_last_only(self, last_only):
-        raise self._exception
-
-    def _build_view_from_events(
-        self, hl_index: int, events: EventRange, last_only: bool
-    ):
-        raise self._exception
-
-
-class BrokenPluginStream(BrokenStream):
-    """A stream class instanciated in place of a plugin, if it can't be
-    instantiated. A warning is issued, but the scan object is not affected,
-    ensuring memory tracker robustness against plugins."""
-
-    def __init__(self, event_stream, plugin_name, exception):
-        super().__init__(event_stream, plugin_name, exception, "broken_plugin")
-
-
-class MissingPluginStream(BrokenStream):
-    """A stream class instanciated in place of a plugin if not installed."""
-
-    def __init__(self, event_stream, plugin_name):
-        super().__init__(
-            event_stream,
-            plugin_name,
-            MissingPluginException(event_stream.name, plugin_name),
-            "missing_plugin",
-        )
-
-
 class Cursor:
     def __init__(self, stream):
         # low level stream (multiplexable events from redis)