blissdata 2.1.0__tar.gz → 2.2.0__tar.gz

{blissdata-2.1.0/src/blissdata.egg-info → blissdata-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: blissdata
-Version: 2.1.0
+Version: 2.2.0
 Summary: Bliss data streaming client
 Maintainer: BCU (ESRF)
 License-Expression: LGPL-3.0-or-later

{blissdata-2.1.0 → blissdata-2.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "blissdata"
-version = "2.1.0"
+version = "2.2.0"
 description = "Bliss data streaming client"
 readme = "README.md"
 license = "LGPL-3.0-or-later"

blissdata-2.2.0/src/blissdata/h5api/h5_scan_map.py

@@ -0,0 +1,133 @@
+import time
+import logging
+from collections.abc import Mapping
+from pathlib import PurePosixPath as Path
+from blissdata.redis_engine.scan import ScanState
+from blissdata.h5map import HDF5ItemType, Group
+
+_logger = logging.getLogger(__name__)
+
+
+class H5ScanMap(Mapping):
+    """H5ScanMap is a wrapper around a blissdata's Scan. It exposes its HDF5
+    mapping with conventional dict API, but also handles automatic refresh of
+    the underlying model from Redis (see _updated_root method).
+
+    IMPORTANT: This is not an H5-like object, it helps manipulating the mapping
+    within a scan. Then H5-like groups and datasets can be implemented on top of
+    it."""
+
+    def __init__(self, scan):
+        self._scan = scan
+        self._root = None
+        self._last_update = time.perf_counter()
+
+    @property
+    def scan(self):
+        return self._scan
+
+    def __len__(self):
+        return len(self._updated_root().children)
+
+    def __iter__(self):
+        yield from self._updated_root().children.keys()
+
+    def __getitem__(self, path: str) -> HDF5ItemType:
+        """Return the HDF5Item associated to that path.
+        Links are not resolved here, instead a SoftLink or ExternalLink is
+        returned and it is caller choice to resolve it or not.
+        """
+        assert path.startswith("/")
+        h5_item = self._updated_root()
+        current_path = Path("/")
+        for part in Path(path).parts:
+            if part == "/":
+                continue
+            current_path /= part
+            if not isinstance(h5_item, Group):
+                raise KeyError(str(current_path))
+            try:
+                h5_item = h5_item.children[part]
+            except KeyError:
+                raise KeyError(str(current_path))
+        return h5_item
+
+    def path_partition(self, path: str) -> tuple[str, HDF5ItemType | None, str]:
+        """Similar to __getitem__, but may return partially resolved path.
+        Goal is to allow link items to be returned along with the remaining
+        part of the path. Links can be resolved before going further.
+
+        Returns:
+            tuple:
+                - path of the returned item
+                - h5map item or None if path doesn't exist
+                - remaining path (empty if item is the actual target)
+
+        Example:
+            /
+            ├─a
+            │ ├─b (link to d)
+            │ └─c
+            └─d
+              └─e
+
+           path_partition("/a/b/c") -> "/a/b/c", Dataset(...), ""
+           path_partition("/a/b/e") -> "/a/b", SoftLink("/d"), "e"
+           path_partition("/a/x/y") -> "/a/x", None, "y"
+        """
+        assert path.startswith("/")
+        h5_item = self._updated_root()
+        item_parts = ["/"]
+        remaining_parts = list(Path(path).parts[1:])
+        while remaining_parts:
+            if isinstance(h5_item, Group):
+                try:
+                    item_parts.append(remaining_parts.pop(0))
+                    h5_item = h5_item.children[item_parts[-1]]
+                except KeyError:
+                    h5_item = None
+                    break
+            else:
+                break
+        remaining_path = str(Path(*remaining_parts)) if remaining_parts else ""
+        return str(Path(*item_parts)), h5_item, remaining_path
+
+    def _updated_root(self) -> Group:
+        """Return an h5map.Group loaded from scan info. Reuse previous answer if
+        not older than 0.2 seconds (polling Redis to 5Hz at most if fine).
+        Moreover, scan state doesn't change that much and the scan's json is
+        only downloaded on changes.
+
+        In case the scan provides no mapping, an empty Group is returned.
+        """
+        scan_state_changed = False
+
+        # Mapping is not expected to be published before scan is prepared, wait
+        while self._scan.state < ScanState.PREPARED:
+            self._scan.update()
+            self._last_update = time.perf_counter()
+            scan_state_changed = True
+
+        # Update whenever the scan is not CLOSED and the last_update is not too
+        # recent.
+        min_update_interval = 0.2
+        now = time.perf_counter()
+        if (
+            self._last_update + min_update_interval < now
+            and self._scan.state < ScanState.CLOSED
+        ):
+            scan_state_changed = self._scan.update(block=False)
+            self._last_update = time.perf_counter()
+
+        # (re)generate mapping tree if necessary
+        if scan_state_changed or self._root is None:
+            try:
+                mappings = self._scan.info.get("h5maps", {})
+                self._root = Group.model_validate(mappings[self._scan.path])
+            except KeyError:
+                _logger.warning(
+                    f"Scan {self._scan.number} has no hdf5 mapping ({self._scan.key})"
+                )
+                return Group()
+
+        return self._root

{blissdata-2.1.0 → blissdata-2.2.0}/src/blissdata/h5api/redis_hdf5.py

@@ -1,20 +1,17 @@
 import re
 import time
+import math
 import functools
 import numpy as np
+from abc import ABC, abstractmethod
 from pathlib import PurePosixPath as Path
 
+import blissdata.h5map as h5m
 from blissdata.beacon.data import BeaconData
 from blissdata.redis_engine.store import DataStore
-from blissdata.h5api import abstract
-from blissdata.h5api.scan_mapping import (
-    MappingNode,
-    GroupNode,
-    DatasetNode,
-    ScanMapper,
-    SoftLinkNode,
-)
 from blissdata.redis_engine.exceptions import EndOfStream, IndexNotYetThereError
+from blissdata.h5api import abstract
+from blissdata.h5api.h5_scan_map import H5ScanMap
 
 
 class Attributes(abstract.Attributes):
@@ -41,9 +38,13 @@ class File(abstract.File):
     file can iterate on live data.
     """
 
-    def __init__(self, filepath: str, data_store: DataStore | None = None):
+    def __init__(
+        self, filepath: str, mode: str = "r", data_store: DataStore | None = None
+    ):
         """If no data_store is provided, but $BEACON_HOST variable is defined,
         then try to query Redis address from that beacon server."""
+        if mode != "r":
+            raise ValueError("Invalid mode; only r is supported with live mode")
         self._filepath = Path(filepath)
         self._closed = False
         if data_store is None:
@@ -81,58 +82,67 @@ class File(abstract.File):
         self._closed = True
 
     def __repr__(self):
-        return f'<{self.__module__}.{type(self).__name__} "{self._filepath.name}">'
+        return f'<Live-HDF5 file "{self._filepath.name}" (mode r)>'
 
     def __len__(self):
         """Return the instant length from the scans that are already there."""
         return len(self.keys())
 
     def __getitem__(self, path: str):
-        scan_path, sub_path = self._split_path(path)
-        path = Path("/") / path
-        if scan_path == "/":
-            return Group(self, path, self.attrs)
-
-        try:
-            scan_mapper = self._get_scan_mapper_by_path(scan_path)
-            node = scan_mapper[str(path)]
-        except KeyError:
-            raise KeyError(f"No such path: {path}")
+        # save originally requested path
+        original_path = path
 
+        # track visited paths to detect link cycles
         visited_paths = []
-        while isinstance(node, SoftLinkNode):
-            visited_paths.append(path)
-            path = Path(node.path).parent / node.target
-            if path in visited_paths:
+        while path:
+            path = Path("/") / path
+            if str(path) in visited_paths:
                 raise KeyError(
-                    f"Link can't be resolved, found cyclic links: {[str(p) for p in visited_paths]}"
+                    f"'{original_path}': path can't be resolved, found link cycle: {[str(p) for p in visited_paths]}"
                 )
+            else:
+                visited_paths.append(str(path))
+
+            scan_path = self._scan_path(path)
+            if scan_path == "/":
+                return Group(self, "/", self.attrs)
+
             try:
-                node = scan_mapper[str(path)]
+                scan_map = self._get_scan_map_by_path(scan_path)
             except KeyError:
-                raise KeyError(f"Broken link, no such path: {path}")
-
-        realpath = node.path  # if softlink where encountered
-        return self._mapping_node_to_h5(realpath, node, scan_mapper.scan)
-
-    def _mapping_node_to_h5(self, path: str, node: MappingNode, scan):
-        if isinstance(node, GroupNode):
-            return Group(self, str(path), node.attrs)
-        elif isinstance(node, DatasetNode):
-            if node.value is not None:
-                return StaticDataset(
-                    self, str(path), attrs=node.attrs, value=node.value
-                )
-            elif node.stream is not None:
-                return StreamDataset(
-                    self, str(path), attrs=node.attrs, stream=scan.streams[node.stream]
+                raise KeyError(f"No such path: '{scan_path}'")
+
+            item_path, item, remain_path = scan_map.path_partition(str(path))
+            if item is None:
+                raise KeyError(f"No such path: '{item_path}'")
+
+            if isinstance(item, h5m.SoftLink):
+                path = Path(item_path).parent / item.target_path / remain_path
+            elif isinstance(item, h5m.ExternalLink):
+                # NOTE no protection against link cycles involving multiple files
+                return File(item.target_file, data_store=self._data_store)[
+                    str(Path(item.target_path) / remain_path)
+                ]
+            elif remain_path:
+                raise KeyError(
+                    f"'{item_path}' is not a Group, can't reach '{str(path)}'"
                 )
             else:
-                raise TypeError(f"Unknown dataset type {node}")
-        elif isinstance(node, SoftLinkNode):
-            raise TypeError("Oops, unresolved SoftLinkNode should not end up there")
+                return self._map_item_to_h5(str(path), item, scan_map.scan)
+
+    def _map_item_to_h5(self, path: str, item: h5m.HDF5ItemType, scan):
+        if isinstance(item, h5m.Group):
+            return Group(self, path, item.attributes)
+        elif isinstance(
+            item, (h5m.Dataset, h5m.ExternalBinaryDataset, h5m.VirtualDataset)
+        ):
+            return dataset_factory(self, path, item, scan)
+        elif isinstance(item, h5m.SoftLink):
+            raise TypeError("Oops, unresolved h5m.SoftLink should not end up there")
+        elif isinstance(item, h5m.ExternalLink):
+            raise TypeError("Oops, unresolved h5m.ExternalLink should not end up there")
         else:
-            raise TypeError(f"Unknown node type {type(node).__name__}")
+            raise TypeError(f"Unknown item type {type(item).__name__}")
 
     def __iter__(self) -> str:
         """Iterate forever: keep waiting for new scans once existing ones are
@@ -140,7 +150,7 @@ class File(abstract.File):
         # iterate through existing scans in that file_path
         ts, keys = self._data_store.search_existing_scans(path=self.filename)
         for key in keys:
-            yield from self._get_scan_mapper_by_key(key)
+            yield from self._get_scan_map_by_key(key)
 
         # iterate forever over the next scans
         while True:
@@ -150,7 +160,7 @@ class File(abstract.File):
             # outside of blissdata (a proper API should be discussed if needed)
             path = self._data_store._redis.json().get(key, "id.path")
             if path == self.filename:
-                yield from self._get_scan_mapper_by_key(key)
+                yield from self._get_scan_map_by_key(key)
 
     def keys(self):
         """Instant list of keys in the file. Can be used to only iterate over
@@ -158,12 +168,12 @@ class File(abstract.File):
         _, keys = self._data_store.search_existing_scans(path=self.filename)
         ret = set()
         for key in keys:
-            ret |= set(self._get_scan_mapper_by_key(key).keys())
+            ret |= set(self._get_scan_map_by_key(key).keys())
         return ret
 
-    def _get_scan_mapper_by_path(self, scan_path: str) -> ScanMapper:
+    def _get_scan_map_by_path(self, scan_path: str) -> H5ScanMap:
         key = self._get_scan_key(scan_path)
-        return self._get_scan_mapper_by_key(key)
+        return self._get_scan_map_by_key(key)
 
     def _get_scan_key(self, scan_path: str) -> str:
         return File._get_scan_key_cached(self._data_store, self.filename, scan_path)
@@ -187,60 +197,60 @@ class File(abstract.File):
                 )
         return keys.pop()
 
-    def _get_scan_mapper_by_key(self, key: str) -> ScanMapper:
-        """Load a scan from Redis, wrap it into a ScanMapper and update
+    def _get_scan_map_by_key(self, key: str) -> H5ScanMap:
+        """Load a scan from Redis, wrap it into a H5ScanMap and update
         cache"""
-        return File._get_scan_mapper_by_key_cached(self._data_store, key)
+        return File._get_scan_map_by_key_cached(self._data_store, key)
 
     @staticmethod
     @functools.lru_cache(maxsize=100)
-    def _get_scan_mapper_by_key_cached(data_store: DataStore, key: str) -> ScanMapper:
+    def _get_scan_map_by_key_cached(data_store: DataStore, key: str) -> H5ScanMap:
         scan = data_store.load_scan(key)
-        return ScanMapper(scan)
+        return H5ScanMap(scan)
 
     def _len_group(self, path: str) -> int:
         """Used by Group.__len__ to query length of a particular path (not only
         the root)"""
-        scan_path, sub_path = self._split_path(path)
+        scan_path = self._scan_path(path)
         if scan_path == "/":
             return len(self)
         else:
             # number of items inside a scan
             try:
-                scan_mapper = self._get_scan_mapper_by_path(scan_path)
-                group_node = scan_mapper[path]
+                scan_map = self._get_scan_map_by_path(scan_path)
+                group_item = scan_map[path]
             except KeyError as e:
                 raise KeyError(f"No such path: {str(scan_path / e.args[0])}")
-            return len(group_node)
+            return len(group_item.children)
 
     def _iter_group(self, path: str) -> str:
         """Used by Group.__iter__ to iterate through a particular path (not
         only the root)."""
-        scan_path, sub_path = self._split_path(path)
+        scan_path = self._scan_path(path)
         if scan_path == "/":
             # iterate over scans (never stops)
             yield from self
         else:
             # iterate inside a scan
             try:
-                scan_mapper = self._get_scan_mapper_by_path(scan_path)
-                group_node = scan_mapper[path]
+                scan_map = self._get_scan_map_by_path(scan_path)
+                group_item = scan_map[path]
             except KeyError as e:
                 raise KeyError(f"No such path: {str(scan_path / e.args[0])}")
-            yield from group_node
-
-    def _split_path(self, path: str) -> tuple[str, str]:
-        """Make path canonical and split it into scan and subscan levels
-        ''             -> '/',      ''
-        '123.1'        -> '/123.1', ''
-        '//123.1///'   -> '/123.1', ''
-        '/123.1/a/b/c' -> '/123.1', 'a/b/c'
+            yield from group_item.children
+
+    def _scan_path(self, path: str) -> str:
+        """Make path canonical and keep scan part only
+        ''             -> '/',
+        '123.1'        -> '/123.1',
+        '//123.1///'   -> '/123.1',
+        '/123.1/a/b/c' -> '/123.1',
         """
         path = Path("/") / path
         if len(path.parts) <= 2:
-            return str(path), ""
+            return str(path)
         else:
-            return str(Path(*path.parts[:2])), str(Path(*path.parts[2:]))
+            return str(Path(*path.parts[:2]))
 
 
 class Node(abstract.Node):
@@ -287,7 +297,11 @@ class Group(Node, abstract.Group):
         return self._path
 
     def __repr__(self):
-        return f'<{self.__module__}.{type(self).__name__} "{self._path}">'
+        try:
+            length = len(self)
+        except Exception:
+            length = "?"
+        return f'<Live-HDF5 group "{self._path}" ({length} members)>'
 
     def __getitem__(self, path: str):
         try:
@@ -304,48 +318,105 @@ class Group(Node, abstract.Group):
         return self._file._len_group(self._path)
 
 
-class StaticDataset(Node, abstract.Dataset):
-    def __init__(self, file: File, path: str, value, attrs: dict):
+class Dataset(Node, abstract.Dataset):
+    def __init__(
+        self,
+        file: File,
+        path: str,
+        attrs: dict,
+        array: ...,
+        virtual: bool = False,
+        external=None,
+    ):
         Node.__init__(self, file, path, attrs)
-        self._value = value
+        self._array = array
+        self._virtual = virtual
+        self._external = external
 
     def __repr__(self):
-        return f'<{self.__module__}.{type(self).__name__} "{self._value}">'
+        if isinstance(self._array, StreamAsArray):
+            shape = f"(_,{','.join([str(i) for i in self._array._stream.shape])})"
+        else:
+            shape = self.shape
+        return f'<Live-HDF5 dataset "{Path(self.name).name}": shape {shape}, type "{self.dtype.str}">'
+
+    @property
+    def is_virtual(self):
+        return self._virtual
+
+    @property
+    def external(self):
+        return self._external
 
     def __getitem__(self, idx):
-        if idx == ():
-            return self._value
-        return self._value[idx]
+        if self.ndim == 0 and idx != ():
+            raise ValueError("Illegal slicing argument for scalar dataspace")
+        return self._array[idx]
 
     def __len__(self):
-        return len(self._value)
+        if self.ndim == 0:
+            raise TypeError("Attempt to take len() of scalar dataset")
+        return len(self._array)
 
     @property
     def dtype(self):
-        raise NotImplementedError
+        return self._array.dtype
 
     @property
     def shape(self):
-        raise NotImplementedError
+        return self._array.shape
+
+    @property
+    def ndim(self):
+        return self._array.ndim
 
     @property
     def size(self):
-        raise NotImplementedError
+        return self._array.size
 
     def __iter__(self):
-        # TODO if scalar:
-        #     raise TypeError("Can't iterate over a scalar dataset")
-        yield from self._value  # TODO only if value is iterable
+        if self.ndim == 0:
+            raise TypeError("Can't iterate over a scalar dataset")
+        yield from self._array
+
+
+class ArrayInterface(ABC):
+    @abstractmethod
+    def __getitem__(self, idx):
+        pass
+
+    @abstractmethod
+    def __len__(self):
+        pass
+
+    @property
+    @abstractmethod
+    def dtype(self):
+        pass
 
     @property
+    @abstractmethod
+    def shape(self):
+        pass
+
+    @property
+    @abstractmethod
     def ndim(self):
-        raise NotImplementedError
+        pass
+
+    @property
+    @abstractmethod
+    def size(self):
+        pass
 
+    @abstractmethod
+    def __iter__(self):
+        pass
 
-class StreamDataset(Node, abstract.Dataset):
-    def __init__(self, file: File, path: str, stream, attrs: dict):
+
+class StreamAsArray(ArrayInterface):
+    def __init__(self, stream):
         assert stream.kind == "array"
-        Node.__init__(self, file, path, attrs)
         self._stream = stream
 
     def __getitem__(self, idx):
@@ -384,22 +455,90 @@ class StreamDataset(Node, abstract.Dataset):
     def shape(self):
         return (len(self),) + self._stream.shape
 
+    @property
+    def ndim(self):
+        return len(self._stream.shape) + 1
+
     @property
     def size(self):
-        return np.prod(self.shape)
+        return math.prod(self.shape)
 
     def __iter__(self):
         cursor = self._stream.cursor()
-        timeout = 1.0
         while True:
             try:
-                view = cursor.read(timeout=timeout)
-                yield from view.get_data()
-                # TODO views should be iterable to choose on their own what's
-                # the best batch size to download (e.g. LimaView)
+                view = cursor.read()
+                yield from view
             except EndOfStream:
                 break
 
+
+class ExternalBinaryAsArray:
+    def __init__(self, item):
+        self.memmaps = []
+        self._shape = tuple(item.shape)
+        self._dtype = np.dtype(item.dtype)
+        for file in item.files:
+            if file.size is None:
+                shape = None
+            else:
+                shape = (file.size // self._dtype.itemsize,)
+            self.memmaps.append(
+                np.memmap(
+                    file.name,
+                    dtype=self._dtype,
+                    mode="r",
+                    offset=file.offset,
+                    shape=shape,
+                )
+            )
+
+        # NOTE naive approach loading files in memory, but memmaps could be
+        # accessed on demand
+        self._data = np.concatenate([m for m in self.memmaps])[: self.size].reshape(
+            self.shape
+        )
+
+    def __getitem__(self, key):
+        return self._data[key]
+
+    def __len__(self):
+        return self._shape[0]
+
+    @property
+    def dtype(self):
+        return self._dtype
+
+    @property
+    def shape(self):
+        return self._shape
+
     @property
     def ndim(self):
-        return len(self.shape)
+        return len(self._shape)
+
+    @property
+    def size(self):
+        return math.prod(self._shape)
+
+    def __iter__(self):
+        yield from self._data
+
+
+def dataset_factory(parent: File, path: str, item: h5m.HDF5ItemType, scan) -> Dataset:
+    virtual = False
+    external = None
+    if isinstance(item, h5m.Dataset):
+        if isinstance(item.value, h5m.Stream):
+            array = StreamAsArray(scan.streams[item.value.stream])
+        else:
+            array = item.value.decode()
+    elif isinstance(item, h5m.ExternalBinaryDataset):
+        external = [(file.name, file.offset, file.size) for file in item.files]
+        array = ExternalBinaryAsArray(item)
+    elif isinstance(item, h5m.VirtualDataset):
+        virtual = True
+        # TODO
+        raise NotImplementedError
+
+    return Dataset(parent, path, item, array, virtual, external)

{blissdata-2.1.0 → blissdata-2.2.0}/src/blissdata/h5map/data_items.py

@@ -64,7 +64,7 @@ class InlineRaw(RootModel):
     def decode(self):
         value = self.model_dump()
         if value.dtype.kind in "UT":
-            return value.tolist()
+            return np.vectorize(lambda x: x.encode(), otypes="O")(value)
         else:
             return value