blissdata 0.3.4__py3-none-any.whl

blissdata/h5api/abstract.py

@@ -0,0 +1,97 @@
+"""Definition of the h5py-like Bliss Data API"""
+
+from collections import abc
+from abc import abstractmethod, abstractproperty
+from typing import Iterator, Optional, Tuple
+from numpy.typing import DTypeLike
+from .utils import types
+
+
+class Attributes(abc.Mapping):
+    """Node attributes."""
+
+
+class Node:
+    """Node in the data tree."""
+
+    @abstractproperty
+    def name(self) -> str:
+        pass
+
+    @abstractproperty
+    def short_name(self) -> str:
+        pass
+
+    @abstractproperty
+    def attrs(self) -> Attributes:
+        pass
+
+    @abstractproperty
+    def parent(self) -> "Group":
+        pass
+
+    @abstractproperty
+    def file(self) -> "File":
+        pass
+
+
+class Group(Node, abc.Mapping):
+    """Node in the data tree which contains other nodes."""
+
+    @abstractmethod
+    def __getitem__(self, key: str) -> Node:
+        pass
+
+    @abstractmethod
+    def __iter__(self) -> Iterator[Node]:
+        pass
+
+    @abstractmethod
+    def __len__(self) -> int:
+        pass
+
+
+class Dataset(Node, abc.Sequence):
+    """Node in the data tree which contains data."""
+
+    @abstractmethod
+    def __getitem__(self, idx: types.DataIndexType) -> types.DataType:
+        pass
+
+    @abstractmethod
+    def __len__(self) -> int:
+        pass
+
+    @abstractproperty
+    def dtype(self) -> DTypeLike:
+        pass
+
+    @abstractproperty
+    def shape(self) -> Tuple[int]:
+        pass
+
+    @abstractproperty
+    def size(self) -> int:
+        pass
+
+    @abstractproperty
+    def ndim(self) -> int:
+        pass
+
+
+class File(Group):
+    """Root node in the data tree."""
+
+    def __enter__(self) -> "File":
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback) -> Optional[bool]:
+        self.close()
+
+    @abstractmethod
+    def close(self) -> None:
+        pass
+
+    @property
+    def parent(self) -> None:
+        return None

blissdata/h5api/dynamic_hdf5.py

@@ -0,0 +1,153 @@
+"""Implementation of the h5py-like Bliss Data API with dynamic HDF5 files"""
+
+from typing import Any, Iterator, Tuple, Type
+from numpy.typing import DTypeLike
+import h5py
+from . import abstract
+from .utils import hdf5
+from .utils import bliss
+from .utils import types
+from .file_arguments import ignore_file_arguments
+
+
+class Attributes(abstract.Attributes):
+    """Node attributes."""
+
+    def __init__(self, retry_handler: hdf5.DynamicHDF5Handler, name: str) -> None:
+        self.__retry_handler = retry_handler
+        self.__name = name
+        super().__init__()
+
+    def __getitem__(self, key: str) -> Any:
+        return self.__retry_handler.get_attr(self.__name, key)
+
+    def __iter__(self) -> Iterator[Any]:
+        yield from self.__retry_handler.iter_attrs(self.__name)
+
+    def __len__(self) -> int:
+        return self.__retry_handler.len_attrs(self.__name)
+
+
+class Node:
+    """Node in the data tree."""
+
+    def __init__(self, retry_handler: hdf5.DynamicHDF5Handler, name: str) -> None:
+        self._retry_handler = retry_handler
+        if not name:
+            name = hdf5.SEP
+        self.__name = name
+        self.__attrs = None
+        self.__parent = None
+        self.__file = None
+        super().__init__()
+
+    @property
+    def name(self) -> str:
+        return self.__name
+
+    @property
+    def short_name(self) -> str:
+        return self.name.rpartition(hdf5.SEP)[-1]
+
+    @property
+    def parent(self) -> "Group":
+        if self.__parent is None:
+            parent_name = self.name.rpartition(hdf5.SEP)[0]
+            self.__parent = Group(self._retry_handler, parent_name)
+        return self.__parent
+
+    @property
+    def file(self) -> "File":
+        if self.__file is None:
+            self.__file = File(self._retry_handler)
+        return self.__file
+
+    @property
+    def attrs(self) -> Attributes:
+        if self.__attrs is None:
+            self.__attrs = Attributes(self._retry_handler, self.name)
+        return self.__attrs
+
+
+class Group(Node, abstract.Group):
+    """Node in the data tree which contains other nodes."""
+
+    def __getitem__(self, key: str) -> Node:
+        if not key.startswith(hdf5.SEP):
+            key = f"{self.name}{hdf5.SEP}{key}"
+        h5py_obj = self._retry_handler.get_item(key)
+        if isinstance(h5py_obj, h5py.Group):
+            return Group(self._retry_handler, h5py_obj.name)
+        else:
+            return Dataset(self._retry_handler, h5py_obj.name)
+
+    def __iter__(self) -> Iterator[Node]:
+        yield from self._retry_handler.iter_item(self.name)
+
+    def __len__(self) -> int:
+        return self._retry_handler.len_item(self.name)
+
+
+class Dataset(Node, abstract.Dataset):
+    """Node in the data tree which contains data."""
+
+    def __getitem__(self, idx: types.DataIndexType) -> types.DataType:
+        return self._retry_handler.slice_dataset(self.name, idx)
+
+    def __iter__(self) -> Iterator[types.DataType]:
+        yield from self._retry_handler.iter_item(self.name)
+
+    def __len__(self) -> int:
+        return self._retry_handler.len_item(self.name)
+
+    @property
+    def dtype(self) -> DTypeLike:
+        return self._retry_handler.getattr_item(self.name, "dtype")
+
+    @property
+    def shape(self) -> Tuple[int]:
+        return self._retry_handler.getattr_item(self.name, "shape")
+
+    @property
+    def size(self) -> int:
+        return self._retry_handler.getattr_item(self.name, "size")
+
+    @property
+    def ndim(self) -> int:
+        return self._retry_handler.getattr_item(self.name, "ndim")
+
+
+class File(Group, abstract.File):
+    """Root node in the data tree."""
+
+    def __init__(
+        self,
+        file: str,
+        hdf5_retry_handler: Type[
+            hdf5.DynamicHDF5Handler
+        ] = bliss.BlissDynamicHDF5Handler,
+        **openargs,
+    ) -> None:
+        if isinstance(file, str):
+            openargs = {
+                k: v
+                for k, v in openargs.items()
+                if k not in ignore_file_arguments("dynamic_hdf5")
+            }
+            file = hdf5_retry_handler(file, **openargs)
+        super().__init__(file, hdf5.SEP)
+
+    def close(self) -> None:
+        self._retry_handler.close()
+
+    @property
+    def parent(self) -> None:
+        return None
+
+    @property
+    def file(self) -> "File":
+        return self
+
+    @property
+    def short_name(self) -> str:
+        return self.name

blissdata/h5api/file_arguments.py

@@ -0,0 +1,28 @@
+"""The `File` class in each context has specific arguments other than the `h5py.File`
+arguments which are specific for that context (static HDF5, dynamic HDF5, ...).
+"""
+from typing import Set
+
+
+_CONTEXT_SPECIFIC_KEYS = {
+    "h5py": {"mode"},
+    "static_hdf5": set(),
+    "dynamic_hdf5": {
+        "hdf5_retry_handler",
+        "retry_timeout",
+        "retry_period",
+        "lima_names",
+        "instrument_name",
+    },
+}
+
+
+def ignore_file_arguments(context: str) -> Set[str]:
+    """Returns all keys with that should be ignore when instantiating the `File`
+    in a specific context"""
+    ignore = set()
+    for key, value in _CONTEXT_SPECIFIC_KEYS.items():
+        if key == context:
+            continue
+        ignore |= value
+    return ignore

blissdata/h5api/static_hdf5.py

@@ -0,0 +1,139 @@
+"""Implementation of the h5py-like Bliss Data API with static HDF5 files"""
+
+from typing import Any, Iterator, Tuple
+from numpy.typing import DTypeLike
+import h5py
+from silx.io import h5py_utils
+from . import abstract
+from .utils import hdf5
+from .utils import types
+from .file_arguments import ignore_file_arguments
+
+
+class Attributes(abstract.Attributes):
+    """Node attributes."""
+
+    def __init__(self, attrs: h5py.AttributeManager) -> None:
+        self.__attrs = attrs
+        super().__init__()
+
+    def __getitem__(self, key: str) -> Any:
+        return self.__attrs[key]
+
+    def __iter__(self) -> Iterator[Any]:
+        yield from self.__attrs
+
+    def __len__(self) -> int:
+        return len(self.__attrs)
+
+
+class Node:
+    """Node in the data tree."""
+
+    def __init__(self, h5py_obj: hdf5.HDF5Item) -> None:
+        self._h5py_obj = h5py_obj
+        self.__attrs = None
+        self.__parent = None
+        self.__file = None
+        super().__init__()
+
+    @property
+    def name(self) -> str:
+        return self._h5py_obj.name
+
+    @property
+    def short_name(self) -> str:
+        return self.name.rpartition(hdf5.SEP)[-1]
+
+    @property
+    def parent(self) -> "Group":
+        if self.__parent is None:
+            self.__parent = Group(self._h5py_obj.parent)
+        return self.__parent
+
+    @property
+    def file(self) -> "File":
+        if self.__file is None:
+            self.__file = File(self._h5py_obj.file)
+        return self.__file
+
+    @property
+    def attrs(self) -> Attributes:
+        if self.__attrs is None:
+            self.__attrs = Attributes(self._h5py_obj.attrs)
+        return self.__attrs
+
+
+class Group(Node, abstract.Group):
+    """Node in the data tree which contains other nodes."""
+
+    def __getitem__(self, key: str) -> Node:
+        h5py_obj = self._h5py_obj[key]
+        if isinstance(h5py_obj, h5py.Group):
+            return Group(h5py_obj)
+        else:
+            return Dataset(h5py_obj)
+
+    def __iter__(self) -> Iterator[Node]:
+        yield from self._h5py_obj
+
+    def __len__(self) -> int:
+        return len(self._h5py_obj)
+
+
+class Dataset(Node, abstract.Dataset):
+    """Node in the data tree which contains data."""
+
+    def __getitem__(self, idx: types.DataIndexType) -> types.DataType:
+        return self._h5py_obj[idx]
+
+    def __iter__(self) -> Iterator[types.DataType]:
+        yield from self._h5py_obj
+
+    def __len__(self) -> int:
+        return len(self._h5py_obj)
+
+    @property
+    def dtype(self) -> DTypeLike:
+        return self._h5py_obj.dtype
+
+    @property
+    def shape(self) -> Tuple[int]:
+        return self._h5py_obj.shape
+
+    @property
+    def size(self) -> int:
+        return self._h5py_obj.size
+
+    @property
+    def ndim(self) -> int:
+        return self._h5py_obj.ndim
+
+
+class File(Group, abstract.File):
+    """Root node in the data tree."""
+
+    def __init__(self, file: str, **openargs) -> None:
+        if isinstance(file, str):
+            openargs = {
+                k: v
+                for k, v in openargs.items()
+                if k not in ignore_file_arguments("dynamic_hdf5")
+            }
+            file = h5py_utils.File(file, **openargs)
+        super().__init__(file)
+
+    def close(self) -> None:
+        self._h5py_obj.close()
+
+    @property
+    def parent(self) -> None:
+        return None
+
+    @property
+    def file(self) -> "File":
+        return self
+
+    @property
+    def short_name(self) -> str:
+        return self.name

blissdata/h5api/utils/bliss.py

@@ -0,0 +1,138 @@
+import os
+import re
+from typing import Optional, Sequence
+
+import h5py
+
+from . import lima
+from . import hdf5
+from .hdf5_retry import RetryError
+
+
+class BlissDynamicHDF5Handler(hdf5.DynamicHDF5Handler):
+    """Each NXentry has a writer status which can be
+    STARTING, RUNNING, SUCCEEDED or FAILED. This status
+    is used as stop criterium for iteration.
+    """
+
+    def __init__(
+        self,
+        *args,
+        lima_names: Sequence[str] = tuple(),
+        instrument_name: Optional[str] = None,
+        **kwargs,
+    ):
+        self._match_lima_group = [
+            re.compile(
+                f"{hdf5.SEP}[0-9]+\\.[0-9]+{hdf5.SEP}instrument{hdf5.SEP}{name}{hdf5.SEP}data"
+            )
+            for name in lima_names
+        ]
+        self._match_lima_dataset = [
+            re.compile(
+                f"{hdf5.SEP}[0-9]+\\.[0-9]+{hdf5.SEP}instrument{hdf5.SEP}{name}{hdf5.SEP}data"
+            )
+            for name in lima_names
+        ]
+        self._non_native_items = dict()
+        self._lima_kwargs = {"user_instrument_name": instrument_name}
+        super().__init__(*args, **kwargs)
+
+    def reset(self) -> None:
+        super().reset()
+        for item in self._non_native_items.values():
+            item.reset()
+
+    def close(self) -> None:
+        for item in self._non_native_items.values():
+            item.close()
+        self._non_native_items = dict()
+        super().close()
+
+    def _check_dataset_before_read(self, item: hdf5.HDF5Dataset):
+        """Check that the last source of a virtual dataset is released by the writer."""
+        if not hasattr(item, "is_virtual"):
+            return
+        if not item.is_virtual:
+            return
+        sources = item.virtual_sources()
+        if not sources:
+            return
+        _, filename, _, _ = sources[-1]
+        if not os.path.isabs(filename):
+            filename = os.path.abspath(
+                os.path.join(os.path.dirname(item.file.filename), filename)
+            )
+        try:
+            with h5py.File(filename, "r"):
+                pass
+        except Exception:
+            raise RetryError(f"cannot open '{filename}'")
+
+    def _get_item(self, name: str) -> hdf5.HDF5Item:
+        h5item = self._non_native_items.get(name)
+        if h5item is not None:
+            return h5item
+
+        try:
+            return super()._get_item(name)
+        except KeyError as e:
+            exception = e
+
+        if any(m.match(name) for m in self._match_lima_dataset):
+            dirname = os.path.dirname(self.file_obj.filename)
+            h5item = lima.LimaDataset(name, dirname, **self._lima_kwargs)
+            self._non_native_items[name] = h5item
+            return h5item
+
+        if any(m.match(name) for m in self._match_lima_group):
+            dirname = os.path.dirname(self.file_obj.filename)
+            h5item = lima.LimaGroup(name, dirname, **self._lima_kwargs)
+            self._non_native_items[name] = h5item
+            return h5item
+
+        raise exception
+
+    @staticmethod
+    def is_group(h5item: hdf5.HDF5Item):
+        return isinstance(h5item, (h5py.Group, h5py.File, lima.LimaGroup))
+
+    def _is_initialized(self, h5item: hdf5.HDF5Item) -> bool:
+        try:
+            if h5item.name == hdf5.SEP:
+                return False
+            scan = [s for s in h5item.name.split(hdf5.SEP) if s][0]
+            nxentry = self.file_obj[scan]
+        except AttributeError:
+            raise RetryError("file is closed")
+        if "end_time" in nxentry:
+            # Last dataset written to the file
+            return True
+        return self._get_writer_status(nxentry) in ("RUNNING", "SUCCEEDED", "FAILED")
+
+    def _is_finished(self, h5item: hdf5.HDF5Item) -> bool:
+        try:
+            if h5item.name == hdf5.SEP:
+                # Assume there could be always more scans coming
+                return False
+            scan = [s for s in h5item.name.split(hdf5.SEP) if s][0]
+            nxentry = self.file_obj[scan]
+        except AttributeError:
+            raise RetryError("file is closed")
+        return "end_time" in nxentry  # Last dataset written to the file
+
+    def _get_writer_status(self, nxentry: h5py.Dataset) -> Optional[str]:
+        nxnote = nxentry.get("writer", None)
+        if nxnote is None:
+            # writer notes not created yet
+            return None
+        status = nxnote.get("status", None)
+        if status is None:
+            # writer status not set yet
+            return None
+        status = status[()]
+        try:
+            status = status.decode()
+        except AttributeError:
+            pass
+        return status