blissdata 0.3.4__py3-none-any.whl

blissdata/h5api/utils/hdf5.py

@@ -0,0 +1,280 @@
+"""Implementation of the h5py-like Bliss Data API with dynamic HDF5 files"""
+
+from numbers import Number
+from typing import Any, Iterator, Optional, Union
+
+import h5py
+from silx.io import h5py_utils
+
+from . import types
+from . import lima
+from .hdf5_retry import retry_file_access
+from .hdf5_retry import ignore_retry_timeout
+from .hdf5_retry import RetryTimeoutError, RetryError, SoftRetryError
+
+
+HDF5Item = Union[h5py.Dataset, h5py.Group, h5py.File, lima.LimaDataset, lima.LimaGroup]
+HDF5Group = Union[h5py.Group, h5py.File, lima.LimaGroup]
+HDF5Dataset = Union[h5py.Dataset, lima.LimaDataset]
+
+
+SEP = "/"
+
+
+class DynamicHDF5Handler:
+    """Object to access an HDF5 file which is re-opened upon re-trying failed IO operations"""
+
+    def __init__(
+        self,
+        file: str,
+        retry_timeout: Optional[Number] = None,
+        retry_period: Optional[Number] = None,
+        **openargs,
+    ):
+        self._file = file
+        self._openargs = openargs
+        self._file_obj = None
+        self._closed = False
+        self._retry_period = retry_period
+        self._retry_options = {
+            "retry_timeout": retry_timeout,
+            "retry_period": retry_period,
+        }
+        self._native_items = dict()
+
+    def close(self) -> None:
+        """Close the HDF5 file, cleanup all HDF5 objects and do not allow re-opening"""
+        self._closed = True
+        self._cleanup()
+
+    def reset(self) -> None:
+        """Close the HDF5 file, cleanup all HDF5 objects but allow re-opening"""
+        self._cleanup()
+
+    def _cleanup(self) -> None:
+        """Close the HDF5 file, cleanup all HDF5 objects"""
+        if self._file_obj is None:
+            return
+        self._file_obj.close()
+        self._file_obj = None
+        self._native_items = dict()
+
+    @property
+    def file_obj(self) -> h5py.File:
+        if self._file_obj is None:
+            if self._closed:
+                raise RuntimeError("File was closed")
+            try:
+                self._file_obj = h5py_utils.File(self._file, **self._openargs)
+            except FileNotFoundError:
+                raise RetryError(f"File {self._file} does not exist (yet)")
+        return self._file_obj
+
+    def get_item(self, name: str) -> HDF5Item:
+        try:
+            return self._retry_get_item(name)
+        except RetryTimeoutError:
+            raise KeyError(name)
+
+    def slice_dataset(self, name: str, idx: types.DataIndexType) -> types.DataType:
+        return self._retry_slice_dataset(name, idx)
+
+    def get_attr(self, name: str, key: str) -> Any:
+        try:
+            return self._retry_get_attr(name, key)
+        except RetryTimeoutError:
+            raise AttributeError(name)
+
+    @ignore_retry_timeout
+    def iter_attrs(self, name: str) -> Iterator[str]:
+        yield from self._retry_iter_attrs(name)
+
+    @ignore_retry_timeout
+    def len_attrs(self, name: str) -> int:
+        return self._retry_len_attrs(name)
+
+    @ignore_retry_timeout
+    def iter_item(self, name: str) -> Iterator[Union[str, types.DataType]]:
+        yield from self._retry_iter_item(name)
+
+    @ignore_retry_timeout
+    def len_item(self, name: str) -> int:
+        return self._retry_len_item(name)
+
+    def getattr_item(self, name: str, attr_name: str) -> Any:
+        return self._retry_getattr_item(name, attr_name)
+
+    @retry_file_access
+    def _retry_get_item(self, name: str) -> HDF5Item:
+        return self._get_item(name)
+
+    @retry_file_access
+    def _retry_slice_dataset(
+        self, name: str, idx: types.DataIndexType
+    ) -> types.DataType:
+        return self._slice_dataset(name, idx)
+
+    @retry_file_access
+    def _retry_get_attr(self, name: str, key: str) -> Any:
+        return self._get_attr(name, key)
+
+    @retry_file_access
+    def _retry_iter_attrs(self, name: str, start_index: int = 0) -> Iterator[str]:
+        yield from self._iter_attrs(name, start_index=start_index)
+
+    @retry_file_access
+    def _retry_len_attrs(self, name: str) -> int:
+        return self._len_attrs(name)
+
+    @retry_file_access
+    def _retry_iter_item(
+        self, name: str, start_index: int = 0
+    ) -> Iterator[Union[str, types.DataType]]:
+        yield from self._iter_item(name, start_index=start_index)
+
+    @retry_file_access
+    def _retry_len_item(self, name: str) -> int:
+        return self._len_item(name)
+
+    @retry_file_access
+    def _retry_getattr_item(self, name: str, attr_name: str) -> Any:
+        return self._getattr_item(name, attr_name)
+
+    def _get_item(self, name: str) -> HDF5Item:
+        item = self._native_items.get(name)
+        if item is not None:
+            return item
+        try:
+            item = self.file_obj[name]
+        except KeyError:
+            parent_name, _, item_name = name.rpartition(SEP)
+            if not parent_name:
+                parent_name = SEP
+            if parent_name == name:
+                raise
+            parent_item = self._get_item(parent_name)
+            if not self._is_finished(parent_item):
+                raise  # item could still be created
+            try:
+                item = parent_item[item_name]  # try one more time
+            except KeyError as e:
+                if not _key_does_not_exist(e):
+                    raise
+                # item will never be created
+                raise RetryTimeoutError(str(e)) from e
+        self._native_items[name] = item
+        return item
+
+    def _slice_dataset(self, name: str, idx: types.DataIndexType) -> types.DataType:
+        item = self._get_item(name)
+        self._check_dataset_before_read(item)
+        return item[idx]
+
+    def _check_dataset_before_read(self, item: HDF5Dataset):
+        pass
+
+    def _get_attr(self, name: str, key: str) -> Any:
+        item = self._get_item(name)
+        try:
+            return item.attrs[key]
+        except KeyError:
+            if not self._is_finished(item):
+                raise  # attribute could still be created
+            try:
+                return item.attrs[key]  # try one more time
+            except KeyError as e:
+                if not _key_does_not_exist(e):
+                    raise
+                # attribute will never be created
+                raise RetryTimeoutError(str(e)) from e
+
+    def _iter_attrs(self, name: str, start_index: int = 0) -> Iterator[str]:
+        item = self._get_item(name)
+        is_complete = self._is_initialized(item)
+        if start_index == 0:
+            yield from item.attrs
+        else:
+            yield from list(item.attrs)[start_index:]
+        if not is_complete:
+            raise RetryError("attributes could still be added")
+
+    def _len_attrs(self, name: str) -> int:
+        item = self._get_item(name)
+        return len(item.attrs)
+
+    def _iter_item(
+        self, name: str, start_index: int = 0
+    ) -> Iterator[Union[str, types.DataType]]:
+        item = self._get_item(name)
+        self._check_dataset_before_read(item)
+        if self.is_group(item):
+            is_complete = self._is_initialized(item)
+            for key in self._iter_group(item, start_index):
+                is_complete = False
+                yield key
+            if not is_complete:
+                raise RetryError("children could still be added to the group")
+        else:
+            is_complete = self._is_finished(item)
+            for data in self._iter_dataset(item, start_index):
+                is_complete = False
+                yield data
+            if not is_complete:
+                raise RetryError("dataset could still grow")
+
+    @staticmethod
+    def is_group(h5item: HDF5Item):
+        return isinstance(h5item, (h5py.Group, h5py.File))
+
+    def _iter_group(self, h5group: HDF5Group, start_index: int) -> Iterator[str]:
+        try:
+            if start_index == 0:
+                yield from h5group
+            else:
+                yield from list(h5group.keys())[start_index:]
+        except ValueError:
+            raise SoftRetryError(f"Failed accessing group {h5group}")
+
+    def _iter_dataset(
+        self, h5dataset: HDF5Dataset, start_index: int
+    ) -> Iterator[types.DataType]:
+        try:
+            if start_index == 0:
+                yield from h5dataset
+            else:
+                for i in range(start_index, len(h5dataset)):
+                    yield h5dataset[i]
+        except ValueError:
+            raise SoftRetryError(f"Failed accessing dataset {h5dataset}")
+
+    def _len_item(self, name: str) -> int:
+        item = self._get_item(name)
+        return len(item)
+
+    def _getattr_item(self, name: str, attr_name: str) -> Any:
+        item = self._get_item(name)
+        return getattr(item, attr_name)
+
+    def _is_initialized(self, h5item: HDF5Item) -> bool:
+        return True
+
+    def _is_finished(self, h5item: HDF5Item) -> bool:
+        return True
+
+
+def _key_does_not_exist(exc: KeyError) -> bool:
+    """
+    A real KeyError (meaning that the key does not exist) gives the following error message:
+
+    .. code
+
+        KeyError: "Unable to synchronously open object (object 'end_time' doesn't exist)"
+
+    An example of a KeyError caused by another reason
+
+    .. code
+
+        KeyError: 'Unable to synchronously open object (addr overflow, addr = 64392369, size = 96, eoa = 64365353)'
+
+    """
+    return "doesn't exist" in str(exc)

blissdata/h5api/utils/hdf5_retry.py

@@ -0,0 +1,98 @@
+import inspect
+from functools import wraps
+from typing import Callable
+
+from silx.io import h5py_utils
+from silx.utils.retry import RetryTimeoutError, RetryError
+
+
+class SoftRetryError(Exception):
+    pass
+
+
+def retry_file_access(method: Callable) -> Callable:
+    return _reset_on_retry_failed(
+        h5py_utils.retry(retry_on_error=_retry_on_error)(_reset_before_retry(method))
+    )
+
+
+def ignore_retry_timeout(method: Callable) -> Callable:
+    if inspect.isgeneratorfunction(method):
+
+        @wraps(method)
+        def wrapper(self, *args, **kw):
+            try:
+                yield from method(self, *args, **kw)
+            except RetryTimeoutError:
+                pass
+
+    else:
+
+        @wraps(method)
+        def wrapper(self, *args, **kw):
+            try:
+                return method(self, *args, **kw)
+            except RetryTimeoutError:
+                pass
+
+    return wrapper
+
+
+def _retry_on_error(e: BaseException) -> bool:
+    return h5py_utils._is_h5py_exception(e) or isinstance(e, RetryError)
+
+
+def _reset_before_retry(method: Callable) -> Callable:
+    if inspect.isgeneratorfunction(method):
+
+        @wraps(method)
+        def wrapper(self, *args, start_index: int = 0, **kw):
+            try:
+                yield from method(self, *args, start_index=start_index, **kw)
+            except SoftRetryError:
+                raise RetryError
+            except Exception as e:
+                if _retry_on_error(e):
+                    self.reset()
+                raise
+
+    else:
+
+        @wraps(method)
+        def wrapper(self, *args, **kw):
+            try:
+                return method(self, *args, **kw)
+            except SoftRetryError:
+                raise RetryError
+            except Exception as e:
+                if _retry_on_error(e):
+                    self.reset()
+                raise
+
+    return wrapper
+
+
+def _reset_on_retry_failed(method: Callable) -> Callable:
+    if inspect.isgeneratorfunction(method):
+
+        @wraps(method)
+        def wrapper(self, *args, start_index: int = 0, **kw):
+            kw.update(self._retry_options)
+            try:
+                yield from method(self, *args, start_index=start_index, **kw)
+            except RetryTimeoutError:
+                self.reset()
+                raise
+
+    else:
+
+        @wraps(method)
+        def wrapper(self, *args, **kw):
+            kw.update(self._retry_options)
+            try:
+                return method(self, *args, **kw)
+            except RetryTimeoutError:
+                self.reset()
+                raise
+
+    return wrapper

blissdata/h5api/utils/lima.py

@@ -0,0 +1,286 @@
+import os
+import re
+from glob import glob
+from collections import abc
+from numbers import Integral
+from typing import Optional, Tuple, Sequence, Iterator
+
+import numpy
+from numpy.typing import DTypeLike
+from silx.utils.retry import RetryError
+from silx.io import h5py_utils
+
+from . import types
+
+SEP = "/"
+
+
+class LimaGroup(abc.Mapping):
+    def __init__(self, name, *args, **kwargs) -> None:
+        self._name = name
+        self._dset_args = args
+        self._dset_kwargs = kwargs
+        self._dataset = None
+        super().__init__()
+
+    def close(self):
+        if self._dataset is None:
+            return
+        self._dataset.close()
+        self._dataset = None
+
+    def reset(self):
+        if self._dataset is None:
+            return
+        self._dataset.reset()
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def attrs(self) -> dict:
+        return {"type": "lima"}
+
+    def __getitem__(self, key: str):
+        if key == "data":
+            if self._dataset is None:
+                self._dataset = LimaDataset(
+                    self.name + SEP + "data", *self._dset_args, **self._dset_kwargs
+                )
+            return self._dataset
+        raise KeyError
+
+    def __iter__(self):
+        yield "data"
+
+    def __len__(self) -> int:
+        return 1
+
+
+class LimaDataset(abc.Sequence):
+    def __init__(
+        self,
+        name: str,
+        dirname: str,
+        url_template: Optional[str] = None,
+        user_detector_name: Optional[str] = None,
+        user_instrument_name: Optional[str] = None,
+    ) -> None:
+        parts = [s for s in name.split(SEP) if s]
+        scan_number = int(parts[0].split(".")[0])
+        bliss_detector_name = parts[-2]
+
+        url_template = lima_url_template(
+            dirname,
+            scan_number,
+            bliss_detector_name,
+            url_template=url_template,
+            user_detector_name=user_detector_name,
+            user_instrument_name=user_instrument_name,
+        )
+        filename_template, self._path_in_file = url_template.split("::")
+        self._search_pattern = filename_template.format(file_index="*")
+        self._match_pattern = re.compile(
+            os.path.basename(filename_template.format(file_index="([0-9]+)"))
+        )
+        self._files = list()
+
+        self._first_shape = None
+        self._points_per_file = None
+        self._dtype = None
+        self._shape = None
+
+        self._active_filename = None
+        self._active_file = None
+        self._active_dset = None
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def attrs(self) -> dict:
+        return dict()
+
+    def reset(self):
+        """Close the activate HDF5 file, cleanup all HDF5 objects and search for lima files"""
+        self._cleanup()
+        self.search()
+
+    def close(self):
+        """Close the activate HDF5 file and cleanup all HDF5 objects"""
+        self._cleanup()
+
+    def _cleanup(self):
+        if self._active_file is None:
+            return
+        self._active_file.close()
+        self._active_filename = None
+        self._active_file = None
+        self._active_dset = None
+
+    def search(self):
+        lima_files = glob(self._search_pattern)
+        lima_nrs = [
+            int(self._match_pattern.search(os.path.basename(s)).group(1))
+            for s in lima_files
+        ]
+        self._files = [filename for _, filename in sorted(zip(lima_nrs, lima_files))]
+        self._shape = None
+
+    def _open(self, file_index: int):
+        try:
+            filename = self._files[file_index]
+        except IndexError:
+            raise RetryError(f"lima file {file_index} does not exist (yet)")
+        if self._active_filename == filename:
+            return
+        self.close()
+        # Lima has the file open until all images are written.
+        # Lima does not flush which means the file is never readable while it is open.
+        # To be sure about the later, try locking the file when reading.
+        # If it succeeds it means lima is done with the file.
+        self._active_file = h5py_utils.File(
+            filename, mode="r"
+        )  # , locking=h5py_utils.HAS_LOCKING_ARGUMENT)
+        self._active_dset = self._active_file[self._path_in_file]
+        self._active_filename = filename
+
+    def __getitem__(self, idx: types.DataIndexType) -> types.DataType:
+        if isinstance(idx, Tuple):
+            idx0 = idx[0]
+            idximage = idx[1:]
+        else:
+            idx0 = idx
+            idximage = tuple()
+
+        dim0scalar = False
+        if isinstance(idx0, Integral):
+            idx0 = numpy.asarray([idx0])
+            dim0scalar = True
+        elif isinstance(idx0, slice):
+            idx0 = numpy.array(range(*idx0.indices(self.shape[0])))
+        elif isinstance(idx0, Sequence):
+            idx0 = numpy.asarray(idx0)
+        elif idx0 is Ellipsis:
+            idx0 = numpy.array(range(self.shape[0]))
+        else:
+            raise TypeError
+
+        result = list()
+
+        for scan_index in idx0:
+            file_index = scan_index // self.point_per_file
+            self._open(file_index)
+            idx = (scan_index % self.point_per_file,) + idximage
+            try:
+                result.append(self._active_dset[idx])
+            except IndexError:
+                raise RetryError(f"Failed slice lima dataset {self._active_dset}")
+
+        if dim0scalar:
+            return result[0]
+        else:
+            return numpy.array(result)
+
+    def __iter__(self) -> Iterator[types.DataType]:
+        for file_index in range(len(self._files)):
+            self._open(file_index)
+            yield from iter(self._active_dset)
+
+    def _cache_dataset_info(self):
+        if not self._files:
+            self.reset()
+        try:
+            filename = self._files[0]
+        except IndexError:
+            raise RetryError("no lima files exists (yet)")
+        with h5py_utils.File(filename, mode="r") as f:
+            dset = f[self._path_in_file]
+            self._points_per_file = len(dset)
+            self._first_shape = dset.shape
+            self._dtype = dset.dtype
+
+    def __len__(self) -> int:
+        return self.shape[0]
+
+    @property
+    def point_per_file(self) -> int:
+        if self._points_per_file is None:
+            self._cache_dataset_info()
+        return self._points_per_file
+
+    @property
+    def dtype(self) -> DTypeLike:
+        if self._dtype is None:
+            self._cache_dataset_info()
+        return self._dtype
+
+    @property
+    def shape(self) -> Tuple[int]:
+        if self._shape is not None:
+            return self._shape
+        if self._first_shape is None:
+            self._cache_dataset_info()
+        shape = list(self._first_shape)
+        shape[0] *= len(self._files)
+        self._shape = tuple(shape)
+        return self._shape
+
+    @property
+    def size(self) -> int:
+        return numpy.prod(self.shape, dtype=int)
+
+    @property
+    def ndim(self) -> int:
+        return 3
+
+
+def lima_url_template(
+    dirname: str,
+    scan_number: int,
+    bliss_detector_name: str,
+    url_template: Optional[str] = None,
+    user_detector_name: Optional[str] = None,
+    user_instrument_name: Optional[str] = None,
+) -> str:
+    if not url_template:
+        if user_instrument_name:
+            url_template = os.path.join(
+                "{dirname}",
+                "scan{scan_number:04d}",
+                "{bliss_detector_name}_{{file_index}}.h5",
+            ) + SEP.join(
+                (
+                    "::",
+                    "entry_0000",
+                    "{user_instrument_name}",
+                    "{user_detector_name}",
+                    "data",
+                )
+            )
+        else:
+            url_template = os.path.join(
+                "{dirname}",
+                "scan{scan_number:04d}",
+                "{bliss_detector_name}_{{file_index}}.h5",
+            ) + SEP.join(("::", "entry_0000", "measurement", "data"))
+    if not user_detector_name:
+        user_detector_name = bliss_detector_name
+    kw = dict()
+    if dirname:
+        kw["dirname"] = dirname
+    if scan_number:
+        kw["scan_number"] = scan_number
+    if bliss_detector_name:
+        kw["bliss_detector_name"] = bliss_detector_name
+    if user_detector_name:
+        kw["user_detector_name"] = user_detector_name
+    if user_instrument_name:
+        kw["user_instrument_name"] = user_instrument_name
+    url_template = url_template.format(**kw)
+    if "{file_index}" not in url_template:
+        raise ValueError("A lima template URL needs '{{file_index}}'")
+    return url_template

blissdata/h5api/utils/types.py

@@ -0,0 +1,13 @@
+from typing import Tuple, Union, Sequence
+from numbers import Number, Integral
+
+try:
+    from types import EllipsisType
+
+    SingleDataIndexType = Union[Integral, slice, Sequence, EllipsisType]
+except ImportError:
+    SingleDataIndexType = Union[Integral, slice, Sequence]
+from numpy.typing import ArrayLike
+
+DataType = Union[bytes, Number, ArrayLike]
+DataIndexType = Union[SingleDataIndexType, Tuple[SingleDataIndexType]]

blissdata/redis/__init__.py

@@ -0,0 +1,12 @@
+"""Manage Redis connections and provide a low-level API to Redis.
+
+The low-level API exposes all Redis commands (https://redis.io/commands)
+as methods of Redis database proxies (`RedisDbProxyBase` and derived classes).
+
+Redis database proxies are created through a Redis connection pool (`RedisDbConnectionPool`).
+
+Redis connection pools are created through a Redis connection manager (`RedisConnectionManager`).
+"""
+
+from .manager import RedisConnectionManager  # noqa: F401
+from .manager import RedisAddress  # noqa: F401