blissdata 0.3.4__py3-none-any.whl

blissdata/data/nodes/lima.py

@@ -0,0 +1,422 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2023 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+import numpy
+import warnings
+import logging
+
+from blissdata.data.nodes.channel import _ChannelDataNodeBase
+from blissdata.data.events import EventData, LimaImageStatusEvent, ImageNotSaved
+from blissdata.settings import QueueObjSetting
+from blissdata.data import lima_image
+
+
+_logger = logging.getLogger(__name__)
+
+
+class LimaDataView:
+    def __init__(self, queue, queue_ref, from_index, to_index, from_stream=False):
+        """
+        :param DataStream queue: acquisition status events
+        :param QueueObjSetting queue_ref: acquisition info
+        :param int from_index:
+        :param int to_index: when <0 we will take the last image
+                             that is "ready" which may change after `update`
+        :param bool from_stream:
+        """
+        self._queue = queue
+        self._queue_ref = queue_ref
+        self.from_index = from_index
+        self._to_index = to_index
+        self.from_stream = from_stream
+        self._status_event = None
+
+    def __getattr__(self, attr):
+        # Get attribute from the status event
+        try:
+            return getattr(self.status_event, attr)
+        except AttributeError:
+            raise AttributeError(attr)
+
+    @property
+    def to_index(self):
+        """Either a fixed number or the last image that is "ready" """
+        self.update()
+        last_index = self.last_image_ready
+
+        if self._to_index >= 0:
+            return min(self._to_index, last_index)
+        else:
+            return last_index
+
+    @property
+    def last_index(self):
+        """WARNING: this is the last image index + 1"""
+        return self.to_index + 1
+
+    @property
+    def connection(self):
+        return self._queue._cnx()
+
+    def update(self):
+        """Get the latest status event from the data stream
+        and add the first element of the reference settings.
+        It is safe to call this as much as you want.
+        """
+        events = self._queue.rev_range(count=1)
+        if events:
+            index, raw = events[-1]  # [0]
+            ev = LimaImageStatusEvent(raw=raw)
+        else:  # Lima acqusition has not yet started.
+            ev = LimaImageStatusEvent({})
+
+        try:
+            ev.info = self.first_ref_data
+        except IndexError:
+            pass
+        ev.connection = self.connection
+        self._status_event = ev
+
+    @property
+    def status_event(self):
+        if self._status_event is None:
+            self.update()
+        return self._status_event
+
+    @property
+    def ref_status(self):
+        warnings.warn(
+            "ref_status is deprecated. Use 'status_event.status' instead.",
+            FutureWarning,
+        )
+        return self.status_event.status
+
+    @property
+    def all_ref_data(self):
+        """
+        :returns list(dict):
+        """
+        return self._queue_ref[0:]
+
+    @property
+    def first_ref_data(self):
+        """
+        :returns dict:
+        """
+        return self._queue_ref[0]
+
+    def is_video_frame_have_meaning(self):
+        """Returns True if the frame number reached from the header from
+        the Lima video have a meaning in the full scan.
+
+        Returns a boolean, else None if this information is not yet known.
+        """
+        self.update()
+        return self.status_event.is_video_frame_have_meaning()
+
+    def get_last_live_image(self):
+        """Returns the last image data from stream within it's frame number.
+
+        If no data is available, the function returns tuple (None, None).
+
+        If camera device is not configured with INTERNAL_TRIGGER_MULTI, and
+        then the reached frame number have no meaning, a None is returned.
+
+        :returns Frame:
+        """
+        if not self.from_stream:
+            # FIXME: It should return None
+            return lima_image.Frame(None, None, None)
+        self.update()
+        return self.status_event.get_last_live_image()
+
+    def get_last_image(self):
+        """Returns the last image from the received one, together with the frame id.
+
+        :returns Frame:
+        """
+        self.update()
+        return self.status_event.get_last_image()
+
+    def get_image(self, image_index_in_scan):
+        """
+        :param int image_index_in_scan:
+        :returns numpy.ndarray:
+        """
+        if image_index_in_scan < 0:
+            raise ValueError("image_index_in_scan cannot be a negative number")
+        self.update()
+        return self.status_event.get_image(image_index_in_scan)
+
+    def __getitem__(self, idx):
+        """Get images from server or file"""
+        if isinstance(idx, slice):
+            start, stop, step = idx.indices(len(self))
+            start += self.from_index
+            stop += self.from_index
+            return numpy.asarray(list(self._image_range(start, stop, step)))
+        elif isinstance(idx, list):
+            idx = numpy.asarray(idx)
+            if isinstance(idx[0].item(), bool):
+                idx = numpy.nonzero(idx)
+            idx += self.from_index
+            return numpy.asarray(list(self._image_iter(idx)))
+        elif isinstance(idx, tuple):
+            # This would slice the image dimensions
+            raise NotImplementedError
+        else:
+            try:
+                idx = int(idx)
+            except Exception as e:
+                raise IndexError from e
+            if self.from_stream and idx == -1:
+                img = self.get_image(-1)
+            else:
+                if idx < 0:
+                    index = self.to_index + 1 + idx
+                    if index < 0:
+                        raise IndexError("No image available")
+                else:
+                    index = self.from_index + idx
+                img = self.get_image(index)
+            if img is None:
+                raise IndexError
+            return img
+
+    def __iter__(self):
+        """Iterator over images from server or file"""
+        yield from self._image_range(self.from_index, self.to_index + 1)
+
+    def _image_range(self, start, stop, step=1):
+        """Iterator over images from server or file"""
+        yield from self._image_iter(range(start, stop, step))
+
+    def _image_iter(self, image_nb_iterator):
+        """Iterator over images from server or file"""
+        for image_index_in_scan in image_nb_iterator:
+            try:
+                img = self.get_image(image_index_in_scan)
+            except IndexError:
+                img = None
+            if img is None:
+                break
+            yield img
+
+    def as_array(self):
+        if len(self) == 1:
+            # To be consistant with ChannelDataNode
+            return list(self)[0]
+        else:
+            return numpy.asarray(list(self))
+
+    def __len__(self):
+        length = self.to_index - self.from_index + 1
+        return 0 if length < 0 else length
+
+    def all_image_references(self, saved=False):
+        """Get the image references.
+
+        :param bool saved: ready or ready and saved
+        :returns list(tuple): file name, path-in-file, image index, file format
+                              File format (HDF5, HDF5BS, EDFLZ4, ...) is not file extension!
+        :raise RuntimeError: images will never be saved
+        """
+        self.update()
+        return self.status_event.all_image_references(saved=saved)
+
+    def image_references(self, image_indices_in_scan, saved=False):
+        """Get the image references.
+
+        :param sequence image_indices_in_scan:
+        :param bool saved: ready or ready and saved
+        :returns list(tuple): file name, path-in-file, image index, file format
+                              File format (HDF5, HDF5BS, EDFLZ4, ...) is not file extension!
+        :raise RuntimeError: some images are not ready or saved yet
+                             or images will never be saved
+        """
+        self.update()
+        return self.status_event.image_references(image_indices_in_scan, saved=saved)
+
+    def image_reference(self, image_index_in_scan, saved=False):
+        """Get the image references.
+
+        :param int image_index_in_scan:
+        :param bool saved: ready or ready and saved
+        :returns tuple: file name, path-in-file, image index, file format
+                        File format (HDF5, HDF5BS, EDFLZ4, ...) is not file extension!
+        :raise RuntimeError: image is not ready or saved yet
+                             or images will never be saved
+        """
+        self.update()
+        return self.status_event.image_reference(image_index_in_scan, saved=saved)
+
+    def iter_image_references(self, image_indices_in_scan=None, saved=False):
+        """Get the image references.
+
+        Stops iterating when it encounters an image that is not
+        ready or saved yet, regardless of how many images you
+        asked for.
+
+        :param sequence image_indices_in_scan:
+        :param bool saved: ready or ready and saved
+        :yields list(tuple): file name, path-in-file, image index, file format
+                             File format (HDF5, HDF5BS, EDFLZ4, ...) is not file extension!
+        :raises RuntimeError: images will never be saved
+        """
+        self.update()
+        yield from self.status_event.iter_image_references(
+            image_indices_in_scan=image_indices_in_scan, saved=saved
+        )
+
+    def get_filenames(self):
+        warnings.warn(
+            "'get_filenames' is deprecated. Use 'all_image_references' instead.",
+            FutureWarning,
+        )
+        return self.all_image_references()
+
+    def _get_filenames(self, ref_data, *image_indices_in_scan):
+        warnings.warn(
+            "'_get_filenames' is deprecated. Use 'image_references' itself.",
+            FutureWarning,
+        )
+        return self.image_references(image_indices_in_scan)
+
+
+class LimaImageChannelDataNode(_ChannelDataNodeBase):
+    _NODE_TYPE = "lima"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # There is good reason for this name not to end with "*_data"
+        # (see DataNodeContainer._subscribe_streams)
+        self._queue_ref = QueueObjSetting(
+            f"{self.db_name}_data_ref", connection=self.db_connection
+        )
+        self.from_stream = False
+        self._local_ref_status = dict()
+        self._stream_image_count = 0
+
+    def store(self, event_dict, cnx=None):
+        """Publish lima reference in Redis"""
+        data = event_dict["data"]
+        if data.get("in_prepare", False):  # in prepare phase
+            ref_data = event_dict["description"]
+            self.info.update(ref_data)
+            self._queue_ref.append(ref_data)
+            self._local_ref_status = data
+            self._local_ref_status["lima_acq_nb"] = self.db_connection.incr(
+                data["server_url"]
+            )
+        else:  # during acquisition
+            self._local_ref_status.update(data)
+            ev = LimaImageStatusEvent(self._local_ref_status)
+            self._queue.add_event(ev, id=self._last_index, cnx=cnx)
+            self._last_index += 1
+
+    def get(self, from_index, to_index=None):
+        """
+        Return a view on data references.
+
+        **from_index** from which image index you want to get
+        **to_index** to which index you want images
+            if to_index is None => only one image which as index from_index
+            if to_index < 0 => to the end of acquisition
+        """
+        return LimaDataView(
+            self._queue,
+            self._queue_ref,
+            from_index,
+            to_index if to_index is not None else from_index,
+            from_stream=self.from_stream,
+        )
+
+    def get_as_array(self, from_index, to_index=None):
+        """Like `get` but ensures the result is a numpy array."""
+        return numpy.asarray(self.get(from_index, to_index).as_array(), self.dtype)
+
+    def decode_raw_events(self, events):
+        """Decode raw stream data and get image URI's.
+
+        :param list((index, raw)) events:
+        :returns EventData:
+        """
+        data = list()
+        first_index = -1
+        description = None
+        if events:
+            # The number of events is NOT equal to the number of images
+            # The number of images can be derived from the event data though
+            # TODO: first_index is only accurate if we use the same DataNode instance!!!
+
+            if len(self._queue_ref) == 0:
+                # Workaround for the issue https://gitlab.esrf.fr/bliss/bliss/-/issues/2128
+                _logger.debug(
+                    "Inconsistency on the Redis data structure. 'image_data_ref' was removed (or is empty)"
+                )
+                return EventData(
+                    first_index=first_index, data=data, description=description
+                )
+
+            ev = LimaImageStatusEvent.merge(events)
+            ev.info = self.first_ref_data
+            first_index = self._stream_image_count
+            try:
+                data = ev.image_reference_range(first_index)
+            except ImageNotSaved:
+                # If images are not supposed to be saved we can't expect to
+                # find references
+                pass
+            self._stream_image_count += len(data)
+            description = ev.status
+        return EventData(first_index=first_index, data=data, description=description)
+
+    @property
+    def all_ref_data(self):
+        """
+        :returns list(dict):
+        """
+        return self._queue_ref[0:]
+
+    @property
+    def first_ref_data(self):
+        """
+        :returns dict:
+        """
+        return self._queue_ref[0]
+
+    @property
+    def images_per_file(self):
+        try:
+            return self.first_ref_data.get("saving_frame_per_file")
+        except IndexError:
+            return None
+
+    @property
+    def images_per_acquisition(self):
+        try:
+            return self.first_ref_data.get("saving_frame_per_acquisition")
+        except IndexError:
+            return None
+
+    def get_db_names(self, **kw):
+        db_names = super().get_db_names(**kw)
+        db_names.append(self.db_name + "_data_ref")
+        events = self._queue.range(count=1)
+        if events:
+            index, raw = events[0]
+            ev = LimaImageStatusEvent(raw=raw)
+            url = ev.server_url
+            if url:
+                db_names.append(url)
+        return db_names
+
+    def get_settings(self):
+        return super().get_settings() + [self._queue_ref]
+
+    def __len__(self):
+        return len(self.get(0, -1))

blissdata/data/nodes/node_ref_channel.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2023 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+from blissdata.data.nodes.channel import ChannelDataNode
+from blissdata.data.events import EventData
+
+
+class NodeRefChannel(ChannelDataNode):
+    """
+    A data node that stores references to other DataNodes. It is intended to be
+    used to keep references e.g. of individual scans to group them together as
+    one group or sequence of scans.
+    """
+
+    _NODE_TYPE = "node_ref_channel"
+
+    def decode_raw_events(self, events):
+        """Decode raw stream data
+
+        :param list((index, raw)) events:
+        :returns EventData:
+        """
+        event_data = super().decode_raw_events(events)
+        nodes = self.get_nodes(*event_data.data)
+        return EventData(
+            first_index=event_data.first_index,
+            data=nodes,
+            description=event_data.description,
+            block_size=event_data.block_size,
+        )

blissdata/data/nodes/proposal.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2023 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+from blissdata.data.nodes.dataset import _DataPolicyNode
+
+
+class ProposalNode(_DataPolicyNode):
+    _NODE_TYPE = "proposal"

blissdata/data/nodes/scan.py

@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2023 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+from blissdata.data.node import DataNodeContainer
+from blissdata.data.nodes.channel import ChannelDataNode, RedisDataExpiredError
+from blissdata.data.nodes.lima import LimaImageChannelDataNode
+from blissdata.streaming_events import StreamEvent
+from blissdata.data.events import (
+    Event,
+    EventType,
+    EventData,
+    EndScanEvent,
+    PreparedScanEvent,
+)
+from blissdata import settings
+from blissdata.common.utils import update_node_info
+
+
+class ScanNode(DataNodeContainer):
+    _NODE_TYPE = "scan"
+
+    _EVENT_TYPE_MAPPING = {
+        EndScanEvent.TYPE.decode("ascii"): EventType.END_SCAN,
+        PreparedScanEvent.TYPE.decode("ascii"): EventType.PREPARED_SCAN,
+    }
+    """Mapping from event name to EventType
+    """
+
+    def __init__(self, name, **kwargs):
+        super().__init__(self._NODE_TYPE, name, **kwargs)
+        self._end_stream = self._create_stream("end")
+        self._prepared_stream = self._create_stream("prepared")
+        # Register to priority as the following way: NEW DATA > PREPARED > NEW NODE > END
+        self._register_stream_priority(self._end_stream.name, 3)
+        self._register_stream_priority(self._prepared_stream.name, 1)
+
+    @property
+    def dataset(self):
+        return self.parent
+
+    def prepared(self, scan_info):
+        """Publish PREPARED event in Redis"""
+        if not self.new_node:
+            return
+        # to avoid to have multiple modification events
+        # TODO: what does the comment above mean?
+        with settings.pipeline(self._prepared_stream, self._info):
+            event = PreparedScanEvent()
+            update_node_info(self, scan_info)
+            self._prepared_stream.add_event(event)
+
+    def end(self, scan_info, exception=None):
+        """Update the info in Redis and publish the END event."""
+        if not self.new_node:
+            return
+        # to avoid to have multiple modification events
+        # TODO: what does the comment above mean?
+        with settings.pipeline(self._end_stream, self._info):
+            event = EndScanEvent(exception=exception)
+            scan_info["end_time"] = event.time
+            scan_info["end_time_str"] = event.strftime
+            scan_info["end_timestamp"] = event.timestamp
+            update_node_info(self, scan_info)
+            self._end_stream.add_event(event)
+
+    def decode_raw_events(self, events):
+        """Decode raw stream data
+
+        :param list((index, raw)) events:
+        :returns EventData:
+        """
+        if not events:
+            return None
+
+        assert len(events) == 1  # Else you are about to lose events
+        event = events[0]
+        timestamp, raw_data = event
+        first_index = self._streamid_to_idx(timestamp)
+        ev = StreamEvent.factory(raw_data)
+        data = type(ev).TYPE.decode()
+        return EventData(first_index=first_index, data=data, description=ev.description)
+
+    def get_db_names(self, **kw):
+        db_names = super().get_db_names(**kw)
+        db_names.append(self._end_stream.name)
+        db_names.append(self._prepared_stream.name)
+        return db_names
+
+    def get_settings(self):
+        return super().get_settings() + [self._end_stream, self._prepared_stream]
+
+    def _subscribe_streams(self, reader, first_index=None, **kw):
+        """Subscribe to all associated streams of this node.
+
+        :param DataStreamReader reader:
+        :param **kw: see DataNodeContainer
+        """
+        super()._subscribe_streams(reader, first_index=first_index, **kw)
+        suffix = self._end_stream.name.rsplit("_", 1)[-1]
+        self._subscribe_stream(
+            suffix, reader, first_index=0, create=True, ignore_excluded=True
+        )
+        suffix = self._prepared_stream.name.rsplit("_", 1)[-1]
+        self._subscribe_stream(
+            suffix, reader, first_index=0, create=True, ignore_excluded=True
+        )
+
+    def get_stream_event_handler(self, stream):
+        """
+        :param DataStream stream:
+        :returns callable:
+        """
+        if stream.name == self._end_stream.name:
+            return self._iter_data_stream_events
+        elif stream.name == self._prepared_stream.name:
+            return self._iter_data_stream_events
+        return super(ScanNode, self).get_stream_event_handler(stream)
+
+    def _iter_data_stream_events(
+        self,
+        reader,
+        events,
+        include_filter=None,
+        exclude_children=None,
+        first_index=None,
+        yield_events=False,
+    ):
+        """
+        :param DataStreamReader reader:
+        :param list(2-tuple) events:
+        :param include_filter: only these nodes are included (all by default)
+        :param exclude_children: ignore children of these nodes recursively
+        :param str or int first_index: Redis stream index (None is now)
+        :param bool yield_events: yield Event or DataNode
+        :yields Event:
+        """
+        for event in events:
+            data = self.decode_raw_events([event])
+            if data is None:
+                return
+            if yield_events and self._included(include_filter):
+                kind = data.data
+                event_id = self._EVENT_TYPE_MAPPING[kind]
+                event = Event(type=event_id, node=self, data=data)
+                yield event
+                if event_id is EventType.END_SCAN:
+                    # Stop reading events from this node's streams
+                    # and the streams of its children
+                    reader.remove_matching_streams(f"{self.db_name}*")
+
+
+def get_data_from_nodes(pipeline, *nodes):
+    scan_channel_get_data_func = dict()  # { channel_name: function }
+    scan_image_get_view = dict()
+    for node in nodes:
+        if isinstance(node, LimaImageChannelDataNode):
+            scan_image_get_view[node.fullname] = node.get(0, -1)
+        elif isinstance(node, ChannelDataNode):
+            chan = node
+            channel_name = chan.fullname
+
+            try:
+                saved_db_connection = chan.db_connection
+                chan.db_connection = pipeline
+                # append channel name and get all data from channel;
+                # as it is in a Redis pipeline, .get() returns the
+                # conversion function only - data will be received
+                # after .execute()
+                scan_channel_get_data_func[channel_name] = chan.get(0, -1)
+            finally:
+                chan.db_connection = saved_db_connection
+
+    result = pipeline.execute()
+
+    for i, (channel_name, get_data_func) in enumerate(
+        scan_channel_get_data_func.items()
+    ):
+        if not result[i]:
+            raise RedisDataExpiredError("Redis data channel expired.")
+
+        if result[i][0][0] != b"1-0":  # first event is 1-0, not 0-0
+            raise RedisDataExpiredError(
+                f"Redis data channel {channel_name} have been trimmed out."
+            )
+
+        yield channel_name, get_data_func(result[i])
+    for channel_name, view in scan_image_get_view.items():
+        yield channel_name, view

blissdata/data/nodes/scan_group.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2023 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+from blissdata.data.nodes.scan import ScanNode
+
+
+class GroupScanNode(ScanNode):
+    _NODE_TYPE = "scan_group"