evefile 0.1.0rc1__py3-none-any.whl → 0.2.0__py3-none-any.whl

evefile/boundaries/evefile.py

@@ -146,9 +146,10 @@ import os
 
 import pandas as pd
 
+import evefile.entities.data
 from evefile.entities.file import File
 from evefile.boundaries.eveh5 import HDF5File
-from evefile.controllers import version_mapping, joining
+from evefile.controllers import version_mapping, joining, timestamp_mapping
 
 
 logger = logging.getLogger(__name__)
@@ -221,6 +222,11 @@ class EveFile(File):
         The keys of the dictionary are the (guaranteed to be unique) HDF
         dataset names, not the "given" names usually familiar to the users.
 
+        **Please note:** For monitors, in contrast to data stored in the
+        :attr:`data` attribute, names are *not unique*. Hence, the only way to
+        address an individual monitor unequivocally is by its ID that is
+        identical to the HDF dataset name.
+
         Each item is an instance of
         :class:`evefile.entities.data.MonitorData`.
 
@@ -261,7 +267,8 @@ class EveFile(File):
     def __init__(self, filename="", load=True):
         super().__init__()
         self.filename = filename
-        self._join_factory = joining.JoinFactory(evefile=self)
+        self._join_factory = joining.JoinFactory(file=self)
+        self._monitor_mapper = timestamp_mapping.Mapper(file=self)
         if load:
             if not filename:
                 raise ValueError("No filename given")
@@ -387,14 +394,24 @@ class EveFile(File):
         if self.metadata.preferred_axis:
             output[0] = self.data[self.metadata.preferred_axis]
         if self.metadata.preferred_channel:
-            output[1] = self.data[self.metadata.preferred_channel]
+            if self.metadata.preferred_channel in self.data:
+                output[1] = self.data[self.metadata.preferred_channel]
+            else:
+                logger.warning(
+                    "Dataset {} not found, possibly an "
+                    "attribute or option?".format(
+                        self.metadata.preferred_channel
+                    )
+                )
         if self.metadata.preferred_normalisation_channel:
             output[2] = self.data[
                 self.metadata.preferred_normalisation_channel
             ]
         return output
 
-    def get_joined_data(self, data=None, mode="AxisOrChannelPositions"):
+    def get_joined_data(
+        self, data=None, mode="AxisOrChannelPositions", include_monitors=False
+    ):
         """
         Retrieve data objects with commensurate dimensions.
 
@@ -420,6 +437,15 @@ class EveFile(File):
 
             Default: "AxisOrChannelPositions"
 
+        include_monitors : :class:`bool`
+            Whether to include (all) monitors in joined data.
+
+            If set to :obj:`True`, all monitors available will automatically
+            be mapped to :obj:`DeviceData <evefile.entities.data.DeviceData>`
+            objects and included in the list of joined data.
+
+            Default: :obj:`False`
+
         Returns
         -------
         data : :class:`list`
@@ -431,10 +457,37 @@ class EveFile(File):
         """
         if not data:
             data = list(self.data.values())
+        data = [
+            (
+                self._convert_str_to_data_object(item)
+                if isinstance(item, str)
+                else item
+            )
+            for item in data
+        ]
+        if include_monitors:
+            monitors = self.get_monitors()
+            if isinstance(monitors, list):
+                data.extend(monitors)
+            else:
+                data.append(monitors)
         joiner = self._join_factory.get_join(mode=mode)
         return joiner.join(data)
 
-    def get_dataframe(self, data=None, mode="AxisOrChannelPositions"):
+    def _convert_str_to_data_object(self, name_or_id=""):
+        try:
+            result = self.data[name_or_id]
+        except KeyError:
+            try:
+                result = self.get_data(name_or_id)
+            except KeyError:
+                # Valid situation: monitor
+                result = self.get_monitors(name_or_id)
+        return result
+
+    def get_dataframe(
+        self, data=None, mode="AxisOrChannelPositions", include_monitors=False
+    ):
         """
         Retrieve Pandas DataFrame with given data objects as columns.
 
@@ -445,6 +498,18 @@ class EveFile(File):
         The names of the columns of the returned DataFrame are the names (not
         IDs) of the respective datasets.
 
+        .. note::
+
+            At least for the time being, for each data object involved only
+            the ``data`` attribute will be contained in the returned
+            DataFrame as a column. This is of particular importance for more
+            complicated data types, such as :class:`NormalizedChannelData
+            <evefile.entities.data.NormalizedChannelData>`,
+            :class:`AverageChannelData
+            <evefile.entities.data.AverageChannelData>`,
+            and :class:`IntervalChannelData
+            <evefile.entities.data.IntervalChannelData>`.
+
         .. important::
 
             While working with a Pandas DataFrame may seem convenient,
@@ -457,12 +522,12 @@ class EveFile(File):
         Parameters
         ----------
         data : :class:`list`
-            (Names/IDs of) data objects whose data should be joined.
+            (Names/IDs of) data objects whose data should be included.
 
             You can provide either names or IDs or the actual data objects.
 
             If no data are given, by default all data available will be
-            joined.
+            included.
 
             Default: :obj:`None`
 
@@ -473,6 +538,19 @@ class EveFile(File):
 
             Default: "AxisOrChannelPositions"
 
+        include_monitors : :class:`bool`
+            Whether to include (all) monitors in the dataframe.
+
+            If set to :obj:`True`, all monitors available will automatically
+            be mapped to :obj:`DeviceData <evefile.entities.data.DeviceData>`
+            objects and included in the dataframe.
+
+            Note that for mapped monitors, their IDs are used as column
+            names rather than the "given" names, as for monitors, names are
+            *not unique*.
+
+            Default: :obj:`False`
+
         Returns
         -------
         dataframe : :class:`pandas.DataFrame`
@@ -484,11 +562,18 @@ class EveFile(File):
         """
         if not data:
             data = list(self.data.values())
-        joined_data = self.get_joined_data(data=data, mode=mode)
+        joined_data = self.get_joined_data(
+            data=data, mode=mode, include_monitors=include_monitors
+        )
+        for item in joined_data:
+            # Ensure IDs are used for monitors, as names are not unique
+            if isinstance(item, evefile.entities.data.DeviceData):
+                item.metadata.name = item.metadata.id
         dataframe = pd.DataFrame(
             {item.metadata.name: item.data for item in joined_data}
         )
-        dataframe.index.name = "PosRef"
+        dataframe.index = joined_data[0].position_counts
+        dataframe.index.name = "position"
         return dataframe
 
     def show_info(self):
@@ -552,3 +637,93 @@ class EveFile(File):
         print("\nMONITORS")
         for item in self.monitors.values():
             print(item)
+
+    def get_monitors(self, monitors=None):
+        """
+        Retrieve monitors as mapped device data objects by ID.
+
+        While you can get the original monitor data objects by accessing the
+        :attr:`monitors <evefile.entities.file.File.monitors>` attribute
+        directly, there, they are stored as
+        :obj:`MonitorData <evefile.entities.data.MonitorData>` objects with
+        timestamps instead of position counts. To relate monitors to the
+        measured data, the former need to be mapped to position counts.
+
+        .. note::
+
+            Monitors, in contrast to data objects stored in the
+            :attr:`data <evefile.entities.file.File.data>` attribute,
+            have *no unique names*. Hence, the only way to unequivocally
+            access a given monitor (or the respective, mapped
+            :obj:`DeviceData <evefile.entities.data.DeviceData>` object) is
+            by means of its unique ID.
+
+
+        Parameters
+        ----------
+        monitors : :class:`str` | :class:`list`
+            Name or list of names of monitors to retrieve
+
+        Returns
+        -------
+        device_data : :class:`evefile.entities.data.Data` | :class:`list`
+            Device data object(s) corresponding to the ID(s).
+
+            In case of a list of data objects, each object is of type
+            :class:`evefile.entities.data.DeviceData`.
+
+        """
+        device_data = []
+        if not monitors:
+            monitors = list(self.monitors)
+        if isinstance(monitors, (list, tuple)):
+            for item in monitors:
+                device_data.append(self._monitor_mapper.map(item))
+        else:
+            device_data.append(self._monitor_mapper.map(monitors))
+        if len(device_data) == 1:
+            device_data = device_data[0]
+        return device_data
+
+    def get_snapshots(self):
+        """
+        Retrieve Pandas DataFrame with snapshots as rows.
+
+        Snapshots serve generally two functions:
+
+        #. Provide base values for axes.
+
+           In case of joining data using :meth:`get_joined_data`, for axes,
+           typically the previous values are used for positions no axes
+           values have been recorded. Snapshots are used if available.
+
+        #. Provide telemetry data for the setup the data were recorded with.
+
+           Snapshots regularly contain many more parameters than motor axes
+           used and detector channels recorded. Generally, this provides a
+           lot of telemetry data regarding the setup used for recording the
+           data.
+
+        The first function is served by the :meth:`get_joined_data` method
+        automatically. The second function can be served by having a look at
+        a summary containing all snapshot data. This is the aim of this
+        method: returning a Pandas DataFrame containing all snapshots as
+        rows and the position counts as columns.
+
+
+        Returns
+        -------
+        snapshot_dataframe : :class:`pandas.DataFrame`
+            Pandas DataFrame containing all snapshots as rows.
+
+            The indices (names of the rows) are the names (not IDs) of the
+            respective snapshot datasets.
+
+        """
+        snapshot_dataframes = []
+        for snapshot in self.snapshots.values():
+            data = dict(zip(snapshot.position_counts, snapshot.data))
+            snapshot_dataframes.append(
+                pd.DataFrame(data=data, index=[snapshot.metadata.name])
+            )
+        return pd.concat(snapshot_dataframes)

evefile/controllers/joining.py

@@ -132,7 +132,7 @@ Monitors have not changed by definition
 ---------------------------------------
 
 A monitor is by definition a device you are observing for changes in its
-values. Hence, if no chance has been recorded (*i.e.*, no newer value is
+values. Hence, if no change has been recorded (*i.e.*, no newer value is
 present), the value hasn't changed. Therefore, monitor data can be
 "filled" with the last known value, and in contrast to axes, we can be
 certain that this is true. (For an axis, you could always argue that if
@@ -165,8 +165,12 @@ NoFill
     have values."
 
     Actually, not a filling, but mathematically an intersection, or,
-    in terms of relational databases, an ``SQL INNER JOIN``. In any case,
-    data are *reduced*.
+    in terms of relational databases, an ``SQL INNER JOIN``. Note, however,
+    that in contrast to the usual terminology of relational databases,
+    no individual "tables" (*i.e.*, datasets) are joined, but only the
+    set unions of channel and axes positions, respectively. In any case,
+    data are *reduced*. Nevertheless, you will most probably end up with
+    ``NaN`` or otherwise missing values. See below for details.
 
 LastFill
     "Use all channel data and fill in the last known position for all axes
@@ -220,7 +224,9 @@ main/standard section yet.
 .. note::
 
     Note that **none of the fill modes guarantees that there are no NaNs**
-    (or comparable null values) in the resulting data.
+    (or comparable null values) in the resulting data. The reason: Not
+    individual axes or channel datasets are joined, but always the set union
+    of all axes positions and all channel positions, respectively.
 
 
 .. note::
@@ -235,9 +241,13 @@ main/standard section yet.
 
     The IDL Cruncher seems to use LastNaNFill combined with applying some
     "dirty" fixes to account for scans using MPSKIP and those scans
-    "monitoring" a motor position via a pseudo-detector. The ``EveHDF``
-    class (DS) uses LastNaNFill as a default as well but does *not* apply
-    some additional post-processing.
+    "monitoring" a motor position via a pseudo-detector.
+
+    The ``EveHDF`` class uses LastNaNFill as a default as well but
+    additionally fills the channel values with the last known value,
+    thus making up data. Therefore, EveHDF is considered harmful with
+    potentially very serious consequences of your data analysis and cannot
+    be recommended for use.
 
     Shall fill modes be something to change in a viewer? And which fill
     modes are used in practice (and do we have any chance to find this out)?
@@ -330,26 +340,20 @@ final joined data array previously determined for all the other datasets
 (that in turn depend on the join mode, of course).
 
 
-A few comments for further development
-======================================
-
-.. important::
-
-    The classes implemented in this module have been copied from the
-    corresponding module in `evedata`_, and here particularly from the
-    "measurement" functional layer. However, the needs in ``evefile`` are
-    different, hence even the basic :class:`Join` class needs to be
-    redesigned. Further information below. Once this has been done,
-    this entire section should be removed.
-
+How to deal with additional attributes?
+=======================================
 
-* Joining should probably take into account all available attributes,
-  not only ``data``, but options as well if present. This of course only
-  applies to ``evefile`` if options of devices are mapped to the
-  respective data objects.
+Joining should take into account all available attributes, not only
+``data``, but options as well if present. This of course only applies to
+``evefile`` if options of devices are mapped to the respective data
+objects.
 
-* Joining should probably take into account monitors that need to be
-  converted to :class:`DeviceData <evefile.entities.data.DeviceData>` before.
+A similar case already implemented: More advanced channels such as average
+and interval channels that come with additional data per each recorded
+value (see :class:`AverageChannelData
+<evefile.entities.data.AverageChannelData>` and
+:class:`IntervalChannelData <evefile.entities.data.IntervalChannelData>`
+for details) have these attributes handled accordingly.
 
 
 Join modes currently implemented
@@ -466,7 +470,6 @@ import logging
 from functools import reduce
 
 import numpy as np
-from numpy import ma
 
 import evefile.entities
 import evefile.entities.data
@@ -499,7 +502,7 @@ class Join:
 
     Attributes
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile object the Join should be performed for.
 
         Although joining is carried out for a small subset of the
@@ -508,7 +511,7 @@ class Join:
 
     Parameters
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile object the join should be performed for.
 
 
@@ -523,7 +526,7 @@ class Join:
 
     .. code-block::
 
-        join = Join(evefile=my_evefile)
+        join = Join(file=my_evefile)
         # Call with data object names
         joined_data = join.join(["name1", "name2"])
         # Call with data objects
@@ -533,12 +536,13 @@ class Join:
 
     """
 
-    def __init__(self, evefile=None):
+    def __init__(self, file=None):
         self._channel_indices = []
         self._axes = []
         self._channels = []
+        self._devices = []
         self._result_positions = None
-        self.evefile = evefile
+        self.file = file
 
     def join(self, data=None):
         """
@@ -574,32 +578,18 @@ class Join:
             Raised if no data are provided
 
         """
-        if not self.evefile:
+        if not self.file:
             raise ValueError("Need an evefile to join data.")
         if not data:
             raise ValueError("Need data to join data.")
-        data = [
-            (
-                self._convert_str_to_data_object(item)
-                if isinstance(item, str)
-                else item
-            )
-            for item in data
-        ]
         return self._join(data=data)
 
-    def _convert_str_to_data_object(self, name_or_id=""):
-        try:
-            result = self.evefile.data[name_or_id]
-        except KeyError:
-            result = self.evefile.get_data(name_or_id)
-        return result
-
     def _join(self, data=None):
         self._sort_data(data)
         self._assign_result_positions()
         self._fill_axes()
         self._fill_channels()
+        self._fill_devices()
         return self._assign_result()
 
     def _sort_data(self, data):
@@ -609,68 +599,35 @@ class Join:
                 self._channel_indices.append(idx)
             if isinstance(item, evefile.entities.data.AxisData):
                 self._axes.append(copy.copy(item))
+            if isinstance(item, evefile.entities.data.DeviceData):
+                self._devices.append(copy.copy(item))
 
     def _assign_result_positions(self):
         pass
 
     def _fill_axes(self):
         for axis in self._axes:
-            if axis.metadata.id in self.evefile.snapshots:
-                self.evefile.snapshots[axis.metadata.id].get_data()
-                axis.position_counts = np.searchsorted(
-                    axis.position_counts,
-                    self.evefile.snapshots[axis.metadata.id].position_counts,
-                )
-                axis.data = np.insert(
-                    axis.data,
-                    axis.position_counts,
-                    self.evefile.snapshots[axis.metadata.id].data,
-                )
-            positions = (
-                np.searchsorted(
-                    axis.position_counts, self._result_positions, side="right"
+            if axis.metadata.id in self.file.snapshots:
+                axis.join(
+                    positions=self._result_positions,
+                    snapshot=self.file.snapshots[axis.metadata.id],
                 )
-                - 1
-            )
-            axis.position_counts = self._result_positions
-            axis.data = axis.data[positions]
-            if np.any(np.where(positions < 0)):
-                axis.data = ma.masked_array(axis.data)
-                axis.data[np.where(positions < 0)] = ma.masked
+            else:
+                axis.join(positions=self._result_positions, fill=True)
 
     def _fill_channels(self):
         for channel in self._channels:
-            if len(self._result_positions) > len(channel.position_counts):
-                original_values = channel.data
-                channel.data = ma.masked_array(
-                    np.zeros(len(self._result_positions))
-                )
-                channel.data = ma.masked_array(channel.data)
-                positions = np.setdiff1d(
-                    self._result_positions, channel.position_counts
-                )
-                channel.data[
-                    np.searchsorted(
-                        self._result_positions, channel.position_counts
-                    ).astype(np.int64)
-                ] = original_values
-                channel.data[
-                    np.searchsorted(self._result_positions, positions).astype(
-                        np.int64
-                    )
-                ] = ma.masked
-            elif len(self._result_positions) < len(channel.position_counts):
-                channel.data = channel.data[
-                    np.searchsorted(
-                        channel.position_counts, self._result_positions
-                    ).astype(np.int64)
-                ]
-            channel.position_counts = self._result_positions
+            channel.join(positions=self._result_positions)
+
+    def _fill_devices(self):
+        for device in self._devices:
+            device.join(positions=self._result_positions)
 
     def _assign_result(self):
         result = [*self._axes]
         for idx, item in enumerate(self._channels):
             result.insert(self._channel_indices[idx], item)
+        result.extend(self._devices)
         return result
 
 
@@ -722,7 +679,7 @@ class ChannelPositions(Join):
 
     Attributes
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile object the join should be performed for.
 
         Although joining may only be carried out for a small subset of the
@@ -732,7 +689,7 @@ class ChannelPositions(Join):
 
     Parameters
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile the join should be performed for.
 
 
@@ -747,7 +704,7 @@ class ChannelPositions(Join):
 
     .. code-block::
 
-        join = ChannelPositions(evefile=my_evefile)
+        join = ChannelPositions(file=my_evefile)
         # Call with data object names
         joined_data = join.join(["name1", "name2"])
         # Call with data objects
@@ -814,7 +771,7 @@ class AxisPositions(Join):
 
     Attributes
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile object the join should be performed for.
 
         Although joining may only be carried out for a small subset of the
@@ -824,7 +781,7 @@ class AxisPositions(Join):
 
     Parameters
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile the join should be performed for.
 
 
@@ -839,7 +796,7 @@ class AxisPositions(Join):
 
     .. code-block::
 
-        join = AxisPositions(evefile=my_evefile)
+        join = AxisPositions(file=my_evefile)
         # Call with data object names
         joined_data = join.join(["name1", "name2"])
         # Call with data objects
@@ -909,7 +866,7 @@ class AxisAndChannelPositions(Join):
 
     Attributes
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile object the join should be performed for.
 
         Although joining may only be carried out for a small subset of the
@@ -919,7 +876,7 @@ class AxisAndChannelPositions(Join):
 
     Parameters
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile the join should be performed for.
 
 
@@ -934,7 +891,7 @@ class AxisAndChannelPositions(Join):
 
     .. code-block::
 
-        join = AxisAndChannelPositions(evefile=my_evefile)
+        join = AxisAndChannelPositions(file=my_evefile)
         # Call with data object names
         joined_data = join.join(["name1", "name2"])
         # Call with data objects
@@ -1006,7 +963,7 @@ class AxisOrChannelPositions(Join):
 
     Attributes
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile object the join should be performed for.
 
         Although joining may only be carried out for a small subset of the
@@ -1016,7 +973,7 @@ class AxisOrChannelPositions(Join):
 
     Parameters
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile the join should be performed for.
 
 
@@ -1031,7 +988,7 @@ class AxisOrChannelPositions(Join):
 
     .. code-block::
 
-        join = AxisOrChannelPositions(evefile=my_evefile)
+        join = AxisOrChannelPositions(file=my_evefile)
         # Call with data object names
         joined_data = join.join(["name1", "name2"])
         # Call with data objects
@@ -1068,13 +1025,13 @@ class JoinFactory:
 
     Attributes
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile the join should be performed for.
 
 
     Parameters
     ----------
-    evefile : :class:`evefile.boundaries.evefile.EveFile`
+    file : :class:`evefile.boundaries.evefile.EveFile`
         EveFile the join should be performed for.
 
 
@@ -1107,8 +1064,8 @@ class JoinFactory:
 
     """
 
-    def __init__(self, evefile=None):
-        self.evefile = evefile
+    def __init__(self, file=None):
+        self.file = file
 
     def get_join(self, mode="Join"):
         """
@@ -1134,5 +1091,5 @@ class JoinFactory:
             Join instance
 
         """
-        instance = globals()[mode](evefile=self.evefile)
+        instance = globals()[mode](file=self.file)
         return instance