dcscope 2.22.1__py3-none-any.whl

dcscope/pipeline/dataslot.py

@@ -0,0 +1,270 @@
+import copy
+import functools
+
+import dclab
+from dclab.features.emodulus.viscosity import KNOWN_MEDIA
+import numpy as np
+
+from ..idiom import SLOPING_FEATURES
+from .. import meta_tool
+from ..util import hashobj
+
+
+class Dataslot(object):
+    """Handles datasets in a pipeline"""
+    _instance_counter = 0
+    _instances = {}
+
+    def __init__(self, path, identifier=None, name=None):
+        Dataslot._instance_counter += 1
+        self.path = path
+        self._dataset = None
+        if identifier is None:
+            identifier = "Dataslot_{}".format(Dataslot._instance_counter)
+            while identifier in Dataslot._instances:
+                Dataslot._instance_counter += 1
+                identifier = "Dataslot_{}".format(Dataslot._instance_counter)
+        cfg = meta_tool.get_rtdc_config(path)
+        if name is None:
+            name = cfg["experiment"]["sample"]
+        #: session-unique identifier of the slot
+        self.identifier = identifier
+        #: user-defined name of the slot
+        self.name = name
+        #: whether to use this slot
+        self.slot_used = True
+        if identifier in Dataslot._instances:
+            raise ValueError("Dataslot with identifier "
+                             + "'{}' already exists!".format(identifier))
+        Dataslot._instances[identifier] = self
+        self.color = random_color()
+        self.fl_name_dict = {"FL-1": "FL-1",
+                             "FL-2": "FL-2",
+                             "FL-3": "FL-3"}
+        is_channel = cfg["setup"]["chip region"] == "channel"
+        self.config = {
+            "crosstalk": {
+                # crosstalk
+                "crosstalk fl12": 0,
+                "crosstalk fl13": 0,
+                "crosstalk fl21": 0,
+                "crosstalk fl23": 0,
+                "crosstalk fl31": 0,
+                "crosstalk fl32": 0,
+            },
+            "emodulus": {
+                # emodulus
+                "emodulus enabled": is_channel,  # False for reservoir
+                "emodulus lut": "LE-2D-FEM-19",
+                "emodulus medium": "undefined",
+                # https://dclab.readthedocs.io/en/latest/sec_av_emodulus.html
+                # possible values are:
+                # - "feature": scenario A
+                # - None: secnario B ("emodulus medium" is "other" or user-def)
+                # - "config": scenario C (temperature taken from config)
+                # - "manual": scenario C (temperature entered manually)
+                "emodulus scenario": "manual",
+                "emodulus temperature": np.nan,
+                "emodulus viscosity": np.nan,
+                "emodulus viscosity model": "buyukurganci-2022",
+            }
+        }
+
+        # use the emodulus medium and temperature values as defaults
+        ds = self.get_dataset()
+        calc = self.config["emodulus"]
+        if "medium" in ds.config["setup"]:
+            calc["emodulus medium"] = ds.config["setup"]["medium"]
+        if "temp" in ds:
+            # use the "temp" feature
+            calc["emodulus scenario"] = "feature"
+        elif "temperature" in ds.config["setup"]:
+            # use the average temperature
+            calc["emodulus temperature"] = ds.config["setup"]["temperature"]
+            calc["emodulus scenario"] = "config"
+
+        #: data file format
+        self.format = ds.format
+
+    def __getstate__(self):
+        state = {"color": self.color,
+                 "crosstalk": self.config["crosstalk"],
+                 "emodulus": self.config["emodulus"],
+                 "fl names": self.fl_name_dict,
+                 "identifier": self.identifier,
+                 "name": self.name,
+                 "path": self.path,
+                 "slot used": self.slot_used,
+                 }
+        return copy.deepcopy(state)
+
+    def __repr__(self):
+        repre = "<Pipeline Slot '{}' at {}>".format(self.identifier,
+                                                    hex(id(self)))
+        return repre
+
+    def __setstate__(self, state):
+        if self.identifier != state["identifier"]:
+            raise ValueError("Identifier mismatch: '{}' vs. '{}'".format(
+                self.identifier, state["identifier"]))
+        self.color = state["color"]
+        self.config["crosstalk"].update(state["crosstalk"])
+        self.config["emodulus"].update(state["emodulus"])
+        self.fl_name_dict = state["fl names"]
+        self.name = state["name"]
+        self.path = state["path"]
+        self.slot_used = state["slot used"]
+
+    @staticmethod
+    def get_slot(slot_id):
+        """Get the slot with the given identifier"""
+        return Dataslot._instances[slot_id]
+
+    @staticmethod
+    def get_instances():
+        return Dataslot._instances
+
+    @staticmethod
+    def remove_slot(slot_id):
+        """Remove a slot taking care of closing any opened files"""
+        slot = Dataslot.get_slot(slot_id)
+        ds = slot._dataset
+        if ds is not None:
+            if isinstance(ds, dclab.rtdc_dataset.RTDC_HDF5):
+                ds.h5file.close()
+        Dataslot._instances.pop(slot_id)
+
+    @property
+    def hash(self):
+        """Return the hash of the slot"""
+        return hashobj(self.__getstate__())
+
+    def _set_emodulus_config(self, dataset):
+        """Set the Young's modulus data options
+
+        The three cases in the dclab docs apply:
+        https://dclab.readthedocs.io/en/latest/sec_av_emodulus.html
+        """
+        # remove any information
+        for key in self.config["emodulus"]:
+            if key in dataset.config["calculation"]:
+                dataset.config["calculation"].pop(key)
+
+        lut = self.config["emodulus"]["emodulus lut"]
+        medium = self.config["emodulus"]["emodulus medium"]
+        visc = self.config["emodulus"]["emodulus viscosity"]
+        visc_model = self.config["emodulus"]["emodulus viscosity model"]
+        scenario = self.config["emodulus"]["emodulus scenario"]
+        if scenario == "config":
+            # Force the temperature from the dataset metadata
+            # (this is a failsafe, the user/developer might not have set it)
+            temp = dataset.config["setup"]["temperature"]
+        elif scenario == "manual":
+            # Only here do we actually need the temperature stored
+            temp = self.config["emodulus"]["emodulus temperature"]
+        else:
+            # Temperature is not used in these scenarios.
+            temp = np.nan
+
+        dataset.config["calculation"]["emodulus lut"] = lut
+        # known media
+        if medium in dclab.features.emodulus.viscosity.KNOWN_MEDIA:
+            dataset.config["calculation"]["emodulus medium"] = medium
+            dataset.config["calculation"]["emodulus viscosity model"] = \
+                visc_model
+        # temperature
+        if not np.isnan(temp):
+            dataset.config["calculation"]["emodulus temperature"] = temp
+        # viscosity
+        if medium not in KNOWN_MEDIA and not np.isnan(visc):
+            dataset.config["calculation"]["emodulus viscosity"] = visc
+            if "emodulus viscosity model" in dataset.config["calculation"]:
+                dataset.config["calculation"].pop("emodulus viscosity model")
+
+    def get_dataset(self):
+        """Return the corresponding dataset
+
+        Returns
+        -------
+        ds: dclab.RTDCBase
+            Loaded dataset
+        """
+        if self._dataset is None:
+            ds = dclab.new_dataset(self.path)
+            self._dataset = ds
+        else:
+            ds = self._dataset
+        self.update_dataset(ds)
+        return ds
+
+    def get_sane_spacing_range(self, feat):
+        """Return sane contour spacing range for this dataset and feature"""
+        return get_sane_contour_spacing_range_for_slot_id(
+            self.identifier, feat)
+
+    def update_dataset(self, dataset):
+        """Update the configuration of an instance of RTDCBase
+
+        This is used to update the configuration for computing
+        the Young's modulus and fluorescence crosstalk.
+        """
+        # emodulus
+        self._set_emodulus_config(dataset)
+
+        # crosstalk
+        if np.sum(list(self.config["crosstalk"].values())):
+            dataset.config["calculation"].update(self.config["crosstalk"])
+        else:
+            # remove any information
+            for key in self.config["crosstalk"]:
+                if key in dataset.config["calculation"]:
+                    dataset.config["calculation"].pop(key)
+
+
+@functools.lru_cache(1000)
+def get_sane_contour_spacing_range_for_slot_id(slot_id, feat):
+    slot = Dataslot.get_instances()[slot_id]
+    ds = slot.get_dataset()
+    sp_min, sp_max = get_sane_contour_spacing_range(feat, ds[feat][:])
+    return sp_min, sp_max
+
+
+def get_sane_contour_spacing_range(feat, data):
+    """Return a sane range for contour spacing for a feature
+
+    Parameters
+    ----------
+    feat: str
+        Name of the feature; If this is in :const:`.SLOPING_FEATURES`,
+        then spacing takes into account first and last item in `data`.
+        Otherwise, the first 10000 elements of `data` are used to
+        guess a sane contour spacing.
+    data: 1d ndarray
+        feature data
+    """
+    if feat in SLOPING_FEATURES:
+        frange = np.abs(data[-1] - data[0])
+        if np.isnan(frange) or np.isinf(frange):
+            invalid = np.logical_or(np.isinf(data), np.isnan(data))
+            data_valid = data[~invalid]
+            frange = np.abs(data_valid[-1] - data_valid[0])
+    else:
+        frange = np.ptp(data[:10000])
+        if np.isnan(frange) or np.isinf(frange):
+            invalid = np.logical_or(np.isinf(data), np.isnan(data))
+            data_valid = data[~invalid]
+            frange = np.ptp(data_valid[:10000])
+    spmin = frange / 1000
+    spmax = frange / 5
+    return spmin, spmax
+
+
+def random_color():
+    color = "#"
+    for _ in range(3):
+        # dark colors (until 200)
+        part = hex(np.random.randint(0, 200))[2:]
+        if len(part) == 1:
+            part += "0"
+        color += part
+    return color.upper()

dcscope/pipeline/filter.py

@@ -0,0 +1,161 @@
+"""Keep record of all filters used"""
+import copy
+
+import dclab
+
+from ..util import hashobj
+
+
+class Filter(object):
+    """Handles filters in a pipeline"""
+    _instance_counter = 0
+    _instances = {}
+
+    def __init__(self, identifier=None, name=None):
+        Filter._instance_counter += 1
+        if identifier is None:
+            identifier = "Filter_{}".format(Filter._instance_counter)
+            while identifier in Filter._instances:
+                Filter._instance_counter += 1
+                identifier = "Filter_{}".format(Filter._instance_counter)
+
+        if name is None:
+            name = identifier
+        #: unique identifier of the filter
+        self.identifier = identifier
+        #: user-defined name of the filter
+        self.name = name
+        #: general filtering arguments are directly passed to
+        #: :class:`dclab.filter.Filter`
+        self.general = {"remove invalid events": False,  # removes nan/inf
+                        "enable filters": True,  # whether to use the filter
+                        }
+        #: filter for limiting number of events
+        self.limit_events = [False, 5000]
+        #: box filters with features as keys; each item is a
+        #: dictionary with the keys "start", "end", "active"
+        self.boxdict = {}
+        #: polygon filter list; each item is an identifier of
+        #: :class:`dclab.PolygonFilter`
+        self.polylist = []
+        if identifier in Filter._instances:
+            raise ValueError("Filter with identifier "
+                             + "'{}' already exists!".format(identifier))
+        Filter._instances[identifier] = self
+
+    def __getstate__(self):
+        state = {
+            "filter used": self.filter_used,
+            "identifier": self.identifier,
+            "limit events bool": self.limit_events[0],
+            "limit events num": self.limit_events[1],
+            "name": self.name,
+            "remove invalid events": self.general["remove invalid events"],
+            "box filters": self.boxdict,
+            "polygon filters": sorted(self.polylist),
+        }
+        return copy.deepcopy(state)
+
+    def __repr__(self):
+        repre = "<Pipeline Filter '{}' at {}>".format(self.identifier,
+                                                      hex(id(self)))
+        return repre
+
+    def __setstate__(self, state):
+        state = copy.deepcopy(state)
+        if self.identifier != state["identifier"]:
+            raise ValueError("Identifier mismatch: '{}' vs. '{}'".format(
+                self.identifier, state["identifier"]))
+        self.general["enable filters"] = state["filter used"]
+        self.limit_events = [state["limit events bool"],
+                             state["limit events num"]]
+        self.name = state["name"]
+        self.general["remove invalid events"] = state["remove invalid events"]
+        self.boxdict = state["box filters"]
+        self.polylist = state["polygon filters"]
+
+    @staticmethod
+    def get_filter(identifier):
+        """Get the filter with the given identifier.
+
+        Notes
+        -----
+        Creates the filter if it does not exist.
+        """
+        if identifier in Filter._instances:
+            f = Filter._instances[identifier]
+        else:
+            f = Filter(identifier=identifier)
+        return f
+
+    @staticmethod
+    def get_instances():
+        return Filter._instances
+
+    @property
+    def filter_used(self):
+        return self.general["enable filters"]
+
+    @filter_used.setter
+    def filter_used(self, b):
+        self.general["enable filters"] = b
+
+    @property
+    def hash(self):
+        """Return the hash of the filter"""
+        return hashobj(self.__getstate__())
+
+    def add_box_filter(self, feature, start, end, active=True):
+        """Add a box filter"""
+        if not dclab.dfn.scalar_feature_exists(feature):
+            raise ValueError("Unknown scalar feature: {}".format(feature))
+        self.boxdict[feature] = {
+            "start": start,
+            "end": end,
+            "active": active}
+
+    def apply_to_dataset(self, dataset):
+        """Convenience function to apply this filter to a dataset
+
+        Parameters
+        ----------
+        dataset: :class:`dclab.RTDCBase`
+            Input dataset
+
+        Returns
+        -------
+        filtered_dataset: RTDCBase
+            Either the input dataset, or a hierarchy child.
+
+        Notes
+        -----
+
+        """
+        self.update_dataset(dataset)
+        dataset.apply_filter()
+
+    def update_dataset(self, dataset):
+        """Update the filtering configuration of a dataset
+
+        Notes
+        -----
+        Due to the design of the filtering pipeline, it is not
+        possible to use manual filters. If any are set, they
+        are removed from the filter.
+        """
+        # remove all previous filters
+        dataset.reset_filter()
+        cfgfilt = dataset.config["filtering"]
+
+        # set general options
+        cfgfilt.update(self.general)
+        cfgfilt["limit events"] = self.limit_events[0] * self.limit_events[1]
+        # set box filters
+        for feat in self.boxdict:
+            if self.boxdict[feat]["active"]:
+                cfgfilt["{} min".format(feat)] = self.boxdict[feat]["start"]
+                cfgfilt["{} max".format(feat)] = self.boxdict[feat]["end"]
+
+        # set polygon filters
+        for pid in self.polylist:
+            dataset.polygon_filter_add(pid)

dcscope/pipeline/filter_ray.py

@@ -0,0 +1,170 @@
+import dclab
+
+
+class FilterRay(object):
+    def __init__(self, slot):
+        """Manages filter-based dataset hierarchies
+
+        Filter rays are used to cache RTDCBase filter-hierarchy
+        children.
+        """
+        #: identifier of the ray (matches the slot)
+        self.identifier = slot.identifier
+        #: slot defining the ray
+        self.slot = slot
+        #: list of RTDCBase (hierarchy children)
+        self.steps = []
+        #: corresponds to hashes of the applied filters
+        self.step_hashes = []
+        # holds the filters (protected so that users use set_filters)
+        self._filters = []
+        # used for testing (incremented when the ray is cut)
+        self._generation = 0
+        # used for checking validity of the ray
+        self._slot_hash = "unset"
+        self._root_child = None
+
+    def __repr__(self):
+        repre = "<Pipeline Filter Ray '{}' at {}>".format(self.identifier,
+                                                          hex(id(self)))
+        return repre
+
+    def _add_step(self, ds, filt):
+        """Add a filter step"""
+        self.step_hashes.append(filt.hash)
+        filt.update_dataset(ds)
+        self.steps.append(ds)
+        return self._new_child(ds, filt)
+
+    def _new_child(self, ds, filt=None, apply_filter=False):
+        identifier = self.slot.identifier
+        if filt is None:
+            identifier += "-root"
+        else:
+            identifier += "-" + filt.identifier + "-child"
+        ds = dclab.rtdc_dataset.RTDC_Hierarchy(
+            ds, apply_filter=apply_filter, identifier=identifier)
+        return ds
+
+    @property
+    def filters(self):
+        """filters currently used by the ray
+
+        Notes
+        -----
+        This list may not be up-to-date. If you would like to
+        get the current list of filters for a dataset, always
+        use :func:`.Pipeline.get_filters_for_slot`.
+        """
+        return self._filters
+
+    @property
+    def root_child(self):
+        """This is the first element in self.steps
+        (Will return a dataset even if self.steps is empty)
+        """
+        if self._slot_hash != self.slot.hash:
+            # reset everything (e.g. emodulus recipe might have changed)
+            self.steps = []
+            self.step_hashes = []
+            self._root_child = self._new_child(self.slot.get_dataset(),
+                                               apply_filter=True)
+            self._slot_hash = self.slot.hash
+        return self._root_child
+
+    def get_final_child(self, rtdc_ds=None, apply_filter=True):
+        """Return the final ray child of `rtdc_ds`
+
+        If `rtdc_ds` is None, then the dataset of the current
+        ray (self.slot) is used. If `rtdc_ds` is given, then
+        no ray caching is performed and the present ray is not
+        modified.
+
+        This is a convenience function used when the filter ray
+        must be applied to a different dataset (not the one in
+        `self.slot`). This is used in DCscope when a filter ray
+        is applied to other data on disk e.g. when computing
+        statistics. For regular use of the filter ray in a
+        pipeline, use :func:`get_dataset`.
+        """
+        filters = self.filters
+
+        if rtdc_ds is None:
+            # normal case
+            external = False
+            rtdc_ds = self.slot.get_dataset()
+            ds = self.root_child
+        else:
+            # ray is applied to other data
+            external = True
+            # do not modify rtdc_ds (create a child to work with)
+            ds = self._new_child(rtdc_ds, apply_filter=True)
+
+        # Dear future self,
+        #
+        # don't even think about filter ray branching.
+        #
+        # Sincerely,
+        # past self
+
+        if filters:
+            # apply all filters
+            for ii, filt in enumerate(filters):
+                # remember the previous hierarchy parent
+                # (ds is always used for the next iteration)
+                prev_ds = ds
+                if external:
+                    # do not touch self.steps or self.step_hashes
+                    filt.update_dataset(ds)
+                    ds = self._new_child(ds, filt)
+                elif len(self.steps) < ii+1:
+                    # just create a new step
+                    ds = self._add_step(ds, filt)
+                elif filt.hash != self.step_hashes[ii]:
+                    # the filter ray is changing here;
+                    # cut it and add a new step
+                    self.steps = self.steps[:ii]
+                    self.step_hashes = self.step_hashes[:ii]
+                    ds = self._add_step(ds, filt)
+                    self._generation += 1  # for testing
+                else:
+                    # the filters match so far
+                    if len(self.steps) > ii + 1:  # next child exists
+                        ds = self.steps[ii + 1]
+                    else:  # next child does not exist
+                        ds = self._new_child(ds, filt)
+            # we now have the entire filter pipeline in self.steps
+            final_ds = prev_ds
+        else:
+            final_ds = rtdc_ds
+        if apply_filter:
+            final_ds.apply_filter()
+        return final_ds
+
+    def get_dataset(self, filters=None, apply_filter=True):
+        """Return the dataset that corresponds to applying these filters
+
+        Parameters
+        ----------
+        filters: list of Filter or None
+            Filters used for computing the dataset hierarchy. If set
+            to None, the current filters in `self.filters` are used.
+        apply_filter: bool
+            Whether to apply all filters and update the metadata of
+            the requested dataset. This should be True if you are
+            intending to work with the resulting data. You can set
+            it to false if you would just like to fetch the dataset,
+            apply some more filters and then call `rejuvenate`
+            yourself.
+        """
+        if filters is not None:
+            # put the filters in place
+            self.set_filters(filters)
+        # compute the final hierarchy child
+        ds = self.get_final_child(apply_filter=apply_filter)
+        return ds
+
+    def set_filters(self, filters):
+        """Set the filters of the current ray"""
+        # only take into account active filters
+        self._filters = [f for f in filters if f.filter_used]