dclab 0.67.0__cp314-cp314-macosx_11_0_arm64.whl

dclab/rtdc_dataset/feat_anc_core/ancillary_feature.py

@@ -0,0 +1,320 @@
+"""Computation of ancillary features
+
+Ancillary features are computed on-the-fly in dclab if the
+required data are available. The features are registered here
+and are computed when `RTDCBase.__getitem__` is called with
+the respective feature name. When `RTDCBase.__contains__` is
+called with the feature name, then the feature is not yet
+computed, but the prerequisites are evaluated:
+
+.. ipython::
+    :okwarning:
+
+    In [1]: import dclab
+
+    In [2]: ds = dclab.new_dataset("data/example.rtdc")
+
+    In [4]: ds.config["calculation"]["emodulus lut"] = "LE-2D-FEM-19"
+
+    In [3]: ds.config["calculation"]["emodulus medium"] = "CellCarrier"
+
+    In [5]: ds.config["calculation"]["emodulus temperature"] = 23.0
+
+    In [6]: ds.config["calculation"]["emodulus viscosity model"] = \
+'buyukurganci-2022'
+
+    In [7]: "emodulus" in ds  # nothing is computed yet
+
+    In [8]: ds["emodulus"]  # now data are computed and cached
+
+Once the data has been computed, `RTDCBase` caches it in
+the `_ancillaries` property dict together with a hash
+that is computed with `AncillaryFeature.hash`. The hash
+is computed from the feature data `req_features` and the
+configuration metadata `req_config`.
+"""
+
+import hashlib
+import warnings
+
+import numpy as np
+
+from ...util import obj2bytes
+from ... import definitions as dfn
+
+
+class BadFeatureSizeWarning(UserWarning):
+    pass
+
+
+class AncillaryFeature:
+    #: All ancillary features registered
+    features = []
+    #: All feature names registered
+    feature_names = []
+
+    def __init__(self, feature_name, method, req_config=None,
+                 req_features=None, req_func=lambda x: True, priority=0,
+                 data=None, identifier=None):
+        """A data feature that is computed from existing data
+
+        Parameters
+        ----------
+        feature_name: str
+            The name of the ancillary feature, e.g. "emodulus".
+        method: callable
+            The method that computes the feature. This method
+            takes an instance of `RTDCBase` as argument.
+        req_config: list
+            Required configuration parameters to compute the feature,
+            e.g. ["calculation", ["emodulus lut", "emodulus viscosity"]]
+        req_features: list
+            Required existing features in the dataset,
+            e.g. ["area_cvx", "deform"]
+        req_func: callable
+            A function that takes an instance of `RTDCBase` as an
+            argument and checks whether any other necessary criteria
+            are met. By default, this is a lambda function that returns
+            True. The function should return False if the necessary
+            criteria are not met. This function may also return a
+            hashable object (via :func:`dclab.util.objstr`) instead of
+            True, if the criteria are subject to change. In this case,
+            the return value is used for identifying the cached
+            ancillary feature.
+
+            .. versionchanged:: 0.27.0
+                Support non-boolean return values for caching purposes.
+
+        priority: int
+            The priority of the feature; if there are multiple
+            AncillaryFeature defined for the same feature_name,
+            then the priority of the features defines which feature
+            returns True in `self.is_available`. A higher value
+            means a higher priority.
+        data: object or BaseModel
+            Any other data relevant for the feature (e.g. the ML
+            model for computing 'ml_score_xxx' features)
+        identifier: None or str
+            A unique identifier (e.g. MD5 hash) of the ancillary
+            feature. For PluginFeatures or ML features, this should
+            be computed at least from the input file and the feature
+            name.
+
+        Notes
+        -----
+        `req_config` and `req_features` are used to test whether the
+        feature can be computed in `self.is_available`.
+        """
+        if req_features is None:
+            req_features = []
+        if req_config is None:
+            req_config = []
+        self.feature_name = feature_name
+        self.method = method
+        self.req_config = req_config
+        self.req_features = req_features
+        self.req_func = req_func
+        self.priority = priority
+        self.data = data
+        self.identifier = identifier
+
+        # register this feature
+        AncillaryFeature.features.append(self)
+        AncillaryFeature.feature_names.append(feature_name)
+
+    def __repr__(self):
+        repre = " ".join([
+            f"<{self.__class__.__name__}",
+            f"'{self.feature_name}'",
+            f"(id {self.identifier[:5]}...)" if self.identifier else "(no ID)",
+            f"with priority {self.priority}",
+            f"at {hex(id(self))}>",
+        ])
+        return repre
+
+    @staticmethod
+    def available_features(rtdc_ds):
+        """Determine available features for an RT-DC dataset
+
+        Parameters
+        ----------
+        rtdc_ds: instance of RTDCBase
+            The dataset to check availability for
+
+        Returns
+        -------
+        features: dict
+            Dictionary with feature names as keys and instances
+            of `AncillaryFeature` as values.
+        """
+        # TODO: This is quite slow.
+        cols = {}
+        for inst in AncillaryFeature.features:
+            if inst.is_available(rtdc_ds):
+                cols[inst.feature_name] = inst
+        return cols
+
+    @staticmethod
+    def get_instances(feature_name):
+        """Return all instances that compute `feature_name`"""
+        feats = []
+        for ft in AncillaryFeature.features:
+            if ft.feature_name == feature_name:
+                feats.append(ft)
+        return feats
+
+    @staticmethod
+    def check_data_size(rtdc_ds, data_dict):
+        """Check the feature data is the correct size. If it isn't, resize it.
+
+        Parameters
+        ----------
+        rtdc_ds: instance of RTDCBase
+            The dataset from which the features are computed
+        data_dict: dict
+            Dictionary with `AncillaryFeature.feature_name` as keys and the
+            computed data features (to be resized) as values.
+
+        Returns
+        -------
+        data_dict: dict
+            Dictionary with `feature_name` as keys and the correctly resized
+            data features as values.
+        """
+        for key in data_dict:
+            dsize = len(rtdc_ds) - len(data_dict[key])
+            if dsize > 0:
+                msg = "Growing feature {} in {} by {} to match event number!"
+                warnings.warn(msg.format(key, rtdc_ds, abs(dsize)),
+                              BadFeatureSizeWarning)
+                data_dict[key] = np.array(data_dict[key], dtype=float)
+                data_dict[key].resize(len(rtdc_ds), refcheck=False)
+                data_dict[key][-dsize:] = np.nan
+            elif dsize < 0:
+                msg = "Shrinking feature {} in {} by {} to match event number!"
+                warnings.warn(msg.format(key, rtdc_ds, abs(dsize)),
+                              BadFeatureSizeWarning)
+                data_dict[key].resize(len(rtdc_ds), refcheck=False)
+            if isinstance(data_dict[key], np.ndarray):
+                data_dict[key].setflags(write=False)
+            elif isinstance(data_dict[key], list):
+                for item in data_dict[key]:
+                    if isinstance(item, np.ndarray):
+                        item.setflags(write=False)
+        return data_dict
+
+    def compute(self, rtdc_ds):
+        """Compute the feature with self.method. All ancillary features that
+        share the same method will also be populated automatically.
+
+        Parameters
+        ----------
+        rtdc_ds: instance of RTDCBase
+            The dataset to compute the feature for
+
+        Returns
+        -------
+        data_dict: dict
+            Dictionary with `AncillaryFeature.feature_name` as keys and the
+            computed data features (read-only) as values.
+        """
+        data_dict = self.method(rtdc_ds)
+        if not isinstance(data_dict, dict):
+            data_dict = {self.feature_name: data_dict}
+        data_dict = AncillaryFeature.check_data_size(rtdc_ds, data_dict)
+        if self.feature_name not in data_dict:
+            raise KeyError(
+                f"I expected the feature '{self.feature_name}' to be a key "
+                + f"in the dictionary returned by {self}. But I found only "
+                + f"the following: {sorted(data_dict.keys())}")
+        for key in data_dict:
+            dfn.check_feature_shape(key, data_dict[key])
+        return data_dict
+
+    def hash(self, rtdc_ds):
+        """Used for identifying an ancillary computation
+
+        The required features, the used configuration keys/values, and
+        the return value of the requirement function are hashed.
+        """
+        hasher = hashlib.md5()
+        # data columns
+        for col in self.req_features:
+            hasher.update(obj2bytes(rtdc_ds[col]))
+        # config keys
+        for sec, keys in self.req_config:
+            for key in keys:
+                val = rtdc_ds.config[sec][key]
+                data = "{}:{}={}".format(sec, key, val)
+                hasher.update(obj2bytes(data))
+        # custom requirement function hash
+        reqret = self.req_func(rtdc_ds)
+        if not isinstance(reqret, bool):
+            # add to hash if not a boolean
+            hasher.update(obj2bytes(reqret))
+        return hasher.hexdigest()
+
+    def is_available(self, rtdc_ds, verbose=False):
+        """Check whether the feature is available
+
+        Parameters
+        ----------
+        rtdc_ds: instance of RTDCBase
+            The dataset to check availability for
+
+        Returns
+        -------
+        available: bool
+            `True`, if feature can be computed with `compute`
+
+        Notes
+        -----
+        This method returns `False` for a feature if there
+        is a feature defined with the same name but with
+        higher priority (even if the feature would be
+        available otherwise).
+        """
+        # Check config keys
+        for item in self.req_config:
+            section, keys = item
+            if section not in rtdc_ds.config:
+                if verbose:
+                    print("{} not in config".format(section))
+                return False
+            else:
+                for key in keys:
+                    if key not in rtdc_ds.config[section]:
+                        if verbose:
+                            print("{} not in config['{}']".format(key,
+                                                                  section))
+                        return False
+        # Check features
+        for col in self.req_features:
+            if col not in rtdc_ds:
+                return False
+        # Check priorities of other features
+        for of in AncillaryFeature.features:
+            if of == self:
+                # nothing to compare
+                continue
+            elif of.feature_name == self.feature_name:
+                # same feature name
+                if of.priority <= self.priority:
+                    # lower priority, ignore
+                    continue
+                else:
+                    # higher priority
+                    if of.is_available(rtdc_ds):
+                        # higher priority is available, thus
+                        # this feature is not available
+                        return False
+                    else:
+                        # higher priority not available
+                        continue
+            else:
+                # other feature
+                continue
+        # Check user-defined function
+        if not self.req_func(rtdc_ds):
+            return False
+        return True

dclab/rtdc_dataset/feat_anc_ml/__init__.py

@@ -0,0 +1,32 @@
+import warnings
+
+
+class MachineLearningFeature:
+    def __init__(self, *args, **kwargs):
+        raise NotImplementedError(
+            "`MachineLearningFeature` has been stripped from dclab since "
+            "version 0.61.0.")
+
+
+def load_ml_feature(*args, **kwargs):
+    raise NotImplementedError(
+        "`load_ml_feature` has been stripped from dclab since "
+        "version 0.61.0.")
+
+
+def load_modc(*args, **kwargs):
+    raise NotImplementedError(
+        "`load_modc` has been stripped from dclab since "
+        "version 0.61.0.")
+
+
+def remove_all_ml_features():
+    warnings.warn("The `remove_all_ml_features` does nothing since it was "
+                  "stripped from dclab version 0.61.0.",
+                  DeprecationWarning)
+
+
+def save_modc(*args, **kwargs):
+    raise NotImplementedError(
+        "`save_modc` has been stripped from dclab since "
+        "version 0.61.0.")

dclab/rtdc_dataset/feat_anc_plugin/__init__.py

@@ -0,0 +1,3 @@
+# flake8: noqa: F401
+from .plugin_feature import (
+    PlugInFeature, load_plugin_feature, remove_all_plugin_features)

dclab/rtdc_dataset/feat_anc_plugin/plugin_feature.py

@@ -0,0 +1,329 @@
+"""
+.. versionadded:: 0.34.0
+"""
+from __future__ import annotations
+
+import hashlib
+import importlib
+import pathlib
+import sys
+from typing import List, Optional
+
+from ...definitions import feat_logic
+from ..feat_anc_core import AncillaryFeature
+
+
+class PluginImportError(BaseException):
+    pass
+
+
+class PlugInFeature(AncillaryFeature):
+    def __init__(self,
+                 feature_name: str,
+                 info: dict,
+                 plugin_path: Optional[str | pathlib.Path] = None):
+        """A user-defined plugin feature
+
+        Parameters
+        ----------
+        feature_name: str
+            name of a feature that matches that defined in `info`
+        info: dict
+            Full plugin recipe (for all features) as given in the
+            `info` dictionary in the plugin file.
+            At least the following keys must be specified:
+
+            - "method": callable function computing the plugin feature
+              values (takes an :class`dclab.rtdc_dataset.core.RTDCBase`
+              as argument)
+            - "feature names": list of plugin feature names provided
+              by the plugin
+
+            The following features are optional:
+
+            - "description": short (one-line) description of the plugin
+            - "long description": long description of the plugin
+            - "feature labels": feature labels used e.g. for plotting
+            - "feature shapes": list of tuples for each feature
+              indicating the shape (this is required only for
+              non-scalar features; for scalar features simply set
+              this to ``None`` or ``(1,)``).
+            - "scalar feature": list of boolean values indicating
+              whether the features are scalar
+            - "config required": configuration keys required to
+              compute the plugin features (see the `req_config`
+              parameter for :class:`.AncillaryFeature`)
+            - "features required": list of feature names required to
+              compute the plugin features (see the `req_features`
+              parameter for :class:`.AncillaryFeature`)
+            - "method check required": additional method that checks
+              whether the features can be computed (see the `req_func`
+              parameter for :class:`.AncillaryFeature`)
+            - "version": version of this plugin (please use
+              semantic verioning)
+
+        plugin_path: str or pathlib.Path, optional
+            path which was used to load the `PlugInFeature` with
+            :func:`load_plugin_feature`.
+
+        Notes
+        -----
+        `PluginFeature` inherits from :class:`AncillaryFeature
+        <dclab.rtdc_dataset.feat_anc_core.ancillary_feature.AncillaryFeature>`.
+        Please read the advanced section on `PluginFeatures` in the dclab docs.
+        """
+        if plugin_path is not None:
+            plugin_path = pathlib.Path(plugin_path)
+
+        #: Plugin feature name
+        self.feature_name = feature_name
+        #: Path to the original plugin file
+        self.plugin_path = plugin_path
+
+        # perform sanity checks
+        self._sanity_check_original_info(info)
+        # keep this for tests
+        self._original_info = info
+
+        # populate `info` dictionary with missing values
+        #: Dictionary containing all information relevant for
+        #: this particular plugin feature instance
+        self.plugin_feature_info = self._process_plugin_info(info)
+
+        # register this plugin feature in definitions
+        # This must be executed before initializing the super class
+        # (AncillaryFeature). If we don't do this, then `remove_plugin_feature`
+        # may end up removing innate features e.g., "deform".
+        feat_logic.feature_register(
+            name=self.feature_name,
+            label=self.plugin_feature_info["feature label"],
+            is_scalar=self.plugin_feature_info["scalar feature"]
+        )
+
+        # Instantiate the super class
+        super(PlugInFeature, self).__init__(
+            feature_name=self.plugin_feature_info["feature name"],
+            method=self.plugin_feature_info["method"],
+            req_config=self.plugin_feature_info["config required"],
+            req_features=self.plugin_feature_info["features required"],
+            req_func=self.plugin_feature_info["method check required"],
+            identifier=self.plugin_feature_info["identifier"],
+        )
+
+    def _process_plugin_info(self, original_info: dict) -> dict:
+        """Return dictionary with all relevant info for this instance
+        """
+        fidx = original_info["feature names"].index(self.feature_name)
+
+        # determine feature label
+        if ("feature labels" in original_info
+                and original_info["feature labels"][fidx]):
+            label = original_info["feature labels"][fidx]
+        else:
+            label = f"Plugin feature {self.feature_name}"
+
+        # determine whether we have a scalar feature
+        if "scalar feature" in original_info:
+            is_scalar = original_info["scalar feature"][fidx]
+        else:
+            is_scalar = True  # default
+
+        if is_scalar:
+            event_shape = (1,)
+        else:
+            if "feature shapes" in original_info:
+                event_shape = original_info["feature shapes"][fidx]
+            else:
+                event_shape = None
+
+        # We assume that the script does not import any other custom
+        # Python scripts.
+        md5hasher = hashlib.md5()
+        if isinstance(self.plugin_path, pathlib.Path):
+            md5hasher.update(self.plugin_path.read_bytes())
+        else:
+            md5hasher.update(original_info["method"].__code__.co_code)
+        md5hasher.update(self.feature_name.encode("utf-8"))
+        md5hasher.update(original_info.get("version", "").encode("utf-8"))
+        for feat in original_info.get("features required", []):
+            md5hasher.update(feat.encode("utf-8"))
+        identifier = md5hasher.hexdigest()
+
+        feature_info = {
+            "method": original_info["method"],
+            "description": original_info.get(
+                "description", "No description provided"),
+            "long description": original_info.get(
+                "long description", "No long description provided."),
+            "feature name": self.feature_name,
+            "feature label": label,
+            "feature shape": event_shape,
+            "features required": original_info.get("features required", []),
+            "config required": original_info.get("config required", []),
+            "method check required": original_info.get(
+                "method check required", lambda x: True),
+            "scalar feature": is_scalar,
+            # allow comparisons with distutil.version.LooseVersion
+            "version": original_info.get("version", "0.0.0-unknown"),
+            "plugin path": self.plugin_path,
+            "identifier": identifier,
+        }
+
+        return feature_info
+
+    def _sanity_check_original_info(self, original_info: dict):
+        """Various checks on the `original_info` attribute dict
+
+        Raises
+        ------
+        ValueError
+            If the parameter `original_info` is not a dict.
+            If the `self.feature_name` is not in
+                `original_info["feature names"]`.
+            If the `method` provided in parameter `original_info`
+                is not callable.
+        """
+        if not isinstance(original_info, dict):
+            raise ValueError(
+                "PlugInFeature parameter for `info` must be a dict, instead "
+                f"a '{type(original_info)}' was given.")
+
+        if not isinstance(original_info["feature names"], list):
+            raise ValueError("'feature names' must be a list, "
+                             f"got '{type(original_info['feature names'])}'")
+
+        if self.feature_name not in original_info["feature names"]:
+            raise ValueError(
+                f"The feature name '{self.feature_name}' is not defined in "
+                + "the `info` dict of the plugin feature"
+                + (f" at {self.plugin_path}" if self.plugin_path else "")
+                + f". Defined names are '{original_info['feature names']}'.")
+
+        if not callable(original_info["method"]):
+            raise ValueError(
+                "The `method` you have provided in the parameter `info` is "
+                f"not callable ('{original_info['method']}' is not "
+                "a function).")
+
+
+def import_plugin_feature_script(
+        plugin_path: str | pathlib.Path) -> dict:
+    """Import the user-defined recipe and return the info dictionary
+
+    Parameters
+    ----------
+    plugin_path: str or Path
+        pathname to a valid dclab plugin script
+
+    Returns
+    -------
+    info: dict
+        Dictionary with the information required to instantiate
+        one (or multiple) :class:`PlugInFeature`.
+
+    Raises
+    ------
+    PluginImportError
+        If the plugin can not be found
+
+    Notes
+    -----
+    One recipe may define multiple plugin features.
+    """
+    path = pathlib.Path(plugin_path)
+    if not path.exists():
+        raise PluginImportError("The plugin could be not be found at "
+                                f"'{plugin_path}'!")
+    try:
+        # insert the plugin directory to sys.path so we can import it
+        sys.path.insert(-1, str(path.parent))
+        sys.dont_write_bytecode = True
+        plugin = importlib.import_module(path.stem)
+    except BaseException as e:
+        raise PluginImportError(
+            f"The plugin {plugin_path} could not be loaded!") from e
+    finally:
+        # undo our path insertion
+        sys.path.pop(0)
+        sys.dont_write_bytecode = False
+
+    return plugin.info
+
+
+def load_plugin_feature(
+        plugin_path: str | pathlib.Path) -> List[PlugInFeature]:
+    """Find and load PlugInFeature(s) from a user-defined recipe
+
+    Parameters
+    ----------
+    plugin_path: str or Path
+        pathname to a valid dclab plugin Python script
+
+    Returns
+    -------
+    plugin_list: list of PlugInFeature
+        list of PlugInFeature instances loaded from `plugin_path`
+
+    Raises
+    ------
+    ValueError
+        If the script dictionary "feature names" are not a list
+
+    Notes
+    -----
+    One recipe may define multiple plugin features.
+
+    See Also
+    --------
+    import_plugin_feature_script: function that imports the plugin script
+    PlugInFeature: class handling the plugin feature information
+    dclab.rtdc_dataset.feat_temp.register_temporary_feature: alternative
+        method for creating user-defined features
+    """
+    info = import_plugin_feature_script(plugin_path)
+    if not isinstance(info["feature names"], list):
+        raise ValueError(
+            "'feature names' must be a list of strings.")
+
+    plugin_list = []
+    for feature_name in info["feature names"]:
+        plugin_list.append(PlugInFeature(feature_name, info, plugin_path))
+    return plugin_list
+
+
+def remove_all_plugin_features():
+    """Convenience function for removing all `PlugInFeature` instances
+
+    See Also
+    --------
+    remove_plugin_feature: remove a single `PlugInFeature` instance
+    """
+    for plugin_instance in reversed(PlugInFeature.features):
+        if isinstance(plugin_instance, PlugInFeature):
+            remove_plugin_feature(plugin_instance)
+
+
+def remove_plugin_feature(plugin_instance: PlugInFeature):
+    """Convenience function for removing a `PlugInFeature` instance
+
+    Parameters
+    ----------
+    plugin_instance: PlugInFeature
+        The `PlugInFeature` instance to be removed from dclab
+
+    Raises
+    ------
+    TypeError
+        If the `plugin_instance` is not a `PlugInFeature` instance
+    """
+    if isinstance(plugin_instance, PlugInFeature):
+        # This check is necessary for situations where the PlugInFeature fails
+        # between updating the `dclab.dfn` file and initialising the
+        # AncillaryFeature
+        if plugin_instance.feature_name in PlugInFeature.feature_names:
+            PlugInFeature.feature_names.remove(plugin_instance.feature_name)
+            feat_logic.feature_deregister(plugin_instance.feature_name)
+        PlugInFeature.features.remove(plugin_instance)
+    else:
+        raise TypeError(f"Type {type(plugin_instance)} should be an instance "
+                        f"of PlugInFeature. '{plugin_instance}' was given.")