dclab 0.62.11__cp313-cp313-win_amd64.whl

dclab/statistics.py

@@ -0,0 +1,203 @@
+"""Statistics computation for RT-DC dataset instances"""
+
+import numpy as np
+import traceback as tb
+import warnings
+
+from . import definitions as dfn
+
+
+class BadMethodWarning(UserWarning):
+    pass
+
+
+class Statistics(object):
+    available_methods = {}
+
+    def __init__(self, name, method, req_feature=False):
+        """A helper class for computing statistics
+
+        All statistical methods are registered in the dictionary
+        `Statistics.available_methods`.
+        """
+        self.method = method
+        self.name = name
+        self.req_feature = req_feature
+        Statistics.available_methods[name] = self
+
+    def __call__(self, **kwargs):
+        data = self._get_data(kwargs)
+        if len(data) == 0:
+            result = np.nan
+        else:
+            try:
+                result = self.method(data)
+            except BaseException:
+                exc = tb.format_exc().replace("\n", "\n    | ")
+                warnings.warn("Failed to compute {} for {}: {}".format(
+                              self.name, kwargs["ds"].title, exc),
+                              BadMethodWarning)
+                result = np.nan
+        return result
+
+    def _get_data(self, kwargs):
+        """Convenience wrapper to get statistics data"""
+        if "ds" not in kwargs:
+            raise ValueError("Keyword argument 'ds' missing.")
+
+        ds = kwargs["ds"]
+
+        if self.req_feature:
+            if "feature" not in kwargs:
+                raise ValueError("Keyword argument 'feature' missing.")
+            return self.get_feature(ds, kwargs["feature"])
+        else:
+            return ds
+
+    def get_feature(self, ds, feat):
+        """Return filtered feature data
+
+        The features are filtered according to the user-defined filters,
+        using the information in `ds.filter.all`. In addition, all
+        `nan` and `inf` values are purged.
+
+        Parameters
+        ----------
+        ds: dclab.rtdc_dataset.RTDCBase
+            The dataset containing the feature
+        feat: str
+            The name of the feature; must be a scalar feature
+        """
+        if ds.config["filtering"]["enable filters"]:
+            x = ds[feat][ds.filter.all]
+        else:
+            x = ds[feat]
+        bad = np.isnan(x) | np.isinf(x)
+        xout = x[~bad]
+        return xout
+
+
+def flow_rate(ds):
+    """Return the flow rate of an RT-DC dataset"""
+    conf = ds.config["setup"]
+    if "flow rate" in conf:
+        return conf["flow rate"]
+    else:
+        return np.nan
+
+
+def get_statistics(ds, methods=None, features=None):
+    """Compute statistics for an RT-DC dataset
+
+    Parameters
+    ----------
+    ds: dclab.rtdc_dataset.RTDCBase
+        The dataset for which to compute the statistics.
+    methods: list of str or None
+        The methods wih which to compute the statistics.
+        The list of available methods is given with
+        `dclab.statistics.Statistics.available_methods.keys()`
+        If set to `None`, statistics for all methods are computed.
+    features: list of str
+        Feature name identifiers are defined by
+        `dclab.definitions.feature_exists`.
+        If set to `None`, statistics for all scalar features
+        available are computed.
+
+    Returns
+    -------
+    header: list of str
+        The header (feature + method names) of the computed statistics.
+    values: list of float
+        The computed statistics.
+    """
+    if methods is None:
+        cls = list(Statistics.available_methods.keys())
+        # sort the features in a usable way
+        avm = Statistics.available_methods
+        me1 = [m for m in cls if not avm[m].req_feature]
+        me2 = [m for m in cls if avm[m].req_feature]
+        methods = me1 + me2
+
+    if features is None:
+        features = ds.features_scalar
+    else:
+        features = [a.lower() for a in features]
+
+    header = []
+    values = []
+
+    # First loop over all methods that do not require a feature
+    for mt in methods:
+        meth = Statistics.available_methods[mt]
+        if not meth.req_feature:
+            values.append(meth(ds=ds))
+            header.append(mt)
+
+    # To make sure that all methods are computed for each feature in a block,
+    # we loop over all features. It would be easier to loop over the methods,
+    # but the ordering of the resulting statistics would not be human-friendly.
+    for ft in features:
+        for mt in methods:
+            meth = Statistics.available_methods[mt]
+            if meth.req_feature:
+                if ft in ds:
+                    values.append(meth(ds=ds, feature=ft))
+                else:
+                    values.append(np.nan)
+                label = dfn.get_feature_label(ft, rtdc_ds=ds)
+                header.append(" ".join([mt, label]))
+
+    return header, values
+
+
+def mode(data):
+    """Compute an intelligent value for the mode
+
+    The most common value in experimental is not very useful if there
+    are a lot of digits after the comma. This method approaches this
+    issue by rounding to bin size that is determined by the
+    Freedman–Diaconis rule.
+
+    Parameters
+    ----------
+    data: 1d ndarray
+        The data for which the mode should be computed.
+
+    Returns
+    -------
+    mode: float
+        The mode computed with the Freedman-Diaconis rule.
+    """
+    # size
+    n = data.shape[0]
+    # interquartile range
+    iqr = np.percentile(data, 75)-np.percentile(data, 25)
+    # Freedman–Diaconis
+    bin_size = 2 * iqr / n**(1/3)
+
+    if bin_size == 0:
+        return np.nan
+
+    # Add bin_size/2, because we want the center of the bin and
+    # not the left corner of the bin.
+    databin = np.round(data/bin_size)*bin_size + bin_size/2
+    u, indices = np.unique(databin, return_inverse=True)
+    mode = u[np.argmax(np.bincount(indices))]
+
+    return mode
+
+
+# Register all the methods
+# Methods that require an axis
+Statistics(name="Mean",   req_feature=True, method=np.average)
+Statistics(name="Median", req_feature=True, method=np.median)
+Statistics(name="Mode",   req_feature=True, method=mode)
+Statistics(name="SD",     req_feature=True, method=np.std)
+# Methods that work on RTDCBase
+Statistics(name="Events",
+           method=lambda mm: np.sum(mm.filter.all))
+Statistics(name="%-gated",
+           method=lambda mm: np.average(mm.filter.all)*100)
+Statistics(name="Flow rate",
+           method=lambda mm: flow_rate(mm))

dclab/util.py

@@ -0,0 +1,156 @@
+"""Utility methods"""
+import functools
+import hashlib
+import numbers
+import pathlib
+import warnings
+
+import h5py
+import numpy as np
+from .rtdc_dataset.config import Configuration, ConfigurationDict
+
+
+if np.lib.NumpyVersion(np.__version__) >= "2.0.0":
+    copy_if_needed = None
+else:
+    copy_if_needed = False
+
+
+class file_monitoring_lru_cache:
+    """Decorator for caching data extracted from files
+
+    The function that is decorated with `file_monitoring_lru_cache`
+    must accept `path` as its first argument. Caching is
+    done with an `lru_cache`. In addition to the full path
+    and the other arguments to the decorated function, the
+    size and the modification time of `path` is used as a
+    key for the decorator.
+    If the path does not exist, no caching is done.
+
+    Use case: Extract and cache metadata from a file on disk
+    that may change.
+    """
+    def __init__(self, maxsize=100):
+        self.lru_cache = functools.lru_cache(maxsize=maxsize)
+        self.cached_wrapper = None
+
+    def __call__(self, func):
+        @self.lru_cache
+        def cached_wrapper(path, path_stats, *args, **kwargs):
+            assert path_stats, "We need stat for validating the cache"
+            return func(path, *args, **kwargs)
+
+        @functools.wraps(func)
+        def wrapper(path, *args, **kwargs):
+            full_path = pathlib.Path(path).resolve()
+            if full_path.exists():
+                path_stat = full_path.stat()
+                return cached_wrapper(
+                    path=full_path,
+                    path_stats=(path_stat.st_mtime_ns, path_stat.st_size),
+                    *args,
+                    **kwargs)
+            else:
+                # `func` will most-likely raise an exception
+                return func(path, *args, **kwargs)
+
+        wrapper.cache_clear = cached_wrapper.cache_clear
+        wrapper.cache_info = cached_wrapper.cache_info
+
+        return wrapper
+
+
+@file_monitoring_lru_cache(maxsize=100)
+def hashfile(fname, blocksize=65536, count=0, constructor=hashlib.md5,
+             hasher_class=None):
+    """Compute md5 hex-hash of a file
+
+    Parameters
+    ----------
+    fname: str or pathlib.Path
+        path to the file
+    blocksize: int
+        block size in bytes read from the file
+        (set to `0` to hash the entire file)
+    count: int
+        number of blocks read from the file
+    hasher_class: callable
+        deprecated, see use `constructor` instead
+    constructor: callable
+        hash algorithm constructor
+    """
+    if hasher_class is not None:
+        warnings.warn("The `hasher_class` argument is deprecated, please use "
+                      "`constructor` instead.")
+        constructor = hasher_class
+
+    path = pathlib.Path(fname)
+
+    hasher = constructor()
+    with path.open('rb') as fd:
+        buf = fd.read(blocksize)
+        ii = 0
+        while len(buf) > 0:
+            hasher.update(buf)
+            buf = fd.read(blocksize)
+            ii += 1
+            if count and ii == count:
+                break
+    return hasher.hexdigest()
+
+
+def hashobj(obj):
+    """Compute md5 hex-hash of a Python object"""
+    return hashlib.md5(obj2bytes(obj)).hexdigest()
+
+
+def obj2bytes(obj):
+    """Bytes representation of an object for hashing
+
+    Note that there is no guarantee that the bytes representation
+    returned is reproducible across sessions. This is currently the
+    case when an :class:`.RTDCBase` instance is passed. There is no
+    opinion on wether/how this should be changed.
+    """
+    if isinstance(obj, str):
+        return obj.encode("utf-8")
+    elif isinstance(obj, pathlib.Path):
+        return obj2bytes(str(obj))
+    elif isinstance(obj, (bool, numbers.Number)):
+        return str(obj).encode("utf-8")
+    elif obj is None:
+        return b"none"
+    elif isinstance(obj, np.ndarray):
+        return obj.tobytes()
+    elif isinstance(obj, tuple):
+        return obj2bytes(list(obj))
+    elif isinstance(obj, list):
+        return b"".join(obj2bytes(o) for o in obj)
+    elif isinstance(obj, dict):
+        return obj2bytes(sorted(obj.items()))
+    elif hasattr(obj, "identifier"):
+        # For RTDCBase, this identifier is not reproducible in-between
+        # sessions. We might want to change this to something that is
+        # reproducible in the future (if the need arises).
+        return obj2bytes(obj.identifier)
+    elif isinstance(obj, h5py.Dataset):
+        # path within the HDF5 file
+        o_name = obj.name
+        # filename
+        o_filename = obj.file.filename
+        _data = [o_name, o_filename]
+        if pathlib.Path(o_filename).exists():
+            # when the file was changed
+            _data.append(pathlib.Path(obj.file.filename).stat().st_mtime)
+            # size of the file
+            _data.append(pathlib.Path(obj.file.filename).stat().st_size)
+        return obj2bytes(_data)
+    elif hasattr(obj, "__array__"):  # must come after h5py.Dataset
+        return obj2bytes(obj.__array__())
+    elif isinstance(obj, Configuration):
+        return obj2bytes(obj.tostring())
+    elif isinstance(obj, ConfigurationDict):
+        return obj2bytes(dict(obj))
+    else:
+        raise ValueError("No rule to convert object '{}' to string.".
+                         format(obj.__class__))

dclab/warn.py

@@ -0,0 +1,15 @@
+
+
+class PipelineWarning(UserWarning):
+    """Super-class for warnings relevant to data analysis
+
+    There are those types of warnings in dclab that are
+    important to the user, because they suggest that the
+    user may not use the correct model (e.g. Young's modulus
+    computation) in his analysis pipeline. All of these
+    warnings should be subclassed from PipelineWarning
+    to allow identifying them in higher-level software
+    such as Shape-Out and to present them correctly to the
+    user.
+    """
+    pass