dclab 0.67.0__cp314-cp314t-macosx_10_13_x86_64.whl

dclab/statistics.py

@@ -0,0 +1,226 @@
+"""Statistics computation for RT-DC dataset instances"""
+
+import numpy as np
+import traceback as tb
+import warnings
+
+from . import definitions as dfn
+
+
+class BadMethodWarning(UserWarning):
+    pass
+
+
+class Statistics(object):
+    available_methods = {}
+
+    def __init__(self, name, method, req_feature=False):
+        """A helper class for computing statistics
+
+        All statistical methods are registered in the dictionary
+        `Statistics.available_methods`.
+        """
+        self.method = method
+        self.name = name
+        self.req_feature = req_feature
+        Statistics.available_methods[name] = self
+
+    def __call__(self, **kwargs):
+        data = self._get_data(kwargs)
+        if len(data) == 0:
+            result = np.nan
+        else:
+            try:
+                result = self.method(data)
+            except BaseException:
+                exc = tb.format_exc().replace("\n", "\n    | ")
+                warnings.warn("Failed to compute {} for {}: {}".format(
+                              self.name, kwargs["ds"].title, exc),
+                              BadMethodWarning)
+                result = np.nan
+        return result
+
+    def _get_data(self, kwargs):
+        """Convenience wrapper to get statistics data"""
+        if "ds" not in kwargs:
+            raise ValueError("Keyword argument 'ds' missing.")
+
+        ds = kwargs["ds"]
+
+        if self.req_feature:
+            if "feature" not in kwargs:
+                raise ValueError("Keyword argument 'feature' missing.")
+            return self.get_feature(ds, kwargs["feature"])
+        else:
+            return ds
+
+    def get_feature(self, ds, feat):
+        """Return filtered feature data
+
+        The features are filtered according to the user-defined filters,
+        using the information in `ds.filter.all`. In addition, all
+        `nan` and `inf` values are purged.
+
+        Parameters
+        ----------
+        ds: dclab.rtdc_dataset.RTDCBase
+            The dataset containing the feature
+        feat: str
+            The name of the feature; must be a scalar feature
+        """
+        if ds.config["filtering"]["enable filters"]:
+            x = ds[feat][ds.filter.all]
+        else:
+            x = ds[feat]
+        bad = np.isnan(x) | np.isinf(x)
+        xout = x[~bad]
+        return xout
+
+
+def flow_rate(ds):
+    """Return the flow rate of an RT-DC dataset"""
+    conf = ds.config["setup"]
+    if "flow rate" in conf:
+        return conf["flow rate"]
+    else:
+        return np.nan
+
+
+def get_statistics(ds, methods=None, features=None, ret_dict=False):
+    """Compute statistics for an RT-DC dataset
+
+    Parameters
+    ----------
+    ds: dclab.rtdc_dataset.RTDCBase
+        The dataset for which to compute the statistics.
+    methods: list of str or None
+        The methods wih which to compute the statistics.
+        The list of available methods is given with
+        :func:`.available_methods.keys`
+        If set to `None`, statistics for all methods are computed.
+    features: list of str
+        Feature name identifiers are defined by
+        :func:`dclab.definitions.feature_exists`.
+        If set to `None`, statistics for all scalar features
+        available are computed.
+    ret_dict: bool
+        Instead of returning ``(header, values)``, return a dictionary
+        with headers as keys.
+
+    Returns
+    -------
+    header: list of str
+        The header (feature + method names) of the computed statistics.
+    values: list of float
+        The computed statistics.
+    """
+    if methods is None:
+        cls = list(Statistics.available_methods.keys())
+        # sort the features in a usable way
+        avm = Statistics.available_methods
+        me1 = [m for m in cls if not avm[m].req_feature]
+        me2 = [m for m in cls if avm[m].req_feature]
+        methods = me1 + me2
+
+    if features is None:
+        features = ds.features_scalar
+    else:
+        features = [a.lower() for a in features]
+
+    header = []
+    values = []
+
+    # First loop over all methods that do not require a feature
+    for mt in methods:
+        meth = Statistics.available_methods[mt]
+        if not meth.req_feature:
+            values.append(meth(ds=ds))
+            header.append(mt)
+
+    # To make sure that all methods are computed for each feature in a block,
+    # we loop over all features. It would be easier to loop over the methods,
+    # but the ordering of the resulting statistics would not be human-friendly.
+    for ft in features:
+        for mt in methods:
+            meth = Statistics.available_methods[mt]
+            if meth.req_feature:
+                if ft in ds:
+                    values.append(meth(ds=ds, feature=ft))
+                else:
+                    values.append(np.nan)
+                label = dfn.get_feature_label(ft, rtdc_ds=ds)
+                header.append(" ".join([mt, label]))
+
+    if ret_dict:
+        return dict(zip(header, values))
+    else:
+        return header, values
+
+
+def mode(data):
+    """Compute an intelligent value for the mode
+
+    The most common value in experimental is not very useful if there
+    are a lot of digits after the comma. This method approaches this
+    issue by rounding to bin size that is determined by the
+    Freedman–Diaconis rule.
+
+    Parameters
+    ----------
+    data: 1d ndarray
+        The data for which the mode should be computed.
+
+    Returns
+    -------
+    mode: float
+        The mode computed with the Freedman-Diaconis rule.
+    """
+    # size
+    n = data.shape[0]
+    # interquartile range
+    iqr = np.percentile(data, 75)-np.percentile(data, 25)
+    # Freedman–Diaconis
+    bin_size = 2 * iqr / n**(1/3)
+
+    if bin_size == 0:
+        return np.nan
+
+    # Add bin_size/2, because we want the center of the bin and
+    # not the left corner of the bin.
+    databin = np.round(data/bin_size)*bin_size + bin_size/2
+    u, indices = np.unique(databin, return_inverse=True)
+    mode = u[np.argmax(np.bincount(indices))]
+
+    return mode
+
+
+# Register all the methods
+# Methods that require an axis
+Statistics(name="Mean",   req_feature=True, method=np.average)
+# Premature-Optimization warning: `np.percentile` also accepts an array
+# of percentiles as the `q` argument, which I would expect to yield better
+# performance than computing percentiles individually. Implementing this
+# would break the way we are defining statistical methods here (One
+# `Statistics` instance per method) and thus requires a considerable
+# amount of work (much more work than writing this text here). It would
+# also make understanding the code more difficult. In addition, computing
+# statistics is not done often and is extremely fast anyway for a few
+# millions of events. Don't optimize this!
+Statistics(name="10th Percentile", req_feature=True,
+           method=lambda data: np.percentile(data, 10))
+Statistics(name="25th Percentile", req_feature=True,
+           method=lambda data: np.percentile(data, 25))
+Statistics(name="Median", req_feature=True, method=np.median)
+Statistics(name="75th Percentile", req_feature=True,
+           method=lambda data: np.percentile(data, 75))
+Statistics(name="90th Percentile", req_feature=True,
+           method=lambda data: np.percentile(data, 90))
+Statistics(name="Mode",   req_feature=True, method=mode)
+Statistics(name="SD",     req_feature=True, method=np.std)
+# Methods that work on RTDCBase
+Statistics(name="Events",
+           method=lambda mm: np.sum(mm.filter.all))
+Statistics(name="%-gated",
+           method=lambda mm: np.average(mm.filter.all)*100)
+Statistics(name="Flow rate",
+           method=lambda mm: flow_rate(mm))

dclab/util.py

@@ -0,0 +1,176 @@
+"""Utility methods"""
+import functools
+import hashlib
+import importlib
+import numbers
+import pathlib
+import warnings
+
+import h5py
+import numpy as np
+from .rtdc_dataset.config import Configuration, ConfigurationDict
+
+
+if np.lib.NumpyVersion(np.__version__) >= "2.0.0":
+    copy_if_needed = None
+else:
+    copy_if_needed = False
+
+
+class file_monitoring_lru_cache:
+    """Decorator for caching data extracted from files
+
+    The function that is decorated with `file_monitoring_lru_cache`
+    must accept `path` as its first argument. Caching is
+    done with an `lru_cache`. In addition to the full path
+    and the other arguments to the decorated function, the
+    size and the modification time of `path` is used as a
+    key for the decorator.
+    If the path does not exist, no caching is done.
+
+    Use case: Extract and cache metadata from a file on disk
+    that may change.
+    """
+    def __init__(self, maxsize=100):
+        self.lru_cache = functools.lru_cache(maxsize=maxsize)
+        self.cached_wrapper = None
+
+    def __call__(self, func):
+        @self.lru_cache
+        def cached_wrapper(path, path_stats, *args, **kwargs):
+            assert path_stats, "We need stat for validating the cache"
+            return func(path, *args, **kwargs)
+
+        @functools.wraps(func)
+        def wrapper(path, *args, **kwargs):
+            full_path = pathlib.Path(path).resolve()
+            if full_path.exists():
+                path_stat = full_path.stat()
+                return cached_wrapper(
+                    path=full_path,
+                    path_stats=(path_stat.st_mtime_ns, path_stat.st_size),
+                    *args,
+                    **kwargs)
+            else:
+                # `func` will most-likely raise an exception
+                return func(path, *args, **kwargs)
+
+        wrapper.cache_clear = cached_wrapper.cache_clear
+        wrapper.cache_info = cached_wrapper.cache_info
+
+        return wrapper
+
+
+class LazyLoader():
+    """Lazy load a module on first attribute access"""
+
+    def __init__(self, modname):
+        self._modname = modname
+        self._mod = None
+
+    def __getattr__(self, attr):
+        """Get attribute from module, load module if not loaded yet"""
+
+        if self._mod is None:
+            # module is unset, load it
+            self._mod = importlib.import_module(self._modname)
+
+        # retry getattr if module was just loaded for first time
+        # call this outside exception handler in case it raises new exception
+        return getattr(self._mod, attr)
+
+
+@file_monitoring_lru_cache(maxsize=100)
+def hashfile(fname, blocksize=65536, count=0, constructor=hashlib.md5,
+             hasher_class=None):
+    """Compute md5 hex-hash of a file
+
+    Parameters
+    ----------
+    fname: str or pathlib.Path
+        path to the file
+    blocksize: int
+        block size in bytes read from the file
+        (set to `0` to hash the entire file)
+    count: int
+        number of blocks read from the file
+    hasher_class: callable
+        deprecated, see use `constructor` instead
+    constructor: callable
+        hash algorithm constructor
+    """
+    if hasher_class is not None:
+        warnings.warn("The `hasher_class` argument is deprecated, please use "
+                      "`constructor` instead.")
+        constructor = hasher_class
+
+    path = pathlib.Path(fname)
+
+    hasher = constructor()
+    with path.open('rb') as fd:
+        buf = fd.read(blocksize)
+        ii = 0
+        while len(buf) > 0:
+            hasher.update(buf)
+            buf = fd.read(blocksize)
+            ii += 1
+            if count and ii == count:
+                break
+    return hasher.hexdigest()
+
+
+def hashobj(obj):
+    """Compute md5 hex-hash of a Python object"""
+    return hashlib.md5(obj2bytes(obj)).hexdigest()
+
+
+def obj2bytes(obj):
+    """Bytes representation of an object for hashing
+
+    Note that there is no guarantee that the bytes representation
+    returned is reproducible across sessions. This is currently the
+    case when an :class:`.RTDCBase` instance is passed. There is no
+    opinion on wether/how this should be changed.
+    """
+    if isinstance(obj, str):
+        return obj.encode("utf-8")
+    elif isinstance(obj, pathlib.Path):
+        return obj2bytes(str(obj))
+    elif isinstance(obj, (bool, numbers.Number)):
+        return str(obj).encode("utf-8")
+    elif obj is None:
+        return b"none"
+    elif isinstance(obj, np.ndarray):
+        return obj.tobytes()
+    elif isinstance(obj, tuple):
+        return obj2bytes(list(obj))
+    elif isinstance(obj, list):
+        return b"".join(obj2bytes(o) for o in obj)
+    elif isinstance(obj, dict):
+        return obj2bytes(sorted(obj.items()))
+    elif hasattr(obj, "identifier"):
+        # For RTDCBase, this identifier is not reproducible in-between
+        # sessions. We might want to change this to something that is
+        # reproducible in the future (if the need arises).
+        return obj2bytes(obj.identifier)
+    elif isinstance(obj, h5py.Dataset):
+        # path within the HDF5 file
+        o_name = obj.name
+        # filename
+        o_filename = obj.file.filename
+        _data = [o_name, o_filename]
+        if pathlib.Path(o_filename).exists():
+            # when the file was changed
+            _data.append(pathlib.Path(obj.file.filename).stat().st_mtime)
+            # size of the file
+            _data.append(pathlib.Path(obj.file.filename).stat().st_size)
+        return obj2bytes(_data)
+    elif hasattr(obj, "__array__"):  # must come after h5py.Dataset
+        return obj2bytes(obj.__array__())
+    elif isinstance(obj, Configuration):
+        return obj2bytes(obj.tostring())
+    elif isinstance(obj, ConfigurationDict):
+        return obj2bytes(dict(obj))
+    else:
+        raise ValueError("No rule to convert object '{}' to string.".
+                         format(obj.__class__))

dclab/warn.py

@@ -0,0 +1,15 @@
+
+
+class PipelineWarning(UserWarning):
+    """Super-class for warnings relevant to data analysis
+
+    There are those types of warnings in dclab that are
+    important to the user, because they suggest that the
+    user may not use the correct model (e.g. Young's modulus
+    computation) in his analysis pipeline. All of these
+    warnings should be subclassed from PipelineWarning
+    to allow identifying them in higher-level software
+    such as DCscope and to present them correctly to the
+    user.
+    """
+    pass

dclab-0.67.0.dist-info/METADATA

@@ -0,0 +1,153 @@
+Metadata-Version: 2.4
+Name: dclab
+Version: 0.67.0
+Summary: Library for real-time deformability cytometry (RT-DC)
+Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
+Maintainer-email: Paul Müller <dev@craban.de>
+License-Expression: GPL-2.0-or-later
+Project-URL: source, https://github.com/DC-Analysis/dclab
+Project-URL: tracker, https://github.com/DC-Analysis/dclab/issues
+Project-URL: documentation, https://dclab.readthedocs.io/en/stable/
+Project-URL: changelog, https://dclab.readthedocs.io/en/stable/sec_changelog.html
+Keywords: RT-DC,deformability,cytometry
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Visualization
+Classifier: Intended Audience :: Science/Research
+Requires-Python: <4,>=3.9
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: h5py<4,>=3.0.0
+Requires-Dist: hdf5plugin<5,>=3.3.1
+Requires-Dist: importlib-resources>=6.0
+Requires-Dist: numpy<3,>=1.21
+Requires-Dist: scipy<2,>=1.10.0
+Provides-Extra: all
+Requires-Dist: dclab[dcor,export,http,s3,tdms]; extra == "all"
+Provides-Extra: dcor
+Requires-Dist: requests<3,>=2.31.0; extra == "dcor"
+Provides-Extra: export
+Requires-Dist: fcswrite>=0.5.1; extra == "export"
+Requires-Dist: av; extra == "export"
+Provides-Extra: http
+Requires-Dist: requests<3,>=2.31.0; extra == "http"
+Provides-Extra: s3
+Requires-Dist: boto3>=1.34.31; extra == "s3"
+Provides-Extra: tdms
+Requires-Dist: imageio[ffmpeg]; extra == "tdms"
+Requires-Dist: nptdms<1.9,>=0.23.0; extra == "tdms"
+Dynamic: license-file
+
+|dclab|
+=======
+
+|PyPI Version| |Build Status| |Coverage Status| |Docs Status|
+
+
+This is a Python library for the post-measurement analysis of
+real-time deformability cytometry (RT-DC) datasets; an essential part of
+the DC Cosmos (
+`DCscope <https://github.com/DC-analysis/DCscope>`__,
+`DCOR <https://github.com/DCOR-dev/dcor_control>`__,
+`DCOR-Aid <https://github.com/DCOR-dev/DCOR-Aid>`__,
+`DCTag <https://github.com/DC-analysis/DCTag>`__,
+`DCKit <https://github.com/DC-analysis/DCKit>`__,
+).
+
+Documentation
+-------------
+The documentation, including the code reference and examples, is available at
+`dclab.readthedocs.io <https://dclab.readthedocs.io/en/stable/>`__.
+
+
+Installation
+------------
+
+::
+
+    pip install dclab[all]
+
+For more options, please check out the `documentation
+<https://dclab.readthedocs.io/en/latest/sec_getting_started.html#installation>`__.
+
+
+Information for developers
+--------------------------
+
+
+Contributing
+~~~~~~~~~~~~
+The main branch for developing dclab is master.
+If you want to make small changes like one-liners,
+documentation, or default values in the configuration,
+you may work on the master branch. If you want to change
+more, please (fork dclab and) create a separate branch,
+e.g. ``my_new_feature_dev``, and create a pull-request
+once you are done making your changes.
+Please make sure to edit the 
+`Changelog <https://github.com/DC-analysis/dclab/blob/master/CHANGELOG>`__.
+
+**Very important:** Please always try to use ::
+
+
+    git pull --rebase
+
+instead of::
+
+    git pull
+
+to prevent non-linearities in the commit history.
+
+Tests
+~~~~~
+dclab is tested using pytest. If you have the time, please write test
+methods for your code and put them in the ``tests`` directory. To run
+the tests, install `pytest` and run::
+
+    pytest tests
+
+
+Docs
+~~~~
+The docs are built with `sphinx <https://www.sphinx-doc.org>`_. Please make
+sure they compile when you change them (this also includes function doc strings)::
+
+    cd docs
+    pip install -r requirements.txt
+    sphinx-build . _build  # open "index.html" in the "_build" directory
+
+
+PEP8
+~~~~
+We use flake8 to enforce coding style::
+
+    pip install flake8
+    flake8 --exclude _version.py dclab
+    flake8 docs
+    flake8 examples
+    flake8 tests
+
+
+Incrementing version
+~~~~~~~~~~~~~~~~~~~~
+Dclab gets its version from the latest git tag.
+If you think that a new version should be published,
+create a tag on the master branch (if you have the necessary
+permissions to do so)::
+
+    git tag -a "0.1.3"
+    git push --tags origin
+
+Appveyor and GitHub Actions will then automatically build source package and wheels
+and publish them on PyPI.
+
+
+.. |dclab| image:: https://raw.github.com/DC-analysis/dclab/master/docs/logo/dclab.png
+.. |PyPI Version| image:: https://img.shields.io/pypi/v/dclab.svg
+   :target: https://pypi.python.org/pypi/dclab
+.. |Build Status| image:: https://img.shields.io/github/actions/workflow/status/DC-analysis/dclab/check.yml
+   :target: https://github.com/DC-analysis/dclab/actions?query=workflow%3AChecks
+.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/DC-analysis/dclab/master.svg
+   :target: https://codecov.io/gh/DC-analysis/dclab
+.. |Docs Status| image:: https://readthedocs.org/projects/dclab/badge/?version=latest
+   :target: https://readthedocs.org/projects/dclab/builds/