dclab 0.67.0__cp314-cp314-macosx_10_13_x86_64.whl

dclab/__init__.py

@@ -0,0 +1,41 @@
+"""Core tools for the analysis of deformability cytometry datasets
+
+Copyright (C) 2015 Paul Müller
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""
+# flake8: noqa: F401
+from . import definitions as dfn
+from . import features
+from . import isoelastics
+from . import lme4
+from .polygon_filter import PolygonFilter
+from . import rtdc_dataset
+from .rtdc_dataset import new_dataset, IntegrityChecker, RTDCWriter
+from .rtdc_dataset.feat_temp import (
+    register_temporary_feature, set_temporary_feature)
+from .rtdc_dataset.feat_anc_ml import (
+    MachineLearningFeature, load_modc, load_ml_feature, save_modc)
+from .rtdc_dataset.feat_anc_plugin.plugin_feature import (
+    PlugInFeature, load_plugin_feature)
+from . import statistics
+from . import util
+
+from ._version import __version__, __version_tuple__
+
+
+# Lazy-load deprecated kde modules 
+kde_contours = util.LazyLoader("dclab.kde_contours")
+kde_methods = util.LazyLoader("dclab.kde_methods")

dclab/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.67.0'
+__version_tuple__ = version_tuple = (0, 67, 0)
+
+__commit_id__ = commit_id = 'g52c420117'

dclab/cached.py

@@ -0,0 +1,97 @@
+"""Cache for fast "recomputation"
+"""
+import functools
+import gc
+import hashlib
+
+import numpy as np
+
+
+MAX_SIZE = 100
+
+
+class Cache:
+    _cache = {}
+    _keys = []
+
+    def __init__(self, func):
+        """
+        A cache that can be used to decorate methods that accept
+        numpy ndarrays as arguments.
+
+        - cache is based on dictionary
+        - md5 hashes of method arguments are the dictionary keys
+        - applicable decorator for all methods in a module
+        - applicable to methods with the same name in different
+          source files
+        - set cache size with `cached.MAX_SIZE`
+        - only one global cache is generated, there are no instances
+          of `Cache`
+
+        Notes
+        -----
+        If you are using other decorators with this decorator, please
+        make sure to apply the `Cache` first (first line before method
+        definition). This wrapper uses name, doc, and filename of the
+        method to identify it. If another wrapper does not implement
+        a unique `__doc__` and is applied to multiple methods, then
+        `Cached` might return values of the wrong method.
+        """
+        self.func = func
+        functools.update_wrapper(self, func)
+
+    def __call__(self, *args, **kwargs):
+        self.ahash = hashlib.md5()
+
+        # hash arguments
+        for arg in args:
+            self._update_hash(arg)
+
+        # hash keyword arguments
+        kwds = list(kwargs.keys())
+        kwds.sort()
+        for k in kwds:
+            self._update_hash(k)
+            self._update_hash(kwargs[k])
+
+        # make sure we are caching for the correct method
+        self._update_hash(self.func.__name__)
+        self._update_hash(self.func.__doc__)
+        self._update_hash(self.func.__code__.co_filename)
+
+        ref = self.ahash.hexdigest()
+
+        if ref in Cache._cache:
+            return Cache._cache[ref]
+        else:
+            data = self.func(*args, **kwargs)
+            Cache._cache[ref] = data
+            Cache._keys.append(ref)
+            if len(Cache._keys) > MAX_SIZE:
+                delref = Cache._keys.pop(0)
+                Cache._cache.pop(delref)
+            return data
+
+    def _update_hash(self, arg):
+        """Takes an argument and updates the hash.
+        The argument can be an np.array, string, or list
+        of things that are convertable to strings.
+        """
+        if isinstance(arg, np.ndarray):
+            self.ahash.update(arg.view(np.uint8))
+        elif isinstance(arg, list):
+            [self._update_hash(a) for a in arg]
+        else:
+            self.ahash.update(str(arg).encode('utf-8'))
+
+    @staticmethod
+    def clear_cache():
+        """Remove all cached objects"""
+        del Cache._keys
+        for k in list(Cache._cache.keys()):
+            it = Cache._cache.pop(k)
+            del it
+        del Cache._cache
+        Cache._keys = []
+        Cache._cache = {}
+        gc.collect()

dclab/cli/__init__.py

@@ -0,0 +1,10 @@
+# flake8: noqa: F401
+"""command line interface"""
+from .common import get_command_log, get_job_info
+from .task_compress import compress, compress_parser
+from .task_condense import condense, condense_dataset, condense_parser
+from .task_join import join, join_parser
+from .task_repack import repack, repack_parser
+from .task_split import split, split_parser
+from .task_tdms2rtdc import tdms2rtdc, tdms2rtdc_parser
+from .task_verify_dataset import verify_dataset, verify_dataset_parser

dclab/cli/common.py

@@ -0,0 +1,237 @@
+import hashlib
+import json
+import numbers
+import pathlib
+import platform
+import time
+import warnings
+
+import h5py
+import numpy as np
+
+try:
+    import imageio
+except ModuleNotFoundError:
+    imageio = None
+
+try:
+    import nptdms
+except ModuleNotFoundError:
+    nptdms = None
+
+from ..rtdc_dataset import fmt_tdms
+from .. import util
+from .._version import version
+
+
+class ExtendedJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, pathlib.Path):
+            return str(obj)
+        elif isinstance(obj, numbers.Integral):
+            return int(obj)
+        elif isinstance(obj, np.bool_):
+            return bool(obj)
+        # Let the base class default method raise the TypeError
+        return json.JSONEncoder.default(self, obj)
+
+
+def assemble_warnings(w):
+    """pretty-format all warnings for logs"""
+    wlog = []
+    for wi in w:
+        wlog.append(f"{wi.category.__name__}")
+        wlog.append(f" in {wi.category.__module__} line {wi.lineno}:")
+        # make sure line is not longer than 100 characters
+        words = str(wi.message).split(" ")
+        wline = "  "
+        for ii in range(len(words)):
+            wline += " " + words[ii]
+            if ii == len(words) - 1:
+                # end
+                wlog.append(wline)
+            elif len(wline + words[ii+1]) + 1 >= 100:
+                # new line
+                wlog.append(wline)
+                wline = "  "
+            # nothing to do here
+    return wlog
+
+
+def get_command_log(paths, custom_dict=None):
+    """Return a json dump of system parameters
+
+    Parameters
+    ----------
+    paths: list of pathlib.Path or str
+        paths of related measurement files; up to 5MB of each of
+        them is md5-hashed and included in the "files" key
+    custom_dict: dict
+        additional user-defined entries; must contain simple
+        Python objects (json.dumps must still work)
+    """
+    if custom_dict is None:
+        custom_dict = {}
+    data = get_job_info()
+    data["files"] = []
+    for ii, pp in enumerate(paths):
+        if pathlib.Path(pp).exists():
+            fdict = {"name": pathlib.Path(pp).name,
+                     # Hash only 5 MB of the input file
+                     "md5-5M": util.hashfile(pp,
+                                             count=80,
+                                             constructor=hashlib.md5),
+                     "index": ii+1
+                     }
+        else:
+            fdict = {"name": f"{pp}",
+                     "index": ii + 1
+                     }
+        data["files"].append(fdict)
+    final_data = {}
+    final_data.update(custom_dict)
+    final_data.update(data)
+    dump = json.dumps(final_data,
+                      sort_keys=True,
+                      indent=2,
+                      cls=ExtendedJSONEncoder).split("\n")
+    return dump
+
+
+def get_job_info():
+    """Return dictionary with current job information
+
+    Returns
+    -------
+    info: dict of dicts
+        Job information including details about time, system,
+        python version, and libraries used.
+    """
+    data = {
+        "utc": {
+            "date": time.strftime("%Y-%m-%d", time.gmtime()),
+            "time": time.strftime("%H:%M:%S", time.gmtime()),
+        },
+        "system": {
+            "info": platform.platform(),
+            "machine": platform.machine(),
+            "name": platform.system(),
+            "release": platform.release(),
+            "version": platform.version(),
+        },
+        "python": {
+            "build": ", ".join(platform.python_build()),
+            "implementation": platform.python_implementation(),
+            "version": platform.python_version(),
+        },
+        "libraries": {
+            "dclab": version,
+            "h5py": h5py.__version__,
+            "numpy": np.__version__,
+        }
+    }
+    if imageio is not None:
+        data["libraries"]["imageio"] = imageio.__version__
+    if nptdms is not None:
+        data["libraries"]["nptdms"] = nptdms.__version__
+    return data
+
+
+def print_info(string):
+    print(f"\033[1m{string}\033[0m")
+
+
+def print_alert(string):
+    print_info(f"\033[33m{string}")
+
+
+def print_violation(string):
+    print_info(f"\033[31m{string}")
+
+
+def setup_task_paths(paths_in, paths_out, allowed_input_suffixes):
+    """Setup directories for a CLI task
+
+    Parameters
+    ----------
+    paths_in: list of str or lsit of pathlib.Path or str or pathlib.Path
+        Input paths
+    paths_out: list of str or list of pathlib.Path or str or pathlib.Path
+        Output paths
+    allowed_input_suffixes: list
+        List of allowed input suffixes (e.g. [".rtdc"])
+
+    Returns
+    -------
+    paths_in: list of pathlib.Path or pathlib.Path
+        Input paths
+    paths_out: list of pathlib.Path or pathlib.Path
+        Output paths
+    paths_temp: list of pathlib.Path or pathlib.Path
+        Temporary paths (working path)
+    """
+    if isinstance(paths_in, list):
+        list_in = True
+    else:
+        paths_in = [paths_in]
+        list_in = False
+
+    if isinstance(paths_out, list):
+        list_out = True
+    else:
+        paths_out = [paths_out]
+        list_out = False
+
+    paths_in = [pathlib.Path(pi) for pi in paths_in]
+    for pi in paths_in:
+        if pi.suffix not in allowed_input_suffixes:
+            raise ValueError(f"Unsupported file type: '{pi.suffix}'")
+
+    paths_out = [pathlib.Path(po) for po in paths_out]
+    for ii, po in enumerate(paths_out):
+        if po.suffix != ".rtdc":
+            paths_out[ii] = po.with_name(po.name + ".rtdc")
+    [po.unlink() for po in paths_out if po.exists()]
+
+    paths_temp = [po.with_suffix(".rtdc~") for po in paths_out]
+    [pt.unlink() for pt in paths_temp if pt.exists()]
+
+    # convert lists back to paths
+    if not list_in:
+        paths_in = paths_in[0]
+
+    if not list_out:
+        paths_out = paths_out[0]
+        paths_temp = paths_temp[0]
+
+    return paths_in, paths_out, paths_temp
+
+
+def skip_empty_image_events(ds, initial=True, final=True):
+    """Set a manual filter to skip inital or final empty image events"""
+    if initial:
+        if (("image" in ds and ds.format == "tdms"
+             and ds.config["fmt_tdms"]["video frame offset"])
+            or ("contour" in ds and np.all(ds["contour"][0] == 0))
+                or ("image" in ds and np.all(ds["image"][0] == 0))):
+            ds.filter.manual[0] = False
+            ds.apply_filter()
+    if final:
+        # This is not easy to test, because we need a corrupt
+        # frame.
+        if "image" in ds:
+            idfin = len(ds) - 1
+            if ds.format == "tdms":
+                with warnings.catch_warnings(record=True) as wfin:
+                    warnings.simplefilter(
+                        "always",
+                        fmt_tdms.event_image.CorruptFrameWarning)
+                    _ = ds["image"][idfin]  # provoke a warning
+                for ww in wfin:
+                    if ww.category == fmt_tdms.event_image.CorruptFrameWarning:
+                        ds.filter.manual[idfin] = False
+                        ds.apply_filter()
+                        break
+            elif np.all(ds["image"][idfin] == 0):
+                ds.filter.manual[idfin] = False
+                ds.apply_filter()

dclab/cli/task_compress.py

@@ -0,0 +1,126 @@
+"""Compress .rtdc files"""
+from __future__ import annotations
+
+import argparse
+import pathlib
+import warnings
+
+import hdf5plugin
+import h5py
+
+from ..rtdc_dataset import rtdc_copy, RTDCWriter
+from .. import util
+from .._version import version
+
+from . import common
+
+
+def compress(
+        path_in: str | pathlib.Path = None,
+        path_out: str | pathlib.Path = None,
+        force: bool = False,
+        check_suffix: bool = True,
+        ret_path: bool = False,
+        ):
+    """Create a new dataset with all features compressed lossless
+
+    Parameters
+    ----------
+    path_in: str or pathlib.Path
+        file to compress
+    path_out: str or pathlib
+        output file path
+    force: bool
+        DEPRECATED
+    check_suffix: bool
+        check suffixes for input and output files
+    ret_path: bool
+        whether to return the output path
+
+    Returns
+    -------
+    path_out: pathlib.Path (optional)
+        output path (with possibly corrected suffix)
+    """
+    cmp_kw = hdf5plugin.Zstd(clevel=5)
+    if path_out is None or path_in is None:
+        parser = compress_parser()
+        args = parser.parse_args()
+        path_in = args.input
+        path_out = args.output
+        force = args.force
+
+    allowed_input_suffixes = [".rtdc"]
+    if not check_suffix:
+        allowed_input_suffixes.append(pathlib.Path(path_in).suffix)
+
+    path_in, path_out, path_temp = common.setup_task_paths(
+        path_in, path_out, allowed_input_suffixes=allowed_input_suffixes)
+
+    if force:
+        warnings.warn(
+            "The `force` keyword argument is deprecated since dclab 0.49.0, "
+            "because compressed HDF5 Datasets are now copied and there "
+            "is no reason to avoid or use force anymore.",
+            DeprecationWarning)
+
+    # command log
+    logs = {"dclab-compress": common.get_command_log(paths=[path_in])}
+
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        with h5py.File(path_in) as h5, h5py.File(path_temp, "w") as hc:
+            rtdc_copy(src_h5file=h5,
+                      dst_h5file=hc,
+                      features="all",
+                      include_basins=True,
+                      include_logs=True,
+                      include_tables=True,
+                      meta_prefix="",
+                      )
+
+            hc.require_group("logs")
+            # rename old dclab-compress logs
+            for lkey in ["dclab-compress", "dclab-compress-warnings"]:
+                if lkey in hc["logs"]:
+                    # This is cached, so no worry calling it multiple times.
+                    md55m = util.hashfile(path_in, count=80)
+                    # rename
+                    hc["logs"][f"{lkey}_{md55m}"] = hc["logs"][lkey]
+                    del hc["logs"][lkey]
+
+        # warnings log
+        if w:
+            logs["dclab-compress-warnings"] = common.assemble_warnings(w)
+
+    # Write log file
+    with RTDCWriter(path_temp,
+                    compression_kwargs=cmp_kw,
+                    mode="append") as hw:
+        for name in logs:
+            hw.store_log(name, logs[name])
+
+    # Finally, rename temp to out
+    path_temp.rename(path_out)
+
+    if ret_path:
+        return path_out
+
+
+def compress_parser():
+    descr = "Create a compressed version of an .rtdc file. This can be " \
+            + "used for saving disk space (loss-less compression). The " \
+            + "data generated during an experiment is usually not compressed."
+    parser = argparse.ArgumentParser(description=descr)
+    parser.add_argument('input', metavar="INPUT", type=str,
+                        help='Input path (.rtdc file)')
+    parser.add_argument('output', metavar="OUTPUT", type=str,
+                        help='Output path (.rtdc file)')
+    parser.add_argument('--force',
+                        dest='force',
+                        action='store_true',
+                        help='DEPRECATED')
+    parser.set_defaults(force=False)
+    parser.add_argument('--version', action='version',
+                        version=f'dclab-compress {version}')
+    return parser