dclab 0.62.11__cp313-cp313-win_amd64.whl

dclab/rtdc_dataset/fmt_hdf5/events.py

@@ -0,0 +1,257 @@
+"""RT-DC hdf5 format"""
+from __future__ import annotations
+
+import warnings
+
+import numbers
+import numpy as np
+
+from ... import definitions as dfn
+from ...util import copy_if_needed
+
+from . import feat_defect
+
+
+class H5ContourEvent:
+    def __init__(self, h5group, length=None):
+        self._length = length
+        self.h5group = h5group
+        # for hashing in util.obj2bytes
+        self.identifier = (h5group.file.filename, h5group["0"].name)
+
+    def __getitem__(self, key):
+        if not isinstance(key, numbers.Integral):
+            # slicing!
+            indices = np.arange(len(self))[key]
+            output = []
+            # populate the output list
+            for evid in indices:
+                output.append(self.h5group[str(evid)][:])
+            return output
+        elif key < 0:
+            return self.__getitem__(key + len(self))
+        else:
+            return self.h5group[str(key)][:]
+
+    def __iter__(self):
+        for idx in range(len(self)):
+            yield self[idx]
+
+    def __len__(self):
+        if self._length is None:
+            # computing the length of an H5Group is slow
+            self._length = len(self.h5group)
+        return self._length
+
+    @property
+    def dtype(self):
+        return self.h5group["0"].dtype
+
+    @property
+    def shape(self):
+        return len(self), np.nan, 2
+
+
+class H5Events:
+    def __init__(self, h5):
+        self.h5file = h5
+        # According to https://github.com/h5py/h5py/issues/1960, we always
+        # have to keep a reference to the HDF5 dataset, otherwise it will
+        # be garbage-collected immediately. In addition to caching the HDF5
+        # datasets, we cache the wrapping classes in the `self._cached_events`
+        # dictionary.
+        self._cached_events = {}
+        self._defective_features = {}
+        self._features_list = None
+
+    @property
+    def _features(self):
+        if self._features_list is None:
+            self._features_list = sorted(self.h5file["events"].keys())
+            # make sure that "trace" is not empty
+            if ("trace" in self._features
+                    and len(self.h5file["events"]["trace"]) == 0):
+                self._features_list.remove("trace")
+        return self._features_list
+
+    def __contains__(self, key):
+        return key in self.keys()
+
+    def __getitem__(self, key):
+        if key not in self._cached_events:
+            # user-level checking is done in core.py
+            assert dfn.feature_exists(key), f"Feature '{key}' does not exist!"
+            data = self.h5file["events"][key]
+            if key == "contour":
+                length = self.h5file.attrs.get("experiment:event count")
+                fdata = H5ContourEvent(data, length=length)
+            elif key == "mask":
+                fdata = H5MaskEvent(data)
+            elif key == "trace":
+                fdata = H5TraceEvent(data)
+            elif data.ndim == 1:
+                fdata = H5ScalarEvent(data)
+            else:
+                # for features like "image", "image_bg" and other non-scalar
+                # ancillary features
+                fdata = data
+            self._cached_events[key] = fdata
+        return self._cached_events[key]
+
+    def __iter__(self):
+        # dict-like behavior
+        for key in self.keys():
+            yield key
+
+    def _is_defective_feature(self, feat):
+        """Whether the stored feature is defective"""
+        if feat not in self._defective_features:
+            defective = False
+            if (feat in feat_defect.DEFECTIVE_FEATURES
+                    and feat in self._features):
+                # feature exists in the HDF5 file
+                # workaround machinery for sorting out defective features
+                defective = feat_defect.DEFECTIVE_FEATURES[feat](self.h5file)
+            self._defective_features[feat] = defective
+        return self._defective_features[feat]
+
+    def keys(self):
+        """Returns list of valid features
+
+        Checks for
+        - defective features: whether the data in the HDF5 file is invalid
+        - existing feature names: dynamic, depending on e.g. plugin features
+        """
+        features = []
+        for key in self._features:
+            # check for defective features
+            if dfn.feature_exists(key) and not self._is_defective_feature(key):
+                features.append(key)
+        return features
+
+
+class H5MaskEvent:
+    """Cast uint8 masks to boolean"""
+
+    def __init__(self, h5dataset):
+        self.h5dataset = h5dataset
+        # identifier required because "mask" is used for computation
+        # of ancillary feature "contour".
+        self.identifier = (self.h5dataset.file.filename, self.h5dataset.name)
+        self.dtype = np.dtype(bool)
+
+    def __array__(self, dtype=np.bool_, copy=copy_if_needed, *args, **kwargs):
+        if dtype is not np.uint8:
+            warnings.warn("Please avoid calling the `__array__` method of the "
+                          "`H5MaskEvent`. It may consume a lot of memory.",
+                          UserWarning)
+        # One of the reasons why we implement __array__ is such that
+        # the data exporter knows this object is sliceable
+        # (see yield_filtered_array_stacks).
+        return self.h5dataset.__array__(dtype=dtype, *args, **kwargs)
+
+    def __getitem__(self, idx):
+        return np.asarray(self.h5dataset[idx], dtype=bool)
+
+    def __iter__(self):
+        for idx in range(len(self)):
+            yield self[idx]
+
+    def __len__(self):
+        return len(self.h5dataset)
+
+    @property
+    def attrs(self):
+        return self.h5dataset.attrs
+
+    @property
+    def shape(self):
+        return self.h5dataset.shape
+
+
+class H5ScalarEvent(np.lib.mixins.NDArrayOperatorsMixin):
+    def __init__(self, h5ds):
+        """Lazy access to a scalar feature with cache"""
+        self.h5ds = h5ds
+        # for hashing in util.obj2bytes
+        self.identifier = (self.h5ds.file.filename, self.h5ds.name)
+        self._array = None
+        self.ndim = 1  # matplotlib might expect this from an array
+        # attrs
+        self._ufunc_attrs = dict(self.h5ds.attrs)
+
+    def __array__(self, dtype=None, copy=copy_if_needed, *args, **kwargs):
+        if self._array is None:
+            self._array = np.asarray(self.h5ds, *args, **kwargs)
+        return np.array(self._array, dtype=dtype, copy=copy)
+
+    def __getitem__(self, idx):
+        return self.__array__()[idx]
+
+    def __len__(self):
+        return len(self.h5ds)
+
+    def _fetch_ufunc_attr(self, uname, ufunc):
+        """A wrapper for calling functions on the scalar feature data
+
+        The ideas are:
+
+        1. If there is a ufunc (max/mean/min) value stored in the dataset
+           attributes, then use this one.
+        2. If the ufunc is computed, it is cached permanently in
+           self._ufunc_attrs
+        """
+        val = self._ufunc_attrs.get(uname, None)
+        if val is None:
+            val = ufunc(self.__array__())
+            self._ufunc_attrs[uname] = val
+        return val
+
+    def max(self, *args, **kwargs):
+        return self._fetch_ufunc_attr("max", np.nanmax)
+
+    def mean(self, *args, **kwargs):
+        return self._fetch_ufunc_attr("mean", np.nanmean)
+
+    def min(self, *args, **kwargs):
+        return self._fetch_ufunc_attr("min", np.nanmin)
+
+    @property
+    def dtype(self):
+        return self.h5ds.dtype
+
+    @property
+    def shape(self):
+        return self.h5ds.shape
+
+
+class H5TraceEvent:
+    def __init__(self, h5group):
+        self.h5group = h5group
+        self._num_traces = None
+        self._shape = None
+
+    def __getitem__(self, idx):
+        return self.h5group[idx]
+
+    def __contains__(self, item):
+        return item in self.h5group
+
+    def __len__(self):
+        if self._num_traces is None:
+            self._num_traces = len(self.h5group)
+        return self._num_traces
+
+    def __iter__(self):
+        for key in sorted(self.h5group.keys()):
+            yield key
+
+    def keys(self):
+        return self.h5group.keys()
+
+    @property
+    def shape(self):
+        if self._shape is None:
+            atrace = list(self.h5group.keys())[0]
+            self._shape = tuple([len(self)] + list(self.h5group[atrace].shape))
+        return self._shape

dclab/rtdc_dataset/fmt_hdf5/feat_defect.py

@@ -0,0 +1,164 @@
+"""RT-DC hdf5 format"""
+from __future__ import annotations
+
+from ...external.packaging import parse as parse_version
+
+
+def get_software_version_from_h5(h5):
+    software_version = h5.attrs.get("setup:software version", "")
+    if isinstance(software_version, bytes):
+        software_version = software_version.decode("utf-8")
+    return software_version
+
+
+def is_defective_feature_aspect(h5):
+    """In Shape-In 2.0.6, there was a wrong variable cast"""
+    software_version = get_software_version_from_h5(h5)
+    return software_version in ["ShapeIn 2.0.6", "ShapeIn 2.0.7"]
+
+
+def is_defective_feature_time(h5):
+    """Shape-In stores the "time" feature as a low-precision float32
+
+    This makes time resolution for large measurements useless,
+    because times are only resolved with four digits after the
+    decimal point. Here, we first check whether the "frame" feature
+    and the [imaging]:"frame rate" configuration are set. If so,
+    then we can compute "time" as an ancillary feature which will
+    be more accurate than its float32 version.
+    """
+    # This is a necessary requirement. If we cannot compute the
+    # ancillary feature, then we cannot ignore (even inaccurate) information.
+    has_ancil = "frame" in h5["events"] and h5.attrs.get("imaging:frame rate",
+                                                         0) != 0
+    if not has_ancil:
+        return False
+
+    # If we have a 32 bit dataset, then things are pretty clear.
+    is_32float = h5["events/time"].dtype.char[-1] == "f"
+    if is_32float:
+        return True
+
+    # Consider the software
+    software_version = get_software_version_from_h5(h5)
+
+    # Only Shape-In stores false data, so we can ignore other recording
+    # software.
+    is_shapein = software_version.count("ShapeIn")
+    if not is_shapein:
+        return False
+
+    # The tricky part: dclab might have analyzed the dataset recorded by
+    # Shape-In, e.g. in a compression step. Since dclab appends its version
+    # string to the software_version, we just have to parse that and make
+    # sure that it is above 0.47.6.
+    last_version = software_version.split("|")[-1].strip()
+    if last_version.startswith("dclab"):
+        dclab_version = last_version.split()[1]
+        if parse_version(dclab_version) < parse_version("0.47.6"):
+            # written with an older version of dclab
+            return True
+
+    # We covered all cases:
+    # - ancillary information are available
+    # - it's not a float32 dataset
+    # - we excluded all non-Shape-In recording software
+    # - it was not written with an older version of dclab
+    return False
+
+
+def is_defective_feature_volume(h5):
+    """dclab computed volume wrong up until version 0.36.1"""
+    # first check if the scripted fix was applied
+    if "dclab_issue_141" in list(h5.get("logs", {}).keys()):
+        return False
+    # if that does not apply, check the software version
+    software_version = get_software_version_from_h5(h5)
+    if software_version:
+        last_version = software_version.split("|")[-1].strip()
+        if last_version.startswith("dclab"):
+            dclab_version = last_version.split()[1]
+            if parse_version(dclab_version) < parse_version("0.37.0"):
+                return True
+    return False
+
+
+def is_defective_feature_inert_ratio(h5):
+    """For long channels, there was an integer overflow until 0.48.1
+
+    The problem here is that not only the channel length, but also
+    the length of the contour play a role. All point coordinates of
+    the contour are summed up and multiplied with one another which
+    leads to integer overflows when computing mu20.
+
+    Thus, this test is only a best guess, but still quite fast.
+
+    See also https://github.com/DC-analysis/dclab/issues/212
+    """
+    # determine whether the image width is larger than 500px
+    # If this file was written with dclab, then we always have the ROI size,
+    # so we don't have to check the actual image.
+    width_large = h5.attrs.get("imaging:roi size x", 0) > 500
+
+    if width_large:
+        # determine whether the software version was outdated
+        software_version = get_software_version_from_h5(h5)
+        if software_version:
+            version_pipeline = [v.strip() for v in software_version.split("|")]
+            last_version = version_pipeline[-1]
+            if last_version.startswith("dclab"):
+                dclab_version = last_version.split()[1]
+                # The fix was implemented in 0.48.2, but this method here
+                # was only implemented in 0.48.3, so we might have leaked
+                # old data into new files.
+                if parse_version(dclab_version) < parse_version("0.48.3"):
+                    return True
+    return False
+
+
+def is_defective_feature_inert_ratio_raw_cvx(h5):
+    """Additional check for `inert_ratio_raw` and `inert_ratio_cvx`
+
+    These features were computed with Shape-In and were very likely
+    computed correctly.
+
+    See https://github.com/DC-analysis/dclab/issues/224
+    """
+    if is_defective_feature_inert_ratio(h5):
+        # Possibly affected. Only return False if Shape-In check is negative
+        software_version = get_software_version_from_h5(h5)
+        version_pipeline = [v.strip() for v in software_version.split("|")]
+        first_version = version_pipeline[0]
+        if first_version.startswith("ShapeIn"):
+            si_version = first_version.split()[1]
+        elif "shapein-acquisition" in h5.get("logs", []):
+            # Later versions of Shape-In do not anymore write "ShapeIn" in the
+            # version string.
+            si_version = first_version
+        else:
+            # Some other software was used to record the data and dclab
+            # very likely stored the wrong inertia ratio.
+            return True
+
+        # We trust Shape-In >= 2.0.5
+        if parse_version(si_version) >= parse_version("2.0.5"):
+            return False
+
+        return True
+
+    return False
+
+
+#: dictionary of defective features, defined by HDF5 attributes;
+#: if a value matches the given HDF5 attribute, the feature is
+#: considered defective
+DEFECTIVE_FEATURES = {
+    # feature: [HDF5_attribute, matching_value]
+    "aspect": is_defective_feature_aspect,
+    "inert_ratio_cvx": is_defective_feature_inert_ratio_raw_cvx,
+    "inert_ratio_prnc": is_defective_feature_inert_ratio,
+    "inert_ratio_raw": is_defective_feature_inert_ratio_raw_cvx,
+    "tilt": is_defective_feature_inert_ratio,
+    "time": is_defective_feature_time,
+    "volume": is_defective_feature_volume,
+}

dclab/rtdc_dataset/fmt_hdf5/logs.py

@@ -0,0 +1,33 @@
+class H5Logs:
+    def __init__(self, h5):
+        self.h5file = h5
+        self._cache_keys = None
+
+    def __getitem__(self, key):
+        if key in self.keys():
+            log = list(self.h5file["logs"][key])
+            if isinstance(log[0], bytes):
+                log = [li.decode("utf") for li in log]
+        else:
+            raise KeyError(
+                f"File {self.h5file.file.filename} does not have the log "
+                f"'{key}'. Available logs are {self.keys()}.")
+        return log
+
+    def __iter__(self):
+        # dict-like behavior
+        for key in self.keys():
+            yield key
+
+    def __len__(self):
+        return len(self.keys())
+
+    def keys(self):
+        if self._cache_keys is None:
+            names = []
+            if "logs" in self.h5file:
+                for key in self.h5file["logs"]:
+                    if self.h5file["logs"][key].size:
+                        names.append(key)
+            self._cache_keys = names
+        return self._cache_keys

dclab/rtdc_dataset/fmt_hdf5/tables.py

@@ -0,0 +1,30 @@
+class H5Tables:
+    def __init__(self, h5):
+        self.h5file = h5
+        self._cache_keys = None
+
+    def __getitem__(self, key):
+        if key in self.keys():
+            tab = self.h5file["tables"][key]
+        else:
+            raise KeyError(f"Table '{key}' not found or empty "
+                           f"in {self.h5file.file.filename}!")
+        return tab
+
+    def __iter__(self):
+        # dict-like behavior
+        for key in self.keys():
+            yield key
+
+    def __len__(self):
+        return len(self.keys())
+
+    def keys(self):
+        if self._cache_keys is None:
+            names = []
+            if "tables" in self.h5file:
+                for key in self.h5file["tables"]:
+                    if self.h5file["tables"][key].size:
+                        names.append(key)
+            self._cache_keys = names
+        return self._cache_keys

dclab/rtdc_dataset/fmt_hierarchy/__init__.py

@@ -0,0 +1,11 @@
+# flake8: noqa: F401
+from .base import RTDC_Hierarchy
+from .events import (
+    ChildTrace, ChildTraceItem, ChildScalar, ChildContour, ChildNDArray,
+    ChildBase
+)
+from .hfilter import HierarchyFilter, HierarchyFilterError
+from .mapper import (
+    map_indices_child2parent, map_indices_child2root,
+    map_indices_root2child, map_indices_parent2child
+)