dclab 0.63.1__tar.gz → 0.64.1__tar.gz

{dclab-0.63.1 → dclab-0.64.1}/CHANGELOG

@@ -1,3 +1,23 @@
+0.64.1
+ - enh: introduce `KernelDensityEstimator.get_at`
+ - enh: add 10th, 25th, 75th, and 90th percentile statistics methods
+ - enh: new `ret_dict` arg in `get_statistics` returns dict instead of tuple
+ - setup: fix finding packages in pyproject.toml
+0.64.0
+ - feat: introduce concept of perishable (e.g. temporally expiring) basins
+ - feat: add `PerishableRecord` to presigned URLs for DCOR basins (#244)
+ - fix: do not export perishable basins (#254)
+ - fix: allow `Basin.verify_basin` to fail
+ - fix: `Basin.__repr__` failed when location did not exist
+ - fix: fetching tables information from DCOR dataset could time out
+ - fix: default to compression level 5 when exporting or writing (#272)
+ - enh: increase timeout every time fetching data from DCOR API times out
+ - docs: update docs `RTDCWriter` I/O
+ - setup: drop support for Python 3.8
+ - tests: add tests for fetching tables from a DCOR dataset (#229)
+ - ref: add `get_contour_lines` method to `KernelDensityEstimator` class
+ - ref: deprecate get_contour and use bin_width_doane_div5 in kde submodule
+ - ref: cache "basins" in `RTDC_DCOR` instead of in `RTDC_DCOR.api`
 0.63.1
  - docs: update documentation and code references for kde submodule
  - enh: cache S3 sessions and clients (test suite duration reduced by 30%)

{dclab-0.63.1 → dclab-0.64.1}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: dclab
-Version: 0.63.1
+Version: 0.64.1
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
-License: GPL-2.0-or-later
+License-Expression: GPL-2.0-or-later
 Project-URL: source, https://github.com/DC-Analysis/dclab
 Project-URL: tracker, https://github.com/DC-Analysis/dclab/issues
 Project-URL: documentation, https://dclab.readthedocs.io/en/stable/
@@ -14,7 +14,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering :: Visualization
 Classifier: Intended Audience :: Science/Research
-Requires-Python: <4,>=3.8
+Requires-Python: <4,>=3.9
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: h5py<4,>=3.0.0

{dclab-0.63.1 → dclab-0.64.1}/dclab/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.63.1'
-__version_tuple__ = version_tuple = (0, 63, 1)
+__version__ = version = '0.64.1'
+__version_tuple__ = version_tuple = (0, 64, 1)

{dclab-0.63.1 → dclab-0.64.1}/dclab/http_utils.py

@@ -254,7 +254,7 @@ class ResoluteRequestsSession(requests.Session):
                 break
         else:
             raise requests.exceptions.ReadTimeout(
-                f"Resolut sesion failed for {args} and {kwargs}!")
+                f"Resolute session failed for {args} and {kwargs}!")
         return resp
 
 

dclab-0.64.1/dclab/kde/base.py

@@ -0,0 +1,459 @@
+import warnings
+
+import numpy as np
+from scipy.interpolate import RegularGridInterpolator as RGI
+
+from .methods import bin_width_doane_div5, get_bad_vals, methods
+from .contours import find_contours_level, get_quantile_levels
+
+
+class ContourSpacingTooLarge(UserWarning):
+    pass
+
+
+class KernelDensityEstimator:
+    def __init__(self, rtdc_ds):
+        self.rtdc_ds = rtdc_ds
+
+    @staticmethod
+    def apply_scale(a, scale, feat):
+        """Helper function for transforming an aray to log-scale
+
+        Parameters
+        ----------
+        a: np.ndarray
+            Input array
+        scale: str
+            If set to "log", take the logarithm of `a`; if set to
+            "linear" return `a` unchanged.
+        feat: str
+            Feature name (required for debugging)
+
+        Returns
+        -------
+        b: np.ndarray
+            The scaled array
+
+        Notes
+        -----
+        If the scale is not "linear", then a new array is returned.
+        All warnings are suppressed when computing `np.log(a)`, as
+        `a` may have negative or nan values.
+        """
+        if scale == "linear":
+            b = a
+        elif scale == "log":
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter("always")
+                b = np.log(a)
+                if len(w):
+                    # Tell the user that the log-transformation issued
+                    # a warning.
+                    warnings.warn(f"Invalid values encounterd in np.log "
+                                  f"while scaling feature '{feat}'!")
+        else:
+            raise ValueError(f"`scale` must be either 'linear' or 'log', "
+                             f"got '{scale}'!")
+        return b
+
+    @staticmethod
+    def get_spacing(a, method, scale="linear", method_kw=None,
+                    feat="undefined", ret_scaled=False):
+        """Convenience function for computing the contour spacing
+
+        Parameters
+        ----------
+        a: ndarray
+            feature data
+        scale: str
+            how the data should be scaled ("log" or "linear")
+        method: callable
+            KDE spacing method to use
+        method_kw: dict
+            keyword arguments to `method`
+        feat: str
+            feature name for debugging
+        ret_scaled: bool
+            whether to return the scaled array of `a`
+        """
+        if method_kw is None:
+            method_kw = {}
+        # Apply scale (no change for linear scale)
+        asc = KernelDensityEstimator.apply_scale(a, scale, feat)
+        # Apply multiplicator
+        acc = method(asc, **method_kw)
+        if ret_scaled:
+            return acc, asc
+        else:
+            return acc
+
+    def get_contour(self, xax="area_um", yax="deform", xacc=None, yacc=None,
+                    kde_type="histogram", kde_kwargs=None, xscale="linear",
+                    yscale="linear"):
+        """Evaluate the kernel density estimate for contour plots
+
+        Parameters
+        ----------
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        xacc: float
+            Contour accuracy in x direction
+        yacc: float
+            Contour accuracy in y direction
+        kde_type: str
+            The KDE method to use
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`.
+
+        Returns
+        -------
+        X, Y, Z : coordinates
+            The kernel density Z evaluated on a rectangular grid (X,Y).
+        """
+        warnings.warn("`get_contour` is deprecated; please use "
+                      "`get_raster` instead", DeprecationWarning)
+        return self.get_raster(
+            xax=xax, yax=yax, xacc=xacc, yacc=yacc,
+            kde_type=kde_type, kde_kwargs=kde_kwargs,
+            xscale=xscale, yscale=yscale
+        )
+
+    def get_contour_lines(self, quantiles=None, xax="area_um", yax="deform",
+                          xacc=None, yacc=None, kde_type="histogram",
+                          kde_kwargs=None, xscale="linear", yscale="linear",
+                          ret_levels=False):
+        """Compute contour lines for a given kernel kensity estimate.
+
+        Parameters
+        ----------
+        quantiles: list or array of floats
+            KDE Quantiles for which contour levels are computed. The
+            values must be between 0 and 1. If set to None, use
+            [0.5, 0.95] as default.
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        xacc: float
+            Contour accuracy in x direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        yacc: float
+            Contour accuracy in y direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        kde_type: str
+            The KDE method to use
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`
+        ret_levels: bool
+            If set to True, return the levels of the contours
+            (default: False)
+
+        Returns
+        -------
+        contour_lines: list of lists (of lists)
+            For every number in `quantiles`, this list contains a list of
+            corresponding contour lines. Each contour line is a 2D
+            array of shape (N, 2), where N is the number of points in the
+            contour line.
+        levels: list of floats
+            The density levels corresponding to each number in `quantiles`.
+            Only returned if `ret_levels` is set to True.
+        """
+        if not quantiles:
+            quantiles = [0.5, 0.95]
+        try:
+            x, y, density = self.get_raster(
+                xax=xax,
+                yax=yax,
+                xacc=xacc,
+                yacc=yacc,
+                xscale=xscale,
+                yscale=yscale,
+                kde_type=kde_type,
+                kde_kwargs=kde_kwargs,
+            )
+        except ValueError:
+            # most-likely there is nothing to compute a contour for
+            return []
+        if density.shape[0] < 3 or density.shape[1] < 3:
+            warnings.warn("Contour not possible; spacing may be too large!",
+                          ContourSpacingTooLarge)
+            return []
+        levels = get_quantile_levels(
+            density=density,
+            x=x,
+            y=y,
+            xp=self.rtdc_ds[xax][self.rtdc_ds.filter.all],
+            yp=self.rtdc_ds[yax][self.rtdc_ds.filter.all],
+            q=np.array(quantiles),
+            normalize=False)
+        contours = []
+        # Normalize levels to [0, 1]
+        nlevels = np.array(levels) / density.max()
+        for nlev in nlevels:
+            # make sure that the contour levels are not at the boundaries
+            if not (np.allclose(nlev, 0, atol=1e-12, rtol=0)
+                    or np.allclose(nlev, 1, atol=1e-12, rtol=0)):
+                cc = find_contours_level(
+                    density, x=x, y=y, level=nlev)
+                contours.append(cc)
+            else:
+                contours.append([])
+        if ret_levels:
+            return contours, levels
+        else:
+            return contours
+
+    def get_raster(self, xax="area_um", yax="deform", xacc=None, yacc=None,
+                   kde_type="histogram", kde_kwargs=None, xscale="linear",
+                   yscale="linear"):
+        """Evaluate the kernel density estimate on a grid
+
+        Parameters
+        ----------
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        xacc: float
+            Contour accuracy in x direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        yacc: float
+            Contour accuracy in y direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        kde_type: str
+            The KDE method to use
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`.
+
+        Returns
+        -------
+        X, Y, Z : coordinates
+            The kernel density Z evaluated on a rectangular grid (X,Y).
+        """
+        if kde_kwargs is None:
+            kde_kwargs = {}
+        xax = xax.lower()
+        yax = yax.lower()
+        kde_type = kde_type.lower()
+        if kde_type not in methods:
+            raise ValueError(f"Not a valid kde type: {kde_type}!")
+
+        # Get data
+        x = self.rtdc_ds[xax][self.rtdc_ds.filter.all]
+        y = self.rtdc_ds[yax][self.rtdc_ds.filter.all]
+
+        xacc_sc, xs = self.get_spacing(
+            a=x,
+            feat=xax,
+            scale=xscale,
+            method=bin_width_doane_div5,
+            ret_scaled=True)
+
+        yacc_sc, ys = self.get_spacing(
+            a=y,
+            feat=yax,
+            scale=yscale,
+            method=bin_width_doane_div5,
+            ret_scaled=True)
+
+        if xacc is None or xacc == 0:
+            xacc = xacc_sc
+
+        if yacc is None or yacc == 0:
+            yacc = yacc_sc
+
+        # Ignore infs and nans
+        bad = get_bad_vals(xs, ys)
+        xc = xs[~bad]
+        yc = ys[~bad]
+
+        xnum = int(np.ceil((xc.max() - xc.min()) / xacc))
+        ynum = int(np.ceil((yc.max() - yc.min()) / yacc))
+
+        xlin = np.linspace(xc.min(), xc.max(), xnum, endpoint=True)
+        ylin = np.linspace(yc.min(), yc.max(), ynum, endpoint=True)
+
+        xmesh, ymesh = np.meshgrid(xlin, ylin, indexing="ij")
+
+        kde_fct = methods[kde_type]
+        if len(x):
+            density = kde_fct(events_x=xs, events_y=ys,
+                              xout=xmesh, yout=ymesh,
+                              **kde_kwargs)
+        else:
+            density = np.array([])
+
+        # Convert mesh back to linear scale if applicable
+        if xscale == "log":
+            xmesh = np.exp(xmesh)
+        if yscale == "log":
+            ymesh = np.exp(ymesh)
+
+        return xmesh, ymesh, density
+
+    def get_scatter(self, xax="area_um", yax="deform", positions=None,
+                    kde_type="histogram", kde_kwargs=None, xscale="linear",
+                    yscale="linear"):
+        """Evaluate the kernel density estimate for scatter plots
+
+        The KDE is evaluated with the `kde_type` function for every point.
+
+        Parameters
+        ----------
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        positions: list of two 1d ndarrays or ndarray of shape (2, N)
+            The positions where the KDE will be computed. Note that
+            the KDE estimate is computed from the points that
+            are set in `self.rtdc_ds.filter.all`.
+        kde_type: str
+            The KDE method to use, see :const:`.kde_methods.methods`
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`.
+
+        Returns
+        -------
+        density : 1d ndarray
+            The kernel density evaluated for the filtered data points.
+        """
+        if kde_kwargs is None:
+            kde_kwargs = {}
+        xax = xax.lower()
+        yax = yax.lower()
+        kde_type = kde_type.lower()
+        if kde_type not in methods:
+            raise ValueError(f"Not a valid kde type: {kde_type}!")
+
+        # Get data
+        x = self.rtdc_ds[xax][self.rtdc_ds.filter.all]
+        y = self.rtdc_ds[yax][self.rtdc_ds.filter.all]
+
+        # Apply scale (no change for linear scale)
+        xs = self.apply_scale(x, xscale, xax)
+        ys = self.apply_scale(y, yscale, yax)
+
+        if positions is None:
+            posx = None
+            posy = None
+        else:
+            posx = self.apply_scale(positions[0], xscale, xax)
+            posy = self.apply_scale(positions[1], yscale, yax)
+
+        kde_fct = methods[kde_type]
+        if len(x):
+            density = kde_fct(events_x=xs, events_y=ys,
+                              xout=posx, yout=posy,
+                              **kde_kwargs)
+        else:
+            density = np.array([])
+
+        return density
+
+    def get_at(self, xax="area_um", yax="deform", positions=None,
+               kde_type="histogram", kde_kwargs=None, xscale="linear",
+               yscale="linear"):
+        """Evaluate the kernel density estimate for specific events
+
+        The KDE is computed via linear interpolation from the output
+        of `get_raster`.
+
+        Parameters
+        ----------
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        positions: list of two 1d ndarrays or ndarray of shape (2, N)
+            The positions where the KDE will be computed. Note that
+            the KDE estimate is computed from the points that
+            are set in `self.rtdc_ds.filter.all`.
+        kde_type: str
+            The KDE method to use, see :const:`.kde_methods.methods`
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`.
+
+        Returns
+        -------
+        density : 1d ndarray
+            The kernel density evaluated for the filtered events.
+        """
+        if kde_kwargs is None:
+            kde_kwargs = {}
+        xax = xax.lower()
+        yax = yax.lower()
+        kde_type = kde_type.lower()
+        if kde_type not in methods:
+            raise ValueError(f"Not a valid kde type: {kde_type}!")
+
+        # Get data
+        x = self.rtdc_ds[xax][self.rtdc_ds.filter.all]
+        y = self.rtdc_ds[yax][self.rtdc_ds.filter.all]
+
+        # Apply scale (no change for linear scale)
+        xs = self.apply_scale(x, xscale, xax)
+        ys = self.apply_scale(y, yscale, yax)
+
+        if positions:
+            xs = self.apply_scale(positions[0], xscale, xax)
+            ys = self.apply_scale(positions[1], yscale, yax)
+
+        if len(x):
+            xr, yr, density_grid = self.get_raster(xax=xax,
+                                                   yax=yax,
+                                                   kde_type=kde_type,
+                                                   kde_kwargs=kde_kwargs,
+                                                   xscale=xscale,
+                                                   yscale=yscale)
+
+            # Apply scale (no change for linear scale)
+            xrs = self.apply_scale(xr, xscale, xax)
+            yrs = self.apply_scale(yr, yscale, yax)
+
+            # 'scipy.interp2d' has been removed in SciPy 1.14.0
+            # https://scipy.github.io/devdocs/tutorial/interpolate/interp_transition_guide.html
+            interp_func = RGI((xrs[:, 0], yrs[0, :]),
+                              density_grid,
+                              method="linear",
+                              bounds_error=False,
+                              fill_value=np.nan)
+            density = interp_func((xs, ys))
+
+        else:
+            density = np.array([])
+
+        return density

{dclab-0.63.1 → dclab-0.64.1}/dclab/kde/methods.py

@@ -56,6 +56,16 @@ def bin_width_doane(a):
     return acc
 
 
+def bin_width_doane_div5(a):
+    """Compute contour spacing based on Doane's formula divided by five
+
+    See Also
+    --------
+    bin_width_doane: method used to compute the bin width
+    """
+    return bin_width_doane(a) / 5
+
+
 def bin_width_percentile(a):
     """Compute contour spacing based on data percentiles
 

{dclab-0.63.1 → dclab-0.64.1}/dclab/rtdc_dataset/core.py

@@ -635,7 +635,7 @@ class RTDCBase(abc.ABC):
             The kernel density Z evaluated on a rectangular grid (X,Y).
         """
         kde_instance = KernelDensityEstimator(rtdc_ds=self)
-        xmesh, ymesh, density = kde_instance.get_contour(
+        xmesh, ymesh, density = kde_instance.get_raster(
             xax=xax, yax=yax, xacc=xacc, yacc=yacc, kde_type=kde_type,
             kde_kwargs=kde_kwargs, xscale=xscale, yscale=yscale
         )
@@ -746,6 +746,8 @@ class RTDCBase(abc.ABC):
                 "ignored_basins": bd_keys,
                 # basin key
                 "key": bdict["key"],
+                # whether the basin is perishable or not
+                "perishable": bdict.get("perishable", False),
             }
 
             # Check whether this basin is supported and exists
@@ -783,12 +785,19 @@ class RTDCBase(abc.ABC):
                     b_cls = bc[bdict["format"]]
                     # Try absolute path
                     bna = b_cls(pp, **kwargs)
-                    if bna.verify_basin():
-                        basins.append(bna)
-                        break
+
+                    try:
+                        absolute_exists = bna.verify_basin()
+                    except BaseException:
+                        pass
+                    else:
+                        if absolute_exists:
+                            basins.append(bna)
+                            break
                     # Try relative path
                     this_path = pathlib.Path(self.path)
                     if this_path.exists():
+
                         # Insert relative path
                         bnr = b_cls(this_path.parent / pp, **kwargs)
                         if bnr.verify_basin():

{dclab-0.63.1 → dclab-0.64.1}/dclab/rtdc_dataset/export.py

@@ -268,8 +268,8 @@ class Export(object):
         compression_kwargs: dict
             Dictionary with the keys "compression" and "compression_opts"
             which are passed to :func:`h5py.H5File.create_dataset`. The
-            default is Zstandard compression with the lowest compression
-            level `hdf5plugin.Zstd(clevel=1)`.
+            default is Zstandard compression with the compression
+            level 5 `hdf5plugin.Zstd(clevel=5)`.
         compression: str or None
             Compression method used for data storage;
             one of [None, "lzf", "gzip", "szip"].
@@ -300,7 +300,7 @@ class Export(object):
             # be backwards-compatible
             compression_kwargs = {"compression": compression}
         if compression_kwargs is None:
-            compression_kwargs = hdf5plugin.Zstd(clevel=1)
+            compression_kwargs = hdf5plugin.Zstd(clevel=5)
         path = pathlib.Path(path)
         # Make sure that path ends with .rtdc
         if path.suffix not in [".rtdc", ".rtdc~"]:
@@ -505,6 +505,11 @@ class Export(object):
                         # defined in. Since we are exporting, it does not
                         # make sense to store these basins in the output file.
                         continue
+                    elif bn_dict.get("perishable"):
+                        # Perishable basins require secret keys or complicated
+                        # logic to execute in order to refresh them. We do not
+                        # store them in the output file.
+                        continue
                     basinmap_orig = bn_dict.get("basin_map")
                     if not filtered:
                         # filtering disabled: just copy basins