dclab 0.67.0__cp314-cp314t-macosx_10_13_x86_64.whl

dclab/kde/__init__.py

@@ -0,0 +1 @@
+from .base import KernelDensityEstimator  # noqa: F401

dclab/kde/base.py

@@ -0,0 +1,459 @@
+import warnings
+
+import numpy as np
+from scipy.interpolate import RegularGridInterpolator as RGI
+
+from .methods import bin_width_doane_div5, get_bad_vals, methods
+from .contours import find_contours_level, get_quantile_levels
+
+
+class ContourSpacingTooLarge(UserWarning):
+    pass
+
+
+class KernelDensityEstimator:
+    def __init__(self, rtdc_ds):
+        self.rtdc_ds = rtdc_ds
+
+    @staticmethod
+    def apply_scale(a, scale, feat):
+        """Helper function for transforming an aray to log-scale
+
+        Parameters
+        ----------
+        a: np.ndarray
+            Input array
+        scale: str
+            If set to "log", take the logarithm of `a`; if set to
+            "linear" return `a` unchanged.
+        feat: str
+            Feature name (required for debugging)
+
+        Returns
+        -------
+        b: np.ndarray
+            The scaled array
+
+        Notes
+        -----
+        If the scale is not "linear", then a new array is returned.
+        All warnings are suppressed when computing `np.log(a)`, as
+        `a` may have negative or nan values.
+        """
+        if scale == "linear":
+            b = a
+        elif scale == "log":
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter("always")
+                b = np.log(a)
+                if len(w):
+                    # Tell the user that the log-transformation issued
+                    # a warning.
+                    warnings.warn(f"Invalid values encounterd in np.log "
+                                  f"while scaling feature '{feat}'!")
+        else:
+            raise ValueError(f"`scale` must be either 'linear' or 'log', "
+                             f"got '{scale}'!")
+        return b
+
+    @staticmethod
+    def get_spacing(a, method, scale="linear", method_kw=None,
+                    feat="undefined", ret_scaled=False):
+        """Convenience function for computing the contour spacing
+
+        Parameters
+        ----------
+        a: ndarray
+            feature data
+        scale: str
+            how the data should be scaled ("log" or "linear")
+        method: callable
+            KDE spacing method to use
+        method_kw: dict
+            keyword arguments to `method`
+        feat: str
+            feature name for debugging
+        ret_scaled: bool
+            whether to return the scaled array of `a`
+        """
+        if method_kw is None:
+            method_kw = {}
+        # Apply scale (no change for linear scale)
+        asc = KernelDensityEstimator.apply_scale(a, scale, feat)
+        # Apply multiplicator
+        acc = method(asc, **method_kw)
+        if ret_scaled:
+            return acc, asc
+        else:
+            return acc
+
+    def get_contour(self, xax="area_um", yax="deform", xacc=None, yacc=None,
+                    kde_type="histogram", kde_kwargs=None, xscale="linear",
+                    yscale="linear"):
+        """Evaluate the kernel density estimate for contour plots
+
+        Parameters
+        ----------
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        xacc: float
+            Contour accuracy in x direction
+        yacc: float
+            Contour accuracy in y direction
+        kde_type: str
+            The KDE method to use
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`.
+
+        Returns
+        -------
+        X, Y, Z : coordinates
+            The kernel density Z evaluated on a rectangular grid (X,Y).
+        """
+        warnings.warn("`get_contour` is deprecated; please use "
+                      "`get_raster` instead", DeprecationWarning)
+        return self.get_raster(
+            xax=xax, yax=yax, xacc=xacc, yacc=yacc,
+            kde_type=kde_type, kde_kwargs=kde_kwargs,
+            xscale=xscale, yscale=yscale
+        )
+
+    def get_contour_lines(self, quantiles=None, xax="area_um", yax="deform",
+                          xacc=None, yacc=None, kde_type="histogram",
+                          kde_kwargs=None, xscale="linear", yscale="linear",
+                          ret_levels=False):
+        """Compute contour lines for a given kernel kensity estimate.
+
+        Parameters
+        ----------
+        quantiles: list or array of floats
+            KDE Quantiles for which contour levels are computed. The
+            values must be between 0 and 1. If set to None, use
+            [0.5, 0.95] as default.
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        xacc: float
+            Contour accuracy in x direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        yacc: float
+            Contour accuracy in y direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        kde_type: str
+            The KDE method to use
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`
+        ret_levels: bool
+            If set to True, return the levels of the contours
+            (default: False)
+
+        Returns
+        -------
+        contour_lines: list of lists (of lists)
+            For every number in `quantiles`, this list contains a list of
+            corresponding contour lines. Each contour line is a 2D
+            array of shape (N, 2), where N is the number of points in the
+            contour line.
+        levels: list of floats
+            The density levels corresponding to each number in `quantiles`.
+            Only returned if `ret_levels` is set to True.
+        """
+        if not quantiles:
+            quantiles = [0.5, 0.95]
+        try:
+            x, y, density = self.get_raster(
+                xax=xax,
+                yax=yax,
+                xacc=xacc,
+                yacc=yacc,
+                xscale=xscale,
+                yscale=yscale,
+                kde_type=kde_type,
+                kde_kwargs=kde_kwargs,
+            )
+        except ValueError:
+            # most-likely there is nothing to compute a contour for
+            return []
+        if density.shape[0] < 3 or density.shape[1] < 3:
+            warnings.warn("Contour not possible; spacing may be too large!",
+                          ContourSpacingTooLarge)
+            return []
+        levels = get_quantile_levels(
+            density=density,
+            x=x,
+            y=y,
+            xp=self.rtdc_ds[xax][self.rtdc_ds.filter.all],
+            yp=self.rtdc_ds[yax][self.rtdc_ds.filter.all],
+            q=np.array(quantiles),
+            normalize=False)
+        contours = []
+        # Normalize levels to [0, 1]
+        nlevels = np.array(levels) / density.max()
+        for nlev in nlevels:
+            # make sure that the contour levels are not at the boundaries
+            if not (np.allclose(nlev, 0, atol=1e-12, rtol=0)
+                    or np.allclose(nlev, 1, atol=1e-12, rtol=0)):
+                cc = find_contours_level(
+                    density, x=x, y=y, level=nlev)
+                contours.append(cc)
+            else:
+                contours.append([])
+        if ret_levels:
+            return contours, levels
+        else:
+            return contours
+
+    def get_raster(self, xax="area_um", yax="deform", xacc=None, yacc=None,
+                   kde_type="histogram", kde_kwargs=None, xscale="linear",
+                   yscale="linear"):
+        """Evaluate the kernel density estimate on a grid
+
+        Parameters
+        ----------
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        xacc: float
+            Contour accuracy in x direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        yacc: float
+            Contour accuracy in y direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        kde_type: str
+            The KDE method to use
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`.
+
+        Returns
+        -------
+        X, Y, Z : coordinates
+            The kernel density Z evaluated on a rectangular grid (X,Y).
+        """
+        if kde_kwargs is None:
+            kde_kwargs = {}
+        xax = xax.lower()
+        yax = yax.lower()
+        kde_type = kde_type.lower()
+        if kde_type not in methods:
+            raise ValueError(f"Not a valid kde type: {kde_type}!")
+
+        # Get data
+        x = self.rtdc_ds[xax][self.rtdc_ds.filter.all]
+        y = self.rtdc_ds[yax][self.rtdc_ds.filter.all]
+
+        xacc_sc, xs = self.get_spacing(
+            a=x,
+            feat=xax,
+            scale=xscale,
+            method=bin_width_doane_div5,
+            ret_scaled=True)
+
+        yacc_sc, ys = self.get_spacing(
+            a=y,
+            feat=yax,
+            scale=yscale,
+            method=bin_width_doane_div5,
+            ret_scaled=True)
+
+        if xacc is None or xacc == 0:
+            xacc = xacc_sc
+
+        if yacc is None or yacc == 0:
+            yacc = yacc_sc
+
+        # Ignore infs and nans
+        bad = get_bad_vals(xs, ys)
+        xc = xs[~bad]
+        yc = ys[~bad]
+
+        xnum = int(np.ceil((xc.max() - xc.min()) / xacc))
+        ynum = int(np.ceil((yc.max() - yc.min()) / yacc))
+
+        xlin = np.linspace(xc.min(), xc.max(), xnum, endpoint=True)
+        ylin = np.linspace(yc.min(), yc.max(), ynum, endpoint=True)
+
+        xmesh, ymesh = np.meshgrid(xlin, ylin, indexing="ij")
+
+        kde_fct = methods[kde_type]
+        if len(x):
+            density = kde_fct(events_x=xs, events_y=ys,
+                              xout=xmesh, yout=ymesh,
+                              **kde_kwargs)
+        else:
+            density = np.array([])
+
+        # Convert mesh back to linear scale if applicable
+        if xscale == "log":
+            xmesh = np.exp(xmesh)
+        if yscale == "log":
+            ymesh = np.exp(ymesh)
+
+        return xmesh, ymesh, density
+
+    def get_scatter(self, xax="area_um", yax="deform", positions=None,
+                    kde_type="histogram", kde_kwargs=None, xscale="linear",
+                    yscale="linear"):
+        """Evaluate the kernel density estimate for scatter plots
+
+        The KDE is evaluated with the `kde_type` function for every point.
+
+        Parameters
+        ----------
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        positions: list of two 1d ndarrays or ndarray of shape (2, N)
+            The positions where the KDE will be computed. Note that
+            the KDE estimate is computed from the points that
+            are set in `self.rtdc_ds.filter.all`.
+        kde_type: str
+            The KDE method to use, see :const:`.kde_methods.methods`
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`.
+
+        Returns
+        -------
+        density : 1d ndarray
+            The kernel density evaluated for the filtered data points.
+        """
+        if kde_kwargs is None:
+            kde_kwargs = {}
+        xax = xax.lower()
+        yax = yax.lower()
+        kde_type = kde_type.lower()
+        if kde_type not in methods:
+            raise ValueError(f"Not a valid kde type: {kde_type}!")
+
+        # Get data
+        x = self.rtdc_ds[xax][self.rtdc_ds.filter.all]
+        y = self.rtdc_ds[yax][self.rtdc_ds.filter.all]
+
+        # Apply scale (no change for linear scale)
+        xs = self.apply_scale(x, xscale, xax)
+        ys = self.apply_scale(y, yscale, yax)
+
+        if positions is None:
+            posx = None
+            posy = None
+        else:
+            posx = self.apply_scale(positions[0], xscale, xax)
+            posy = self.apply_scale(positions[1], yscale, yax)
+
+        kde_fct = methods[kde_type]
+        if len(x):
+            density = kde_fct(events_x=xs, events_y=ys,
+                              xout=posx, yout=posy,
+                              **kde_kwargs)
+        else:
+            density = np.array([])
+
+        return density
+
+    def get_at(self, xax="area_um", yax="deform", positions=None,
+               kde_type="histogram", kde_kwargs=None, xscale="linear",
+               yscale="linear"):
+        """Evaluate the kernel density estimate for specific events
+
+        The KDE is computed via linear interpolation from the output
+        of `get_raster`.
+
+        Parameters
+        ----------
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        positions: list of two 1d ndarrays or ndarray of shape (2, N)
+            The positions where the KDE will be computed. Note that
+            the KDE estimate is computed from the points that
+            are set in `self.rtdc_ds.filter.all`.
+        kde_type: str
+            The KDE method to use, see :const:`.kde_methods.methods`
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`.
+
+        Returns
+        -------
+        density : 1d ndarray
+            The kernel density evaluated for the filtered events.
+        """
+        if kde_kwargs is None:
+            kde_kwargs = {}
+        xax = xax.lower()
+        yax = yax.lower()
+        kde_type = kde_type.lower()
+        if kde_type not in methods:
+            raise ValueError(f"Not a valid kde type: {kde_type}!")
+
+        # Get data
+        x = self.rtdc_ds[xax][self.rtdc_ds.filter.all]
+        y = self.rtdc_ds[yax][self.rtdc_ds.filter.all]
+
+        # Apply scale (no change for linear scale)
+        xs = self.apply_scale(x, xscale, xax)
+        ys = self.apply_scale(y, yscale, yax)
+
+        if positions:
+            xs = self.apply_scale(positions[0], xscale, xax)
+            ys = self.apply_scale(positions[1], yscale, yax)
+
+        if len(x):
+            xr, yr, density_grid = self.get_raster(xax=xax,
+                                                   yax=yax,
+                                                   kde_type=kde_type,
+                                                   kde_kwargs=kde_kwargs,
+                                                   xscale=xscale,
+                                                   yscale=yscale)
+
+            # Apply scale (no change for linear scale)
+            xrs = self.apply_scale(xr, xscale, xax)
+            yrs = self.apply_scale(yr, yscale, yax)
+
+            # 'scipy.interp2d' has been removed in SciPy 1.14.0
+            # https://scipy.github.io/devdocs/tutorial/interpolate/interp_transition_guide.html
+            interp_func = RGI((xrs[:, 0], yrs[0, :]),
+                              density_grid,
+                              method="linear",
+                              bounds_error=False,
+                              fill_value=np.nan)
+            density = interp_func((xs, ys))
+
+        else:
+            density = np.array([])
+
+        return density

dclab/kde/contours.py

@@ -0,0 +1,222 @@
+
+import numpy as np
+
+from ..external.skimage.measure import find_contours, points_in_poly
+import scipy.interpolate as spint
+
+from .methods import get_bad_vals
+
+
+def find_contours_level(density, x, y, level, closed=False):
+    """Find iso-valued density contours for a given level value
+
+    Parameters
+    ----------
+    density: 2d ndarray of shape (M, N)
+        Kernel density estimate (KDE) for which to compute the contours
+    x: 2d ndarray of shape (M, N) or 1d ndarray of size M
+        X-values corresponding to `density`
+    y: 2d ndarray of shape (M, N) or 1d ndarray of size M
+        Y-values corresponding to `density`
+    level: float between 0 and 1
+        Value along which to find contours in `density` relative
+        to its maximum
+    closed: bool
+        Whether to close contours at the KDE support boundaries
+
+    Returns
+    -------
+    contours: list of ndarrays of shape (P, 2)
+        Contours found for the given level value
+
+    See Also
+    --------
+    skimage.measure.find_contours: Contour finding algorithm used
+    """
+    if level >= 1 or level <= 0:
+        raise ValueError("`level` must be in (0,1), got '{}'!".format(level))
+    # level relative to maximum
+    level = level * density.max()
+    # xy coordinates
+    if len(x.shape) == 2:
+        assert np.all(x[:, 0] == x[:, 1])
+        x = x[:, 0]
+    if len(y.shape) == 2:
+        assert np.all(y[0, :] == y[1, :])
+        y = y[0, :]
+    if closed:
+        # find closed contours
+        density = np.pad(density, ((1, 1), (1, 1)), mode="constant")
+        offset = 1
+    else:
+        # leave contours open at kde boundary
+        offset = 0
+
+    conts_idx = find_contours(density, level)
+    conts_xy = []
+
+    for cc in conts_idx:
+        cx = np.interp(x=cc[:, 0]-offset,
+                       xp=range(x.size),
+                       fp=x)
+        cy = np.interp(x=cc[:, 1]-offset,
+                       xp=range(y.size),
+                       fp=y)
+        conts_xy.append(np.stack((cx, cy), axis=1))
+
+    return conts_xy
+
+
+def get_quantile_levels(density, x, y, xp, yp, q, normalize=True):
+    """Compute density levels for given quantiles by interpolation
+
+    For a given 2D density, compute the density levels at which
+    the resulting contours contain the fraction `1-q` of all
+    data points. E.g. for a measurement of 1000 events, all
+    contours at the level corresponding to a quantile of
+    `q=0.95` (95th percentile) contain 50 events (5%).
+
+    Parameters
+    ----------
+    density: 2d ndarray of shape (M, N)
+        Kernel density estimate for which to compute the contours
+    x: 2d ndarray of shape (M, N) or 1d ndarray of size M
+        X-values corresponding to `density`
+    y: 2d ndarray of shape (M, N) or 1d ndarray of size M
+        Y-values corresponding to `density`
+    xp: 1d ndarray of size D
+        Event x-data from which to compute the quantile
+    yp: 1d ndarray of size D
+        Event y-data from which to compute the quantile
+    q: array_like or float between 0 and 1
+        Quantile along which to find contours in `density` relative
+        to its maximum
+    normalize: bool
+        Whether output levels should be normalized to the maximum
+        of `density`
+
+    Returns
+    -------
+    level: np.ndarray or float
+        Contours level(s) corresponding to the given quantile
+
+    Notes
+    -----
+    NaN-values events in `xp` and `yp` are ignored.
+    """
+    # xy coordinates
+    if len(x.shape) == 2:
+        assert np.all(x[:, 0] == x[:, 1])
+        x = x[:, 0]
+    if len(y.shape) == 2:
+        assert np.all(y[0, :] == y[1, :])
+        y = y[0, :]
+
+    # remove bad events
+    bad = get_bad_vals(xp, yp)
+    xp = xp[~bad]
+    yp = yp[~bad]
+
+    # Normalize interpolation data such that the spacing for
+    # x and y is about the same during interpolation.
+    x_norm = x.max()
+    x = x / x_norm
+    xp = xp / x_norm
+
+    y_norm = y.max()
+    y = y / y_norm
+    yp = yp / y_norm
+
+    # Perform interpolation
+    dp = spint.interpn((x, y), density,
+                       (xp, yp),
+                       method='linear',
+                       bounds_error=False,
+                       fill_value=0)
+
+    if normalize:
+        dp /= density.max()
+
+    if not np.isscalar(q):
+        q = np.array(q)
+    plev = np.nanpercentile(dp, q=q*100)
+    return plev
+
+
+def _find_quantile_level(density, x, y, xp, yp, quantile, acc=.01,
+                         ret_err=False):
+    """Find density level for a given data quantile by iteration
+
+    Parameters
+    ----------
+    density: 2d ndarray of shape (M, N)
+        Kernel density estimate for which to compute the contours
+    x: 2d ndarray of shape (M, N) or 1d ndarray of size M
+        X-values corresponding to `density`
+    y: 2d ndarray of shape (M, N) or 1d ndarray of size M
+        Y-values corresponding to `density`
+    xp: 1d ndarray of size D
+        Event x-data from which to compute the quantile
+    yp: 1d ndarray of size D
+        Event y-data from which to compute the quantile
+    quantile: float between 0 and 1
+        Quantile along which to find contours in `density` relative
+        to its maximum
+    acc: float
+        Desired absolute accuracy (stopping criterion) of the
+        contours
+    ret_err: bool
+        If True, also return the absolute error
+
+    Returns
+    -------
+    level: float
+        Contours level corresponding to the given quantile
+
+    Notes
+    -----
+    A much more faster method (using interpolation) is implemented in
+    :func:`get_quantile_levels`.
+    NaN-values events in `xp` and `yp` are ignored.
+
+    See Also
+    --------
+    skimage.measure.find_contours: Contour finding algorithm
+    """
+    if quantile >= 1 or quantile <= 0:
+        raise ValueError("Invalid value for `quantile`: {}".format(quantile))
+
+    # remove bad events
+    bad = get_bad_vals(xp, yp)
+    xp = xp[~bad]
+    yp = yp[~bad]
+    points = np.concatenate((xp.reshape(-1, 1), yp.reshape(-1, 1)), axis=1)
+
+    # initial guess
+    level = quantile
+    # error of current iteration
+    err = 1
+    # iteration factor (guarantees convergence)
+    itfac = 1
+    # total number of events
+    nev = xp.size
+
+    while np.abs(err) > acc:
+        # compute contours
+        conts = find_contours_level(density, x, y, level, closed=True)
+        # compute number of points in contour
+        isin = 0
+        pi = np.array(points, copy=True)
+        for cc in conts:
+            pinc = points_in_poly(points=pi, verts=cc)
+            isin += np.sum(pinc)
+            # ignore these points for the other contours
+            pi = pi[~pinc]
+        err = quantile - (nev - isin) / nev
+        level += err * itfac
+        itfac *= .9
+
+    if ret_err:
+        return level, err
+    else:
+        return level