dclab 0.67.0__cp314-cp314-macosx_10_13_x86_64.whl

dclab/kde/methods.py

@@ -0,0 +1,313 @@
+"""Kernel Density Estimation methods"""
+
+import numpy as np
+from scipy.interpolate import RectBivariateSpline
+from scipy.stats import gaussian_kde, skew
+
+from ..cached import Cache
+from ..external.statsmodels.nonparametric.kernel_density import KDEMultivariate
+
+
+def bin_num_doane(a):
+    """Compute number of bins based on Doane's formula
+
+    Notes
+    -----
+    If the bin width cannot be determined, then a bin
+    number of 5 is returned.
+
+    See Also
+    --------
+    bin_width_doane: method used to compute the bin width
+    """
+    bad = np.isnan(a) | np.isinf(a)
+    data = a[~bad]
+    acc = bin_width_doane(a)
+    if acc == 0 or np.isnan(acc):
+        num = 5
+    else:
+        num = int(np.round((data.max() - data.min()) / acc))
+    return num
+
+
+def bin_width_doane(a):
+    """Compute contour spacing based on Doane's formula
+
+    References
+    ----------
+    - `<https://en.wikipedia.org/wiki/Histogram#Number_of_bins_and_width>`_
+    - `<https://stats.stackexchange.com/questions/55134/
+      doanes-formula-for-histogram-binning>`_
+
+    Notes
+    -----
+    Doane's formula is actually designed for histograms. This
+    function is kept here for backwards-compatibility reasons.
+    It is highly recommended to use :func:`bin_width_percentile`
+    instead.
+    """
+    bad = np.isnan(a) | np.isinf(a)
+    data = a[~bad]
+    n = data.size
+    g1 = skew(data)
+    sigma_g1 = np.sqrt(6 * (n - 2) / ((n + 1) * (n + 3)))
+    k = 1 + np.log2(n) + np.log2(1 + np.abs(g1) / sigma_g1)
+    acc = (data.max() - data.min()) / k
+    return acc
+
+
+def bin_width_doane_div5(a):
+    """Compute contour spacing based on Doane's formula divided by five
+
+    See Also
+    --------
+    bin_width_doane: method used to compute the bin width
+    """
+    return bin_width_doane(a) / 5
+
+
+def bin_width_percentile(a):
+    """Compute contour spacing based on data percentiles
+
+    The 10th and the 90th percentile of the input data are taken.
+    The spacing then computes to the difference between those
+    two percentiles divided by 23.
+
+    Notes
+    -----
+    The Freedman–Diaconis rule uses the interquartile range and
+    normalizes to the third root of len(a). Such things do not
+    work very well for RT-DC data, because len(a) is huge. Here
+    we use just the top and bottom 10th percentiles with a fixed
+    normalization.
+    """
+    bad = np.isnan(a) | np.isinf(a)
+    data = a[~bad]
+    start = np.percentile(data, 10)
+    end = np.percentile(data, 90)
+    acc = (end - start) / 23
+    return acc
+
+
+def get_bad_vals(x, y):
+    return np.isnan(x) | np.isinf(x) | np.isnan(y) | np.isinf(y)
+
+
+def ignore_nan_inf(kde_method):
+    """Ignores nans and infs from the input data
+
+    Invalid positions in the resulting density are set to nan.
+    """
+    def new_kde_method(events_x, events_y, xout=None, yout=None,
+                       *args, **kwargs):
+        bad_in = get_bad_vals(events_x, events_y)
+        if xout is None:
+            density = np.zeros_like(events_x, dtype=np.float64)
+            bad_out = bad_in
+            xo = yo = None
+        else:
+            density = np.zeros_like(xout, dtype=np.float64)
+            bad_out = get_bad_vals(xout, yout)
+            xo = xout[~bad_out]
+            yo = yout[~bad_out]
+        # Filter events
+        ev_x = events_x[~bad_in]
+        ev_y = events_y[~bad_in]
+        density[~bad_out] = kde_method(ev_x, ev_y,
+                                       xo, yo,
+                                       *args, **kwargs)
+        density[bad_out] = np.nan
+        return density
+
+    doc_add = "\n    Notes\n" +\
+              "    -----\n" +\
+              "    This is a wrapped version that ignores nan and inf values."
+    new_kde_method.__doc__ = kde_method.__doc__ + doc_add
+
+    return new_kde_method
+
+
+@ignore_nan_inf
+@Cache
+def kde_gauss(events_x, events_y, xout=None, yout=None):
+    """ Gaussian Kernel Density Estimation
+
+    Parameters
+    ----------
+    events_x, events_y: 1D ndarray
+        The input points for kernel density estimation. Input
+        is flattened automatically.
+    xout, yout: ndarray
+        The coordinates at which the KDE should be computed.
+        If set to none, input coordinates are used.
+
+    Returns
+    -------
+    density: ndarray, same shape as `xout`
+        The KDE for the points in (xout, yout)
+
+    See Also
+    --------
+    `scipy.stats.gaussian_kde`
+    """
+    valid_combi = ((xout is None and yout is None) or
+                   (xout is not None and yout is not None)
+                   )
+    if not valid_combi:
+        raise ValueError("Both `xout` and `yout` must be (un)set.")
+
+    if xout is None and yout is None:
+        xout = events_x
+        yout = events_y
+
+    try:
+        estimator = gaussian_kde([events_x.flatten(), events_y.flatten()])
+        density = estimator.evaluate([xout.flatten(), yout.flatten()])
+    except np.linalg.LinAlgError:
+        # LinAlgError occurs when matrix to solve is singular (issue #117)
+        density = np.zeros(xout.shape)*np.nan
+    return density.reshape(xout.shape)
+
+
+@ignore_nan_inf
+@Cache
+def kde_histogram(events_x, events_y, xout=None, yout=None, bins=None):
+    """ Histogram-based Kernel Density Estimation
+
+    Parameters
+    ----------
+    events_x, events_y: 1D ndarray
+        The input points for kernel density estimation. Input
+        is flattened automatically.
+    xout, yout: ndarray
+        The coordinates at which the KDE should be computed.
+        If set to none, input coordinates are used.
+    bins: tuple (binsx, binsy)
+        The number of bins to use for the histogram.
+
+    Returns
+    -------
+    density: ndarray, same shape as `xout`
+        The KDE for the points in (xout, yout)
+
+    See Also
+    --------
+    `numpy.histogram2d`
+    `scipy.interpolate.RectBivariateSpline`
+    """
+    valid_combi = ((xout is None and yout is None) or
+                   (xout is not None and yout is not None)
+                   )
+    if not valid_combi:
+        raise ValueError("Both `xout` and `yout` must be (un)set.")
+
+    if xout is None and yout is None:
+        xout = events_x
+        yout = events_y
+
+    if bins is None:
+        bins = (max(5, bin_num_doane(events_x)),
+                max(5, bin_num_doane(events_y)))
+
+    # Compute the histogram
+    hist2d, xedges, yedges = np.histogram2d(x=events_x,
+                                            y=events_y,
+                                            bins=bins,
+                                            density=True)
+    xip = xedges[1:]-(xedges[1]-xedges[0])/2
+    yip = yedges[1:]-(yedges[1]-yedges[0])/2
+
+    estimator = RectBivariateSpline(x=xip, y=yip, z=hist2d)
+    density = estimator.ev(xout, yout)
+    density[density < 0] = 0
+
+    return density.reshape(xout.shape)
+
+
+def kde_none(events_x, events_y, xout=None, yout=None):
+    """No Kernel Density Estimation
+
+    Parameters
+    ----------
+    events_x, events_y: 1D ndarray
+        The input points for kernel density estimation. Input
+        is flattened automatically.
+    xout, yout: ndarray
+        The coordinates at which the KDE should be computed.
+        If set to none, input coordinates are used.
+
+    Returns
+    -------
+    density: ndarray, same shape as `xout`
+        The KDE for the points in (xout, yout)
+
+    Notes
+    -----
+    This method is a convenience method that always returns ones in the shape
+    that the other methods in this module produce.
+    """
+    valid_combi = ((xout is None and yout is None) or
+                   (xout is not None and yout is not None)
+                   )
+    if not valid_combi:
+        raise ValueError("Both `xout` and `yout` must be (un)set.")
+
+    if xout is None and yout is None:
+        xout = events_x
+        _ = events_y
+
+    return np.ones(xout.shape)
+
+
+@ignore_nan_inf
+@Cache
+def kde_multivariate(events_x, events_y, xout=None, yout=None, bw=None):
+    """ Multivariate Kernel Density Estimation
+
+    Parameters
+    ----------
+    events_x, events_y: 1D ndarray
+        The input points for kernel density estimation. Input
+        is flattened automatically.
+    bw: tuple (bwx, bwy) or None
+        The bandwith for kernel density estimation.
+    xout, yout: ndarray
+        The coordinates at which the KDE should be computed.
+        If set to none, input coordinates are used.
+
+    Returns
+    -------
+    density: ndarray, same shape as `xout`
+        The KDE for the points in (xout, yout)
+
+    See Also
+    --------
+    `statsmodels.nonparametric.kernel_density.KDEMultivariate`
+    """
+    valid_combi = ((xout is None and yout is None) or
+                   (xout is not None and yout is not None)
+                   )
+    if not valid_combi:
+        raise ValueError("Both `xout` and `yout` must be (un)set.")
+
+    if xout is None and yout is None:
+        xout = events_x
+        yout = events_y
+    if bw is None:
+        # divide by 2 to make it comparable to histogram KDE
+        bw = (bin_width_doane(events_x) / 2,
+              bin_width_doane(events_y) / 2)
+
+    positions = np.vstack([xout.flatten(), yout.flatten()])
+    estimator_ly = KDEMultivariate(data=[events_x.flatten(),
+                                         events_y.flatten()],
+                                   var_type='cc', bw=bw)
+
+    density = estimator_ly.pdf(positions)
+    return density.reshape(xout.shape)
+
+
+methods = {"gauss": kde_gauss,
+           "histogram": kde_histogram,
+           "none": kde_none,
+           "multivariate": kde_multivariate}

dclab/kde_contours.py

@@ -0,0 +1,10 @@
+import warnings
+
+from .kde.contours import (  # noqa: F401
+    find_contours_level, _find_quantile_level, get_quantile_levels
+)
+
+
+warnings.warn("`dclab.kde_contours` is deprecated; please use "
+              "the `dclab.kde.contours` instead",
+              DeprecationWarning)

dclab/kde_methods.py

@@ -0,0 +1,11 @@
+import warnings
+
+from .kde.methods import (  # noqa: F401
+    bin_num_doane, bin_width_doane, bin_width_percentile, get_bad_vals,
+    ignore_nan_inf, kde_gauss, kde_histogram, kde_multivariate, kde_none,
+    methods
+)
+
+warnings.warn("`dclab.kde_methods` is deprecated; please use "
+              "the `dclab.kde.methods` instead",
+              DeprecationWarning)

dclab/lme4/__init__.py

@@ -0,0 +1,5 @@
+"""A wrapper around R with the lme4 package"""
+from . import rsetup, wrapr  # noqa: F401
+from .wrapr import Rlme4, bootstrapped_median_distributions  # noqa: F401
+from .rsetup import (  # noqa: F401
+    set_r_lib_path, get_r_path, get_r_version, require_lme4, set_r_path)

dclab/lme4/lme4_template.R

@@ -0,0 +1,94 @@
+require(stats);
+require(lme4);
+
+model_name <- "<MODEL_NAME>"
+cat("OUTPUT model:", model_name, "#*#\n")
+
+func_model <- "feature ~ group + (1 + group | repetition)"
+func_nullmodel <- "feature ~ (1 + group | repetition)"
+
+# These are the feature, group, and repetition arrays that are set by dclab
+# via templates.
+feature <- c(<FEATURES>)
+group <- c(<GROUPS>)
+repetition <- c(<REPETITIONS>)
+
+data <- data.frame(feature, group, repetition)
+
+if (model_name == "glmer+loglink") {
+    Model <- glmer(func_model, data, family=Gamma(link='log'))
+    NullModel <- glmer(func_nullmodel, data, family=Gamma(link='log'))
+} else if (model_name == "lmer") {
+    Model <- lmer(func_model, data)
+    NullModel <- lmer(func_nullmodel, data)
+} else {
+    stop("Invalid model_name:", model_name)
+}
+
+# Anova analysis (increase verbosity by making models global)
+# Using anova is a very conservative way of determining
+# p values.
+res_anova <- anova(Model, NullModel)
+cat("OUTPUT r anova: ")
+res_anova
+cat("#*#\n")
+
+pvalue <- res_anova$"Pr(>Chisq)"[2]
+cat("OUTPUT anova p-value:", pvalue, "#*#\n")
+
+model_summary <- summary(Model)
+cat("OUTPUT r model summary:")
+model_summary
+cat("#*#\n")
+
+model_coefficients <- coef(Model)
+cat("OUTPUT r model coefficients:")
+model_coefficients
+cat("#*#\n")
+
+fe_reps <- model_coefficients$repetition
+
+effects <- data.frame(coef(model_summary))
+
+fe_icept <- effects$Estimate[1]
+
+fe_treat <- effects$Estimate[2]
+
+if (model_name == "glmer+loglink") {
+    # transform back from log
+    fe_treat <- exp(fe_icept + fe_treat) - exp(fe_icept)
+    fe_icept <- exp(fe_icept)
+    fe_reps[, 2] = exp(fe_reps[, 1] + fe_reps[, 2]) - exp(fe_reps[, 1])
+    fe_reps[, 1] = exp(fe_reps[, 1])
+}
+
+cat("OUTPUT fixed effects intercept:", fe_icept, "#*#\n")
+cat("OUTPUT fixed effects treatment:", fe_treat, "#*#\n")
+cat("OUTPUT fixed effects repetitions:")
+fe_reps
+cat("#*#\n")
+
+# convergence
+
+# convergence warnings in lme4
+is_warning_generated <- function(m) {
+  df <- summary(m)
+  !is.null(df$optinfo$conv$lme4$messages) &&
+           grepl('failed to converge', df$optinfo$conv$lme4$messages)
+}
+lme4_not_converged <- is_warning_generated(Model)
+
+# convergence code by the optimizer
+lme4l <- model_summary$optinfo$conv$lme4
+if (length(lme4l) == 0) {
+    # the optimizer probably does not know
+    conv_code <- 0
+} else if (is.null(lme4l$code)) {
+    # NULL means 0
+    conv_code <- 0
+} else {
+    conv_code <- lme4l$code
+}
+
+cat("OUTPUT model converged:", (conv_code == 0) && !lme4_not_converged, "#*#\n")
+cat("OUTPUT lme4 messages:", lme4l$optinfo$conv$lme4$messages, "#*#\n")

dclab/lme4/rsetup.py

@@ -0,0 +1,204 @@
+import logging
+import os
+import pathlib
+import shutil
+import subprocess as sp
+
+logger = logging.getLogger(__name__)
+
+_has_lme4 = None
+_has_r = None
+
+
+class CommandFailedError(BaseException):
+    """Used when `run_command` encounters an error"""
+    pass
+
+
+class RNotFoundError(BaseException):
+    pass
+
+
+def get_r_path():
+    """Return the path of the R executable"""
+    # Maybe the user set the executable already?
+    r_exec = os.environ.get("R_EXEC")
+    if r_exec is not None:
+        r_exec = pathlib.Path(r_exec)
+        if r_exec.is_file():
+            return r_exec
+
+    # Try to get the executable using which
+    r_exec = shutil.which("R")
+    if r_exec is not None:
+        r_exec = pathlib.Path(r_exec)
+        return r_exec
+
+    # Try to determine the path to the executable from R_HOME
+    r_home = os.environ.get("R_HOME")
+    if r_home and not pathlib.Path(r_home).is_dir():
+        logger.warning(f"R_HOME Directory does not exist: {r_home}")
+        r_home = None
+
+    if r_home is None:
+        raise RNotFoundError(
+            "Cannot find R, please set the `R_HOME` environment variable "
+            "or use `set_r_path`.")
+
+    r_home = pathlib.Path(r_home)
+
+    # search for the R executable
+    for rr in [
+        r_home / "bin" / "R",
+        r_home / "bin" / "x64" / "R",
+    ]:
+        if rr.is_file():
+            return rr
+        rr_win = rr.with_name("R.exe")
+        if rr_win.is_file():
+            return rr_win
+    else:
+        raise RNotFoundError(
+            f"Could not find R binary in '{r_home}'")
+
+
+def get_r_script_path():
+    """Return the path to the Rscript executable"""
+    return get_r_path().with_name("Rscript")
+
+
+def get_r_version():
+    """Return the full R version string"""
+    require_r()
+    cmd = (str(get_r_path()), "--version")
+    logger.debug(f"Looking for R version with: {' '.join(cmd)}")
+    r_version = run_command(
+        cmd,
+        env={"R_LIBS_USER": os.environ.get("R_LIBS_USER", "")},
+    )
+    r_version = r_version.split(os.linesep)
+    if r_version[0].startswith("WARNING"):
+        r_version = r_version[1]
+    else:
+        r_version = r_version[0]
+    logger.info(f"R version found: {r_version}")
+    # get the actual version string
+    if r_version.startswith("R version "):
+        r_version = r_version.split(" ", 2)[2]
+    return r_version.strip()
+
+
+def has_lme4():
+    """Return True if the lme4 package is installed"""
+    global _has_lme4
+    if _has_lme4:
+        return True
+    require_r()
+    for pkg in ["lme4", "statmod", "nloptr"]:
+        res = run_command(
+            (str(get_r_path()), "-q", "-e", f"system.file(package='{pkg}')"),
+            env={"R_LIBS_USER": os.environ.get("R_LIBS_USER", "")},
+            )
+        if not res.split("[1]")[1].count(pkg):
+            avail = False
+            break
+    else:
+        avail = _has_lme4 = True
+    return avail
+
+
+def has_r():
+    """Return True if R is available"""
+    global _has_r
+    if _has_r:
+        return True
+    try:
+        hasr = get_r_path().is_file()
+    except RNotFoundError:
+        hasr = False
+    if hasr:
+        _has_r = True
+    return hasr
+
+
+def require_lme4():
+    """Install the lme4 package (if not already installed)
+
+    Besides ``lme4``, this also installs ``nloptr`` and ``statmod``.
+    The packages are installed to the user data directory
+    given in :const:`lib_path` from the http://cran.rstudio.org mirror.
+    """
+    install_command = ("install.packages("
+                       "c('statmod','nloptr','lme4'),"
+                       "repos='http://cran.rstudio.org'"
+                       ")"
+                       )
+    require_r()
+    if not has_lme4():
+        run_command(cmd=(get_r_path(), "-e", install_command),
+                    env={"R_LIBS_USER": os.environ.get("R_LIBS_USER", "")},
+                    )
+
+
+def require_r():
+    """Make sure R is installed an R HOME is set"""
+    if not has_r():
+        raise RNotFoundError("Cannot find R, please set its path with the "
+                             "`set_r_path` function or set the `RHOME` "
+                             "environment variable.")
+
+
+def run_command(cmd, **kwargs):
+    """Run a command via subprocess"""
+    if hasattr(sp, "STARTUPINFO"):
+        # On Windows, subprocess calls will pop up a command window by
+        # default when run from Pyinstaller with the ``--noconsole``
+        # option. Avoid this distraction.
+        si = sp.STARTUPINFO()
+        si.dwFlags |= sp.STARTF_USESHOWWINDOW
+        # Windows doesn't search the path by default. Pass it an
+        # environment so it will.
+        env = os.environ
+    else:
+        si = None
+        env = None
+
+    kwargs.setdefault("text", True)
+    kwargs.setdefault("stderr", sp.STDOUT)
+    if env is not None:
+        if "env" in kwargs:
+            env.update(kwargs.pop("env"))
+        kwargs["env"] = env
+    kwargs["startupinfo"] = si
+
+    # Convert paths to strings
+    cmd = [str(cc) for cc in cmd]
+
+    try:
+        tmp = sp.check_output(cmd, **kwargs)
+    except sp.CalledProcessError as e:
+        raise CommandFailedError(f"The command '{' '.join(cmd)}' failed with "
+                                 f"exit code {e.returncode}: {e.output}")
+
+    return tmp.strip()
+
+
+def set_r_lib_path(r_lib_path):
+    """Add given directory to the R_LIBS_USER environment variable"""
+    paths = os.environ.get("R_LIBS_USER", "").split(os.pathsep)
+    paths = [p for p in paths if p]
+    paths.append(str(r_lib_path).strip())
+    os.environ["R_LIBS_USER"] = os.pathsep.join(list(set(paths)))
+
+
+def set_r_path(r_path):
+    """Set the path of the R executable/binary"""
+    tmp = run_command((str(r_path), "RHOME"))
+
+    r_home = tmp.split(os.linesep)
+    if r_home[0].startswith("WARNING"):
+        res = r_home[1]
+    else:
+        res = r_home[0].strip()
+    os.environ["R_HOME"] = res
+    os.environ["R_EXEC"] = str(pathlib.Path(r_path).resolve())