dclab 0.61.2__cp312-cp312-macosx_10_9_x86_64.whl → 2.18.0__cp312-cp312-macosx_10_9_x86_64.whl

dclab/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.61.2'
-__version_tuple__ = version_tuple = (0, 61, 2)
+__version__ = version = '2.18.0'
+__version_tuple__ = version_tuple = (2, 18, 0)

dclab/lme4/__init__.py

@@ -1,5 +1,5 @@
 """A wrapper around R with the lme4 package"""
-from . import rsetup, wrapr  # noqa: F401
-from .wrapr import Rlme4, bootstrapped_median_distributions  # noqa: F401
-from .rsetup import (  # noqa: F401
-    get_r_path, get_r_version, require_lme4, set_r_path)
+# flake8: noqa: F401
+from . import rlibs, rsetup, wrapr
+from .wrapr import Rlme4, bootstrapped_median_distributions
+from .rsetup import get_r_path, get_r_version, install_lme4, set_r_path

dclab/lme4/rlibs.py

@@ -0,0 +1,93 @@
+import importlib
+import os
+import warnings
+
+from ..external.packaging import parse as parse_version
+
+#: Minimum R version
+#: This is actually a dependency for rpy2, because the API changed then
+#: (ffi.error: symbol 'R_tryCatchError' not found in library).
+R_MIN_VERSION = "3.6.0"
+
+#: Minimum rpy2 version
+RPY2_MIN_VERSION = "2.9.4"
+
+R_SUBMODULES = [
+    "rpy2.robjects.packages",
+    "rpy2.situation",
+    "rpy2.robjects.vectors",
+]
+
+R_SUBMODULES_3 = [
+    "rpy2.rinterface_lib.callbacks",
+]
+
+
+class RPY2UnavailableError(BaseException):
+    pass
+
+
+class RPY2ImportError(RPY2UnavailableError):
+    pass
+
+
+class RPY2OutdatedError(RPY2UnavailableError):
+    pass
+
+
+class RUnavailableError(BaseException):
+    pass
+
+
+class ROutdatedError(RUnavailableError):
+    pass
+
+
+class MockRPackage:
+    def __init__(self, exception):
+        self.exception = exception
+
+    def __getattr__(self, item):
+        raise self.exception
+
+
+def import_r_submodules():
+    importlib.import_module("rpy2.situation")
+    r_home = rpy2.situation.get_r_home()
+    if r_home is not None:
+        if os.environ.get("R_HOME", None) is None:
+            # set R_HOME globally (https://github.com/rpy2/rpy2/issues/796)
+            os.environ["R_HOME"] = r_home
+        if rpy2_is_version_3:
+            mods = R_SUBMODULES + R_SUBMODULES_3
+        else:
+            mods = R_SUBMODULES
+        try:
+            for sm in mods:
+                importlib.import_module(sm)
+        except rpy2.rinterface_lib.openrlib.ffi.error as exc:
+            # This error happens when the installed R version is too old:
+            # "ffi.error: symbol 'R_tryCatchError' not found in library"
+            raise ROutdatedError(
+                f"Encountered '{exc.__class__.__name__}: {exc}'. "
+                f"Please make sure you have 'R>={R_MIN_VERSION}'!")
+
+
+try:
+    rpy2 = importlib.import_module("rpy2")
+    if parse_version(rpy2.__version__) < parse_version(RPY2_MIN_VERSION):
+        raise RPY2OutdatedError(f"Please install 'rpy2>={RPY2_MIN_VERSION}'!")
+except ImportError:
+    rpy2 = MockRPackage(
+        RPY2ImportError(f"Please install 'rpy2>={RPY2_MIN_VERSION}'!"))
+    rpy2_is_version_3 = False
+except BaseException as e:
+    rpy2 = MockRPackage(e)
+    rpy2_is_version_3 = False
+else:
+    rpy2_is_version_3 = parse_version(rpy2.__version__) >= parse_version("3.0")
+    try:
+        import_r_submodules()
+    except RUnavailableError as e:
+        warnings.warn("There is an issue with the linked R version: "
+                      + f"{e.__class__.__name__}: {e}")

dclab/lme4/rsetup.py

@@ -1,142 +1,185 @@
 import logging
 import os
-import pathlib
 import subprocess as sp
-import sys
 
-logger = logging.getLogger(__name__)
+from .rlibs import (
+    RUnavailableError, rpy2, rpy2_is_version_3, import_r_submodules)
 
-_has_lme4 = None
-_has_r = None
+# Disable rpy2 logger because of unnecessary prints to stdout
+logging.getLogger("rpy2.rinterface_lib.callbacks").disabled = True
 
 
 class RNotFoundError(BaseException):
     pass
 
 
-def get_r_path():
-    """Return the path of the R executable"""
-    # Maybe the user set the executable already?
-    r_exec = os.environ.get("R_EXEC")
-    if r_exec is not None:
-        r_exec = pathlib.Path(r_exec)
-        if r_exec.is_file():
-            return r_exec
-
-    # Try to determine the path to the executable from R_HOME
-    r_home = os.environ.get("R_HOME")
-    if not pathlib.Path(r_home).is_dir():
-        logger.warning(f"R_HOME Directory does not exist: {r_home}")
-        r_home = None
-    if r_home is None:
-        cmd = ("R", "RHOME")
-        try:
-            tmp = sp.check_output(cmd, universal_newlines=True)
-            # may raise FileNotFoundError, WindowsError, etc
-            r_home = tmp.split(os.linesep)
-        except BaseException:
-            pass
+class AutoRConsole(object):
+    """Helper class for catching R console output"""
+    lock = False
+    perform_lock = rpy2_is_version_3
+
+    def __init__(self):
+        """
+        By default, this console always returns "yes" when asked a
+        question. If you need something different, you can subclass
+        and override `consoleread` fucntion. The console stream is
+        recorded in `self.stream`.
+        """
+        self.stream = [["init", "Starting RConsole class\n"]]
+        if AutoRConsole.perform_lock:
+            if AutoRConsole.lock:
+                raise ValueError("Only one RConsole instance allowed!")
+            AutoRConsole.lock = True
+            self.original_funcs = {
+                "consoleread": rpy2.rinterface_lib.callbacks.consoleread,
+                "consolewrite_print":
+                    rpy2.rinterface_lib.callbacks.consolewrite_print,
+                "consolewrite_warnerror":
+                    rpy2.rinterface_lib.callbacks.consolewrite_warnerror,
+            }
+            rpy2.rinterface_lib.callbacks.consoleread = self.consoleread
+            rpy2.rinterface_lib.callbacks.consolewrite_print = \
+                self.consolewrite_print
+            rpy2.rinterface_lib.callbacks.showmessage = \
+                self.consolewrite_print
+
+            rpy2.rinterface_lib.callbacks.consolewrite_warnerror = \
+                self.consolewrite_warnerror
+        # Set locale (to get always English messages)
+        rpy2.robjects.r('Sys.setlocale("LC_MESSAGES", "C")')
+        rpy2.robjects.r('Sys.setlocale("LC_CTYPE", "C")')
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        if AutoRConsole.perform_lock:
+            AutoRConsole.lock = False
+            rpy2.rinterface_lib.callbacks.consoleread = \
+                self.original_funcs["consoleread"]
+            rpy2.rinterface_lib.callbacks.consolewrite_print = \
+                self.original_funcs["consolewrite_print"]
+            rpy2.rinterface_lib.callbacks.consolewrite_warnerror = \
+                self.original_funcs["consolewrite_warnerror"]
+
+    def close(self):
+        """Remove the rpy2 monkeypatches"""
+        self.__exit__()
+
+    def consoleread(self, prompt):
+        """Read user input, returns "yes" by default"""
+        self.write_to_stream("consoleread", prompt + "YES")
+        return "yes"
+
+    def consolewrite_print(self, s):
+        self.write_to_stream("consolewrite_print", s)
+
+    def consolewrite_warnerror(self, s):
+        self.write_to_stream("consolewrite_warnerror", s)
+
+    def write_to_stream(self, topic, s):
+        prev_topic = self.stream[-1][0]
+        same_topic = prev_topic == topic
+        unfinished_line = self.stream[-1][1][-1] not in ["\n", "\r"]
+        if same_topic and unfinished_line:
+            # append to previous line
+            self.stream[-1][1] += s
         else:
-            if r_home[0].startswith("WARNING"):
-                r_home = r_home[1].strip()
-            else:
-                r_home = r_home[0].strip()
-    if r_home is None:
-        raise RNotFoundError(
-            "Cannot find R, please set the `R_HOME` environment variable "
-            "or use `set_r_path`.")
-
-    r_home = pathlib.Path(r_home)
-
-    if sys.platform == "win32" and "64 bit" in sys.version:
-        r_exec = r_home / "bin" / "x64" / "R.exe"
-    else:
-        r_exec = r_home / "bin" / "R"
-    if not r_exec.is_file():
-        raise RNotFoundError(
-            f"Expected R binary at '{r_exec}' does not exist!")
-    logger.info(f"R path: {r_exec}")
-    return r_exec
+            self.stream.append([topic, s])
+
+    def get_prints(self):
+        prints = []
+        for line in self.stream:
+            if line[0] == "consolewrite_print":
+                prints.append(line[1].strip())
+        return prints
+
+    def get_warnerrors(self):
+        warnerrors = []
+        for line in self.stream:
+            if line[0] == "consolewrite_warnerror":
+                warnerrors.append(line[1].strip())
+        return warnerrors
 
 
-def get_r_script_path():
-    """Return the path to the Rscript executable"""
-    return get_r_path().with_name("Rscript")
+def check_r():
+    """Make sure R is installed an R HOME is set"""
+    if not has_r():
+        raise RNotFoundError("Cannot find R, please set its path with the "
+                             + "`set_r_path` function.")
+
+
+def get_r_path():
+    """Get the path of the R executable/binary from rpy2"""
+    r_home = rpy2.situation.get_r_home()
+    return rpy2.situation.get_r_exec(r_home)
 
 
 def get_r_version():
-    """Return the full R version string"""
-    require_r()
-    cmd = ("R", "--version")
-    logger.debug(f"Looking for R version with: {cmd}")
-    tmp = sp.check_output(cmd, stderr=sp.STDOUT)
-    r_version = tmp.decode("ascii", "ignore").split(os.linesep)
-    if r_version[0].startswith("WARNING"):
-        r_version = r_version[1]
-    else:
-        r_version = r_version[0]
-    logger.info(f"R version found: {r_version}")
-    # get the actual version string
-    if r_version.startswith("R version "):
-        r_version = r_version.split(" ", 2)[2]
-    return r_version.strip()
+    check_r()
+    ver_string = rpy2.situation.r_version_from_subprocess().strip()
+    if ver_string:
+        # get the actual version string
+        if ver_string.startswith("R version "):
+            ver_string = ver_string.split(" ")[2]
+    return ver_string
 
 
 def has_lme4():
     """Return True if the lme4 package is installed"""
-    global _has_lme4
-    if _has_lme4:
-        return True
-    require_r()
-    for pkg in ["lme4", "statmod", "nloptr"]:
-        res = run_command(("R", "-q", "-e", f"system.file(package='{pkg}')"))
-        if not res.split("[1]")[1].count(pkg):
-            avail = False
-            break
-    else:
-        avail = _has_lme4 = True
-    return avail
+    check_r()
+    lme4_there = rpy2.robjects.packages.isinstalled("lme4")
+    statmod_there = rpy2.robjects.packages.isinstalled("statmod")
+    nloptr_there = rpy2.robjects.packages.isinstalled("nloptr")
+    return lme4_there and statmod_there and nloptr_there
 
 
 def has_r():
     """Return True if R is available"""
-    global _has_r
-    if _has_r:
-        return True
     try:
-        hasr = bool(get_r_path())
-    except RNotFoundError:
+        hasr = rpy2.situation.get_r_home() is not None
+    except RUnavailableError:
         hasr = False
-    if hasr:
-        _has_r = True
     return hasr
 
 
-def require_lme4():
+def import_lme4():
+    check_r()
+    if has_lme4():
+        lme4pkg = rpy2.robjects.packages.importr("lme4")
+    else:
+        raise ValueError(
+            "The R package 'lme4' is not installed, please install it via "
+            + "`dclab.lme4.rsetup.install_lme4()` or by executing "
+            + "in a shell: R -e " + '"install.packages(' + "'lme4', "
+            + "repos='http://cran.rstudio.org')" + '"')
+    return lme4pkg
+
+
+def install_lme4():
     """Install the lme4 package (if not already installed)
 
-    Besides ``lme4``, this also installs ``nloptr`` and ``statmod``.
     The packages are installed to the user data directory
-    given in :const:`lib_path` from the http://cran.rstudio.org mirror.
+    given in :const:`lib_path`.
     """
-    require_r()
+    check_r()
     if not has_lme4():
-        run_command((
-            "R", "-e", "install.packages(c('statmod','nloptr','lme4'),"
-                       "repos='http://cran.rstudio.org')"))
-
-
-def require_r():
-    """Make sure R is installed an R HOME is set"""
-    if not has_r():
-        raise RNotFoundError("Cannot find R, please set its path with the "
-                             "`set_r_path` function or set the `RHOME` "
-                             "environment variable.")
+        # import R's utility package
+        utils = rpy2.robjects.packages.importr('utils')
+        # select the first mirror in the list
+        utils.chooseCRANmirror(ind=1)
+        # install lme4 to user data directory (say yes to user dir install)
+        with AutoRConsole() as rc:
+            # install statmod and nloptr first
+            # (Doesn't R have package dependencies?!)
+            utils.install_packages(
+                rpy2.robjects.vectors.StrVector(["statmod", "nloptr", "lme4"]))
+        return rc
 
 
-def run_command(cmd):
-    """Run a command via subprocess"""
-    if hasattr(sp, "STARTUPINFO"):
+def set_r_path(r_path):
+    """Set the path of the R executable/binary for rpy2"""
+    if hasattr(sp, 'STARTUPINFO'):
         # On Windows, subprocess calls will pop up a command window by
         # default when run from Pyinstaller with the ``--noconsole``
         # option. Avoid this distraction.
@@ -149,26 +192,16 @@ def run_command(cmd):
         si = None
         env = None
 
-    # Convert paths to strings
-    cmd = [str(cc) for cc in cmd]
-
-    tmp = sp.check_output(cmd,
+    tmp = sp.check_output((r_path, 'RHOME'),
                           startupinfo=si,
                           env=env,
-                          stderr=sp.STDOUT,
                           text=True,
                           )
-    return tmp
-
-
-def set_r_path(r_path):
-    """Set the path of the R executable/binary"""
-    tmp = run_command((r_path, "RHOME"))
 
     r_home = tmp.split(os.linesep)
-    if r_home[0].startswith("WARNING"):
+    if r_home[0].startswith('WARNING'):
         res = r_home[1]
     else:
         res = r_home[0].strip()
     os.environ["R_HOME"] = res
-    os.environ["R_EXEC"] = str(pathlib.Path(r_path).resolve())
+    import_r_submodules()

dclab/lme4/wrapr.py

@@ -1,19 +1,18 @@
 """R lme4 wrapper"""
-import logging
 import numbers
-import pathlib
-import tempfile
+import warnings
 
-import importlib_resources
 import numpy as np
 
 from .. import definitions as dfn
 from ..rtdc_dataset.core import RTDCBase
 
+from .rlibs import rpy2
 from . import rsetup
 
 
-logger = logging.getLogger(__name__)
+class Lme4InstallWarning(UserWarning):
+    pass
 
 
 class Rlme4(object):
@@ -39,12 +38,19 @@ class Rlme4(object):
         #: list of [RTDCBase, column, repetition, chip_region]
         self.data = []
 
+        #: model function
+        self.r_func_model = "feature ~ group + (1 + group | repetition)"
+        #: null model function
+        self.r_func_nullmodel = "feature ~ (1 + group | repetition)"
+
         self.set_options(model=model, feature=feature)
 
         # Make sure that lme4 is available
         if not rsetup.has_lme4():
-            logger.info("Installing lme4, this may take a while!")
-            rsetup.require_lme4()
+            warnings.warn("Installing lme4, this may take a while!",
+                          Lme4InstallWarning)
+            rsetup.install_lme4()
+        rsetup.import_lme4()
 
     def add_dataset(self, ds, group, repetition):
         """Add a dataset to the analysis list
@@ -61,8 +67,8 @@ class Rlme4(object):
 
         Notes
         -----
-        - For each repetition, there must be a "treatment" (``1``) and a
-          "control" (``0``) group.
+        - For each repetition, there must be a "treatment" and a
+          "control" ``group``.
         - If you would like to perform a differential feature analysis,
           then you need to pass at least a reservoir and a channel
           dataset (with same parameters for `group` and `repetition`).
@@ -96,10 +102,10 @@ class Rlme4(object):
         The response variable is modeled using two linear mixed effect
         models:
 
-        - model: "feature ~ group + (1 + group | repetition)"
-          (random intercept + random slope model)
-        - the null model: "feature ~ (1 + group | repetition)"
-          (without the fixed effect introduced by the "treatment" group).
+        - model :const:`Rlme4.r_func_model` (random intercept +
+          random slope model)
+        - the null model :const:`Rlme4.r_func_nullmodel` (without
+          the fixed effect introduced by the "treatment" group).
 
         Both models are compared in R using "anova" (from the
         R-package "stats" :cite:`Everitt1992`) which performs a
@@ -127,16 +133,16 @@ class Rlme4(object):
         results: dict
             Dictionary with the results of the fitting process:
 
-            - "anova p-value": Anova likelihood ratio test (significance)
+            - "anova p-value": Anova likelyhood ratio test (significance)
             - "feature": name of the feature used for the analysis
               ``self.feature``
             - "fixed effects intercept": Mean of ``self.feature`` for all
               controls; In the case of the "glmer+loglink" model, the intercept
-              is already back transformed from log space.
+              is already backtransformed from log space.
             - "fixed effects treatment": The fixed effect size between the mean
               of the controls and the mean of the treatments relative to
               "fixed effects intercept"; In the case of the "glmer+loglink"
-              model, the fixed effect is already back transformed from log
+              model, the fixed effect is already backtransformed from log
               space.
             - "fixed effects repetitions": The effects (intercept and
               treatment) for each repetition. The first axis defines
@@ -153,10 +159,11 @@ class Rlme4(object):
             - "model": model name used for the analysis ``self.model``
             - "model converged": boolean indicating whether the model
               converged
-            - "r model summary": Summary of the model
-            - "r model coefficients": Model coefficient table
-            - "r script": the R script used
-            - "r output": full output of the R script
+            - "r anova": Anova model (exposed from R)
+            - "r model summary": Summary of the model (exposed from R)
+            - "r model coefficients": Model coefficient table (exposed from R)
+            - "r stderr": errors and warnings from R
+            - "r stdout": standard output from R
         """
         self.set_options(model=model, feature=feature)
         self.check_data()
@@ -175,38 +182,105 @@ class Rlme4(object):
                 groups.append(dd[1])
                 repetitions.append(dd[2])
 
-        # concatenate and populate arrays for R
-        features_c = np.concatenate(features)
-        groups_c = np.zeros(len(features_c), dtype=str)
-        repetitions_c = np.zeros(len(features_c), dtype=int)
-        pos = 0
-        for ii in range(len(features)):
-            size = len(features[ii])
-            groups_c[pos:pos+size] = groups[ii][0]
-            repetitions_c[pos:pos+size] = repetitions[ii]
-            pos += size
-
-        # Run R with the given template script
-        rscript = importlib_resources.read_text("dclab.lme4",
-                                                "lme4_template.R")
-        _, script_path = tempfile.mkstemp(prefix="dclab_lme4_", suffix=".R",
-                                          text=True)
-        script_path = pathlib.Path(script_path)
-        rscript = rscript.replace("<MODEL_NAME>", self.model)
-        rscript = rscript.replace("<FEATURES>", arr2str(features_c))
-        rscript = rscript.replace("<REPETITIONS>", arr2str(repetitions_c))
-        rscript = rscript.replace("<GROUPS>", arr2str(groups_c))
-        script_path.write_text(rscript, encoding="utf-8")
-
-        result = rsetup.run_command((rsetup.get_r_script_path(), script_path))
-
-        ret_dict = self.parse_result(result)
-        ret_dict["is differential"] = self.is_differential()
-        ret_dict["feature"] = self.feature
-        ret_dict["r script"] = rscript
-        ret_dict["r output"] = result
-        assert ret_dict["model"] == self.model
-
+        # Fire up R
+        with rsetup.AutoRConsole() as ac:
+            r = rpy2.robjects.r
+
+            # Load lme4
+            rpy2.robjects.packages.importr("lme4")
+
+            # Concatenate huge arrays for R
+            r_features = rpy2.robjects.FloatVector(np.concatenate(features))
+            _groups = []
+            _repets = []
+            for ii in range(len(features)):
+                _groups.append(np.repeat(groups[ii], len(features[ii])))
+                _repets.append(np.repeat(repetitions[ii], len(features[ii])))
+            r_groups = rpy2.robjects.StrVector(np.concatenate(_groups))
+            r_repetitions = rpy2.robjects.IntVector(np.concatenate(_repets))
+
+            # Register groups and repetitions
+            rpy2.robjects.globalenv["feature"] = r_features
+            rpy2.robjects.globalenv["group"] = r_groups
+            rpy2.robjects.globalenv["repetition"] = r_repetitions
+
+            # Create a dataframe which contains all the data
+            r_data = r["data.frame"](r_features, r_groups, r_repetitions)
+
+            # Random intercept and random slope model
+            if self.model == 'glmer+loglink':
+                r_model = r["glmer"](self.r_func_model, r_data,
+                                     family=r["Gamma"](link='log'))
+                r_nullmodel = r["glmer"](self.r_func_nullmodel, r_data,
+                                         family=r["Gamma"](link='log'))
+            else:  # lmer
+                r_model = r["lmer"](self.r_func_model, r_data)
+                r_nullmodel = r["lmer"](self.r_func_nullmodel, r_data)
+
+            # Anova analysis (increase verbosity by making models global)
+            # Using anova is a very conservative way of determining
+            # p values.
+            rpy2.robjects.globalenv["Model"] = r_model
+            rpy2.robjects.globalenv["NullModel"] = r_nullmodel
+            r_anova = r("anova(Model, NullModel)")
+            try:
+                pvalue = r_anova.rx2["Pr(>Chisq)"][1]
+            except ValueError:  # rpy2 2.9.4
+                pvalue = r_anova[7][1]
+            r_model_summary = r["summary"](r_model)
+            r_model_coefficients = r["coef"](r_model)
+            try:
+                fe_reps = np.array(r_model_coefficients.rx2["repetition"])
+            except ValueError:  # rpy2 2.9.4
+                fe_reps = np.concatenate((
+                    np.array(r_model_coefficients[0][0]).reshape(1, -1),
+                    np.array(r_model_coefficients[0][1]).reshape(1, -1)),
+                    axis=0)
+
+            r_effects = r["data.frame"](r["coef"](r_model_summary))
+            try:
+                fe_icept = r_effects.rx2["Estimate"][0]
+                fe_treat = r_effects.rx2["Estimate"][1]
+            except ValueError:  # rpy2 2.9.4
+                fe_icept = r_effects[0][0]
+                fe_treat = r_effects[0][1]
+            if self.model == "glmer+loglink":
+                # transform back from log
+                fe_treat = np.exp(fe_icept + fe_treat) - np.exp(fe_icept)
+                fe_icept = np.exp(fe_icept)
+                fe_reps[:, 1] = np.exp(fe_reps[:, 0] + fe_reps[:, 1]) \
+                    - np.exp(fe_reps[:, 0])
+                fe_reps[:, 0] = np.exp(fe_reps[:, 0])
+
+            # convergence
+            try:
+                lme4l = r_model_summary.rx2["optinfo"].rx2["conv"].rx2["lme4"]
+            except ValueError:  # rpy2 2.9.4
+                lme4l = r_model_summary[17][3][1]
+
+            if lme4l and "code" in lme4l.names:
+                try:
+                    conv_code = lme4l.rx2["code"]
+                except ValueError:  # rpy2 2.9.4
+                    conv_code = lme4l[0]
+            else:
+                conv_code = 0
+
+        ret_dict = {
+            "anova p-value": pvalue,
+            "feature": self.feature,
+            "fixed effects intercept": fe_icept,
+            "fixed effects treatment": fe_treat,  # aka "fixed effect"
+            "fixed effects repetitions": fe_reps,
+            "is differential": self.is_differential(),
+            "model": self.model,
+            "model converged": conv_code == 0,
+            "r anova": r_anova,
+            "r model summary": r_model_summary,
+            "r model coefficients": r_model_coefficients,
+            "r stderr": ac.get_warnerrors(),
+            "r stdout": ac.get_prints(),
+        }
         return ret_dict
 
     def get_differential_dataset(self):
@@ -214,7 +288,7 @@ class Rlme4(object):
 
         The most famous use case is differential deformation. The idea
         is that you cannot tell what the difference in deformation
-        from channel to reservoir, because you never measure the
+        from channel to reservoir is, because you never measure the
         same object in the reservoir and the channel. You usually just
         have two distributions. Comparing distributions is possible
         via bootstrapping. And then, instead of running the lme4
@@ -288,34 +362,6 @@ class Rlme4(object):
         else:
             return False
 
-    def parse_result(self, result):
-        resd = result.split("OUTPUT")
-        ret_dict = {}
-        for item in resd:
-            string = item.split("#*#")[0]
-            key, value = string.split(":", 1)
-            key = key.strip()
-            value = value.strip().replace("\n\n", "\n")
-
-            if key == "fixed effects repetitions":
-                rows = value.split("\n")[1:]
-                reps = []
-                for row in rows:
-                    reps.append([float(vv) for vv in row.split()[1:]])
-                value = np.array(reps).transpose()
-            elif key == "model converged":
-                value = value == "TRUE"
-            elif value == "NA":
-                value = np.nan
-            else:
-                try:
-                    value = float(value)
-                except ValueError:
-                    pass
-
-            ret_dict[key] = value
-        return ret_dict
-
     def set_options(self, model=None, feature=None):
         """Set analysis options"""
         if model is not None:
@@ -326,16 +372,6 @@ class Rlme4(object):
             self.feature = feature
 
 
-def arr2str(a):
-    """Convert an array to a string"""
-    if isinstance(a.dtype.type, np.integer):
-        return ",".join(str(dd) for dd in a.tolist())
-    elif a.dtype.type == np.str_:
-        return ",".join(f"'{dd}'" for dd in a.tolist())
-    else:
-        return ",".join(f"{dd:.16g}" for dd in a.tolist())
-
-
 def bootstrapped_median_distributions(a, b, bs_iter=1000, rs=117):
     """Compute the bootstrapped distributions for two arrays.
 
@@ -345,7 +381,7 @@ def bootstrapped_median_distributions(a, b, bs_iter=1000, rs=117):
         Input data
     bs_iter: int
         Number of bootstrapping iterations to perform
-        (output size).
+        (outtput size).
     rs: int
         Random state seed for random number generator
 
@@ -360,7 +396,7 @@ def bootstrapped_median_distributions(a, b, bs_iter=1000, rs=117):
 
     Notes
     -----
-    From a programmatic point of view, it would have been better
+    From a programmatical point of view, it would have been better
     to implement this method for just one input array (because of
     redundant code). However, due to historical reasons (testing
     and comparability to Shape-Out 1), bootstrapping is done

{dclab-0.61.2.dist-info → dclab-2.18.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dclab
-Version: 0.61.2
+Version: 2.18.0
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -23,7 +23,7 @@ Requires-Dist: importlib-resources>=6.0
 Requires-Dist: numpy<3,>=1.21
 Requires-Dist: scipy<2,>=1.10.0
 Provides-Extra: all
-Requires-Dist: dclab[dcor,export,http,s3,tdms]; extra == "all"
+Requires-Dist: dclab[dcor,export,http,lme4,s3,tdms]; extra == "all"
 Provides-Extra: dcor
 Requires-Dist: requests<3,>=2.31.0; extra == "dcor"
 Provides-Extra: export
@@ -31,6 +31,8 @@ Requires-Dist: fcswrite>=0.5.0; extra == "export"
 Requires-Dist: imageio[ffmpeg]; extra == "export"
 Provides-Extra: http
 Requires-Dist: requests<3,>=2.31.0; extra == "http"
+Provides-Extra: lme4
+Requires-Dist: rpy2>=2.9.4; extra == "lme4"
 Provides-Extra: s3
 Requires-Dist: boto3>=1.34.31; extra == "s3"
 Provides-Extra: tdms

{dclab-0.61.2.dist-info → dclab-2.18.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dclab/kde_methods.py,sha256=awlqkj819VRJzArG6Fx6myDa6qGpLoZ8S9IJ1J1YECA,9206
-dclab/_version.py,sha256=jCcQYeQUVxJlyYParhmvl3GfmgLyh6_duOusvZWWoSs,413
+dclab/_version.py,sha256=QyMxKDzFsqXDA7bVrQozJznaASbsECKe57aBBNAljsA,413
 dclab/util.py,sha256=FcRsWcyKkKkfX2nZCShjJ5wPccr7U7NHsIDd_WEws6w,5141
 dclab/downsampling.pyx,sha256=OK7zbgGLl5gVyoU8ZBHo9EWwb8C9ChavmLNEvQvC9T0,7258
 dclab/__init__.py,sha256=1mskJAUo8HbvDhJRJFbcFB6HccFeqoRUCEHLuS64t_g,812
@@ -9,7 +9,7 @@ dclab/http_utils.py,sha256=TGqgiaUV-Z6Wiae__Yze-uHRWX57846ZnlV10ajzv2E,10562
 dclab/polygon_filter.py,sha256=qexmo-rXe06CUPZhN6EMJy4y4B5gXZeqejdvIB2arOE,13480
 dclab/statistics.py,sha256=tJDqPlY_Jw2Hhl-s7ugMBSZAxcRuPu4LQuBAZBXz7t8,6355
 dclab/kde_contours.py,sha256=5K7PzZz-FtSGvW4IR2tLpbEKKqCSsSTQPsupu7zIsPg,6745
-dclab/downsampling.cpython-312-darwin.so,sha256=zM2vp6uhNKEm3683X3C1agPoTF4eVwi0SZy2ol1aVL8,259136
+dclab/downsampling.cpython-312-darwin.so,sha256=8Mu_2whAnEOv_GSzu97Po69Nt1ccewisS-Iqp5C4iO0,259136
 dclab/isoelastics/iso_LE-2D-FEM-19-volume-deform.txt,sha256=vTcazOlOXo3BQ0NQtGB_IdHKA0neOLXZ_d3JuMU--RE,83358
 dclab/isoelastics/iso_HE-3D-FEM-22-area_um-deform.txt,sha256=IjULG1KO-hClaYPoJJnwPbF2TkS9i9jxF1tbhhQTClY,71350
 dclab/isoelastics/iso_HE-2D-FEM-22-volume-deform.txt,sha256=aLOWxev_hvyP2yWquomcogn3Ze4SBfvqz-qmTOOibNw,97512
@@ -34,10 +34,10 @@ dclab/features/emodulus/scale_linear.py,sha256=5tZfOpG4QSSEMlwSoA_3XRQuY0tiZ3CSm
 dclab/features/emodulus/lut_HE-3D-FEM-22.txt,sha256=1Sg4ys0ykT7Q_jFM28XS0uJ85nZd_ha7BDXdN2rsPIA,47248
 dclab/features/emodulus/load.py,sha256=Q9rII-7Om-f0m183VNeJUxNno7at1reigKtbzdrWgjE,8557
 dclab/features/emodulus/viscosity.py,sha256=aBm1pQa9cFAOBnO07UWMj87bw6fQpmW1IlWWD0MEtgw,9438
-dclab/lme4/lme4_template.R,sha256=CEXQIquvYCla9dCvRYgiBemI6fiVgAKnJTetJA2LAtk,2570
-dclab/lme4/__init__.py,sha256=dj9ZR9WrxIXi704kOVsT2FCRkMhQMmp106mDXMP-aoI,256
-dclab/lme4/rsetup.py,sha256=emTCJavJAjKLy54B2qjQEw4_h3FXD6oUp0BRieon7EY,5036
-dclab/lme4/wrapr.py,sha256=rdIc2hS8GhgdU9WFA6pLzohJGlBga-mkm60qqqk6VO4,15017
+dclab/lme4/__init__.py,sha256=Y_oqYEqNnHCjxfdzkoP0ZXPCQx6XSKqwt1pAgdGF2dA,235
+dclab/lme4/rsetup.py,sha256=siVWXt0niXAqD20d2quOjVzaZc4-zF0i81G_HRZE6q0,6951
+dclab/lme4/wrapr.py,sha256=MzJDUZ7L_5xDWVtl5vrDgTJoum2vrxfP0dYwPMcHIPY,16977
+dclab/lme4/rlibs.py,sha256=3oHSrSykeQB9ldvrV4Ite9EffIURUwCo3bn4uBpVeSk,2539
 dclab/cli/task_tdms2rtdc.py,sha256=u0L1Fq9rXIeQG9b72SuUIh_qYC6fG2xXxht9_rcdCao,8283
 dclab/cli/__init__.py,sha256=84YzzV6aE_NY-o7wvqgvUoxBLvIOEXpSUbkVcGRyzQ0,483
 dclab/cli/task_condense.py,sha256=uNZzm04VuQOXJi6uXPmaLdQCk0g8ONueiO4p67yJv0k,8546
@@ -61,17 +61,17 @@ dclab/external/statsmodels/nonparametric/kernel_density.py,sha256=3UyzLuZS68TkNT
 dclab/external/statsmodels/nonparametric/__init__.py,sha256=-GEWgwsF27ems5WTBvR1zo4SWJ0pRTWyHVagnIYer3g,43
 dclab/external/statsmodels/nonparametric/kernels.py,sha256=fuy4kStFz2ZA9pqgfUb4cly-YBpXLu4TJ9-ZujayuIw,1075
 dclab/external/skimage/measure.py,sha256=y1idCqD9TUxp3-QnOiWR_d674OKaeqBJ4MN2-gVP6ro,247
-dclab/external/skimage/_pnpoly.cpython-312-darwin.so,sha256=S8m8J1d4qVSQ8ht4_vF5T03jrW5ix7bHrDiYatwJn44,222776
+dclab/external/skimage/_pnpoly.cpython-312-darwin.so,sha256=QevOsMePM73uwzfnRPAVcgnFs_thMF1iN9R2N3Ksy64,222776
 dclab/external/skimage/LICENSE,sha256=ivsSBvn3c0R9mOctWRRdza7C7wdZSRYgCVxlVqUdlB8,1452
 dclab/external/skimage/pnpoly.py,sha256=r8hFNiTz5XlUoNZjosqA0iyv1FPn0l7ewbplgFgkdaw,1347
 dclab/external/skimage/_find_contours.py,sha256=16v5eeTZBmevG8SSuXtJ6yUpVPhwfSmtc8pDD0nuuOU,9340
 dclab/external/skimage/__init__.py,sha256=-B2QUKHAFzQuBWuuKvPDC5JIl0Zb-x3OGmbwPaE9VwQ,72
-dclab/external/skimage/_find_contours_cy.cpython-312-darwin.so,sha256=5fTIJOXpu8iFA5GDMC4WM_7dhQeRyx3QLy53DSK73M8,204520
+dclab/external/skimage/_find_contours_cy.cpython-312-darwin.so,sha256=2kub7ht3_Dx6HDfdLa5-RTkeeXG_-nai4ooY_vtiAfQ,204520
 dclab/external/skimage/_pnpoly.pyx,sha256=Qdn6xPazDschBqbr46DzB75MB2MnqvdnoTSBMK7kUGE,2504
 dclab/external/skimage/_find_contours_cy.pyx,sha256=pZJOBhMHzYEMkcz4WQVyjn7jDNrdjCfet47FU1hRAxk,7161
 dclab/external/skimage/_shared/geometry.pxd,sha256=kRsu9ifv_rL3kbRIgSLf86p0hn2oTMp6s013lZ9bBZM,346
 dclab/external/skimage/_shared/__init__.py,sha256=2sHZwTtJSlMTa3Q2YSvQW7jrPLMUSqDJQa-ROe5zfcw,37
-dclab/external/skimage/_shared/geometry.cpython-312-darwin.so,sha256=wia_nS-MBJy17QrJIbwkJ2x2SA_zp6oT0SwanKzDDKo,39216
+dclab/external/skimage/_shared/geometry.cpython-312-darwin.so,sha256=CdXgHOQZTU-tUUsnE7TALAj_5g3jK9KVvbXEfVqk8RI,39216
 dclab/external/skimage/_shared/geometry.pyx,sha256=miCHUh6mBDbRRIoaF_0xAER1MRzsCAzFdlYQZhV7RmE,1667
 dclab/definitions/feat_logic.py,sha256=SXsSlAusgtE3uXcPu84dQwYZ07zxmV37DmPednA3_dM,5823
 dclab/definitions/meta_parse.py,sha256=YdaTdM8DAMsIFn5ITH9OHYGTXeAOBGWtx22oVjxXcWk,2393
@@ -130,9 +130,9 @@ dclab/rtdc_dataset/fmt_tdms/event_image.py,sha256=-jp7Z-N91e4ieumYQ1huMicj7PMJqw
 dclab/rtdc_dataset/fmt_tdms/event_trace.py,sha256=Vkym0QKSw2mq1XZl5n8wDkgHXmaZwQGiMAV5AuRSJkE,5215
 dclab/rtdc_dataset/fmt_tdms/exc.py,sha256=WzrMqnyrzp8gsT8Pf7JKqGGv43ewx7d_qgtirURppRI,813
 dclab/rtdc_dataset/fmt_tdms/event_contour.py,sha256=kjo0wJx9F0gmmOOyR0NoLw6VEtSl3h63WXXkcbfnoS8,9627
-dclab-0.61.2.dist-info/RECORD,,
-dclab-0.61.2.dist-info/LICENSE,sha256=1mLfjOTOaeiMSGPJiF5hHnMQfKX88QVeZpCCXwJGj3k,18131
-dclab-0.61.2.dist-info/WHEEL,sha256=Gslco0u-UdyhQslriEMCnIh5wENI6Dd1FlwyhvZM_HQ,110
-dclab-0.61.2.dist-info/entry_points.txt,sha256=eOpjgznu-eW-9utUpLU-77O5098YyUEgGF3ksGMdtec,273
-dclab-0.61.2.dist-info/top_level.txt,sha256=irvwZMgs1edY1Zj60ZFk7Almb9Zhk4k6E6aC4YPFnnM,6
-dclab-0.61.2.dist-info/METADATA,sha256=QibArMbIi19J5t670ST0BhxxT94ruSFTQ5J2nif3F18,4754
+dclab-2.18.0.dist-info/RECORD,,
+dclab-2.18.0.dist-info/LICENSE,sha256=1mLfjOTOaeiMSGPJiF5hHnMQfKX88QVeZpCCXwJGj3k,18131
+dclab-2.18.0.dist-info/WHEEL,sha256=Gslco0u-UdyhQslriEMCnIh5wENI6Dd1FlwyhvZM_HQ,110
+dclab-2.18.0.dist-info/entry_points.txt,sha256=eOpjgznu-eW-9utUpLU-77O5098YyUEgGF3ksGMdtec,273
+dclab-2.18.0.dist-info/top_level.txt,sha256=irvwZMgs1edY1Zj60ZFk7Almb9Zhk4k6E6aC4YPFnnM,6
+dclab-2.18.0.dist-info/METADATA,sha256=WTNyL58XdjHtE11bxXQWGntyi981Fv5pWGaxXshj1Iw,4824

dclab/lme4/lme4_template.R

@@ -1,94 +0,0 @@
-require(stats);
-require(lme4);
-
-model_name <- "<MODEL_NAME>"
-cat("OUTPUT model:", model_name, "#*#\n")
-
-func_model <- "feature ~ group + (1 + group | repetition)"
-func_nullmodel <- "feature ~ (1 + group | repetition)"
-
-# These are the feature, group, and repetition arrays that are set by dclab
-# via templates.
-feature <- c(<FEATURES>)
-group <- c(<GROUPS>)
-repetition <- c(<REPETITIONS>)
-
-data <- data.frame(feature, group, repetition)
-
-if (model_name == "glmer+loglink") {
-    Model <- glmer(func_model, data, family=Gamma(link='log'))
-    NullModel <- glmer(func_nullmodel, data, family=Gamma(link='log'))
-} else if (model_name == "lmer") {
-    Model <- lmer(func_model, data)
-    NullModel <- lmer(func_nullmodel, data)
-} else {
-    stop("Invalid model_name:", model_name)
-}
-
-# Anova analysis (increase verbosity by making models global)
-# Using anova is a very conservative way of determining
-# p values.
-res_anova <- anova(Model, NullModel)
-cat("OUTPUT r anova: ")
-res_anova
-cat("#*#\n")
-
-pvalue <- res_anova$"Pr(>Chisq)"[2]
-cat("OUTPUT anova p-value:", pvalue, "#*#\n")
-
-model_summary <- summary(Model)
-cat("OUTPUT r model summary:")
-model_summary
-cat("#*#\n")
-
-model_coefficients <- coef(Model)
-cat("OUTPUT r model coefficients:")
-model_coefficients
-cat("#*#\n")
-
-fe_reps <- model_coefficients$repetition
-
-effects <- data.frame(coef(model_summary))
-
-fe_icept <- effects$Estimate[1]
-
-fe_treat <- effects$Estimate[2]
-
-if (model_name == "glmer+loglink") {
-    # transform back from log
-    fe_treat <- exp(fe_icept + fe_treat) - exp(fe_icept)
-    fe_icept <- exp(fe_icept)
-    fe_reps[, 2] = exp(fe_reps[, 1] + fe_reps[, 2]) - exp(fe_reps[, 1])
-    fe_reps[, 1] = exp(fe_reps[, 1])
-}
-
-cat("OUTPUT fixed effects intercept:", fe_icept, "#*#\n")
-cat("OUTPUT fixed effects treatment:", fe_treat, "#*#\n")
-cat("OUTPUT fixed effects repetitions:")
-fe_reps
-cat("#*#\n")
-
-# convergence
-
-# convergence warnings in lme4
-is_warning_generated <- function(m) {
-  df <- summary(m)
-  !is.null(df$optinfo$conv$lme4$messages) &&
-           grepl('failed to converge', df$optinfo$conv$lme4$messages)
-}
-lme4_not_converged <- is_warning_generated(Model)
-
-# convergence code by the optimizer
-lme4l <- model_summary$optinfo$conv$lme4
-if (length(lme4l) == 0) {
-    # the optimizer probably does not know
-    conv_code <- 0
-} else if (is.null(lme4l$code)) {
-    # NULL means 0
-    conv_code <- 0
-} else {
-    conv_code <- lme4l$code
-}
-
-cat("OUTPUT model converged:", (conv_code == 0) && !lme4_not_converged, "#*#\n")
-cat("OUTPUT lme4 messages:", lme4l$optinfo$conv$lme4$messages, "#*#\n")