dclab 0.62.10__cp313-cp313-macosx_10_13_x86_64.whl → 2.18.0__cp313-cp313-macosx_10_13_x86_64.whl

dclab/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.62.10'
-__version_tuple__ = version_tuple = (0, 62, 10)
+__version__ = version = '2.18.0'
+__version_tuple__ = version_tuple = (2, 18, 0)

dclab/definitions/meta_const.py

@@ -161,17 +161,7 @@ CFG_METADATA = {
         ["subtract mean", fbool, "Subtract mean before processing"],
         # pipeline_kws
         ["filter name", str, "Fourier filter used"],
-        # qpretrieve defines the keyword argument `filter_size_interpretation`
-        # for determining the filter size in Fourier space. In DC, we
-        # need a well-defined value for the filter size. The most logical
-        # choice is to interpret the filter size as "frequency index", which
-        # is independent of the image shape and yields a good approximation
-        # of the actual resolution one can expect. The default value
-        # ("sideband distance") is a good choice for general QPI analysis,
-        # but there is no meaningful information one could extract from it
-        # by just looking at the number. Thus, the "filter size" that we
-        # see here corresponds to a filter size set in qpretrieve where
-        # `filter_size_interpretation="frequency index"`.
+        # Corresponds to `filter_size_interpretation="frequency index"`
         ["filter size", float, "Fourier filter size [1/pix]"],
         ["scale to filter", fboolorfloat, "Scale QPI data to filter size"],
         # x, y coordinates, don't set if you wish None to be the default

dclab/http_utils.py

@@ -27,14 +27,6 @@ REGEXP_HTTP_URL = re.compile(
 )
 
 
-class ETagNotInResponseHeaderWarning(UserWarning):
-    """Used for cases where the requests.Response does not contain an ETag"""
-
-
-class ConnectionTimeoutWarning(UserWarning):
-    """Used when a connection fails or times out"""
-
-
 class HTTPFile(io.IOBase):
     def __init__(self, url, chunk_size=2**18, keep_chunks=200):
         """Chunk-cached access to a URL supporting range requests
@@ -86,8 +78,7 @@ class HTTPFile(io.IOBase):
             etag = resp.headers.get("etag", "").strip("'").strip('"')
             if len(etag) < 5:
                 etag = None
-                warnings.warn(f"Got empty ETag header for {self.url}",
-                              ETagNotInResponseHeaderWarning)
+                warnings.warn(f"Got empty ETag header for {self.url}")
             self._etag = etag
 
     @property
@@ -247,8 +238,7 @@ class ResoluteRequestsSession(requests.Session):
                     requests.exceptions.ConnectTimeout,
                     requests.urllib3.exceptions.ConnectionError,
                     requests.urllib3.exceptions.ReadTimeoutError) as e:
-                warnings.warn(f"Encountered {e} for {args} {kwargs}",
-                              ConnectionTimeoutWarning)
+                warnings.warn(f"Encountered {e} for {args} {kwargs}")
                 continue
             else:
                 break

dclab/lme4/__init__.py

@@ -1,5 +1,5 @@
 """A wrapper around R with the lme4 package"""
-from . import rsetup, wrapr  # noqa: F401
-from .wrapr import Rlme4, bootstrapped_median_distributions  # noqa: F401
-from .rsetup import (  # noqa: F401
-    set_r_lib_path, get_r_path, get_r_version, require_lme4, set_r_path)
+# flake8: noqa: F401
+from . import rlibs, rsetup, wrapr
+from .wrapr import Rlme4, bootstrapped_median_distributions
+from .rsetup import get_r_path, get_r_version, install_lme4, set_r_path

dclab/lme4/rlibs.py

@@ -0,0 +1,93 @@
+import importlib
+import os
+import warnings
+
+from ..external.packaging import parse as parse_version
+
+#: Minimum R version
+#: This is actually a dependency for rpy2, because the API changed then
+#: (ffi.error: symbol 'R_tryCatchError' not found in library).
+R_MIN_VERSION = "3.6.0"
+
+#: Minimum rpy2 version
+RPY2_MIN_VERSION = "2.9.4"
+
+R_SUBMODULES = [
+    "rpy2.robjects.packages",
+    "rpy2.situation",
+    "rpy2.robjects.vectors",
+]
+
+R_SUBMODULES_3 = [
+    "rpy2.rinterface_lib.callbacks",
+]
+
+
+class RPY2UnavailableError(BaseException):
+    pass
+
+
+class RPY2ImportError(RPY2UnavailableError):
+    pass
+
+
+class RPY2OutdatedError(RPY2UnavailableError):
+    pass
+
+
+class RUnavailableError(BaseException):
+    pass
+
+
+class ROutdatedError(RUnavailableError):
+    pass
+
+
+class MockRPackage:
+    def __init__(self, exception):
+        self.exception = exception
+
+    def __getattr__(self, item):
+        raise self.exception
+
+
+def import_r_submodules():
+    importlib.import_module("rpy2.situation")
+    r_home = rpy2.situation.get_r_home()
+    if r_home is not None:
+        if os.environ.get("R_HOME", None) is None:
+            # set R_HOME globally (https://github.com/rpy2/rpy2/issues/796)
+            os.environ["R_HOME"] = r_home
+        if rpy2_is_version_3:
+            mods = R_SUBMODULES + R_SUBMODULES_3
+        else:
+            mods = R_SUBMODULES
+        try:
+            for sm in mods:
+                importlib.import_module(sm)
+        except rpy2.rinterface_lib.openrlib.ffi.error as exc:
+            # This error happens when the installed R version is too old:
+            # "ffi.error: symbol 'R_tryCatchError' not found in library"
+            raise ROutdatedError(
+                f"Encountered '{exc.__class__.__name__}: {exc}'. "
+                f"Please make sure you have 'R>={R_MIN_VERSION}'!")
+
+
+try:
+    rpy2 = importlib.import_module("rpy2")
+    if parse_version(rpy2.__version__) < parse_version(RPY2_MIN_VERSION):
+        raise RPY2OutdatedError(f"Please install 'rpy2>={RPY2_MIN_VERSION}'!")
+except ImportError:
+    rpy2 = MockRPackage(
+        RPY2ImportError(f"Please install 'rpy2>={RPY2_MIN_VERSION}'!"))
+    rpy2_is_version_3 = False
+except BaseException as e:
+    rpy2 = MockRPackage(e)
+    rpy2_is_version_3 = False
+else:
+    rpy2_is_version_3 = parse_version(rpy2.__version__) >= parse_version("3.0")
+    try:
+        import_r_submodules()
+    except RUnavailableError as e:
+        warnings.warn("There is an issue with the linked R version: "
+                      + f"{e.__class__.__name__}: {e}")

dclab/lme4/rsetup.py

@@ -1,156 +1,185 @@
 import logging
 import os
-import pathlib
-import shutil
 import subprocess as sp
 
-logger = logging.getLogger(__name__)
+from .rlibs import (
+    RUnavailableError, rpy2, rpy2_is_version_3, import_r_submodules)
 
-_has_lme4 = None
-_has_r = None
-
-
-class CommandFailedError(BaseException):
-    """Used when `run_command` encounters an error"""
-    pass
+# Disable rpy2 logger because of unnecessary prints to stdout
+logging.getLogger("rpy2.rinterface_lib.callbacks").disabled = True
 
 
 class RNotFoundError(BaseException):
     pass
 
 
-def get_r_path():
-    """Return the path of the R executable"""
-    # Maybe the user set the executable already?
-    r_exec = os.environ.get("R_EXEC")
-    if r_exec is not None:
-        r_exec = pathlib.Path(r_exec)
-        if r_exec.is_file():
-            return r_exec
-
-    # Try to get the executable using which
-    r_exec = shutil.which("R")
-    if r_exec is not None:
-        r_exec = pathlib.Path(r_exec)
-        return r_exec
-
-    # Try to determine the path to the executable from R_HOME
-    r_home = os.environ.get("R_HOME")
-    if r_home and not pathlib.Path(r_home).is_dir():
-        logger.warning(f"R_HOME Directory does not exist: {r_home}")
-        r_home = None
-
-    if r_home is None:
-        raise RNotFoundError(
-            "Cannot find R, please set the `R_HOME` environment variable "
-            "or use `set_r_path`.")
-
-    r_home = pathlib.Path(r_home)
-
-    # search for the R executable
-    for rr in [
-        r_home / "bin" / "R",
-        r_home / "bin" / "x64" / "R",
-    ]:
-        if rr.is_file():
-            return rr
-        rr_win = rr.with_name("R.exe")
-        if rr_win.is_file():
-            return rr_win
-    else:
-        raise RNotFoundError(
-            f"Could not find R binary in '{r_home}'")
+class AutoRConsole(object):
+    """Helper class for catching R console output"""
+    lock = False
+    perform_lock = rpy2_is_version_3
+
+    def __init__(self):
+        """
+        By default, this console always returns "yes" when asked a
+        question. If you need something different, you can subclass
+        and override `consoleread` fucntion. The console stream is
+        recorded in `self.stream`.
+        """
+        self.stream = [["init", "Starting RConsole class\n"]]
+        if AutoRConsole.perform_lock:
+            if AutoRConsole.lock:
+                raise ValueError("Only one RConsole instance allowed!")
+            AutoRConsole.lock = True
+            self.original_funcs = {
+                "consoleread": rpy2.rinterface_lib.callbacks.consoleread,
+                "consolewrite_print":
+                    rpy2.rinterface_lib.callbacks.consolewrite_print,
+                "consolewrite_warnerror":
+                    rpy2.rinterface_lib.callbacks.consolewrite_warnerror,
+            }
+            rpy2.rinterface_lib.callbacks.consoleread = self.consoleread
+            rpy2.rinterface_lib.callbacks.consolewrite_print = \
+                self.consolewrite_print
+            rpy2.rinterface_lib.callbacks.showmessage = \
+                self.consolewrite_print
+
+            rpy2.rinterface_lib.callbacks.consolewrite_warnerror = \
+                self.consolewrite_warnerror
+        # Set locale (to get always English messages)
+        rpy2.robjects.r('Sys.setlocale("LC_MESSAGES", "C")')
+        rpy2.robjects.r('Sys.setlocale("LC_CTYPE", "C")')
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        if AutoRConsole.perform_lock:
+            AutoRConsole.lock = False
+            rpy2.rinterface_lib.callbacks.consoleread = \
+                self.original_funcs["consoleread"]
+            rpy2.rinterface_lib.callbacks.consolewrite_print = \
+                self.original_funcs["consolewrite_print"]
+            rpy2.rinterface_lib.callbacks.consolewrite_warnerror = \
+                self.original_funcs["consolewrite_warnerror"]
+
+    def close(self):
+        """Remove the rpy2 monkeypatches"""
+        self.__exit__()
+
+    def consoleread(self, prompt):
+        """Read user input, returns "yes" by default"""
+        self.write_to_stream("consoleread", prompt + "YES")
+        return "yes"
+
+    def consolewrite_print(self, s):
+        self.write_to_stream("consolewrite_print", s)
+
+    def consolewrite_warnerror(self, s):
+        self.write_to_stream("consolewrite_warnerror", s)
+
+    def write_to_stream(self, topic, s):
+        prev_topic = self.stream[-1][0]
+        same_topic = prev_topic == topic
+        unfinished_line = self.stream[-1][1][-1] not in ["\n", "\r"]
+        if same_topic and unfinished_line:
+            # append to previous line
+            self.stream[-1][1] += s
+        else:
+            self.stream.append([topic, s])
+
+    def get_prints(self):
+        prints = []
+        for line in self.stream:
+            if line[0] == "consolewrite_print":
+                prints.append(line[1].strip())
+        return prints
+
+    def get_warnerrors(self):
+        warnerrors = []
+        for line in self.stream:
+            if line[0] == "consolewrite_warnerror":
+                warnerrors.append(line[1].strip())
+        return warnerrors
+
+
+def check_r():
+    """Make sure R is installed an R HOME is set"""
+    if not has_r():
+        raise RNotFoundError("Cannot find R, please set its path with the "
+                             + "`set_r_path` function.")
 
 
-def get_r_script_path():
-    """Return the path to the Rscript executable"""
-    return get_r_path().with_name("Rscript")
+def get_r_path():
+    """Get the path of the R executable/binary from rpy2"""
+    r_home = rpy2.situation.get_r_home()
+    return rpy2.situation.get_r_exec(r_home)
 
 
 def get_r_version():
-    """Return the full R version string"""
-    require_r()
-    cmd = (str(get_r_path()), "--version")
-    logger.debug(f"Looking for R version with: {' '.join(cmd)}")
-    r_version = run_command(
-        cmd,
-        env={"R_LIBS_USER": os.environ.get("R_LIBS_USER", "")},
-    )
-    r_version = r_version.split(os.linesep)
-    if r_version[0].startswith("WARNING"):
-        r_version = r_version[1]
-    else:
-        r_version = r_version[0]
-    logger.info(f"R version found: {r_version}")
-    # get the actual version string
-    if r_version.startswith("R version "):
-        r_version = r_version.split(" ", 2)[2]
-    return r_version.strip()
+    check_r()
+    ver_string = rpy2.situation.r_version_from_subprocess().strip()
+    if ver_string:
+        # get the actual version string
+        if ver_string.startswith("R version "):
+            ver_string = ver_string.split(" ")[2]
+    return ver_string
 
 
 def has_lme4():
     """Return True if the lme4 package is installed"""
-    global _has_lme4
-    if _has_lme4:
-        return True
-    require_r()
-    for pkg in ["lme4", "statmod", "nloptr"]:
-        res = run_command(
-            (str(get_r_path()), "-q", "-e", f"system.file(package='{pkg}')"),
-            env={"R_LIBS_USER": os.environ.get("R_LIBS_USER", "")},
-            )
-        if not res.split("[1]")[1].count(pkg):
-            avail = False
-            break
-    else:
-        avail = _has_lme4 = True
-    return avail
+    check_r()
+    lme4_there = rpy2.robjects.packages.isinstalled("lme4")
+    statmod_there = rpy2.robjects.packages.isinstalled("statmod")
+    nloptr_there = rpy2.robjects.packages.isinstalled("nloptr")
+    return lme4_there and statmod_there and nloptr_there
 
 
 def has_r():
     """Return True if R is available"""
-    global _has_r
-    if _has_r:
-        return True
     try:
-        hasr = get_r_path().is_file()
-    except RNotFoundError:
+        hasr = rpy2.situation.get_r_home() is not None
+    except RUnavailableError:
         hasr = False
-    if hasr:
-        _has_r = True
     return hasr
 
 
-def require_lme4():
+def import_lme4():
+    check_r()
+    if has_lme4():
+        lme4pkg = rpy2.robjects.packages.importr("lme4")
+    else:
+        raise ValueError(
+            "The R package 'lme4' is not installed, please install it via "
+            + "`dclab.lme4.rsetup.install_lme4()` or by executing "
+            + "in a shell: R -e " + '"install.packages(' + "'lme4', "
+            + "repos='http://cran.rstudio.org')" + '"')
+    return lme4pkg
+
+
+def install_lme4():
     """Install the lme4 package (if not already installed)
 
-    Besides ``lme4``, this also installs ``nloptr`` and ``statmod``.
     The packages are installed to the user data directory
-    given in :const:`lib_path` from the http://cran.rstudio.org mirror.
+    given in :const:`lib_path`.
     """
-    install_command = ("install.packages("
-                       "c('statmod','nloptr','lme4'),"
-                       "repos='http://cran.rstudio.org'"
-                       ")"
-                       )
-    require_r()
+    check_r()
     if not has_lme4():
-        run_command(cmd=(get_r_path(), "-e", install_command),
-                    env={"R_LIBS_USER": os.environ.get("R_LIBS_USER", "")},
-                    )
+        # import R's utility package
+        utils = rpy2.robjects.packages.importr('utils')
+        # select the first mirror in the list
+        utils.chooseCRANmirror(ind=1)
+        # install lme4 to user data directory (say yes to user dir install)
+        with AutoRConsole() as rc:
+            # install statmod and nloptr first
+            # (Doesn't R have package dependencies?!)
+            utils.install_packages(
+                rpy2.robjects.vectors.StrVector(["statmod", "nloptr", "lme4"]))
+        return rc
 
 
-def require_r():
-    """Make sure R is installed an R HOME is set"""
-    if not has_r():
-        raise RNotFoundError("Cannot find R, please set its path with the "
-                             "`set_r_path` function or set the `RHOME` "
-                             "environment variable.")
-
-
-def run_command(cmd, **kwargs):
-    """Run a command via subprocess"""
-    if hasattr(sp, "STARTUPINFO"):
+def set_r_path(r_path):
+    """Set the path of the R executable/binary for rpy2"""
+    if hasattr(sp, 'STARTUPINFO'):
         # On Windows, subprocess calls will pop up a command window by
         # default when run from Pyinstaller with the ``--noconsole``
         # option. Avoid this distraction.
@@ -163,42 +192,16 @@ def run_command(cmd, **kwargs):
         si = None
         env = None
 
-    kwargs.setdefault("text", True)
-    kwargs.setdefault("stderr", sp.STDOUT)
-    if env is not None:
-        if "env" in kwargs:
-            env.update(kwargs.pop("env"))
-        kwargs["env"] = env
-    kwargs["startupinfo"] = si
-
-    # Convert paths to strings
-    cmd = [str(cc) for cc in cmd]
-
-    try:
-        tmp = sp.check_output(cmd, **kwargs)
-    except sp.CalledProcessError as e:
-        raise CommandFailedError(f"The command '{' '.join(cmd)}' failed with "
-                                 f"exit code {e.returncode}: {e.output}")
-
-    return tmp.strip()
-
-
-def set_r_lib_path(r_lib_path):
-    """Add given directory to the R_LIBS_USER environment variable"""
-    paths = os.environ.get("R_LIBS_USER", "").split(os.pathsep)
-    paths = [p for p in paths if p]
-    paths.append(str(r_lib_path).strip())
-    os.environ["R_LIBS_USER"] = os.pathsep.join(list(set(paths)))
-
-
-def set_r_path(r_path):
-    """Set the path of the R executable/binary"""
-    tmp = run_command((str(r_path), "RHOME"))
+    tmp = sp.check_output((r_path, 'RHOME'),
+                          startupinfo=si,
+                          env=env,
+                          text=True,
+                          )
 
     r_home = tmp.split(os.linesep)
-    if r_home[0].startswith("WARNING"):
+    if r_home[0].startswith('WARNING'):
         res = r_home[1]
     else:
         res = r_home[0].strip()
     os.environ["R_HOME"] = res
-    os.environ["R_EXEC"] = str(pathlib.Path(r_path).resolve())
+    import_r_submodules()