isolate 0.22.0__py3-none-any.whl

isolate/__init__.py

@@ -0,0 +1,3 @@
+from isolate.registry import prepare_environment  # noqa: F401
+
+from ._version import __version__, version_tuple  # noqa: F401

isolate/_isolate_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.22.0'
+__version_tuple__ = version_tuple = (0, 22, 0)
+
+__commit_id__ = commit_id = None

isolate/_version.py

@@ -0,0 +1,6 @@
+try:
+    from ._isolate_version import version as __version__  # type: ignore[import]
+    from ._isolate_version import version_tuple  # type: ignore[import]
+except ImportError:
+    __version__ = "UNKNOWN"
+    version_tuple = (0, 0, __version__)  # type: ignore[assignment]

isolate/backends/__init__.py

@@ -0,0 +1,2 @@
+from isolate.backends._base import *  # noqa: F403
+from isolate.backends.settings import IsolateSettings  # noqa: F401

isolate/backends/_base.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import (
+    Any,
+    Callable,
+    ClassVar,
+    Generic,
+    Iterator,
+    TypeVar,
+)
+
+from isolate.backends.settings import DEFAULT_SETTINGS, IsolateSettings
+from isolate.logs import Log, LogLevel, LogSource
+
+__all__ = [
+    "BasicCallable",
+    "CallResultType",
+    "EnvironmentConnection",
+    "BaseEnvironment",
+    "EnvironmentCreationError",
+]
+
+ConnectionKeyType = TypeVar("ConnectionKeyType")
+CallResultType = TypeVar("CallResultType")
+BasicCallable = Callable[[], CallResultType]
+
+
+class EnvironmentCreationError(Exception):
+    """Raised when the environment cannot be created."""
+
+
+class BaseEnvironment(Generic[ConnectionKeyType]):
+    """Represents a managed environment definition for an isolatation backend
+    that can be used to run Python code with different set of dependencies."""
+
+    BACKEND_NAME: ClassVar[str | None] = None
+
+    settings: IsolateSettings = DEFAULT_SETTINGS
+
+    @classmethod
+    def from_config(
+        cls,
+        config: dict[str, Any],
+        settings: IsolateSettings = DEFAULT_SETTINGS,
+    ) -> BaseEnvironment:
+        """Create a new environment from the given configuration."""
+        raise NotImplementedError
+
+    @property
+    def key(self) -> str:
+        """A unique identifier for this environment (combination of requirements,
+        python version and other relevant information) that can be used for caching
+        and identification purposes."""
+        raise NotImplementedError
+
+    def create(self, *, force: bool = False) -> ConnectionKeyType:
+        """Setup the given environment, and return all the information needed
+        for establishing a connection to it. If `force` flag is set, then even
+        if the environment is cached; it will be tried to be re-built."""
+        raise NotImplementedError
+
+    def destroy(self, connection_key: ConnectionKeyType) -> None:
+        """Dismantle this environment. Might raise an exception if the environment
+        does not exist."""
+        raise NotImplementedError
+
+    def exists(self) -> bool:
+        """Return True if the environment already exists."""
+        raise NotImplementedError
+
+    def open_connection(
+        self, connection_key: ConnectionKeyType
+    ) -> EnvironmentConnection:
+        """Return a new connection to the environment with using the
+        `connection_key`."""
+        raise NotImplementedError
+
+    @contextmanager
+    def connect(self) -> Iterator[EnvironmentConnection]:
+        """Create the given environment (if it already doesn't exist) and establish a
+        connection to it."""
+        connection_key = self.create()
+        with self.open_connection(connection_key) as connection:
+            yield connection
+
+    def apply_settings(self, settings: IsolateSettings) -> None:
+        """Apply the new settings to this environment."""
+        self.settings = settings
+
+    def log(
+        self,
+        message: str,
+        *,
+        level: LogLevel = LogLevel.DEBUG,
+        source: LogSource = LogSource.BUILDER,
+    ) -> None:
+        """Log a message."""
+        log_msg = Log(message, level=level, source=source, bound_env=self)
+        self.settings.log(log_msg)
+
+
+@dataclass
+class EnvironmentConnection:
+    environment: BaseEnvironment
+
+    def __enter__(self) -> EnvironmentConnection:
+        return self
+
+    def __exit__(self, *exc_info):
+        return None
+
+    def run(
+        self,
+        executable: BasicCallable,
+        *args: Any,
+        **kwargs: Any,
+    ) -> CallResultType:  # type: ignore[type-var]
+        """Run the given executable inside the environment, and return the result.
+        If the executable raises an exception, then it will be raised directly."""
+        raise NotImplementedError
+
+    def log(
+        self,
+        message: str,
+        *,
+        level: LogLevel = LogLevel.TRACE,
+        source: LogSource = LogSource.BRIDGE,
+    ) -> None:
+        """Log a message through the bound environment."""
+        self.environment.log(message, level=level, source=source)

isolate/backends/common.py

@@ -0,0 +1,259 @@
+from __future__ import annotations
+
+import errno
+import hashlib
+import os
+import select
+import shutil
+import sysconfig
+import threading
+import time
+from contextlib import contextmanager, suppress
+from functools import lru_cache
+from pathlib import Path
+from types import ModuleType
+from typing import Callable, Iterator
+
+# For ensuring that the lock is created and not forgotten
+# (e.g. the process which acquires it crashes, so it is never
+# released), we are going to check the lock file's mtime every
+# _REVOKE_LOCK_DELAY seconds. If the mtime is older than that
+# value, we are going to assume the lock is stale and revoke it.
+_REVOKE_LOCK_DELAY = 30
+
+
+@contextmanager
+def lock_build_path(path: Path, lock_dir: Path) -> Iterator[None]:
+    """Try to acquire a lock for all operations on the given 'path'. This guarantees
+    that the path will not be modified by any other process while the lock is held."""
+    lock_file = (lock_dir / path.name).with_suffix(".lock")
+    while not _try_acquire(lock_file):
+        time.sleep(0.05)
+        continue
+
+    with _keep_lock_alive(lock_file):
+        yield
+
+
+@contextmanager
+def _keep_lock_alive(lock_file: Path) -> Iterator[None]:
+    """Keep the lock file alive by updating its mtime as long
+    as we are doing something in the cache."""
+    event = threading.Event()
+
+    def _keep_alive(per_beat_delay: float = 1) -> None:
+        while not event.wait(per_beat_delay):
+            lock_file.touch()
+        lock_file.unlink()
+
+    thread = threading.Thread(target=_keep_alive)
+    try:
+        thread.start()
+        yield
+    finally:
+        event.set()
+        thread.join()
+
+
+def _try_acquire(lock_file: Path) -> bool:
+    with suppress(FileNotFoundError):
+        mtime = lock_file.stat().st_mtime
+        if time.time() - mtime > _REVOKE_LOCK_DELAY:
+            # The lock file exists, but it may be stale. Check the
+            # mtime and if it is too old, revoke it.
+            lock_file.unlink()
+
+    try:
+        lock_file.touch(exist_ok=False)
+    except FileExistsError:
+        return False
+    else:
+        return True
+
+
+def get_executable_path(search_path: Path, executable_name: str) -> Path:
+    """Return the path for the executable named 'executable_name' under
+    the '/bin' directory of 'search_path'."""
+
+    bin_dir = (search_path / "bin").as_posix()
+    executable_path = shutil.which(executable_name, path=bin_dir)
+    if executable_path is None:
+        raise FileNotFoundError(
+            f"Could not find '{executable_name}' in '{search_path}'. "
+            f"Is the virtual environment corrupted?"
+        )
+
+    return Path(executable_path)
+
+
+_CHECK_FOR_TERMINATION_DELAY = 0.05
+HookT = Callable[[str], None]
+
+
+def _io_observer(
+    hooks: dict[int, HookT],
+    termination_event: threading.Event,
+) -> threading.Thread:
+    """Starts a new thread that reads from the specified file descriptors
+    and calls the bound hook function for each line until the EOF is reached
+    or the termination event is set.
+
+    Caller is responsible for joining the thread.
+    """
+
+    followed_fds = list(hooks.keys())
+    for fd in followed_fds:
+        if os.get_blocking(fd):
+            raise NotImplementedError(
+                "All the hooked file descriptors must be non-blocking."
+            )
+
+    def forward_lines(fd: int) -> None:
+        hook = hooks[fd]
+        with open(fd, closefd=False, errors="backslashreplace") as stream:
+            # TODO: we probably should pass the real line endings
+            raw_data = stream.read()
+            if not raw_data:
+                return  # Nothing to read
+
+            for line in raw_data.splitlines():
+                # TODO: parse the lines to include `extra={...}` added by the logger?
+                hook(line)
+
+    def _reader():
+        while not termination_event.is_set():
+            # The observed file descriptors may be closed by the
+            # underlying process at any given time. So before we
+            # make a select call, we need to check if the file
+            # descriptors are still valid and remove the ones
+            # that are not.
+            for fd in followed_fds.copy():
+                try:
+                    os.fstat(fd)
+                except OSError as exc:
+                    if exc.errno == errno.EBADF:
+                        followed_fds.remove(fd)
+
+            if not followed_fds:
+                # All the file descriptors are closed, so we can
+                # stop the thread.
+                return
+
+            ready, _, _ = select.select(
+                # rlist=
+                followed_fds,
+                # wlist=
+                [],
+                # xlist=
+                [],
+                # timeout=
+                _CHECK_FOR_TERMINATION_DELAY,
+            )
+            for fd in ready:
+                forward_lines(fd)
+
+    observer_thread = threading.Thread(target=_reader)
+    observer_thread.start()
+    return observer_thread
+
+
+def _unblocked_pipe() -> tuple[int, int]:
+    """Create a pair of unblocked pipes. This is actually
+    the same as os.pipe2(os.O_NONBLOCK), but that is not
+    available in MacOS so we have to do it manually."""
+
+    read_fd, write_fd = os.pipe()
+    os.set_blocking(read_fd, False)
+    os.set_blocking(write_fd, False)
+    return read_fd, write_fd
+
+
+@contextmanager
+def logged_io(
+    stdout_hook: HookT,
+    stderr_hook: HookT | None = None,
+    log_hook: HookT | None = None,
+) -> Iterator[tuple[int, int, int]]:
+    """Open two new streams (for stdout and stderr, respectively) and start relaying all
+    the output from them to the given hooks."""
+
+    stdout_reader_fd, stdout_writer_fd = _unblocked_pipe()
+    stderr_reader_fd, stderr_writer_fd = _unblocked_pipe()
+    log_reader_fd, log_writer_fd = _unblocked_pipe()
+
+    termination_event = threading.Event()
+    io_observer = _io_observer(
+        hooks={
+            stdout_reader_fd: stdout_hook,
+            stderr_reader_fd: stderr_hook or stdout_hook,
+            log_reader_fd: log_hook or stdout_hook,
+        },
+        termination_event=termination_event,
+    )
+    try:
+        yield stdout_writer_fd, stderr_writer_fd, log_writer_fd
+    finally:
+        termination_event.set()
+        try:
+            # The observer thread checks the termination event in every
+            # _CHECK_FOR_TERMINATION_DELAY seconds. We need to wait at least
+            # more than that to make sure that it has a chance to terminate
+            # properly.
+            io_observer.join(timeout=_CHECK_FOR_TERMINATION_DELAY * 3)
+        except TimeoutError:
+            raise RuntimeError("Log observers did not terminate in time.")
+
+
+@lru_cache(maxsize=None)
+def sha256_digest_of(*unique_fields: str | bytes) -> str:
+    """Return the SHA256 digest that corresponds to the combined version
+    of 'unique_fields. The order is preserved."""
+
+    def _normalize(text: str | bytes) -> bytes:
+        if isinstance(text, str):
+            return text.encode()
+        else:
+            return text
+
+    join_char = b"\n"
+    inner_text = join_char.join(map(_normalize, unique_fields))
+    return hashlib.sha256(inner_text).hexdigest()
+
+
+def active_python() -> str:
+    """Return the active Python version that can be used for caching
+    and re-creating this environment. Currently only covers major and
+    minor versions (like 3.9); patch versions are ignored (like 3.9.4)."""
+    return sysconfig.get_python_version()
+
+
+def optional_import(module_name: str) -> ModuleType:
+    """Try to import the given module, and fail if it is not available
+    with an informative error message that includes the installations
+    instructions."""
+
+    import importlib
+
+    try:
+        return importlib.import_module(module_name)
+    except ImportError as exc:
+        raise ImportError(
+            "isolate must be installed with the 'build' extras for "
+            f"accessing {module_name!r} import functionality. Please try: "
+            f"'$ pip install \"isolate[build]\"' to install it."
+        ) from exc
+
+
+@lru_cache(4)
+def get_executable(command: str, home: str | None = None) -> Path:
+    for path in [home, None]:
+        binary_path = shutil.which(command, path=path)
+        if binary_path is not None:
+            return Path(binary_path)
+    # TODO: we should probably show some instructions on how you
+    # can install conda here.
+    raise FileNotFoundError(
+        f"Could not find the {command} executable. "
+        f"If the {command} executable is not available by default, please point "
+        f"isolate to the path where the {command} binary is available '{home}'."
+    )

isolate/backends/conda.py

@@ -0,0 +1,215 @@
+from __future__ import annotations
+
+import copy
+import os
+import subprocess
+import tempfile
+from dataclasses import dataclass, field
+from functools import partial
+from pathlib import Path
+from typing import Any, ClassVar
+
+from isolate.backends import BaseEnvironment, EnvironmentCreationError
+from isolate.backends.common import (
+    active_python,
+    get_executable,
+    logged_io,
+    optional_import,
+    sha256_digest_of,
+)
+from isolate.backends.settings import DEFAULT_SETTINGS, IsolateSettings
+from isolate.connections import PythonIPC
+from isolate.logs import LogLevel
+
+# Specify paths where conda and mamba binaries might reside
+_CONDA_COMMAND = os.environ.get("CONDA_EXE", "conda")
+_MAMBA_COMMAND = os.environ.get("MAMBA_EXE", "micromamba")
+_ISOLATE_CONDA_HOME = os.getenv("ISOLATE_CONDA_HOME")
+_ISOLATE_MAMBA_HOME = os.getenv("ISOLATE_MAMBA_HOME")
+_ISOLATE_DEFAULT_RESOLVER = os.getenv("ISOLATE_DEFAULT_RESOLVER", "mamba")
+
+# Conda accepts the following version specifiers: =, ==, >=, <=, >, <, !=
+_POSSIBLE_CONDA_VERSION_IDENTIFIERS = (
+    "=",
+    "<",
+    ">",
+    "!",
+)
+
+
+@dataclass
+class CondaEnvironment(BaseEnvironment[Path]):
+    BACKEND_NAME: ClassVar[str] = "conda"
+
+    environment_definition: dict[str, Any] = field(default_factory=dict)
+    python_version: str | None = None
+    tags: list[str] = field(default_factory=list)
+    _exec_home: str | None = _ISOLATE_MAMBA_HOME
+    _exec_command: str | None = _MAMBA_COMMAND
+
+    @classmethod
+    def from_config(
+        cls,
+        config: dict[str, Any],
+        settings: IsolateSettings = DEFAULT_SETTINGS,
+    ) -> BaseEnvironment:
+        processing_config = copy.deepcopy(config)
+        processing_config.setdefault("python_version", active_python())
+        resolver = processing_config.pop("resolver", _ISOLATE_DEFAULT_RESOLVER)
+        if resolver == "conda":
+            _exec_home = _ISOLATE_CONDA_HOME
+            _exec_command = _CONDA_COMMAND
+        elif resolver == "mamba":
+            _exec_home = _ISOLATE_MAMBA_HOME
+            _exec_command = _MAMBA_COMMAND
+        else:
+            raise Exception(f"Conda resolver of type {resolver} is not supported")
+        if "env_dict" in processing_config:
+            definition = processing_config.pop("env_dict")
+        elif "env_yml_str" in processing_config:
+            yaml = optional_import("yaml")
+
+            definition = yaml.safe_load(processing_config.pop("env_yml_str"))
+        elif "packages" in processing_config:
+            definition = {
+                "dependencies": processing_config.pop("packages"),
+            }
+        else:
+            raise ValueError(
+                "Either 'env_dict', 'env_yml_str' or 'packages' must be specified"
+            )
+
+        dependencies = definition.setdefault("dependencies", [])
+        if _depends_on(dependencies, "python"):
+            raise ValueError(
+                "Python version can not be specified by the environment but rather ",
+                " it needs to be passed as `python_version` option to the environment.",
+            )
+
+        dependencies.append(f"python={processing_config['python_version']}")
+
+        # Extend pip dependencies and channels if they are specified.
+        if "pip" in processing_config:
+            if not _depends_on(dependencies, "pip"):
+                dependencies.append("pip")
+
+            try:
+                dependency_group = next(
+                    dependency
+                    for dependency in dependencies
+                    if isinstance(dependency, dict) and "pip" in dependency
+                )
+            except StopIteration:
+                dependency_group = {"pip": []}
+                dependencies.append(dependency_group)
+
+            dependency_group["pip"].extend(processing_config.pop("pip"))
+
+        if "channels" in processing_config:
+            definition.setdefault("channels", [])
+            definition["channels"].extend(processing_config.pop("channels"))
+
+        environment = cls(
+            environment_definition=definition,
+            _exec_home=_exec_home,
+            _exec_command=_exec_command,
+            **processing_config,
+        )
+        environment.apply_settings(settings)
+        return environment
+
+    @property
+    def key(self) -> str:
+        return sha256_digest_of(
+            repr(self.environment_definition),
+            self.python_version,
+            self._exec_command,
+            *sorted(self.tags),
+        )
+
+    def create(self, *, force: bool = False) -> Path:
+        env_path = self.settings.cache_dir_for(self)
+        with self.settings.cache_lock_for(env_path):
+            if env_path.exists() and not force:
+                return env_path
+
+            self.log(f"Creating the environment at '{env_path}'")
+            with tempfile.NamedTemporaryFile(mode="w", suffix=".yml") as tf:
+                yaml = optional_import("yaml")
+                yaml.dump(self.environment_definition, tf)
+                tf.flush()
+
+                try:
+                    self._run_create(str(env_path), tf.name)
+                except subprocess.SubprocessError as exc:
+                    raise EnvironmentCreationError(
+                        f"Failure during 'conda create': {exc}"
+                    )
+
+            self.log(f"New environment cached at '{env_path}'")
+            return env_path
+
+    def destroy(self, connection_key: Path) -> None:
+        with self.settings.cache_lock_for(connection_key):
+            # It might be destroyed already (when we are awaiting
+            # for the lock to be released).
+            if not connection_key.exists():
+                return
+
+            self._run_destroy(str(connection_key))
+
+    def _run_create(self, env_path: str, env_name: str) -> None:
+        if self._exec_command == "conda":
+            self._run_conda(
+                "env", "create", "--yes", "--prefix", env_path, "-f", env_name
+            )
+        else:
+            self._run_conda("env", "create", "--prefix", env_path, "-f", env_name)
+
+    def _run_destroy(self, connection_key: str) -> None:
+        self._run_conda("remove", "--yes", "--all", "--prefix", connection_key)
+
+    def _run_conda(self, *args: Any) -> None:
+        conda_executable = get_executable(self._exec_command, self._exec_home)
+        with logged_io(partial(self.log, level=LogLevel.INFO)) as (stdout, stderr, _):
+            subprocess.check_call(
+                [conda_executable, *args],
+                stdout=stdout,
+                stderr=stderr,
+            )
+
+    def exists(self) -> bool:
+        path = self.settings.cache_dir_for(self)
+        return path.exists()
+
+    def open_connection(self, connection_key: Path) -> PythonIPC:
+        return PythonIPC(self, connection_key)
+
+
+def _depends_on(
+    dependencies: list[str | dict[str, list[str]]],
+    package_name: str,
+) -> bool:
+    for dependency in dependencies:
+        if isinstance(dependency, dict):
+            # It is a dependency group like pip: [...]
+            continue
+
+        # Get rid of all whitespace characters (python = 3.8 becomes python=3.8)
+        package = dependency.replace(" ", "")
+        if not package.startswith(package_name):
+            continue
+
+        # Ensure that the package name matches perfectly and not only
+        # at the prefix level. Examples:
+        #  - python # OK
+        #  - python=3.8 # OK
+        #  - python>=3.8 # OK
+        #  - python-user-toolkit # NOT OK
+        #  - pythonhelp!=1.0 # NOT OK
+        suffix = package[len(package_name) :]
+        if suffix and suffix[0] not in _POSSIBLE_CONDA_VERSION_IDENTIFIERS:
+            continue
+
+        return True
+    return False