isolate 0.22.0__py3-none-any.whl

isolate/backends/container.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, ClassVar
+
+from isolate.backends import BaseEnvironment
+from isolate.backends.common import sha256_digest_of
+from isolate.backends.settings import DEFAULT_SETTINGS, IsolateSettings
+from isolate.connections import PythonIPC
+
+
+@dataclass
+class ContainerizedPythonEnvironment(BaseEnvironment[Path]):
+    BACKEND_NAME: ClassVar[str] = "container"
+
+    image: dict[str, Any] = field(default_factory=dict)
+    python_version: str | None = None
+    requirements: list[str] = field(default_factory=list)
+    tags: list[str] = field(default_factory=list)
+    resolver: str | None = None
+
+    @classmethod
+    def from_config(
+        cls,
+        config: dict[str, Any],
+        settings: IsolateSettings = DEFAULT_SETTINGS,
+    ) -> BaseEnvironment:
+        environment = cls(**config)
+        environment.apply_settings(settings)
+        if environment.resolver not in ("uv", None):
+            raise ValueError(
+                "Only 'uv' is supported as a resolver for container environments."
+            )
+        return environment
+
+    @property
+    def key(self) -> str:
+        extras = []
+        if self.resolver is not None:
+            extras.append(f"resolver={self.resolver}")
+
+        # dockerfile_str is always there, but the validation is handled by the
+        # controller.
+        dockerfile_str = self.image.get("dockerfile_str", "")
+        return sha256_digest_of(
+            dockerfile_str,
+            *self.requirements,
+            *sorted(self.tags),
+            *extras,
+        )
+
+    def create(self, *, force: bool = False) -> Path:
+        return Path(sys.exec_prefix)
+
+    def destroy(self, connection_key: Path) -> None:
+        raise NotImplementedError("ContainerizedPythonEnvironment cannot be destroyed")
+
+    def exists(self) -> bool:
+        return True
+
+    def open_connection(self, connection_key: Path) -> PythonIPC:
+        return PythonIPC(self, connection_key)

isolate/backends/local.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, ClassVar
+
+from isolate.backends import BaseEnvironment
+from isolate.backends.common import sha256_digest_of
+from isolate.backends.settings import DEFAULT_SETTINGS, IsolateSettings
+from isolate.connections import PythonIPC
+
+
+@dataclass
+class LocalPythonEnvironment(BaseEnvironment[Path]):
+    BACKEND_NAME: ClassVar[str] = "local"
+
+    @classmethod
+    def from_config(
+        cls,
+        config: dict[str, Any],
+        settings: IsolateSettings = DEFAULT_SETTINGS,
+    ) -> BaseEnvironment:
+        environment = cls(**config)
+        environment.apply_settings(settings)
+        return environment
+
+    @property
+    def key(self) -> str:
+        return sha256_digest_of(sys.exec_prefix)
+
+    def create(self, *, force: bool = False) -> Path:
+        if force is True:
+            raise NotImplementedError(
+                "LocalPythonEnvironment cannot be forcibly created"
+            )
+        return Path(sys.exec_prefix)
+
+    def destroy(self, connection_key: Path) -> None:
+        raise NotImplementedError("LocalPythonEnvironment cannot be destroyed")
+
+    def exists(self) -> bool:
+        return True
+
+    def open_connection(self, connection_key: Path) -> PythonIPC:
+        return PythonIPC(self, connection_key)

isolate/backends/pyenv.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import functools
+import os
+import shutil
+import subprocess
+from dataclasses import dataclass
+from functools import partial
+from pathlib import Path
+from typing import Any, ClassVar
+
+from isolate.backends import BaseEnvironment, EnvironmentCreationError
+from isolate.backends.common import logged_io
+from isolate.backends.settings import DEFAULT_SETTINGS, IsolateSettings
+from isolate.connections import PythonIPC
+from isolate.logs import LogLevel
+
+_PYENV_EXECUTABLE_NAME = "pyenv"
+_PYENV_EXECUTABLE_PATH = os.environ.get("ISOLATE_PYENV_EXECUTABLE")
+
+
+@dataclass
+class PyenvEnvironment(BaseEnvironment[Path]):
+    BACKEND_NAME: ClassVar[str] = "pyenv"
+
+    python_version: str
+
+    @classmethod
+    def from_config(
+        cls,
+        config: dict[str, Any],
+        settings: IsolateSettings = DEFAULT_SETTINGS,
+    ) -> BaseEnvironment:
+        environment = cls(**config)
+        environment.apply_settings(settings)
+        return environment
+
+    @property
+    def key(self) -> str:
+        return os.path.join("versions", self.python_version)
+
+    def create(self, *, force: bool = False) -> Path:
+        pyenv = _get_pyenv_executable()
+        env_path = self.settings.cache_dir_for(self)
+        with self.settings.cache_lock_for(env_path):
+            # PyEnv installs* the Python versions under $root/versions/$version, where
+            # we use versions/$version as the key and $root as the base directory
+            # (for pyenv).
+            #
+            # [0]: https://github.com/pyenv/pyenv#locating-pyenv-provided-python-installations
+            pyenv_root = env_path.parent.parent
+            prefix = self._try_get_prefix(pyenv, pyenv_root)
+            if prefix is None or force:
+                self._install_python(pyenv, pyenv_root)
+                prefix = self._try_get_prefix(pyenv, pyenv_root)
+                if not prefix:
+                    raise EnvironmentCreationError(
+                        f"Python {self.python_version} must have been installed by now."
+                    )
+
+            assert prefix is not None
+            return prefix
+
+    def _try_get_prefix(self, pyenv: Path, root_path: Path) -> Path | None:
+        try:
+            prefix = subprocess.check_output(
+                [pyenv, "prefix", self.python_version],
+                env={**os.environ, "PYENV_ROOT": str(root_path)},
+                text=True,
+                stderr=subprocess.PIPE,
+            )
+        except subprocess.CalledProcessError as exc:
+            if "not installed" in exc.stderr:
+                return None
+            raise EnvironmentCreationError(
+                f"Failed to get the prefix for Python {self.python_version}.\n"
+                f"{exc.stdout}\n{exc.stderr}"
+            )
+
+        return Path(prefix.strip())
+
+    def _install_python(self, pyenv: Path, root_path: Path) -> None:
+        with logged_io(partial(self.log, level=LogLevel.INFO)) as (stdout, stderr, _):
+            try:
+                subprocess.check_call(
+                    [pyenv, "install", "--skip-existing", self.python_version],
+                    env={**os.environ, "PYENV_ROOT": str(root_path)},
+                    stdout=stdout,
+                    stderr=stderr,
+                )
+            except subprocess.CalledProcessError:
+                raise EnvironmentCreationError(
+                    f"Failed to install Python {self.python_version} via pyenv.\n"
+                )
+
+    def destroy(self, connection_key: Path) -> None:
+        pyenv = _get_pyenv_executable()
+        with self.settings.cache_lock_for(connection_key):
+            # It might be destroyed already (when we are awaiting
+            # for the lock to be released).
+            if not connection_key.exists():
+                return None
+
+            pyenv_root = connection_key.parent.parent
+            with logged_io(self.log) as (stdout, stderr, _):
+                subprocess.check_call(
+                    [pyenv, "uninstall", "-f", connection_key.name],
+                    env={**os.environ, "PYENV_ROOT": str(pyenv_root)},
+                    stdout=stdout,
+                    stderr=stderr,
+                )
+
+    def exists(self) -> bool:
+        pyenv = _get_pyenv_executable()
+        cache_dir = self.settings.cache_dir_for(self)
+        with self.settings.cache_lock_for(cache_dir):
+            pyenv_root = cache_dir.parent.parent
+            prefix = self._try_get_prefix(pyenv, pyenv_root)
+            return prefix is not None
+
+    def open_connection(self, connection_key: Path) -> PythonIPC:
+        return PythonIPC(self, connection_key)
+
+
+@functools.lru_cache(1)
+def _get_pyenv_executable() -> Path:
+    if _PYENV_EXECUTABLE_PATH:
+        if not os.path.exists(_PYENV_EXECUTABLE_PATH):
+            raise EnvironmentCreationError(
+                "Path to pyenv executable not found! ISOLATE_PYENV_EXECUTABLE "
+                f"variable: {_PYENV_EXECUTABLE_PATH!r}"
+            )
+        return Path(_PYENV_EXECUTABLE_PATH)
+
+    pyenv_path = shutil.which(_PYENV_EXECUTABLE_NAME)
+    if pyenv_path is None:
+        raise FileNotFoundError(
+            "Could not find the pyenv executable. If pyenv is not already installed "
+            "in your system, please install it first. If it is not in your PATH, "
+            "then point ISOLATE_PYENV_COMMAND to the absolute path of the "
+            "pyenv executable."
+        )
+    return Path(pyenv_path)

isolate/backends/remote.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import copy
+import json
+from dataclasses import dataclass
+from typing import Any, ClassVar, List
+
+import grpc
+
+from isolate.backends import (
+    BaseEnvironment,
+    BasicCallable,
+    CallResultType,
+    EnvironmentConnection,
+)
+from isolate.backends.common import sha256_digest_of
+from isolate.backends.settings import DEFAULT_SETTINGS, IsolateSettings
+from isolate.server import interface
+from isolate.server.definitions import (
+    BoundFunction,
+    EnvironmentDefinition,
+    IsolateStub,
+)
+
+
+@dataclass
+class IsolateServer(BaseEnvironment[List[EnvironmentDefinition]]):
+    BACKEND_NAME: ClassVar[str] = "isolate-server"
+
+    host: str
+    target_environments: list[dict[str, Any]]
+
+    @classmethod
+    def from_config(
+        cls,
+        config: dict[str, Any],
+        settings: IsolateSettings = DEFAULT_SETTINGS,
+    ) -> BaseEnvironment:
+        environment = cls(**config)
+        environment.apply_settings(settings)
+
+        return environment
+
+    @property
+    def key(self) -> str:
+        return sha256_digest_of(
+            self.host,
+            json.dumps(self.target_environments),
+        )
+
+    def create(self, *, force: bool = False) -> list[EnvironmentDefinition]:
+        if force is True:
+            raise NotImplementedError(
+                "Only individual environments can be forcibly created, please set "
+                "them up manually by using the 'force_create' flag on the "
+                "environment definition."
+            )
+
+        envs = []
+        for env in self.target_environments:
+            if not env.get("kind") or not env.get("configuration"):
+                raise RuntimeError(f"`kind` or `configuration` key missing in: {env}")
+            configuration = copy.deepcopy(env["configuration"])
+            force_create = configuration.pop("force_create", False)
+            envs.append(
+                EnvironmentDefinition(
+                    kind=env["kind"],
+                    configuration=interface.to_struct(env["configuration"]),
+                    force=force_create,
+                )
+            )
+        return envs
+
+    def exists(self) -> bool:
+        return False
+
+    def open_connection(
+        self,
+        connection_key: list[EnvironmentDefinition],
+    ) -> IsolateServerConnection:
+        return IsolateServerConnection(self, self.host, connection_key)
+
+
+@dataclass
+class IsolateServerConnection(EnvironmentConnection):
+    host: str
+    definitions: list[EnvironmentDefinition]
+    _channel: grpc.Channel | None = None
+
+    def _acquire_channel(self) -> None:
+        self._channel = grpc.insecure_channel(self.host)
+
+    def _release_channel(self) -> None:
+        if self._channel:
+            self._channel.close()
+            self._channel = None
+
+    def __exit__(self, *args: Any) -> None:
+        self._release_channel()
+
+    def run(
+        self,
+        executable: BasicCallable,
+        *args: Any,
+        **kwargs: Any,
+    ) -> CallResultType:  # type: ignore[type-var]
+        if self._channel is None:
+            self._acquire_channel()
+
+        stub = IsolateStub(self._channel)
+        request = BoundFunction(
+            function=interface.to_serialized_object(
+                executable,
+                method=self.environment.settings.serialization_method,
+                was_it_raised=False,
+            ),
+            environments=self.definitions,
+            stream_logs=True,  # Default to streaming logs
+        )
+
+        return_value = []
+        for result in stub.Run(request):
+            for raw_log in result.logs:
+                log = interface.from_grpc(raw_log)
+                self.log(log.message, level=log.level, source=log.source)
+
+            if result.is_complete:
+                return_value.append(interface.from_grpc(result.result))
+
+        if len(return_value) == 0:
+            raise RuntimeError(
+                "No result object was received from the server"
+                " (it never set is_complete to True)."
+            )
+        elif len(return_value) > 1:
+            raise RuntimeError(
+                "Multiple result objects were received from the server"
+                " (it set is_complete to True multiple times)."
+            )
+        else:
+            return return_value[0]

isolate/backends/settings.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import os
+import shutil
+import tempfile
+from contextlib import contextmanager
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import TYPE_CHECKING, Callable, Iterator
+
+from platformdirs import user_cache_dir
+
+from isolate.backends.common import lock_build_path
+from isolate.logs import Log, LogLevel, LogSource
+
+if TYPE_CHECKING:
+    from isolate.backends import BaseEnvironment
+
+_SYSTEM_TEMP_DIR = Path(tempfile.gettempdir())
+_STRICT_CACHE = os.getenv("ISOLATE_STRICT_CACHE", "0") == "1"
+
+
+@dataclass(frozen=True)
+class IsolateSettings:
+    cache_dir: Path = Path(user_cache_dir("isolate", "isolate"))
+    serialization_method: str = "pickle"
+    log_hook: Callable[[Log], None] = print
+    strict_cache: bool = _STRICT_CACHE
+
+    def log(self, log: Log) -> None:
+        self.log_hook(self._infer_log_level(log))
+
+    def _infer_log_level(self, log: Log) -> Log:
+        """Infer the log level if it's correctly set."""
+        if log.level not in (LogLevel.STDOUT, LogLevel.STDERR):
+            # We should only infer the log level for stdout/stderr logs.
+            return log
+
+        if log.source in (LogSource.BUILDER, LogSource.BRIDGE):
+            return replace(log, level=LogLevel.TRACE)
+
+        line = log.message.lower()
+
+        if line.startswith("error") or "[error]" in line:
+            return replace(log, level=LogLevel.ERROR)
+        if line.startswith("warning") or "[warning]" in line:
+            return replace(log, level=LogLevel.WARNING)
+        if line.startswith("warn") or "[warn]" in line:
+            return replace(log, level=LogLevel.WARNING)
+        if line.startswith("info") or "[info]" in line:
+            return replace(log, level=LogLevel.INFO)
+        if line.startswith("debug") or "[debug]" in line:
+            return replace(log, level=LogLevel.DEBUG)
+        if line.startswith("trace") or "[trace]" in line:
+            return replace(log, level=LogLevel.TRACE)
+
+        # Default all to INFO level, even STDERR
+        return replace(log, level=LogLevel.INFO)
+
+    def _get_temp_base(self) -> Path:
+        """Return the base path for creating temporary files/directories.
+
+        If the isolate cache directory is in a different device than the
+        system temp base (e.g. /tmp), then it will return a new directory
+        under the cache directory."""
+
+        cache_stat = self.cache_dir.stat()
+        system_stat = _SYSTEM_TEMP_DIR.stat()
+        if cache_stat.st_dev == system_stat.st_dev:
+            return _SYSTEM_TEMP_DIR
+
+        if _SYSTEM_TEMP_DIR.samefile(self.cache_dir):
+            path = _SYSTEM_TEMP_DIR / "isolate"
+        else:
+            # This is quite important since if we have a shared cache
+            # disk, then /tmp is going to be in a different disk than
+            # the cache directory, which would make it impossible to
+            # rename() atomically.
+            path = self.cache_dir / "tmp"
+
+        path.mkdir(exist_ok=True, parents=True)
+        return path
+
+    def _get_lock_dir(self) -> Path:
+        """Return a directory which can be used for storing file-based locks."""
+        lock_dir = self._get_temp_base() / "locks"
+        lock_dir.mkdir(exist_ok=True, parents=True)
+        return lock_dir
+
+    @contextmanager
+    def cache_lock_for(self, path: Path) -> Iterator[Path]:
+        """Create a lock for accessing (and operating on) the given path. This
+        means whenever the context manager is entered, the path can be freely
+        modified and accessed without any other process interfering."""
+
+        with lock_build_path(path, self._get_lock_dir()):
+            try:
+                yield path
+            except BaseException:
+                # If anything goes wrong, we have to clean up the
+                # directory (we can't leave it as a corrupted build).
+                shutil.rmtree(path, ignore_errors=True)
+                raise
+
+    def cache_dir_for(self, backend: BaseEnvironment) -> Path:
+        """Return a directory which can be used for caching the given
+        environment's artifacts."""
+        backend_name = backend.BACKEND_NAME
+        assert backend_name is not None
+
+        environment_base_path = self.cache_dir / backend_name
+        environment_base_path.mkdir(exist_ok=True, parents=True)
+        return environment_base_path / backend.key
+
+    def completion_marker_for(self, path: Path) -> Path:
+        return path / ".isolate.completed"
+
+    replace = replace
+
+
+DEFAULT_SETTINGS = IsolateSettings()