isolate 0.12.15__tar.gz → 0.12.16__tar.gz

{isolate-0.12.15 → isolate-0.12.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: isolate
-Version: 0.12.15
+Version: 0.12.16
 Summary: Managed isolated environments for Python
 Author-email: Features & Labels <hello@fal.ai>
 Project-URL: Issues, https://github.com/fal-ai/isolate/issues

{isolate-0.12.15 → isolate-0.12.16}/src/isolate/_isolate_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.12.15'
-__version_tuple__ = version_tuple = (0, 12, 15)
+__version__ = version = '0.12.16'
+__version_tuple__ = version_tuple = (0, 12, 16)

{isolate-0.12.15 → isolate-0.12.16}/src/isolate/backends/common.py

@@ -171,23 +171,26 @@ def _unblocked_pipe() -> tuple[int, int]:
 def logged_io(
     stdout_hook: HookT,
     stderr_hook: HookT | None = None,
-) -> Iterator[tuple[int, int]]:
+    log_hook: HookT | None = None,
+) -> Iterator[tuple[int, int, int]]:
     """Open two new streams (for stdout and stderr, respectively) and start relaying all
     the output from them to the given hooks."""
 
     stdout_reader_fd, stdout_writer_fd = _unblocked_pipe()
     stderr_reader_fd, stderr_writer_fd = _unblocked_pipe()
+    log_reader_fd, log_writer_fd = _unblocked_pipe()
 
     termination_event = threading.Event()
     io_observer = _io_observer(
         hooks={
             stdout_reader_fd: stdout_hook,
             stderr_reader_fd: stderr_hook or stdout_hook,
+            log_reader_fd: log_hook or stdout_hook,
         },
         termination_event=termination_event,
     )
     try:
-        yield stdout_writer_fd, stderr_writer_fd
+        yield stdout_writer_fd, stderr_writer_fd, log_writer_fd
     finally:
         termination_event.set()
         try:

{isolate-0.12.15 → isolate-0.12.16}/src/isolate/backends/conda.py

@@ -171,7 +171,7 @@ class CondaEnvironment(BaseEnvironment[Path]):
 
     def _run_conda(self, *args: Any) -> None:
         conda_executable = get_executable(self._exec_command, self._exec_home)
-        with logged_io(partial(self.log, level=LogLevel.INFO)) as (stdout, stderr):
+        with logged_io(partial(self.log, level=LogLevel.INFO)) as (stdout, stderr, _):
             subprocess.check_call(
                 [conda_executable, *args],
                 stdout=stdout,

{isolate-0.12.15 → isolate-0.12.16}/src/isolate/backends/pyenv.py

@@ -80,7 +80,7 @@ class PyenvEnvironment(BaseEnvironment[Path]):
         return Path(prefix.strip())
 
     def _install_python(self, pyenv: Path, root_path: Path) -> None:
-        with logged_io(partial(self.log, level=LogLevel.INFO)) as (stdout, stderr):
+        with logged_io(partial(self.log, level=LogLevel.INFO)) as (stdout, stderr, _):
             try:
                 subprocess.check_call(
                     [pyenv, "install", "--skip-existing", self.python_version],
@@ -102,7 +102,7 @@ class PyenvEnvironment(BaseEnvironment[Path]):
                 return None
 
             pyenv_root = connection_key.parent.parent
-            with logged_io(self.log) as (stdout, stderr):
+            with logged_io(self.log) as (stdout, stderr, _):
                 subprocess.check_call(
                     [pyenv, "uninstall", "-f", connection_key.name],
                     env={**os.environ, "PYENV_ROOT": str(pyenv_root)},

{isolate-0.12.15 → isolate-0.12.16}/src/isolate/backends/virtualenv.py

@@ -110,7 +110,7 @@ class VirtualPythonEnvironment(BaseEnvironment[Path]):
         for extra_index_url in self.extra_index_urls:
             pip_cmd.extend(["--extra-index-url", extra_index_url])
 
-        with logged_io(partial(self.log, level=LogLevel.INFO)) as (stdout, stderr):
+        with logged_io(partial(self.log, level=LogLevel.INFO)) as (stdout, stderr, _):
             try:
                 subprocess.check_call(
                     pip_cmd,

{isolate-0.12.15 → isolate-0.12.16}/src/isolate/connections/_local/_base.py

@@ -17,7 +17,7 @@ from typing import (
 
 from isolate.backends.common import get_executable_path, logged_io
 from isolate.connections.common import AGENT_SIGNATURE
-from isolate.logs import LogLevel
+from isolate.logs import LogLevel, LogSource
 
 if TYPE_CHECKING:
     from isolate.backends import BaseEnvironment
@@ -113,14 +113,22 @@ class PythonExecutionBase(Generic[ConnectionType]):
 
         python_executable = get_executable_path(self.environment_path, "python")
         with logged_io(
-            partial(self.handle_agent_log, level=LogLevel.STDOUT),
-            partial(self.handle_agent_log, level=LogLevel.STDERR),
-        ) as (stdout, stderr):
+            partial(
+                self.handle_agent_log, source=LogSource.USER, level=LogLevel.STDOUT
+            ),
+            partial(
+                self.handle_agent_log, source=LogSource.USER, level=LogLevel.STDERR
+            ),
+            partial(
+                self.handle_agent_log, source=LogSource.BRIDGE, level=LogLevel.TRACE
+            ),
+        ) as (stdout, stderr, log_fd):
             yield subprocess.Popen(
-                self.get_python_cmd(python_executable, connection),
+                self.get_python_cmd(python_executable, connection, log_fd),
                 env=self.get_env_vars(),
                 stdout=stdout,
                 stderr=stderr,
+                pass_fds=(log_fd,),
                 text=True,
             )
 
@@ -158,11 +166,12 @@ class PythonExecutionBase(Generic[ConnectionType]):
         self,
         executable: Path,
         connection: ConnectionType,
+        log_fd: int,
     ) -> list[str | Path]:
         """Return the command to run the agent process with."""
         raise NotImplementedError
 
-    def handle_agent_log(self, line: str, level: LogLevel) -> None:
+    def handle_agent_log(self, line: str, level: LogLevel, source: LogSource) -> None:
         """Handle a log line emitted by the agent process. The level will be either
         STDOUT or STDERR."""
         raise NotImplementedError

{isolate-0.12.15 → isolate-0.12.16}/src/isolate/connections/grpc/_base.py

@@ -136,13 +136,17 @@ class LocalPythonGRPC(PythonExecutionBase[str], GRPCExecutionBase):
         self,
         executable: Path,
         connection: str,
+        log_fd: int,
     ) -> List[Union[str, Path]]:
         return [
             executable,
             agent_startup.__file__,
             agent.__file__,
             connection,
+            "--log-fd",
+            str(log_fd),
         ]
 
-    def handle_agent_log(self, line: str, level: LogLevel) -> None:
-        self.log(line, level=level, source=LogSource.USER)
+    def handle_agent_log(self, line: str, level: LogLevel, source: LogSource) -> None:
+        print(f"[{source}] [{level}] {line}")
+        self.log(line, level=level, source=source)

{isolate-0.12.15 → isolate-0.12.16}/src/isolate/connections/grpc/agent.py

@@ -2,16 +2,16 @@
 
 from __future__ import annotations
 
+import os
+import sys
 import traceback
 from argparse import ArgumentParser
 from concurrent import futures
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import (
     Any,
-    Generator,
     Iterable,
     Iterator,
-    cast,
 )
 
 import grpc
@@ -21,8 +21,7 @@ from isolate.backends.common import sha256_digest_of
 from isolate.connections.common import SerializationError, serialize_object
 from isolate.connections.grpc import definitions
 from isolate.connections.grpc.configuration import get_default_options
-from isolate.connections.grpc.interface import from_grpc, to_grpc
-from isolate.logs import Log, LogLevel, LogSource
+from isolate.connections.grpc.interface import from_grpc
 
 
 @dataclass
@@ -30,16 +29,19 @@ class AbortException(Exception):
     message: str
 
 
-@dataclass
 class AgentServicer(definitions.AgentServicer):
-    _run_cache: dict[str, Any] = field(default_factory=dict)
+    def __init__(self, log_fd: int | None = None):
+        super().__init__()
+
+        self._run_cache: dict[str, Any] = {}
+        self._log = sys.stdout if log_fd is None else os.fdopen(log_fd, "w")
 
     def Run(
         self,
         request: definitions.FunctionCall,
         context: ServicerContext,
     ) -> Iterator[definitions.PartialRunResult]:
-        yield from self.log(f"A connection has been established: {context.peer()}!")
+        self.log(f"A connection has been established: {context.peer()}!")
 
         extra_args = []
         if request.HasField("setup_func"):
@@ -53,16 +55,16 @@ class AgentServicer(definitions.AgentServicer):
                         result,
                         was_it_raised,
                         stringized_tb,
-                    ) = yield from self.execute_function(
+                    ) = self.execute_function(
                         request.setup_func,
                         "setup",
                     )
 
                     if was_it_raised:
-                        yield from self.log(
+                        self.log(
                             "The setup function has thrown an error. Aborting the run."
                         )
-                        yield from self.send_object(
+                        yield self.send_object(
                             request.setup_func.method,
                             result,
                             was_it_raised,
@@ -78,12 +80,12 @@ class AgentServicer(definitions.AgentServicer):
             extra_args.append(self._run_cache[cache_key])
 
         try:
-            result, was_it_raised, stringized_tb = yield from self.execute_function(
+            result, was_it_raised, stringized_tb = self.execute_function(
                 request.function,
                 "function",
                 extra_args=extra_args,
             )
-            yield from self.send_object(
+            yield self.send_object(
                 request.function.method,
                 result,
                 was_it_raised,
@@ -98,7 +100,7 @@ class AgentServicer(definitions.AgentServicer):
         function_kind: str,
         *,
         extra_args: Iterable[Any] = (),
-    ) -> Generator[definitions.PartialRunResult, None, Any]:
+    ) -> tuple[Any, bool, str | None]:
         if function.was_it_raised:
             raise AbortException(
                 f"The {function_kind} function must be callable, "
@@ -110,7 +112,7 @@ class AgentServicer(definitions.AgentServicer):
             # depickling is basically involves code execution from the *user*.
             function = from_grpc(function)
         except SerializationError:
-            yield from self.log(traceback.format_exc(), level=LogLevel.ERROR)
+            self.log(traceback.format_exc())
             raise AbortException(
                 f"The {function_kind} function could not be deserialized."
             )
@@ -121,7 +123,7 @@ class AgentServicer(definitions.AgentServicer):
                 f"not {type(function).__name__}."
             )
 
-        yield from self.log(f"Starting the execution of the {function_kind} function.")
+        self.log(f"Starting the execution of the {function_kind} function.")
 
         was_it_raised = False
         stringized_tb = None
@@ -133,7 +135,7 @@ class AgentServicer(definitions.AgentServicer):
             num_frames = len(traceback.extract_stack()[:-5])
             stringized_tb = "".join(traceback.format_exc(limit=-num_frames))
 
-        yield from self.log(f"Completed the execution of the {function_kind} function.")
+        self.log(f"Completed the execution of the {function_kind} function.")
         return result, was_it_raised, stringized_tb
 
     def send_object(
@@ -142,46 +144,38 @@ class AgentServicer(definitions.AgentServicer):
         result: object,
         was_it_raised: bool,
         stringized_tb: str | None,
-    ) -> Generator[definitions.PartialRunResult, None, Any]:
+    ) -> definitions.PartialRunResult:
         try:
             definition = serialize_object(serialization_method, result)
         except SerializationError:
             if stringized_tb:
-                yield from self.log(
-                    stringized_tb, source=LogSource.USER, level=LogLevel.STDERR
-                )
+                print(stringized_tb, file=sys.stderr)
             raise AbortException(
                 "Error while serializing the execution result "
                 f"(object of type {type(result)})."
             )
         except BaseException:
-            yield from self.log(traceback.format_exc(), level=LogLevel.ERROR)
+            self.log(traceback.format_exc())
             raise AbortException(
                 "An unexpected error occurred while serializing the result."
             )
 
-        yield from self.log("Sending the result.")
+        self.log("Sending the result.")
         serialized_obj = definitions.SerializedObject(
             method=serialization_method,
             definition=definition,
             was_it_raised=was_it_raised,
             stringized_traceback=stringized_tb,
         )
-        yield definitions.PartialRunResult(
+        return definitions.PartialRunResult(
             result=serialized_obj,
             is_complete=True,
             logs=[],
         )
 
-    def log(
-        self,
-        message: str,
-        level: LogLevel = LogLevel.TRACE,
-        source: LogSource = LogSource.BRIDGE,
-    ) -> Iterator[definitions.PartialRunResult]:
-        log = to_grpc(Log(message, level=level, source=source))
-        log = cast(definitions.Log, log)
-        yield definitions.PartialRunResult(result=None, is_complete=False, logs=[log])
+    def log(self, message: str) -> None:
+        self._log.write(message)
+        self._log.flush()
 
     def abort_with_msg(
         self,
@@ -211,10 +205,10 @@ def create_server(address: str) -> grpc.Server:
     return server
 
 
-def run_agent(address: str) -> int:
+def run_agent(address: str, log_fd: int | None = None) -> int:
     """Run the agent servicer on the given address."""
     server = create_server(address)
-    servicer = AgentServicer()
+    servicer = AgentServicer(log_fd=log_fd)
 
     # This function just calls some methods on the server
     # and register a generic handler for the bridge. It does
@@ -229,9 +223,10 @@ def run_agent(address: str) -> int:
 def main() -> int:
     parser = ArgumentParser()
     parser.add_argument("address", type=str)
+    parser.add_argument("--log-fd", type=int)
 
     options = parser.parse_args()
-    return run_agent(options.address)
+    return run_agent(options.address, log_fd=options.log_fd)
 
 
 if __name__ == "__main__":

{isolate-0.12.15 → isolate-0.12.16}/src/isolate/connections/ipc/_base.py

@@ -202,6 +202,7 @@ class PythonIPC(PythonExecutionBase[AgentListener], IsolatedProcessConnection):
         self,
         executable: Path,
         connection: AgentListener,
+        log_fd: int,
     ) -> list[str | Path]:
         assert isinstance(connection.address, tuple)
         return [
@@ -214,21 +215,9 @@ class PythonIPC(PythonExecutionBase[AgentListener], IsolatedProcessConnection):
             # the connection with the bridge.
             "--serialization-backend",
             self.environment.settings.serialization_method,
+            "--log-fd",
+            str(log_fd),
         ]
 
-    def handle_agent_log(self, line: str, level: LogLevel) -> None:
-        # TODO: we probably should create a new fd and pass it as
-        # one of the the arguments to the child process. Then everything
-        # from that fd can be automatically logged as originating from the
-        # bridge.
-
-        # Agent can produce [trace] messages, so change the log
-        # level to it if this does not originate from the user.
-        if line.startswith("[trace]"):
-            line = line.replace("[trace]", "", 1)
-            level = LogLevel.TRACE
-            source = LogSource.BRIDGE
-        else:
-            source = LogSource.USER
-
+    def handle_agent_log(self, line: str, level: LogLevel, source: LogSource) -> None:
         self.log(line, level=level, source=source)

{isolate-0.12.15 → isolate-0.12.16}/src/isolate/connections/ipc/agent.py

@@ -15,6 +15,8 @@
 # one being the actual result of the given callable, and the other one is a boolean flag
 # indicating whether the callable has raised an exception or not.
 
+from __future__ import annotations
+
 import base64
 import importlib
 import os
@@ -24,7 +26,7 @@ import traceback
 from argparse import ArgumentParser
 from contextlib import closing
 from multiprocessing.connection import Client
-from typing import TYPE_CHECKING, Any, Callable, ContextManager, Tuple
+from typing import TYPE_CHECKING, Any, Callable, ContextManager
 
 if TYPE_CHECKING:
     # Somhow mypy can't figure out that `ConnectionWrapper`
@@ -47,13 +49,13 @@ else:
     from multiprocessing.connection import ConnectionWrapper
 
 
-def decode_service_address(address: str) -> Tuple[str, int]:
+def decode_service_address(address: str) -> tuple[str, int]:
     host, port = base64.b64decode(address).decode("utf-8").rsplit(":", 1)
     return host, int(port)
 
 
 def child_connection(
-    serialization_method: str, address: Tuple[str, int]
+    serialization_method: str, address: tuple[str, int]
 ) -> ContextManager[ConnectionWrapper]:
     serialization_backend = importlib.import_module(serialization_method)
     return closing(
@@ -70,7 +72,11 @@ DEBUG_TIMEOUT = 60 * 15
 
 
 def run_client(
-    serialization_method: str, address: Tuple[str, int], *, with_pdb: bool = False
+    serialization_method: str,
+    address: tuple[str, int],
+    *,
+    with_pdb: bool = False,
+    log_fd: int | None = None,
 ) -> None:
     # Debug Mode
     # ==========
@@ -96,13 +102,22 @@ def run_client(
 
         pdb.set_trace()
 
-    print(f"[trace] Trying to create a connection to {address}")
+    if log_fd is None:
+        _log = sys.stdout
+    else:
+        _log = os.fdopen(log_fd, "w")
+
+    def log(_msg):
+        _log.write(_msg)
+        _log.flush()
+
+    log(f"Trying to create a connection to {address}")
     # TODO(feat): this should probably run in a loop instead of
     # receiving a single function and then exitting immediately.
     with child_connection(serialization_method, address) as connection:
-        print(f"[trace] Created child connection to {address}")
+        log(f"Created child connection to {address}")
         callable = connection.recv()
-        print(f"[trace] Received the callable at {address}")
+        log(f"Received the callable at {address}")
 
         result = None
         did_it_raise = False
@@ -149,12 +164,11 @@ def _get_shell_bootstrap() -> str:
 
 
 def main() -> int:
-    print(f"[trace] Starting the isolated process at PID {os.getpid()}")
-
     parser = ArgumentParser()
     parser.add_argument("listen_at")
     parser.add_argument("--with-pdb", action="store_true", default=False)
     parser.add_argument("--serialization-backend", default="pickle")
+    parser.add_argument("--log-fd", type=int)
 
     options = parser.parse_args()
     if IS_DEBUG_MODE:
@@ -178,7 +192,12 @@ def main() -> int:
 
     serialization_method = options.serialization_backend
     address = decode_service_address(options.listen_at)
-    run_client(serialization_method, address, with_pdb=options.with_pdb)
+    run_client(
+        serialization_method,
+        address,
+        with_pdb=options.with_pdb,
+        log_fd=options.log_fd,
+    )
     return 0
 
 

{isolate-0.12.15 → isolate-0.12.16}/src/isolate.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: isolate
-Version: 0.12.15
+Version: 0.12.16
 Summary: Managed isolated environments for Python
 Author-email: Features & Labels <hello@fal.ai>
 Project-URL: Issues, https://github.com/fal-ai/isolate/issues