isolate 0.26.4__tar.gz → 0.26.6__tar.gz

{isolate-0.26.4/src/isolate.egg-info → isolate-0.26.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: isolate
-Version: 0.26.4
+Version: 0.26.6
 Summary: Managed isolated environments for Python
 Author-email: Features & Labels <hello@fal.ai>
 Project-URL: Issues, https://github.com/fal-ai/isolate/issues

{isolate-0.26.4 → isolate-0.26.6}/pyproject.toml

@@ -76,3 +76,8 @@ keep-runtime-typing = true
 
 [tool.ruff.lint]
 select = ["E", "F", "W", "PLC", "PLE", "PLW", "I", "UP"]
+
+[tool.pytest.ini_options]
+addopts = "-ra --durations=50"
+asyncio_mode = "auto"
+testpaths = "tests"

{isolate-0.26.4 → isolate-0.26.6}/src/isolate/_isolate_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.26.4'
-__version_tuple__ = version_tuple = (0, 26, 4)
+__version__ = version = '0.26.6'
+__version_tuple__ = version_tuple = (0, 26, 6)
 
-__commit_id__ = commit_id = 'gce17302a5'
+__commit_id__ = commit_id = 'g6d4ff5277'

{isolate-0.26.4 → isolate-0.26.6}/src/isolate/backends/common.py

@@ -166,13 +166,15 @@ def _close_fd(fd: int) -> None:
 
 
 def _unblocked_pipe() -> tuple[int, int]:
-    """Create a pair of unblocked pipes. This is actually
-    the same as os.pipe2(os.O_NONBLOCK), but that is not
-    available in MacOS so we have to do it manually."""
+    """Create a pipe whose read end is non-blocking, so the reader thread in
+    `_io_observer` can `select()` + `read()` without risk of hanging on a
+    partial-line edge case. The write end stays blocking so the worker's
+    stdout/stderr applies backpressure (waits for the reader to drain the
+    kernel pipe buffer) instead of raising BlockingIOError when the buffer
+    fills up — which happens reliably with chunky tracebacks on Python 3.14."""
 
     read_fd, write_fd = os.pipe()
     os.set_blocking(read_fd, False)
-    os.set_blocking(write_fd, False)
     return read_fd, write_fd
 
 

{isolate-0.26.4 → isolate-0.26.6}/src/isolate/connections/grpc/_base.py

@@ -185,14 +185,26 @@ class LocalPythonGRPC(PythonExecutionBase[str], GRPCExecutionBase):
                     # already finished
                     return_code = self._process.returncode
                 else:
-                    print("Terminating the agent process...")
+                    self.log(
+                        "Terminating the agent process...",
+                        level=LogLevel.INFO,
+                        source=LogSource.BRIDGE,
+                    )
                     self._process.terminate()
                     return_code = self._process.wait(
                         timeout=PROCESS_SHUTDOWN_TIMEOUT_SECONDS
                     )
-                    print("Agent process shutdown gracefully")
+                    self.log(
+                        "Agent process shutdown gracefully",
+                        level=LogLevel.INFO,
+                        source=LogSource.BRIDGE,
+                    )
             except Exception as exc:
-                print(f"Failed to shutdown the agent process gracefully: {exc}")
+                self.log(
+                    f"Failed to shutdown the agent process gracefully: {exc}",
+                    level=LogLevel.ERROR,
+                    source=LogSource.BRIDGE,
+                )
                 self._process.kill()
                 return_code = self._process.wait()
 

{isolate-0.26.4 → isolate-0.26.6}/src/isolate/logger.py

@@ -1,5 +1,6 @@
 import json
 import os
+import sys
 from datetime import datetime, timezone
 from typing import Dict
 
@@ -55,6 +56,10 @@ class IsolateLogger:
             try:
                 _labels = json.loads(raw)
             except json.JSONDecodeError:
-                print("Failed to parse ISOLATE_LOG_LABELS")
+                # Write to stderr so a malformed env var does not become a
+                # constant first stdout line and break Vector's content-based
+                # file fingerprint (collision-prone first lines cause silent
+                # log skipping; see infra commit b9f0f2237).
+                sys.stderr.write("Failed to parse ISOLATE_LOG_LABELS\n")
 
         return cls.with_env_expanded(labels=_labels)

{isolate-0.26.4 → isolate-0.26.6}/src/isolate/server/server.py

@@ -36,6 +36,15 @@ from isolate.server import definitions, health
 from isolate.server.health_server import HealthServicer
 from isolate.server.interface import from_grpc, to_grpc
 
+# Server-level logger for events that aren't tied to a RunTask (startup,
+# signals, auth interceptor, lifecycle). Routes to stdout as JSON via
+# IsolateLogger.log -> print(json.dumps(record)). Critical: this makes the
+# very first line written to isolate-server.stdout a JSON record with a
+# microsecond-precision logged_at, so Vector's default content-based file
+# fingerprint (checksum, lines: 1) is unique per alloc instead of colliding
+# on the constant "Started listening at 0.0.0.0:NNNNN" prefix.
+_SERVER_LOGGER = IsolateLogger.from_env()
+
 EMPTY_MESSAGE_INTERVAL = float(os.getenv("ISOLATE_EMPTY_MESSAGE_INTERVAL", "600"))
 SKIP_EMPTY_LOGS = os.getenv("ISOLATE_SKIP_EMPTY_LOGS") == "1"
 MAX_GRPC_WAIT_TIMEOUT = float(os.getenv("ISOLATE_MAX_GRPC_WAIT_TIMEOUT", "10.0"))
@@ -442,7 +451,12 @@ class IsolateServicer(definitions.IsolateServicer):
         task.future = self._thread_pool.submit(self._run_task_in_background, task)
         task_id = str(uuid.uuid4())
 
-        print(f"Submitted a task {task_id}")
+        _SERVER_LOGGER.log(
+            LogLevel.INFO,
+            f"Submitted a task {task_id}",
+            source=LogSource.BRIDGE,
+            line_labels={},
+        )
 
         self.background_tasks[task_id] = task
 
@@ -452,7 +466,12 @@ class IsolateServicer(definitions.IsolateServicer):
                 msg += f" error: {exc!r}"
             else:
                 msg += f" result: {future.result()!r}"
-            print(msg)
+            _SERVER_LOGGER.log(
+                LogLevel.INFO,
+                msg,
+                source=LogSource.BRIDGE,
+                line_labels={},
+            )
             self.background_tasks.pop(task_id, None)
 
         task.future.add_done_callback(_callback)
@@ -518,7 +537,12 @@ class IsolateServicer(definitions.IsolateServicer):
     ) -> definitions.CancelResponse:
         task_id = request.task_id
 
-        print(f"Canceling task {task_id}")
+        _SERVER_LOGGER.log(
+            LogLevel.INFO,
+            f"Canceling task {task_id}",
+            source=LogSource.BRIDGE,
+            line_labels={},
+        )
         task = self.background_tasks.get(task_id)
         if task is not None:
             task.cancel()
@@ -527,14 +551,29 @@ class IsolateServicer(definitions.IsolateServicer):
 
     def shutdown(self) -> None:
         if self._shutting_down:
-            print("Shutdown already in progress...")
+            _SERVER_LOGGER.log(
+                LogLevel.INFO,
+                "Shutdown already in progress...",
+                source=LogSource.BRIDGE,
+                line_labels={},
+            )
             return
 
         self._shutting_down = True
         task_count = len(self.background_tasks)
-        print(f"Shutting down, canceling {task_count} tasks...")
+        _SERVER_LOGGER.log(
+            LogLevel.INFO,
+            f"Shutting down, canceling {task_count} tasks...",
+            source=LogSource.BRIDGE,
+            line_labels={},
+        )
         self.cancel_tasks()
-        print("All tasks canceled.")
+        _SERVER_LOGGER.log(
+            LogLevel.INFO,
+            "All tasks canceled.",
+            source=LogSource.BRIDGE,
+            line_labels={},
+        )
 
     def watch_queue_until_completed(
         self, queue: Queue, is_completed: Callable[[], bool]
@@ -703,11 +742,21 @@ class SingleTaskInterceptor(ServerBoundInterceptor):
             def _wrapper(request: Any, context: grpc.ServicerContext) -> Any:
                 def termination() -> None:
                     if is_run:
-                        print("Stopping server since run is finished")
+                        _SERVER_LOGGER.log(
+                            LogLevel.INFO,
+                            "Stopping server since run is finished",
+                            source=LogSource.BRIDGE,
+                            line_labels={},
+                        )
                         self.servicer.shutdown()
                         # Stop the server after the Run task is finished
                         self.server.stop(grace=0.1)
-                        print("Server stopped")
+                        _SERVER_LOGGER.log(
+                            LogLevel.INFO,
+                            "Server stopped",
+                            source=LogSource.BRIDGE,
+                            line_labels={},
+                        )
 
                     elif is_submit:
                         # Wait until the task_id is assigned
@@ -731,10 +780,20 @@ class SingleTaskInterceptor(ServerBoundInterceptor):
                             def _stop(*args):
                                 # Small sleep to make sure the cancellation is processed
                                 time.sleep(0.3)
-                                print("Stopping server since the task is finished")
+                                _SERVER_LOGGER.log(
+                                    LogLevel.INFO,
+                                    "Stopping server since the task is finished",
+                                    source=LogSource.BRIDGE,
+                                    line_labels={},
+                                )
                                 self.servicer.shutdown()
                                 self.server.stop(grace=0.1)
-                                print("Server stopped")
+                                _SERVER_LOGGER.log(
+                                    LogLevel.INFO,
+                                    "Server stopped",
+                                    source=LogSource.BRIDGE,
+                                    line_labels={},
+                                )
 
                             # Add a callback which will stop the server
                             # after the task is finished
@@ -771,7 +830,12 @@ class ControllerAuthInterceptor(ServerBoundInterceptor):
         ]
 
         if handler_call_details.method in skipped_auth_methods:
-            print(f"[debug] Skipping authentication for {handler_call_details.method}")
+            _SERVER_LOGGER.log(
+                LogLevel.DEBUG,
+                f"Skipping authentication for {handler_call_details.method}",
+                source=LogSource.BRIDGE,
+                line_labels={},
+            )
             # Let these requests pass through without authentication
             return continuation(handler_call_details)
 
@@ -816,9 +880,12 @@ def main(argv: list[str] | None = None) -> None:
         # Set an interceptor to only accept requests with the correct auth key
         interceptors.append(ControllerAuthInterceptor(controller_auth_key))
     else:
-        print(
-            "[WARN] ISOLATE_CONTROLLER_AUTH_KEY is not set, all requests will be "
-            "accepted without authentication."
+        _SERVER_LOGGER.log(
+            LogLevel.WARNING,
+            "ISOLATE_CONTROLLER_AUTH_KEY is not set, all requests will be "
+            "accepted without authentication.",
+            source=LogSource.BRIDGE,
+            line_labels={},
         )
 
     server = grpc.server(
@@ -839,6 +906,12 @@ def main(argv: list[str] | None = None) -> None:
         definitions.register_isolate(servicer, server)
         health.register_health(HealthServicer(), server)
 
+        # Signal handlers intentionally use bare print(): IsolateLogger.log()
+        # calls datetime.now() and json.dumps() which allocate memory, and
+        # heap allocation in a Python signal handler is unsafe (can deadlock
+        # if the main thread is mid-allocation). These messages also fire at
+        # shutdown — well past the first stdout line — so they don't affect
+        # Vector's file fingerprint.
         def handle_termination(*args):
             print("Termination signal received, shutting down...")
             servicer.shutdown()
@@ -853,11 +926,21 @@ def main(argv: list[str] | None = None) -> None:
         signal.signal(signal.SIGCHLD, handle_child_termination)
 
         server.add_insecure_port(f"[::]:{options.port}")
-        print(f"Started listening at {options.host}:{options.port}")
+        _SERVER_LOGGER.log(
+            LogLevel.INFO,
+            f"Started listening at {options.host}:{options.port}",
+            source=LogSource.BRIDGE,
+            line_labels={},
+        )
 
         server.start()
         server.wait_for_termination()
-        print("Server shut down")
+        _SERVER_LOGGER.log(
+            LogLevel.INFO,
+            "Server shut down",
+            source=LogSource.BRIDGE,
+            line_labels={},
+        )
 
 
 if __name__ == "__main__":

{isolate-0.26.4 → isolate-0.26.6/src/isolate.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: isolate
-Version: 0.26.4
+Version: 0.26.6
 Summary: Managed isolated environments for Python
 Author-email: Features & Labels <hello@fal.ai>
 Project-URL: Issues, https://github.com/fal-ai/isolate/issues

{isolate-0.26.4 → isolate-0.26.6}/src/isolate.egg-info/SOURCES.txt

@@ -79,6 +79,7 @@ tests/test_connections_grpc_base.py
 tests/test_isolate.py
 tests/test_log.py
 tests/test_log_masking.py
+tests/test_logged_io_pipe.py
 tests/test_logger.py
 tests/test_serialization.py
 tests/test_server.py

{isolate-0.26.4 → isolate-0.26.6}/tests/test_connections_grpc_base.py

@@ -1,7 +1,7 @@
 import asyncio
 import os
 from pathlib import Path
-from unittest.mock import Mock, patch
+from unittest.mock import Mock, call, patch
 
 import grpc
 import pytest
@@ -51,11 +51,23 @@ def test_abort_agent_logs_return_code_for_graceful_termination(tmp_path: Path) -
     process.terminate.assert_called_once()
     process.wait.assert_called_once()
     process.kill.assert_not_called()
-    connection.log.assert_called_once_with(
-        "Isolate agent gracefully terminated",
-        level=LogLevel.INFO,
-        source=LogSource.BRIDGE,
-    )
+    assert connection.log.call_args_list == [
+        call(
+            "Terminating the agent process...",
+            level=LogLevel.INFO,
+            source=LogSource.BRIDGE,
+        ),
+        call(
+            "Agent process shutdown gracefully",
+            level=LogLevel.INFO,
+            source=LogSource.BRIDGE,
+        ),
+        call(
+            "Isolate agent gracefully terminated",
+            level=LogLevel.INFO,
+            source=LogSource.BRIDGE,
+        ),
+    ]
     assert connection._process is None
 
 
@@ -73,11 +85,23 @@ def test_abort_agent_logs_return_code_after_kill_fallback(tmp_path: Path) -> Non
     process.terminate.assert_called_once()
     process.kill.assert_called_once()
     process.wait.assert_called_once()
-    connection.log.assert_called_once_with(
-        "Isolate agent forcefully killed",
-        level=LogLevel.INFO,
-        source=LogSource.BRIDGE,
-    )
+    assert connection.log.call_args_list == [
+        call(
+            "Terminating the agent process...",
+            level=LogLevel.INFO,
+            source=LogSource.BRIDGE,
+        ),
+        call(
+            "Failed to shutdown the agent process gracefully: terminate failed",
+            level=LogLevel.ERROR,
+            source=LogSource.BRIDGE,
+        ),
+        call(
+            "Isolate agent forcefully killed",
+            level=LogLevel.INFO,
+            source=LogSource.BRIDGE,
+        ),
+    ]
     assert connection._process is None
 
 

isolate-0.26.6/tests/test_logged_io_pipe.py

@@ -0,0 +1,132 @@
+"""Regression tests for the pipe wiring used by `logged_io`.
+
+Background: `_unblocked_pipe()` is consumed by `logged_io()`, where the
+*read* end is drained from a reader thread (`_io_observer`) using
+`select()` + `os.read()`. Only the read end needs to be non-blocking
+(so the post-`select` read can never hang on a partial-line edge case).
+
+Historically both ends were marked non-blocking. That makes the *write*
+end raise `BlockingIOError` (errno EAGAIN) the moment the kernel pipe
+buffer (~64 KB on Linux) fills up before the reader thread drains it —
+which is exactly what happens when a worker emits a chunky traceback.
+With a blocking write end, writers apply backpressure instead.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import threading
+
+import pytest
+from isolate.backends.common import _unblocked_pipe, logged_io
+
+
+def test_unblocked_pipe_write_end_is_blocking():
+    """The write end must be blocking so that worker stdout/stderr applies
+    backpressure instead of raising BlockingIOError when the pipe buffer fills."""
+    read_fd, write_fd = _unblocked_pipe()
+    try:
+        assert not os.get_blocking(read_fd), "read end must be non-blocking"
+        assert os.get_blocking(write_fd), (
+            "write end must be blocking so the worker gets backpressure "
+            "instead of BlockingIOError when the pipe buffer fills up"
+        )
+    finally:
+        os.close(read_fd)
+        os.close(write_fd)
+
+
+def test_writing_more_than_pipe_buffer_does_not_raise_blocking_io_error():
+    """Direct reproducer: write more than the kernel pipe buffer (64 KB on
+    Linux) through a Python BufferedWriter on top of the pipe's write end,
+    while a reader thread drains it slowly. With a non-blocking write end
+    this raises `BlockingIOError`; with a blocking write end the writer
+    transparently waits for the reader to make room.
+    """
+    read_fd, write_fd = _unblocked_pipe()
+    drained = bytearray()
+    drain_done = threading.Event()
+
+    def drain():
+        # Slow reader: read in small chunks with a `select` loop, mimicking
+        # `_io_observer`'s pattern but slower so the pipe buffer fills up.
+        import select as _select
+
+        while not drain_done.is_set():
+            ready, _, _ = _select.select([read_fd], [], [], 0.05)
+            if read_fd in ready:
+                try:
+                    chunk = os.read(read_fd, 4096)
+                except BlockingIOError:
+                    continue
+                if not chunk:
+                    return
+                drained.extend(chunk)
+
+    reader = threading.Thread(target=drain)
+    reader.start()
+    try:
+        # 512 KB is well past the 64 KB pipe buffer on Linux.
+        payload = b"x" * (512 * 1024)
+        with open(write_fd, "wb", closefd=False) as stream:
+            stream.write(payload)
+            stream.flush()
+        # Give the reader a moment to catch up.
+        deadline_loops = 50
+        while len(drained) < len(payload) and deadline_loops > 0:
+            deadline_loops -= 1
+            drain_done.wait(0.05)
+    finally:
+        drain_done.set()
+        os.close(write_fd)
+        reader.join(timeout=2)
+        os.close(read_fd)
+
+    assert len(drained) == len(
+        payload
+    ), f"reader only drained {len(drained)} of {len(payload)} bytes"
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="pipes / pass_fds are POSIX-y")
+def test_logged_io_subprocess_emitting_large_output_does_not_crash():
+    """End-to-end reproducer mirroring how `logged_io` is used in
+    `isolate.connections._local._base`: a child process writes a chunky
+    payload (~256 KB) to stdout. With a non-blocking write end the child's
+    Python runtime raises `BlockingIOError` when flushing stdout; with
+    backpressure the child completes cleanly and the parent receives
+    every line."""
+
+    captured: list[str] = []
+    lock = threading.Lock()
+
+    def record(line: str) -> None:
+        with lock:
+            captured.append(line)
+
+    n_lines = 4000
+    line_template = "trace-{idx:08d}-" + ("y" * 50)
+    child_script = (
+        "import sys\n"
+        f"for i in range({n_lines}):\n"
+        f"    sys.stdout.write({line_template!r}.format(idx=i) + '\\n')\n"
+        "sys.stdout.flush()\n"
+    )
+
+    with logged_io(record) as (stdout, stderr, log_fd):
+        proc = subprocess.run(
+            [sys.executable, "-u", "-c", child_script],
+            stdout=stdout,
+            stderr=stderr,
+            pass_fds=(log_fd,),
+            text=True,
+            check=False,
+        )
+
+    assert proc.returncode == 0, (
+        f"child exited with {proc.returncode}; with a non-blocking write end "
+        "Python's stdout buffer raises BlockingIOError on large flushes"
+    )
+    # All `n_lines` should have been captured by the observer thread.
+    assert len(captured) == n_lines, f"expected {n_lines} lines, got {len(captured)}"