geny-executor 2.65.0__tar.gz → 2.65.2__tar.gz

{geny_executor-2.65.0 → geny_executor-2.65.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geny-executor
-Version: 2.65.0
+Version: 2.65.2
 Summary: Harness-engineered agent pipeline library with 21-stage dual-abstraction architecture, built on the Anthropic API
 Project-URL: Homepage, https://github.com/CocoRoF/geny-executor
 Project-URL: Repository, https://github.com/CocoRoF/geny-executor

{geny_executor-2.65.0 → geny_executor-2.65.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "geny-executor"
-version = "2.65.0"
+version = "2.65.2"
 description = "Harness-engineered agent pipeline library with 21-stage dual-abstraction architecture, built on the Anthropic API"
 readme = "README.md"
 license = "Apache-2.0"

{geny_executor-2.65.0 → geny_executor-2.65.2}/src/geny_executor/llm_client/_cli_runtime.py

@@ -71,6 +71,12 @@ logger = logging.getLogger(__name__)
 # discarded, so the line is unrecoverable). Delegated heavy work makes
 # large tool results routine, so the default is deliberately generous:
 # 32 MiB (a cap, not an allocation — memory is used only per actual line).
+#: How often the stream reader looks up from the pipe to check whether
+#: the child is still alive. Only ever costs a wakeup when no line has
+#: arrived, so a busy stream never pays it.
+_EXIT_POLL_S = 0.25
+
+
 def _cli_stream_limit() -> int:
     raw = os.environ.get("GENY_CLI_STREAM_LIMIT", "").strip()
     try:
@@ -221,6 +227,22 @@ class CLIProcessRunner:
     cwd: Optional[str] = None
     timeout_s: float = 300.0
     kill_grace_s: float = 2.0
+    #: How long to keep reading stdout AFTER the child has exited.
+    #:
+    #: A pipe reaches EOF when the last writer closes it — which is not
+    #: the same event as "the child exited". The CLI spawns MCP servers
+    #: as its own children, and they inherit its stdout; one that
+    #: outlives it (or ignores the parent-death signal) holds the write
+    #: end open, so ``readline()`` blocks on a pipe nothing will ever
+    #: write to again. Waiting on EOF alone therefore parks the turn for
+    #: the FULL ``timeout_s`` — 2026-08-10 in production that was a dead
+    #: CLI (rc=0, output complete) and a turn that hung until a
+    #: host-side stall guard abandoned it minutes later.
+    #:
+    #: After exit, anything still in flight is bytes already written to
+    #: the pipe buffer, which drain immediately. This grace is generous
+    #: for that and short enough that a leaked FD costs seconds.
+    exit_drain_grace_s: float = 5.0
 
     def __post_init__(self) -> None:
         if not self.binary:
@@ -301,9 +323,15 @@ class CLIProcessRunner:
         stderr_task = asyncio.create_task(_collect_stderr(proc.stderr, stderr_buf))
 
         try:
-            async for line in _aiter_lines(proc.stdout, timeout_s=self.timeout_s, start_t=t0):
+            async for line in _aiter_lines(
+                proc.stdout,
+                timeout_s=self.timeout_s,
+                start_t=t0,
+                proc=proc,
+                drain_grace_s=self.exit_drain_grace_s,
+            ):
                 yield line
-            rc = await proc.wait()
+            rc = await self._reap(proc)
         except CLITimeout:
             await self._kill_tree(proc)
             raise
@@ -355,10 +383,50 @@ class CLIProcessRunner:
             timeout=self.timeout_s,
         )
 
+    # ------------------------------------------------------------- reap
+    async def _reap(self, proc: asyncio.subprocess.Process) -> int:
+        """Exit status, without betting the turn on ``proc.wait()``.
+
+        ``wait()`` completes only when the child has exited AND every
+        pipe has disconnected. A survivor holding the inherited stdout
+        satisfies the first and blocks the second forever, so awaiting
+        it here would re-introduce exactly the hang the read loop just
+        escaped.
+
+        So: give ``wait()`` a short window, and if it does not land,
+        take the returncode the child watcher already recorded and kill
+        the process group — which is also what finally releases the
+        leaked descriptor.
+        """
+        try:
+            return await asyncio.wait_for(proc.wait(), timeout=self.kill_grace_s)
+        except asyncio.TimeoutError:
+            pass
+        rc = proc.returncode
+        if rc is None:
+            # Still genuinely running — the caller's ladder owns it.
+            return -1
+        logger.warning(
+            "CLI exited rc=%s but a pipe stayed open — killing the process "
+            "group to release it", rc,
+        )
+        await self._kill_tree(proc, force=True)
+        return rc
+
     # ------------------------------------------------------------- kill
-    async def _kill_tree(self, proc: asyncio.subprocess.Process) -> None:
-        """Send SIGTERM, wait grace, then SIGKILL the process group."""
-        if proc.returncode is not None:
+    async def _kill_tree(
+        self, proc: asyncio.subprocess.Process, *, force: bool = False
+    ) -> None:
+        """Send SIGTERM, wait grace, then SIGKILL the process group.
+
+        ``force`` keeps going when the direct child is already reaped:
+        the group can still hold survivors it spawned — an MCP server
+        sitting on the inherited stdout — and those are precisely what
+        needs signalling. Without it the early return below reads "the
+        child is gone, nothing to kill", which is exactly wrong for the
+        case that leaks.
+        """
+        if proc.returncode is not None and not force:
             return
         try:
             if sys.platform != "win32":
@@ -512,33 +580,92 @@ async def _aiter_lines(
     *,
     timeout_s: float,
     start_t: float,
+    proc: Optional[asyncio.subprocess.Process] = None,
+    drain_grace_s: float = 5.0,
 ) -> AsyncIterator[bytes]:
+    """Yield stdout lines until EOF — or until the child is gone and quiet.
+
+    Once the child has exited, EOF is no longer guaranteed to arrive at
+    all: an inherited write end can keep the pipe open indefinitely. So
+    the read budget collapses to ``drain_grace_s`` from the moment the
+    exit is observed, and the iterator ends on the first quiet moment
+    instead of parking on ``timeout_s``. Bytes already in the pipe are
+    still delivered — draining a buffer is instant next to that grace.
+    """
     if stream is None:
         return
-    while True:
-        elapsed = time.monotonic() - start_t
-        remaining = timeout_s - elapsed
-        if remaining <= 0:
-            raise CLITimeout(f"stream timeout after {timeout_s:.1f}s")
-        try:
-            line = await asyncio.wait_for(stream.readline(), timeout=remaining)
-        except asyncio.TimeoutError as e:
-            raise CLITimeout(f"stream readline timeout after {timeout_s:.1f}s") from e
-        except ValueError as e:
-            # asyncio raises ValueError("Separator is found, but chunk is
-            # longer than limit") when ONE line exceeds the StreamReader
-            # limit — and discards the buffered bytes, so the line is
-            # unrecoverable. With the 32 MiB default this is near
-            # impossible; if it still happens, losing ONE event beats
-            # killing the whole delegated turn. Log loudly and continue.
-            logger.warning(
-                "CLI stream line exceeded the %d-byte limit — skipping one "
-                "event and continuing (%s)", _cli_stream_limit(), e,
+    read_task: Optional[asyncio.Task[bytes]] = None
+    died_at: Optional[float] = None
+    try:
+        while True:
+            now = time.monotonic()
+            remaining = timeout_s - (now - start_t)
+            if remaining <= 0:
+                raise CLITimeout(f"stream timeout after {timeout_s:.1f}s")
+
+            # One read task, carried across iterations: a readline that
+            # loses the race must NOT be discarded, or the bytes it is
+            # mid-way through consuming are lost.
+            if read_task is None:
+                read_task = asyncio.ensure_future(stream.readline())
+
+            if died_at is None and proc is not None and proc.returncode is not None:
+                died_at = now
+            if died_at is not None:
+                grace_left = drain_grace_s - (now - died_at)
+                if grace_left <= 0:
+                    logger.warning(
+                        "CLI exited but stdout stayed open for %.1fs — ending "
+                        "the stream (an inherited pipe write end is still "
+                        "held; the answer is complete, the FD is not).",
+                        drain_grace_s,
+                    )
+                    return
+                budget = min(remaining, grace_left)
+            else:
+                # Poll rather than await the child: ``proc.wait()`` cannot
+                # be the death signal here, because asyncio only completes
+                # it once every pipe has ALSO disconnected — which is the
+                # exact condition a leaked stdout FD prevents. The
+                # returncode, by contrast, is set the moment the child
+                # watcher reaps, pipes or no pipes.
+                budget = min(remaining, _EXIT_POLL_S)
+
+            done, _pending = await asyncio.wait(
+                {read_task}, timeout=budget, return_when=asyncio.FIRST_COMPLETED
             )
-            continue
-        if not line:
-            return
-        yield line
+
+            if read_task not in done:
+                if died_at is not None or budget < remaining:
+                    # Either draining after exit, or just a poll tick with
+                    # budget left — go round again.
+                    continue
+                raise CLITimeout(f"stream readline timeout after {timeout_s:.1f}s")
+
+            try:
+                line = read_task.result()
+            except ValueError as e:
+                # asyncio raises ValueError("Separator is found, but chunk is
+                # longer than limit") when ONE line exceeds the StreamReader
+                # limit — and discards the buffered bytes, so the line is
+                # unrecoverable. With the 32 MiB default this is near
+                # impossible; if it still happens, losing ONE event beats
+                # killing the whole delegated turn. Log loudly and continue.
+                logger.warning(
+                    "CLI stream line exceeded the %d-byte limit — skipping one "
+                    "event and continuing (%s)", _cli_stream_limit(), e,
+                )
+                read_task = None
+                continue
+            finally:
+                if read_task is not None and read_task.done():
+                    read_task = None
+            if not line:
+                return
+            yield line
+    finally:
+        if read_task is not None and not read_task.done():
+            read_task.cancel()
 
 
 async def _drain_stdin(

{geny_executor-2.65.0 → geny_executor-2.65.2}/src/geny_executor/llm_client/claude_code.py

@@ -113,6 +113,29 @@ def _classify_cli_result(result: CLIResult, *, cli_version: str = "") -> APIErro
     )
 
 
+def _process_alive(proc: Any) -> bool:
+    """Is this child still running, really?
+
+    ``asyncio.subprocess.Process.returncode`` is bookkeeping: it stays None
+    until the event loop reaps the child. A process that already exited can
+    therefore look alive indefinitely. Ask the kernel instead.
+    """
+    if getattr(proc, "returncode", None) is not None:
+        return False
+    pid = getattr(proc, "pid", None)
+    if not pid:
+        return False
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True          # exists, owned by someone else — still alive
+    except OSError:
+        return False
+    return True
+
+
 class ClaudeCodeCLIClient(BaseClient):
     """Subprocess-backed Claude Code client."""
 
@@ -276,7 +299,20 @@ class ClaudeCodeCLIClient(BaseClient):
         The spare is only valid for an IDENTICAL argv (model, MCP config,
         session resume flags, permissions — everything). Any drift means
         the prewarmed process was booted with stale config: discard it
-        and spawn fresh. Also discards a spare that died while idle.
+        and spawn fresh.
+
+        It must also still be ALIVE, and ``returncode`` is not enough to
+        know that. It is only set once asyncio has reaped the child; a
+        process that exited without the transport noticing keeps
+        ``returncode is None`` forever, so a corpse reads as a healthy
+        spare. The turn then hands its prompt to a dead pipe and waits —
+        which is exactly what happened in production: the CLI started,
+        listed its tools, exited, and every subsequent turn stalled until a
+        watchdog abandoned it. Turning the prewarm off made the same
+        session answer in 11 s.
+
+        ``kill(pid, 0)`` costs a syscall and answers the question the
+        bookkeeping cannot.
         """
         spare = self._spare
         if spare is None:
@@ -286,7 +322,7 @@ class ClaudeCodeCLIClient(BaseClient):
         if expire_task is not None:
             expire_task.cancel()
         proc = spare["proc"]
-        if spare["argv"] != list(argv) or proc.returncode is not None:
+        if spare["argv"] != list(argv) or not _process_alive(proc):
             self._discard_spare_proc(spare)
             return None
         return proc

{geny_executor-2.65.0 → geny_executor-2.65.2}/tests/_fixtures/fake_claude.py

@@ -474,6 +474,32 @@ def _stream_then_hang(argv: List[str]) -> int:
     return 0
 
 
+def _leaky_child_then_exit(argv: List[str]) -> int:
+    """Answer in full, exit 0 — and leave a child holding stdout open.
+
+    This is the production failure of 2026-08-10, reduced. The real CLI
+    spawns MCP servers as its own children; they inherit its stdout. When
+    the CLI exits, the pipe does NOT reach EOF, because the write end is
+    still open in the survivor. A reader that waits for EOF alone waits
+    forever on a process that finished minutes ago — the turn hangs with
+    a complete answer sitting in it.
+
+    The forked child holds the FD and sleeps. It writes nothing: the
+    point is the open descriptor, not the data.
+    """
+    pid = os.fork()
+    if pid == 0:  # child — inherits stdout, outlives the parent
+        try:
+            os.setsid()
+        except OSError:
+            pass
+        time.sleep(600)
+        os._exit(0)
+    rc = _ok_stream_event(argv)
+    sys.stdout.flush()
+    return rc
+
+
 def _permission_fail(argv: List[str]) -> int:
     sys.stderr.write("permission denied: tool Bash blocked by policy\n")
     return 1
@@ -520,6 +546,7 @@ SCENARIOS = {
     "ok_result_envelope": _ok_result_envelope,
     "stream_unknown_lines": _stream_unknown_lines,
     "stream_then_hang": _stream_then_hang,
+    "leaky_child_then_exit": _leaky_child_then_exit,
     "message_form_auth_failed": _message_form_auth_failed,
     "auth_fail": _auth_fail,
     "permission_fail": _permission_fail,

geny_executor-2.65.2/tests/llm_client/unit/test_cli_leaked_stdout.py

@@ -0,0 +1,135 @@
+"""A dead CLI must not be able to park a turn on its own stdout.
+
+Production, 2026-08-10. Turns stopped answering. The stack said the
+pipeline was inside ``_call_streaming``, awaiting the client's async
+generator; the CLI trace said the very same invocation had exited
+``rc=0`` with a complete answer, seconds in. Both were true.
+
+A pipe reaches EOF when the LAST writer closes it, and the CLI is not
+the only writer: it spawns MCP servers as children and they inherit its
+stdout. One that outlives it — a bridge that misses the parent-death
+signal, a server mid-request — holds the write end open, and
+``readline()`` blocks on a pipe with nothing left to write to it. The
+reader waited for EOF, so it waited for the full ``timeout_s``: 30
+minutes of spinner over an answer that was already complete.
+
+The fix reaps the child concurrently and, once it is gone, gives stdout
+only ``exit_drain_grace_s`` to produce anything more. Buffered bytes
+still arrive — draining a buffer is instant next to that grace.
+
+``leaky_child_then_exit`` reproduces the shape exactly: full stream-json
+answer, clean exit, one forked child sitting on the inherited FD.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import signal
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "src"))
+
+import pytest
+
+from geny_executor.llm_client._cli_runtime import CLIProcessRunner
+
+
+FAKE_CLAUDE = str(
+    (Path(__file__).resolve().parents[2] / "_fixtures" / "fake_claude.py")
+)
+
+STREAM_ARGV = ["--print", "--output-format", "stream-json"]
+
+
+def _runner(scenario: str, **kwargs) -> CLIProcessRunner:
+    defaults = dict(binary=FAKE_CLAUDE, timeout_s=30.0, kill_grace_s=0.5)
+    defaults.update(kwargs)
+    return CLIProcessRunner(
+        env_extras={"FAKE_CLAUDE_SCENARIO": scenario}, **defaults
+    )
+
+
+async def _collect(runner: CLIProcessRunner) -> list[bytes]:
+    return [line async for line in runner.stream(STREAM_ARGV)]
+
+
+@pytest.mark.asyncio
+async def test_exited_child_with_leaked_stdout_ends_the_stream() -> None:
+    """The stream ends on the drain grace, not on ``timeout_s``.
+
+    ``timeout_s`` is 30s and the grace 1s; the leaked child sleeps 600s.
+    Before the fix this test hung for the full 30s and then raised
+    ``CLITimeout``, losing a complete answer.
+    """
+    runner = _runner("leaky_child_then_exit", timeout_s=30.0,
+                     exit_drain_grace_s=1.0)
+    t0 = time.monotonic()
+    lines = await asyncio.wait_for(_collect(runner), timeout=20.0)
+    elapsed = time.monotonic() - t0
+
+    assert elapsed < 10.0, f"stream waited {elapsed:.1f}s on a dead child"
+    # The answer survived: ending early must not truncate delivered output.
+    assert lines, "no output collected"
+    joined = b"".join(lines)
+    assert b"content_block_delta" in joined or b"assistant" in joined
+
+
+@pytest.mark.asyncio
+async def test_drain_grace_is_measured_from_child_exit() -> None:
+    """A longer grace costs exactly that much more, and no more.
+
+    Pins the shape of the wait: the cost of a leaked FD is the grace,
+    not the timeout — which is the whole point of the change.
+    """
+    runner = _runner("leaky_child_then_exit", timeout_s=30.0,
+                     exit_drain_grace_s=3.0)
+    t0 = time.monotonic()
+    await asyncio.wait_for(_collect(runner), timeout=20.0)
+    elapsed = time.monotonic() - t0
+    assert 2.0 <= elapsed < 12.0, f"grace not honoured (took {elapsed:.1f}s)"
+
+
+@pytest.mark.asyncio
+async def test_healthy_stream_is_unaffected() -> None:
+    """No grandchild, no grace: a clean CLI still ends on real EOF.
+
+    The guard must not shorten a normal stream — the common path has to
+    stay exactly as fast as it was.
+    """
+    runner = _runner("ok_stream_event", timeout_s=30.0,
+                     exit_drain_grace_s=5.0)
+    t0 = time.monotonic()
+    lines = await asyncio.wait_for(_collect(runner), timeout=20.0)
+    elapsed = time.monotonic() - t0
+    assert lines
+    assert elapsed < 5.0, (
+        f"clean stream paid the drain grace ({elapsed:.1f}s) — it should "
+        "end on EOF immediately"
+    )
+
+
+@pytest.mark.asyncio
+async def test_slow_but_alive_child_is_not_cut_off() -> None:
+    """The grace applies only AFTER exit — a slow live child keeps its
+    full budget.
+
+    ``stream_then_hang`` emits one delta and sleeps 600s while alive, so
+    the only thing that may stop it is ``timeout_s``. If the drain grace
+    leaked into the alive path, this would end in ~1s with a CLITimeout
+    raised far too early.
+    """
+    from geny_executor.llm_client._cli_runtime import CLITimeout
+
+    runner = _runner("stream_then_hang", timeout_s=4.0,
+                     exit_drain_grace_s=1.0)
+    t0 = time.monotonic()
+    with pytest.raises(CLITimeout):
+        await asyncio.wait_for(_collect(runner), timeout=25.0)
+    elapsed = time.monotonic() - t0
+    assert elapsed >= 3.0, (
+        f"alive child was cut off after {elapsed:.1f}s — the drain grace "
+        "must not apply before exit"
+    )

geny_executor-2.65.2/tests/unit/test_cli_hot_spare_liveness.py

@@ -0,0 +1,101 @@
+"""The hot spare must be alive, not merely un-reaped.
+
+The prewarm exists for time-to-first-token: after a turn, a `claude` process
+is started so the next turn skips Node boot, auth and MCP startup. The turn
+then hands its prompt to that process over stdin.
+
+`Process.returncode` is bookkeeping — it stays None until the event loop
+reaps the child — so a process that already exited reads as a healthy spare
+forever. In production the CLI started, listed its tools, exited, and every
+following turn stalled until a watchdog abandoned it at 320 s. The same
+session answered in 11 s with the prewarm disabled.
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from geny_executor.llm_client.claude_code import (
+    ClaudeCodeCLIClient,
+    _process_alive,
+)
+
+
+class _Proc:
+    def __init__(self, pid, returncode=None):
+        self.pid = pid
+        self.returncode = returncode
+
+
+def _client() -> ClaudeCodeCLIClient:
+    return ClaudeCodeCLIClient.__new__(ClaudeCodeCLIClient)
+
+
+def _with_spare(argv, proc):
+    c = _client()
+    c._spare = {"proc": proc, "argv": list(argv), "runner": None}
+    c._discarded = []
+    c._discard_spare_proc = lambda spare: c._discarded.append(spare)
+    return c
+
+
+# ── the liveness probe itself ───────────────────────────────────────
+
+def test_a_running_process_is_alive():
+    assert _process_alive(_Proc(os.getpid())) is True
+
+
+def test_a_reaped_process_is_not():
+    assert _process_alive(_Proc(os.getpid(), returncode=0)) is False
+
+
+def test_a_vanished_process_is_not_alive_despite_no_returncode():
+    """THE bug. returncode is None because nobody reaped it; the process is
+    gone all the same."""
+    pid = os.fork()
+    if pid == 0:                      # child
+        os._exit(0)
+    os.waitpid(pid, 0)                # reap outside asyncio's view
+
+    assert _process_alive(_Proc(pid)) is False
+
+
+def test_no_pid_is_not_alive():
+    assert _process_alive(_Proc(None)) is False
+
+
+# ── what _take_spare does with it ───────────────────────────────────
+
+def test_a_live_matching_spare_is_handed_over():
+    proc = _Proc(os.getpid())
+    c = _with_spare(["claude", "--model", "opus"], proc)
+
+    assert c._take_spare(["claude", "--model", "opus"]) is proc
+    assert c._spare is None, "the spare was handed over AND kept"
+
+
+def test_a_dead_spare_is_discarded_not_reused():
+    pid = os.fork()
+    if pid == 0:
+        os._exit(0)
+    os.waitpid(pid, 0)
+    c = _with_spare(["claude"], _Proc(pid))
+
+    assert c._take_spare(["claude"]) is None
+    assert len(c._discarded) == 1
+
+
+def test_a_spare_booted_with_different_argv_is_discarded():
+    """Model, MCP config, resume flags — any drift means stale config."""
+    c = _with_spare(["claude", "--model", "opus"], _Proc(os.getpid()))
+
+    assert c._take_spare(["claude", "--model", "haiku"]) is None
+    assert len(c._discarded) == 1
+
+
+def test_no_spare_is_not_an_error():
+    c = _client()
+    c._spare = None
+    assert c._take_spare(["claude"]) is None