agentic-runner 2.6.0__tar.gz → 2.6.1__tar.gz

{agentic_runner-2.6.0 → agentic_runner-2.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: agentic-runner
-Version: 2.6.0
+Version: 2.6.1
 Summary: The Agentic OS Runner: an activity-only Temporal executor for workspaces, Agent Runtimes and verb seams
 License-Expression: AGPL-3.0-only
 License-File: LICENSE

{agentic_runner-2.6.0 → agentic_runner-2.6.1}/src/agentic_runner/__init__.py

@@ -7,6 +7,6 @@ stay on the platform; this package imports none of them, and a CI partition test
 (``tests/test_package_partition.py``) fails the build if that changes.
 """
 
-__version__ = "2.6.0"
+__version__ = "2.6.1"
 
 __all__ = ["__version__"]

{agentic_runner-2.6.0 → agentic_runner-2.6.1}/src/agentic_runner/activities.py

@@ -38,6 +38,7 @@ from uuid import UUID
 from temporalio import activity
 from temporalio.exceptions import ApplicationError
 
+from agentic_runner.attempts import AttemptRecords, PriorAttemptAliveError, fence_work_record
 from agentic_runner.callback import (
     CALLBACK_TOKEN_ENV,
     AnnotateRequest,
@@ -1321,6 +1322,7 @@ def wipe_contract_residue(
 
 # Where a Runner-side routing refusal lands in the trail (PRD issue 42, 17 A11).
 _ROUTING_EVIDENCE_SOURCE = "runner.routing"
+PRIOR_ATTEMPT_ALIVE_EVIDENCE_SOURCE = "runner.prior_attempt_alive"
 DIRECTIVE_TOKEN_EVIDENCE_SOURCE = "runner.directive_token"
 AGENT_RUNTIME_EVIDENCE_SOURCE = "runner.agent_runtime"
 
@@ -1435,6 +1437,7 @@ class RunnerRalphActivities:
         mcp_spawner: Spawner | None = None,
         egress_resolver: Resolver | None = None,
         tool_server_health: ToolServerHealthLog | None = None,
+        attempt_records: AttemptRecords | None = None,
     ) -> None:
         self._fastapi_client = fastapi_client
         # Wall-clock source for per-Directive runtime, accrued against the Budget's
@@ -1494,6 +1497,9 @@ class RunnerRalphActivities:
         self._mcp_spawner = mcp_spawner
         self._egress_resolver = egress_resolver
         self._tool_server_health = tool_server_health
+        # The retry fence (runner-repo 05). None fences nothing: a process that never
+        # registered has no state directory to keep the records in.
+        self._attempt_records = attempt_records
 
     def activity_callables(self) -> list[Callable[..., Any]]:
         """Return decorated activity callables for Temporal worker registration."""
@@ -1561,6 +1567,37 @@ class RunnerRalphActivities:
             else "runner"
         )
 
+    @contextlib.asynccontextmanager
+    async def _fenced(self, work_record_id: str) -> AsyncIterator[None]:
+        """Refuse this attempt's spawns while an earlier attempt's group lives (RR-05).
+
+        The refusal is retryable on purpose: the earlier attempt either finishes or is
+        killed, and the next retry then finds the Workspace free.
+        """
+
+        if self._attempt_records is None:
+            yield
+            return
+        attempt = activity.info().attempt if activity.in_activity() else 1
+        try:
+            with fence_work_record(self._attempt_records, work_record_id, attempt=attempt):
+                yield
+        except PriorAttemptAliveError as refusal:
+            await _raise_recorded(
+                refusal,
+                self._fastapi_client,
+                work_record_id,
+                source=PRIOR_ATTEMPT_ALIVE_EVIDENCE_SOURCE,
+                actor=self._actor(),
+                payload={
+                    "event": "directive.prior_attempt_alive",
+                    "work_record_id": work_record_id,
+                    "attempt": attempt,
+                    "prior_attempt": refusal.prior.attempt,
+                    "prior_pgid": refusal.prior.pgid,
+                },
+            )
+
     @activity.defn(name="create_or_update_branch_pr")
     async def create_or_update_branch_pr(
         self,
@@ -1570,7 +1607,9 @@ class RunnerRalphActivities:
         await self._assert_routed(request)
         asked: list[QuestionAsked] = []
         try:
-            return _with_question(await self._run_branch_pr_directive(request, asked), asked)
+            async with self._fenced(request.work_record_id):
+                ran = await self._run_branch_pr_directive(request, asked)
+            return _with_question(ran, asked)
         except HookRefusedError:
             # A Runner Hook at or before `pre_runtime` exited non-zero, `pre_directive`
             # being the reject gate (PRD issue 45). Refusing the Directive is the point,
@@ -3811,7 +3850,7 @@ class RunnerRalphActivities:
     async def run_verifier(self, request: VerifierRunInput) -> VerifierRunOutput:
         """Run product-owned verifier config and record bounded redacted evidence."""
         await self._assert_routed(request)
-        async with _liveness_heartbeats():
+        async with _liveness_heartbeats(), self._fenced(request.work_record_id):
             if self._runtime_dependencies_ready():
                 state = await self._runtime_state(
                     request.work_record_id, workspace_path=request.workspace_path
@@ -3909,7 +3948,7 @@ class RunnerRalphActivities:
                     sandbox=verifier_sandbox,
                 )
                 try:
-                    # Run in a thread: subprocess.run inline would block the worker
+                    # Run in a thread: waiting on the verifier inline would block the worker
                     # event loop for up to the verifier timeout, starving the liveness
                     # heartbeats above (and every other activity on this pod).
                     result = await asyncio.to_thread(
@@ -3981,7 +4020,8 @@ class RunnerRalphActivities:
         repair a verifier failure, then re-push. Exactly one runtime turn (ADR-0007)."""
         await self._assert_routed(request)
         try:
-            return await self._run_fix_directive(request)
+            async with self._fenced(request.work_record_id):
+                return await self._run_fix_directive(request)
         except HookRefusedError as refusal:
             # Same posture as the first Directive: a hook refusal ends the loop
             # attributably instead of raising into an endless retry (PRD issue 45).
@@ -4071,25 +4111,26 @@ class RunnerRalphActivities:
             )
 
         try:
-            ran = await self._run_in_place_directive(
-                request,
-                _InPlaceDirective(
-                    lifecycle_source="ralph.execute_member_directive",
-                    run_source="ralph.member_directive.codex_run",
-                    push_source="ralph.member_directive.git_commit_push",
-                    commit_message=(
-                        f"feat(task): directive {request.directive_number} by Agent "
-                        f"{request.agent_id} for work record {request.work_record_id}"
-                    ),
-                    push_summary=(
-                        f"Push Directive {request.directive_number} ({request.role or 'member'}"
-                        f", woken by {request.wake_reason or 'pending messages'}) to "
-                        f"{request.repository}."
+            async with self._fenced(request.work_record_id):
+                ran = await self._run_in_place_directive(
+                    request,
+                    _InPlaceDirective(
+                        lifecycle_source="ralph.execute_member_directive",
+                        run_source="ralph.member_directive.codex_run",
+                        push_source="ralph.member_directive.git_commit_push",
+                        commit_message=(
+                            f"feat(task): directive {request.directive_number} by Agent "
+                            f"{request.agent_id} for work record {request.work_record_id}"
+                        ),
+                        push_summary=(
+                            f"Push Directive {request.directive_number} ({request.role or 'member'}"
+                            f", woken by {request.wake_reason or 'pending messages'}) to "
+                            f"{request.repository}."
+                        ),
+                        prompt=prompt,
+                        push_when_clean=False,
                     ),
-                    prompt=prompt,
-                    push_when_clean=False,
-                ),
-            )
+                )
         except HookRefusedError as refusal:
             return MemberDirectiveOutput(
                 work_record_id=request.work_record_id,
@@ -4163,7 +4204,8 @@ class RunnerRalphActivities:
                 summary="directive unavailable: runtime dependencies missing",
             )
         try:
-            return await self._run_learning_directive(request)
+            async with self._fenced(request.work_record_id):
+                return await self._run_learning_directive(request)
         except HookRefusedError as refusal:
             return self._learning_output(
                 request,

agentic_runner-2.6.1/src/agentic_runner/attempts.py

@@ -0,0 +1,258 @@
+"""One live process group per Work Record, fenced across activity attempts (runner-repo 05).
+
+A lost heartbeat makes Temporal schedule a retry while the first attempt's coroutine,
+and the harness it spawned, may still be running in this same process. Both would then
+write one Workspace, which ADR-0015 §2 and ADR-0012 assume never happens. So every process
+group a Work Record's Directive, verifier run or Runner Hook spawns is recorded as
+``{pgid, start_time, attempt}`` under the Runner's state directory, and the next spawn
+for that Work Record is refused while the recorded group's leader is still alive.
+
+Liveness is pid *plus* start time, as Paperclip fences its restarts: a pid alone is
+recycled, and a record naming a recycled pid must not block a Work Record forever. The
+record lives in the state directory, never the Workspace, because the Agent writes the
+Workspace and could forge or delete its own fence there.
+
+The binding is a context variable rather than a parameter: the spawn sites sit behind
+the Agent Runtime port, the Runner Hook runner and the verifier's command runner, and
+none of them knows which Work Record it serves. ``asyncio.to_thread`` copies the
+context, so the verifier's thread sees the activity's binding too.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import re
+import subprocess
+import sys
+from collections.abc import Iterator
+from contextvars import ContextVar
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Final
+
+__all__ = [
+    "ATTEMPTS_DIR",
+    "AttemptRecord",
+    "AttemptRecords",
+    "PriorAttemptAliveError",
+    "fence_work_record",
+    "process_start_time",
+    "recorded_group",
+    "refuse_if_prior_attempt_alive",
+]
+
+ATTEMPTS_DIR: Final[str] = "attempts"
+_DIR_MODE: Final[int] = 0o700
+_FILE_MODE: Final[int] = 0o600
+# A Work Record id names a file here, so it must not be able to name one anywhere else.
+_SAFE_ID: Final[re.Pattern[str]] = re.compile(r"[A-Za-z0-9_-][A-Za-z0-9._-]*")
+
+
+@dataclass(frozen=True, slots=True)
+class AttemptRecord:
+    pgid: int
+    start_time: str
+    attempt: int
+
+
+class PriorAttemptAliveError(Exception):
+    """A spawn was refused: an earlier attempt's process group for this Work Record lives."""
+
+    def __init__(self, work_record_id: str, prior: AttemptRecord) -> None:
+        super().__init__(
+            f"work record {work_record_id}: attempt {prior.attempt} still runs "
+            f"(process group {prior.pgid})"
+        )
+        self.work_record_id = work_record_id
+        self.prior = prior
+
+
+def process_start_time(pid: int) -> str | None:
+    """When ``pid`` started, in the platform's own units, or None if no such process.
+
+    Only ever compared for equality with an earlier reading of the same pid on the same
+    host, so the units need not agree across platforms: clock ticks since boot on Linux,
+    the ``lstart`` text on macOS (one-second resolution, which a pid recycled within the
+    same second would defeat -- not a case a harness spawn produces).
+    """
+
+    if sys.platform == "linux":
+        return _linux_start_time(pid)
+    return _ps_start_time(pid)
+
+
+def _linux_start_time(pid: int) -> str | None:
+    try:
+        stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8", errors="replace")
+    except (FileNotFoundError, ProcessLookupError):
+        return None
+    # Field 2 is the command name in parentheses and may itself hold spaces or ')', so
+    # the fields are counted from after its *last* ')': field 3 is index 0, 22 is 19.
+    return stat[stat.rindex(")") + 2 :].split()[19]
+
+
+def _ps_start_time(pid: int) -> str | None:
+    completed = subprocess.run(
+        ["ps", "-o", "lstart=", "-p", str(pid)],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    started = completed.stdout.strip()
+    return started if completed.returncode == 0 and started else None
+
+
+class AttemptRecords:
+    """The ``attempts/`` directory of one Runner's state directory."""
+
+    def __init__(self, state_dir: Path, *, workspace_root: Path) -> None:
+        self._dir = state_dir / ATTEMPTS_DIR
+        if self._dir.resolve().is_relative_to(workspace_root.resolve()):
+            raise ValueError(
+                f"attempt records at {self._dir} would sit under the workspace root "
+                f"{workspace_root}, where an Agent could forge them"
+            )
+
+    def path(self, work_record_id: str) -> Path:
+        if not _SAFE_ID.fullmatch(work_record_id):
+            raise ValueError(f"work record id is not a file name: {work_record_id!r}")
+        return self._dir / f"{work_record_id}.json"
+
+    def refuse_if_alive(self, work_record_id: str) -> None:
+        """Raise if a recorded group for this Work Record lives; clear a stale record."""
+
+        path = self.path(work_record_id)
+        record = _read(path)
+        if record is None:
+            return
+        if _alive(record):
+            raise PriorAttemptAliveError(work_record_id, record)
+        _unlink_if(path, record)
+
+    def record(self, work_record_id: str, *, pgid: int, attempt: int) -> AttemptRecord | None:
+        """Claim the Work Record for a just-spawned group; None if it already exited.
+
+        Exclusive create, so two attempts that both passed the pre-spawn check in the
+        same instant cannot both hold the claim: the second finds the first's record
+        and is refused.
+        """
+
+        start_time = process_start_time(pgid)
+        if start_time is None:
+            return None
+        record = AttemptRecord(pgid=pgid, start_time=start_time, attempt=attempt)
+        path = self.path(work_record_id)
+        self._dir.mkdir(mode=_DIR_MODE, parents=True, exist_ok=True)
+        while True:
+            try:
+                fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, _FILE_MODE)
+            except FileExistsError:
+                self.refuse_if_alive(work_record_id)
+                continue
+            with os.fdopen(fd, "w", encoding="utf-8") as handle:
+                json.dump(asdict(record), handle)
+            return record
+
+    def release(self, work_record_id: str, record: AttemptRecord) -> None:
+        """Remove our own record once its group leader has exited.
+
+        Kept while the leader lives -- a cancel whose kill did not land -- so the next
+        attempt is still refused rather than let in beside it.
+        """
+
+        if not _alive(record):
+            _unlink_if(self.path(work_record_id), record)
+
+    def sweep(self) -> None:
+        """At Runner start: drop records whose process is gone, keep the live ones.
+
+        A workstation Runner restarted without its children leaves them running in the
+        Workspace; their records stay so the check still refuses until they exit.
+        """
+
+        if not self._dir.is_dir():
+            return
+        for path in self._dir.glob("*.json"):
+            record = _read(path)
+            if record is None or not _alive(record):
+                path.unlink(missing_ok=True)
+
+
+def _read(path: Path) -> AttemptRecord | None:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+        return AttemptRecord(
+            pgid=int(payload["pgid"]),
+            start_time=str(payload["start_time"]),
+            attempt=int(payload["attempt"]),
+        )
+    except FileNotFoundError:
+        return None
+    except (ValueError, KeyError, TypeError):
+        # Half-written by a Runner that died mid-write: it fences nothing it can name.
+        return AttemptRecord(pgid=0, start_time="", attempt=0)
+
+
+def _alive(record: AttemptRecord) -> bool:
+    return record.pgid > 0 and process_start_time(record.pgid) == record.start_time
+
+
+def _unlink_if(path: Path, record: AttemptRecord) -> None:
+    # Another attempt may have replaced a stale record between our read and this unlink;
+    # only the record we judged is ours to remove.
+    if _read(path) == record:
+        path.unlink(missing_ok=True)
+
+
+@dataclass(frozen=True, slots=True)
+class _Fence:
+    records: AttemptRecords
+    work_record_id: str
+    attempt: int
+
+
+_FENCE: ContextVar[_Fence | None] = ContextVar("agentic_runner_attempt_fence", default=None)
+
+
+@contextlib.contextmanager
+def fence_work_record(
+    records: AttemptRecords, work_record_id: str, *, attempt: int
+) -> Iterator[None]:
+    """Fence every process group spawned inside this block to ``work_record_id``."""
+
+    token = _FENCE.set(_Fence(records=records, work_record_id=work_record_id, attempt=attempt))
+    try:
+        yield
+    finally:
+        _FENCE.reset(token)
+
+
+def refuse_if_prior_attempt_alive() -> None:
+    """Called before a spawn; a no-op outside a fenced block."""
+
+    fence = _FENCE.get()
+    if fence is not None:
+        fence.records.refuse_if_alive(fence.work_record_id)
+
+
+@contextlib.contextmanager
+def recorded_group(pgid: int) -> Iterator[None]:
+    """Hold the Work Record's record for a spawned group while the block waits on it.
+
+    Raises :class:`PriorAttemptAliveError` on entry if another attempt claimed the Work
+    Record since the pre-spawn check; the caller kills the group it just spawned.
+    """
+
+    fence = _FENCE.get()
+    record = (
+        None
+        if fence is None
+        else fence.records.record(fence.work_record_id, pgid=pgid, attempt=fence.attempt)
+    )
+    try:
+        yield
+    finally:
+        if fence is not None and record is not None:
+            fence.records.release(fence.work_record_id, record)

{agentic_runner-2.6.0 → agentic_runner-2.6.1}/src/agentic_runner/runtime/verifier_command.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
+import contextlib
 import hashlib
 import json
 import os
 import re
+import signal
 import subprocess
 import time
 from collections.abc import Callable, Mapping
@@ -11,6 +13,11 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Protocol
 
+from agentic_runner.attempts import (
+    PriorAttemptAliveError,
+    recorded_group,
+    refuse_if_prior_attempt_alive,
+)
 from agentic_runner_contracts.redaction import redact_secret_like_text
 
 DEFAULT_OUTPUT_LIMIT_BYTES = 16_384
@@ -91,21 +98,44 @@ def _subprocess_runner(
     that cannot separate uids still verifies, and they arrive together from
     ``DirectiveSandbox.spawn_kwargs`` — the uid drop in subprocess's own C fork-exec path,
     only the rlimits in the Python hook.
+
+    Its own session, killed as a group on timeout, like a Directive: the test suite
+    forks workers into the Workspace, and the attempt fence (``attempts.py``) records
+    the group by its pgid.
     """
 
-    return subprocess.run(
+    refuse_if_prior_attempt_alive()
+    with subprocess.Popen(
         list(argv),
         cwd=cwd,
         env=env,
-        timeout=timeout_seconds,
-        check=False,
-        capture_output=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
         text=True,
+        start_new_session=True,
         preexec_fn=preexec_fn,
         user=user,
         group=group,
         extra_groups=extra_groups,
-    )
+    ) as process:
+        try:
+            with recorded_group(process.pid):
+                try:
+                    stdout, stderr = process.communicate(timeout=timeout_seconds)
+                except subprocess.TimeoutExpired:
+                    _kill_group(process)
+                    raise
+        except PriorAttemptAliveError:
+            _kill_group(process)
+            raise
+    return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr)
+
+
+def _kill_group(process: subprocess.Popen[str]) -> None:
+    with contextlib.suppress(ProcessLookupError, PermissionError):
+        os.killpg(process.pid, signal.SIGKILL)
+    process.kill()
+    process.communicate()
 
 
 def run(

{agentic_runner-2.6.0 → agentic_runner-2.6.1}/src/agentic_runner/service.py

@@ -44,6 +44,7 @@ from temporalio.worker import (
 
 from agentic_runner import __version__ as runner_version
 from agentic_runner.activities import RunnerRalphActivities
+from agentic_runner.attempts import AttemptRecords
 from agentic_runner.child_watcher import install_stop_tolerant_child_watcher
 from agentic_runner.config import RunnerConfig
 from agentic_runner.config import load as load_config
@@ -1021,6 +1022,10 @@ def _activities(
     )
     static_git_token = os.getenv("AGENTIC_OS_GIT_TOKEN") or None
     isolation = build_contract_isolation(settings, can_separate_uids=can_change_uid)
+    attempt_records = AttemptRecords(config.state_dir, workspace_root=settings.WORKSPACE_ROOT)
+    # Only the dead go: a live record is a harness this process lost on restart, still
+    # writing its Workspace, and the next attempt must keep being refused (RR-05).
+    attempt_records.sweep()
     ralph = RunnerRalphActivities(
         fastapi,
         runtime_context_resolver=WorkerRuntimeContextResolver(fastapi),
@@ -1056,6 +1061,7 @@ def _activities(
         message_store=messages,
         workflow_signaller=signaller,
         tool_server_health=tool_servers,
+        attempt_records=attempt_records,
     )
     return [
         *ralph.activity_callables(),

{agentic_runner-2.6.0 → agentic_runner-2.6.1}/src/agentic_runner/workers/_runtime_support.py

@@ -19,6 +19,11 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Final
 
+from agentic_runner.attempts import (
+    PriorAttemptAliveError,
+    recorded_group,
+    refuse_if_prior_attempt_alive,
+)
 from agentic_runner.callback import CALLBACK_SOCKET_ENV, CALLBACK_TOKEN_ENV
 from agentic_runner.egress import EGRESS_ENV_NAMES
 from agentic_runner.llm_proxy import PROXY_ENV_NAMES
@@ -110,6 +115,7 @@ async def run_subprocess_exec(
     output_limit_bytes: int,
     sandbox: DirectiveSandbox | None = None,
 ) -> SubprocessResult:
+    refuse_if_prior_attempt_alive()
     process = await asyncio.create_subprocess_exec(
         *argv,
         cwd=cwd,
@@ -121,14 +127,23 @@ async def run_subprocess_exec(
         **({} if sandbox is None else sandbox.spawn_kwargs()),
     )
     try:
-        stdout_bytes, stderr_bytes, stdout_truncated, stderr_truncated = await asyncio.wait_for(
-            _collect_process_output(process, stdin=stdin, output_limit_bytes=output_limit_bytes),
-            timeout=timeout_seconds,
-        )
-    except TimeoutError:
-        await _terminate_process_tree(process)
-        raise
-    except asyncio.CancelledError:
+        with recorded_group(process.pid):
+            try:
+                (
+                    stdout_bytes,
+                    stderr_bytes,
+                    stdout_truncated,
+                    stderr_truncated,
+                ) = await asyncio.wait_for(
+                    _collect_process_output(
+                        process, stdin=stdin, output_limit_bytes=output_limit_bytes
+                    ),
+                    timeout=timeout_seconds,
+                )
+            except (TimeoutError, asyncio.CancelledError):
+                await _terminate_process_tree(process)
+                raise
+    except PriorAttemptAliveError:
         await _terminate_process_tree(process)
         raise