codex-autorunner 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

codex_autorunner/core/flows/transition.py

@@ -45,22 +45,7 @@ def resolve_flow_transition(
     inner_status = engine.get("status")
     reason_code = engine.get("reason_code")
 
-    # 1) Worker liveness overrides for active flows.
-    if (
-        record.status in (FlowRunStatus.RUNNING, FlowRunStatus.STOPPING)
-        and not health.is_alive
-    ):
-        new_status = (
-            FlowRunStatus.STOPPED
-            if record.status == FlowRunStatus.STOPPING
-            else FlowRunStatus.FAILED
-        )
-        state = ensure_reason_summary(state, status=new_status, default="Worker died")
-        return TransitionDecision(
-            status=new_status, finished_at=now, state=state, note="worker-dead"
-        )
-
-    # 2) Inner engine reconciliation (worker is alive or not required).
+    # 1) Inner engine completion takes priority over worker liveness for active flows.
     if record.status == FlowRunStatus.RUNNING:
         if inner_status == "paused":
             state = ensure_reason_summary(state, status=FlowRunStatus.PAUSED)
@@ -79,10 +64,32 @@ def resolve_flow_transition(
                 note="engine-completed",
             )
 
+        # 2) Worker liveness overrides for active flows (only if engine not completed).
+        if not health.is_alive:
+            new_status = FlowRunStatus.FAILED
+            state = ensure_reason_summary(
+                state, status=new_status, default="Worker died"
+            )
+            return TransitionDecision(
+                status=new_status, finished_at=now, state=state, note="worker-dead"
+            )
+
         return TransitionDecision(
             status=FlowRunStatus.RUNNING, finished_at=None, state=state, note="running"
         )
 
+    # Handle STOPPING case separately - worker liveness check still applies.
+    if record.status == FlowRunStatus.STOPPING and not health.is_alive:
+        state = ensure_reason_summary(
+            state, status=FlowRunStatus.STOPPED, default="Worker stopped"
+        )
+        return TransitionDecision(
+            status=FlowRunStatus.STOPPED,
+            finished_at=now,
+            state=state,
+            note="worker-dead",
+        )
+
     if record.status == FlowRunStatus.PAUSED:
         if inner_status == "completed":
             return TransitionDecision(

codex_autorunner/core/flows/ux_helpers.py

@@ -4,7 +4,7 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Callable, Optional, Protocol
 
-from ...tickets.files import list_ticket_paths
+from ...tickets.files import list_ticket_paths, ticket_is_done
 from .models import FlowEventType, FlowRunRecord
 from .store import FlowStore
 from .worker_process import (
@@ -58,6 +58,18 @@ def _ticket_dir(repo_root: Path) -> Path:
     return repo_root.resolve() / ".codex-autorunner" / "tickets"
 
 
+def ticket_progress(repo_root: Path) -> dict[str, int]:
+    ticket_dir = _ticket_dir(repo_root)
+    ticket_paths = list_ticket_paths(ticket_dir)
+    total = len(ticket_paths)
+    done = 0
+    if total:
+        for path in ticket_paths:
+            if ticket_is_done(path):
+                done += 1
+    return {"done": done, "total": total}
+
+
 def bootstrap_check(
     repo_root: Path,
     github_service_factory: Optional[Callable[[Path], GitHubServiceProtocol]] = None,
@@ -219,13 +231,15 @@ def build_flow_status_snapshot(
         "last_event_at": last_event_at,
         "worker_health": health,
         "effective_current_ticket": effective_ticket,
+        "ticket_progress": ticket_progress(repo_root),
         "state": updated_state,
     }
 
 
-def ensure_worker(repo_root: Path, run_id: str) -> dict:
+def ensure_worker(repo_root: Path, run_id: str, is_terminal: bool = False) -> dict:
     health = check_worker_health(repo_root, run_id)
-    if health.status in {"dead", "mismatch", "invalid"}:
+    # Only clear metadata for dead/mismatch/invalid workers if not terminal
+    if not is_terminal and health.status in {"dead", "mismatch", "invalid"}:
         try:
             clear_worker_metadata(health.artifact_path.parent)
         except Exception:
@@ -252,6 +266,7 @@ __all__ = [
     "format_issue_as_markdown",
     "issue_md_has_content",
     "issue_md_path",
+    "ticket_progress",
     "seed_issue_from_github",
     "seed_issue_from_text",
 ]

codex_autorunner/core/flows/worker_process.py

@@ -152,7 +152,8 @@ def check_worker_health(
     try:
         data = json.loads(metadata_path.read_text(encoding="utf-8"))
         pid = int(data.get("pid")) if data.get("pid") is not None else None
-        cmd = data.get("cmd") or []
+        raw_cmd = data.get("cmd") or []
+        cmd = [str(part) for part in raw_cmd] if isinstance(raw_cmd, list) else []
     except Exception:
         return FlowWorkerHealth(
             status="invalid",
@@ -166,7 +167,7 @@ def check_worker_health(
         return FlowWorkerHealth(
             status="invalid",
             pid=pid,
-            cmdline=cmd if isinstance(cmd, list) else [],
+            cmdline=cmd,
             artifact_path=metadata_path,
             message="missing or invalid PID",
         )
@@ -175,19 +176,19 @@ def check_worker_health(
         return FlowWorkerHealth(
             status="dead",
             pid=pid,
-            cmdline=cmd if isinstance(cmd, list) else [],
+            cmdline=cmd,
             artifact_path=metadata_path,
             message="worker PID not running",
         )
 
-    expected_cmd = _build_worker_cmd(entrypoint, run_id)
+    expected_cmd = cmd or _build_worker_cmd(entrypoint, run_id)
     actual_cmd = _read_process_cmdline(pid)
     if actual_cmd is None:
         # Can't inspect cmdline; trust the PID check.
         return FlowWorkerHealth(
             status="alive",
             pid=pid,
-            cmdline=cmd if isinstance(cmd, list) else [],
+            cmdline=cmd,
             artifact_path=metadata_path,
             message="worker running (cmdline unknown)",
         )
@@ -198,7 +199,7 @@ def check_worker_health(
             pid=pid,
             cmdline=actual_cmd,
             artifact_path=metadata_path,
-            message="worker PID command does not match expected",
+            message="worker PID command does not match stored metadata",
         )
 
     return FlowWorkerHealth(
@@ -210,6 +211,31 @@ def check_worker_health(
     )
 
 
+def register_worker_metadata(
+    repo_root: Path,
+    run_id: str,
+    *,
+    artifacts_root: Optional[Path] = None,
+    pid: Optional[int] = None,
+    cmd: Optional[list[str]] = None,
+    entrypoint: str = "codex_autorunner",
+) -> Path:
+    normalized_run_id = _normalized_run_id(run_id)
+    artifacts_dir = _worker_artifacts_dir(repo_root, normalized_run_id, artifacts_root)
+
+    resolved_pid = pid or os.getpid()
+    resolved_cmd = cmd or _read_process_cmdline(resolved_pid)
+    if not resolved_cmd:
+        resolved_cmd = _build_worker_cmd(entrypoint, normalized_run_id)
+
+    _write_worker_metadata(
+        _worker_metadata_path(artifacts_dir),
+        resolved_pid,
+        resolved_cmd,
+    )
+    return artifacts_dir
+
+
 def spawn_flow_worker(
     repo_root: Path,
     run_id: str,

codex_autorunner/core/hub.py

@@ -1,11 +1,13 @@
+import asyncio
 import dataclasses
 import enum
 import logging
 import re
 import shutil
+import threading
 import time
 from pathlib import Path
-from typing import Callable, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 from ..bootstrap import seed_repo_files
 from ..discovery import DiscoveryRecord, discover_and_init
@@ -18,7 +20,6 @@ from ..manifest import (
 )
 from .archive import archive_worktree_snapshot, build_snapshot_id
 from .config import HubConfig, RepoConfig, derive_repo_config, load_hub_config
-from .engine import AppServerSupervisorFactory, BackendFactory, Engine
 from .git_utils import (
     GitError,
     git_available,
@@ -29,15 +30,22 @@ from .git_utils import (
     git_upstream_status,
     run_git,
 )
+from .lifecycle_events import LifecycleEvent, LifecycleEventEmitter, LifecycleEventStore
 from .locks import DEFAULT_RUNNER_CMD_HINTS, assess_lock, process_alive
+from .ports.backend_orchestrator import (
+    BackendOrchestrator as BackendOrchestratorProtocol,
+)
 from .runner_controller import ProcessRunnerController, SpawnRunnerFn
+from .runtime import RuntimeContext
 from .state import RunnerState, load_state, now_iso
+from .types import AppServerSupervisorFactory, BackendFactory
 from .utils import atomic_write
 
 logger = logging.getLogger("codex_autorunner.hub")
 
 BackendFactoryBuilder = Callable[[Path, RepoConfig], BackendFactory]
 AppServerSupervisorFactoryBuilder = Callable[[RepoConfig], AppServerSupervisorFactory]
+BackendOrchestratorBuilder = Callable[[Path, RepoConfig], BackendOrchestratorProtocol]
 
 
 def _git_failure_detail(proc) -> str:
@@ -205,27 +213,25 @@ class RepoRunner:
         app_server_supervisor_factory_builder: Optional[
             AppServerSupervisorFactoryBuilder
         ] = None,
+        backend_orchestrator_builder: Optional[BackendOrchestratorBuilder] = None,
         agent_id_validator: Optional[Callable[[str], str]] = None,
     ):
         self.repo_id = repo_id
-        backend_factory = (
-            backend_factory_builder(repo_root, repo_config)
-            if backend_factory_builder is not None
-            else None
-        )
-        app_server_supervisor_factory = (
-            app_server_supervisor_factory_builder(repo_config)
-            if app_server_supervisor_factory_builder is not None
+        backend_orchestrator = (
+            backend_orchestrator_builder(repo_root, repo_config)
+            if backend_orchestrator_builder is not None
             else None
         )
-        self._engine = Engine(
-            repo_root,
+        if backend_orchestrator is None:
+            raise ValueError(
+                "backend_orchestrator_builder is required for HubSupervisor"
+            )
+        self._ctx = RuntimeContext(
+            repo_root=repo_root,
             config=repo_config,
-            backend_factory=backend_factory,
-            app_server_supervisor_factory=app_server_supervisor_factory,
-            agent_id_validator=agent_id_validator,
+            backend_orchestrator=backend_orchestrator,
         )
-        self._controller = ProcessRunnerController(self._engine, spawn_fn=spawn_fn)
+        self._controller = ProcessRunnerController(self._ctx, spawn_fn=spawn_fn)
 
     @property
     def running(self) -> bool:
@@ -254,6 +260,7 @@ class HubSupervisor:
         app_server_supervisor_factory_builder: Optional[
             AppServerSupervisorFactoryBuilder
         ] = None,
+        backend_orchestrator_builder: Optional[BackendOrchestratorBuilder] = None,
         agent_id_validator: Optional[Callable[[str], str]] = None,
     ):
         self.hub_config = hub_config
@@ -264,11 +271,22 @@ class HubSupervisor:
         self._app_server_supervisor_factory_builder = (
             app_server_supervisor_factory_builder
         )
+        self._backend_orchestrator_builder = backend_orchestrator_builder
         self._agent_id_validator = agent_id_validator
         self.state = load_hub_state(self.state_path, self.hub_config.root)
         self._list_cache_at: Optional[float] = None
         self._list_cache: Optional[List[RepoSnapshot]] = None
+        self._list_lock = threading.Lock()
+        self._lifecycle_emitter = LifecycleEventEmitter(hub_config.root)
+        self._lifecycle_task_lock = threading.Lock()
+        self._lifecycle_stop_event = threading.Event()
+        self._lifecycle_thread: Optional[threading.Thread] = None
+        self._dispatch_interceptor_task: Optional[asyncio.Task] = None
+        self._dispatch_interceptor_stop_event: Optional[threading.Event] = None
+        self._dispatch_interceptor_thread: Optional[threading.Thread] = None
         self._reconcile_startup()
+        self._start_lifecycle_event_processor()
+        self._start_dispatch_interceptor()
 
     @classmethod
     def from_path(
@@ -279,12 +297,14 @@ class HubSupervisor:
         app_server_supervisor_factory_builder: Optional[
             AppServerSupervisorFactoryBuilder
         ] = None,
+        backend_orchestrator_builder: Optional[BackendOrchestratorBuilder] = None,
     ) -> "HubSupervisor":
         config = load_hub_config(path)
         return cls(
             config,
             backend_factory_builder=backend_factory_builder,
             app_server_supervisor_factory_builder=app_server_supervisor_factory_builder,
+            backend_orchestrator_builder=backend_orchestrator_builder,
         )
 
     def scan(self) -> List[RepoSnapshot]:
@@ -296,16 +316,17 @@ class HubSupervisor:
         return snapshots
 
     def list_repos(self, *, use_cache: bool = True) -> List[RepoSnapshot]:
-        if use_cache and self._list_cache and self._list_cache_at is not None:
-            if time.monotonic() - self._list_cache_at < 2.0:
-                return self._list_cache
-        manifest, records = self._manifest_records(manifest_only=True)
-        snapshots = self._build_snapshots(records)
-        self.state = HubState(last_scan_at=self.state.last_scan_at, repos=snapshots)
-        save_hub_state(self.state_path, self.state, self.hub_config.root)
-        self._list_cache = snapshots
-        self._list_cache_at = time.monotonic()
-        return snapshots
+        with self._list_lock:
+            if use_cache and self._list_cache and self._list_cache_at is not None:
+                if time.monotonic() - self._list_cache_at < 2.0:
+                    return self._list_cache
+            manifest, records = self._manifest_records(manifest_only=True)
+            snapshots = self._build_snapshots(records)
+            self.state = HubState(last_scan_at=self.state.last_scan_at, repos=snapshots)
+            save_hub_state(self.state_path, self.state, self.hub_config.root)
+            self._list_cache = snapshots
+            self._list_cache_at = time.monotonic()
+            return snapshots
 
     def _reconcile_startup(self) -> None:
         try:
@@ -320,23 +341,18 @@ class HubSupervisor:
                 repo_config = derive_repo_config(
                     self.hub_config, record.absolute_path, load_env=False
                 )
-                backend_factory = (
-                    self._backend_factory_builder(record.absolute_path, repo_config)
-                    if self._backend_factory_builder is not None
-                    else None
-                )
-                app_server_supervisor_factory = (
-                    self._app_server_supervisor_factory_builder(repo_config)
-                    if self._app_server_supervisor_factory_builder is not None
+                backend_orchestrator = (
+                    self._backend_orchestrator_builder(
+                        record.absolute_path, repo_config
+                    )
+                    if self._backend_orchestrator_builder is not None
                     else None
                 )
                 controller = ProcessRunnerController(
-                    Engine(
-                        record.absolute_path,
+                    RuntimeContext(
+                        repo_root=record.absolute_path,
                         config=repo_config,
-                        backend_factory=backend_factory,
-                        app_server_supervisor_factory=app_server_supervisor_factory,
-                        agent_id_validator=self._agent_id_validator,
+                        backend_orchestrator=backend_orchestrator,
                     )
                 )
                 controller.reconcile()
@@ -890,6 +906,7 @@ class HubSupervisor:
             app_server_supervisor_factory_builder=(
                 self._app_server_supervisor_factory_builder
             ),
+            backend_orchestrator_builder=self._backend_orchestrator_builder,
             agent_id_validator=self._agent_id_validator,
         )
         self._runners[repo_id] = runner
@@ -933,8 +950,148 @@ class HubSupervisor:
         return snapshot
 
     def _invalidate_list_cache(self) -> None:
-        self._list_cache = None
-        self._list_cache_at = None
+        with self._list_lock:
+            self._list_cache = None
+            self._list_cache_at = None
+
+    @property
+    def lifecycle_emitter(self) -> LifecycleEventEmitter:
+        return self._lifecycle_emitter
+
+    @property
+    def lifecycle_store(self) -> LifecycleEventStore:
+        return self._lifecycle_emitter._store
+
+    def trigger_pma_from_lifecycle_event(self, event: LifecycleEvent) -> None:
+        if event.processed:
+            return
+        event_id = event.event_id
+        if event_id is None:
+            return
+        self.lifecycle_store.mark_processed(event_id)
+        self.lifecycle_store.prune_processed(keep_last=50)
+        logger.info(
+            "PMA wakeup triggered by lifecycle event: type=%s repo_id=%s run_id=%s",
+            event.event_type.value,
+            event.repo_id,
+            event.run_id,
+        )
+
+    def process_lifecycle_events(self) -> None:
+        events = self.lifecycle_store.get_unprocessed(limit=100)
+        if not events:
+            return
+        for event in events:
+            try:
+                self.trigger_pma_from_lifecycle_event(event)
+            except Exception as exc:
+                logger.exception(
+                    "Failed to process lifecycle event %s: %s", event.event_id, exc
+                )
+
+    def _start_lifecycle_event_processor(self) -> None:
+        if self._lifecycle_thread is not None:
+            return
+
+        def _process_loop():
+            while not self._lifecycle_stop_event.wait(5.0):
+                try:
+                    self.process_lifecycle_events()
+                except Exception:
+                    logger.exception("Error in lifecycle event processor")
+
+        self._lifecycle_thread = threading.Thread(
+            target=_process_loop, daemon=True, name="lifecycle-event-processor"
+        )
+        self._lifecycle_thread.start()
+
+    def _stop_lifecycle_event_processor(self) -> None:
+        if self._lifecycle_thread is None:
+            return
+        self._lifecycle_stop_event.set()
+        self._lifecycle_thread.join(timeout=2.0)
+        self._lifecycle_thread = None
+
+    def shutdown(self) -> None:
+        self._stop_lifecycle_event_processor()
+        self._stop_dispatch_interceptor()
+
+    def _start_dispatch_interceptor(self) -> None:
+        if not self.hub_config.pma.enabled:
+            return
+        if not self.hub_config.pma.dispatch_interception_enabled:
+            return
+        if self._dispatch_interceptor_thread is not None:
+            return
+
+        import asyncio
+        from typing import TYPE_CHECKING
+
+        if TYPE_CHECKING:
+            pass
+
+        def _run_interceptor():
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+            from .pma_dispatch_interceptor import run_dispatch_interceptor
+
+            stop_event = threading.Event()
+            self._dispatch_interceptor_stop_event = stop_event
+
+            async def run_until_stop():
+                task = None
+                try:
+                    task = await run_dispatch_interceptor(
+                        hub_root=self.hub_config.root,
+                        supervisor=self,
+                        interval_seconds=5.0,
+                        on_intercept=self._on_dispatch_intercept,
+                    )
+                    while not stop_event.is_set():
+                        await asyncio.sleep(0.1)
+                except asyncio.CancelledError:
+                    pass
+                finally:
+                    if task is not None and not task.done():
+                        task.cancel()
+                    if task is not None:
+                        try:
+                            await task
+                        except (asyncio.CancelledError, Exception):
+                            pass
+
+            loop.run_until_complete(run_until_stop())
+            loop.close()
+
+        self._dispatch_interceptor_thread = threading.Thread(
+            target=_run_interceptor, daemon=True, name="pma-dispatch-interceptor"
+        )
+        self._dispatch_interceptor_thread.start()
+
+    def _stop_dispatch_interceptor(self) -> None:
+        if self._dispatch_interceptor_stop_event is not None:
+            self._dispatch_interceptor_stop_event.set()
+        if self._dispatch_interceptor_thread is not None:
+            self._dispatch_interceptor_thread.join(timeout=2.0)
+            self._dispatch_interceptor_thread = None
+            self._dispatch_interceptor_stop_event = None
+
+    def _on_dispatch_intercept(self, event_id: str, result: Any) -> None:
+        logger.info(
+            "Dispatch intercepted: event_id=%s action=%s reason=%s",
+            event_id,
+            (
+                result.get("action")
+                if isinstance(result, dict)
+                else getattr(result, "action", None)
+            ),
+            (
+                result.get("reason")
+                if isinstance(result, dict)
+                else getattr(result, "reason", None)
+            ),
+        )
 
     def _snapshot_from_record(self, record: DiscoveryRecord) -> RepoSnapshot:
         repo_path = record.absolute_path