codex-autorunner 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

codex_autorunner/core/flows/reasons.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from .models import FlowRunStatus
+
+MAX_REASON_SUMMARY_LEN = 120
+
+
+def _truncate(text: str, max_len: int = MAX_REASON_SUMMARY_LEN) -> str:
+    if len(text) <= max_len:
+        return text
+    return f"{text[:max_len].rstrip()}…"
+
+
+def ensure_reason_summary(
+    state: Any,
+    *,
+    status: FlowRunStatus,
+    error_message: Optional[str] = None,
+    default: Optional[str] = None,
+) -> dict[str, Any]:
+    """Ensure state includes a short reason_summary when stopping/pausing/failing."""
+    normalized: dict[str, Any] = dict(state) if isinstance(state, dict) else {}
+    existing = normalized.get("reason_summary")
+    if isinstance(existing, str) and existing.strip():
+        return normalized
+
+    reason: Optional[str] = None
+    engine = normalized.get("ticket_engine")
+    if isinstance(engine, dict):
+        engine_reason = engine.get("reason")
+        if isinstance(engine_reason, str) and engine_reason.strip():
+            reason = engine_reason.strip()
+
+    if not reason and isinstance(error_message, str) and error_message.strip():
+        reason = error_message.strip()
+
+    if not reason:
+        if default:
+            reason = default
+        else:
+            fallback = {
+                FlowRunStatus.PAUSED: "Paused",
+                FlowRunStatus.FAILED: "Failed",
+                FlowRunStatus.STOPPED: "Stopped",
+            }
+            reason = fallback.get(status)
+
+    if reason:
+        normalized["reason_summary"] = _truncate(reason)
+    return normalized

codex_autorunner/core/flows/reconciler.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from ..locks import FileLockBusy, file_lock
+from .models import FlowRunRecord, FlowRunStatus
+from .store import UNSET, FlowStore
+from .transition import resolve_flow_transition
+from .worker_process import FlowWorkerHealth, check_worker_health, clear_worker_metadata
+
+_logger = logging.getLogger(__name__)
+
+_ACTIVE_STATUSES = (
+    FlowRunStatus.RUNNING,
+    FlowRunStatus.STOPPING,
+    FlowRunStatus.PAUSED,
+)
+
+
+@dataclass
+class FlowReconcileSummary:
+    checked: int = 0
+    active: int = 0
+    updated: int = 0
+    locked: int = 0
+    errors: int = 0
+
+
+@dataclass
+class FlowReconcileResult:
+    records: list[FlowRunRecord]
+    summary: FlowReconcileSummary
+
+
+def _reconcile_lock_path(repo_root: Path, run_id: str) -> Path:
+    return repo_root / ".codex-autorunner" / "flows" / run_id / "reconcile.lock"
+
+
+def _ensure_worker_not_stale(health: FlowWorkerHealth) -> None:
+    if health.status in {"dead", "mismatch", "invalid"}:
+        try:
+            clear_worker_metadata(health.artifact_path.parent)
+        except Exception:
+            _logger.debug("Failed to clear worker metadata: %s", health.artifact_path)
+
+
+def reconcile_flow_run(
+    repo_root: Path,
+    record: FlowRunRecord,
+    store: FlowStore,
+    *,
+    logger: Optional[logging.Logger] = None,
+) -> tuple[FlowRunRecord, bool, bool]:
+    if record.status not in _ACTIVE_STATUSES:
+        return record, False, False
+
+    lock_path = _reconcile_lock_path(repo_root, record.id)
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        with file_lock(lock_path, blocking=False):
+            health = check_worker_health(repo_root, record.id)
+            decision = resolve_flow_transition(record, health)
+
+            if (
+                decision.status == record.status
+                and decision.finished_at == record.finished_at
+                and decision.state == (record.state or {})
+            ):
+                return record, False, False
+
+            (logger or _logger).info(
+                "Reconciling flow %s: %s -> %s (%s)",
+                record.id,
+                record.status.value,
+                decision.status.value,
+                decision.note or "reconcile",
+            )
+
+            updated = store.update_flow_run_status(
+                run_id=record.id,
+                status=decision.status,
+                state=decision.state,
+                finished_at=decision.finished_at if decision.finished_at else UNSET,
+            )
+            _ensure_worker_not_stale(health)
+            return (updated or record), bool(updated), False
+    except FileLockBusy:
+        return record, False, True
+    except Exception as exc:
+        (logger or _logger).warning("Failed to reconcile flow %s: %s", record.id, exc)
+        return record, False, False
+
+
+def reconcile_flow_runs(
+    repo_root: Path,
+    *,
+    flow_type: Optional[str] = None,
+    logger: Optional[logging.Logger] = None,
+) -> FlowReconcileResult:
+    db_path = repo_root / ".codex-autorunner" / "flows.db"
+    if not db_path.exists():
+        return FlowReconcileResult(records=[], summary=FlowReconcileSummary())
+    from ..config import load_repo_config
+
+    config = load_repo_config(repo_root)
+    store = FlowStore(db_path, durable=config.durable_writes)
+    summary = FlowReconcileSummary()
+    records: list[FlowRunRecord] = []
+    try:
+        store.initialize()
+        for record in store.list_flow_runs(flow_type=flow_type):
+            if record.status in _ACTIVE_STATUSES:
+                summary.active += 1
+                summary.checked += 1
+                record, updated, locked = reconcile_flow_run(
+                    repo_root, record, store, logger=logger
+                )
+                if updated:
+                    summary.updated += 1
+                if locked:
+                    summary.locked += 1
+            records.append(record)
+    except Exception as exc:
+        summary.errors += 1
+        (logger or _logger).warning("Flow reconcile run failed: %s", exc)
+    finally:
+        try:
+            store.close()
+        except Exception:
+            pass
+    return FlowReconcileResult(records=records, summary=summary)

codex_autorunner/core/flows/runtime.py

@@ -5,23 +5,38 @@ from typing import Any, Callable, Dict, Optional, Set, cast
 
 from .definition import FlowDefinition, StepFn, StepFn2, StepFn3
 from .models import FlowEvent, FlowEventType, FlowRunRecord, FlowRunStatus
+from .reasons import ensure_reason_summary
 from .store import FlowStore, now_iso
 
 _logger = logging.getLogger(__name__)
 
 
+LifecycleEventCallback = Optional[Callable[[str, str, str, Dict[str, Any]], None]]
+
+
 class FlowRuntime:
     def __init__(
         self,
         definition: FlowDefinition,
         store: FlowStore,
         emit_event: Optional[Callable[[FlowEvent], None]] = None,
+        emit_lifecycle_event: LifecycleEventCallback = None,
     ):
         self.definition = definition
         self.store = store
         self.emit_event = emit_event
+        self.emit_lifecycle_event = emit_lifecycle_event
         self._stop_check_interval = 0.5
 
+    def _emit_lifecycle(
+        self, event_type: str, repo_id: str, run_id: str, data: Dict[str, Any]
+    ) -> None:
+        if self.emit_lifecycle_event:
+            try:
+                self.emit_lifecycle_event(event_type, repo_id, run_id, data)
+            except Exception as exc:
+                _logger.exception("Error emitting lifecycle event: %s", exc)
+
     def _emit(
         self,
         event_type: FlowEventType,
@@ -97,14 +112,21 @@ class FlowRuntime:
                 if record.stop_requested:
                     self._emit(FlowEventType.FLOW_STOPPED, run_id)
                     now = now_iso()
+                    state = ensure_reason_summary(
+                        dict(record.state or {}),
+                        status=FlowRunStatus.STOPPED,
+                        default="Stopped by user",
+                    )
                     updated = self.store.update_flow_run_status(
                         run_id=run_id,
                         status=FlowRunStatus.STOPPED,
                         finished_at=now,
+                        state=state,
                     )
                     if not updated:
                         raise RuntimeError(f"Failed to stop flow run {run_id}")
                     record = updated
+                    self._emit_lifecycle("flow_stopped", "", run_id, {})
                     break
 
                 step_id = next_steps.pop()
@@ -128,17 +150,24 @@ class FlowRuntime:
                 data={"error": str(e)},
             )
             now = now_iso()
+            state = ensure_reason_summary(
+                dict(record.state or {}),
+                status=FlowRunStatus.FAILED,
+                error_message=str(e),
+            )
             updated = self.store.update_flow_run_status(
                 run_id=run_id,
                 status=FlowRunStatus.FAILED,
                 finished_at=now,
                 error_message=str(e),
+                state=state,
             )
             if not updated:
                 raise RuntimeError(
                     f"Failed to update flow run {run_id} to failed state"
                 ) from e
             record = updated
+            self._emit_lifecycle("flow_failed", "", run_id, {"error": str(e)})
             return record
 
     async def _execute_step(
@@ -257,6 +286,7 @@ class FlowRuntime:
                         f"Failed to update flow run after step {step_id}"
                     )
                 record = updated
+                self._emit_lifecycle("flow_completed", "", record.id, {})
 
             elif outcome.status == FlowRunStatus.FAILED:
                 self._emit(
@@ -267,12 +297,17 @@ class FlowRuntime:
                 )
 
                 now = now_iso()
+                state = ensure_reason_summary(
+                    dict(record.state or {}),
+                    status=FlowRunStatus.FAILED,
+                    error_message=outcome.error,
+                )
                 updated = self.store.update_flow_run_status(
                     run_id=record.id,
                     status=FlowRunStatus.FAILED,
                     finished_at=now,
                     error_message=outcome.error,
-                    state=record.state,
+                    state=state,
                     current_step=None,
                 )
                 if not updated:
@@ -280,6 +315,9 @@ class FlowRuntime:
                         f"Failed to update flow run after step {step_id}"
                     )
                 record = updated
+                self._emit_lifecycle(
+                    "flow_failed", "", record.id, {"error": outcome.error or ""}
+                )
 
             elif outcome.status == FlowRunStatus.STOPPED:
                 self._emit(
@@ -290,11 +328,15 @@ class FlowRuntime:
                 )
 
                 now = now_iso()
+                state = ensure_reason_summary(
+                    dict(record.state or {}),
+                    status=FlowRunStatus.STOPPED,
+                )
                 updated = self.store.update_flow_run_status(
                     run_id=record.id,
                     status=FlowRunStatus.STOPPED,
                     finished_at=now,
-                    state=record.state,
+                    state=state,
                     current_step=None,
                 )
                 if not updated:
@@ -302,6 +344,7 @@ class FlowRuntime:
                         f"Failed to update flow run after step {step_id}"
                     )
                 record = updated
+                self._emit_lifecycle("flow_stopped", "", record.id, {})
 
             elif outcome.status == FlowRunStatus.PAUSED:
                 self._emit(
@@ -311,10 +354,14 @@ class FlowRuntime:
                     step_id=step_id,
                 )
 
+                state = ensure_reason_summary(
+                    dict(record.state or {}),
+                    status=FlowRunStatus.PAUSED,
+                )
                 updated = self.store.update_flow_run_status(
                     run_id=record.id,
                     status=FlowRunStatus.PAUSED,
-                    state=record.state,
+                    state=state,
                     current_step=step_id,
                 )
                 if not updated:
@@ -322,6 +369,7 @@ class FlowRuntime:
                         f"Failed to update flow run after step {step_id}"
                     )
                 record = updated
+                self._emit_lifecycle("flow_paused", "", record.id, {})
 
             return record
 
@@ -335,12 +383,17 @@ class FlowRuntime:
             )
 
             now = now_iso()
+            state = ensure_reason_summary(
+                dict(record.state or {}),
+                status=FlowRunStatus.FAILED,
+                error_message=str(e),
+            )
             updated = self.store.update_flow_run_status(
                 run_id=record.id,
                 status=FlowRunStatus.FAILED,
                 finished_at=now,
                 error_message=str(e),
-                state=record.state,
+                state=state,
                 current_step=None,
             )
             if not updated:

codex_autorunner/core/flows/store.py

@@ -1,12 +1,15 @@
+from __future__ import annotations
+
 import json
 import logging
 import sqlite3
 import threading
 from contextlib import contextmanager
-from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Dict, Generator, List, Optional, cast
 
+from ..sqlite_utils import SQLITE_PRAGMAS, SQLITE_PRAGMAS_DURABLE
+from ..time_utils import now_iso
 from .models import (
     FlowArtifact,
     FlowEvent,
@@ -21,18 +24,22 @@ SCHEMA_VERSION = 2
 UNSET = object()
 
 
-def now_iso() -> str:
-    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-
 class FlowStore:
-    def __init__(self, db_path: Path):
+    def __init__(self, db_path: Path, durable: bool = False):
         self.db_path = db_path
+        self._durable = durable
         self._local: threading.local = threading.local()
 
+    def __enter__(self) -> FlowStore:
+        self.initialize()
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        self.close()
+
     def _get_conn(self) -> sqlite3.Connection:
         if not hasattr(self._local, "conn"):
-            # Ensure parent directory exists so sqlite can create/open the file.
+            # Ensure parent directory exists so sqlite can create/open file.
             try:
                 self.db_path.parent.mkdir(parents=True, exist_ok=True)
             except Exception:
@@ -42,6 +49,9 @@ class FlowStore:
                 self.db_path, check_same_thread=False, isolation_level=None
             )
             self._local.conn.row_factory = sqlite3.Row
+            pragmas = SQLITE_PRAGMAS_DURABLE if self._durable else SQLITE_PRAGMAS
+            for pragma in pragmas:
+                self._local.conn.execute(pragma)
         return cast(sqlite3.Connection, self._local.conn)
 
     @contextmanager
@@ -384,6 +394,131 @@ class FlowStore:
         rows = conn.execute(query, params).fetchall()
         return [self._row_to_flow_event(row) for row in rows]
 
+    def get_events_by_types(
+        self,
+        run_id: str,
+        event_types: list[FlowEventType],
+        *,
+        after_seq: Optional[int] = None,
+        limit: Optional[int] = None,
+    ) -> List[FlowEvent]:
+        """Return events for a run filtered to specific event types."""
+        if not event_types:
+            return []
+        conn = self._get_conn()
+        placeholders = ", ".join("?" for _ in event_types)
+        query = f"""
+            SELECT *
+            FROM flow_events
+            WHERE run_id = ? AND event_type IN ({placeholders})
+        """
+        params: List[Any] = [run_id, *[t.value for t in event_types]]
+
+        if after_seq is not None:
+            query += " AND seq > ?"
+            params.append(after_seq)
+
+        query += " ORDER BY seq ASC"
+
+        if limit is not None:
+            query += " LIMIT ?"
+            params.append(limit)
+
+        rows = conn.execute(query, params).fetchall()
+        return [self._row_to_flow_event(row) for row in rows]
+
+    def get_events_by_type(
+        self,
+        run_id: str,
+        event_type: FlowEventType,
+        *,
+        after_seq: Optional[int] = None,
+        limit: Optional[int] = None,
+    ) -> List[FlowEvent]:
+        return self.get_events_by_types(
+            run_id, [event_type], after_seq=after_seq, limit=limit
+        )
+
+    def get_last_event_meta(self, run_id: str) -> tuple[Optional[int], Optional[str]]:
+        conn = self._get_conn()
+        row = conn.execute(
+            "SELECT seq, timestamp FROM flow_events WHERE run_id = ? ORDER BY seq DESC LIMIT 1",
+            (run_id,),
+        ).fetchone()
+        if row is None:
+            return None, None
+        return row["seq"], row["timestamp"]
+
+    def get_last_event_seq_by_types(
+        self, run_id: str, event_types: list[FlowEventType]
+    ) -> Optional[int]:
+        if not event_types:
+            return None
+        conn = self._get_conn()
+        placeholders = ", ".join("?" for _ in event_types)
+        params = [run_id, *[t.value for t in event_types]]
+        row = conn.execute(
+            f"""
+            SELECT seq
+            FROM flow_events
+            WHERE run_id = ? AND event_type IN ({placeholders})
+            ORDER BY seq DESC
+            LIMIT 1
+            """,
+            params,
+        ).fetchone()
+        if row is None:
+            return None
+        return cast(int, row["seq"])
+
+    def get_last_event_by_type(
+        self, run_id: str, event_type: FlowEventType
+    ) -> Optional[FlowEvent]:
+        conn = self._get_conn()
+        row = conn.execute(
+            """
+            SELECT *
+            FROM flow_events
+            WHERE run_id = ? AND event_type = ?
+            ORDER BY seq DESC
+            LIMIT 1
+            """,
+            (run_id, event_type.value),
+        ).fetchone()
+        if row is None:
+            return None
+        return self._row_to_flow_event(row)
+
+    def get_latest_step_progress_current_ticket(
+        self, run_id: str, *, after_seq: Optional[int] = None, limit: int = 50
+    ) -> Optional[str]:
+        """Return the most recent step_progress.data.current_ticket for a run.
+
+        This is intentionally lightweight to support UI polling endpoints.
+        """
+        conn = self._get_conn()
+        query = """
+            SELECT seq, data
+            FROM flow_events
+            WHERE run_id = ? AND event_type = ?
+        """
+        params: List[Any] = [run_id, FlowEventType.STEP_PROGRESS.value]
+        if after_seq is not None:
+            query += " AND seq > ?"
+            params.append(after_seq)
+        query += " ORDER BY seq DESC LIMIT ?"
+        params.append(limit)
+        rows = conn.execute(query, params).fetchall()
+        for row in rows:
+            try:
+                data = json.loads(row["data"] or "{}")
+            except Exception:
+                data = {}
+            current_ticket = data.get("current_ticket")
+            if isinstance(current_ticket, str) and current_ticket.strip():
+                return current_ticket.strip()
+        return None
+
     def create_artifact(
         self,
         artifact_id: str,

codex_autorunner/core/flows/transition.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 from typing import Any, Optional
 
 from codex_autorunner.core.flows.models import FlowRunRecord, FlowRunStatus
+from codex_autorunner.core.flows.reasons import ensure_reason_summary
 from codex_autorunner.core.flows.store import now_iso
 
 
@@ -44,23 +45,10 @@ def resolve_flow_transition(
     inner_status = engine.get("status")
     reason_code = engine.get("reason_code")
 
-    # 1) Worker liveness overrides for active flows.
-    if (
-        record.status in (FlowRunStatus.RUNNING, FlowRunStatus.STOPPING)
-        and not health.is_alive
-    ):
-        new_status = (
-            FlowRunStatus.STOPPED
-            if record.status == FlowRunStatus.STOPPING
-            else FlowRunStatus.FAILED
-        )
-        return TransitionDecision(
-            status=new_status, finished_at=now, state=state, note="worker-dead"
-        )
-
-    # 2) Inner engine reconciliation (worker is alive or not required).
+    # 1) Inner engine completion takes priority over worker liveness for active flows.
     if record.status == FlowRunStatus.RUNNING:
         if inner_status == "paused":
+            state = ensure_reason_summary(state, status=FlowRunStatus.PAUSED)
             return TransitionDecision(
                 status=FlowRunStatus.PAUSED,
                 finished_at=None,
@@ -76,10 +64,32 @@ def resolve_flow_transition(
                 note="engine-completed",
             )
 
+        # 2) Worker liveness overrides for active flows (only if engine not completed).
+        if not health.is_alive:
+            new_status = FlowRunStatus.FAILED
+            state = ensure_reason_summary(
+                state, status=new_status, default="Worker died"
+            )
+            return TransitionDecision(
+                status=new_status, finished_at=now, state=state, note="worker-dead"
+            )
+
         return TransitionDecision(
             status=FlowRunStatus.RUNNING, finished_at=None, state=state, note="running"
         )
 
+    # Handle STOPPING case separately - worker liveness check still applies.
+    if record.status == FlowRunStatus.STOPPING and not health.is_alive:
+        state = ensure_reason_summary(
+            state, status=FlowRunStatus.STOPPED, default="Worker stopped"
+        )
+        return TransitionDecision(
+            status=FlowRunStatus.STOPPED,
+            finished_at=now,
+            state=state,
+            note="worker-dead",
+        )
+
     if record.status == FlowRunStatus.PAUSED:
         if inner_status == "completed":
             return TransitionDecision(
@@ -98,6 +108,7 @@ def resolve_flow_transition(
             engine.pop("reason", None)
             engine.pop("reason_details", None)
             engine.pop("reason_code", None)
+            state.pop("reason_summary", None)
             engine["status"] = "running"
             state["ticket_engine"] = engine
             return TransitionDecision(
@@ -115,6 +126,7 @@ def resolve_flow_transition(
                 note="paused-worker-dead",
             )
 
+        state = ensure_reason_summary(state, status=FlowRunStatus.PAUSED)
         return TransitionDecision(
             status=FlowRunStatus.PAUSED, finished_at=None, state=state, note="paused"
         )