codex-autorunner 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

codex_autorunner/core/pma_dispatches.py

@@ -0,0 +1,224 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+import yaml
+
+from ..tickets.frontmatter import parse_markdown_frontmatter
+from .time_utils import now_iso
+from .utils import atomic_write
+
+logger = logging.getLogger(__name__)
+
+PMA_DISPATCHES_DIRNAME = "dispatches"
+PMA_DISPATCH_ALLOWED_PRIORITIES = {"info", "warn", "action"}
+PMA_DISPATCH_ALLOWED_EXTS = {".md", ".markdown"}
+
+
+@dataclass(frozen=True)
+class PmaDispatch:
+    dispatch_id: str
+    title: str
+    body: str
+    priority: str
+    links: list[dict[str, str]]
+    created_at: str
+    resolved_at: Optional[str]
+    source_turn_id: Optional[str]
+    path: Path
+
+
+def _dispatches_root(hub_root: Path) -> Path:
+    return hub_root / ".codex-autorunner" / "pma" / PMA_DISPATCHES_DIRNAME
+
+
+def ensure_pma_dispatches_dir(hub_root: Path) -> Path:
+    root = _dispatches_root(hub_root)
+    root.mkdir(parents=True, exist_ok=True)
+    return root
+
+
+def _normalize_priority(value: Any) -> str:
+    if not isinstance(value, str):
+        return "info"
+    priority = value.strip().lower()
+    if priority in PMA_DISPATCH_ALLOWED_PRIORITIES:
+        return priority
+    return "info"
+
+
+def _normalize_links(value: Any) -> list[dict[str, str]]:
+    if not isinstance(value, list):
+        return []
+    links: list[dict[str, str]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        label = item.get("label")
+        href = item.get("href")
+        if not isinstance(label, str) or not isinstance(href, str):
+            continue
+        label = label.strip()
+        href = href.strip()
+        if not label or not href:
+            continue
+        links.append({"label": label, "href": href})
+    return links
+
+
+def _created_at_from_path(path: Path) -> str:
+    try:
+        stamp = path.stat().st_mtime
+    except OSError:
+        return now_iso()
+    return datetime.fromtimestamp(stamp, tz=timezone.utc).isoformat()
+
+
+def _parse_iso(value: Optional[str]) -> Optional[datetime]:
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        return datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except Exception:
+        return None
+
+
+def parse_pma_dispatch(path: Path) -> tuple[Optional[PmaDispatch], list[str]]:
+    errors: list[str] = []
+    try:
+        raw = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        return None, [f"Failed to read dispatch: {exc}"]
+
+    data, body = parse_markdown_frontmatter(raw)
+    if not isinstance(data, dict):
+        data = {}
+
+    raw_title = data.get("title")
+    title = raw_title if isinstance(raw_title, str) else ""
+    body = body or ""
+    priority = _normalize_priority(data.get("priority"))
+    links = _normalize_links(data.get("links"))
+    created_at = (
+        data.get("created_at") if isinstance(data.get("created_at"), str) else None
+    )
+    resolved_at = (
+        data.get("resolved_at") if isinstance(data.get("resolved_at"), str) else None
+    )
+    source_turn_id = (
+        data.get("source_turn_id")
+        if isinstance(data.get("source_turn_id"), str)
+        else None
+    )
+
+    if not created_at:
+        created_at = _created_at_from_path(path)
+
+    dispatch_id = path.stem
+
+    dispatch = PmaDispatch(
+        dispatch_id=dispatch_id,
+        title=title,
+        body=body,
+        priority=priority,
+        links=links,
+        created_at=created_at,
+        resolved_at=resolved_at,
+        source_turn_id=source_turn_id,
+        path=path,
+    )
+    return dispatch, errors
+
+
+def list_pma_dispatches(
+    hub_root: Path,
+    *,
+    include_resolved: bool = False,
+    limit: Optional[int] = None,
+) -> list[PmaDispatch]:
+    root = ensure_pma_dispatches_dir(hub_root)
+    dispatches: list[PmaDispatch] = []
+    for child in sorted(root.iterdir(), key=lambda p: p.name):
+        if child.suffix.lower() not in PMA_DISPATCH_ALLOWED_EXTS:
+            continue
+        dispatch, errors = parse_pma_dispatch(child)
+        if errors or dispatch is None:
+            continue
+        if not include_resolved and dispatch.resolved_at:
+            continue
+        dispatches.append(dispatch)
+
+    dispatches.sort(
+        key=lambda d: _parse_iso(d.created_at)
+        or datetime.min.replace(tzinfo=timezone.utc),
+        reverse=True,
+    )
+    if isinstance(limit, int) and limit > 0:
+        dispatches = dispatches[:limit]
+    return dispatches
+
+
+def list_pma_dispatches_for_turn(hub_root: Path, turn_id: str) -> list[PmaDispatch]:
+    if not isinstance(turn_id, str) or not turn_id:
+        return []
+    dispatches = list_pma_dispatches(hub_root, include_resolved=False)
+    return [d for d in dispatches if d.source_turn_id == turn_id]
+
+
+def find_pma_dispatch_path(hub_root: Path, dispatch_id: str) -> Optional[Path]:
+    if not isinstance(dispatch_id, str) or not dispatch_id:
+        return None
+    safe_id = dispatch_id.strip()
+    if not safe_id or "/" in safe_id or "\\" in safe_id:
+        return None
+    root = ensure_pma_dispatches_dir(hub_root)
+    for ext in PMA_DISPATCH_ALLOWED_EXTS:
+        candidate = root / f"{safe_id}{ext}"
+        if candidate.exists():
+            return candidate
+    return None
+
+
+def resolve_pma_dispatch(path: Path) -> tuple[Optional[PmaDispatch], list[str]]:
+    dispatch, errors = parse_pma_dispatch(path)
+    if errors or dispatch is None:
+        return dispatch, errors
+
+    if dispatch.resolved_at:
+        return dispatch, []
+
+    data = {
+        "title": dispatch.title,
+        "priority": dispatch.priority,
+        "links": dispatch.links,
+        "created_at": dispatch.created_at,
+        "resolved_at": now_iso(),
+        "source_turn_id": dispatch.source_turn_id,
+    }
+    frontmatter = yaml.safe_dump(data, sort_keys=False).strip()
+    content = f"---\n{frontmatter}\n---\n\n{dispatch.body.strip()}\n"
+    try:
+        atomic_write(path, content)
+    except OSError as exc:
+        return dispatch, [f"Failed to resolve dispatch: {exc}"]
+
+    updated, parse_errors = parse_pma_dispatch(path)
+    if parse_errors:
+        return updated, parse_errors
+    return updated, []
+
+
+__all__ = [
+    "PmaDispatch",
+    "PMA_DISPATCHES_DIRNAME",
+    "ensure_pma_dispatches_dir",
+    "parse_pma_dispatch",
+    "list_pma_dispatches",
+    "list_pma_dispatches_for_turn",
+    "find_pma_dispatch_path",
+    "resolve_pma_dispatch",
+]

codex_autorunner/core/pma_lane_worker.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any, Awaitable, Callable, Optional
+
+from .pma_queue import PmaQueue, PmaQueueItem
+
+logger = logging.getLogger(__name__)
+
+PmaLaneExecutor = Callable[[PmaQueueItem], Awaitable[dict[str, Any]]]
+PmaLaneResultHook = Callable[
+    [PmaQueueItem, dict[str, Any]],
+    Optional[Awaitable[None]],
+]
+
+
+class PmaLaneWorker:
+    """Background worker that drains a PMA lane queue."""
+
+    def __init__(
+        self,
+        lane_id: str,
+        queue: PmaQueue,
+        executor: PmaLaneExecutor,
+        *,
+        log: Optional[logging.Logger] = None,
+        on_result: Optional[PmaLaneResultHook] = None,
+        poll_interval_seconds: float = 1.0,
+    ) -> None:
+        self.lane_id = lane_id
+        self._queue = queue
+        self._executor = executor
+        self._log = log or logger
+        self._on_result = on_result
+        self._poll_interval_seconds = max(0.1, poll_interval_seconds)
+        self._task: Optional[asyncio.Task[None]] = None
+        self._cancel_event = asyncio.Event()
+        self._lock = asyncio.Lock()
+
+    @property
+    def is_running(self) -> bool:
+        return self._task is not None and not self._task.done()
+
+    async def start(self) -> bool:
+        async with self._lock:
+            if self.is_running:
+                return False
+            self._cancel_event = asyncio.Event()
+            self._task = asyncio.create_task(self._run())
+            self._log.info("PMA lane worker started (lane_id=%s)", self.lane_id)
+            return True
+
+    async def stop(self) -> None:
+        async with self._lock:
+            task = self._task
+            if task is None:
+                return
+            self._cancel_event.set()
+        if task is not None:
+            try:
+                await task
+            finally:
+                self._log.info("PMA lane worker stopped (lane_id=%s)", self.lane_id)
+
+    async def _run(self) -> None:
+        await self._queue.replay_pending(self.lane_id)
+        while not self._cancel_event.is_set():
+            item = await self._queue.dequeue(self.lane_id)
+            if item is None:
+                await self._queue.wait_for_lane_item(
+                    self.lane_id,
+                    self._cancel_event,
+                    poll_interval_seconds=self._poll_interval_seconds,
+                )
+                continue
+
+            if self._cancel_event.is_set():
+                await self._queue.fail_item(item, "cancelled by lane stop")
+                await self._notify(item, {"status": "error", "detail": "lane stopped"})
+                continue
+
+            self._log.info(
+                "PMA lane item started (lane_id=%s item_id=%s)",
+                self.lane_id,
+                item.item_id,
+            )
+            try:
+                result = await self._executor(item)
+                await self._queue.complete_item(item, result)
+                self._log.info(
+                    "PMA lane item completed (lane_id=%s item_id=%s status=%s)",
+                    self.lane_id,
+                    item.item_id,
+                    result.get("status") if isinstance(result, dict) else None,
+                )
+                await self._notify(item, result)
+            except Exception as exc:
+                self._log.exception("Failed to process PMA queue item %s", item.item_id)
+                error_result = {"status": "error", "detail": str(exc)}
+                await self._queue.fail_item(item, str(exc))
+                self._log.info(
+                    "PMA lane item failed (lane_id=%s item_id=%s error=%s)",
+                    self.lane_id,
+                    item.item_id,
+                    str(exc),
+                )
+                await self._notify(item, error_result)
+
+    async def _notify(self, item: PmaQueueItem, result: dict[str, Any]) -> None:
+        if self._on_result is None:
+            return
+        try:
+            maybe = self._on_result(item, result)
+            if asyncio.iscoroutine(maybe):
+                await maybe
+        except Exception:
+            self._log.exception(
+                "PMA lane result hook failed (lane_id=%s item_id=%s)",
+                self.lane_id,
+                item.item_id,
+            )

codex_autorunner/core/pma_queue.py

@@ -83,8 +83,10 @@ class PmaQueue:
         self._lane_queues: dict[str, asyncio.Queue[PmaQueueItem]] = {}
         self._lane_locks: dict[str, asyncio.Lock] = {}
         self._lane_events: dict[str, asyncio.Event] = {}
+        self._lane_known_ids: dict[str, set[str]] = {}
         self._replayed_lanes: set[str] = set()
         self._lock = asyncio.Lock()
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
 
     def _lane_queue_path(self, lane_id: str) -> Path:
         safe_lane_id = lane_id.replace(":", "__COLON__").replace("/", "__SLASH__")
@@ -115,6 +117,21 @@ class PmaQueue:
             self._lane_queues[lane_id] = queue
         return queue
 
+    def _ensure_lane_known_ids(self, lane_id: str) -> set[str]:
+        known = self._lane_known_ids.get(lane_id)
+        if known is None:
+            known = set()
+            self._lane_known_ids[lane_id] = known
+        return known
+
+    def _record_loop(self) -> None:
+        if self._loop is not None:
+            return
+        try:
+            self._loop = asyncio.get_running_loop()
+        except RuntimeError:
+            return
+
     async def enqueue(
         self,
         lane_id: str,
@@ -122,6 +139,7 @@ class PmaQueue:
         payload: dict[str, Any],
     ) -> tuple[PmaQueueItem, Optional[str]]:
         async with self._lock:
+            self._record_loop()
             existing = await self._find_by_idempotency_key(lane_id, idempotency_key)
             if existing:
                 if existing.state in (QueueItemState.PENDING, QueueItemState.RUNNING):
@@ -139,10 +157,36 @@ class PmaQueue:
             await self._append_to_file(item)
             queue = self._ensure_lane_queue(lane_id)
             await queue.put(item)
+            self._ensure_lane_known_ids(lane_id).add(item.item_id)
             self._ensure_lane_event(lane_id).set()
             return item, None
 
+    def enqueue_sync(
+        self,
+        lane_id: str,
+        idempotency_key: str,
+        payload: dict[str, Any],
+    ) -> tuple[PmaQueueItem, Optional[str]]:
+        existing = self._find_by_idempotency_key_sync(lane_id, idempotency_key)
+        if existing:
+            if existing.state in (QueueItemState.PENDING, QueueItemState.RUNNING):
+                dedupe_item = PmaQueueItem.create(
+                    lane_id=lane_id,
+                    idempotency_key=idempotency_key,
+                    payload=payload,
+                )
+                dedupe_item.state = QueueItemState.DEDUPED
+                dedupe_item.dedupe_reason = f"duplicate_of_{existing.item_id}"
+                self._append_to_file_sync(dedupe_item)
+                return dedupe_item, f"duplicate of {existing.item_id}"
+
+        item = PmaQueueItem.create(lane_id, idempotency_key, payload)
+        self._append_to_file_sync(item)
+        self._notify_in_memory_enqueue(item)
+        return item, None
+
     async def dequeue(self, lane_id: str) -> Optional[PmaQueueItem]:
+        self._record_loop()
         queue = self._lane_queues.get(lane_id)
         if queue is None or queue.empty():
             return None
@@ -206,11 +250,15 @@ class PmaQueue:
         return cancelled
 
     async def replay_pending(self, lane_id: str) -> int:
+        self._record_loop()
         if lane_id in self._replayed_lanes:
             return 0
         self._replayed_lanes.add(lane_id)
 
         items = await self.list_items(lane_id)
+        known = self._ensure_lane_known_ids(lane_id)
+        for item in items:
+            known.add(item.item_id)
         pending = [item for item in items if item.state == QueueItemState.PENDING]
         if not pending:
             return 0
@@ -222,29 +270,45 @@ class PmaQueue:
         return len(pending)
 
     async def wait_for_lane_item(
-        self, lane_id: str, cancel_event: Optional[asyncio.Event] = None
+        self,
+        lane_id: str,
+        cancel_event: Optional[asyncio.Event] = None,
+        *,
+        poll_interval_seconds: float = 1.0,
     ) -> bool:
+        self._record_loop()
         event = self._ensure_lane_event(lane_id)
         if event.is_set():
             event.clear()
             return True
 
-        wait_tasks = [asyncio.create_task(event.wait())]
-        if cancel_event is not None:
-            wait_tasks.append(asyncio.create_task(cancel_event.wait()))
+        poll_interval = max(0.1, poll_interval_seconds)
+        while True:
+            wait_tasks = [asyncio.create_task(event.wait())]
+            if cancel_event is not None:
+                wait_tasks.append(asyncio.create_task(cancel_event.wait()))
 
-        done, pending = await asyncio.wait(
-            wait_tasks, return_when=asyncio.FIRST_COMPLETED
-        )
-        for task in pending:
-            task.cancel()
+            try:
+                done, pending = await asyncio.wait(
+                    wait_tasks,
+                    timeout=poll_interval,
+                    return_when=asyncio.FIRST_COMPLETED,
+                )
+            finally:
+                for task in wait_tasks:
+                    if not task.done():
+                        task.cancel()
 
-        if cancel_event is not None and cancel_event.is_set():
-            return False
+            if cancel_event is not None and cancel_event.is_set():
+                return False
 
-        if event.is_set():
-            event.clear()
-        return True
+            if event.is_set():
+                event.clear()
+                return True
+
+            added = await self._refresh_lane_from_disk(lane_id)
+            if added:
+                return True
 
     async def list_items(self, lane_id: str) -> list[PmaQueueItem]:
         path = self._lane_queue_path(lane_id)
@@ -253,10 +317,11 @@ class PmaQueue:
 
         items: list[PmaQueueItem] = []
         async with self._ensure_lane_lock(lane_id):
-            try:
-                content = path.read_text(encoding="utf-8")
-            except OSError:
-                return []
+            with file_lock(self._lane_queue_lock_path(lane_id)):
+                try:
+                    content = path.read_text(encoding="utf-8")
+                except OSError:
+                    return []
 
             for line in content.strip().splitlines():
                 line = line.strip()
@@ -270,6 +335,29 @@ class PmaQueue:
 
         return items
 
+    async def _refresh_lane_from_disk(self, lane_id: str) -> int:
+        items = await self.list_items(lane_id)
+        if not items:
+            return 0
+
+        known = self._ensure_lane_known_ids(lane_id)
+        new_pending: list[PmaQueueItem] = []
+        for item in items:
+            if item.item_id in known:
+                continue
+            known.add(item.item_id)
+            if item.state == QueueItemState.PENDING:
+                new_pending.append(item)
+
+        if not new_pending:
+            return 0
+
+        queue = self._ensure_lane_queue(lane_id)
+        for item in new_pending:
+            await queue.put(item)
+        self._ensure_lane_event(lane_id).set()
+        return len(new_pending)
+
     async def _find_by_idempotency_key(
         self, lane_id: str, idempotency_key: str
     ) -> Optional[PmaQueueItem]:
@@ -324,6 +412,67 @@ class PmaQueue:
                 if updated:
                     path.write_text("\n".join(lines) + "\n", encoding="utf-8")
 
+    def _append_to_file_sync(self, item: PmaQueueItem) -> None:
+        path = self._lane_queue_path(item.lane_id)
+        with file_lock(self._lane_queue_lock_path(item.lane_id)):
+            path.parent.mkdir(parents=True, exist_ok=True)
+            line = json.dumps(item.to_dict(), separators=(",", ":")) + "\n"
+            with path.open("a", encoding="utf-8") as f:
+                f.write(line)
+
+    def _read_items_sync(self, lane_id: str) -> list[PmaQueueItem]:
+        path = self._lane_queue_path(lane_id)
+        if not path.exists():
+            return []
+
+        items: list[PmaQueueItem] = []
+        with file_lock(self._lane_queue_lock_path(lane_id)):
+            try:
+                content = path.read_text(encoding="utf-8")
+            except OSError:
+                return []
+
+        for line in content.strip().splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                data = json.loads(line)
+                items.append(PmaQueueItem.from_dict(data))
+            except (json.JSONDecodeError, ValueError):
+                continue
+
+        return items
+
+    def _find_by_idempotency_key_sync(
+        self, lane_id: str, idempotency_key: str
+    ) -> Optional[PmaQueueItem]:
+        items = self._read_items_sync(lane_id)
+        for item in items:
+            if item.idempotency_key == idempotency_key and item.state in (
+                QueueItemState.PENDING,
+                QueueItemState.RUNNING,
+            ):
+                return item
+        return None
+
+    def _notify_in_memory_enqueue(self, item: PmaQueueItem) -> None:
+        self._ensure_lane_known_ids(item.lane_id).add(item.item_id)
+        queue = self._lane_queues.get(item.lane_id)
+        event = self._lane_events.get(item.lane_id)
+        loop = self._loop
+        if loop is None or loop.is_closed() or queue is None or event is None:
+            return
+
+        def _enqueue() -> None:
+            queue.put_nowait(item)
+            event.set()
+
+        try:
+            loop.call_soon_threadsafe(_enqueue)
+        except RuntimeError:
+            return
+
     async def get_lane_stats(self, lane_id: str) -> dict[str, Any]:
         items = await self.list_items(lane_id)
         by_state: dict[str, int] = {}

codex_autorunner/core/pma_reactive.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import json
+import logging
+import time
+from pathlib import Path
+from typing import Any, Optional
+
+from .locks import file_lock
+from .utils import atomic_write
+
+PMA_REACTIVE_STATE_FILENAME = "reactive_state.json"
+
+logger = logging.getLogger(__name__)
+
+
+def default_pma_reactive_state() -> dict[str, Any]:
+    return {
+        "version": 1,
+        "last_enqueued": {},
+    }
+
+
+class PmaReactiveStore:
+    def __init__(self, hub_root: Path) -> None:
+        self._path = (
+            hub_root / ".codex-autorunner" / "pma" / PMA_REACTIVE_STATE_FILENAME
+        )
+
+    def _lock_path(self) -> Path:
+        return self._path.with_suffix(self._path.suffix + ".lock")
+
+    def load(self) -> dict[str, Any]:
+        with file_lock(self._lock_path()):
+            state = self._load_unlocked()
+            if state is not None:
+                return state
+            state = default_pma_reactive_state()
+            self._save_unlocked(state)
+            return state
+
+    def check_and_update(self, key: str, debounce_seconds: int) -> bool:
+        """
+        Return True if enqueue is allowed; otherwise False if debounced.
+        Updates the last_enqueued timestamp when allowed.
+        """
+        now = time.time()
+        with file_lock(self._lock_path()):
+            state = self._load_unlocked() or default_pma_reactive_state()
+            last_enqueued = state.get("last_enqueued")
+            if not isinstance(last_enqueued, dict):
+                last_enqueued = {}
+                state["last_enqueued"] = last_enqueued
+
+            last = last_enqueued.get(key)
+            if debounce_seconds > 0 and isinstance(last, (int, float)):
+                if now - float(last) < debounce_seconds:
+                    return False
+
+            last_enqueued[key] = now
+            self._save_unlocked(state)
+        return True
+
+    def _load_unlocked(self) -> Optional[dict[str, Any]]:
+        if not self._path.exists():
+            return None
+        try:
+            raw = self._path.read_text(encoding="utf-8")
+        except OSError as exc:
+            logger.warning(
+                "Failed to read PMA reactive state at %s: %s", self._path, exc
+            )
+            return None
+        try:
+            data = json.loads(raw)
+        except json.JSONDecodeError:
+            return default_pma_reactive_state()
+        if not isinstance(data, dict):
+            return default_pma_reactive_state()
+        return data
+
+    def _save_unlocked(self, state: dict[str, Any]) -> None:
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        atomic_write(self._path, json.dumps(state, indent=2) + "\n")
+
+
+__all__ = [
+    "PMA_REACTIVE_STATE_FILENAME",
+    "PmaReactiveStore",
+    "default_pma_reactive_state",
+]