codex-autorunner 0.1.2__py3-none-any.whl → 1.0.0__py3-none-any.whl

codex_autorunner/core/flows/worker_process.py

@@ -0,0 +1,242 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import IO, Literal, Optional, Tuple
+
+_WORKER_METADATA_FILENAME = "worker.json"
+
+
+@dataclass
+class FlowWorkerHealth:
+    status: Literal["absent", "alive", "dead", "invalid", "mismatch"]
+    pid: Optional[int]
+    cmdline: list[str]
+    artifact_path: Path
+    message: Optional[str] = None
+
+    @property
+    def is_alive(self) -> bool:
+        return self.status == "alive"
+
+
+def _normalized_run_id(run_id: str) -> str:
+    return str(uuid.UUID(str(run_id)))
+
+
+def _worker_artifacts_dir(
+    repo_root: Path, run_id: str, artifacts_root: Optional[Path] = None
+) -> Path:
+    repo_root = repo_root.resolve()
+    base_artifacts = (
+        artifacts_root
+        if artifacts_root is not None
+        else repo_root / ".codex-autorunner" / "flows"
+    )
+    artifacts_dir = base_artifacts / _normalized_run_id(run_id)
+    artifacts_dir.mkdir(parents=True, exist_ok=True)
+    return artifacts_dir
+
+
+def _worker_metadata_path(artifacts_dir: Path) -> Path:
+    return artifacts_dir / _WORKER_METADATA_FILENAME
+
+
+def _build_worker_cmd(entrypoint: str, run_id: str) -> list[str]:
+    normalized_run_id = _normalized_run_id(run_id)
+    return [
+        sys.executable,
+        "-m",
+        entrypoint,
+        "flow",
+        "worker",
+        "--run-id",
+        normalized_run_id,
+    ]
+
+
+def _pid_is_running(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        # Process exists but we may not own it.
+        return True
+    except OSError:
+        return False
+    return True
+
+
+def _read_process_cmdline(pid: int) -> list[str] | None:
+    proc_path = Path(f"/proc/{pid}/cmdline")
+    if proc_path.exists():
+        try:
+            raw = proc_path.read_bytes()
+            return [part for part in raw.decode().split("\0") if part]
+        except Exception:
+            pass
+
+    try:
+        out = subprocess.check_output(
+            ["ps", "-p", str(pid), "-o", "command="],
+            stderr=subprocess.DEVNULL,
+        )
+        cmd = out.decode().strip()
+        if cmd:
+            return cmd.split()
+    except Exception:
+        return None
+    return None
+
+
+def _cmdline_matches(expected: list[str], actual: list[str]) -> bool:
+    if not expected or not actual:
+        return False
+    if len(actual) >= len(expected) and actual[-len(expected) :] == expected:
+        return True
+    expected_str = " ".join(expected)
+    actual_str = " ".join(actual)
+    return expected_str in actual_str
+
+
+def _write_worker_metadata(path: Path, pid: int, cmd: list[str]) -> None:
+    data = {
+        "pid": pid,
+        "cmd": cmd,
+        "cwd": os.getcwd(),
+    }
+    path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+    # Also emit a plain PID file for quick inspection.
+    pid_path = path.with_suffix(".pid")
+    pid_path.write_text(str(pid), encoding="utf-8")
+
+
+def clear_worker_metadata(artifacts_dir: Path) -> None:
+    for name in (
+        _WORKER_METADATA_FILENAME,
+        f"{Path(_WORKER_METADATA_FILENAME).stem}.pid",
+    ):
+        try:
+            (artifacts_dir / name).unlink()
+        except FileNotFoundError:
+            pass
+        except Exception:
+            pass
+
+
+def check_worker_health(
+    repo_root: Path,
+    run_id: str,
+    *,
+    artifacts_root: Optional[Path] = None,
+    entrypoint: str = "codex_autorunner",
+) -> FlowWorkerHealth:
+    artifacts_dir = _worker_artifacts_dir(repo_root, run_id, artifacts_root)
+    metadata_path = _worker_metadata_path(artifacts_dir)
+
+    if not metadata_path.exists():
+        return FlowWorkerHealth(
+            status="absent",
+            pid=None,
+            cmdline=[],
+            artifact_path=metadata_path,
+            message="worker metadata missing",
+        )
+
+    try:
+        data = json.loads(metadata_path.read_text(encoding="utf-8"))
+        pid = int(data.get("pid")) if data.get("pid") is not None else None
+        cmd = data.get("cmd") or []
+    except Exception:
+        return FlowWorkerHealth(
+            status="invalid",
+            pid=None,
+            cmdline=[],
+            artifact_path=metadata_path,
+            message="worker metadata unreadable",
+        )
+
+    if not pid or pid <= 0:
+        return FlowWorkerHealth(
+            status="invalid",
+            pid=pid,
+            cmdline=cmd if isinstance(cmd, list) else [],
+            artifact_path=metadata_path,
+            message="missing or invalid PID",
+        )
+
+    if not _pid_is_running(pid):
+        return FlowWorkerHealth(
+            status="dead",
+            pid=pid,
+            cmdline=cmd if isinstance(cmd, list) else [],
+            artifact_path=metadata_path,
+            message="worker PID not running",
+        )
+
+    expected_cmd = _build_worker_cmd(entrypoint, run_id)
+    actual_cmd = _read_process_cmdline(pid)
+    if actual_cmd is None:
+        # Can't inspect cmdline; trust the PID check.
+        return FlowWorkerHealth(
+            status="alive",
+            pid=pid,
+            cmdline=cmd if isinstance(cmd, list) else [],
+            artifact_path=metadata_path,
+            message="worker running (cmdline unknown)",
+        )
+
+    if not _cmdline_matches(expected_cmd, actual_cmd):
+        return FlowWorkerHealth(
+            status="mismatch",
+            pid=pid,
+            cmdline=actual_cmd,
+            artifact_path=metadata_path,
+            message="worker PID command does not match expected",
+        )
+
+    return FlowWorkerHealth(
+        status="alive",
+        pid=pid,
+        cmdline=actual_cmd,
+        artifact_path=metadata_path,
+        message="worker running",
+    )
+
+
+def spawn_flow_worker(
+    repo_root: Path,
+    run_id: str,
+    *,
+    artifacts_root: Optional[Path] = None,
+    entrypoint: str = "codex_autorunner",
+) -> Tuple[subprocess.Popen, IO[bytes], IO[bytes]]:
+    """Spawn a detached flow worker with consistent artifacts/log layout."""
+
+    normalized_run_id = _normalized_run_id(run_id)
+    repo_root = repo_root.resolve()
+    artifacts_dir = _worker_artifacts_dir(repo_root, normalized_run_id, artifacts_root)
+
+    stdout_path = artifacts_dir / "worker.out.log"
+    stderr_path = artifacts_dir / "worker.err.log"
+
+    stdout_handle = stdout_path.open("ab")
+    stderr_handle = stderr_path.open("ab")
+
+    cmd = _build_worker_cmd(entrypoint, normalized_run_id)
+
+    proc = subprocess.Popen(
+        cmd,
+        cwd=repo_root,
+        stdout=stdout_handle,
+        stderr=stderr_handle,
+    )
+
+    _write_worker_metadata(_worker_metadata_path(artifacts_dir), proc.pid, cmd)
+    return proc, stdout_handle, stderr_handle

codex_autorunner/core/hub.py

@@ -137,7 +137,8 @@ def load_hub_state(state_path: Path, hub_root: Path) -> HubState:
         import json
 
         payload = json.loads(data)
-    except Exception:
+    except Exception as exc:
+        logger.warning("Failed to parse hub state from %s: %s", state_path, exc)
         return HubState(last_scan_at=None, repos=[])
     last_scan_at = payload.get("last_scan_at")
     repos_payload = payload.get("repos") or []
@@ -168,7 +169,13 @@ def load_hub_state(state_path: Path, hub_root: Path) -> HubState:
                 runner_pid=entry.get("runner_pid"),
             )
             repos.append(repo)
-        except Exception:
+        except Exception as exc:
+            repo_id = entry.get("id", "unknown")
+            logger.warning(
+                "Failed to load repo snapshot for id=%s from hub state: %s",
+                repo_id,
+                exc,
+            )
             continue
     return HubState(last_scan_at=last_scan_at, repos=repos)
 
@@ -759,10 +766,10 @@ class HubSupervisor:
         if not repo:
             raise ValueError(f"Repo {repo_id} not found in manifest")
         repo_root = (self.hub_config.root / repo.path).resolve()
-        state_path = repo_root / ".codex-autorunner" / "state.sqlite3"
-        if not allow_uninitialized and not state_path.exists():
+        tickets_dir = repo_root / ".codex-autorunner" / "tickets"
+        if not allow_uninitialized and not tickets_dir.exists():
             raise ValueError(f"Repo {repo_id} is not initialized")
-        if not state_path.exists():
+        if not tickets_dir.exists():
             return None
         repo_config = derive_repo_config(self.hub_config, repo_root, load_env=False)
         runner = RepoRunner(
@@ -781,7 +788,7 @@ class HubSupervisor:
         records: List[DiscoveryRecord] = []
         for entry in manifest.repos:
             repo_path = (self.hub_config.root / entry.path).resolve()
-            initialized = (repo_path / ".codex-autorunner" / "state.sqlite3").exists()
+            initialized = (repo_path / ".codex-autorunner" / "tickets").exists()
             records.append(
                 DiscoveryRecord(
                     repo=entry,
@@ -821,9 +828,8 @@ class HubSupervisor:
         lock_status = read_lock_status(lock_path)
 
         runner_state: Optional[RunnerState] = None
-        state_path = repo_path / ".codex-autorunner" / "state.sqlite3"
-        if record.initialized and state_path.exists():
-            runner_state = load_state(state_path)
+        if record.initialized:
+            runner_state = load_state(repo_path / ".codex-autorunner" / "state.sqlite3")
 
         is_clean: Optional[bool] = None
         if record.exists_on_disk and git_available(repo_path):

codex_autorunner/core/locks.py

@@ -1,5 +1,6 @@
 import errno
 import json
+import logging
 import os
 import socket
 import subprocess
@@ -27,6 +28,7 @@ class LockAssessment:
 
 
 DEFAULT_RUNNER_CMD_HINTS = ("codex_autorunner.cli", "codex-autorunner", "car ")
+logger = logging.getLogger(__name__)
 
 
 def process_alive(pid: int) -> bool:
@@ -48,6 +50,7 @@ def process_is_zombie(pid: int) -> bool:
             check=False,
         )
     except Exception:
+        logger.debug("Failed to check process status for pid %s", pid, exc_info=True)
         return False
     if result.returncode != 0:
         return False
@@ -65,6 +68,7 @@ def process_command(pid: int) -> Optional[str]:
             check=False,
         )
     except Exception:
+        logger.debug("Failed to inspect process command for pid %s", pid, exc_info=True)
         return None
     if result.returncode != 0:
         return None

codex_autorunner/core/prompt.py

@@ -15,12 +15,20 @@ def _display_path(root: Path, path: Path) -> str:
 
 
 def build_doc_paths(config: Config) -> Mapping[str, str]:
+    def _safe_path(*keys: str) -> str:
+        for key in keys:
+            try:
+                return _display_path(config.root, config.doc_path(key))
+            except KeyError:
+                continue
+        return ""
+
     return {
-        "todo": _display_path(config.root, config.doc_path("todo")),
-        "progress": _display_path(config.root, config.doc_path("progress")),
-        "opinions": _display_path(config.root, config.doc_path("opinions")),
-        "spec": _display_path(config.root, config.doc_path("spec")),
-        "summary": _display_path(config.root, config.doc_path("summary")),
+        "todo": _safe_path("todo", "active_context"),
+        "progress": _safe_path("progress", "decisions"),
+        "opinions": _safe_path("opinions"),
+        "spec": _safe_path("spec"),
+        "summary": _safe_path("summary"),
     }
 
 
@@ -64,8 +72,8 @@ def build_final_summary_prompt(
 
     doc_paths = build_doc_paths(config)
     doc_contents = {
-        "todo": docs.read_doc("todo"),
-        "progress": docs.read_doc("progress"),
+        "todo": docs.read_doc("todo") or docs.read_doc("active_context"),
+        "progress": docs.read_doc("progress") or docs.read_doc("decisions"),
         "opinions": docs.read_doc("opinions"),
         "spec": docs.read_doc("spec"),
         "summary": docs.read_doc("summary"),

codex_autorunner/core/redaction.py

@@ -0,0 +1,29 @@
+import re
+from typing import List, Tuple
+
+_REDACTIONS: List[Tuple[re.Pattern[str], str]] = [
+    # OpenAI-like keys.
+    (re.compile(r"\bsk-[A-Za-z0-9]{20,}\b"), "sk-[REDACTED]"),
+    # GitHub personal access tokens.
+    (re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b"), "gh_[REDACTED]"),
+    # AWS access key ids (best-effort).
+    (re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "AKIA[REDACTED]"),
+    # JWT-ish blobs.
+    (
+        re.compile(
+            r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"
+        ),
+        "[JWT_REDACTED]",
+    ),
+]
+
+
+def redact_text(text: str) -> str:
+    redacted = text
+    for pattern, replacement in _REDACTIONS:
+        redacted = pattern.sub(replacement, redacted)
+    return redacted
+
+
+def get_redaction_patterns() -> List[str]:
+    return [pattern.pattern for pattern, _ in _REDACTIONS]

codex_autorunner/core/review_context.py

@@ -97,13 +97,10 @@ def build_spec_progress_review_context(
         remaining = 0
 
     def doc_label(name: str) -> str:
-        mapping = {
-            "spec": "SPEC.md",
-            "progress": "PROGRESS.md",
-            "todo": "TODO.md",
-            "summary": "SUMMARY.md",
-        }
-        return mapping.get(name.lower(), name)
+        try:
+            return engine.config.doc_path(name).relative_to(engine.repo_root).as_posix()
+        except Exception:
+            return name
 
     def read_doc(name: str) -> str:
         try:
@@ -122,7 +119,7 @@ def build_spec_progress_review_context(
 
     primary_list = [doc for doc in primary_docs if isinstance(doc, str)] or [
         "spec",
-        "progress",
+        "active_context",
     ]
     primary_set = {doc.lower() for doc in primary_list}
 

codex_autorunner/core/run_index.py

@@ -197,6 +197,8 @@ class RunIndexStore:
         *,
         log_path: str,
         run_log_path: str,
+        actor: Optional[dict[str, Any]] = None,
+        mode: Optional[dict[str, Any]] = None,
     ) -> dict[str, Any]:
         with open_sqlite(self._path) as conn:
             self._ensure_schema(conn)
@@ -207,6 +209,10 @@ class RunIndexStore:
                 entry["started_at"] = now_iso()
                 entry["log_path"] = log_path
                 entry["run_log_path"] = run_log_path
+                if actor is not None:
+                    entry["actor"] = actor
+                if mode is not None:
+                    entry["mode"] = mode
             elif marker == "end":
                 entry["end_offset"] = offset[1] if offset else None
                 entry["finished_at"] = now_iso()

codex_autorunner/core/runner_process.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 import atexit
+import logging
 import subprocess
 import sys
 from pathlib import Path
 from typing import Set
 
 _process_registry: Set[subprocess.Popen] = set()
+logger = logging.getLogger(__name__)
 
 
 def build_runner_cmd(repo_root: Path, *, action: str, once: bool = False) -> list[str]:
@@ -41,14 +43,15 @@ def cleanup_processes() -> None:
             try:
                 proc.terminate()
             except Exception:
-                pass
+                logger.debug("Failed to terminate runner process", exc_info=True)
             try:
                 proc.wait(timeout=5)
             except Exception:
+                logger.debug("Runner process wait timed out", exc_info=True)
                 try:
                     proc.kill()
                 except Exception:
-                    pass
+                    logger.debug("Failed to kill runner process", exc_info=True)
     _process_registry.clear()
 
 

codex_autorunner/core/state.py

@@ -1,6 +1,5 @@
 import dataclasses
 import json
-import logging
 from contextlib import contextmanager
 from datetime import datetime, timezone
 from pathlib import Path
@@ -9,8 +8,6 @@ from typing import Any, Iterator, Optional
 from .locks import file_lock
 from .sqlite_utils import open_sqlite
 
-_logger = logging.getLogger(__name__)
-
 
 @dataclasses.dataclass
 class RunnerState:
@@ -100,75 +97,6 @@ def now_iso() -> str:
     return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
 
-def _coerce_int(value: Any) -> Optional[int]:
-    if isinstance(value, int) and not isinstance(value, bool):
-        return value
-    return None
-
-
-def _coerce_str(value: Any) -> Optional[str]:
-    if isinstance(value, str) and value:
-        return value
-    return None
-
-
-def _load_legacy_state_json(path: Path) -> Optional[RunnerState]:
-    try:
-        payload = json.loads(path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError):
-        return None
-    if not isinstance(payload, dict):
-        return None
-    state = RunnerState(
-        last_run_id=_coerce_int(payload.get("last_run_id")),
-        status=_coerce_str(payload.get("status")) or "idle",
-        last_exit_code=_coerce_int(payload.get("last_exit_code")),
-        last_run_started_at=_coerce_str(payload.get("last_run_started_at")),
-        last_run_finished_at=_coerce_str(payload.get("last_run_finished_at")),
-        autorunner_agent_override=_coerce_str(payload.get("autorunner_agent_override")),
-        autorunner_model_override=_coerce_str(payload.get("autorunner_model_override")),
-        autorunner_effort_override=_coerce_str(
-            payload.get("autorunner_effort_override")
-        ),
-        autorunner_approval_policy=_coerce_str(
-            payload.get("autorunner_approval_policy")
-        ),
-        autorunner_sandbox_mode=_coerce_str(payload.get("autorunner_sandbox_mode")),
-        autorunner_workspace_write_network=(
-            payload.get("autorunner_workspace_write_network")
-            if isinstance(payload.get("autorunner_workspace_write_network"), bool)
-            else None
-        ),
-        runner_stop_after_runs=_coerce_int(payload.get("runner_stop_after_runs")),
-        runner_pid=_coerce_int(payload.get("runner_pid")),
-    )
-    sessions: dict[str, SessionRecord] = {}
-    sessions_payload = payload.get("sessions")
-    if isinstance(sessions_payload, dict):
-        for session_id, record_payload in sessions_payload.items():
-            if not isinstance(session_id, str) or not session_id:
-                continue
-            record = (
-                SessionRecord.from_dict(record_payload)
-                if isinstance(record_payload, dict)
-                else None
-            )
-            if record is not None:
-                sessions[session_id] = record
-    repo_to_session: dict[str, str] = {}
-    repo_payload = payload.get("repo_to_session")
-    if isinstance(repo_payload, dict):
-        for repo_key, session_id in repo_payload.items():
-            if not isinstance(repo_key, str) or not repo_key:
-                continue
-            if not isinstance(session_id, str) or not session_id:
-                continue
-            repo_to_session[repo_key] = session_id
-    state.sessions = sessions
-    state.repo_to_session = repo_to_session
-    return state
-
-
 def _ensure_state_schema(conn) -> None:
     with conn:
         conn.execute(
@@ -270,22 +198,6 @@ def _apply_overrides(state: RunnerState, raw: Optional[str]) -> None:
 
 
 def load_state(state_path: Path) -> RunnerState:
-    legacy_path = state_path.with_name("state.json")
-    # Legacy JSON migration (remove after old state.json is retired).
-    if not state_path.exists() and legacy_path.exists():
-        migrated = _load_legacy_state_json(legacy_path)
-        if migrated is not None:
-            try:
-                save_state(state_path, migrated)
-                return migrated
-            except Exception:
-                _logger.warning(
-                    "Failed to migrate legacy state from %s to %s. The original JSON file is preserved.",
-                    legacy_path,
-                    state_path,
-                    exc_info=True,
-                )
-                raise
     with open_sqlite(state_path) as conn:
         _ensure_state_schema(conn)
         row = conn.execute(

codex_autorunner/core/static_assets.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from contextlib import ExitStack
+from importlib import resources
+from pathlib import Path
+from typing import Optional
+
+# Keep the required asset list close to the core boundary so core modules do not
+# import from codex_autorunner.web.*
+_REQUIRED_STATIC_ASSETS = (
+    "index.html",
+    "styles.css",
+    "bootstrap.js",
+    "loader.js",
+    "app.js",
+    "vendor/xterm.js",
+    "vendor/xterm-addon-fit.js",
+    "vendor/xterm.css",
+)
+
+
+def missing_static_assets(static_dir: Path) -> list[str]:
+    missing: list[str] = []
+    for rel_path in _REQUIRED_STATIC_ASSETS:
+        try:
+            if not (static_dir / rel_path).exists():
+                missing.append(rel_path)
+        except OSError:
+            missing.append(rel_path)
+    return missing
+
+
+def resolve_static_dir() -> tuple[Path, Optional[ExitStack]]:
+    """Locate packaged static assets without importing codex_autorunner.web."""
+
+    static_root = resources.files("codex_autorunner").joinpath("static")
+    if isinstance(static_root, Path):
+        if static_root.exists():
+            return static_root, None
+        fallback = Path(__file__).resolve().parent.parent / "static"
+        return fallback, None
+
+    stack = ExitStack()
+    try:
+        static_path = stack.enter_context(resources.as_file(static_root))
+    except Exception:
+        stack.close()
+        fallback = Path(__file__).resolve().parent.parent / "static"
+        return fallback, None
+    if static_path.exists():
+        return static_path, stack
+
+    stack.close()
+    fallback = Path(__file__).resolve().parent.parent / "static"
+    return fallback, None