codex-autorunner 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

codex_autorunner/core/sqlite_utils.py

@@ -13,19 +13,28 @@ SQLITE_PRAGMAS = (
     "PRAGMA temp_store=MEMORY;",
 )
 
+SQLITE_PRAGMAS_DURABLE = (
+    "PRAGMA journal_mode=WAL;",
+    "PRAGMA synchronous=FULL;",
+    "PRAGMA foreign_keys=ON;",
+    "PRAGMA busy_timeout=5000;",
+    "PRAGMA temp_store=MEMORY;",
+)
+
 
-def connect_sqlite(path: Path) -> sqlite3.Connection:
+def connect_sqlite(path: Path, durable: bool = False) -> sqlite3.Connection:
     path.parent.mkdir(parents=True, exist_ok=True)
     conn = sqlite3.connect(path)
     conn.row_factory = sqlite3.Row
-    for pragma in SQLITE_PRAGMAS:
+    pragmas = SQLITE_PRAGMAS_DURABLE if durable else SQLITE_PRAGMAS
+    for pragma in pragmas:
         conn.execute(pragma)
     return conn
 
 
 @contextmanager
-def open_sqlite(path: Path) -> Iterator[sqlite3.Connection]:
-    conn = connect_sqlite(path)
+def open_sqlite(path: Path, durable: bool = False) -> Iterator[sqlite3.Connection]:
+    conn = connect_sqlite(path, durable=durable)
     try:
         yield conn
     finally:

codex_autorunner/core/state.py

@@ -1,12 +1,12 @@
 import dataclasses
 import json
 from contextlib import contextmanager
-from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Iterator, Optional
 
 from .locks import file_lock
 from .sqlite_utils import open_sqlite
+from .time_utils import now_iso
 
 
 @dataclasses.dataclass
@@ -93,10 +93,6 @@ class SessionRecord:
         }
 
 
-def now_iso() -> str:
-    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-
 def _ensure_state_schema(conn) -> None:
     with conn:
         conn.execute(
@@ -197,8 +193,8 @@ def _apply_overrides(state: RunnerState, raw: Optional[str]) -> None:
         state.runner_stop_after_runs = runner_stop_after_runs
 
 
-def load_state(state_path: Path) -> RunnerState:
-    with open_sqlite(state_path) as conn:
+def load_state(state_path: Path, durable: bool = False) -> RunnerState:
+    with open_sqlite(state_path, durable=durable) as conn:
         _ensure_state_schema(conn)
         row = conn.execute(
             """
@@ -253,9 +249,9 @@ def load_state(state_path: Path) -> RunnerState:
         return state
 
 
-def save_state(state_path: Path, state: RunnerState) -> None:
+def save_state(state_path: Path, state: RunnerState, durable: bool = False) -> None:
     overrides_json = _encode_overrides(state)
-    with open_sqlite(state_path) as conn:
+    with open_sqlite(state_path, durable=durable) as conn:
         _ensure_state_schema(conn)
         updated_at = now_iso()
         with conn:
@@ -343,9 +339,10 @@ def persist_session_registry(
     state_path: Path,
     sessions: dict[str, SessionRecord],
     repo_to_session: dict[str, str],
+    durable: bool = False,
 ) -> None:
     with state_lock(state_path):
-        with open_sqlite(state_path) as conn:
+        with open_sqlite(state_path, durable=durable) as conn:
             _ensure_state_schema(conn)
             with conn:
                 conn.execute("DELETE FROM sessions")

codex_autorunner/core/state_roots.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Mapping, Optional
+
+from .path_utils import ConfigPathError, resolve_config_path
+
+GLOBAL_STATE_ROOT_ENV = "CAR_GLOBAL_STATE_ROOT"
+
+
+def _read_global_root_from_config(raw: Optional[Mapping[str, Any]]) -> Optional[str]:
+    if not raw:
+        return None
+    state_roots = raw.get("state_roots")
+    if not isinstance(state_roots, Mapping):
+        return None
+    value = state_roots.get("global")
+    return value if isinstance(value, str) else None
+
+
+def resolve_global_state_root(
+    *,
+    config: Optional[Any] = None,
+    repo_root: Optional[Path] = None,
+    scope: str = "state_roots.global",
+) -> Path:
+    """Resolve the global state root used for cross-repo caches and locks."""
+    base_root = repo_root
+    raw_config = None
+    if config is not None:
+        base_root = getattr(config, "root", None) or base_root
+        raw_config = getattr(config, "raw", None)
+
+    if base_root is None:
+        base_root = Path.cwd()
+
+    env_value = os.environ.get(GLOBAL_STATE_ROOT_ENV)
+    raw_value = env_value or _read_global_root_from_config(raw_config)
+    if raw_value:
+        try:
+            return resolve_config_path(
+                raw_value,
+                base_root,
+                allow_absolute=True,
+                allow_home=True,
+                scope=scope,
+            )
+        except ConfigPathError as exc:
+            raise ConfigPathError(str(exc), path=raw_value, scope=scope) from exc
+
+    return Path.home() / ".codex-autorunner"
+
+
+def resolve_repo_state_root(repo_root: Path) -> Path:
+    """Return the repo-local state root (.codex-autorunner)."""
+    return repo_root / ".codex-autorunner"
+
+
+def resolve_hub_templates_root(hub_root: Path) -> Path:
+    """Return the hub-scoped templates root."""
+    return hub_root / ".codex-autorunner" / "templates"

codex_autorunner/core/supervisor_protocol.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+class AppServerSupervisorProtocol:
+    async def get_client(self, workspace_root: Path) -> Any:
+        raise NotImplementedError
+
+    async def close_all(self) -> None:
+        raise NotImplementedError
+
+    async def prune_idle(self) -> int:
+        raise NotImplementedError

codex_autorunner/core/templates/__init__.py

@@ -0,0 +1,39 @@
+from .git_mirror import (
+    FetchedTemplate,
+    NetworkUnavailableError,
+    RefNotFoundError,
+    RepoNotConfiguredError,
+    TemplateNotFoundError,
+    TemplateRef,
+    ensure_git_mirror,
+    fetch_template,
+    parse_template_ref,
+)
+from .provenance import inject_provenance
+from .scan_cache import (
+    TemplateScanRecord,
+    get_scan_record,
+    scan_lock,
+    scan_lock_path,
+    scan_record_path,
+    write_scan_record,
+)
+
+__all__ = [
+    "FetchedTemplate",
+    "NetworkUnavailableError",
+    "RefNotFoundError",
+    "RepoNotConfiguredError",
+    "TemplateNotFoundError",
+    "TemplateRef",
+    "ensure_git_mirror",
+    "fetch_template",
+    "parse_template_ref",
+    "inject_provenance",
+    "TemplateScanRecord",
+    "get_scan_record",
+    "scan_lock",
+    "scan_lock_path",
+    "scan_record_path",
+    "write_scan_record",
+]

codex_autorunner/core/templates/git_mirror.py

@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import dataclasses
+from pathlib import Path
+from typing import Optional
+
+from ..config import TemplateRepoConfig
+from ..git_utils import GitError, run_git
+from ..state_roots import resolve_hub_templates_root
+
+
+class RepoNotConfiguredError(Exception):
+    def __init__(self, repo_id: str, *, detail: Optional[str] = None) -> None:
+        message = f"Template repo not configured: {repo_id}"
+        if detail:
+            message = f"{message} ({detail})"
+        super().__init__(message)
+        self.repo_id = repo_id
+        self.detail = detail
+
+
+class TemplateNotFoundError(Exception):
+    def __init__(self, repo_id: str, path: str, ref: str) -> None:
+        super().__init__(f"Template not found: repo_id={repo_id} path={path} ref={ref}")
+        self.repo_id = repo_id
+        self.path = path
+        self.ref = ref
+
+
+class RefNotFoundError(Exception):
+    def __init__(self, repo_id: str, ref: str) -> None:
+        super().__init__(f"Ref not found: repo_id={repo_id} ref={ref}")
+        self.repo_id = repo_id
+        self.ref = ref
+
+
+class NetworkUnavailableError(Exception):
+    def __init__(
+        self,
+        repo_id: str,
+        ref: str,
+        path: str,
+        *,
+        detail: Optional[str] = None,
+    ) -> None:
+        message = (
+            "Template fetch failed and cache is unavailable: "
+            f"repo_id={repo_id} ref={ref} path={path}"
+        )
+        if detail:
+            message = f"{message} ({detail})"
+        super().__init__(message)
+        self.repo_id = repo_id
+        self.ref = ref
+        self.path = path
+        self.detail = detail
+
+
+@dataclasses.dataclass(frozen=True)
+class TemplateRef:
+    repo_id: str
+    path: str
+    ref: Optional[str]
+
+
+@dataclasses.dataclass(frozen=True)
+class FetchedTemplate:
+    repo_id: str
+    url: str
+    trusted: bool
+    path: str
+    ref: str
+    commit_sha: str
+    blob_sha: str
+    content: str
+
+
+def parse_template_ref(raw: str) -> TemplateRef:
+    """Parse canonical template reference strings: REPO_ID:PATH[@REF]."""
+    if ":" not in raw:
+        raise ValueError("template ref must be formatted as REPO_ID:PATH[@REF]")
+    repo_id, remainder = raw.split(":", 1)
+    if not repo_id:
+        raise ValueError("template ref missing repo_id")
+    if not remainder:
+        raise ValueError("template ref missing path")
+
+    path: str
+    ref: Optional[str]
+    if "@" in remainder:
+        path, ref = remainder.rsplit("@", 1)
+        if not ref:
+            raise ValueError("template ref missing ref after '@'")
+    else:
+        path, ref = remainder, None
+
+    if not path:
+        raise ValueError("template ref missing path")
+
+    return TemplateRef(repo_id=repo_id, path=path, ref=ref)
+
+
+def ensure_git_mirror(repo: TemplateRepoConfig, hub_root: Path) -> Path:
+    templates_root = resolve_hub_templates_root(hub_root)
+    mirror_path = templates_root / "git" / f"{repo.id}.git"
+    if mirror_path.exists():
+        _ensure_origin_remote(mirror_path, repo.url)
+        return mirror_path
+
+    mirror_path.parent.mkdir(parents=True, exist_ok=True)
+    run_git(["init", "--bare", str(mirror_path)], mirror_path.parent, check=True)
+    _ensure_origin_remote(mirror_path, repo.url)
+    return mirror_path
+
+
+def _ensure_origin_remote(mirror_path: Path, url: str) -> None:
+    try:
+        proc = run_git(["remote", "get-url", "origin"], mirror_path, check=False)
+    except GitError:
+        proc = None
+    if proc and proc.returncode == 0:
+        current = (proc.stdout or "").strip()
+        if current and current != url:
+            run_git(["remote", "set-url", "origin", url], mirror_path, check=True)
+    else:
+        run_git(["remote", "add", "origin", url], mirror_path, check=True)
+    _configure_mirror_remote(mirror_path)
+
+
+def _configure_mirror_remote(mirror_path: Path) -> None:
+    run_git(
+        ["config", "remote.origin.fetch", "+refs/*:refs/*"],
+        mirror_path,
+        check=True,
+    )
+    run_git(["config", "remote.origin.mirror", "true"], mirror_path, check=True)
+
+
+def fetch_template(
+    *,
+    repo: TemplateRepoConfig,
+    hub_root: Path,
+    template_ref: str,
+    fetch_timeout_seconds: int = 30,
+) -> FetchedTemplate:
+    parsed = parse_template_ref(template_ref)
+    if parsed.repo_id != repo.id:
+        raise RepoNotConfiguredError(
+            parsed.repo_id,
+            detail=f"expected repo_id {repo.id}",
+        )
+
+    ref = parsed.ref or repo.default_ref
+    mirror_path = ensure_git_mirror(repo, hub_root)
+
+    fetch_error: Optional[str] = None
+    try:
+        run_git(
+            ["fetch", "--prune", "origin"],
+            mirror_path,
+            timeout_seconds=fetch_timeout_seconds,
+            check=True,
+        )
+    except GitError as exc:
+        fetch_error = str(exc)
+
+    try:
+        commit_sha = _resolve_commit(mirror_path, repo.id, ref)
+        blob_sha = _resolve_blob(mirror_path, commit_sha, parsed.path, repo.id, ref)
+        content = _read_blob(mirror_path, blob_sha)
+    except (RefNotFoundError, TemplateNotFoundError) as exc:
+        if fetch_error:
+            raise NetworkUnavailableError(
+                repo.id,
+                ref,
+                parsed.path,
+                detail=fetch_error,
+            ) from exc
+        raise
+
+    return FetchedTemplate(
+        repo_id=repo.id,
+        url=repo.url,
+        trusted=repo.trusted,
+        path=parsed.path,
+        ref=ref,
+        commit_sha=commit_sha,
+        blob_sha=blob_sha,
+        content=content,
+    )
+
+
+def _resolve_commit(mirror_path: Path, repo_id: str, ref: str) -> str:
+    try:
+        proc = run_git(
+            ["rev-parse", f"{ref}^{{commit}}"],
+            mirror_path,
+            check=True,
+        )
+    except GitError as exc:
+        raise RefNotFoundError(repo_id, ref) from exc
+    return (proc.stdout or "").strip()
+
+
+def _resolve_blob(
+    mirror_path: Path,
+    commit_sha: str,
+    path: str,
+    repo_id: str,
+    ref: str,
+) -> str:
+    try:
+        proc = run_git(
+            ["ls-tree", commit_sha, "--", path],
+            mirror_path,
+            check=True,
+        )
+    except GitError as exc:
+        raise TemplateNotFoundError(repo_id, path, ref) from exc
+
+    raw = (proc.stdout or "").strip()
+    if not raw:
+        raise TemplateNotFoundError(repo_id, path, ref)
+
+    # Format: "<mode> <type> <sha>\t<path>"
+    parts = raw.split()
+    if len(parts) < 3:
+        raise TemplateNotFoundError(repo_id, path, ref)
+    return parts[2]
+
+
+def _read_blob(mirror_path: Path, blob_sha: str) -> str:
+    proc = run_git(["cat-file", "-p", blob_sha], mirror_path, check=True)
+    return proc.stdout or ""

codex_autorunner/core/templates/provenance.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from typing import Optional
+
+import yaml
+
+from .git_mirror import FetchedTemplate
+from .scan_cache import TemplateScanRecord
+
+
+def inject_provenance(
+    content: str,
+    fetched: FetchedTemplate,
+    scan_record: Optional[TemplateScanRecord],
+) -> str:
+    """Inject deterministic provenance keys into ticket frontmatter.
+
+    Args:
+        content: Ticket markdown content
+        fetched: The fetched template metadata
+        scan_record: Optional scan record for the template
+
+    Returns:
+        Updated markdown content with provenance keys in frontmatter
+
+    Notes:
+        - Does not embed scan evidence
+        - Does not add timestamps (avoids nondeterminism)
+        - Creates frontmatter if missing
+        - Merges with existing frontmatter without clobbering unrelated keys
+    """
+    from ...tickets.frontmatter import split_markdown_frontmatter
+
+    fm_yaml, body = split_markdown_frontmatter(content)
+
+    data = {}
+    if fm_yaml is not None:
+        try:
+            parsed = yaml.safe_load(fm_yaml)
+            if isinstance(parsed, dict):
+                data = parsed
+        except yaml.YAMLError:
+            pass
+
+    data["template"] = f"{fetched.repo_id}:{fetched.path}@{fetched.ref}"
+    data["template_commit"] = fetched.commit_sha
+    data["template_blob"] = fetched.blob_sha
+    data["template_trusted"] = fetched.trusted
+
+    if scan_record is not None:
+        data["template_scan"] = scan_record.decision
+    else:
+        data["template_scan"] = "skipped"
+
+    rendered = yaml.safe_dump(data, sort_keys=False).rstrip()
+    return f"---\n{rendered}\n---\n{body}"

codex_autorunner/core/templates/scan_cache.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import dataclasses
+import json
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Iterator, Optional
+
+from ..locks import FileLock
+from ..state_roots import resolve_hub_templates_root
+from ..utils import atomic_write
+
+
+@dataclasses.dataclass(frozen=True)
+class TemplateScanRecord:
+    blob_sha: str
+    repo_id: str
+    path: str
+    ref: str
+    commit_sha: str
+    trusted: bool
+    decision: str
+    severity: str
+    reason: str
+    evidence: Optional[list[str]]
+    scanned_at: str
+    scanner: Optional[dict[str, str]]
+
+    def to_dict(self, *, include_evidence: bool = True) -> dict[str, Any]:
+        payload: dict[str, Any] = {
+            "blob_sha": self.blob_sha,
+            "repo_id": self.repo_id,
+            "path": self.path,
+            "ref": self.ref,
+            "commit_sha": self.commit_sha,
+            "trusted": self.trusted,
+            "decision": self.decision,
+            "severity": self.severity,
+            "reason": self.reason,
+            "scanned_at": self.scanned_at,
+        }
+        if include_evidence and self.evidence:
+            payload["evidence"] = list(self.evidence)
+        if self.scanner:
+            payload["scanner"] = dict(self.scanner)
+        return payload
+
+    @staticmethod
+    def from_dict(payload: dict[str, Any]) -> "TemplateScanRecord":
+        return TemplateScanRecord(
+            blob_sha=str(payload.get("blob_sha", "")),
+            repo_id=str(payload.get("repo_id", "")),
+            path=str(payload.get("path", "")),
+            ref=str(payload.get("ref", "")),
+            commit_sha=str(payload.get("commit_sha", "")),
+            trusted=bool(payload.get("trusted", False)),
+            decision=str(payload.get("decision", "")),
+            severity=str(payload.get("severity", "")),
+            reason=str(payload.get("reason", "")),
+            evidence=_coerce_evidence(payload.get("evidence")),
+            scanned_at=str(payload.get("scanned_at", "")),
+            scanner=_coerce_scanner(payload.get("scanner")),
+        )
+
+
+def _coerce_evidence(value: Any) -> Optional[list[str]]:
+    if not value:
+        return None
+    if isinstance(value, list):
+        return [str(item) for item in value]
+    return [str(value)]
+
+
+def _coerce_scanner(value: Any) -> Optional[dict[str, str]]:
+    if not value or not isinstance(value, dict):
+        return None
+    return {str(key): str(val) for key, val in value.items()}
+
+
+def _scan_root(hub_root: Path) -> Path:
+    return resolve_hub_templates_root(hub_root) / "scans"
+
+
+def scan_record_path(hub_root: Path, blob_sha: str) -> Path:
+    return _scan_root(hub_root) / f"{blob_sha}.json"
+
+
+def scan_lock_path(hub_root: Path, blob_sha: str) -> Path:
+    return _scan_root(hub_root) / "locks" / f"{blob_sha}.lock"
+
+
+def get_scan_record(hub_root: Path, blob_sha: str) -> Optional[TemplateScanRecord]:
+    path = scan_record_path(hub_root, blob_sha)
+    if not path.exists():
+        return None
+    with path.open("r", encoding="utf-8") as handle:
+        payload = json.load(handle)
+    if not isinstance(payload, dict):
+        return None
+    return TemplateScanRecord.from_dict(payload)
+
+
+def write_scan_record(record: TemplateScanRecord, hub_root: Path) -> None:
+    path = scan_record_path(hub_root, record.blob_sha)
+    payload = record.to_dict(include_evidence=False)
+    if record.evidence:
+        payload["evidence_redacted"] = True
+    atomic_write(path, json.dumps(payload, indent=2) + "\n")
+
+
+@contextmanager
+def scan_lock(hub_root: Path, blob_sha: str) -> Iterator[None]:
+    path = scan_lock_path(hub_root, blob_sha)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    lock = FileLock(path)
+    lock.acquire()
+    try:
+        yield
+    finally:
+        lock.release()

codex_autorunner/core/text_delta_coalescer.py

@@ -41,3 +41,57 @@ class TextDeltaCoalescer:
 
     def clear(self) -> None:
         self._buffer = ""
+
+
+class StreamingTextCoalescer:
+    """
+    Coalesce small streaming text deltas into larger, readable chunks.
+
+    - Flushes whole lines as soon as a newline is observed.
+    - Flushes buffered text when it grows past `min_flush_chars` and ends at a
+      natural boundary (whitespace or sentence punctuation).
+    - Enforces an upper bound on the buffer size to avoid unbounded growth.
+    """
+
+    def __init__(self, min_flush_chars: int = 32, max_buffer_chars: int = 2048):
+        self._buffer: str = ""
+        self._min_flush_chars = max(1, int(min_flush_chars))
+        self._max_buffer_chars = max(self._min_flush_chars, int(max_buffer_chars))
+
+    def add(self, delta: Optional[str]) -> list[str]:
+        chunks: list[str] = []
+        if not isinstance(delta, str) or not delta:
+            return chunks
+
+        self._buffer += delta
+        self._flush_complete_lines(chunks)
+        self._flush_if_boundary(chunks)
+        self._flush_if_oversized(chunks)
+        return chunks
+
+    def flush(self) -> list[str]:
+        if not self._buffer:
+            return []
+        chunk = self._buffer
+        self._buffer = ""
+        return [chunk]
+
+    def _flush_complete_lines(self, chunks: list[str]) -> None:
+        while "\n" in self._buffer:
+            line, remainder = self._buffer.split("\n", 1)
+            # Preserve the newline boundary so the caller keeps the same text.
+            chunks.append(f"{line}\n")
+            self._buffer = remainder
+
+    def _flush_if_boundary(self, chunks: list[str]) -> None:
+        if len(self._buffer) < self._min_flush_chars:
+            return
+        last_char = self._buffer[-1]
+        if last_char.isspace() or last_char in ".!?;:":
+            chunks.append(self._buffer)
+            self._buffer = ""
+
+    def _flush_if_oversized(self, chunks: list[str]) -> None:
+        while len(self._buffer) > self._max_buffer_chars:
+            chunks.append(self._buffer[: self._max_buffer_chars])
+            self._buffer = self._buffer[self._max_buffer_chars :]