sdxloop 0.1.0__py3-none-any.whl

sdxloop/engine/store.py

@@ -0,0 +1,257 @@
+"""SQLite state store: runs, tasks, phase attempts, events.
+
+One WAL-mode database at ``<state_dir>/state.db``. A checkpoint row is
+written after every state transition, which is what makes ``resume`` safe:
+a phase whose result was never committed is simply re-run from its start.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import time
+from collections.abc import Iterator
+from pathlib import Path
+
+from sdxloop.engine.model import (
+    PlanModel,
+    RunRecord,
+    RunState,
+    TaskRecord,
+    TaskSpec,
+    TaskState,
+)
+from sdxloop.errors import StateError
+from sdxloop_worker.protocol import Event
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS runs (
+    run_id     TEXT PRIMARY KEY,
+    outcome    TEXT NOT NULL,
+    state      TEXT NOT NULL,
+    config_json TEXT NOT NULL DEFAULT '{}',
+    created_at REAL NOT NULL,
+    updated_at REAL NOT NULL
+);
+CREATE TABLE IF NOT EXISTS tasks (
+    run_id     TEXT NOT NULL,
+    task_id    TEXT NOT NULL,
+    order_idx  INTEGER NOT NULL,
+    state      TEXT NOT NULL,
+    spec_json  TEXT NOT NULL,
+    plan_json  TEXT,
+    revisions  INTEGER NOT NULL DEFAULT 0,
+    replans    INTEGER NOT NULL DEFAULT 0,
+    last_feedback TEXT NOT NULL DEFAULT '',
+    session_id TEXT,
+    PRIMARY KEY (run_id, task_id)
+);
+CREATE TABLE IF NOT EXISTS phase_attempts (
+    id         INTEGER PRIMARY KEY AUTOINCREMENT,
+    run_id     TEXT NOT NULL,
+    task_id    TEXT,
+    phase      TEXT NOT NULL,
+    attempt    INTEGER NOT NULL,
+    status     TEXT NOT NULL,
+    output_json TEXT,
+    started_at REAL NOT NULL,
+    ended_at   REAL NOT NULL
+);
+CREATE TABLE IF NOT EXISTS events (
+    seq       INTEGER PRIMARY KEY AUTOINCREMENT,
+    run_id    TEXT NOT NULL,
+    ts        REAL NOT NULL,
+    type      TEXT NOT NULL,
+    job_id    TEXT,
+    data_json TEXT NOT NULL DEFAULT '{}'
+);
+CREATE INDEX IF NOT EXISTS idx_events_run ON events (run_id, seq);
+"""
+
+
+class StateStore:
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(path, check_same_thread=False)
+        self._conn.row_factory = sqlite3.Row
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.executescript(_SCHEMA)
+        self._conn.commit()
+
+    def close(self) -> None:
+        self._conn.close()
+
+    # -- runs --------------------------------------------------------------
+
+    def create_run(self, run_id: str, outcome: str, config_json: str = "{}") -> RunRecord:
+        now = time.time()
+        try:
+            self._conn.execute(
+                "INSERT INTO runs (run_id, outcome, state, config_json, created_at, updated_at)"
+                " VALUES (?, ?, 'created', ?, ?, ?)",
+                (run_id, outcome, config_json, now, now),
+            )
+        except sqlite3.IntegrityError as exc:
+            raise StateError(f"run {run_id} already exists") from exc
+        self._conn.commit()
+        return RunRecord(
+            run_id=run_id, outcome=outcome, state="created", created_at=now, updated_at=now
+        )
+
+    def set_run_state(self, run_id: str, state: RunState) -> None:
+        cursor = self._conn.execute(
+            "UPDATE runs SET state = ?, updated_at = ? WHERE run_id = ?",
+            (state, time.time(), run_id),
+        )
+        if cursor.rowcount == 0:
+            raise StateError(f"unknown run {run_id}")
+        self._conn.commit()
+
+    def get_run(self, run_id: str) -> RunRecord:
+        row = self._conn.execute("SELECT * FROM runs WHERE run_id = ?", (run_id,)).fetchone()
+        if row is None:
+            raise StateError(f"unknown run {run_id}")
+        return RunRecord(
+            run_id=row["run_id"],
+            outcome=row["outcome"],
+            state=row["state"],
+            created_at=row["created_at"],
+            updated_at=row["updated_at"],
+        )
+
+    def list_runs(self) -> list[RunRecord]:
+        rows = self._conn.execute("SELECT * FROM runs ORDER BY created_at DESC").fetchall()
+        return [
+            RunRecord(
+                run_id=row["run_id"],
+                outcome=row["outcome"],
+                state=row["state"],
+                created_at=row["created_at"],
+                updated_at=row["updated_at"],
+            )
+            for row in rows
+        ]
+
+    # -- tasks -------------------------------------------------------------
+
+    def save_tasks(self, run_id: str, specs: list[TaskSpec]) -> None:
+        for index, spec in enumerate(specs):
+            self._conn.execute(
+                "INSERT OR REPLACE INTO tasks (run_id, task_id, order_idx, state, spec_json)"
+                " VALUES (?, ?, ?, 'pending', ?)",
+                (run_id, spec.id, index, spec.model_dump_json()),
+            )
+        self._conn.commit()
+
+    def update_task(self, run_id: str, task: TaskRecord) -> None:
+        cursor = self._conn.execute(
+            "UPDATE tasks SET state = ?, plan_json = ?, revisions = ?, replans = ?,"
+            " last_feedback = ?, session_id = ? WHERE run_id = ? AND task_id = ?",
+            (
+                task.state,
+                task.plan.model_dump_json() if task.plan else None,
+                task.revisions,
+                task.replans,
+                task.last_feedback,
+                task.session_id,
+                run_id,
+                task.spec.id,
+            ),
+        )
+        if cursor.rowcount == 0:
+            raise StateError(f"unknown task {task.spec.id} in run {run_id}")
+        self._conn.commit()
+
+    def get_tasks(self, run_id: str) -> list[TaskRecord]:
+        rows = self._conn.execute(
+            "SELECT * FROM tasks WHERE run_id = ? ORDER BY order_idx", (run_id,)
+        ).fetchall()
+        records: list[TaskRecord] = []
+        for row in rows:
+            state: TaskState = row["state"]
+            records.append(
+                TaskRecord(
+                    spec=TaskSpec.model_validate_json(row["spec_json"]),
+                    state=state,
+                    revisions=row["revisions"],
+                    replans=row["replans"],
+                    last_feedback=row["last_feedback"],
+                    session_id=row["session_id"],
+                    plan=(
+                        PlanModel.model_validate_json(row["plan_json"])
+                        if row["plan_json"]
+                        else None
+                    ),
+                )
+            )
+        return records
+
+    # -- phase attempts ----------------------------------------------------
+
+    def record_phase(
+        self,
+        run_id: str,
+        phase: str,
+        *,
+        task_id: str | None,
+        attempt: int,
+        status: str,
+        output_json: str | None,
+        started_at: float,
+    ) -> None:
+        self._conn.execute(
+            "INSERT INTO phase_attempts"
+            " (run_id, task_id, phase, attempt, status, output_json, started_at, ended_at)"
+            " VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+            (run_id, task_id, phase, attempt, status, output_json, started_at, time.time()),
+        )
+        self._conn.commit()
+
+    def phase_attempts(self, run_id: str, task_id: str | None = None) -> list[sqlite3.Row]:
+        if task_id is None:
+            return list(
+                self._conn.execute(
+                    "SELECT * FROM phase_attempts WHERE run_id = ? ORDER BY id", (run_id,)
+                )
+            )
+        return list(
+            self._conn.execute(
+                "SELECT * FROM phase_attempts WHERE run_id = ? AND task_id = ? ORDER BY id",
+                (run_id, task_id),
+            )
+        )
+
+    # -- events ------------------------------------------------------------
+
+    def append_event(self, event: Event) -> None:
+        self._conn.execute(
+            "INSERT INTO events (run_id, ts, type, job_id, data_json) VALUES (?, ?, ?, ?, ?)",
+            (event.run_id, event.ts, event.type, event.job_id, json.dumps(event.data)),
+        )
+        self._conn.commit()
+
+    def events(
+        self,
+        run_id: str,
+        *,
+        after_seq: int = 0,
+        type_prefix: str | None = None,
+    ) -> Iterator[tuple[int, Event]]:
+        query = "SELECT * FROM events WHERE run_id = ? AND seq > ?"
+        params: list[object] = [run_id, after_seq]
+        if type_prefix:
+            query += " AND type LIKE ?"
+            params.append(type_prefix + "%")
+        query += " ORDER BY seq"
+        for row in self._conn.execute(query, params):
+            yield (
+                row["seq"],
+                Event(
+                    ts=row["ts"],
+                    run_id=row["run_id"],
+                    job_id=row["job_id"],
+                    type=row["type"],
+                    data=json.loads(row["data_json"]),
+                ),
+            )

sdxloop/errors.py

@@ -0,0 +1,73 @@
+"""sdxloop exception hierarchy."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+
+class SdxloopError(Exception):
+    """Base class for all sdxloop errors."""
+
+
+class ConfigError(SdxloopError):
+    """Invalid or unloadable configuration."""
+
+
+class SbxError(SdxloopError):
+    """An sbx CLI invocation failed."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        argv: Sequence[str] = (),
+        returncode: int | None = None,
+        stderr: str = "",
+    ) -> None:
+        super().__init__(message)
+        self.argv = list(argv)
+        self.returncode = returncode
+        self.stderr = stderr
+
+    def __str__(self) -> str:
+        base = super().__str__()
+        parts = [base]
+        if self.argv:
+            parts.append(f"argv={' '.join(self.argv)}")
+        if self.returncode is not None:
+            parts.append(f"rc={self.returncode}")
+        if self.stderr.strip():
+            parts.append(f"stderr={self.stderr.strip()}")
+        return " | ".join(parts)
+
+
+class SbxNotFoundError(SbxError):
+    """The sbx binary is missing, or a referenced sandbox does not exist."""
+
+
+class ProvisionError(SdxloopError):
+    """Sandbox pair provisioning failed."""
+
+
+class WorkerError(SdxloopError):
+    """The in-sandbox worker failed or returned an invalid result."""
+
+
+class WorkerTimeoutError(WorkerError):
+    """The worker exceeded its job timeout and was killed."""
+
+
+class ProtocolError(SdxloopError):
+    """Host/worker protocol violation (bad event line, missing result, ...)."""
+
+
+class GithubOpsError(SdxloopError):
+    """A GitHub operation failed in the github-ops sandbox."""
+
+
+class BudgetExceededError(SdxloopError):
+    """A run or task exhausted one of its budgets."""
+
+
+class StateError(SdxloopError):
+    """Invalid state transition or corrupted persisted state."""

sdxloop/events.py

@@ -0,0 +1,87 @@
+"""Host-side event plumbing: a synchronous pub/sub bus over protocol Events."""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+from collections.abc import Callable
+from typing import Any, Protocol, runtime_checkable
+
+from sdxloop_worker.protocol import Event
+
+logger = logging.getLogger(__name__)
+
+Subscriber = Callable[[Event], None]
+
+
+@runtime_checkable
+class Hook(Protocol):
+    """Extension point: receives every event published on the bus.
+
+    Implementations must be fast and must not raise; exceptions are caught
+    and logged so one misbehaving hook cannot break a run.
+    """
+
+    def on_event(self, event: Event) -> None: ...
+
+
+class EventBus:
+    """Synchronous fan-out of protocol events to subscribers.
+
+    Subscriber exceptions are isolated: they are logged and swallowed so
+    telemetry consumers can never fail a run.
+    """
+
+    def __init__(self) -> None:
+        self._subscribers: list[Subscriber] = []
+
+    def subscribe(self, fn: Subscriber) -> Callable[[], None]:
+        """Register a subscriber; returns an unsubscribe callable."""
+        self._subscribers.append(fn)
+
+        def unsubscribe() -> None:
+            with contextlib.suppress(ValueError):
+                self._subscribers.remove(fn)
+
+        return unsubscribe
+
+    def attach_hook(self, hook: Hook) -> Callable[[], None]:
+        return self.subscribe(hook.on_event)
+
+    def publish(self, event: Event) -> None:
+        for fn in list(self._subscribers):
+            try:
+                fn(event)
+            except Exception:
+                logger.exception("event subscriber %r failed for %s", fn, event.type)
+
+    def emit(
+        self,
+        type: str,
+        run_id: str,
+        job_id: str | None = None,
+        **data: Any,
+    ) -> Event:
+        """Construct an Event stamped now, publish it, and return it."""
+        event = Event.now(type, run_id, job_id=job_id, **data)
+        self.publish(event)
+        return event
+
+
+class HostEventTypes:
+    """Host-emitted event types (the worker's live in protocol.EventTypes)."""
+
+    RUN_START = "run.start"
+    RUN_STATE = "run.state"
+    RUN_END = "run.end"
+    TASK_START = "task.start"
+    TASK_STATE = "task.state"
+    TASK_END = "task.end"
+    PHASE_START = "phase.start"
+    PHASE_END = "phase.end"
+    SANDBOX_PROVISION_START = "sandbox.provision_start"
+    SANDBOX_READY = "sandbox.ready"
+    SANDBOX_CLEANUP = "sandbox.cleanup"
+
+
+__all__ = ["Event", "EventBus", "Hook", "HostEventTypes", "Subscriber"]

sdxloop/gh/__init__.py

@@ -0,0 +1,6 @@
+"""Host-side GitHub operations (executed in the github-ops sandbox)."""
+
+from sdxloop.gh.ops import GithubOps, IssueRef, PrRef
+from sdxloop.gh.reporter import GithubReporterHook
+
+__all__ = ["GithubOps", "GithubReporterHook", "IssueRef", "PrRef"]

sdxloop/gh/ops.py

@@ -0,0 +1,140 @@
+"""Typed facade over github.op jobs running in the github-ops sandbox.
+
+The host never talks to GitHub with the user PAT directly — every operation
+becomes a ``github.op`` JobRequest submitted to the github sandbox, which is
+the only environment holding ``GH_TOKEN``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+from sdxloop.errors import GithubOpsError
+from sdxloop.ids import new_job_id
+from sdxloop.worker.client import WorkerClient
+from sdxloop_worker.protocol import JobRequest
+
+
+class IssueRef(BaseModel):
+    number: int
+    url: str
+
+
+class PrRef(BaseModel):
+    number: int
+    url: str
+
+
+class GithubOps:
+    def __init__(
+        self,
+        client: WorkerClient,
+        run_id: str,
+        *,
+        timeout_s: float = 120.0,
+    ) -> None:
+        self.client = client
+        self.run_id = run_id
+        self.timeout_s = timeout_s
+
+    def _op(self, op: str, params: dict[str, Any]) -> Any:
+        job = JobRequest(
+            job_id=new_job_id(),
+            run_id=self.run_id,
+            kind="github.op",
+            op=op,
+            params=params,
+            timeout_s=self.timeout_s,
+        )
+        result = self.client.submit(job)
+        if result.status != "ok":
+            assert result.error is not None
+            raise GithubOpsError(
+                f"github op {op} failed: {result.error.type}: {result.error.message}"
+            )
+        return result.output_json
+
+    def issue_create(
+        self,
+        repo: str,
+        title: str,
+        body: str = "",
+        labels: list[str] | None = None,
+    ) -> IssueRef:
+        params: dict[str, Any] = {"repo": repo, "title": title, "body": body}
+        if labels:
+            params["labels"] = labels
+        return IssueRef.model_validate(self._op("issue.create", params))
+
+    def issue_comment(self, repo: str, number: int, body: str) -> str:
+        data = self._op("issue.comment", {"repo": repo, "number": number, "body": body})
+        return str(data.get("url", ""))
+
+    def pr_create(
+        self,
+        repo: str,
+        base: str,
+        head: str,
+        title: str,
+        body: str = "",
+        *,
+        draft: bool = False,
+    ) -> PrRef:
+        return PrRef.model_validate(
+            self._op(
+                "pr.create",
+                {
+                    "repo": repo,
+                    "base": base,
+                    "head": head,
+                    "title": title,
+                    "body": body,
+                    "draft": draft,
+                },
+            )
+        )
+
+    def pr_comment(self, repo: str, number: int, body: str) -> str:
+        data = self._op("pr.comment", {"repo": repo, "number": number, "body": body})
+        return str(data.get("url", ""))
+
+    def contents_read(self, repo: str, path: str, ref: str | None = None) -> str:
+        params: dict[str, Any] = {"repo": repo, "path": path}
+        if ref:
+            params["ref"] = ref
+        data = self._op("contents.read", params)
+        return str(data.get("content", ""))
+
+    def status_create(
+        self,
+        repo: str,
+        sha: str,
+        state: str,
+        *,
+        context: str = "sdxloop",
+        description: str = "",
+        target_url: str = "",
+    ) -> None:
+        params: dict[str, Any] = {"repo": repo, "sha": sha, "state": state, "context": context}
+        if description:
+            params["description"] = description
+        if target_url:
+            params["target_url"] = target_url
+        self._op("status.create", params)
+
+    def repo_get(self, repo: str) -> dict[str, Any]:
+        data = self._op("repo.get", {"repo": repo})
+        assert isinstance(data, dict)
+        return data
+
+    def search_issues(self, query: str, per_page: int = 30) -> list[dict[str, Any]]:
+        data = self._op("search.issues", {"query": query, "per_page": per_page})
+        return data if isinstance(data, list) else []
+
+    def raw(self, method: str, path: str, body: dict[str, Any] | None = None) -> Any:
+        params: dict[str, Any] = {"method": method, "path": path}
+        if body is not None:
+            params["body"] = body
+        return self._op("raw.api", params)

sdxloop/gh/reporter.py

@@ -0,0 +1,75 @@
+"""GithubReporterHook: mirrors run progress to a GitHub tracking issue.
+
+The default consumer of the github-ops sandbox. When ``github.report_repo``
+is configured, the hook opens a tracking issue at run start, comments as
+tasks finish, and posts a final summary at run end. It never raises — a
+reporting failure must not fail a run — and the EventBus isolates it anyway.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from sdxloop.events import Event, HostEventTypes
+from sdxloop.gh.ops import GithubOps, IssueRef
+
+logger = logging.getLogger(__name__)
+
+
+class GithubReporterHook:
+    def __init__(self, ops: GithubOps, repo: str) -> None:
+        self.ops = ops
+        self.repo = repo
+        self.issue: IssueRef | None = None
+        self._task_lines: list[str] = []
+
+    # -- Hook protocol -----------------------------------------------------
+
+    def on_event(self, event: Event) -> None:
+        try:
+            if event.type == HostEventTypes.RUN_START:
+                self._on_run_start(event)
+            elif event.type == HostEventTypes.TASK_END:
+                self._on_task_end(event)
+            elif event.type == HostEventTypes.RUN_END:
+                self._on_run_end(event)
+        except Exception:
+            logger.warning("github reporting failed for %s", event.type, exc_info=True)
+
+    # -- handlers ----------------------------------------------------------
+
+    def _on_run_start(self, event: Event) -> None:
+        outcome = str(event.data.get("outcome", ""))
+        body = (
+            f"sdxloop run `{event.run_id}` started.\n\n"
+            f"**Outcome:**\n\n> {outcome}\n\n"
+            "Progress is reported as comments on this issue."
+        )
+        self.issue = self.ops.issue_create(
+            self.repo,
+            title=f"sdxloop run {event.run_id}",
+            body=body,
+            labels=["sdxloop"],
+        )
+
+    def _on_task_end(self, event: Event) -> None:
+        if self.issue is None:
+            return
+        task_id = event.data.get("task_id", "?")
+        title = event.data.get("title", "")
+        state = event.data.get("state", "?")
+        marker = "✅" if state == "done" else "❌"
+        line = f"{marker} `{task_id}` {title} — **{state}**"
+        self._task_lines.append(line)
+        self.ops.issue_comment(self.repo, self.issue.number, line)
+
+    def _on_run_end(self, event: Event) -> None:
+        if self.issue is None:
+            return
+        state = event.data.get("state", "?")
+        summary = "\n".join(self._task_lines) or "_no tasks were executed_"
+        self.ops.issue_comment(
+            self.repo,
+            self.issue.number,
+            f"Run `{event.run_id}` finished: **{state}**\n\n{summary}",
+        )

sdxloop/ids.py

@@ -0,0 +1,35 @@
+"""Identifier generation for runs, jobs, and tasks."""
+
+from __future__ import annotations
+
+import re
+import secrets
+
+# Crockford-flavored base32: lowercase, no i/l/o/u lookalikes.
+_ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz"
+
+_RUN_ID_RE = re.compile(r"^r[0-9abcdefghjkmnpqrstvwxyz]{8}$")
+
+
+def _token(length: int) -> str:
+    return "".join(secrets.choice(_ALPHABET) for _ in range(length))
+
+
+def new_run_id() -> str:
+    """A short, sandbox-name-safe run id, e.g. ``r7k2m9qp3``-style."""
+    return "r" + _token(8)
+
+
+def is_run_id(value: str) -> bool:
+    return _RUN_ID_RE.fullmatch(value) is not None
+
+
+def new_job_id() -> str:
+    return "j" + _token(10)
+
+
+def task_id(index: int) -> str:
+    """Deterministic task ids within a run: t1, t2, ..."""
+    if index < 1:
+        raise ValueError("task index starts at 1")
+    return f"t{index}"

sdxloop/sbx/__init__.py

@@ -0,0 +1,6 @@
+"""Typed subprocess layer over the Docker Sandboxes ``sbx`` CLI."""
+
+from sdxloop.sbx.cli import SbxCLI
+from sdxloop.sbx.models import ExecResult, SandboxInfo, SandboxSpec, SecretSpec
+
+__all__ = ["ExecResult", "SandboxInfo", "SandboxSpec", "SbxCLI", "SecretSpec"]