sdxloop 0.1.0__py3-none-any.whl

sdxloop/config.py

@@ -0,0 +1,180 @@
+"""Configuration loading with layered precedence.
+
+Precedence (highest wins): ``SDXLOOP_*`` environment variables >
+``./sdxloop.toml`` > ``pyproject.toml [tool.sdxloop]`` > built-in defaults.
+
+Nested keys use ``__`` in environment variables, e.g.
+``SDXLOOP_BUDGETS__MAX_TASKS=5``. Values are parsed as TOML scalars where
+possible (so ``true``, ``42``, ``1.5`` get real types) and fall back to
+strings.
+"""
+
+from __future__ import annotations
+
+import os
+import tomllib
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from sdxloop.errors import ConfigError
+
+ENV_PREFIX = "SDXLOOP_"
+
+# SDXLOOP_-prefixed variables consumed by the *worker process* rather than
+# host configuration; the env config layer must not treat them as settings.
+RESERVED_ENV_KEYS = frozenset({"worker_backend", "echo_script"})
+
+WorkerTransport = Literal["stream", "poll"]
+SecretStrategy = Literal["proxy", "plain-env"]
+
+
+class _ConfigModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class SandboxConfig(_ConfigModel):
+    template: str | None = None
+    workspace: Path | None = None
+    extra_allow_domains: list[str] = Field(default_factory=list)
+
+
+class GithubConfig(_ConfigModel):
+    report_repo: str | None = None
+
+
+class Budgets(_ConfigModel):
+    max_revisions_per_task: int = 2
+    max_replans_per_task: int = 1
+    max_tasks: int = 20
+    max_wall_clock_s: float = 7200.0
+    per_job_timeout_s: float = 900.0
+
+
+class Config(_ConfigModel):
+    model: str = "auto"
+    app_name: str = "sdxloop"
+    state_dir: Path = Path(".sdxloop")
+    keep_sandboxes: bool = False
+    worker_transport: WorkerTransport = "stream"
+    secret_strategy: SecretStrategy = "proxy"
+    # Advanced: in-sandbox interpreter for the worker, and whether to run the
+    # install flow. Overridden in tests/e2e via SDXLOOP_WORKER_PYTHON /
+    # SDXLOOP_INSTALL_WORKERS.
+    worker_python: str = "/home/agent/.sdxloop/venv/bin/python"
+    install_workers: bool = True
+    sandbox: SandboxConfig = Field(default_factory=SandboxConfig)
+    github: GithubConfig = Field(default_factory=GithubConfig)
+    budgets: Budgets = Field(default_factory=Budgets)
+
+
+def _read_toml(path: Path) -> dict[str, Any]:
+    try:
+        with path.open("rb") as f:
+            return tomllib.load(f)
+    except tomllib.TOMLDecodeError as exc:
+        raise ConfigError(f"invalid TOML in {path}: {exc}") from exc
+
+
+def _pyproject_layer(cwd: Path) -> dict[str, Any]:
+    path = cwd / "pyproject.toml"
+    if not path.is_file():
+        return {}
+    data = _read_toml(path)
+    tool = data.get("tool", {})
+    if not isinstance(tool, dict):
+        return {}
+    section = tool.get("sdxloop", {})
+    return section if isinstance(section, dict) else {}
+
+
+def _sdxloop_toml_layer(cwd: Path) -> dict[str, Any]:
+    path = cwd / "sdxloop.toml"
+    if not path.is_file():
+        return {}
+    return _read_toml(path)
+
+
+def _parse_env_value(raw: str) -> Any:
+    """Parse an env var as a TOML scalar; fall back to the raw string."""
+    try:
+        return tomllib.loads(f"v = {raw}")["v"]
+    except tomllib.TOMLDecodeError:
+        return raw
+
+
+def _env_layer(env: Mapping[str, str]) -> dict[str, Any]:
+    layer: dict[str, Any] = {}
+    for key, raw in env.items():
+        if not key.startswith(ENV_PREFIX):
+            continue
+        path = key[len(ENV_PREFIX) :].lower().split("__")
+        if not all(path) or path[0] in RESERVED_ENV_KEYS:
+            continue
+        node = layer
+        for part in path[:-1]:
+            node = node.setdefault(part, {})
+            if not isinstance(node, dict):
+                raise ConfigError(f"conflicting env overrides at {key}")
+        node[path[-1]] = _parse_env_value(raw)
+    return layer
+
+
+def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
+    merged = dict(base)
+    for key, value in override.items():
+        if isinstance(value, dict) and isinstance(merged.get(key), dict):
+            merged[key] = _deep_merge(merged[key], value)
+        else:
+            merged[key] = value
+    return merged
+
+
+def _flatten(data: dict[str, Any], prefix: str = "") -> dict[str, Any]:
+    flat: dict[str, Any] = {}
+    for key, value in data.items():
+        dotted = f"{prefix}{key}"
+        if isinstance(value, dict):
+            flat.update(_flatten(value, f"{dotted}."))
+        else:
+            flat[dotted] = value
+    return flat
+
+
+def load_config_with_sources(
+    cwd: Path | None = None,
+    env: Mapping[str, str] | None = None,
+) -> tuple[Config, dict[str, str]]:
+    """Load config and report, per dotted key, which layer supplied it."""
+    cwd = cwd or Path.cwd()
+    env = os.environ if env is None else env
+
+    layers: list[tuple[str, dict[str, Any]]] = [
+        ("pyproject.toml", _pyproject_layer(cwd)),
+        ("sdxloop.toml", _sdxloop_toml_layer(cwd)),
+        ("env", _env_layer(env)),
+    ]
+
+    merged: dict[str, Any] = {}
+    sources: dict[str, str] = {}
+    for name, layer in layers:
+        merged = _deep_merge(merged, layer)
+        for dotted in _flatten(layer):
+            sources[dotted] = name
+
+    try:
+        config = Config.model_validate(merged)
+    except ValidationError as exc:
+        raise ConfigError(f"invalid sdxloop configuration: {exc}") from exc
+
+    for dotted in _flatten(config.model_dump()):
+        sources.setdefault(dotted, "default")
+    return config, sources
+
+
+def load_config(cwd: Path | None = None, env: Mapping[str, str] | None = None) -> Config:
+    """Load the effective sdxloop configuration for ``cwd``."""
+    config, _ = load_config_with_sources(cwd=cwd, env=env)
+    return config

sdxloop/engine/__init__.py

@@ -0,0 +1,26 @@
+"""The sdxloop loop engine."""
+
+from sdxloop.engine.engine import LoopEngine, run_outcome
+from sdxloop.engine.model import (
+    PlanModel,
+    RunRecord,
+    RunResult,
+    TaskGraph,
+    TaskRecord,
+    TaskSpec,
+    Verdict,
+)
+from sdxloop.engine.store import StateStore
+
+__all__ = [
+    "LoopEngine",
+    "PlanModel",
+    "RunRecord",
+    "RunResult",
+    "StateStore",
+    "TaskGraph",
+    "TaskRecord",
+    "TaskSpec",
+    "Verdict",
+    "run_outcome",
+]

sdxloop/engine/engine.py

@@ -0,0 +1,380 @@
+"""LoopEngine: drives DECOMPOSE → (PLAN → EXECUTE → SCRUTINIZE → VERIFY →
+VALIDATE)* under budgets, with SQLite checkpointing after every transition.
+
+Failure semantics:
+
+- Budget exhaustion (revisions/replans) fails the *task*; dependents are
+  skipped and the run continues, finishing ``failed`` if any task failed.
+- Infrastructure errors (worker/sbx crashes) propagate after state is
+  persisted — equivalent to a kill. ``resume()`` re-provisions a fresh
+  sandbox pair (sandboxes are cattle; the workspace and SQLite state
+  persist on the host) and continues from the last committed transition:
+  a phase whose result was never committed re-runs from its start.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import Callable, Sequence
+
+from sdxloop.config import Config, load_config
+from sdxloop.engine.model import (
+    RESUMABLE_RUN_STATES,
+    RunResult,
+    RunState,
+    TaskRecord,
+    TaskState,
+)
+from sdxloop.engine.phases import PhaseRunner, clip
+from sdxloop.engine.store import StateStore
+from sdxloop.errors import BudgetExceededError, SdxloopError, StateError
+from sdxloop.events import EventBus, Hook, HostEventTypes
+from sdxloop.gh.ops import GithubOps
+from sdxloop.gh.reporter import GithubReporterHook
+from sdxloop.ids import new_run_id
+from sdxloop.sbx.cli import SbxCLI
+from sdxloop.sbx.provision import Provisioner
+from sdxloop.worker.client import WorkerClient
+
+
+class LoopEngine:
+    def __init__(
+        self,
+        config: Config | None = None,
+        *,
+        store: StateStore | None = None,
+        bus: EventBus | None = None,
+        hooks: Sequence[Hook] = (),
+        sbx: SbxCLI | None = None,
+        worker_python: str | None = None,
+        install_workers: bool | None = None,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> None:
+        self.config = config or load_config()
+        self.store = store or StateStore(self.config.state_dir / "state.db")
+        self.bus = bus or EventBus()
+        self.sbx = sbx or SbxCLI(app_name=self.config.app_name or None)
+        self.worker_python = (
+            worker_python if worker_python is not None else (self.config.worker_python)
+        )
+        self.install_workers = (
+            install_workers if install_workers is not None else self.config.install_workers
+        )
+        self.clock = clock
+        for hook in hooks:
+            self.bus.attach_hook(hook)
+        self.bus.subscribe(self._persist_event)
+
+    def _persist_event(self, event: object) -> None:
+        from sdxloop_worker.protocol import Event
+
+        assert isinstance(event, Event)
+        self.store.append_event(event)
+
+    # -- public API --------------------------------------------------------
+
+    def start(self, outcome: str, *, run_id: str | None = None) -> RunResult:
+        run_id = run_id or new_run_id()
+        self.store.create_run(run_id, outcome, self.config.model_dump_json())
+        self.bus.emit(HostEventTypes.RUN_START, run_id, outcome=outcome)
+        return self._drive(run_id, outcome)
+
+    def resume(self, run_id: str) -> RunResult:
+        run = self.store.get_run(run_id)
+        if run.state not in RESUMABLE_RUN_STATES:
+            raise StateError(f"run {run_id} is {run.state}; only unfinished runs can resume")
+        self.bus.emit(HostEventTypes.RUN_START, run_id, outcome=run.outcome, resumed=True)
+        return self._drive(run_id, run.outcome)
+
+    def cancel(self, run_id: str) -> None:
+        self.store.get_run(run_id)  # raises for unknown runs
+        self.store.set_run_state(run_id, "cancelled")
+
+    # -- run driver --------------------------------------------------------
+
+    def _drive(self, run_id: str, outcome: str) -> RunResult:
+        deadline = self.clock() + self.config.budgets.max_wall_clock_s
+        self._set_run_state(run_id, "provisioning")
+        provisioner = Provisioner(self.sbx, self.config, self.bus)
+        pair = provisioner.ensure_pair(run_id)
+        try:
+            with pair:
+                agent = WorkerClient(
+                    pair.agent,
+                    self.bus,
+                    transport=self.config.worker_transport,
+                    python=self.worker_python,
+                )
+                github = WorkerClient(
+                    pair.github,
+                    self.bus,
+                    transport=self.config.worker_transport,
+                    python=self.worker_python,
+                )
+                if self.install_workers:
+                    agent.install(extras="copilot")
+                    github.install(extras="")
+                detach = self._attach_reporter(github, run_id)
+                try:
+                    phases = PhaseRunner(agent, self.config, run_id, outcome)
+                    state = self._run_phases(run_id, phases, deadline)
+                finally:
+                    detach()
+        except SdxloopError:
+            # State is already persisted; the exception is the kill signal.
+            raise
+        self._set_run_state(run_id, state)
+        tasks = self.store.get_tasks(run_id)
+        self.bus.emit(HostEventTypes.RUN_END, run_id, state=state)
+        return RunResult(run_id=run_id, state=state, tasks=tasks)
+
+    def _attach_reporter(self, github: WorkerClient, run_id: str) -> Callable[[], None]:
+        repo = self.config.github.report_repo
+        if not repo:
+            return lambda: None
+        hook = GithubReporterHook(GithubOps(github, run_id), repo)
+        return self.bus.attach_hook(hook)
+
+    def _run_phases(self, run_id: str, phases: PhaseRunner, deadline: float) -> RunState:
+        tasks = self.store.get_tasks(run_id)
+        if not tasks:
+            self._set_run_state(run_id, "decomposing")
+            started = time.time()
+            graph = phases.decompose()
+            if len(graph.tasks) > self.config.budgets.max_tasks:
+                raise BudgetExceededError(
+                    f"decomposition produced {len(graph.tasks)} tasks "
+                    f"(max {self.config.budgets.max_tasks})"
+                )
+            ordered = graph.topo_order()
+            self.store.save_tasks(run_id, ordered)
+            self.store.record_phase(
+                run_id,
+                "decompose",
+                task_id=None,
+                attempt=1,
+                status="ok",
+                output_json=graph.model_dump_json(),
+                started_at=started,
+            )
+            tasks = self.store.get_tasks(run_id)
+
+        self._set_run_state(run_id, "running")
+        failed_ids: set[str] = {t.spec.id for t in tasks if t.state == "failed"}
+        skipped_ids: set[str] = {t.spec.id for t in tasks if t.state == "skipped"}
+        for task in tasks:
+            if task.terminal:
+                failed_ids |= {task.spec.id} if task.state == "failed" else set()
+                continue
+            blocked = [d for d in task.spec.depends_on if d in failed_ids | skipped_ids]
+            if blocked:
+                task.state = "skipped"
+                skipped_ids.add(task.spec.id)
+                self.store.update_task(run_id, task)
+                self._emit_task_end(run_id, task)
+                continue
+            self._run_task(run_id, phases, task, deadline)
+            if task.state == "failed":
+                failed_ids.add(task.spec.id)
+
+        self._set_run_state(run_id, "finalizing")
+        return "failed" if failed_ids or skipped_ids else "completed"
+
+    # -- task state machine ------------------------------------------------
+
+    def _run_task(
+        self,
+        run_id: str,
+        phases: PhaseRunner,
+        task: TaskRecord,
+        deadline: float,
+    ) -> None:
+        budgets = self.config.budgets
+        if task.state == "pending":
+            self._set_task_state(run_id, task, "planning")
+        self.bus.emit(
+            HostEventTypes.TASK_START, run_id, task_id=task.spec.id, title=task.spec.title
+        )
+        # Resume mapping: executing/scrutinizing restart at executing (the
+        # plan is persisted); a missing plan always restarts at planning.
+        if task.state in ("executing", "scrutinizing") and task.plan is None:
+            self._set_task_state(run_id, task, "planning")
+        if task.state in ("verifying", "validating") and task.plan is None:
+            self._set_task_state(run_id, task, "planning")
+        if task.state == "scrutinizing":
+            self._set_task_state(run_id, task, "executing")
+
+        while not task.terminal:
+            self._check_cancelled_and_clock(run_id, deadline)
+            if task.state == "planning":
+                self._phase_plan(run_id, phases, task)
+            elif task.state == "executing":
+                self._phase_execute_and_scrutinize(run_id, phases, task, budgets)
+            elif task.state == "verifying":
+                self._phase_verify(run_id, phases, task, budgets)
+            elif task.state == "validating":
+                self._phase_validate(run_id, phases, task, budgets)
+            else:  # pragma: no cover - defensive
+                raise StateError(f"task {task.spec.id} in unexpected state {task.state}")
+
+        self._emit_task_end(run_id, task)
+
+    def _phase_plan(self, run_id: str, phases: PhaseRunner, task: TaskRecord) -> None:
+        started = time.time()
+        plan = phases.plan(task)
+        task.plan = plan
+        self.store.record_phase(
+            run_id,
+            "plan",
+            task_id=task.spec.id,
+            attempt=task.replans + 1,
+            status="ok",
+            output_json=plan.model_dump_json(),
+            started_at=started,
+        )
+        self._set_task_state(run_id, task, "executing")
+
+    def _phase_execute_and_scrutinize(
+        self,
+        run_id: str,
+        phases: PhaseRunner,
+        task: TaskRecord,
+        budgets: object,
+    ) -> None:
+        assert task.plan is not None
+        started = time.time()
+        result = phases.execute(task, task.plan)
+        task.session_id = result.session_id
+        executor_report = clip(result.output_text)
+        self.store.record_phase(
+            run_id,
+            "execute",
+            task_id=task.spec.id,
+            attempt=task.revisions + 1,
+            status="ok",
+            output_json=json.dumps({"report": executor_report, "session_id": result.session_id}),
+            started_at=started,
+        )
+        self._set_task_state(run_id, task, "scrutinizing")
+
+        started = time.time()
+        verdict = phases.scrutinize(task, task.plan, executor_report)
+        self.store.record_phase(
+            run_id,
+            "scrutinize",
+            task_id=task.spec.id,
+            attempt=task.revisions + 1,
+            status=verdict.verdict,
+            output_json=verdict.model_dump_json(),
+            started_at=started,
+        )
+        if verdict.verdict == "revise":
+            self._register_revision(run_id, task, verdict.feedback or "scrutiny found issues")
+        else:
+            task.last_feedback = ""
+            self._set_task_state(run_id, task, "verifying")
+
+    def _phase_verify(
+        self,
+        run_id: str,
+        phases: PhaseRunner,
+        task: TaskRecord,
+        budgets: object,
+    ) -> None:
+        assert task.plan is not None
+        started = time.time()
+        passed, feedback = phases.verify(task, task.plan)
+        self.store.record_phase(
+            run_id,
+            "verify",
+            task_id=task.spec.id,
+            attempt=task.revisions + 1,
+            status="ok" if passed else "failed",
+            output_json=json.dumps({"passed": passed, "feedback": clip(feedback)}),
+            started_at=started,
+        )
+        if passed:
+            self._set_task_state(run_id, task, "validating")
+        else:
+            self._register_revision(run_id, task, feedback)
+
+    def _phase_validate(
+        self,
+        run_id: str,
+        phases: PhaseRunner,
+        task: TaskRecord,
+        budgets: object,
+    ) -> None:
+        started = time.time()
+        verdict = phases.validate(task)
+        self.store.record_phase(
+            run_id,
+            "validate",
+            task_id=task.spec.id,
+            attempt=task.replans + 1,
+            status=verdict.verdict,
+            output_json=verdict.model_dump_json(),
+            started_at=started,
+        )
+        if verdict.verdict == "accept":
+            self._set_task_state(run_id, task, "done")
+            return
+        task.replans += 1
+        if task.replans > self.config.budgets.max_replans_per_task:
+            task.last_feedback = verdict.feedback
+            self._set_task_state(run_id, task, "failed")
+            return
+        task.last_feedback = verdict.feedback or "validation rejected the result"
+        task.plan = None
+        task.revisions = 0
+        self._set_task_state(run_id, task, "planning")
+
+    def _register_revision(self, run_id: str, task: TaskRecord, feedback: str) -> None:
+        task.revisions += 1
+        task.last_feedback = feedback
+        if task.revisions > self.config.budgets.max_revisions_per_task:
+            self._set_task_state(run_id, task, "failed")
+        else:
+            self._set_task_state(run_id, task, "executing")
+
+    # -- bookkeeping -------------------------------------------------------
+
+    def _check_cancelled_and_clock(self, run_id: str, deadline: float) -> None:
+        if self.store.get_run(run_id).state == "cancelled":
+            raise StateError(f"run {run_id} was cancelled")
+        if self.clock() > deadline:
+            self._set_run_state(run_id, "failed")
+            raise BudgetExceededError(
+                f"run {run_id} exceeded max_wall_clock_s={self.config.budgets.max_wall_clock_s}"
+            )
+
+    def _set_run_state(self, run_id: str, state: RunState) -> None:
+        self.store.set_run_state(run_id, state)
+        self.bus.emit(HostEventTypes.RUN_STATE, run_id, state=state)
+
+    def _set_task_state(self, run_id: str, task: TaskRecord, state: TaskState) -> None:
+        task.state = state
+        self.store.update_task(run_id, task)
+        self.bus.emit(
+            HostEventTypes.TASK_STATE,
+            run_id,
+            task_id=task.spec.id,
+            state=state,
+            revisions=task.revisions,
+            replans=task.replans,
+        )
+
+    def _emit_task_end(self, run_id: str, task: TaskRecord) -> None:
+        self.bus.emit(
+            HostEventTypes.TASK_END,
+            run_id,
+            task_id=task.spec.id,
+            title=task.spec.title,
+            state=task.state,
+        )
+
+
+def run_outcome(outcome: str, config: Config | None = None) -> RunResult:
+    """Convenience one-shot API: run an outcome with default wiring."""
+    return LoopEngine(config or load_config()).start(outcome)