stargate-cli 1.0.0__py3-none-any.whl

stargate/fanout.py

@@ -0,0 +1,1152 @@
+"""Fan-out runs: architect a DAG, execute ready nodes, then review integration."""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from collections.abc import Callable
+from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .agent import invoke_agent
+from .commit import commit_run
+from .config import blocking_severities, commit_enabled, prompt_dirs, render_prompt
+from .core import RunContext, StargateError, git_quiet, print_output, short_name
+from .run import (
+    budget_spent,
+    complete_stage,
+    create_worktree,
+    enter_stage,
+    load_run,
+    make_context,
+    save_state,
+    snapshot,
+    unique_branch,
+    warn_if_dirty,
+    worktree_fingerprint,
+)
+from .stages import finish, plan_tests, review_and_finish, run_tests
+
+TASK_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,47}\Z")
+MAX_FANOUT_TASKS_DEFAULT = 8
+MAX_PARALLEL_TASKS_DEFAULT = 2
+
+
+@dataclass(frozen=True)
+class FanoutTask:
+    id: str
+    task: str
+    depends_on: tuple[str, ...]
+    acceptance: tuple[str, ...]
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "task": self.task,
+            "depends_on": list(self.depends_on),
+            "acceptance": list(self.acceptance),
+        }
+
+
+def _positive_setting(config: dict[str, Any], key: str, default: int) -> int:
+    value = config.get("settings", {}).get(key, default)
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        raise StargateError(f"settings.{key} must be a positive integer.")
+    return value
+
+
+def fanout_limits(
+    config: dict[str, Any], parallel_override: int | None = None
+) -> tuple[int, int]:
+    max_tasks = _positive_setting(
+        config, "max_fanout_tasks", MAX_FANOUT_TASKS_DEFAULT
+    )
+    max_parallel = (
+        parallel_override
+        if parallel_override is not None
+        else _positive_setting(
+            config, "max_parallel_tasks", MAX_PARALLEL_TASKS_DEFAULT
+        )
+    )
+    if (
+        isinstance(max_parallel, bool)
+        or not isinstance(max_parallel, int)
+        or max_parallel < 1
+    ):
+        raise StargateError("--max-parallel-tasks must be a positive integer.")
+    # The configured graph ceiling is not a concurrency ceiling. The scheduler
+    # naturally limits active workers to the number of ready graph nodes.
+    return max_tasks, max_parallel
+
+
+def parse_task_graph(
+    raw: str, max_tasks: int | None
+) -> tuple[str, list[FanoutTask]]:
+    """Validate the architect's JSON and return tasks in topological order."""
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise StargateError(
+            "Fan-out architect did not return valid JSON "
+            f"(line {exc.lineno}, column {exc.colno}): {exc.msg}"
+        ) from exc
+    if not isinstance(data, dict):
+        raise StargateError("Fan-out architect output must be one JSON object.")
+
+    name_value = data.get("name")
+    name = name_value.strip() if isinstance(name_value, str) else ""
+    if not short_name(name):
+        raise StargateError("Fan-out architect output needs a usable string 'name'.")
+
+    raw_tasks = data.get("tasks")
+    if not isinstance(raw_tasks, list) or not raw_tasks:
+        raise StargateError("Fan-out architect output needs a non-empty 'tasks' list.")
+    if max_tasks is not None and len(raw_tasks) > max_tasks:
+        extra = raw_tasks[max_tasks]
+        extra_id = extra.get("id") if isinstance(extra, dict) else None
+        where = f"tasks[{max_tasks}]"
+        if extra_id is not None:
+            where += f" ({extra_id!r})"
+        raise StargateError(
+            f"Fan-out graph contains {len(raw_tasks)} tasks; "
+            f"settings.max_fanout_tasks is {max_tasks}. "
+            f"First excess entry: {where}."
+        )
+
+    tasks: list[FanoutTask] = []
+    seen: set[str] = set()
+    for index, item in enumerate(raw_tasks, 1):
+        where = f"tasks[{index - 1}]"
+        if not isinstance(item, dict):
+            raise StargateError(f"{where} must be an object.")
+        task_id = item.get("id")
+        if not isinstance(task_id, str) or not TASK_ID.fullmatch(task_id):
+            raise StargateError(
+                f"{where}.id ({task_id!r}) must match "
+                "[a-z0-9][a-z0-9-]{0,47}."
+            )
+        if task_id in seen:
+            raise StargateError(
+                f"{where}.id duplicates fan-out task {task_id!r}."
+            )
+        seen.add(task_id)
+        task_where = f"Fan-out task {task_id!r} at {where}"
+
+        description = item.get("task")
+        if not isinstance(description, str) or not description.strip():
+            raise StargateError(f"{task_where}.task must be a non-empty string.")
+        dependencies = item.get("depends_on", [])
+        if not isinstance(dependencies, list) or not all(
+            isinstance(value, str) for value in dependencies
+        ):
+            raise StargateError(
+                f"{task_where}.depends_on must be a list of task IDs."
+            )
+        if len(set(dependencies)) != len(dependencies):
+            duplicate = next(
+                value
+                for offset, value in enumerate(dependencies)
+                if value in dependencies[:offset]
+            )
+            raise StargateError(
+                f"{task_where}.depends_on contains duplicate task ID "
+                f"{duplicate!r}."
+            )
+        acceptance = item.get("acceptance", [])
+        if (
+            not isinstance(acceptance, list)
+            or not all(
+                isinstance(value, str) and value.strip() for value in acceptance
+            )
+        ):
+            raise StargateError(
+                f"{task_where}.acceptance must be a list of non-empty strings."
+            )
+        tasks.append(
+            FanoutTask(
+                id=task_id,
+                task=description.strip(),
+                depends_on=tuple(dependencies),
+                acceptance=tuple(value.strip() for value in acceptance),
+            )
+        )
+
+    by_id = {task.id: task for task in tasks}
+    for task in tasks:
+        missing = [dep for dep in task.depends_on if dep not in by_id]
+        if missing:
+            raise StargateError(
+                f"Fan-out task {task.id!r} depends on unknown task(s): "
+                f"{', '.join(missing)}."
+            )
+        if task.id in task.depends_on:
+            raise StargateError(f"Fan-out task {task.id!r} depends on itself.")
+
+    position = {task.id: index for index, task in enumerate(tasks)}
+    indegree = {task.id: len(task.depends_on) for task in tasks}
+    children: dict[str, list[str]] = {task.id: [] for task in tasks}
+    for task in tasks:
+        for dependency in task.depends_on:
+            children[dependency].append(task.id)
+    ready = [task.id for task in tasks if indegree[task.id] == 0]
+    order: list[str] = []
+    while ready:
+        ready.sort(key=position.__getitem__)
+        task_id = ready.pop(0)
+        order.append(task_id)
+        for child in children[task_id]:
+            indegree[child] -= 1
+            if indegree[child] == 0:
+                ready.append(child)
+    if len(order) != len(tasks):
+        visiting: dict[str, int] = {}
+        visited: set[str] = set()
+        stack: list[str] = []
+
+        def find_cycle(task_id: str) -> list[str] | None:
+            visiting[task_id] = len(stack)
+            stack.append(task_id)
+            for dependency in by_id[task_id].depends_on:
+                if dependency in visiting:
+                    return [*stack[visiting[dependency] :], dependency]
+                if dependency not in visited:
+                    cycle = find_cycle(dependency)
+                    if cycle is not None:
+                        return cycle
+            stack.pop()
+            visiting.pop(task_id)
+            visited.add(task_id)
+            return None
+
+        cycle = next(
+            (
+                found
+                for task in tasks
+                if task.id not in visited
+                and (found := find_cycle(task.id)) is not None
+            ),
+            [],
+        )
+        raise StargateError(
+            "Fan-out task dependencies contain a cycle: "
+            f"{' -> '.join(cycle)}."
+        )
+    return name, [by_id[task_id] for task_id in order]
+
+
+def normalized_graph(name: str, tasks: list[FanoutTask]) -> str:
+    graph_tasks: list[dict[str, Any]] = []
+    for task in tasks:
+        item = task.as_dict()
+        if not task.acceptance:
+            item.pop("acceptance")
+        graph_tasks.append(item)
+    return json.dumps(
+        {"name": name, "tasks": graph_tasks},
+        indent=2,
+    ) + "\n"
+
+
+def _validate_frozen_graph(
+    ctx: RunContext, name: str, tasks: list[FanoutTask]
+) -> None:
+    if not ctx.fanout:
+        return
+    expected_order = ctx.fanout.get("order")
+    records = ctx.fanout.get("tasks")
+    if (
+        ctx.fanout.get("name") != name
+        or
+        expected_order != [task.id for task in tasks]
+        or not isinstance(records, dict)
+    ):
+        raise StargateError(
+            "tasks.json no longer matches the fan-out graph frozen in state.json."
+        )
+    for task in tasks:
+        record = records.get(task.id)
+        if not isinstance(record, dict) or any(
+            record.get(key) != value
+            for key, value in task.as_dict().items()
+        ):
+            raise StargateError(
+                "tasks.json no longer matches the fan-out graph frozen in state.json."
+            )
+
+
+def _new_task_records(
+    ctx: RunContext, name: str, tasks: list[FanoutTask]
+) -> dict[str, Any]:
+    records: dict[str, dict[str, Any]] = {}
+    reserved_branches: set[str] = set()
+    for task in tasks:
+        branch_base = f"{ctx.branch}-{task.id}"
+        branch = unique_branch(ctx.repo, branch_base)
+        suffix = 2
+        while branch in reserved_branches:
+            branch = unique_branch(ctx.repo, f"{branch_base}-{suffix}")
+            suffix += 1
+        reserved_branches.add(branch)
+        records[task.id] = {
+            **task.as_dict(),
+            "status": "pending",
+            "stage": "pending",
+            "branch": branch,
+            "worktree": str(ctx.worktree.parent / f"{ctx.run_id}-{task.id}"),
+            "base_commit": "",
+            "commit": "",
+            "test_exit": None,
+            "tokens_used": 0,
+            "test_artifacts": [],
+            "error": None,
+        }
+    return {
+        "name": name,
+        "order": [task.id for task in tasks],
+        "tasks": records,
+        "integrated": [],
+    }
+
+
+def _merge(worktree: Path, branch: str, purpose: str) -> None:
+    # A process can die after Git writes MERGE_HEAD but before our failure path
+    # gets to `merge --abort`. Generated worktrees have no legitimate reason to
+    # retain that state between scheduler invocations, so make an old merge
+    # recoverable before attempting the next one.
+    if _merge_residue(worktree):
+        print_output(f"Recovering an interrupted merge in {worktree}.")
+        _reset_worktree(worktree, "HEAD", clean_untracked=False)
+
+    tracked = git_quiet(
+        worktree, "status", "--porcelain", "--untracked-files=no"
+    ).strip()
+    if tracked:
+        raise StargateError(
+            f"Could not merge {branch!r} while {purpose}: {worktree} has "
+            "uncommitted tracked changes."
+        )
+
+    proc = subprocess.run(
+        ["git", "merge", "--no-edit", branch],
+        cwd=str(worktree),
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    if proc.stdout:
+        print_output(
+            proc.stdout, end="" if proc.stdout.endswith("\n") else "\n"
+        )
+    if proc.returncode == 0:
+        return
+    abort = subprocess.run(
+        ["git", "merge", "--abort"],
+        cwd=str(worktree),
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    cleanup_error = ""
+    tracked = git_quiet(
+        worktree, "status", "--porcelain", "--untracked-files=no"
+    ).strip()
+    if _merge_residue(worktree) or tracked:
+        try:
+            _reset_worktree(worktree, "HEAD", clean_untracked=False)
+        except StargateError as exc:
+            cleanup_error = str(exc)
+    residue = _merge_residue(worktree)
+    tracked = git_quiet(
+        worktree, "status", "--porcelain", "--untracked-files=no"
+    ).strip()
+    if residue or tracked:
+        abort_tail = "\n".join((abort.stdout or "").strip().splitlines()[-5:])
+        cleanup_error = cleanup_error or abort_tail or "Git left merge state behind."
+    tail = "\n".join((proc.stdout or "").strip().splitlines()[-20:])
+    raise StargateError(
+        f"Could not merge {branch!r} while {purpose}."
+        + (f"\nLast Git output:\n{tail}" if tail else "")
+        + (
+            f"\nCould not restore a clean worktree: {cleanup_error}"
+            if cleanup_error
+            else ""
+        )
+    )
+
+
+def _merge_residue(worktree: Path) -> bool:
+    merge_head = subprocess.run(
+        ["git", "rev-parse", "-q", "--verify", "MERGE_HEAD"],
+        cwd=str(worktree),
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    unmerged = subprocess.run(
+        ["git", "ls-files", "-u"],
+        cwd=str(worktree),
+        text=True,
+        capture_output=True,
+    )
+    return merge_head.returncode == 0 or bool(unmerged.stdout.strip())
+
+
+def _reset_worktree(
+    worktree: Path, revision: str, *, clean_untracked: bool
+) -> None:
+    commands = [["reset", "--hard", revision]]
+    if clean_untracked:
+        # Task worktrees are generated scratch space. An unfinished attempt must
+        # not silently influence the next developer, including via ignored
+        # build output or a nested repository.
+        commands.append(["clean", "-ffdx"])
+    for command in commands:
+        proc = subprocess.run(
+            ["git", *command],
+            cwd=str(worktree),
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
+        if proc.returncode:
+            detail = "\n".join((proc.stdout or "").strip().splitlines()[-10:])
+            raise StargateError(
+                f"Could not reset task worktree {worktree} with "
+                f"'git {' '.join(command)}'."
+                + (f"\nLast Git output:\n{detail}" if detail else "")
+            )
+
+
+def _assert_branch(worktree: Path, expected: str, purpose: str) -> None:
+    proc = subprocess.run(
+        ["git", "symbolic-ref", "--short", "HEAD"],
+        cwd=str(worktree),
+        text=True,
+        capture_output=True,
+    )
+    checked_out = proc.stdout.strip() if proc.returncode == 0 else "(detached HEAD)"
+    if checked_out != expected:
+        raise StargateError(
+            f"{purpose} worktree {worktree} is on {checked_out!r}, "
+            f"not {expected!r}."
+        )
+
+
+def _ref_commit(repo: Path, branch: str) -> str:
+    proc = subprocess.run(
+        [
+            "git",
+            "rev-parse",
+            "--verify",
+            "--end-of-options",
+            f"refs/heads/{branch}^{{commit}}",
+        ],
+        cwd=str(repo),
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+    )
+    return proc.stdout.strip() if proc.returncode == 0 else ""
+
+
+def _is_ancestor(repo: Path, ancestor: str, descendant: str) -> bool:
+    return subprocess.run(
+        ["git", "merge-base", "--is-ancestor", ancestor, descendant],
+        cwd=str(repo),
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    ).returncode == 0
+
+
+def _task_context(ctx: RunContext, record: dict[str, Any]) -> RunContext:
+    task_artifacts = ctx.artifacts / "tasks" / record["id"]
+    task_artifacts.mkdir(parents=True, exist_ok=True)
+    task_ctx = RunContext(
+        repo=ctx.repo,
+        config=ctx.config,
+        run_id=ctx.run_id,
+        slug=record["id"],
+        branch=record["branch"],
+        base_ref=ctx.base_ref,
+        base_commit=record["base_commit"] or ctx.base_commit,
+        worktree=Path(record["worktree"]),
+        artifacts=task_artifacts,
+        task=record["task"],
+        tokens_used=int(record.get("tokens_used", 0)),
+        test_artifacts=set(record.get("test_artifacts", [])),
+        commit=str(record.get("commit") or ""),
+        mode="fanout-task",
+    )
+    task_ctx.test_command = ctx.test_command
+    task_ctx.test_source = ctx.test_source
+    task_ctx.detected = ctx.detected
+    return task_ctx
+
+
+def _prepare_task(ctx: RunContext, record: dict[str, Any]) -> RunContext:
+    dependencies = [ctx.fanout["tasks"][item] for item in record["depends_on"]]
+    missing = [item["id"] for item in dependencies if not item.get("commit")]
+    if missing:
+        raise StargateError(
+            f"Task {record['id']!r} has uncommitted dependencies: {', '.join(missing)}."
+        )
+
+    initial_base = dependencies[0]["commit"] if dependencies else ctx.base_commit
+    recorded_base = str(record.get("base_commit") or "")
+    preparation_base = recorded_base or initial_base
+    task_ctx = _task_context(ctx, record)
+    task_ctx.base_commit = preparation_base
+    existed = task_ctx.worktree.exists() or bool(
+        _ref_commit(ctx.repo, task_ctx.branch)
+    )
+    create_worktree(
+        task_ctx,
+        recover_recorded_fanout_branch=bool(record.get("branch_created")),
+    )
+    if not record.get("branch_created"):
+        # Persist ownership only after Git creates the branch successfully. A
+        # competing process that takes the planned name must still be refused.
+        record["branch_created"] = True
+        save_state(ctx, "running")
+    _assert_branch(task_ctx.worktree, task_ctx.branch, "Task")
+
+    if existed:
+        print_output(
+            f"Clean retry for task {record['id']}: resetting {task_ctx.worktree} "
+            f"to {preparation_base[:12]} and discarding the unfinished "
+            "attempt's edits."
+        )
+        _reset_worktree(task_ctx.worktree, preparation_base, clean_untracked=True)
+
+    head = git_quiet(task_ctx.worktree, "rev-parse", "HEAD").strip()
+    if head != preparation_base:
+        raise StargateError(
+            f"Task {record['id']!r} branch {task_ctx.branch!r} starts at "
+            f"{head}, not its recorded base {preparation_base}."
+        )
+
+    if recorded_base:
+        missing_from_base = [
+            dependency["id"]
+            for dependency in dependencies
+            if not _is_ancestor(ctx.repo, dependency["commit"], recorded_base)
+        ]
+        if missing_from_base:
+            raise StargateError(
+                f"Task {record['id']!r} recorded base does not contain "
+                f"dependency task(s): {', '.join(missing_from_base)}."
+            )
+    else:
+        for dependency in dependencies:
+            branch_head = _ref_commit(ctx.repo, dependency["branch"])
+            if branch_head != dependency["commit"]:
+                raise StargateError(
+                    f"Dependency task {dependency['id']!r} branch "
+                    f"{dependency['branch']!r} is at {branch_head or '(missing)'}, "
+                    f"not its recorded commit {dependency['commit']}."
+                )
+            _merge(
+                task_ctx.worktree,
+                dependency["commit"],
+                f"preparing task {record['id']!r} from dependency "
+                f"{dependency['id']!r}",
+            )
+        task_ctx.base_commit = git_quiet(
+            task_ctx.worktree, "rev-parse", "HEAD"
+        ).strip()
+        record["base_commit"] = task_ctx.base_commit
+
+    missing_from_head = [
+        dependency["id"]
+        for dependency in dependencies
+        if not _is_ancestor(ctx.repo, dependency["commit"], task_ctx.base_commit)
+    ]
+    if missing_from_head:
+        raise StargateError(
+            f"Task {record['id']!r} worktree does not contain dependency "
+            f"task(s): {', '.join(missing_from_head)}."
+        )
+    record["stage"] = "ready"
+    return task_ctx
+
+
+def _task_prompt(overall_task: str, task: FanoutTask) -> str:
+    acceptance = "\n".join(f"- {item}" for item in task.acceptance) or "- Not specified"
+    dependencies = ", ".join(task.depends_on) or "none"
+    return (
+        f"OVERALL FAN-OUT GOAL:\n{overall_task}\n\n"
+        f"ASSIGNED TASK ID: {task.id}\n"
+        f"DEPENDENCIES ALREADY PRESENT: {dependencies}\n\n"
+        f"TASK:\n{task.task}\n\n"
+        f"ACCEPTANCE CRITERIA:\n{acceptance}"
+    )
+
+
+def _execute_task(
+    outer: RunContext,
+    task_ctx: RunContext,
+    task: FanoutTask,
+    prompts: list[Path],
+    graph: str,
+) -> dict[str, Any]:
+    task_ctx.stage = "developer"
+    save_state(task_ctx, "running")
+    before = worktree_fingerprint(task_ctx)
+    prompt = render_prompt(
+        prompts,
+        "developer",
+        task=_task_prompt(outer.task, task),
+        base_ref=task_ctx.base_commit,
+        plan=graph,
+    )
+    print_output(f"\n=== TASK {task.id}: DEVELOPER ===")
+    invoke_agent(
+        task_ctx,
+        "developer",
+        prompt,
+        task_ctx.worktree,
+        task_ctx.artifacts / "developer.txt",
+    )
+    # Persist usage immediately after the billable phase. The outer scheduler
+    # reconciles this total into its own state once the future completes.
+    save_state(task_ctx, "running")
+    if worktree_fingerprint(task_ctx) == before:
+        raise StargateError(
+            f"Developer changed nothing for fan-out task {task.id!r} in "
+            f"{task_ctx.worktree}."
+        )
+    complete_stage(task_ctx, "developer")
+    test_exit, _ = run_tests(task_ctx, "task")
+    outcome = commit_run(task_ctx, "TASK_COMPLETE", test_exit)
+    if outcome == "failed":
+        raise StargateError(
+            task_ctx.commit_error or f"Could not commit fan-out task {task.id!r}."
+        )
+    if not task_ctx.commit:
+        raise StargateError(f"Fan-out task {task.id!r} produced no commit.")
+    result = {
+        "commit": task_ctx.commit,
+        "test_exit": test_exit,
+        "tokens_used": task_ctx.tokens_used,
+        "test_artifacts": sorted(task_ctx.test_artifacts),
+    }
+    (task_ctx.artifacts / "result.json").write_text(
+        json.dumps(result, indent=2) + "\n"
+    )
+    save_state(task_ctx, "task_complete")
+    return result
+
+
+def _recover_task(ctx: RunContext, record: dict[str, Any]) -> bool:
+    state_path = ctx.artifacts / "tasks" / record["id"] / "state.json"
+    if not state_path.exists():
+        return False
+    try:
+        state = json.loads(state_path.read_text())
+    except (OSError, ValueError):
+        return False
+    if not isinstance(state, dict):
+        return False
+    branch_head = _ref_commit(ctx.repo, record["branch"])
+    if not branch_head:
+        return False
+    state_commit = str(state.get("commit") or "")
+    if state.get("status") == "task_complete" and state_commit:
+        if state_commit != branch_head:
+            return False
+    commit = branch_head
+    message_proc = subprocess.run(
+        ["git", "show", "-s", "--format=%B", commit],
+        cwd=str(ctx.repo),
+        text=True,
+        capture_output=True,
+    )
+    if message_proc.returncode:
+        return False
+    trailers = {
+        key: value
+        for line in message_proc.stdout.splitlines()
+        if ": " in line
+        for key, value in [line.split(": ", 1)]
+    }
+    base_commit = str(record.get("base_commit") or "")
+    if (
+        trailers.get("Stargate-Run-Id") != ctx.run_id
+        or trailers.get("Stargate-Verdict") != "TASK_COMPLETE"
+        or trailers.get("Stargate-Base-Commit") != base_commit
+        or not base_commit
+        or not _is_ancestor(ctx.repo, base_commit, commit)
+    ):
+        return False
+    result_path = state_path.parent / "result.json"
+    try:
+        result = json.loads(result_path.read_text())
+    except (OSError, ValueError):
+        result = {}
+    if not isinstance(result, dict) or (
+        result.get("commit") and result.get("commit") != commit
+    ):
+        result = {}
+    try:
+        tokens_used = max(
+            0,
+            int(record.get("tokens_used", 0)),
+            int(state.get("tokens_used", 0)),
+            int(result.get("tokens_used", 0)),
+        )
+    except (TypeError, ValueError):
+        return False
+    test_exit = result.get("test_exit")
+    if test_exit is None and "Stargate-Tests-Exit" in trailers:
+        try:
+            test_exit = int(trailers["Stargate-Tests-Exit"])
+        except ValueError:
+            test_exit = None
+    record.update(
+        status="complete",
+        stage="complete",
+        commit=commit,
+        test_exit=test_exit,
+        tokens_used=tokens_used,
+        test_artifacts=list(
+            result.get("test_artifacts", state.get("test_artifacts", []))
+        ),
+        error=None,
+    )
+    print_output(
+        f"Recovered completed task {record['id']} at {commit[:12]} from its "
+        "task branch."
+    )
+    return True
+
+
+def _run_scheduler(
+    ctx: RunContext,
+    tasks: list[FanoutTask],
+    prompts: list[Path],
+    graph: str,
+    max_parallel: int,
+) -> int | None:
+    enter_stage(ctx, "tasks")
+    by_id = {task.id: task for task in tasks}
+    records = ctx.fanout["tasks"]
+    for task in tasks:
+        record = records[task.id]
+        if record["status"] == "complete":
+            continue
+        previous_tokens = int(record.get("tokens_used", 0))
+        if _recover_task(ctx, record):
+            ctx.tokens_used += max(
+                0, int(record.get("tokens_used", 0)) - previous_tokens
+            )
+            continue
+        record["status"] = "pending"
+        record["stage"] = "pending"
+        record["error"] = None
+    save_state(ctx, "running")
+
+    running: dict[Future[dict[str, Any]], tuple[str, RunContext]] = {}
+    budget_reached = False
+    interrupted: BaseException | None = None
+    executor = ThreadPoolExecutor(
+        max_workers=max_parallel, thread_name_prefix="stargate-task"
+    )
+    try:
+        while True:
+            if not budget_reached:
+                pending = [
+                    task.id
+                    for task in tasks
+                    if records[task.id]["status"] == "pending"
+                ]
+                next_phase = f"fan-out task {pending[0]}" if pending else "integration"
+                if budget_spent(ctx, next_phase):
+                    budget_reached = True
+                    print_output(
+                        "The cap is enforced between task agents. Up to "
+                        f"{max_parallel} concurrently started task agent(s) may "
+                        "finish above it; no more task agents will be scheduled.",
+                        file=sys.stderr,
+                    )
+
+            if not budget_reached:
+                for task in tasks:
+                    if len(running) >= max_parallel:
+                        break
+                    record = records[task.id]
+                    if record["status"] != "pending":
+                        continue
+                    if not all(
+                        records[dependency]["status"] == "complete"
+                        for dependency in task.depends_on
+                    ):
+                        continue
+                    try:
+                        task_ctx = _prepare_task(ctx, record)
+                    except StargateError as exc:
+                        record.update(
+                            status="failed", stage="prepare", error=str(exc)
+                        )
+                        save_state(ctx, "running")
+                        continue
+                    record.update(status="running", stage="developer", error=None)
+                    save_state(ctx, "running")
+                    future = executor.submit(
+                        _execute_task, ctx, task_ctx, by_id[task.id], prompts, graph
+                    )
+                    running[future] = (task.id, task_ctx)
+                    print_output(
+                        f"Scheduled {task.id} on {record['branch']} "
+                        f"({len(running)}/{max_parallel} slots)"
+                    )
+            if not running:
+                break
+
+            finished, _ = wait(running, return_when=FIRST_COMPLETED)
+            for future in finished:
+                task_id, task_ctx = running.pop(future)
+                record = records[task_id]
+                try:
+                    result = future.result()
+                except BaseException as exc:
+                    previous_tokens = int(record.get("tokens_used", 0))
+                    record.update(
+                        status="failed",
+                        stage=task_ctx.stage,
+                        tokens_used=task_ctx.tokens_used,
+                        test_artifacts=sorted(task_ctx.test_artifacts),
+                        error=f"{type(exc).__name__}: {exc}",
+                    )
+                    ctx.tokens_used += max(
+                        0,
+                        task_ctx.tokens_used - previous_tokens,
+                    )
+                    print_output(
+                        f"\nTask {task_id} failed: {exc}", file=sys.stderr
+                    )
+                else:
+                    previous_tokens = int(record.get("tokens_used", 0))
+                    record.update(
+                        status="complete",
+                        stage="complete",
+                        commit=result["commit"],
+                        test_exit=result["test_exit"],
+                        tokens_used=result["tokens_used"],
+                        test_artifacts=result["test_artifacts"],
+                        error=None,
+                    )
+                    ctx.tokens_used += max(
+                        0, int(result["tokens_used"]) - previous_tokens
+                    )
+                    print_output(
+                        f"\nTask {task_id} complete at "
+                        f"{result['commit'][:12]}"
+                    )
+                save_state(ctx, "running")
+    except BaseException as exc:
+        interrupted = exc
+    finally:
+        executor.shutdown(wait=True, cancel_futures=True)
+
+    if interrupted is not None:
+        for future, (task_id, task_ctx) in running.items():
+            record = records[task_id]
+            previous_tokens = int(record.get("tokens_used", 0))
+            if future.cancelled():
+                record.update(
+                    status="pending",
+                    stage="interrupted",
+                    error="Interrupted before the task agent started; safe to retry.",
+                )
+                continue
+            try:
+                result = future.result()
+            except BaseException:
+                record.update(
+                    status="pending",
+                    stage="interrupted",
+                    tokens_used=task_ctx.tokens_used,
+                    test_artifacts=sorted(task_ctx.test_artifacts),
+                    error=(
+                        "Interrupted before task completion; retry will use the "
+                        "clean-retry policy."
+                    ),
+                )
+                ctx.tokens_used += max(
+                    0, task_ctx.tokens_used - previous_tokens
+                )
+            else:
+                record.update(
+                    status="complete",
+                    stage="complete",
+                    commit=result["commit"],
+                    test_exit=result["test_exit"],
+                    tokens_used=result["tokens_used"],
+                    test_artifacts=result["test_artifacts"],
+                    error=None,
+                )
+                ctx.tokens_used += max(
+                    0, int(result["tokens_used"]) - previous_tokens
+                )
+        for task in tasks:
+            record = records[task.id]
+            if record["status"] == "running":
+                record.update(
+                    status="pending",
+                    stage="interrupted",
+                    error=(
+                        "Interrupted while scheduling the task; retry will use "
+                        "the clean-retry policy."
+                    ),
+                )
+        save_state(
+            ctx,
+            "failed",
+            f"{type(interrupted).__name__}: {interrupted}",
+        )
+        raise interrupted
+
+    if budget_reached:
+        if all(records[task.id]["status"] == "complete" for task in tasks):
+            complete_stage(ctx, "tasks")
+        save_state(ctx, "budget_exceeded")
+        return 4
+
+    failed = [task.id for task in tasks if records[task.id]["status"] == "failed"]
+    for task in tasks:
+        record = records[task.id]
+        if record["status"] == "pending":
+            record.update(
+                status="blocked",
+                stage="dependencies",
+                error="A dependency did not complete.",
+            )
+    blocked = [task.id for task in tasks if records[task.id]["status"] == "blocked"]
+    if failed or blocked:
+        save_state(ctx, "failed", "One or more fan-out tasks did not complete.")
+        details = [*failed, *(f"{task_id} (blocked)" for task_id in blocked)]
+        raise StargateError("Fan-out tasks did not complete: " + ", ".join(details))
+    complete_stage(ctx, "tasks")
+    return None
+
+
+def _integrate(ctx: RunContext, tasks: list[FanoutTask]) -> None:
+    enter_stage(ctx, "integration")
+    print("\n=== INTEGRATION ===")
+    create_worktree(ctx)
+    _assert_branch(ctx.worktree, ctx.branch, "Integration")
+    integrated = ctx.fanout.setdefault("integrated", [])
+    for task in tasks:
+        if task.id in integrated:
+            continue
+        record = ctx.fanout["tasks"][task.id]
+        branch_head = _ref_commit(ctx.repo, record["branch"])
+        if branch_head != record["commit"]:
+            raise StargateError(
+                f"Task {task.id!r} branch {record['branch']!r} is at "
+                f"{branch_head or '(missing)'}, not its recorded commit "
+                f"{record['commit']}."
+            )
+        _merge(
+            ctx.worktree,
+            record["commit"],
+            f"integrating task {task.id!r} from branch {record['branch']!r}",
+        )
+        integrated.append(task.id)
+        ctx.commit = git_quiet(ctx.worktree, "rev-parse", "HEAD").strip()
+        save_state(ctx, "running")
+    complete_stage(ctx, "integration")
+
+
+def _load_or_create_graph(
+    ctx: RunContext,
+    prompts: list[Path],
+    max_tasks: int,
+) -> tuple[list[FanoutTask], str]:
+    """Load or create the frozen graph, enforcing current configured limits."""
+    graph_path = ctx.artifacts / "tasks.json"
+    architect_complete = "architect" in ctx.done
+    if architect_complete:
+        if not ctx.fanout:
+            raise StargateError(
+                "tasks.json no longer matches the fan-out graph frozen in state.json."
+            )
+        if graph_path.exists():
+            raw = graph_path.read_text()
+            print(f"\n=== ARCHITECT (skipped, reusing {graph_path}) ===")
+        else:
+            order = ctx.fanout.get("order")
+            records = ctx.fanout.get("tasks")
+            if not isinstance(order, list) or not isinstance(records, dict):
+                raise StargateError(
+                    "tasks.json no longer matches the fan-out graph frozen in "
+                    "state.json."
+                )
+            frozen_tasks: list[dict[str, Any]] = []
+            for task_id in order:
+                record = records.get(task_id)
+                if not isinstance(record, dict):
+                    raise StargateError(
+                        "tasks.json no longer matches the fan-out graph frozen in "
+                        "state.json."
+                    )
+                frozen_task = {
+                    key: record.get(key)
+                    for key in ("id", "task", "depends_on")
+                }
+                if record.get("acceptance"):
+                    frozen_task["acceptance"] = record["acceptance"]
+                frozen_tasks.append(frozen_task)
+            raw = json.dumps(
+                {"name": ctx.fanout.get("name"), "tasks": frozen_tasks}
+            )
+            print(f"\n=== ARCHITECT (skipped, restoring {graph_path}) ===")
+    else:
+        enter_stage(ctx, "architect")
+        architect_prompt = render_prompt(
+            prompts,
+            "fanout",
+            task=ctx.task,
+            base_ref=ctx.base_ref,
+            max_tasks=str(max_tasks),
+        )
+        print("\n=== ARCHITECT: FAN-OUT ===")
+        raw = invoke_agent(
+            ctx,
+            "architect",
+            architect_prompt,
+            ctx.repo,
+            ctx.artifacts / "architect-tasks.json",
+        ).strip()
+
+    name, tasks = parse_task_graph(raw, max_tasks)
+    graph = normalized_graph(name, tasks)
+    _validate_frozen_graph(ctx, name, tasks)
+    graph_path.write_text(graph)
+    if not ctx.named_by_user and ctx.tag and not ctx.fanout:
+        proposed = f"stargate/{short_name(name)}-{ctx.tag}"
+        ctx.branch = unique_branch(ctx.repo, proposed)
+        print(f"Branch:   {ctx.branch}   (named by the architect)")
+        # Freeze the architect-selected parent before deriving task branches.
+        save_state(ctx, "running")
+    if not ctx.fanout:
+        ctx.fanout = _new_task_records(ctx, name, tasks)
+    if not architect_complete:
+        complete_stage(ctx, "architect")
+    return tasks, graph
+
+
+def orchestrate_fanout(
+    args: argparse.Namespace,
+    script_dir: Path,
+    config: dict[str, Any],
+    repo: Path,
+    on_context: Callable[[RunContext], None] | None = None,
+) -> int:
+    """Run or resume the opt-in DAG workflow.
+
+    `on_context` receives the run being orchestrated once it is loaded or
+    created and validated, so a caller handling a failure can act on that exact
+    run instead of re-deriving it from the runs directory.
+    """
+    resuming = args.command == "resume"
+    commit_error = (
+        "Fan-out requires commits to move work between task branches; "
+        "remove --no-commit and enable settings.commit."
+    )
+
+    # Invocation errors must not reserve a new run or rewrite an existing run's
+    # resumable status. A resume still validates the frozen/explicit config
+    # after loading it below.
+    if resuming and getattr(args, "redo", []):
+        raise StargateError("--redo is not supported for fan-out runs.")
+    if args.no_commit:
+        raise StargateError(commit_error)
+    if not resuming:
+        blocking_severities(config)
+        if not commit_enabled(config):
+            raise StargateError(commit_error)
+        max_tasks, max_parallel = fanout_limits(
+            config, args.max_parallel_tasks
+        )
+
+    if resuming:
+        ctx = load_run(repo, args.run_id, config, use_frozen=args.config is None)
+        blocking_severities(ctx.config)
+        if not commit_enabled(ctx.config):
+            raise StargateError(commit_error)
+        max_tasks, max_parallel = fanout_limits(
+            ctx.config, args.max_parallel_tasks
+        )
+        prompts = [ctx.artifacts / "prompts"]
+        print(f"\nResuming fan-out {ctx.run_id}: {ctx.task}")
+    else:
+        warn_if_dirty(repo)
+        ctx = make_context(
+            repo, config, args.task, args.base_ref, args.name,
+            task_source=getattr(args, "from_url", None) or "",
+        )
+        ctx.mode = "fanout"
+        prompts = snapshot(ctx, prompt_dirs(config, script_dir))
+        save_state(ctx, "running")
+
+    print(f"\nRun ID:   {ctx.run_id}")
+    print(f"Mode:     fan-out ({max_parallel} parallel tasks)")
+    print(f"Base:     {ctx.base_ref} @ {ctx.base_commit[:12]}")
+    print(f"Branch:   {ctx.branch}")
+    print(f"Worktree: {ctx.worktree}   (integration)")
+    print(f"Artifacts:{ctx.artifacts}")
+
+    # Announced only past invocation and config validation: an error raised
+    # before any orchestration happened must leave an existing run's artifacts
+    # exactly as its last completed attempt wrote them.
+    if on_context is not None:
+        on_context(ctx)
+
+    try:
+        plan_tests(ctx)
+        tasks, graph = _load_or_create_graph(ctx, prompts, max_tasks)
+        scheduler_exit = _run_scheduler(
+            ctx, tasks, prompts, graph, max_parallel
+        )
+        if scheduler_exit is not None:
+            # `finish` gives this resumable terminal path the same summary and
+            # RESULT block as every other budget stop. The integration worktree
+            # may not exist yet, so create only its untouched base branch. Do
+            # not put a terminal commit on that branch: a later resume must
+            # integrate task work directly on the frozen base, without a
+            # historical BUDGET_EXCEEDED marker beneath it.
+            create_worktree(ctx)
+            _assert_branch(ctx.worktree, ctx.branch, "Integration")
+            exit_code = finish(
+                ctx,
+                ctx.task,
+                "BUDGET_EXCEEDED",
+                None,
+                commit=False,
+                publish=getattr(args, "pr", False),
+            )
+            print(
+                f"\nResume with a larger token budget: stargate resume {ctx.run_id}",
+                file=sys.stderr,
+            )
+            return exit_code
+        _integrate(ctx, tasks)
+        test_exit, test_report = run_tests(ctx, "integration")
+        return review_and_finish(
+            ctx,
+            args,
+            prompts,
+            task=ctx.task,
+            plan=graph,
+            test_exit=test_exit,
+            test_report=test_report,
+            commit=True,
+        )
+    except (StargateError, KeyboardInterrupt) as exc:
+        save_state(ctx, "failed", f"{type(exc).__name__}: {exc}")
+        print(f"\nResume with: stargate resume {ctx.run_id}", file=sys.stderr)
+        raise