prizmkit 1.1.149 → 1.1.151

package/bundled/dev-pipeline/prizmkit_runtime/reset_preserve.py

@@ -0,0 +1,822 @@
+"""Non-destructive failed-chain reset for the unified Python runtime."""
+
+from __future__ import annotations
+
+import copy
+import json
+import os
+import shlex
+import shutil
+import tempfile
+import time
+from contextlib import ExitStack
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .checkpoint_state import (
+    load_checkpoint_state,
+    rewind_checkpoint_for_implementation,
+)
+from .gitops import (
+    BranchContext,
+    WorktreePolicy,
+    WorktreeRuntimeContext,
+    create_branch_from_base,
+    ensure_linked_worktree,
+    local_branch_exists,
+    registered_worktrees,
+    worktree_runtime_context,
+)
+from .runner_models import RunnerFamily
+from .task_checkout import (
+    CHECKOUT_MODE_BRANCH,
+    CHECKOUT_MODE_WORKTREE,
+    TaskCheckout,
+    TaskCheckoutError,
+    load_task_checkout,
+    replace_active_task_checkout_locked,
+    task_checkout_guard,
+)
+
+
+@dataclass(frozen=True)
+class PreservePlanItem:
+    """Validated plan item used by preserve-runtime target discovery."""
+
+    item_id: str
+    title: str
+    slug: str
+    status: str
+    dependencies: tuple[str, ...]
+    index: int
+
+
+@dataclass(frozen=True)
+class PreservePlanSnapshot:
+    """Validated selected plan plus failed-chain targets."""
+
+    data: dict[str, Any]
+    items: tuple[PreservePlanItem, ...]
+    failed_roots: tuple[PreservePlanItem, ...]
+    auto_skipped_descendants: tuple[PreservePlanItem, ...]
+
+
+@dataclass(frozen=True)
+class RootRecovery:
+    """Preflighted recovery operation for one failed root."""
+
+    item: PreservePlanItem
+    checkout: TaskCheckout
+    status_path: Path
+    status_data: dict[str, Any]
+    replacement_checkout: TaskCheckout | None = None
+    replacement_worktree: WorktreeRuntimeContext | None = None
+    source_artifact: Path | None = None
+    target_artifact: Path | None = None
+    checkpoint_target: Path | None = None
+    checkpoint_data: dict[str, Any] | None = None
+
+    @property
+    def replaces_branch(self) -> bool:
+        return self.replacement_checkout is not None
+
+
+@dataclass(frozen=True)
+class PreserveRuntimeResetResult:
+    """Observable result of one preserve-runtime reset invocation."""
+
+    ok: bool
+    failed_roots: tuple[str, ...] = ()
+    auto_skipped_descendants: tuple[str, ...] = ()
+    reused_checkouts: tuple[str, ...] = ()
+    replacement_checkouts: tuple[tuple[str, str, str], ...] = ()
+    blockers: tuple[str, ...] = ()
+    no_targets: bool = False
+
+    def lines(self, family: RunnerFamily, list_path: Path) -> tuple[str, ...]:
+        if self.no_targets:
+            return (
+                f"No failed {family.kind} tasks require preserve-runtime recovery.",
+                "No plan, status, checkout, checkpoint, branch, worktree, session, or artifact state changed.",
+            )
+        if not self.ok:
+            return (
+                "Preserve-runtime reset blocked before task-state recovery:",
+                *(f"ERROR: {blocker}" for blocker in self.blockers),
+            )
+
+        lines = [
+            "Preserve-runtime reset complete.",
+            f"Failed roots restored: {', '.join(self.failed_roots)}",
+        ]
+        if self.auto_skipped_descendants:
+            lines.append(
+                "Causal auto-skipped descendants restored: "
+                + ", ".join(self.auto_skipped_descendants)
+            )
+        else:
+            lines.append("Causal auto-skipped descendants restored: none")
+        if self.reused_checkouts:
+            lines.append("Active checkouts reused: " + ", ".join(self.reused_checkouts))
+        for item_id, old_branch, new_branch in self.replacement_checkouts:
+            lines.append(f"Replacement checkout: {item_id}: {old_branch} -> {new_branch}")
+        lines.extend(
+            (
+                "Existing sessions, checkpoints, artifacts, branches, worktrees, and task work were not deleted.",
+                "Reset did not start a runner. Run separately:",
+                "python3 ./.prizmkit/dev-pipeline/cli.py "
+                f"{family.kind} run {shlex.quote(str(list_path))}",
+            )
+        )
+        return tuple(lines)
+
+
+def run_preserve_runtime_reset(
+    family: RunnerFamily,
+    list_path: Path,
+    project_root: Path,
+) -> PreserveRuntimeResetResult:
+    """Restore every failed root and its causal auto-skipped descendants."""
+    try:
+        initial_bytes = list_path.read_bytes()
+        initial = _load_plan_snapshot(family, list_path, initial_bytes)
+    except (OSError, ValueError) as exc:
+        return PreserveRuntimeResetResult(False, blockers=(str(exc),))
+
+    if not initial.failed_roots:
+        return PreserveRuntimeResetResult(True, no_targets=True)
+
+    blockers: list[str] = []
+    recoveries: list[RootRecovery] = []
+    with ExitStack() as stack:
+        for item in initial.failed_roots:
+            try:
+                stack.enter_context(task_checkout_guard(family.state_dir, item.item_id))
+            except (RuntimeError, TaskCheckoutError) as exc:
+                blockers.append(f"{item.item_id}: checkout lifecycle is busy: {exc}")
+        if blockers:
+            return _blocked(initial, blockers)
+
+        try:
+            locked_bytes = list_path.read_bytes()
+        except OSError as exc:
+            return _blocked(initial, [f"Cannot reload selected plan under task guards: {exc}"])
+        if locked_bytes != initial_bytes:
+            return _blocked(initial, ["Selected plan changed while preserve-runtime reset acquired task guards"])
+        try:
+            locked = _load_plan_snapshot(family, list_path, locked_bytes)
+        except ValueError as exc:
+            return _blocked(initial, [str(exc)])
+
+        registrations = registered_worktrees(project_root)
+        for item in locked.failed_roots:
+            recovery, item_blockers = _preflight_root(
+                family,
+                item,
+                project_root,
+                registrations,
+            )
+            blockers.extend(item_blockers)
+            if recovery is not None:
+                recoveries.append(recovery)
+
+        blockers.extend(_writability_blockers(list_path, recoveries))
+        if blockers:
+            return _blocked(locked, blockers)
+
+        try:
+            _apply_recovery(family, locked, recoveries, project_root, list_path)
+        except (OSError, RuntimeError, TaskCheckoutError, ValueError) as exc:
+            return _blocked(
+                locked,
+                [
+                    "Recovery apply failed after successful preflight; existing task work was not cleaned or deleted: "
+                    f"{exc}"
+                ],
+            )
+
+    reused = tuple(recovery.item.item_id for recovery in recoveries if not recovery.replaces_branch)
+    replacements = tuple(
+        (
+            recovery.item.item_id,
+            recovery.checkout.active_dev_branch,
+            recovery.replacement_checkout.active_dev_branch,
+        )
+        for recovery in recoveries
+        if recovery.replacement_checkout is not None
+    )
+    return PreserveRuntimeResetResult(
+        True,
+        failed_roots=tuple(item.item_id for item in initial.failed_roots),
+        auto_skipped_descendants=tuple(
+            item.item_id for item in initial.auto_skipped_descendants
+        ),
+        reused_checkouts=reused,
+        replacement_checkouts=replacements,
+    )
+
+
+def _blocked(
+    snapshot: PreservePlanSnapshot,
+    blockers: list[str],
+) -> PreserveRuntimeResetResult:
+    return PreserveRuntimeResetResult(
+        False,
+        failed_roots=tuple(item.item_id for item in snapshot.failed_roots),
+        auto_skipped_descendants=tuple(
+            item.item_id for item in snapshot.auto_skipped_descendants
+        ),
+        blockers=tuple(blockers),
+    )
+
+
+def _load_plan_snapshot(
+    family: RunnerFamily,
+    list_path: Path,
+    raw_bytes: bytes | None = None,
+) -> PreservePlanSnapshot:
+    try:
+        data = json.loads((raw_bytes if raw_bytes is not None else list_path.read_bytes()).decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise ValueError(f"Invalid {family.kind} plan JSON: {exc}") from exc
+    if not isinstance(data, dict):
+        raise ValueError(f"Invalid {family.kind} plan: root must be an object")
+    raw_items = data.get(family.list_key)
+    if not isinstance(raw_items, list):
+        raise ValueError(f"Invalid {family.kind} plan: {family.list_key} must be an array")
+
+    items: list[PreservePlanItem] = []
+    seen: set[str] = set()
+    errors: list[str] = []
+    for index, raw_item in enumerate(raw_items):
+        if not isinstance(raw_item, dict):
+            errors.append(f"item at index {index} must be an object")
+            continue
+        item_id = _normalize_item_id(str(raw_item.get("id") or ""), family.id_prefix)
+        if not item_id:
+            errors.append(f"item at index {index} has no valid id")
+            continue
+        if item_id in seen:
+            errors.append(f"duplicate item id: {item_id}")
+            continue
+        seen.add(item_id)
+        raw_dependencies = raw_item.get("dependencies", [])
+        if raw_dependencies is None:
+            raw_dependencies = []
+        if not isinstance(raw_dependencies, list) or any(
+            not isinstance(value, str) for value in raw_dependencies
+        ):
+            errors.append(f"{item_id}: dependencies must be an array of task IDs")
+            continue
+        dependencies = tuple(
+            _normalize_item_id(value, family.id_prefix) for value in raw_dependencies
+        )
+        if any(not value for value in dependencies):
+            errors.append(f"{item_id}: dependencies contain an invalid task ID")
+            continue
+        title = str(raw_item.get("title") or "")
+        items.append(
+            PreservePlanItem(
+                item_id=item_id,
+                title=title,
+                slug=_item_slug(family, item_id, title),
+                status=str(raw_item.get("status") or ""),
+                dependencies=dependencies,
+                index=index,
+            )
+        )
+
+    known_ids = {item.item_id for item in items}
+    for item in items:
+        for dependency in item.dependencies:
+            if dependency not in known_ids:
+                errors.append(f"{item.item_id}: unknown dependency {dependency}")
+            elif dependency == item.item_id:
+                errors.append(f"{item.item_id}: cannot depend on itself")
+    cycle = _dependency_cycle(items)
+    if cycle:
+        errors.append("dependency cycle: " + " -> ".join(cycle))
+    if errors:
+        raise ValueError(f"Invalid {family.kind} plan: " + "; ".join(errors))
+
+    failed_roots = tuple(item for item in items if item.status == "failed")
+    restored_ids = {item.item_id for item in failed_roots}
+    descendants: list[PreservePlanItem] = []
+    changed = True
+    while changed:
+        changed = False
+        for item in items:
+            if item.item_id in restored_ids or item.status != "auto_skipped":
+                continue
+            if any(dependency in restored_ids for dependency in item.dependencies):
+                restored_ids.add(item.item_id)
+                descendants.append(item)
+                changed = True
+
+    return PreservePlanSnapshot(
+        data=data,
+        items=tuple(items),
+        failed_roots=failed_roots,
+        auto_skipped_descendants=tuple(descendants),
+    )
+
+
+def _dependency_cycle(items: list[PreservePlanItem]) -> tuple[str, ...]:
+    dependencies = {item.item_id: item.dependencies for item in items}
+    visiting: set[str] = set()
+    visited: set[str] = set()
+    stack: list[str] = []
+
+    def visit(item_id: str) -> tuple[str, ...]:
+        visiting.add(item_id)
+        stack.append(item_id)
+        for dependency in dependencies.get(item_id, ()):
+            if dependency not in dependencies:
+                continue
+            if dependency in visiting:
+                start = stack.index(dependency)
+                return tuple((*stack[start:], dependency))
+            if dependency not in visited:
+                cycle = visit(dependency)
+                if cycle:
+                    return cycle
+        stack.pop()
+        visiting.remove(item_id)
+        visited.add(item_id)
+        return ()
+
+    for item in items:
+        if item.item_id not in visited:
+            cycle = visit(item.item_id)
+            if cycle:
+                return cycle
+    return ()
+
+
+def _preflight_root(
+    family: RunnerFamily,
+    item: PreservePlanItem,
+    project_root: Path,
+    registrations,
+) -> tuple[RootRecovery | None, list[str]]:
+    blockers: list[str] = []
+    status_path = family.state_dir / item.item_id / "status.json"
+    try:
+        status_data = json.loads(status_path.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        status_data = None
+        blockers.append(f"{item.item_id}: runtime status file is missing: {status_path}")
+    except (OSError, json.JSONDecodeError) as exc:
+        status_data = None
+        blockers.append(f"{item.item_id}: runtime status file is invalid: {exc}")
+    if status_data is not None and not isinstance(status_data, dict):
+        blockers.append(f"{item.item_id}: runtime status root must be an object")
+        status_data = None
+
+    try:
+        checkout = load_task_checkout(family.state_dir, family.task_type, item.item_id)
+    except TaskCheckoutError as exc:
+        checkout = None
+        blockers.append(f"{item.item_id}: {exc}")
+    if checkout is None:
+        blockers.append(f"{item.item_id}: active checkout identity is missing")
+    elif not checkout.is_active:
+        blockers.append(
+            f"{item.item_id}: checkout state is {checkout.state}, expected active"
+        )
+    if checkout is None or not checkout.is_active or status_data is None:
+        return None, blockers
+
+    if not local_branch_exists(project_root, checkout.base_branch):
+        blockers.append(
+            f"{item.item_id}: recorded base branch is missing: {checkout.base_branch}"
+        )
+
+    branch_exists = local_branch_exists(project_root, checkout.active_dev_branch)
+    active_registration = [
+        registration
+        for registration in registrations
+        if registration.branch == checkout.active_dev_branch
+    ]
+    execution_root = project_root
+    old_runtime: WorktreeRuntimeContext | None = None
+    if checkout.checkout_mode == CHECKOUT_MODE_WORKTREE:
+        recorded_worktree = Path(checkout.worktree_path).resolve()
+        old_runtime = worktree_runtime_context(
+            BranchContext(
+                checkout.base_branch,
+                checkout.active_dev_branch,
+                project_root,
+            ),
+            WorktreePolicy(True, True, True, recorded_worktree.parent),
+        )
+        if not old_runtime.is_cleanup_path_safe() or old_runtime.worktree_path != recorded_worktree:
+            blockers.append(
+                f"{item.item_id}: recorded worktree path is unsafe or inconsistent: {recorded_worktree}"
+            )
+        execution_root = recorded_worktree
+        if branch_exists:
+            if not recorded_worktree.is_dir():
+                blockers.append(
+                    f"{item.item_id}: recorded worktree directory is missing: {recorded_worktree}"
+                )
+            if not any(
+                registration.path == recorded_worktree
+                and registration.branch == checkout.active_dev_branch
+                for registration in registrations
+            ):
+                blockers.append(
+                    f"{item.item_id}: recorded branch/worktree registration is inconsistent"
+                )
+        elif any(registration.path == recorded_worktree for registration in registrations):
+            blockers.append(
+                f"{item.item_id}: missing branch still has a conflicting registered worktree"
+            )
+    elif checkout.checkout_mode == CHECKOUT_MODE_BRANCH:
+        if branch_exists and any(
+            registration.path != project_root.resolve()
+            for registration in active_registration
+        ):
+            blockers.append(
+                f"{item.item_id}: branch checkout is owned by a different linked worktree"
+            )
+    else:
+        blockers.append(
+            f"{item.item_id}: unsupported checkout mode: {checkout.checkout_mode}"
+        )
+
+    replacement_checkout: TaskCheckout | None = None
+    replacement_worktree: WorktreeRuntimeContext | None = None
+    if not branch_exists and not blockers:
+        try:
+            replacement_branch, replacement_worktree = _replacement_identity(
+                family,
+                item,
+                checkout,
+                project_root,
+                registrations,
+                old_runtime,
+            )
+            replacement_checkout = TaskCheckout.active(
+                family.task_type,
+                item.item_id,
+                replacement_branch,
+                checkout.base_branch,
+                checkout_mode=checkout.checkout_mode,
+                worktree_path=(
+                    str(replacement_worktree.worktree_path)
+                    if replacement_worktree is not None
+                    else ""
+                ),
+            )
+        except (TaskCheckoutError, ValueError) as exc:
+            blockers.append(str(exc))
+
+    checkpoint_pairs = _checkpoint_candidates(
+        family,
+        item,
+        project_root,
+        execution_root,
+    )
+    for checkpoint_path, checkpoint_root in checkpoint_pairs:
+        state = load_checkpoint_state(
+            checkpoint_path,
+            project_root=checkpoint_root,
+        )
+        if not state.valid:
+            blockers.append(
+                f"{item.item_id}: invalid checkpoint {checkpoint_path}: "
+                f"{state.error_code}: {state.error_message}"
+            )
+        elif not state.semantic.sequence_valid:
+            blockers.append(
+                f"{item.item_id}: invalid checkpoint sequence {checkpoint_path}: "
+                f"{state.semantic.error_message or 'mandatory stage sequence is invalid'}"
+            )
+
+    source_artifact: Path | None = None
+    target_artifact = _artifact_path(family, item, execution_root)
+    checkpoint_target: Path | None = None
+    checkpoint_data: dict[str, Any] | None = None
+    if replacement_checkout is not None and not blockers:
+        if checkout.checkout_mode == CHECKOUT_MODE_WORKTREE:
+            source_artifact = _existing_artifact_source(
+                family,
+                item,
+                execution_root,
+                project_root,
+            )
+            if replacement_worktree is None:
+                blockers.append(f"{item.item_id}: replacement worktree was not resolved")
+            else:
+                target_artifact = _artifact_path(
+                    family,
+                    item,
+                    replacement_worktree.worktree_path,
+                )
+                if target_artifact.exists():
+                    blockers.append(
+                        f"{item.item_id}: replacement artifact destination already exists: {target_artifact}"
+                    )
+        else:
+            source_artifact = _existing_artifact_source(
+                family,
+                item,
+                project_root,
+                project_root,
+            )
+            target_artifact = _artifact_path(family, item, project_root)
+
+        source_checkpoint = (
+            source_artifact / "workflow-checkpoint.json"
+            if source_artifact is not None
+            else None
+        )
+        if source_checkpoint is not None and source_checkpoint.is_file() and not blockers:
+            try:
+                source_data = json.loads(source_checkpoint.read_text(encoding="utf-8"))
+                checkpoint_target = target_artifact / "workflow-checkpoint.json"
+                checkpoint_data = rewind_checkpoint_for_implementation(
+                    source_data,
+                    path=checkpoint_target,
+                    project_root=(
+                        replacement_worktree.worktree_path
+                        if replacement_worktree is not None
+                        else project_root
+                    ),
+                )
+            except (OSError, json.JSONDecodeError, ValueError) as exc:
+                blockers.append(
+                    f"{item.item_id}: checkpoint cannot be safely rewound: {exc}"
+                )
+
+    if blockers:
+        return None, blockers
+    return (
+        RootRecovery(
+            item=item,
+            checkout=checkout,
+            status_path=status_path,
+            status_data=status_data,
+            replacement_checkout=replacement_checkout,
+            replacement_worktree=replacement_worktree,
+            source_artifact=source_artifact,
+            target_artifact=target_artifact,
+            checkpoint_target=checkpoint_target,
+            checkpoint_data=checkpoint_data,
+        ),
+        [],
+    )
+
+
+def _replacement_identity(
+    family: RunnerFamily,
+    item: PreservePlanItem,
+    checkout: TaskCheckout,
+    project_root: Path,
+    registrations,
+    old_runtime: WorktreeRuntimeContext | None,
+) -> tuple[str, WorktreeRuntimeContext | None]:
+    stamp = time.strftime("%Y%m%d%H%M%S", time.gmtime())
+    worktree_root = old_runtime.worktree_root if old_runtime is not None else None
+    registered_branches = {registration.branch for registration in registrations}
+    registered_paths = {registration.path for registration in registrations}
+    for suffix in range(1000):
+        ending = "" if suffix == 0 else f"-{suffix}"
+        branch = f"{family.branch_prefix}/{item.item_id}-resume-{stamp}{ending}"
+        if local_branch_exists(project_root, branch) or branch in registered_branches:
+            continue
+        runtime = None
+        if checkout.checkout_mode == CHECKOUT_MODE_WORKTREE:
+            if worktree_root is None:
+                continue
+            runtime = worktree_runtime_context(
+                BranchContext(checkout.base_branch, branch, project_root),
+                WorktreePolicy(True, True, True, worktree_root),
+            )
+            if (
+                not runtime.is_cleanup_path_safe()
+                or runtime.worktree_path.exists()
+                or runtime.worktree_path.resolve() in registered_paths
+            ):
+                continue
+        return branch, runtime
+    raise ValueError(f"{item.item_id}: cannot allocate a collision-safe replacement checkout")
+
+
+def _checkpoint_candidates(
+    family: RunnerFamily,
+    item: PreservePlanItem,
+    project_root: Path,
+    execution_root: Path,
+) -> tuple[tuple[Path, Path], ...]:
+    candidates: list[tuple[Path, Path]] = []
+    seen: set[Path] = set()
+    for root in (execution_root, project_root):
+        checkpoint = _artifact_path(family, item, root) / "workflow-checkpoint.json"
+        resolved = checkpoint.resolve()
+        if resolved in seen or not checkpoint.is_file():
+            continue
+        seen.add(resolved)
+        candidates.append((checkpoint, root))
+    return tuple(candidates)
+
+
+def _existing_artifact_source(
+    family: RunnerFamily,
+    item: PreservePlanItem,
+    primary_root: Path,
+    project_root: Path,
+) -> Path | None:
+    for root in (primary_root, project_root):
+        candidate = _artifact_path(family, item, root)
+        if candidate.is_dir():
+            return candidate
+    return None
+
+
+def _artifact_path(
+    family: RunnerFamily,
+    item: PreservePlanItem,
+    execution_root: Path,
+) -> Path:
+    artifact_name = item.slug if family.kind == "feature" else item.item_id
+    return execution_root / ".prizmkit" / family.artifact_root_name / artifact_name
+
+
+def _writability_blockers(
+    list_path: Path,
+    recoveries: list[RootRecovery],
+) -> list[str]:
+    blockers: list[str] = []
+    for path, label in ((list_path, "selected plan"),):
+        if not _path_writable(path):
+            blockers.append(f"{label} is not writable: {path}")
+    for recovery in recoveries:
+        checks = [
+            (recovery.status_path, f"{recovery.item.item_id} runtime status"),
+            (
+                recovery.status_path.parent / "checkout.json",
+                f"{recovery.item.item_id} checkout identity",
+            ),
+        ]
+        if recovery.checkpoint_target is not None and recovery.replacement_worktree is None:
+            checks.append(
+                (
+                    recovery.checkpoint_target,
+                    f"{recovery.item.item_id} workflow checkpoint",
+                )
+            )
+        if recovery.replacement_worktree is not None:
+            checks.append(
+                (
+                    recovery.replacement_worktree.worktree_root,
+                    f"{recovery.item.item_id} replacement worktree root",
+                )
+            )
+        for path, label in checks:
+            if not _path_writable(path):
+                blockers.append(f"{label} is not writable: {path}")
+    return blockers
+
+
+def _path_writable(path: Path) -> bool:
+    if path.exists():
+        return os.access(path, os.W_OK) and os.access(path.parent, os.W_OK)
+    parent = path.parent
+    while not parent.exists() and parent != parent.parent:
+        parent = parent.parent
+    return parent.exists() and os.access(parent, os.W_OK)
+
+
+def _apply_recovery(
+    family: RunnerFamily,
+    snapshot: PreservePlanSnapshot,
+    recoveries: list[RootRecovery],
+    project_root: Path,
+    list_path: Path,
+) -> None:
+    for recovery in recoveries:
+        replacement = recovery.replacement_checkout
+        if replacement is None:
+            continue
+        branch_result = create_branch_from_base(
+            project_root,
+            replacement.active_dev_branch,
+            replacement.base_branch,
+        )
+        if branch_result.return_code != 0:
+            detail = branch_result.stderr.strip() or branch_result.stdout.strip()
+            raise RuntimeError(
+                f"{recovery.item.item_id}: replacement branch creation failed: {detail or 'unknown Git error'}"
+            )
+        if recovery.replacement_worktree is not None:
+            setup = ensure_linked_worktree(recovery.replacement_worktree)
+            if not setup.ok:
+                detail = "; ".join(
+                    result.stderr.strip() or result.stdout.strip()
+                    for result in setup.results
+                    if result.return_code != 0
+                )
+                raise RuntimeError(
+                    f"{recovery.item.item_id}: replacement worktree creation failed: "
+                    f"{detail or setup.reason}"
+                )
+            if recovery.source_artifact is not None and recovery.target_artifact is not None:
+                recovery.target_artifact.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copytree(
+                    recovery.source_artifact,
+                    recovery.target_artifact,
+                    symlinks=True,
+                )
+
+    for recovery in recoveries:
+        if recovery.checkpoint_target is not None and recovery.checkpoint_data is not None:
+            _atomic_write_json(recovery.checkpoint_target, recovery.checkpoint_data)
+
+    for recovery in recoveries:
+        if recovery.replacement_checkout is not None:
+            replace_active_task_checkout_locked(
+                family.state_dir,
+                recovery.checkout,
+                recovery.replacement_checkout,
+            )
+
+    for recovery in recoveries:
+        status_data = copy.deepcopy(recovery.status_data)
+        status_data["retry_count"] = 0
+        status_data["infra_error_count"] = 0
+        status_data["continuation_pending"] = True
+        if recovery.replacement_checkout is not None:
+            status_data["active_dev_branch"] = (
+                recovery.replacement_checkout.active_dev_branch
+            )
+            status_data["base_branch"] = recovery.replacement_checkout.base_branch
+        _atomic_write_json(recovery.status_path, status_data)
+
+    plan_data = copy.deepcopy(snapshot.data)
+    raw_items = plan_data.get(family.list_key)
+    if not isinstance(raw_items, list):
+        raise ValueError(f"Invalid {family.kind} plan during apply")
+    restored_ids = {
+        item.item_id
+        for item in (*snapshot.failed_roots, *snapshot.auto_skipped_descendants)
+    }
+    for raw_item in raw_items:
+        if not isinstance(raw_item, dict):
+            continue
+        item_id = _normalize_item_id(str(raw_item.get("id") or ""), family.id_prefix)
+        if item_id in restored_ids:
+            raw_item["status"] = "pending"
+    _atomic_write_json(list_path, plan_data)
+
+
+def _atomic_write_json(path: Path, data: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    descriptor = -1
+    temporary = ""
+    try:
+        descriptor, temporary = tempfile.mkstemp(
+            prefix=f".{path.name}.",
+            suffix=".tmp",
+            dir=path.parent,
+        )
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            descriptor = -1
+            json.dump(data, handle, indent=2, ensure_ascii=False)
+            handle.write("\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary, path)
+    except OSError:
+        if descriptor >= 0:
+            os.close(descriptor)
+        if temporary:
+            try:
+                os.unlink(temporary)
+            except OSError:
+                pass
+        raise
+
+
+def _normalize_item_id(value: str, prefix: str) -> str:
+    upper = value.strip().upper()
+    if not upper.startswith(prefix.upper()):
+        return ""
+    suffix = upper[len(prefix) :]
+    if not suffix.isdigit():
+        return ""
+    return f"{prefix}{int(suffix):03d}"
+
+
+def _item_slug(family: RunnerFamily, item_id: str, title: str) -> str:
+    if family.kind != "feature":
+        return item_id
+    import re
+
+    numeric = item_id.removeprefix("F-").zfill(3)
+    slug = re.sub(r"[^a-z0-9\s-]", "", title.lower())
+    slug = re.sub(r"\s+", "-", slug.strip())
+    slug = re.sub(r"-+", "-", slug).strip("-") or "feature"
+    return f"{numeric}-{slug}"