devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/recovery/dag.py

@@ -0,0 +1,2609 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import sqlite3
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib import error as urllib_error
+from urllib import request as urllib_request
+
+from pydantic import BaseModel
+
+from ..agentic_prompts import load_agentic_prompt_lines
+from ..agentic_runtime import run_agent_step
+from ..devflow_state import publish_devflow_state
+from ..implementation.dag import LocalSetupContract, _get_docker_service_logs
+from ..llm.cli_stream import llm_sessions_db
+from ..implementation.test_runtime import (
+    discover_story_scoped_test_paths,
+    load_story_test_runtime_contract,
+    normalize_recovery_story_runtime_contract,
+    persist_story_runtime_contract,
+    resolve_story_runtime_contract,
+    story_test_runtime_contract_path,
+)
+from ..stores.execution_store import ExecutionStore
+from ..vendor.datalumina_genai.core.nodes.agent import AgentConfig, AgentNode
+from ..vendor.datalumina_genai.core.nodes.base import Node
+from ..vendor.datalumina_genai.core.nodes.router import BaseRouter, RouterNode
+from ..vendor.datalumina_genai.core.schema import NodeConfig, WorkflowSchema
+from ..vendor.datalumina_genai.core.task import TaskContext
+from ..vendor.datalumina_genai.core.workflow import Workflow
+from .models import (
+    FailedQueueItemArtifact,
+    RecoveryInvestigationArtifact,
+    RecoveryNonConvergenceArtifact,
+    RecoveryDiagnosisArtifact,
+    RecoverySuccessCriterion,
+    RecoveryExecutionArtifact,
+    PreReplayCheckArtifact,
+    RecoveryOutcomeArtifact,
+    RemediationPlanArtifact,
+    ReenqueueArtifact,
+    RecoveryHandoffArtifact,
+    SystemicPatternArtifact,
+    CodeRootCauseArtifact,
+    RemediationResultArtifact,
+)
+
+DAG_ID = "post_queue_failure_recovery_dag"
+_CURRENT_STORE: ExecutionStore | None = None
+_CURRENT_RUN_ID: str | None = None
+_CURRENT_STRENGTH: str | None = None
+_CURRENT_REPO_ROOT: Path | None = None
+
+_QUEUE_TABLE_BY_KIND = {
+    "scope": ("scope_queue", "scope_queue_id"),
+    "idea_creation": ("idea_creation_queue", "idea_creation_queue_id"),
+    "idea": ("idea_queue", "idea_queue_id"),
+    "story": ("story_queue", "story_queue_id"),
+    "integration": ("integration_queue", "integration_queue_id"),
+    "recovery": ("recovery_queue", "recovery_queue_id"),
+}
+
+
+class FailureRecoveryDagEvent(BaseModel):
+    repo_root: str
+    project_id: str
+    queue_type: str
+    item_id: str
+
+
+@dataclass(frozen=True)
+class FailureRecoveryDagResult:
+    exit_code: int
+    run_id: str
+    outcome: dict[str, Any]
+    message: str
+
+
+def _store_run() -> tuple[ExecutionStore, str]:
+    if _CURRENT_STORE is None or _CURRENT_RUN_ID is None:
+        raise RuntimeError("recovery workflow missing runtime bindings")
+    return _CURRENT_STORE, _CURRENT_RUN_ID
+
+
+def _repo_root() -> Path:
+    if _CURRENT_REPO_ROOT is None:
+        raise RuntimeError("recovery workflow missing repo root binding")
+    return _CURRENT_REPO_ROOT
+
+
+def _persist_node(*, node_id: str, node_name: str, fn):
+    store, run_id = _store_run()
+    node_exec_id = store.create_node_attempt(run_id=run_id, node_id=node_id, node_name=node_name, attempt=1)
+    try:
+        output, task_context = fn(node_exec_id)
+    except Exception as exc:
+        store.mark_node_finished(node_exec_id=node_exec_id, status="failed", error={"message": str(exc)})
+        raise
+    store.mark_node_finished(node_exec_id=node_exec_id, status="succeeded", output=output)
+    return task_context
+
+
+def _recovery_display_path(recovery_id: str) -> str:
+    return f"recovery:recovery_{recovery_id}"
+
+
+def _publish(project_id: str, run_id: str, state: str, status: str, summary: str, error: str | None = None, recovery_id: str | None = None) -> None:
+    publish_devflow_state(
+        project_id=project_id,
+        run_id=run_id,
+        current_state=state,
+        current_status=status,
+        run_summary=summary,
+        error_message=error,
+        display="project",
+        display_path=_recovery_display_path(recovery_id or run_id),
+    )
+
+
+def _publish_node(project_id: str, run_id: str, summary: str, recovery_id: str | None = None) -> None:
+    publish_devflow_state(
+        project_id=project_id,
+        run_id=run_id,
+        current_state="running",
+        current_status="processing",
+        run_summary=summary,
+        display="project",
+        display_path=_recovery_display_path(recovery_id or run_id),
+    )
+
+
+def _normalize_text(value: Any) -> str:
+    return str(value or "").strip().lower()
+
+
+_SOFT_PROVENANCE_REASON_MARKERS = (
+    "provenance",
+    "byte identity",
+    "byte-ident",
+    "byte-for-byte",
+    "forensic",
+    "checksum",
+    "digest",
+    "hash certainty",
+    "identity certainty",
+    "cannot prove byte identity",
+    "cannot prove identity",
+    "cannot prove provenance",
+    "cannot verify provenance",
+    "audit trail",
+    "metadata certainty",
+)
+
+_HARD_BLOCK_REASON_MARKERS = (
+    "manual review",
+    "human",
+    "operator",
+    "approval",
+    "contradict",
+    "conflict",
+    "inconsistent",
+    "mismatch",
+    "failed",
+    "failure",
+    "not viable",
+    "unsafe",
+    "missing artifact",
+    "missing runtime",
+    "unauthorized",
+    "forbidden",
+    "denied",
+    "exhausted",
+)
+
+
+def _is_soft_provenance_reason(reason: str) -> bool:
+    normalized = _normalize_text(reason)
+    return bool(normalized) and any(marker in normalized for marker in _SOFT_PROVENANCE_REASON_MARKERS) and not any(
+        marker in normalized for marker in _HARD_BLOCK_REASON_MARKERS
+    )
+
+
+def _verification_allows_reenqueue(
+    *,
+    execution: RecoveryExecutionArtifact,
+    verified: PreReplayCheckArtifact,
+    diagnosis: RecoveryDiagnosisArtifact | None,
+) -> bool:
+    if execution.outcome != "reenqueued":
+        return False
+    if verified.ready:
+        return True
+    if diagnosis is not None and diagnosis.suggested_action == "manual_review_required":
+        return False
+    if not verified.checks:
+        return False
+    blocking_reasons = [str(reason).strip() for reason in verified.blocking_reasons if str(reason).strip()]
+    if not blocking_reasons:
+        return False
+    return all(_is_soft_provenance_reason(reason) for reason in blocking_reasons)
+
+
+def _normalized_failure_signature(*, failure_message: str | None, failure_context: dict[str, Any] | None) -> str:
+    ctx = failure_context if isinstance(failure_context, dict) else {}
+    message = str(failure_message or "").strip()
+    error_type = str(ctx.get("error_type") or "").strip()
+    failed_stage = str(ctx.get("failed_stage") or "").strip().lower()
+    actual_failed_node = str(ctx.get("actual_failed_node") or "").strip().lower() or failed_stage
+    if "Prompt is too long" in message:
+        base = "prompt_too_long"
+    elif "NameError" in message:
+        base = f"name_error:{error_type or 'unknown'}"
+    elif error_type:
+        base = error_type.lower().replace(" ", "_")[:80]
+    else:
+        base = message[:80].lower().replace(" ", "_")
+    return f"{actual_failed_node}:{base}" if actual_failed_node else base
+
+
+_RECOVERY_CHURN_GATE_THRESHOLD = 3
+_RECOVERY_CHURN_GATE_VERSION = 1
+
+
+def _durable_recovery_identity(item: FailedQueueItemArtifact) -> str:
+    if item.queue_type == "story" and str(item.story_id or "").strip():
+        return f"story:{str(item.story_id or '').strip()}"
+    raw = item.raw_row if isinstance(item.raw_row, dict) else {}
+    for key in ("scope_id", "idea_id", "integration_id"):
+        value = str(raw.get(key) or "").strip()
+        if value:
+            return f"{item.queue_type}:{value}"
+    return f"{item.queue_type}:{item.item_id}"
+
+
+def _recovery_churn_key(*, item: FailedQueueItemArtifact, failure_signature: str) -> str:
+    return (
+        f"recovery_churn_gate:v{_RECOVERY_CHURN_GATE_VERSION}:"
+        f"{_durable_recovery_identity(item)}:{failure_signature}:no_material_change"
+    )
+
+
+def _recovery_churn_fingerprint_inputs(*, item: FailedQueueItemArtifact, failure_signature: str) -> dict[str, Any]:
+    return {
+        "surface": "recovery_churn_gate",
+        "version": _RECOVERY_CHURN_GATE_VERSION,
+        "queue_type": item.queue_type,
+        "durable_identity": _durable_recovery_identity(item),
+        "failure_signature": failure_signature,
+        "no_material_change": True,
+    }
+
+
+def _load_recovery_churn_gate_state(
+    *,
+    store: ExecutionStore,
+    project_id: str,
+    item: FailedQueueItemArtifact,
+    failure_signature: str,
+) -> dict[str, Any]:
+    fingerprint = store._fingerprint_from_inputs(  # type: ignore[attr-defined]
+        _recovery_churn_fingerprint_inputs(item=item, failure_signature=failure_signature)
+    )
+    with store._connect() as conn:
+        row = conn.execute(
+            (
+                "SELECT error_task_id, status, occurrence_count "
+                "FROM error_tasks WHERE project_id=? AND fingerprint=? "
+                "ORDER BY created_at DESC LIMIT 1"
+            ),
+            (project_id, fingerprint),
+        ).fetchone()
+    return {
+        "fingerprint": fingerprint,
+        "churn_key": _recovery_churn_key(item=item, failure_signature=failure_signature),
+        "error_task_id": None if row is None else str(row["error_task_id"] or ""),
+        "status": None if row is None else str(row["status"] or ""),
+        "occurrence_count": 0 if row is None else int(row["occurrence_count"] or 0),
+        "threshold": _RECOVERY_CHURN_GATE_THRESHOLD,
+        "threshold_met": row is not None and int(row["occurrence_count"] or 0) >= _RECOVERY_CHURN_GATE_THRESHOLD,
+    }
+
+
+def _record_recovery_churn_strike(
+    *,
+    store: ExecutionStore,
+    project_id: str,
+    run_id: str,
+    item: FailedQueueItemArtifact,
+    failure_signature: str,
+    message: str,
+) -> dict[str, Any]:
+    error_task_id = store.create_error_task_from_failure(
+        project_id=project_id,
+        run_id=run_id,
+        plane="process_error",
+        source_kind="recovery",
+        source_ref=item.item_id,
+        title=f"Recovery churn gate: {_durable_recovery_identity(item)}",
+        severity="high",
+        error_type="recovery_churn_no_material_change",
+        message=message,
+        stacktrace=None,
+        next_steps=[
+            "Review repeated recovery churn on the same durable identity.",
+            "Apply a material fix before retrying recovery again.",
+        ],
+        fingerprint_inputs=_recovery_churn_fingerprint_inputs(item=item, failure_signature=failure_signature),
+    )
+    state = _load_recovery_churn_gate_state(
+        store=store,
+        project_id=project_id,
+        item=item,
+        failure_signature=failure_signature,
+    )
+    state["error_task_id"] = error_task_id
+    return state
+
+
+def _decode_failure_context_blob(raw: Any) -> dict[str, Any]:
+    if isinstance(raw, dict):
+        return dict(raw)
+    try:
+        payload = json.loads(str(raw or "{}") or "{}")
+        return payload if isinstance(payload, dict) else {}
+    except Exception:
+        return {}
+
+
+def _story_churn_source_item_ids(*, conn, item: FailedQueueItemArtifact) -> list[str]:
+    if item.queue_type == "story" and str(item.story_id or "").strip():
+        rows = conn.execute(
+            "SELECT story_queue_id FROM story_queue WHERE story_id=?",
+            (str(item.story_id or "").strip(),),
+        ).fetchall()
+        ids = [str(row["story_queue_id"] or "") for row in rows if str(row["story_queue_id"] or "").strip()]
+        if ids:
+            return ids
+    return [item.item_id]
+
+
+def _collect_failure_evidence(
+    *,
+    item: FailedQueueItemArtifact,
+    investigation: RecoveryInvestigationArtifact | None = None,
+    extra_evidence: list[str] | None = None,
+) -> list[str]:
+    evidence: list[str] = []
+    failure_context = item.failure_context if isinstance(item.failure_context, dict) else {}
+    for raw in (
+        item.failure_message,
+        failure_context.get("error"),
+        failure_context.get("error_type"),
+        failure_context.get("failed_stage"),
+        json.dumps(failure_context, sort_keys=True) if failure_context else None,
+        None if investigation is None else investigation.summary,
+        None if investigation is None else investigation.failure_nature,
+        *(extra_evidence or []),
+    ):
+        text = str(raw or "").strip()
+        if text:
+            evidence.append(text)
+    return evidence
+
+
+def _load_json_file(path: Path) -> dict[str, Any] | None:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+    return payload if isinstance(payload, dict) else None
+
+
+def _preflight_artifact_paths(*, repo_root: Path, item: FailedQueueItemArtifact) -> list[Path]:
+    candidates: list[Path] = []
+    failure_context = item.failure_context if isinstance(item.failure_context, dict) else {}
+    for raw_path in failure_context.get("artifact_paths") or []:
+        text = str(raw_path or "").strip()
+        if not text:
+            continue
+        path = Path(text)
+        candidates.append(path if path.is_absolute() else repo_root / path)
+    story_id = str(item.story_id or "").strip()
+    if story_id:
+        candidates.append(repo_root / ".devflow" / "stories" / story_id / "preflight.json")
+    seen: set[str] = set()
+    ordered: list[Path] = []
+    for candidate in candidates:
+        key = str(candidate)
+        if key in seen:
+            continue
+        seen.add(key)
+        ordered.append(candidate)
+    return ordered
+
+
+def _load_preflight_health_failure_details(*, repo_root: Path, item: FailedQueueItemArtifact) -> dict[str, Any] | None:
+    failure_context = item.failure_context if isinstance(item.failure_context, dict) else {}
+    if str(failure_context.get("failed_stage") or "").strip().lower() != "preflight":
+        return None
+    for artifact_path in _preflight_artifact_paths(repo_root=repo_root, item=item):
+        payload = _load_json_file(artifact_path)
+        if payload is None:
+            continue
+        blocking_issues = payload.get("blocking_issues") if isinstance(payload.get("blocking_issues"), list) else []
+        health_issues = [issue for issue in blocking_issues if isinstance(issue, dict) and str(issue.get("kind") or "").strip() == "health_check_failed"]
+        if not health_issues:
+            continue
+        return {
+            "artifact_path": artifact_path,
+            "report": payload,
+            "health_issues": health_issues,
+        }
+    return None
+
+
+def _get_recovery_runtime(failure_context: dict[str, Any] | None) -> dict[str, Any]:
+    if not isinstance(failure_context, dict):
+        return {"strategy_history": [], "last_success": None}
+    payload = failure_context.get("recovery_runtime")
+    if not isinstance(payload, dict):
+        return {"strategy_history": [], "last_success": None}
+    return {
+        "strategy_history": list(payload.get("strategy_history") or []),
+        "last_success": payload.get("last_success"),
+    }
+
+
+def _write_recovery_runtime(failure_context: dict[str, Any] | None, runtime: dict[str, Any]) -> dict[str, Any]:
+    payload = dict(failure_context or {})
+    payload["recovery_runtime"] = {
+        "strategy_history": list(runtime.get("strategy_history") or []),
+        "last_success": runtime.get("last_success"),
+    }
+    return payload
+
+
+def _persist_queue_failure_context(*, store: ExecutionStore, item: FailedQueueItemArtifact, failure_context: dict[str, Any]) -> None:
+    mapping = _QUEUE_TABLE_BY_KIND.get(item.queue_type)
+    if mapping is None:
+        return
+    table, id_col = mapping
+    now = int(__import__("time").time())
+    with store._connect() as conn:
+        conn.execute(
+            f"UPDATE {table} SET failure_context_json=?, updated_at=? WHERE {id_col}=?",
+            (json.dumps(failure_context, sort_keys=True), now, item.item_id),
+        )
+
+
+def _dedupe_strings(values: list[Any]) -> list[str]:
+    seen: set[str] = set()
+    ordered: list[str] = []
+    for raw in values:
+        value = str(raw or "").strip()
+        if not value or value in seen:
+            continue
+        seen.add(value)
+        ordered.append(value)
+    return ordered
+
+
+def _normalize_success_criteria(
+    *,
+    verification_targets: list[RecoverySuccessCriterion] | list[dict[str, Any]] | None,
+    fallback_targets: list[RecoverySuccessCriterion] | list[dict[str, Any]] | None,
+) -> list[RecoverySuccessCriterion]:
+    chosen = verification_targets or fallback_targets or [
+        {
+            "criterion": "Recovery action completed",
+            "oracle": "The queue item state matches the selected recovery strategy family.",
+        }
+    ]
+    normalized: list[RecoverySuccessCriterion] = []
+    for target in chosen:
+        criterion = RecoverySuccessCriterion.model_validate(target)
+        if not str(criterion.oracle or "").strip():
+            raise ValueError("Recovery success criteria must include a non-empty oracle.")
+        normalized.append(criterion)
+    return normalized
+
+
+def _recoverable_story_state_root(*, repo_root: Path) -> Path:
+    return repo_root / ".devflow" / "stories"
+
+
+def _recovery_handoff_artifact_path(*, repo_root: Path, item: FailedQueueItemArtifact) -> Path:
+    story_id = str(item.story_id or "").strip()
+    if story_id:
+        safe_story_id = re.sub(r"[^A-Za-z0-9_.-]+", "_", story_id) or "unknown_story"
+        return _recoverable_story_state_root(repo_root=repo_root) / safe_story_id / "recovery_handoff.json"
+    safe_item_id = re.sub(r"[^A-Za-z0-9_.-]+", "_", item.item_id) or "unknown_item"
+    return repo_root / ".devflow" / "recovery_handoffs" / item.queue_type / f"{safe_item_id}.json"
+
+
+def _extract_log_refs(payload: Any) -> tuple[list[str], list[str]]:
+    log_paths: list[str] = []
+    session_ids: list[str] = []
+
+    def _walk(value: Any, *, key: str | None = None) -> None:
+        if isinstance(value, dict):
+            for child_key, child_value in value.items():
+                _walk(child_value, key=str(child_key))
+            return
+        if isinstance(value, list):
+            for item in value:
+                _walk(item, key=key)
+            return
+        if not isinstance(value, str):
+            return
+        lowered = str(key or "").lower()
+        if lowered in {"log_path", "logfile", "log_file", "journal_path"} and value.strip():
+            log_paths.append(value.strip())
+        elif lowered == "session_id" and value.strip():
+            session_ids.append(value.strip())
+
+    _walk(payload)
+    return _dedupe_strings(log_paths), _dedupe_strings(session_ids)
+
+
+def _read_log_excerpt(*, path: Path, max_lines: int = 60, max_chars: int = 6000) -> str:
+    try:
+        if path.suffix == ".jsonl":
+            lines = path.read_text(encoding="utf-8").splitlines()[-max_lines:]
+            rendered: list[str] = []
+            for raw in lines:
+                try:
+                    record = json.loads(raw)
+                except Exception:
+                    rendered.append(raw)
+                    continue
+                stream = str(record.get("stream") or "log").strip()
+                line = str(record.get("line") or "").rstrip()
+                if line:
+                    rendered.append(f"[{stream}] {line}")
+            return "\\n".join(rendered)[-max_chars:]
+        return path.read_text(encoding="utf-8")[-max_chars:]
+    except Exception:
+        return ""
+
+
+def _load_llm_session_log_refs(*, run_id: str | None, node_exec_id: str | None) -> tuple[list[str], list[str]]:
+    db_path = llm_sessions_db()
+    if not db_path.exists() or (not run_id and not node_exec_id):
+        return [], []
+    query = "SELECT session_id, log_path FROM dev_journal_entries WHERE "
+    params: list[str] = []
+    clauses: list[str] = []
+    if node_exec_id:
+        clauses.append("node_exec_id=?")
+        params.append(node_exec_id)
+    if run_id:
+        clauses.append("run_id=?")
+        params.append(run_id)
+    query += " OR ".join(clauses) + " ORDER BY started_at DESC LIMIT 6"
+    try:
+        conn = sqlite3.connect(str(db_path))
+        conn.row_factory = sqlite3.Row
+        try:
+            rows = conn.execute(query, tuple(params)).fetchall()
+        finally:
+            conn.close()
+    except Exception:
+        return [], []
+    log_paths = [str(row["log_path"] or "").strip() for row in rows if str(row["log_path"] or "").strip()]
+    session_ids = [str(row["session_id"] or "").strip() for row in rows if str(row["session_id"] or "").strip()]
+    return _dedupe_strings(log_paths), _dedupe_strings(session_ids)
+
+
+def _gather_log_first_recovery_evidence(*, store: ExecutionStore, repo_root: Path, item: FailedQueueItemArtifact) -> dict[str, Any]:
+    failure_context = item.failure_context if isinstance(item.failure_context, dict) else {}
+    run_id = str(failure_context.get("implementation_run_id") or "").strip()
+    failed_stage = str(failure_context.get("actual_failed_node") or failure_context.get("failed_stage") or "").strip()
+    if not run_id or not failed_stage:
+        return {"available": False, "source": None, "refs": [], "session_ids": [], "excerpt": "", "stage": failed_stage or None}
+    node = store.get_latest_node_attempt(run_id=run_id, node_id=failed_stage)
+    node_output = node.get("output") if isinstance(node, dict) and isinstance(node.get("output"), dict) else {}
+    log_paths, session_ids = _extract_log_refs(node_output)
+    journal_log_paths, journal_session_ids = _load_llm_session_log_refs(run_id=run_id, node_exec_id=None if not isinstance(node, dict) else str(node.get("node_exec_id") or "") or None)
+    log_paths = _dedupe_strings([*log_paths, *journal_log_paths])
+    session_ids = _dedupe_strings([*session_ids, *journal_session_ids])
+    excerpt = ""
+    used_refs: list[str] = []
+    for raw_path in log_paths:
+        path = Path(raw_path).expanduser()
+        if not path.is_absolute():
+            path = repo_root / raw_path
+        if not path.exists() or not path.is_file():
+            continue
+        excerpt = _read_log_excerpt(path=path)
+        if excerpt:
+            used_refs.append(str(path))
+            break
+    refs = _dedupe_strings([*used_refs, *session_ids])
+    source = "streamed_agent_logs" if used_refs else ("llm_session_journal" if session_ids else None)
+    return {
+        "available": bool(source),
+        "source": source,
+        "refs": refs,
+        "session_ids": session_ids,
+        "excerpt": excerpt,
+        "stage": failed_stage,
+        "implementation_run_id": run_id,
+    }
+
+
+def _enrich_investigation_with_log_evidence(*, investigation: RecoveryInvestigationArtifact, log_evidence: dict[str, Any]) -> RecoveryInvestigationArtifact:
+    if not log_evidence.get("available"):
+        return investigation
+    evidence = list(investigation.evidence or [])
+    source = str(log_evidence.get("source") or "streamed_agent_logs")
+    source_ref = ", ".join(str(ref) for ref in (log_evidence.get("refs") or []) if str(ref).strip())
+    source_line = f"primary_evidence_source={source}" + (f" ({source_ref})" if source_ref else "")
+    if source_line not in evidence:
+        evidence.insert(0, source_line)
+    return investigation.model_copy(update={
+        "evidence": evidence[:8],
+        "primary_evidence_source": investigation.primary_evidence_source or source,
+        "primary_evidence_refs": list(investigation.primary_evidence_refs or []) or [str(ref) for ref in (log_evidence.get("refs") or []) if str(ref).strip()],
+        "primary_log_insight": investigation.primary_log_insight or investigation.summary,
+    })
+
+
+def _persist_recovery_handoff_artifact(
+    *,
+    repo_root: Path,
+    recovery_run_id: str,
+    item: FailedQueueItemArtifact,
+    investigation: RecoveryInvestigationArtifact | None,
+    diagnosis: RecoveryDiagnosisArtifact | None,
+    execution: RecoveryExecutionArtifact | None,
+    pre_replay: PreReplayCheckArtifact | None,
+) -> Path | None:
+    useful = any(
+        [
+            investigation is not None,
+            diagnosis is not None,
+            execution is not None and bool(str(execution.verification_summary or execution.execution_summary or "").strip()),
+            pre_replay is not None and bool(pre_replay.blocking_reasons),
+        ]
+    )
+    if not useful:
+        return None
+    failure_context = item.failure_context if isinstance(item.failure_context, dict) else {}
+    disproven_dead_ends: list[str] = []
+    if investigation is not None and investigation.non_convergence is not None and investigation.non_convergence.unchanged_test_surface:
+        disproven_dead_ends.append("Repeated retries did not materially change the failing surface.")
+    if investigation is not None and investigation.non_convergence is not None and investigation.non_convergence.wrong_seam:
+        disproven_dead_ends.append("Prior recovery attempts stayed on the wrong seam.")
+    payload = RecoveryHandoffArtifact(
+        queue_type=item.queue_type,
+        item_id=item.item_id,
+        story_id=item.story_id,
+        implementation_run_id=str(failure_context.get("implementation_run_id") or "").strip() or None,
+        failed_stage=str(failure_context.get("actual_failed_node") or failure_context.get("failed_stage") or "").strip() or None,
+        primary_evidence_source=(None if investigation is None else investigation.primary_evidence_source) or (None if execution is None else execution.primary_evidence_source),
+        primary_evidence_refs=([] if investigation is None else investigation.primary_evidence_refs) or ([] if execution is None else execution.primary_evidence_refs),
+        key_log_insight=(None if investigation is None else investigation.primary_log_insight) or (None if execution is None else execution.primary_log_insight),
+        failing_surface_summary=(None if investigation is None else investigation.summary) or str(item.failure_message or execution.execution_summary if execution else item.item_id),
+        likely_seam=(None if investigation is None else investigation.affected_boundary) or str(failure_context.get("actual_failed_node") or failure_context.get("failed_stage") or diagnosis.strategy if diagnosis else "").strip() or None,
+        disproven_dead_ends=disproven_dead_ends[:4],
+        verification_blockers=[] if pre_replay is None else [str(reason) for reason in pre_replay.blocking_reasons if str(reason).strip()][:4],
+        non_convergence_insight=None if investigation is None or investigation.non_convergence is None else investigation.non_convergence.reason,
+        produced_by_recovery_run_id=recovery_run_id,
+    )
+    path = _recovery_handoff_artifact_path(repo_root=repo_root, item=item)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload.model_dump(), indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return path
+
+
+def _build_diagnosis(
+    *,
+    item: FailedQueueItemArtifact,
+    investigation: RecoveryInvestigationArtifact | None,
+    prior_execution: RecoveryExecutionArtifact | None = None,
+    prior_verification: PreReplayCheckArtifact | None = None,
+    attempt: int = 1,
+) -> tuple[RecoveryDiagnosisArtifact, RemediationPlanArtifact]:
+    diagnosis, _envelope = run_agent_step(
+        repo_root=_repo_root(),
+        stage_name="recovery_diagnosis",
+        output_model=RecoveryDiagnosisArtifact,
+        context_payload={
+            "failed_item": item.model_dump(),
+            "investigation": None if investigation is None else investigation.model_dump(),
+            "prior_execution": None if prior_execution is None else prior_execution.model_dump(),
+            "prior_verification": None if prior_verification is None else prior_verification.model_dump(),
+            "attempt": attempt,
+        },
+        guidance=load_agentic_prompt_lines("recovery_diagnosis"),
+        timeout_seconds=300,
+        strength=_CURRENT_STRENGTH,
+    )
+    targets = _normalize_success_criteria(
+        verification_targets=diagnosis.verification_targets,
+        fallback_targets=None if investigation is None else investigation.success_criteria,
+    )
+    enforced = diagnosis.model_copy(
+        update={
+            "queue_type": item.queue_type,
+            "item_id": item.item_id,
+            "summary": str(diagnosis.summary or (None if investigation is None else investigation.summary) or "Recovery diagnosis").strip(),
+            "rationale": str(diagnosis.rationale or (None if investigation is None else investigation.failure_nature) or "Recovery diagnosis").strip(),
+            "verification_targets": targets,
+            "replay_path": str(diagnosis.replay_path or (None if investigation is None else investigation.replay_path) or "") or None,
+            "evidence": [str(entry) for entry in (diagnosis.evidence or (None if investigation is None else investigation.evidence) or []) if str(entry).strip()][:8],
+        }
+    )
+    plan = RemediationPlanArtifact(
+        queue_type=item.queue_type,
+        action=enforced.suggested_action,
+        summary=enforced.summary,
+        steps=[criterion.criterion for criterion in enforced.verification_targets],
+        replay_path=enforced.replay_path,
+    )
+    return enforced, plan
+
+
+def _record_recovery_attempt(
+    *,
+    item: FailedQueueItemArtifact,
+    diagnosis: RecoveryDiagnosisArtifact | None = None,
+    success: bool = False,
+    failure_signature: str | None = None,
+    material_change: bool | None = None,
+    remediation_artifact: str | None = None,
+) -> dict[str, Any]:
+    runtime = _get_recovery_runtime(item.failure_context if isinstance(item.failure_context, dict) else {})
+    if diagnosis is not None:
+        entry: dict[str, Any] = {
+            "strategy": diagnosis.strategy,
+            "summary": diagnosis.summary,
+            "success": success,
+        }
+        if failure_signature is not None:
+            entry["failure_signature"] = failure_signature
+        if material_change is not None:
+            entry["material_change"] = material_change
+        if remediation_artifact is not None:
+            entry["remediation_artifact"] = remediation_artifact
+        runtime["strategy_history"].append(entry)
+    if success and diagnosis is not None:
+        runtime["last_success"] = {
+            "strategy": diagnosis.strategy,
+            "failure_signature": failure_signature,
+            "material_change": material_change,
+        }
+    return _write_recovery_runtime(item.failure_context if isinstance(item.failure_context, dict) else {}, runtime)
+
+
+_DOWNSTREAM_RECOVERY_STAGES = {"redreview", "security", "verifygreen", "gitcommit(refactor)"}
+
+_SCHEMA_RUNTIME_DRIFT_MARKERS = (
+    "schema drift",
+    "runtime drift",
+    "contract drift",
+    "migration mismatch",
+    "serialization",
+    "deserial",
+    "no tests collected",
+    "collected 0 items",
+    "pytest import error",
+    "vitest",
+)
+
+
+def _build_non_convergence_analysis(
+    *,
+    item: FailedQueueItemArtifact,
+    failure_signature: str,
+    occurrence_count: int,
+    remediation_artifact: str | None = None,
+) -> RecoveryNonConvergenceArtifact:
+    failure_context = item.failure_context if isinstance(item.failure_context, dict) else {}
+    runtime = _get_recovery_runtime(failure_context)
+    history = [entry for entry in (runtime.get("strategy_history") or []) if isinstance(entry, dict)]
+    matching_history = [
+        entry
+        for entry in history
+        if str(entry.get("failure_signature") or "").strip() == failure_signature or not str(entry.get("failure_signature") or "").strip()
+    ]
+    attempts_reviewed = max(int(occurrence_count or 0), len(matching_history))
+    recent_attempts = matching_history[-max(attempts_reviewed, _RECOVERY_CHURN_GATE_THRESHOLD) :]
+    recent_failure_signatures = {str(entry.get("failure_signature") or "").strip() for entry in recent_attempts if str(entry.get("failure_signature") or "").strip()}
+    same_failure_surface = bool(failure_signature) and (not recent_failure_signatures or recent_failure_signatures == {failure_signature})
+    unchanged_test_surface = same_failure_surface and all(entry.get("material_change") is False for entry in recent_attempts)
+    measurable_progress = any(bool(entry.get("success")) or entry.get("material_change") is True for entry in recent_attempts)
+    failed_stage = str(failure_context.get("failed_stage") or "").strip().lower() or None
+    downstream_blocker = bool(failed_stage and failed_stage in _DOWNSTREAM_RECOVERY_STAGES)
+    strategy_set = {str(entry.get("strategy") or "").strip() for entry in recent_attempts if str(entry.get("strategy") or "").strip()}
+    artifact_set = {
+        str(entry.get("remediation_artifact") or "").strip()
+        for entry in recent_attempts
+        if str(entry.get("remediation_artifact") or "").strip()
+    }
+    combined_text = "\n".join(
+        [
+            str(item.failure_message or ""),
+            str(failure_context.get("error") or ""),
+            str(failure_context.get("error_type") or ""),
+            str(failure_context.get("failed_stage") or ""),
+            *(str(entry.get("summary") or "") for entry in recent_attempts),
+            *(str(entry.get("remediation_artifact") or "") for entry in recent_attempts),
+        ]
+    ).lower()
+    schema_or_runtime_drift = any(marker in combined_text for marker in _SCHEMA_RUNTIME_DRIFT_MARKERS)
+    wrong_seam = bool(
+        same_failure_surface
+        and not measurable_progress
+        and not downstream_blocker
+        and not schema_or_runtime_drift
+        and (len(strategy_set) == 1 or len(artifact_set) == 1 or remediation_artifact)
+    )
+    summary_bits: list[str] = []
+    if unchanged_test_surface:
+        summary_bits.append("the same failure surface stayed unchanged across attempts")
+    if wrong_seam:
+        summary_bits.append("recovery kept editing the wrong seam")
+    if downstream_blocker and failed_stage:
+        summary_bits.append(f"the blocking node stayed downstream at {failed_stage} rather than the green boundary")
+    if schema_or_runtime_drift:
+        summary_bits.append("schema/runtime drift kept invalidating the attempted fixes")
+    if not measurable_progress:
+        summary_bits.append("there was no measurable improvement across the three attempts")
+    if not summary_bits:
+        summary_bits.append("three attempts did not materially change the failing boundary")
+    evidence: list[str] = [
+        f"failure_signature={failure_signature}",
+        f"attempts_reviewed={attempts_reviewed}",
+    ]
+    if failed_stage:
+        evidence.append(f"failed_stage={failed_stage}")
+    if strategy_set:
+        evidence.append("strategies=" + ", ".join(sorted(strategy_set)))
+    if artifact_set:
+        evidence.append("remediation_artifacts=" + ", ".join(sorted(artifact_set)))
+    if unchanged_test_surface:
+        evidence.append("all reviewed attempts recorded material_change=false")
+    if remediation_artifact and remediation_artifact not in artifact_set:
+        evidence.append(f"current_remediation_artifact={remediation_artifact}")
+    return RecoveryNonConvergenceArtifact(
+        summary="Non-convergence analysis for repeated recovery churn.",
+        reason="; ".join(summary_bits),
+        attempts_reviewed=attempts_reviewed,
+        same_failure_surface=same_failure_surface,
+        unchanged_test_surface=unchanged_test_surface,
+        wrong_seam=wrong_seam,
+        downstream_blocker=downstream_blocker,
+        downstream_blocker_stage=failed_stage if downstream_blocker else None,
+        schema_or_runtime_drift=schema_or_runtime_drift,
+        measurable_progress=measurable_progress,
+        evidence=evidence[:8],
+    )
+
+
+def _build_churn_gate_investigation(
+    *,
+    item: FailedQueueItemArtifact,
+    failure_signature: str,
+    occurrence_count: int,
+    threshold: int,
+    churn_key: str | None = None,
+    remediation_artifact: str | None = None,
+) -> RecoveryInvestigationArtifact:
+    failure_context = item.failure_context if isinstance(item.failure_context, dict) else {}
+    failed_stage = str(failure_context.get("failed_stage") or "").strip().lower() or None
+    non_convergence = _build_non_convergence_analysis(
+        item=item,
+        failure_signature=failure_signature,
+        occurrence_count=occurrence_count,
+        remediation_artifact=remediation_artifact,
+    )
+    underlying_issue = str(item.failure_message or failure_context.get("error") or failure_signature or "Repeated recovery failure").strip()
+    evidence = _collect_failure_evidence(item=item, extra_evidence=[*non_convergence.evidence, churn_key or ""])
+    return RecoveryInvestigationArtifact(
+        queue_type=item.queue_type,
+        item_id=item.item_id,
+        summary=(
+            f"Recovery churn gate hit after {occurrence_count}/{threshold} no-progress attempts on "
+            f"{failed_stage or failure_signature or item.item_id}."
+        ),
+        failure_nature=underlying_issue,
+        evidence=evidence[:8],
+        primary_evidence_source="recovery_churn_history",
+        primary_log_insight=non_convergence.reason,
+        affected_boundary=failed_stage or failure_signature or item.queue_type,
+        likely_failed_stage=failed_stage,
+        confidence="high" if occurrence_count >= threshold else "medium",
+        recovery_goal="Explain both the underlying failure and why implementation did not converge before any further replay.",
+        success_criteria=[
+            {
+                "criterion": "Underlying failure boundary identified",
+                "oracle": "The investigation names the concrete failing node/stage and the defect surface that stayed broken.",
+            },
+            {
+                "criterion": "Non-convergence reason identified",
+                "oracle": "The investigation explains why three attempts made no measurable progress and whether the repeated work stayed on the wrong seam, on an unchanged test surface, or behind downstream/runtime drift.",
+            },
+        ],
+        verification_evidence=[
+            f"Churn gate occurrence count reached {occurrence_count}/{threshold}.",
+            non_convergence.reason,
+        ],
+        replay_path="manual_review_required",
+        escalation_conditions=[
+            "Do not requeue again until the non-convergence reason has a concrete fix plan.",
+            "Require a materially different intervention when the same story/node has failed three times with no measurable improvement.",
+        ],
+        non_convergence=non_convergence,
+    )
+
+
+def _build_story_replay_metadata(*, item: FailedQueueItemArtifact, diagnosis: RecoveryDiagnosisArtifact | None, execution: RecoveryExecutionArtifact | None) -> dict[str, Any] | None:
+    if item.queue_type != "story":
+        return None
+    failure_context = item.failure_context if isinstance(item.failure_context, dict) else {}
+    prior_run_id = str(failure_context.get("implementation_run_id") or "").strip()
+    failed_stage = str(failure_context.get("failed_stage") or "").strip().lower()
+    if not prior_run_id or not failed_stage:
+        return None
+    canonical_order = [
+        "normalize",
+        "preflight",
+        "dependencyassessment",
+        "storyimplementationplanning",
+        "testdesign",
+        "red",
+        "redreview",
+        "storysufficiencyreconciliation",
+        "green",
+        "refactor",
+        "security",
+        "gitcommit(refactor)",
+    ]
+    if failed_stage not in canonical_order:
+        return None
+    invalidated = {failed_stage}
+    invalidated.update(str(item).strip().lower() for item in (failure_context.get("invalidated_stages") or []) if str(item).strip())
+    for criterion in ((execution.success_criteria if execution is not None else None) or []):
+        criterion_text = f"{criterion.criterion} {criterion.oracle}".lower()
+        for stage_id in canonical_order:
+            if stage_id in criterion_text and stage_id != failed_stage:
+                invalidated.add(stage_id)
+    valid_prior = [stage_id for stage_id in canonical_order if stage_id not in invalidated and canonical_order.index(stage_id) < canonical_order.index(failed_stage)]
+    repaired_artifacts = failure_context.get("repaired_artifacts")
+    if not isinstance(repaired_artifacts, list):
+        repaired_artifacts = []
+    replay_metadata = {
+        "resume_from_stage": failed_stage,
+        "valid_prior_stages": valid_prior,
+        "invalidated_stages": [stage_id for stage_id in canonical_order if stage_id in invalidated],
+        "repaired_artifacts": repaired_artifacts,
+        "prior_run_id": prior_run_id,
+        "strategy": None if diagnosis is None else diagnosis.strategy,
+    }
+    return replay_metadata
+
+
+_TEST_RUNTIME_STAGE_IDS = {"red", "redreview", "green", "verifygreen", "gitcommit(refactor)"}
+_TEST_RUNTIME_FAILURE_PATTERNS = (
+    "story test validation failed",
+    "story-scoped test sufficiency failed",
+    "insufficient_story_tests",
+    "test validation",
+    "no story-scoped tests discovered",
+    "no tests collected",
+    "collected 0 items",
+    "pytest",
+    "vitest",
+)
+
+_PYTESTMARK_BLOCK_START_RE = re.compile(r"^\s*pytestmark\s*=\s*\[")
+_OPEN_FROM_IMPORT_RE = re.compile(r"^\s*from\b.+\bimport\s*\(\s*$")
+
+
+def _count_parens(text: str) -> int:
+    return text.count("(") - text.count(")")
+
+
+def _find_pytestmark_block(lines: list[str], *, start_index: int) -> tuple[int, int] | None:
+    for index in range(start_index, len(lines)):
+        if not _PYTESTMARK_BLOCK_START_RE.match(lines[index]):
+            continue
+        balance = lines[index].count("[") - lines[index].count("]")
+        end_index = index + 1
+        while balance > 0 and end_index < len(lines):
+            balance += lines[end_index].count("[") - lines[end_index].count("]")
+            end_index += 1
+        if balance == 0:
+            return index, end_index
+        return None
+    return None
+
+
+def _repair_malformed_story_pytestmark_import_block(*, text: str) -> str | None:
+    lines = text.splitlines()
+    for import_start, line in enumerate(lines):
+        if not _OPEN_FROM_IMPORT_RE.match(line):
+            continue
+        block = _find_pytestmark_block(lines, start_index=import_start + 1)
+        if block is None:
+            continue
+        pytestmark_start, pytestmark_end = block
+        paren_balance = _count_parens(line)
+        closing_index: int | None = None
+        cursor = import_start + 1
+        while cursor < len(lines):
+            if pytestmark_start <= cursor < pytestmark_end:
+                cursor = pytestmark_end
+                continue
+            paren_balance += _count_parens(lines[cursor])
+            if paren_balance <= 0:
+                closing_index = cursor
+                break
+            cursor += 1
+        if closing_index is None:
+            continue
+        repaired_lines = lines[:pytestmark_start] + lines[pytestmark_end:]
+        adjusted_closing_index = closing_index - (pytestmark_end - pytestmark_start)
+        insert_at = adjusted_closing_index + 1
+        pytestmark_lines = lines[pytestmark_start:pytestmark_end]
+        prefix = repaired_lines[:insert_at]
+        suffix = repaired_lines[insert_at:]
+        updated_lines = [
+            *prefix,
+            *([""] if prefix and prefix[-1].strip() else []),
+            *pytestmark_lines,
+            *([""] if suffix and suffix[0].strip() else []),
+            *suffix,
+        ]
+        updated_text = "\n".join(updated_lines).rstrip() + "\n"
+        if updated_text == text:
+            continue
+        try:
+            compile(updated_text, "<recovered_story_test>", "exec")
+        except SyntaxError:
+            continue
+        return updated_text
+    return None
+
+
+def _repair_story_test_files_for_runtime_failure(*, repo_root: Path, test_paths: list[str]) -> list[str]:
+    changed: list[str] = []
+    for raw_path in test_paths:
+        relative = str(raw_path or "").strip()
+        if not relative or not relative.endswith(".py"):
+            continue
+        candidate = repo_root / relative
+        if not candidate.exists() or not candidate.is_file():
+            continue
+        try:
+            original = candidate.read_text(encoding="utf-8")
+            compile(original, str(candidate), "exec")
+            continue
+        except SyntaxError:
+            pass
+        except Exception:
+            continue
+        repaired = _repair_malformed_story_pytestmark_import_block(text=original)
+        if repaired is None:
+            continue
+        candidate.write_text(repaired, encoding="utf-8")
+        changed.append(relative)
+    return changed
+
+
+def _maybe_repair_story_test_runtime_contract(*, repo_root: Path, item: FailedQueueItemArtifact) -> dict[str, Any] | None:
+    if item.queue_type != "story":
+        return None
+    story_id = str(item.story_id or "").strip()
+    if not story_id:
+        return None
+    failure_context = item.failure_context if isinstance(item.failure_context, dict) else {}
+    failed_stage = str(failure_context.get("failed_stage") or "").strip().lower()
+    if failed_stage not in _TEST_RUNTIME_STAGE_IDS:
+        return None
+    current_contract = load_story_test_runtime_contract(repo_root=repo_root, story_id=story_id) or {}
+    current_story_uuid = str(current_contract.get("story_uuid") or "").strip() or None
+    failure_blob = " ".join(
+        str(part or "")
+        for part in (
+            item.failure_message,
+            failure_context.get("error"),
+            failure_context.get("error_type"),
+        )
+    ).lower()
+    fresh_test_paths = [str(path) for path in (current_contract.get("test_paths") or []) if str(path).strip()]
+    if not fresh_test_paths:
+        fresh_test_paths = discover_story_scoped_test_paths(
+            repo_root=repo_root,
+            story_id=story_id,
+            story_uuid=current_story_uuid,
+        )
+    if current_contract:
+        fresh_contract = dict(current_contract)
+        if fresh_test_paths:
+            fresh_contract["test_paths"] = fresh_test_paths
+    else:
+        fresh_contract = resolve_story_runtime_contract(
+            repo_root=repo_root,
+            story_id=story_id,
+            story_uuid=current_story_uuid,
+            test_paths=fresh_test_paths,
+            prefer_story_contract=False,
+        )
+        fresh_contract = normalize_recovery_story_runtime_contract(
+            repo_root=repo_root,
+            contract=fresh_contract,
+        )
+        fresh_contract["source"] = "recovery_repair"
+    repaired_test_files = _repair_story_test_files_for_runtime_failure(
+        repo_root=repo_root,
+        test_paths=fresh_test_paths,
+    )
+
+    comparable_keys = ("framework", "cwd", "run_cmd", "env", "setup_cmd", "test_paths")
+    current_fingerprint = {key: current_contract.get(key) for key in comparable_keys}
+    fresh_fingerprint = {key: fresh_contract.get(key) for key in comparable_keys}
+    looks_like_runtime_failure = any(pattern in failure_blob for pattern in _TEST_RUNTIME_FAILURE_PATTERNS)
+    if not looks_like_runtime_failure and current_fingerprint == fresh_fingerprint and current_contract and not repaired_test_files:
+        return None
+
+    path = story_test_runtime_contract_path(repo_root=repo_root, story_id=story_id)
+    updated = current_fingerprint != fresh_fingerprint or not current_contract or bool(repaired_test_files)
+    if updated:
+        if current_fingerprint != fresh_fingerprint or not current_contract:
+            path = persist_story_runtime_contract(repo_root=repo_root, contract=fresh_contract)
+    return {
+        "story_id": story_id,
+        "path": str(path),
+        "updated": updated,
+        "previous_contract": current_contract or None,
+        "runtime_contract": fresh_contract,
+        "files_changed": repaired_test_files,
+        "reason": "test_runtime_boundary_repaired",
+    }
+
+
+def _load_local_setup_contract(repo_root: Path) -> tuple[Path, LocalSetupContract] | None:
+    path = repo_root / ".devflow" / "local_setup.json"
+    payload = _load_json_file(path)
+    if payload is None:
+        return None
+    try:
+        return path, LocalSetupContract.model_validate(payload)
+    except Exception:
+        return None
+
+
+def _check_health_endpoint(url: str, expected_status: int) -> bool:
+    try:
+        with urllib_request.urlopen(url, timeout=5) as response:
+            return int(response.status) == int(expected_status)
+    except urllib_error.HTTPError as exc:
+        return int(exc.code) == int(expected_status)
+    except Exception:
+        return False
+
+
+def _collect_preflight_repair_files(*, repo_root: Path, service_hints: list[str]) -> dict[str, str]:
+    candidates = [
+        ".devflow/local_setup.json",
+        "docker-compose.yml",
+        "docker-compose.yaml",
+        "compose.yml",
+        "compose.yaml",
+        "package.json",
+        "pnpm-workspace.yaml",
+        "pnpm-workspace.yml",
+        "pnpm-lock.yaml",
+        "package-lock.json",
+        "yarn.lock",
+        "pyproject.toml",
+        "uv.lock",
+        "requirements.txt",
+        "requirements-dev.txt",
+        "vite.config.ts",
+        "vite.config.js",
+        "vite.config.mjs",
+        "next.config.js",
+        "next.config.mjs",
+        "next.config.ts",
+    ]
+    files: dict[str, str] = {}
+    for relative in candidates:
+        path = repo_root / relative
+        if not path.exists() or not path.is_file():
+            continue
+        try:
+            files[relative] = path.read_text(encoding="utf-8")
+        except Exception:
+            continue
+    hints = {hint for hint in service_hints if hint}
+    for child in sorted(repo_root.iterdir()):
+        if not child.is_dir() or child.name.startswith("."):
+            continue
+        child_name = child.name.lower()
+        if hints and not any(hint in child_name or child_name in hint for hint in hints):
+            continue
+        for name in ("package.json", "pyproject.toml", "vite.config.ts", "vite.config.js", "next.config.js", "next.config.ts"):
+            path = child / name
+            if not path.exists() or not path.is_file():
+                continue
+            rel = str(path.relative_to(repo_root))
+            if rel in files:
+                continue
+            try:
+                files[rel] = path.read_text(encoding="utf-8")
+            except Exception:
+                continue
+    return files
+
+
+def _select_recovery_health_checks(*, contract: LocalSetupContract, failing_urls: set[str]) -> list[HealthCheckEntry]:
+    checks = [hc for hc in contract.health_checks if hc.url in failing_urls] if failing_urls else list(contract.health_checks)
+    return checks or list(contract.health_checks)
+
+
+def _verify_recovery_health_checks(*, checks: list[HealthCheckEntry]) -> tuple[list[str], list[str]]:
+    verification_checks: list[str] = []
+    blocking_reasons: list[str] = []
+    for health_check in checks:
+        if _check_health_endpoint(health_check.url, health_check.expected_status):
+            verification_checks.append(f"{health_check.name} returned {health_check.expected_status} at {health_check.url}")
+        else:
+            blocking_reasons.append(f"{health_check.name} still failed health verification at {health_check.url}")
+    return verification_checks, blocking_reasons
+
+
+def _maybe_repair_story_preflight_health_boundary(*, repo_root: Path, item: FailedQueueItemArtifact) -> dict[str, Any] | None:
+    details = _load_preflight_health_failure_details(repo_root=repo_root, item=item)
+    if details is None:
+        return None
+    setup = _load_local_setup_contract(repo_root)
+    if setup is None:
+        return {
+            "ready": False,
+            "blocking_reasons": ["Cannot repair preflight health failure because .devflow/local_setup.json is missing or invalid."],
+            "files_changed": [],
+            "artifact_path": str(details["artifact_path"]),
+            "verification_checks": [],
+            "repair_result": None,
+            "start_command_executed": False,
+        }
+    setup_path, contract = setup
+    failing_urls = {
+        str(issue.get("message") or "").split(": ", 1)[1].split(" did not return ", 1)[0].strip()
+        for issue in details["health_issues"]
+        if ": " in str(issue.get("message") or "") and " did not return " in str(issue.get("message") or "")
+    }
+    failing_checks = _select_recovery_health_checks(contract=contract, failing_urls=failing_urls)
+    initial_verification_checks, initial_blocking_reasons = _verify_recovery_health_checks(checks=failing_checks)
+    if not initial_blocking_reasons:
+        return {
+            "ready": True,
+            "blocking_reasons": [],
+            "files_changed": [],
+            "artifact_path": str(details["artifact_path"]),
+            "verification_checks": initial_verification_checks
+            + [
+                "Skipped local setup start command because the previously failing health checks are already healthy."
+            ],
+            "repair_result": None,
+            "start_command_executed": False,
+        }
+    service_hints = [str(hc.name or "").split("-", 1)[0].strip().lower() for hc in failing_checks]
+    logs_by_service = {
+        hint: _get_docker_service_logs(hint, repo_root)
+        for hint in service_hints
+        if hint
+    }
+    repair_result, _envelope = run_agent_step(
+        repo_root=repo_root,
+        stage_name="recovery_preflight_health_repo_repair",
+        output_model=RemediationResultArtifact,
+        context_payload={
+            "failed_item": item.model_dump(),
+            "preflight_report_path": str(details["artifact_path"]),
+            "preflight_report": details["report"],
+            "health_issues": details["health_issues"],
+            "local_setup_path": str(setup_path),
+            "local_setup": contract.model_dump(),
+            "service_logs": logs_by_service,
+            "files_to_change": _collect_preflight_repair_files(repo_root=repo_root, service_hints=service_hints),
+        },
+        guidance=load_agentic_prompt_lines("recovery_preflight_health_repo_repair"),
+        timeout_seconds=600,
+        strength=_CURRENT_STRENGTH,
+    )
+    files_written: list[str] = []
+    for patch in repair_result.file_patches or []:
+        target = repo_root / patch.path
+        try:
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_text(patch.content, encoding="utf-8")
+            files_written.append(patch.path)
+        except Exception:
+            continue
+    repair_result = repair_result.model_copy(update={"fix_applied": bool(files_written), "files_changed": files_written or repair_result.files_changed})
+    if not repair_result.fix_applied:
+        return {
+            "ready": False,
+            "blocking_reasons": ["Preflight health repair produced no file changes, so recovery cannot verify a real repo/config fix."],
+            "files_changed": [],
+            "artifact_path": str(details["artifact_path"]),
+            "verification_checks": [],
+            "repair_result": repair_result,
+            "start_command_executed": False,
+        }
+    refreshed_setup = _load_local_setup_contract(repo_root)
+    if refreshed_setup is not None:
+        _setup_path, contract = refreshed_setup
+        failing_checks = _select_recovery_health_checks(contract=contract, failing_urls=failing_urls)
+    verification_checks, blocking_reasons = _verify_recovery_health_checks(checks=failing_checks)
+    start_command_executed = False
+    if blocking_reasons:
+        subprocess.run(
+            ["/bin/sh", "-lc", contract.start_command],
+            cwd=str(repo_root),
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=600,
+        )
+        start_command_executed = True
+        verification_checks, blocking_reasons = _verify_recovery_health_checks(checks=failing_checks)
+    else:
+        verification_checks.append(
+            "Skipped local setup start command because the previously failing health checks are already healthy after the repo/config repair."
+        )
+    return {
+        "ready": not blocking_reasons,
+        "blocking_reasons": blocking_reasons,
+        "files_changed": files_written,
+        "artifact_path": str(details["artifact_path"]),
+        "verification_checks": verification_checks,
+        "repair_result": repair_result,
+        "start_command_executed": start_command_executed,
+    }
+
+
+class LoadFailedQueueItemNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        def _run(_node_exec_id: str):
+            event = task_context.event
+            store, run_id = _store_run()
+            row = None
+            payload_ref = None
+            story_id = None
+            repo_root = Path(str(event.repo_root))
+            queue_type = str(event.queue_type)
+            item_id = str(event.item_id)
+            if queue_type == "scope":
+                row = store.get_scope_queue_item(scope_queue_id=item_id)
+                payload_ref = None if row is None else str(row.get("scope_payload_path") or "") or None
+            elif queue_type == "idea_creation":
+                row = store.get_idea_creation_queue_item(idea_creation_queue_id=item_id)
+                payload_ref = None if row is None else str(row.get("idea_payload_path") or "") or None
+            elif queue_type == "idea":
+                row = store.get_idea_queue_item(idea_queue_id=item_id)
+                payload_ref = None if row is None else str(row.get("idea_payload_path") or "") or None
+            elif queue_type == "story":
+                row = store.get_story_queue_item(story_queue_id=item_id)
+                if row is not None:
+                    artifact = store.get_artifact(artifact_id=str(row.get("story_artifact_id") or ""))
+                    payload_ref = None if artifact is None else str(artifact.get("uri") or "") or None
+                    story_id = str(row.get("story_id") or "") or None
+            elif queue_type == "integration":
+                row = store.get_integration_queue_item(integration_queue_id=item_id)
+                payload_ref = None if row is None else str(row.get("integration_payload_path") or "") or None
+            else:
+                raise ValueError(f"unsupported queue_type={queue_type}")
+            if row is None:
+                raise ValueError(f"queue item not found: {queue_type}:{item_id}")
+            artifact = FailedQueueItemArtifact(
+                queue_type=queue_type,  # type: ignore[arg-type]
+                item_id=item_id,
+                project_id=row.get("project_id"),
+                dfs_project_id=row.get("dfs_project_id"),
+                enqueue_run_id=str(row.get("enqueue_run_id") or ""),
+                status=str(row.get("status") or ""),
+                title=str(row.get("title") or item_id),
+                payload_ref=payload_ref,
+                payload_exists=bool(payload_ref and Path(payload_ref).exists()),
+                story_id=story_id,
+                failure_message=row.get("failure_message"),
+                failure_context=dict(row.get("failure_context") or {}),
+                raw_row=dict(row),
+            )
+            task_context.metadata["failed_item"] = artifact
+            _publish_node(artifact.dfs_project_id or event.project_id, run_id, "Loading failure", recovery_id=item_id)
+            _publish(artifact.dfs_project_id or event.project_id, run_id, "running", "processing", f"Loading failed {queue_type} queue item", recovery_id=item_id)
+            self.save_output(artifact)
+            return artifact.model_dump(), task_context
+        return _persist_node(node_id="load_failed_item", node_name="LoadFailedQueueItem", fn=_run)
+
+
+class SystemicPatternAnalysisNode(Node):
+    """Deterministic node: queries DB for sibling failures sharing the same error signature.
+
+    Produces SystemicPatternArtifact stored in task_context.metadata["systemic_pattern"].
+    Also sets metadata["churn_detected"] when the durable recovery churn gate is already
+    at or above threshold for the same durable identity + normalized failure signature.
+    """
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        def _run(_node_exec_id: str):
+            store, run_id = _store_run()
+            item: FailedQueueItemArtifact = task_context.metadata["failed_item"]
+            project_id = item.dfs_project_id or task_context.event.project_id
+            _publish_node(project_id, run_id, "Analysing failure pattern", recovery_id=item.item_id)
+
+            failure_context = item.failure_context if isinstance(item.failure_context, dict) else {}
+            failure_message = str(item.failure_message or "").strip()
+            error_type = str(failure_context.get("error_type") or "").strip()
+            failed_stage = str(failure_context.get("failed_stage") or "").strip()
+
+            failure_signature = _normalized_failure_signature(
+                failure_message=failure_message,
+                failure_context=failure_context,
+            )
+
+            queue_type = item.queue_type
+            item_id = item.item_id
+
+            affected_item_ids: list[str] = []
+            sample_failure_messages: list[str] = []
+            failed_stages: list[str] = []
+            churn_state = _load_recovery_churn_gate_state(
+                store=store,
+                project_id=project_id,
+                item=item,
+                failure_signature=failure_signature,
+            )
+            churn_detected = bool(churn_state["threshold_met"])
+
+            with store._connect() as conn:
+                # --- Find sibling failed items in the source queue with matching failure_message ---
+                if queue_type == "story":
+                    table = "story_queue"
+                    id_col = "story_queue_id"
+                elif queue_type == "idea_creation":
+                    table = "idea_creation_queue"
+                    id_col = "idea_creation_queue_id"
+                elif queue_type == "idea":
+                    table = "idea_queue"
+                    id_col = "idea_queue_id"
+                elif queue_type == "scope":
+                    table = "scope_queue"
+                    id_col = "scope_queue_id"
+                elif queue_type == "integration":
+                    table = "integration_queue"
+                    id_col = "integration_queue_id"
+                else:
+                    table = None
+                    id_col = None
+
+                if table and failure_message:
+                    # Search by LIKE on failure_message (primary)
+                    sig_fragment = failure_message[:60].replace("%", "")
+                    sibling_rows = conn.execute(
+                        f"SELECT {id_col} as item_id, failure_message, failure_context_json FROM {table} "
+                        f"WHERE status='failed' AND failure_message LIKE ? LIMIT 100",
+                        (f"%{sig_fragment}%",),
+                    ).fetchall()
+                    for r in sibling_rows:
+                        sid = str(r["item_id"] or "")
+                        if sid and sid not in affected_item_ids:
+                            affected_item_ids.append(sid)
+                        msg = str(r["failure_message"] or "")
+                        if msg and msg not in sample_failure_messages:
+                            sample_failure_messages.append(msg)
+                        # Extract failed_stage from failure_context_json
+                        try:
+                            fc = json.loads(str(r["failure_context_json"] or "{}") or "{}")
+                            fs = str(fc.get("failed_stage") or "").strip()
+                            if fs and fs not in failed_stages:
+                                failed_stages.append(fs)
+                        except Exception:
+                            pass
+
+                # --- Also search recovery_queue for matching failure signature ---
+                rq_rows = conn.execute(
+                    "SELECT source_item_id, failure_message FROM recovery_queue "
+                    "WHERE failure_message LIKE ? LIMIT 100",
+                    (f"%{failure_message[:60].replace('%', '')}%",) if failure_message else ("%",),
+                ).fetchall()
+                for r in rq_rows:
+                    sid = str(r["source_item_id"] or "")
+                    if sid and sid not in affected_item_ids:
+                        affected_item_ids.append(sid)
+
+            # Include current item if not already in list
+            if item_id not in affected_item_ids:
+                affected_item_ids.insert(0, item_id)
+            if failure_message and failure_message not in sample_failure_messages:
+                sample_failure_messages.insert(0, failure_message)
+            if failed_stage and failed_stage not in failed_stages:
+                failed_stages.insert(0, failed_stage)
+
+            total_affected = len(affected_item_ids)
+            is_systemic = total_affected >= 3
+
+            pattern = SystemicPatternArtifact(
+                failure_signature=failure_signature,
+                is_systemic=is_systemic,
+                affected_queue_type=queue_type,
+                affected_item_ids=affected_item_ids,
+                sample_failure_messages=sample_failure_messages[:5],
+                failed_stages=failed_stages[:10],
+                total_affected=total_affected,
+                pattern_summary=(
+                    f"{'Systemic' if is_systemic else 'Isolated'} failure: {failure_signature}. "
+                    f"{total_affected} affected item(s) in {queue_type} queue."
+                ),
+            )
+
+            task_context.metadata["systemic_pattern"] = pattern
+            if churn_detected:
+                task_context.metadata["churn_detected"] = True
+                task_context.metadata["churn"] = {
+                    "detected": True,
+                    "failure_signature": failure_signature,
+                    "occurrence_count": int(churn_state["occurrence_count"]),
+                    "threshold": int(churn_state["threshold"]),
+                    "error_task_id": churn_state["error_task_id"],
+                    "churn_key": churn_state["churn_key"],
+                    "story_id": item.story_id,
+                    "item_id": item.item_id,
+                }
+
+            self.save_output(pattern)
+            return pattern.model_dump(), task_context
+
+        return _persist_node(node_id="systemic_pattern_analysis", node_name="SystemicPatternAnalysis", fn=_run)
+
+
+class AgenticFailureInvestigationNode(AgentNode):
+    def get_agent_config(self) -> AgentConfig:
+        return AgentConfig(instructions="Investigate the nature of a process failure, identify the affected boundary, and define what successful recovery would look like.", output_type=RecoveryInvestigationArtifact)
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        def _run(_node_exec_id: str):
+            item: FailedQueueItemArtifact = task_context.metadata["failed_item"]
+            if task_context.metadata.get("churn_detected"):
+                churn = task_context.metadata.get("churn") or {}
+                artifact = _build_churn_gate_investigation(
+                    item=item,
+                    failure_signature=str(churn.get("failure_signature") or ""),
+                    occurrence_count=int(churn.get("occurrence_count") or 0),
+                    threshold=int(churn.get("threshold") or _RECOVERY_CHURN_GATE_THRESHOLD),
+                    churn_key=str(churn.get("churn_key") or "") or None,
+                )
+                task_context.metadata["investigation"] = artifact
+                self.save_output(artifact)
+                return artifact.model_dump(), task_context
+            _publish_node(item.dfs_project_id or task_context.event.project_id, _store_run()[1], "Investigating failure", recovery_id=item.item_id)
+            store, _run_id = _store_run()
+            repo_root = Path(str(task_context.event.repo_root))
+            log_evidence = _gather_log_first_recovery_evidence(store=store, repo_root=repo_root, item=item)
+            artifact, _envelope = run_agent_step(
+                repo_root=repo_root,
+                stage_name="recovery_failure_investigation",
+                output_model=RecoveryInvestigationArtifact,
+                context_payload={"failed_item": item.model_dump(), "log_evidence": log_evidence},
+                guidance=load_agentic_prompt_lines("recovery_failure_investigation"),
+                timeout_seconds=300,
+                strength=_CURRENT_STRENGTH,
+            )
+            artifact = _enrich_investigation_with_log_evidence(investigation=artifact, log_evidence=log_evidence)
+            task_context.metadata["investigation"] = artifact
+            task_context.metadata["log_evidence"] = log_evidence
+            self.save_output(artifact)
+            return artifact.model_dump(), task_context
+        return _persist_node(node_id="failure_investigation", node_name="AgenticFailureInvestigation", fn=_run)
+
+
+class RootCauseCodeInvestigationNode(AgentNode):
+    """AgentNode: given a systemic pattern, reads key source files and asks the LLM to
+    identify the root cause and propose a concrete fix plan.
+
+    Output: CodeRootCauseArtifact
+    """
+
+    def get_agent_config(self) -> AgentConfig:
+        return AgentConfig(
+            instructions="Investigate the root cause of a systemic failure pattern in the DevFlow codebase. Identify the exact code location, describe the problem, and propose a specific fix plan.",
+            output_type=CodeRootCauseArtifact,
+        )
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        def _run(_node_exec_id: str):
+            event = task_context.event
+            store, run_id = _store_run()
+            item: FailedQueueItemArtifact = task_context.metadata["failed_item"]
+            pattern: SystemicPatternArtifact = task_context.metadata["systemic_pattern"]
+            repo_root = Path(str(event.repo_root))
+            project_id = item.dfs_project_id or event.project_id
+
+            _publish_node(project_id, run_id, "Investigating root cause", recovery_id=item.item_id)
+
+            # Deterministically read key source files before calling agent
+            key_source_files: dict[str, str] = {}
+
+            # 1. agentic_runtime.py — always include
+            _read_file_into(key_source_files, repo_root / "src" / "devflow_engine" / "agentic_runtime.py")
+
+            # 2. implementation/dag.py — Red/Green context-building sections
+            impl_dag = repo_root / "src" / "devflow_engine" / "implementation" / "dag.py"
+            if impl_dag.exists():
+                try:
+                    full = impl_dag.read_text(encoding="utf-8")
+                    # Extract lines around _build_red_generation_context and _build_green_generation_context
+                    lines = full.splitlines()
+                    relevant: list[str] = []
+                    capture = False
+                    for i, line in enumerate(lines):
+                        if "_build_red_generation_context" in line or "_build_green_generation_context" in line:
+                            capture = True
+                        if capture:
+                            relevant.append(line)
+                            # Stop after a reasonable chunk (50 lines per section)
+                            if len(relevant) > 200:
+                                relevant.append("... [truncated]")
+                                break
+                    key_source_files["implementation/dag.py (red/green context sections)"] = "\n".join(relevant[:200])
+                except Exception:
+                    pass
+
+            # 3. List .claude/agents/ directory sizes if present
+            agents_dir = repo_root / ".claude" / "agents"
+            if agents_dir.exists():
+                sizes: list[str] = []
+                try:
+                    for p in sorted(agents_dir.iterdir()):
+                        if p.is_file():
+                            sizes.append(f"{p.name}: {p.stat().st_size} bytes")
+                except Exception:
+                    pass
+                key_source_files[".claude/agents/ directory"] = "\n".join(sizes) if sizes else "(empty)"
+
+            # 4. CLI config
+            cli_config = ""
+            config_path = Path.home() / ".devflow" / "config.toml"
+            if config_path.exists():
+                try:
+                    cli_config = config_path.read_text(encoding="utf-8")
+                except Exception:
+                    cli_config = "(unreadable)"
+
+            artifact, _envelope = run_agent_step(
+                repo_root=repo_root,
+                stage_name="recovery_root_cause_investigation",
+                output_model=CodeRootCauseArtifact,
+                context_payload={
+                    "systemic_pattern": pattern.model_dump(),
+                    "failed_item": item.model_dump(),
+                    "key_source_files": key_source_files,
+                    "cli_config": cli_config,
+                },
+                guidance=load_agentic_prompt_lines("recovery_root_cause_investigation"),
+                timeout_seconds=600,
+                strength=_CURRENT_STRENGTH,
+            )
+            task_context.metadata["root_cause"] = artifact
+            self.save_output(artifact)
+            return artifact.model_dump(), task_context
+
+        return _persist_node(node_id="root_cause_investigation", node_name="RootCauseCodeInvestigation", fn=_run)
+
+
+def _read_file_into(dest: dict[str, str], path: Path) -> None:
+    """Helper: read a file and store its content in dest under its relative path."""
+    try:
+        if path.exists():
+            dest[str(path.name)] = path.read_text(encoding="utf-8")
+    except Exception:
+        pass
+
+
+class RemediationExecutionNode(AgentNode):
+    """AgentNode: given CodeRootCauseArtifact, applies the fix.
+
+    After parsing the agent response:
+    - Writes file_patches to disk at repo_root / patch.path
+    - Sets fix_applied=True if files were written
+
+    Output: RemediationResultArtifact
+    """
+
+    def get_agent_config(self) -> AgentConfig:
+        return AgentConfig(
+            instructions="Apply a diagnosed code/config fix to resolve a systemic DevFlow failure. Produce file patches with complete corrected file contents.",
+            output_type=RemediationResultArtifact,
+        )
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        def _run(_node_exec_id: str):
+            event = task_context.event
+            store, run_id = _store_run()
+            item: FailedQueueItemArtifact = task_context.metadata["failed_item"]
+            root_cause: CodeRootCauseArtifact = task_context.metadata["root_cause"]
+            pattern: SystemicPatternArtifact = task_context.metadata["systemic_pattern"]
+            repo_root = Path(str(event.repo_root))
+            project_id = item.dfs_project_id or event.project_id
+
+            _publish_node(project_id, run_id, "Applying remediation", recovery_id=item.item_id)
+
+            # Deterministically read current content of files the fix will touch
+            files_to_change: dict[str, str] = {}
+            if root_cause.root_cause_location:
+                # root_cause_location is like "src/devflow_engine/agentic_runtime.py:run_agent_step"
+                location_path = root_cause.root_cause_location.split(":")[0].strip()
+                candidate = repo_root / location_path
+                _read_file_into(files_to_change, candidate)
+
+            # Also read files listed in files_inspected (capped at 3)
+            for fi in (root_cause.files_inspected or [])[:3]:
+                _read_file_into(files_to_change, repo_root / fi)
+
+            artifact, _envelope = run_agent_step(
+                repo_root=repo_root,
+                stage_name="recovery_remediation_execution",
+                output_model=RemediationResultArtifact,
+                context_payload={
+                    "root_cause": root_cause.model_dump(),
+                    "systemic_pattern": pattern.model_dump(),
+                    "files_to_change": files_to_change,
+                    "affected_items": pattern.affected_item_ids,
+                },
+                guidance=load_agentic_prompt_lines("recovery_remediation_execution"),
+                timeout_seconds=900,
+                strength=_CURRENT_STRENGTH,
+            )
+
+            # Write file patches to disk
+            files_written: list[str] = []
+            for patch in (artifact.file_patches or []):
+                try:
+                    target = repo_root / patch.path
+                    target.parent.mkdir(parents=True, exist_ok=True)
+                    target.write_text(patch.content, encoding="utf-8")
+                    files_written.append(patch.path)
+                except Exception as exc:
+                    # Log but don't fail the node; note it in verification
+                    pass
+
+            if files_written:
+                artifact = artifact.model_copy(update={"fix_applied": True, "files_changed": files_written})
+
+            task_context.metadata["remediation_result"] = artifact
+            self.save_output(artifact)
+            return artifact.model_dump(), task_context
+
+        return _persist_node(node_id="remediation_execution", node_name="RemediationExecution", fn=_run)
+
+
+class BulkReenqueueNode(Node):
+    """Deterministic node: after remediation, bulk-re-enqueues all items listed in
+    RemediationResultArtifact.items_to_requeue and dead-letters items in items_to_dead_letter.
+    """
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        def _run(_node_exec_id: str):
+            event = task_context.event
+            store, run_id = _store_run()
+            item: FailedQueueItemArtifact = task_context.metadata["failed_item"]
+            remediation: RemediationResultArtifact = task_context.metadata["remediation_result"]
+            pattern: SystemicPatternArtifact = task_context.metadata["systemic_pattern"]
+            project_id = item.dfs_project_id or event.project_id
+
+            _publish_node(project_id, run_id, "Re-enqueuing fixed items", recovery_id=item.item_id)
+
+            queue_type = pattern.affected_queue_type
+            requeued: list[str] = []
+            dead_lettered: list[str] = []
+            errors: list[str] = []
+
+            for qid in (remediation.items_to_requeue or []):
+                try:
+                    if queue_type == "story":
+                        store.retry_story_queue_item(project_id=project_id, story_queue_id=qid, preserve_failure_context=False)
+                    elif queue_type == "idea_creation":
+                        store.retry_idea_creation_queue_item(project_id=project_id, idea_creation_queue_id=qid, preserve_failure_context=False)
+                    elif queue_type == "idea":
+                        store.retry_idea_queue_item(project_id=project_id, idea_queue_id=qid, preserve_failure_context=False)
+                    elif queue_type == "scope":
+                        store.retry_scope_queue_item(project_id=project_id, scope_queue_id=qid, preserve_failure_context=False)
+                    elif queue_type == "integration":
+                        store.retry_integration_queue_item(project_id=project_id, integration_queue_id=qid, preserve_failure_context=False)
+                    requeued.append(qid)
+                except Exception as exc:
+                    errors.append(f"{qid}: {exc}")
+
+            # Mark dead-letter items as failed in queue (set failure_message to indicate dead-lettered)
+            for qid in (remediation.items_to_dead_letter or []):
+                try:
+                    with store._connect() as conn:
+                        now_ts = int(__import__("time").time())
+                        if queue_type == "story":
+                            conn.execute(
+                                "UPDATE story_queue SET status='failed', failure_message=?, updated_at=? WHERE story_queue_id=?",
+                                ("dead_lettered_by_recovery", now_ts, qid),
+                            )
+                        elif queue_type == "idea_creation":
+                            conn.execute(
+                                "UPDATE idea_creation_queue SET status='failed', failure_message=?, updated_at=? WHERE idea_creation_queue_id=?",
+                                ("dead_lettered_by_recovery", now_ts, qid),
+                            )
+                        elif queue_type == "idea":
+                            conn.execute(
+                                "UPDATE idea_queue SET status='failed', failure_message=?, updated_at=? WHERE idea_queue_id=?",
+                                ("dead_lettered_by_recovery", now_ts, qid),
+                            )
+                        elif queue_type == "scope":
+                            conn.execute(
+                                "UPDATE scope_queue SET status='failed', failure_message=?, updated_at=? WHERE scope_queue_id=?",
+                                ("dead_lettered_by_recovery", now_ts, qid),
+                            )
+                        elif queue_type == "integration":
+                            conn.execute(
+                                "UPDATE integration_queue SET status='failed', failure_message=?, updated_at=? WHERE integration_queue_id=?",
+                                ("dead_lettered_by_recovery", now_ts, qid),
+                            )
+                    dead_lettered.append(qid)
+                except Exception as exc:
+                    errors.append(f"dead_letter {qid}: {exc}")
+
+            outcome = "reenqueued" if requeued else "blocked"
+            task_context.metadata["outcome"] = outcome
+            task_context.metadata["bulk_reenqueue_result"] = {
+                "requeued": requeued,
+                "dead_lettered": dead_lettered,
+                "errors": errors,
+            }
+
+            summary = f"Bulk re-enqueue: {len(requeued)} re-queued, {len(dead_lettered)} dead-lettered, {len(errors)} errors."
+            result = {"requeued": requeued, "dead_lettered": dead_lettered, "errors": errors, "outcome": outcome, "summary": summary}
+            self.save_output(result)
+            return result, task_context
+
+        return _persist_node(node_id="bulk_reenqueue", node_name="BulkReenqueue", fn=_run)
+
+
+# ---------------------------------------------------------------------------
+# Router redesign
+# ---------------------------------------------------------------------------
+
+class _RouteLoopGuard(RouterNode):
+    def determine_next_node(self, task_context: TaskContext) -> Node | None:
+        if task_context.metadata.get("churn_detected"):
+            task_context.metadata["outcome"] = "blocked"
+            task_context.metadata["delegation_summary"] = None
+            failure_signature = str((task_context.metadata.get("churn") or {}).get("failure_signature") or "same normalized failure")
+            churn_key = str((task_context.metadata.get("churn") or {}).get("churn_key") or "")
+            occurrence_count = int((task_context.metadata.get("churn") or {}).get("occurrence_count") or 0)
+            threshold = int((task_context.metadata.get("churn") or {}).get("threshold") or _RECOVERY_CHURN_GATE_THRESHOLD)
+            task_context.metadata["pre_replay"] = PreReplayCheckArtifact(
+                queue_type=task_context.metadata["failed_item"].queue_type,
+                ready=False,
+                checks=[],
+                blocking_reasons=[
+                    (
+                        f"CHURN: durable recovery gate blocked {failure_signature} "
+                        f"after {occurrence_count}/{threshold} no-material-change strikes"
+                        + (f" ({churn_key})" if churn_key else "")
+                    )
+                ],
+            )
+            return PublishRecoveryStateNode(task_context=task_context)
+        return None
+
+
+class _RouteSystemic(RouterNode):
+    def determine_next_node(self, task_context: TaskContext) -> Node | None:
+        pattern: SystemicPatternArtifact | None = task_context.metadata.get("systemic_pattern")
+        if pattern and pattern.is_systemic:
+            return RootCauseCodeInvestigationNode(task_context=task_context)
+        return None
+
+
+class _RouteIsolatedProcess(RouterNode):
+    def determine_next_node(self, task_context: TaskContext) -> Node | None:
+        return AgenticRecoveryDiagnosisNode(task_context=task_context)
+
+
+class SystemicVsIsolatedRouter(BaseRouter):
+    def __init__(self) -> None:
+        self.routes = [_RouteLoopGuard(), _RouteSystemic(), _RouteIsolatedProcess()]
+        self.fallback = PublishRecoveryStateNode()
+
+
+# ---------------------------------------------------------------------------
+# Legacy router (kept for backwards-compat in case referenced externally)
+# ---------------------------------------------------------------------------
+
+class _RouteCodeError(RouterNode):
+    def determine_next_node(self, task_context: TaskContext) -> Node | None:
+        if False:
+            return AgenticRecoveryExecutionNode(task_context=task_context)
+        return None
+
+
+class _RouteProcessError(RouterNode):
+    def determine_next_node(self, task_context: TaskContext) -> Node | None:
+        if True:
+            return AgenticRecoveryDiagnosisNode(task_context=task_context)
+        return None
+
+
+class FailureTypeRouter(BaseRouter):
+    def __init__(self) -> None:
+        self.routes = [_RouteCodeError(), _RouteProcessError()]
+        self.fallback = PublishRecoveryStateNode()
+
+
+class AgenticRecoveryDiagnosisNode(AgentNode):
+    def get_agent_config(self) -> AgentConfig:
+        return AgentConfig(instructions="Diagnose the best recovery strategy for a failed queue item.", output_type=RecoveryDiagnosisArtifact)
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        def _run(_node_exec_id: str):
+            item: FailedQueueItemArtifact = task_context.metadata["failed_item"]
+            investigation: RecoveryInvestigationArtifact | None = task_context.metadata.get("investigation")
+            _publish_node(item.dfs_project_id or task_context.event.project_id, _store_run()[1], "Diagnosing recovery", recovery_id=item.item_id)
+            diagnosis, plan = _build_diagnosis(item=item, investigation=investigation)
+            task_context.metadata["diagnosis"] = diagnosis
+            task_context.metadata["plan"] = plan
+            self.save_output(diagnosis)
+            return {"diagnosis": diagnosis.model_dump(), "plan": plan.model_dump()}, task_context
+        return _persist_node(node_id="recovery_diagnosis", node_name="AgenticRecoveryDiagnosis", fn=_run)
+
+
+class AgenticRecoveryExecutionNode(AgentNode):
+    def get_agent_config(self) -> AgentConfig:
+        return AgentConfig(instructions="Execute recovery and verify the outcome against agent-defined success criteria inside the same node.", output_type=RecoveryExecutionArtifact)
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        def _run(_node_exec_id: str):
+            event = task_context.event
+            store, run_id = _store_run()
+            item: FailedQueueItemArtifact = task_context.metadata["failed_item"]
+            diagnosis: RecoveryDiagnosisArtifact | None = task_context.metadata.get("diagnosis")
+            plan: RemediationPlanArtifact | None = task_context.metadata.get("plan")
+            investigation: RecoveryInvestigationArtifact | None = task_context.metadata.get("investigation")
+            repo_root = Path(str(event.repo_root))
+            _publish_node(item.dfs_project_id or event.project_id, run_id, "Executing recovery", recovery_id=item.item_id)
+            last_reenqueue: ReenqueueArtifact | None = None
+            last_execution: RecoveryExecutionArtifact | None = None
+            last_verified: PreReplayCheckArtifact | None = None
+
+            preflight_health_repair = _maybe_repair_story_preflight_health_boundary(repo_root=repo_root, item=item)
+            if preflight_health_repair is not None:
+                failure_signature = _normalized_failure_signature(
+                    failure_message=item.failure_message,
+                    failure_context=item.failure_context if isinstance(item.failure_context, dict) else {},
+                )
+                health_repair_diagnosis = RecoveryDiagnosisArtifact(
+                    queue_type=item.queue_type,
+                    item_id=item.item_id,
+                    strategy="preflight_health_repair_recovery",
+                    summary="Repair repo/config causing preflight health failure.",
+                    rationale="Preflight evidence contains health_check_failed blockers that require a bounded repo/config repair before replay.",
+                    verification_targets=[
+                        {
+                            "criterion": "Failing health endpoint returns the expected status",
+                            "oracle": "The previously failing preflight boundary is reachable and returns the expected HTTP status after the repair.",
+                            "evidence_ref": preflight_health_repair["artifact_path"],
+                        }
+                    ],
+                    suggested_action="repair_artifact_then_requeue",
+                )
+                plan = RemediationPlanArtifact(
+                    queue_type=item.queue_type,
+                    action="repair_artifact_then_requeue",
+                    summary="Repair repo/config causing preflight health failure, verify the boundary, then requeue.",
+                    steps=[criterion.criterion for criterion in health_repair_diagnosis.verification_targets],
+                    remediation_artifact=preflight_health_repair["artifact_path"],
+                )
+                task_context.metadata["diagnosis"] = health_repair_diagnosis
+                task_context.metadata["plan"] = plan
+                repair_result: RemediationResultArtifact | None = preflight_health_repair.get("repair_result")
+                if not preflight_health_repair["ready"]:
+                    execution = RecoveryExecutionArtifact(
+                        queue_type=item.queue_type,
+                        item_id=item.item_id,
+                        outcome="blocked",
+                        execution_summary=(
+                            "Preflight health repo/config repair did not restore the failing boundary."
+                            if repair_result is not None and repair_result.fix_applied
+                            else "Preflight health repo/config repair could not produce a bounded fix."
+                        ),
+                        preserve_failure_context=True,
+                        attempts_used=1,
+                        success_criteria=health_repair_diagnosis.verification_targets,
+                        verification_summary="; ".join(preflight_health_repair["blocking_reasons"]),
+                    )
+                    last_verified = PreReplayCheckArtifact(
+                        queue_type=item.queue_type,
+                        ready=False,
+                        checks=preflight_health_repair["verification_checks"],
+                        blocking_reasons=preflight_health_repair["blocking_reasons"],
+                    )
+                    updated_context = _record_recovery_attempt(
+                        item=item,
+                        diagnosis=health_repair_diagnosis,
+                        success=False,
+                        failure_signature=failure_signature,
+                        material_change=bool(preflight_health_repair["files_changed"]),
+                        remediation_artifact=preflight_health_repair["artifact_path"],
+                    )
+                    item.failure_context = updated_context
+                    _persist_queue_failure_context(store=store, item=item, failure_context=updated_context)
+                    task_context.metadata["preflight_health_repair"] = preflight_health_repair
+                    task_context.metadata["pre_replay"] = last_verified
+                    task_context.metadata["outcome"] = "blocked"
+                    task_context.metadata["recovery_execution"] = execution
+                    self.save_output(execution)
+                    return {
+                        "outcome": "blocked",
+                        "recovery_execution": execution.model_dump(),
+                        "pre_replay": last_verified.model_dump(),
+                        "preflight_health_repair": {
+                            "files_changed": preflight_health_repair["files_changed"],
+                            "artifact_path": preflight_health_repair["artifact_path"],
+                        },
+                    }, task_context
+                health_repair_changed_files = bool(preflight_health_repair["files_changed"])
+                execution = RecoveryExecutionArtifact(
+                    queue_type=item.queue_type,
+                    item_id=item.item_id,
+                    outcome="reenqueued",
+                    execution_summary=(
+                        "Repaired repo/config for preflight health failure and re-verified the boundary."
+                        if health_repair_changed_files
+                        else "Re-verified the preflight health boundary and skipped local setup bootstrap because the stack was already healthy."
+                    ),
+                    preserve_failure_context=True,
+                    attempts_used=1,
+                    success_criteria=health_repair_diagnosis.verification_targets,
+                    verification_summary="; ".join(preflight_health_repair["verification_checks"]),
+                )
+                updated_context = _record_recovery_attempt(
+                    item=item,
+                    diagnosis=health_repair_diagnosis,
+                    success=True,
+                    failure_signature=failure_signature,
+                    material_change=health_repair_changed_files,
+                    remediation_artifact=preflight_health_repair["artifact_path"],
+                )
+                updated_context["preflight_health_repair"] = {
+                    "artifact_path": preflight_health_repair["artifact_path"],
+                    "files_changed": preflight_health_repair["files_changed"],
+                }
+                item.failure_context = updated_context
+                _persist_queue_failure_context(store=store, item=item, failure_context=updated_context)
+                replay_metadata = _build_story_replay_metadata(item=item, diagnosis=health_repair_diagnosis, execution=execution)
+                row = store.retry_story_queue_item(
+                    project_id=event.project_id,
+                    story_queue_id=item.item_id,
+                    preserve_failure_context=True,
+                    replay_metadata=replay_metadata,
+                )
+                failure_context = dict(item.failure_context if isinstance(item.failure_context, dict) else {})
+                failure_context["preflight_health_repair"] = {
+                    "artifact_path": preflight_health_repair["artifact_path"],
+                    "files_changed": preflight_health_repair["files_changed"],
+                }
+                if replay_metadata is not None:
+                    failure_context["replay"] = replay_metadata
+                last_reenqueue = ReenqueueArtifact(
+                    queue_type=item.queue_type,
+                    item_id=item.item_id,
+                    status=str(row.get("status") or "queued"),
+                    failure_context=failure_context,
+                    replay_metadata=replay_metadata,
+                )
+                last_verified = PreReplayCheckArtifact(
+                    queue_type=item.queue_type,
+                    ready=True,
+                    checks=preflight_health_repair["verification_checks"],
+                    blocking_reasons=[],
+                )
+                task_context.metadata["preflight_health_repair"] = preflight_health_repair
+                task_context.metadata["reenqueue"] = last_reenqueue
+                task_context.metadata["pre_replay"] = last_verified
+                task_context.metadata["outcome"] = "reenqueued"
+                task_context.metadata["recovery_execution"] = execution
+                self.save_output(execution)
+                return {
+                    "outcome": "reenqueued",
+                    "recovery_execution": execution.model_dump(),
+                    "pre_replay": last_verified.model_dump(),
+                    "preflight_health_repair": {
+                        "files_changed": preflight_health_repair["files_changed"],
+                        "artifact_path": preflight_health_repair["artifact_path"],
+                    },
+                }, task_context
+
+            runtime_contract_repair = _maybe_repair_story_test_runtime_contract(repo_root=repo_root, item=item)
+            if runtime_contract_repair is not None:
+                failure_signature = _normalized_failure_signature(
+                    failure_message=item.failure_message,
+                    failure_context=item.failure_context if isinstance(item.failure_context, dict) else {},
+                )
+                repaired_test_files = [str(path) for path in (runtime_contract_repair.get("files_changed") or []) if str(path).strip()]
+                contract_updated = bool(
+                    (runtime_contract_repair.get("previous_contract") or None) is None
+                    or any(
+                        (runtime_contract_repair.get("previous_contract") or {}).get(key) != runtime_contract_repair["runtime_contract"].get(key)
+                        for key in ("framework", "cwd", "run_cmd", "env", "setup_cmd", "test_paths")
+                    )
+                )
+                runtime_repair_diagnosis = diagnosis or RecoveryDiagnosisArtifact(
+                    queue_type=item.queue_type,
+                    item_id=item.item_id,
+                    strategy="artifact_regeneration_recovery",
+                    summary="Repair runtime contract",
+                    rationale="Story runtime contract repair is the bounded automated fix path.",
+                    suggested_action="repair_artifact_then_requeue",
+                )
+                runtime_repair_diagnosis = runtime_repair_diagnosis.model_copy(update={"strategy": "artifact_regeneration_recovery"})
+                if plan is None:
+                    plan = RemediationPlanArtifact(
+                        queue_type=item.queue_type,
+                        action="repair_artifact_then_requeue",
+                        summary="Repair story test runtime contract and requeue.",
+                    )
+                if not runtime_contract_repair["updated"]:
+                    churn_message = (
+                        "Recovery runtime repair produced no material runtime-contract delta "
+                        f"for {_durable_recovery_identity(item)} on {failure_signature}."
+                    )
+                    churn_state = _record_recovery_churn_strike(
+                        store=store,
+                        project_id=event.project_id,
+                        run_id=run_id,
+                        item=item,
+                        failure_signature=failure_signature,
+                        message=churn_message,
+                    )
+                    noop_cycles = int(churn_state["occurrence_count"])
+                    updated_context = _record_recovery_attempt(
+                        item=item,
+                        diagnosis=runtime_repair_diagnosis,
+                        success=False,
+                        failure_signature=failure_signature,
+                        material_change=False,
+                        remediation_artifact=runtime_contract_repair["path"],
+                    )
+                    item.failure_context = updated_context
+                    task_context.metadata["failed_item"] = item
+                    _persist_queue_failure_context(store=store, item=item, failure_context=updated_context)
+                    task_context.metadata["durable_churn_key"] = churn_state["churn_key"]
+                    task_context.metadata["durable_churn_error_task_id"] = churn_state["error_task_id"]
+                    if churn_state["threshold_met"]:
+                        investigation = _build_churn_gate_investigation(
+                            item=item,
+                            failure_signature=failure_signature,
+                            occurrence_count=noop_cycles,
+                            threshold=_RECOVERY_CHURN_GATE_THRESHOLD,
+                            churn_key=str(churn_state.get("churn_key") or "") or None,
+                            remediation_artifact=str(runtime_contract_repair["path"]),
+                        )
+                        task_context.metadata["investigation"] = investigation
+                        plan = plan.model_copy(
+                            update={
+                                "action": "manual_review_required",
+                                "summary": "Durable recovery churn gate blocked repeated no-op runtime repair.",
+                                "remediation_artifact": runtime_contract_repair["path"],
+                            }
+                        )
+                        task_context.metadata["plan"] = plan
+                        execution = RecoveryExecutionArtifact(
+                            queue_type=item.queue_type,
+                            item_id=item.item_id,
+                            outcome="blocked",
+                            execution_summary=(
+                                "Blocked repeated story runtime repair because the same normalized failure produced no material runtime-contract delta."
+                            ),
+                            preserve_failure_context=True,
+                            attempts_used=noop_cycles,
+                            success_criteria=[
+                                {
+                                    "criterion": "Durable churn gate blocks repeated no-op runtime repair",
+                                    "oracle": "Recovery hard-blocks after 3 strikes on the same durable identity with no material runtime-contract delta.",
+                                    "evidence_ref": runtime_contract_repair["path"],
+                                }
+                            ],
+                            verification_summary=(
+                                f"CHURN: durable recovery gate blocked {failure_signature} after {noop_cycles}/{_RECOVERY_CHURN_GATE_THRESHOLD} no-material-change strikes"
+                                f" ({churn_state['churn_key']})."
+                            ),
+                        )
+                        last_verified = PreReplayCheckArtifact(
+                            queue_type=item.queue_type,
+                            ready=False,
+                            checks=[],
+                            blocking_reasons=[
+                                f"CHURN: durable recovery gate blocked {failure_signature} after {noop_cycles}/{_RECOVERY_CHURN_GATE_THRESHOLD} no-material-change strikes"
+                                f" ({churn_state['churn_key']})."
+                            ],
+                        )
+                        task_context.metadata["runtime_contract_update"] = runtime_contract_repair
+                        task_context.metadata["pre_replay"] = last_verified
+                        task_context.metadata["outcome"] = "blocked"
+                        task_context.metadata["recovery_execution"] = execution
+                        task_context.metadata["churn_detected"] = True
+                        task_context.metadata["churn"] = {
+                            "detected": True,
+                            "failure_signature": failure_signature,
+                            "noop_cycles": noop_cycles,
+                            "occurrence_count": noop_cycles,
+                            "threshold": _RECOVERY_CHURN_GATE_THRESHOLD,
+                            "error_task_id": churn_state["error_task_id"],
+                            "churn_key": churn_state["churn_key"],
+                            "reason": "no_material_runtime_contract_delta",
+                        }
+                        self.save_output(execution)
+                        return {
+                            "outcome": "blocked",
+                            "recovery_execution": execution.model_dump(),
+                            "pre_replay": last_verified.model_dump(),
+                            "runtime_contract_update": runtime_contract_repair,
+                        }, task_context
+                execution_summary_parts: list[str] = []
+                success_criteria = []
+                verification_checks = []
+                remediation_artifact = str(runtime_contract_repair["path"])
+                if contract_updated:
+                    execution_summary_parts.append(f"Updated story test runtime contract at {runtime_contract_repair['path']}.")
+                    success_criteria.append(
+                        {
+                            "criterion": "Story runtime contract updated",
+                            "oracle": "The canonical story test_runtime.json matches the corrected run pattern.",
+                            "evidence_ref": runtime_contract_repair["path"],
+                        }
+                    )
+                    verification_checks.append("story runtime contract updated")
+                if repaired_test_files:
+                    execution_summary_parts.append(
+                        "Repaired malformed story-scoped pytest file(s): " + ", ".join(repaired_test_files) + "."
+                    )
+                    success_criteria.append(
+                        {
+                            "criterion": "Malformed story-scoped pytest file repaired",
+                            "oracle": "Injected pytestmark blocks no longer sit inside an open Python import list and the repaired file compiles.",
+                            "evidence_ref": repaired_test_files[0],
+                        }
+                    )
+                    verification_checks.append("malformed story-scoped pytest file repaired")
+                    remediation_artifact = repaired_test_files[0]
+                plan = plan.model_copy(
+                    update={
+                        "action": "repair_artifact_then_requeue",
+                        "summary": " ".join(execution_summary_parts).strip(),
+                        "remediation_artifact": remediation_artifact,
+                    }
+                )
+                task_context.metadata["plan"] = plan
+                execution = RecoveryExecutionArtifact(
+                    queue_type=item.queue_type,
+                    item_id=item.item_id,
+                    outcome="reenqueued",
+                    execution_summary=" ".join(execution_summary_parts).strip(),
+                    preserve_failure_context=True,
+                    attempts_used=1,
+                    success_criteria=[
+                        *success_criteria,
+                        {
+                            "criterion": "Story re-enqueued",
+                            "oracle": "The story queue row returns to queued so the next run uses the repaired runtime boundary.",
+                        },
+                    ],
+                    verification_summary="; ".join([*verification_checks, "story re-enqueued"]),
+                )
+                updated_context = _record_recovery_attempt(
+                    item=item,
+                    diagnosis=runtime_repair_diagnosis,
+                    success=True,
+                    failure_signature=failure_signature,
+                    material_change=True,
+                    remediation_artifact=runtime_contract_repair["path"],
+                )
+                updated_context["runtime_contract_update"] = {
+                    "path": runtime_contract_repair["path"],
+                    "source": runtime_contract_repair["runtime_contract"].get("source"),
+                    "files_changed": repaired_test_files,
+                }
+                item.failure_context = updated_context
+                _persist_queue_failure_context(store=store, item=item, failure_context=updated_context)
+                replay_metadata = _build_story_replay_metadata(item=item, diagnosis=diagnosis, execution=execution)
+                row = store.retry_story_queue_item(
+                    project_id=event.project_id,
+                    story_queue_id=item.item_id,
+                    preserve_failure_context=True,
+                    replay_metadata=replay_metadata,
+                )
+                failure_context = dict(item.failure_context if isinstance(item.failure_context, dict) else {})
+                failure_context["runtime_contract_update"] = {
+                    "path": runtime_contract_repair["path"],
+                    "source": runtime_contract_repair["runtime_contract"].get("source"),
+                    "files_changed": repaired_test_files,
+                }
+                if replay_metadata is not None:
+                    failure_context["replay"] = replay_metadata
+                last_reenqueue = ReenqueueArtifact(
+                    queue_type=item.queue_type,
+                    item_id=item.item_id,
+                    status=str(row.get("status") or "queued"),
+                    failure_context=failure_context,
+                    replay_metadata=replay_metadata,
+                )
+                last_verified = PreReplayCheckArtifact(
+                    queue_type=item.queue_type,
+                    ready=True,
+                    checks=[*verification_checks, "story queue item reset to queued"],
+                    blocking_reasons=[],
+                )
+                task_context.metadata["runtime_contract_update"] = runtime_contract_repair
+                task_context.metadata["reenqueue"] = last_reenqueue
+                task_context.metadata["pre_replay"] = last_verified
+                task_context.metadata["outcome"] = "reenqueued"
+                task_context.metadata["recovery_execution"] = execution
+                self.save_output(execution)
+                return {
+                    "outcome": "reenqueued",
+                    "recovery_execution": execution.model_dump(),
+                    "pre_replay": last_verified.model_dump(),
+                    "runtime_contract_update": runtime_contract_repair,
+                }, task_context
+
+            for attempt in range(1, 4):
+                if diagnosis is None or plan is None:
+                    diagnosis, plan = _build_diagnosis(item=item, investigation=investigation, attempt=attempt)
+                    task_context.metadata["diagnosis"] = diagnosis
+                    task_context.metadata["plan"] = plan
+                try:
+                    execution, _envelope = run_agent_step(
+                        repo_root=repo_root,
+                        stage_name="recovery_execution",
+                        output_model=RecoveryExecutionArtifact,
+                        context_payload={
+                            "failed_item": item.model_dump(),
+                            "diagnosis": None if diagnosis is None else diagnosis.model_dump(),
+                            "success_criteria": [] if diagnosis is None else [c.model_dump() for c in diagnosis.verification_targets],
+                            "plan": None if plan is None else plan.model_dump(),
+                            "attempt": attempt,
+                            "previous_reenqueue": None if last_reenqueue is None else last_reenqueue.model_dump(),
+                            "previous_execution": None if last_execution is None else last_execution.model_dump(),
+                        },
+                        guidance=load_agentic_prompt_lines("recovery_execution"),
+                        timeout_seconds=300,
+                        strength=_CURRENT_STRENGTH,
+                    )
+                except Exception as exc:
+                    error_summary = f"recovery_execution agent step failed (attempt {attempt}): {exc}"
+                    task_context.metadata["outcome"] = "reenqueued"
+                    task_context.metadata["recovery_execution_error"] = error_summary
+                    if item.queue_type == "scope":
+                        store.retry_scope_queue_item(project_id=event.project_id, scope_queue_id=item.item_id, preserve_failure_context=False)
+                    elif item.queue_type == "idea_creation":
+                        store.retry_idea_creation_queue_item(project_id=event.project_id, idea_creation_queue_id=item.item_id, preserve_failure_context=False)
+                    elif item.queue_type == "idea":
+                        store.retry_idea_queue_item(project_id=event.project_id, idea_queue_id=item.item_id, preserve_failure_context=False)
+                    elif item.queue_type == "integration":
+                        store.retry_integration_queue_item(project_id=event.project_id, integration_queue_id=item.item_id, preserve_failure_context=False)
+                    else:
+                        store.retry_story_queue_item(project_id=event.project_id, story_queue_id=item.item_id, preserve_failure_context=False, replay_metadata=None)
+                    fallback_artifact = RecoveryExecutionArtifact(
+                        queue_type=item.queue_type,
+                        item_id=item.item_id,
+                        outcome="reenqueued",
+                        execution_summary=error_summary,
+                        attempts_used=attempt,
+                    )
+                    self.save_output(fallback_artifact)
+                    return {"outcome": "reenqueued", "recovery_execution": fallback_artifact.model_dump(), "recovery_execution_error": error_summary}, task_context
+                execution.attempts_used = attempt
+                last_execution = execution
+                if execution.outcome == "delegated":
+                    updated_context = _record_recovery_attempt(item=item, diagnosis=diagnosis, success=False)
+                    item.failure_context = updated_context
+                    _persist_queue_failure_context(store=store, item=item, failure_context=updated_context)
+                    task_context.metadata["outcome"] = "delegated"
+                    task_context.metadata["delegation_summary"] = execution.delegation_summary or execution.execution_summary
+                    task_context.metadata["recovery_execution"] = execution
+                    self.save_output(execution)
+                    return {"outcome": "delegated", "recovery_execution": execution.model_dump()}, task_context
+                if execution.outcome == "blocked":
+                    if attempt < 3:
+                        diagnosis, plan = _build_diagnosis(
+                            item=item,
+                            investigation=investigation,
+                            prior_execution=execution,
+                            attempt=attempt + 1,
+                        )
+                        task_context.metadata["diagnosis"] = diagnosis
+                        task_context.metadata["plan"] = plan
+                        continue
+                    task_context.metadata["outcome"] = "blocked"
+                    task_context.metadata["recovery_execution"] = execution
+                    self.save_output(execution)
+                    return {"outcome": "blocked", "recovery_execution": execution.model_dump()}, task_context
+                preserve = execution.preserve_failure_context if plan is None else plan.preserve_failure_context
+                if preserve:
+                    failure_context = _record_recovery_attempt(item=item, diagnosis=diagnosis, success=False)
+                    item.failure_context = failure_context
+                    _persist_queue_failure_context(store=store, item=item, failure_context=failure_context)
+                replay_metadata = _build_story_replay_metadata(item=item, diagnosis=diagnosis, execution=execution)
+                if item.queue_type == "scope":
+                    row = store.retry_scope_queue_item(project_id=event.project_id, scope_queue_id=item.item_id, preserve_failure_context=preserve)
+                elif item.queue_type == "idea_creation":
+                    row = store.retry_idea_creation_queue_item(project_id=event.project_id, idea_creation_queue_id=item.item_id, preserve_failure_context=preserve)
+                elif item.queue_type == "idea":
+                    row = store.retry_idea_queue_item(project_id=event.project_id, idea_queue_id=item.item_id, preserve_failure_context=preserve)
+                elif item.queue_type == "integration":
+                    row = store.retry_integration_queue_item(project_id=event.project_id, integration_queue_id=item.item_id, preserve_failure_context=preserve)
+                else:
+                    row = store.retry_story_queue_item(
+                        project_id=event.project_id,
+                        story_queue_id=item.item_id,
+                        preserve_failure_context=preserve,
+                        replay_metadata=replay_metadata,
+                    )
+                failure_context = item.failure_context if preserve else None
+                if isinstance(failure_context, dict) and replay_metadata is not None:
+                    failure_context = dict(failure_context)
+                    failure_context["replay"] = replay_metadata
+                last_reenqueue = ReenqueueArtifact(
+                    queue_type=item.queue_type,
+                    item_id=item.item_id,
+                    status=str(row.get("status") or "queued"),
+                    failure_context=failure_context,
+                    replay_metadata=replay_metadata,
+                )
+                task_context.metadata["reenqueue"] = last_reenqueue
+                _publish_node(item.dfs_project_id or event.project_id, run_id, "Verifying recovery", recovery_id=item.item_id)
+                verified, _verify_env = run_agent_step(
+                    repo_root=repo_root,
+                    stage_name="recovery_execution_verification",
+                    output_model=PreReplayCheckArtifact,
+                    context_payload={
+                        "failed_item": item.model_dump(),
+                        "diagnosis": None if diagnosis is None else diagnosis.model_dump(),
+                        "success_criteria": [] if diagnosis is None else [c.model_dump() for c in diagnosis.verification_targets],
+                        "execution": execution.model_dump(),
+                        "reenqueue": last_reenqueue.model_dump(),
+                        "attempt": attempt,
+                    },
+                    guidance=load_agentic_prompt_lines("recovery_execution_verification"),
+                    timeout_seconds=300,
+                    strength=_CURRENT_STRENGTH,
+                )
+                last_verified = verified
+                task_context.metadata["pre_replay"] = verified
+                if _verification_allows_reenqueue(execution=execution, verified=verified, diagnosis=diagnosis):
+                    if preserve:
+                        failure_context = _record_recovery_attempt(item=item, diagnosis=diagnosis, success=True)
+                        item.failure_context = failure_context
+                        _persist_queue_failure_context(store=store, item=item, failure_context=failure_context)
+                    task_context.metadata["outcome"] = "reenqueued"
+                    task_context.metadata["recovery_execution"] = execution
+                    self.save_output(execution)
+                    return {"outcome": "reenqueued", "recovery_execution": execution.model_dump(), "pre_replay": verified.model_dump()}, task_context
+                if attempt < 3:
+                    diagnosis, plan = _build_diagnosis(
+                        item=item,
+                        investigation=investigation,
+                        prior_execution=execution,
+                        prior_verification=verified,
+                        attempt=attempt + 1,
+                    )
+                    task_context.metadata["diagnosis"] = diagnosis
+                    task_context.metadata["plan"] = plan
+            task_context.metadata["outcome"] = "blocked"
+            artifact = last_execution or RecoveryExecutionArtifact(queue_type=item.queue_type, item_id=item.item_id, outcome="blocked", execution_summary="Recovery attempts exhausted", attempts_used=3)
+            task_context.metadata["recovery_execution"] = artifact
+            if last_verified is not None:
+                task_context.metadata["pre_replay"] = last_verified
+            self.save_output(artifact)
+            return {"outcome": "blocked", "recovery_execution": artifact.model_dump(), "pre_replay": None if last_verified is None else last_verified.model_dump()}, task_context
+        return _persist_node(node_id="recovery_execution", node_name="AgenticRecoveryExecution", fn=_run)
+
+
+class PublishRecoveryStateNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        def _run(_node_exec_id: str):
+            event = task_context.event
+            _store, run_id = _store_run()
+            item: FailedQueueItemArtifact = task_context.metadata["failed_item"]
+            _publish_node(item.dfs_project_id or event.project_id, run_id, "Publishing outcome", recovery_id=item.item_id)
+            investigation: RecoveryInvestigationArtifact | None = task_context.metadata.get("investigation")
+            plan: RemediationPlanArtifact | None = task_context.metadata.get("plan")
+            reenqueue: ReenqueueArtifact | None = task_context.metadata.get("reenqueue")
+            recovery_execution: RecoveryExecutionArtifact | None = task_context.metadata.get("recovery_execution")
+            outcome = str(task_context.metadata.get("outcome") or "blocked")
+            pre_replay: PreReplayCheckArtifact | None = task_context.metadata.get("pre_replay")
+            if outcome == "reenqueued":
+                summary = f"Recovery re-enqueued failed {item.queue_type} item {item.item_id}."
+                _publish(item.dfs_project_id or event.project_id, run_id, "idle", "completed", summary, recovery_id=item.item_id)
+                exit_code = 0
+            elif outcome == "delegated":
+                summary = str(task_context.metadata.get("delegation_summary") or f"Delegated {item.queue_type} item {item.item_id} to code error recovery.")
+                _publish(item.dfs_project_id or event.project_id, run_id, "idle", "completed", summary, recovery_id=item.item_id)
+                exit_code = 0
+            else:
+                reason = "recovery blocked"
+                if pre_replay is not None and pre_replay.blocking_reasons:
+                    reason = "; ".join(pre_replay.blocking_reasons)
+                elif recovery_execution is not None and recovery_execution.verification_summary:
+                    reason = recovery_execution.verification_summary
+                elif investigation is not None and investigation.summary:
+                    reason = investigation.summary
+                elif plan is not None and plan.summary:
+                    reason = plan.summary
+                if investigation is not None and investigation.non_convergence is not None:
+                    reason = f"{reason}; non-convergence: {investigation.non_convergence.reason}" if reason else investigation.non_convergence.reason
+                summary = f"Recovery blocked for failed {item.queue_type} item {item.item_id}: {reason}"
+                _publish(item.dfs_project_id or event.project_id, run_id, "failed", "blocked", summary, reason, recovery_id=item.item_id)
+                exit_code = 2
+            diagnosis: RecoveryDiagnosisArtifact | None = task_context.metadata.get("diagnosis")
+            handoff_path = _persist_recovery_handoff_artifact(
+                repo_root=Path(str(event.repo_root)),
+                recovery_run_id=run_id,
+                item=item,
+                investigation=investigation,
+                diagnosis=diagnosis,
+                execution=recovery_execution,
+                pre_replay=pre_replay,
+            )
+            if recovery_execution is not None and handoff_path is not None:
+                recovery_execution = recovery_execution.model_copy(update={
+                    "recovery_handoff_artifact_path": str(handoff_path),
+                    "recovery_handoff_summary": f"Persisted compact recovery handoff at {handoff_path}",
+                })
+                task_context.metadata["recovery_execution"] = recovery_execution
+            artifact = RecoveryOutcomeArtifact(
+                queue_type=item.queue_type,
+                item_id=item.item_id,
+                project_id=item.project_id,
+                outcome=outcome,  # type: ignore[arg-type]
+                summary=summary,
+                investigation=investigation,
+                plan=plan,
+                reenqueue=reenqueue,
+                recovery_handoff_artifact_path=None if handoff_path is None else str(handoff_path),
+            )
+            task_context.metadata["result"] = artifact.model_dump()
+            task_context.metadata["message"] = json.dumps(artifact.model_dump(), sort_keys=True)
+            task_context.metadata["exit_code"] = exit_code
+            self.save_output(artifact)
+            return {"outcome": outcome, "summary": summary, "exit_code": exit_code}, task_context
+        return _persist_node(node_id="publish_recovery_state", node_name="PublishRecoveryState", fn=_run)
+
+
+class FailureRecoveryWorkflow(Workflow):
+    workflow_schema = WorkflowSchema(
+        description="Post-queue-drain failure recovery DAG — systemic-aware",
+        event_schema=FailureRecoveryDagEvent,
+        start=LoadFailedQueueItemNode,
+        nodes=[
+            NodeConfig(node=LoadFailedQueueItemNode, connections=[SystemicPatternAnalysisNode]),
+            NodeConfig(node=SystemicPatternAnalysisNode, connections=[AgenticFailureInvestigationNode]),
+            NodeConfig(node=AgenticFailureInvestigationNode, connections=[SystemicVsIsolatedRouter]),
+            NodeConfig(node=SystemicVsIsolatedRouter, connections=[PublishRecoveryStateNode, RootCauseCodeInvestigationNode, AgenticRecoveryDiagnosisNode], is_router=True),
+            NodeConfig(node=RootCauseCodeInvestigationNode, connections=[RemediationExecutionNode]),
+            NodeConfig(node=RemediationExecutionNode, connections=[BulkReenqueueNode]),
+            NodeConfig(node=BulkReenqueueNode, connections=[PublishRecoveryStateNode]),
+            NodeConfig(node=AgenticRecoveryDiagnosisNode, connections=[AgenticRecoveryExecutionNode]),
+            NodeConfig(node=AgenticRecoveryExecutionNode, connections=[PublishRecoveryStateNode]),
+            NodeConfig(node=PublishRecoveryStateNode, connections=[]),
+        ],
+    )
+
+
+def run_failure_recovery_dag(*, repo_root: Path, store: ExecutionStore, project_id: str, queue_type: str, item_id: str, run_id: str | None = None, strength: str | None = None) -> FailureRecoveryDagResult:
+    owns_run = run_id is None
+    if run_id is None:
+        run_id = store.create_run(dag_id=DAG_ID, dag_version="v2", root_correlation_id=f"corr_recovery_{queue_type}_{item_id}", config={"project_id": project_id, "queue_type": queue_type, "item_id": item_id})
+        store.mark_run_started(run_id=run_id)
+    _publish(project_id, run_id, "running", "processing", f"Recovering failed {queue_type} queue item", recovery_id=item_id)
+    wf = FailureRecoveryWorkflow()
+    global _CURRENT_STORE, _CURRENT_RUN_ID, _CURRENT_STRENGTH, _CURRENT_REPO_ROOT
+    _CURRENT_STORE = store
+    _CURRENT_RUN_ID = run_id
+    _CURRENT_STRENGTH = strength
+    _CURRENT_REPO_ROOT = repo_root
+    try:
+        ctx = wf.run({"repo_root": str(repo_root), "project_id": project_id, "queue_type": queue_type, "item_id": item_id})
+    except Exception as exc:
+        if owns_run:
+            store.mark_run_finished(run_id=run_id, status="failed")
+        _publish(project_id, run_id, "failed", "failed", f"Recovery failed for {queue_type}:{item_id}", str(exc), recovery_id=item_id)
+        raise
+    finally:
+        _CURRENT_STORE = None
+        _CURRENT_RUN_ID = None
+        _CURRENT_STRENGTH = None
+        _CURRENT_REPO_ROOT = None
+    exit_code = int(ctx.metadata.get("exit_code") or 0)
+    if owns_run:
+        store.mark_run_finished(run_id=run_id, status="succeeded" if exit_code == 0 else "failed")
+    return FailureRecoveryDagResult(exit_code=exit_code, run_id=run_id, outcome=dict(ctx.metadata.get("result") or {}), message=str(ctx.metadata.get("message") or ""))