devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/worker.py

@@ -0,0 +1,1086 @@
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from devflow_engine.errors.runtime_observability import build_runtime_observability
+from devflow_engine.idea.idea_creation_dag import run_idea_creation_dag
+from devflow_engine.idea.story_pipeline import run_idea_to_devflow_stories_dag
+
+from .devflow_state import publish_devflow_state
+from .playwright_workflow import run_post_integration_playwright_dag
+from .process import dag as process_dag
+from .recovery.dag import run_failure_recovery_dag
+from .scope_idea.dag import run_scope_to_idea_dag
+from .source_doc_mutation_dag import run_source_doc_mutation_dag
+from .source_docs_updater import process_source_doc_update_once
+from .stores.execution_store import ExecutionStore
+from .worker_guard import ensure_worker_start_allowed
+
+
+@dataclass(frozen=True)
+class WorkerStartResult:
+    project_id: str
+    worker_id: str
+    processed: dict[str, Any] | None
+    report: dict[str, Any]
+
+
+@dataclass(frozen=True)
+class WorkerDrainResult:
+    project_id: str
+    processed_count: int
+    processed: list[dict[str, Any]]
+    stopped_reason: str
+    final_report: dict[str, Any]
+
+
+def _queue_label(queue_type: str | None) -> str:
+    return {
+        'error': 'Errors',
+        'recovery': 'Recovery',
+        'scope': 'Scopes',
+        'idea_creation': 'Idea creation',
+        'idea': 'Ideas',
+        'story': 'Stories',
+        'source_doc_mutation': 'Source doc mutations',
+        'integration': 'Integration',
+        'source_docs': 'Source docs',
+        None: 'Queue',
+    }.get(queue_type, 'Queue')
+
+
+def _publish_queue_boundary_state(*, project_id: str, run_id: str | None, queue_type: str | None, next_queue_type: str | None) -> None:
+    if next_queue_type is None:
+        summary = f"{_queue_label(queue_type)} queue completed"
+    else:
+        summary = f"{_queue_label(queue_type)} queue completed · next: {_queue_label(next_queue_type)}"
+    publish_devflow_state(
+        project_id=project_id,
+        run_id=run_id,
+        current_state='idle',
+        current_status='completed',
+        run_summary=summary,
+        display='project',
+        display_path=f"queue:{next_queue_type or queue_type or 'none'}",
+    )
+
+
+def _has_processed_successful_integration(processed: list[dict[str, Any]]) -> bool:
+    return any(item.get("queue_type") == "integration" and "error" not in item for item in processed)
+
+
+def _integration_drained_without_recovery(*, project_id: str, store: ExecutionStore) -> bool:
+    with store._connect() as conn:
+        integration_row = conn.execute(
+            (
+                "SELECT integration_queue_id FROM integration_queue "
+                "WHERE project_id=? AND status IN ('queued','claimed','in_progress','failed') LIMIT 1"
+            ),
+            (project_id,),
+        ).fetchone()
+        if integration_row is not None:
+            return False
+        recovery_row = store._select_active_recovery_queue_row(
+            conn,
+            project_id=project_id,
+            statuses=("queued", "claimed", "in_progress", "blocked"),
+            source_queue_type="integration",
+            columns="rq.recovery_queue_id",
+        )
+        return recovery_row is None
+
+
+def _maybe_run_post_integration_playwright_workflow(
+    *,
+    project_id: str,
+    repo_root: Path,
+    store: ExecutionStore,
+    processed: list[dict[str, Any]],
+    last_run_id: str | None,
+) -> str | None:
+    if any(item.get("queue_type") == "post_integration_playwright" for item in processed):
+        return last_run_id
+    if not _has_processed_successful_integration(processed):
+        return last_run_id
+    if not _integration_drained_without_recovery(project_id=project_id, store=store):
+        return last_run_id
+
+    result = run_post_integration_playwright_dag(
+        repo_root=repo_root,
+        store=store,
+        project_id=project_id,
+        trigger_run_id=last_run_id,
+    )
+    processed.append(
+        {
+            "queue_type": "post_integration_playwright",
+            "item_id": result.run_id,
+            "status": "completed" if result.exit_code == 0 else "failed",
+            "pipeline_dir": str(result.pipeline_dir),
+        }
+    )
+    return result.run_id
+
+
+def _load_story_payload(*, store: ExecutionStore, story_queue_id: str) -> tuple[dict[str, Any], dict[str, Any]]:
+    item = store.get_story_queue_item(story_queue_id=story_queue_id)
+    if item is None:
+        raise ValueError(f"story_queue_id not found: {story_queue_id}")
+    artifact = store.get_artifact(artifact_id=str(item["story_artifact_id"]))
+    if artifact is None:
+        raise ValueError(f"artifact not found: {item['story_artifact_id']}")
+
+    payload = dict(artifact.get("metadata") or {})
+    artifact_uri = str(artifact.get("uri") or "")
+    if artifact_uri:
+        path = Path(artifact_uri)
+        if path.exists():
+            payload = json.loads(path.read_text(encoding="utf-8"))
+
+    story = {
+        "story_id": payload.get("story_id"),
+        "story_uuid": payload.get("story_uuid"),
+        "title": payload.get("title"),
+        "project_id": item.get("project_id"),
+        "contract": payload,
+    }
+    failure_context = item.get("failure_context") if isinstance(item.get("failure_context"), dict) else {}
+    replay = failure_context.get("replay") if isinstance(failure_context.get("replay"), dict) else None
+    if replay:
+        story["replay"] = replay
+    return item, story
+
+
+def _load_scope_payload(*, store: ExecutionStore, scope_queue_id: str) -> dict[str, Any]:
+    item = store.get_scope_queue_item(scope_queue_id=scope_queue_id)
+    if item is None:
+        raise ValueError(f"scope_queue_id not found: {scope_queue_id}")
+    return item
+
+
+def _load_idea_creation_payload(*, store: ExecutionStore, idea_creation_queue_id: str) -> dict[str, Any]:
+    item = store.get_idea_creation_queue_item(idea_creation_queue_id=idea_creation_queue_id)
+    if item is None:
+        raise ValueError(f"idea_creation_queue_id not found: {idea_creation_queue_id}")
+    return item
+
+
+def _load_idea_payload(*, store: ExecutionStore, idea_queue_id: str) -> dict[str, Any]:
+    item = store.get_idea_queue_item(idea_queue_id=idea_queue_id)
+    if item is None:
+        raise ValueError(f"idea_queue_id not found: {idea_queue_id}")
+    return item
+
+
+def _load_source_doc_mutation_payload(*, store: ExecutionStore, source_doc_mutation_queue_id: str) -> dict[str, Any]:
+    item = store.get_source_doc_mutation_queue_item(source_doc_mutation_queue_id=source_doc_mutation_queue_id)
+    if item is None:
+        raise ValueError(f"source_doc_mutation_queue_id not found: {source_doc_mutation_queue_id}")
+    return item
+
+
+def _load_integration_payload(*, store: ExecutionStore, integration_queue_id: str) -> dict[str, Any]:
+    item = store.get_integration_queue_item(integration_queue_id=integration_queue_id)
+    if item is None:
+        raise ValueError(f'integration_queue_id not found: {integration_queue_id}')
+    return item
+
+
+def _resolve_integration_payload_path(*, repo_root: Path, item: dict[str, Any]) -> tuple[Path | None, dict[str, Any] | None]:
+    raw_path = str(item.get("integration_payload_path") or "").strip()
+    if raw_path:
+        candidate = Path(raw_path).expanduser()
+        if not candidate.is_absolute():
+            candidate = repo_root / candidate
+        try:
+            resolved = candidate.resolve()
+        except Exception:
+            resolved = candidate
+        if resolved.is_file():
+            try:
+                payload = json.loads(resolved.read_text(encoding="utf-8"))
+            except Exception:
+                payload = None
+            if isinstance(payload, dict):
+                return resolved, payload
+    return None, None
+
+
+def _load_recovery_payload(*, store: ExecutionStore, recovery_queue_id: str) -> dict[str, Any]:
+    item = store.get_recovery_queue_item(recovery_queue_id=recovery_queue_id)
+    if item is None:
+        raise ValueError(f"recovery_queue_id not found: {recovery_queue_id}")
+    return item
+
+
+def _enqueue_recovery_request(
+    *,
+    project_id: str,
+    repo_root: Path,
+    store: ExecutionStore,
+    source_queue_type: str,
+    source_item_id: str,
+    enqueue_run_id: str,
+    title: str,
+    failure_message: str | None,
+    failure_context: dict[str, Any] | None,
+) -> str:
+    recovery_root = repo_root / ".devflow" / "recovery" / "requests"
+    recovery_root.mkdir(parents=True, exist_ok=True)
+    request_path = recovery_root / f"{source_queue_type}_{source_item_id}_recovery_request.json"
+    request_payload = {
+        "contract": "recovery_queue_request.v1",
+        "project_id": project_id,
+        "source_queue_type": source_queue_type,
+        "source_item_id": source_item_id,
+        "enqueue_run_id": enqueue_run_id,
+        "title": title,
+        "failure_message": failure_message,
+        "failure_context": failure_context or {},
+    }
+    request_path.write_text(json.dumps(request_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return store.enqueue_recovery_task(
+        project_id=project_id,
+        enqueue_run_id=enqueue_run_id,
+        source_queue_type=source_queue_type,
+        source_item_id=source_item_id,
+        title=f"Recovery: {title}",
+        recovery_request_path=str(request_path),
+        failure_context=failure_context,
+    )
+
+
+def _implementation_failure_routes_to_error_solving(failure_context: dict[str, Any] | None) -> bool:
+    ctx = failure_context if isinstance(failure_context, dict) else {}
+    return (
+        str(ctx.get("kind") or "") == "implementation_failure.v1"
+        and str(ctx.get("classification_hint") or "") == "project_code"
+    )
+
+
+def _create_error_task_for_implementation_failure(
+    *,
+    store: ExecutionStore,
+    project_id: str,
+    run_id: str,
+    story_queue_id: str,
+    story_id: str,
+    failure_message: str | None,
+    failure_context: dict[str, Any],
+) -> str:
+    failure_kind = str(failure_context.get("failure_kind") or "implementation_failure")
+    failed_stage = str(failure_context.get("failed_stage") or failure_context.get("stage") or "implementation")
+    return store.create_error_task_from_failure(
+        project_id=project_id,
+        run_id=run_id,
+        plane="implementation",
+        title=f"Implementation failure: {story_id or story_queue_id}",
+        severity="high",
+        source_kind="implementation_failure.v1",
+        source_ref=story_id or story_queue_id,
+        error_type=failure_kind,
+        message=failure_message or f"{failed_stage} failed",
+        stacktrace="",
+        next_steps=[
+            "Run devflow doctor for triage.",
+            "Run devflow error solve for project-code repair.",
+        ],
+        fingerprint_inputs={
+            "kind": "implementation_failure.v1",
+            "project_id": project_id,
+            "story_id": story_id,
+            "story_queue_id": story_queue_id,
+            "failed_stage": failed_stage,
+            "actual_failed_node": failure_context.get("actual_failed_node"),
+            "failure_kind": failure_kind,
+        },
+    )
+
+
+_STORY_CHURN_THRESHOLD = 3
+
+
+def _normalize_story_failure_node(node_id: str | None) -> str | None:
+    raw = str(node_id or "").strip()
+    if not raw:
+        return None
+    if "." in raw:
+        raw = raw.split(".", 1)[0]
+    return raw
+
+
+def _story_actual_failed_node(*, failure_context: dict[str, Any] | None, failed_stage: str | None = None) -> str | None:
+    ctx = failure_context if isinstance(failure_context, dict) else {}
+    return _normalize_story_failure_node(
+        str(ctx.get("actual_failed_node") or "").strip()
+        or str(failed_stage or "").strip()
+        or str(ctx.get("failed_stage") or "").strip()
+    )
+
+
+def _build_story_churn_state(
+    *,
+    story_id: str,
+    prior_failure_context: dict[str, Any] | None,
+    failure_context: dict[str, Any] | None,
+    failed_stage: str | None,
+) -> dict[str, Any]:
+    current_node = _story_actual_failed_node(failure_context=failure_context, failed_stage=failed_stage) or "unknown"
+    prior_ctx = prior_failure_context if isinstance(prior_failure_context, dict) else {}
+    prior_state = prior_ctx.get("churn_state") if isinstance(prior_ctx.get("churn_state"), dict) else {}
+    prior_story_id = str(prior_state.get("story_id") or story_id).strip() or story_id
+    prior_node = _normalize_story_failure_node(
+        str(prior_state.get("failing_node") or "").strip()
+        or str(prior_ctx.get("actual_failed_node") or "").strip()
+        or str(prior_ctx.get("failed_stage") or "").strip()
+    )
+
+    count = 1
+    if prior_story_id == story_id and prior_node == current_node:
+        count = int(prior_state.get("same_story_same_node_failures") or 0) + 1
+
+    return {
+        "story_id": story_id,
+        "failing_node": current_node,
+        "same_story_same_node_failures": count,
+        "threshold": _STORY_CHURN_THRESHOLD,
+        "blocked_for_churn": count >= _STORY_CHURN_THRESHOLD,
+        "requires_human_intervention": count >= _STORY_CHURN_THRESHOLD,
+    }
+
+
+def _peek_next_queue_type(*, project_id: str, store: ExecutionStore) -> str | None:
+    with store._connect() as conn:
+        recovery_row = store._select_active_recovery_queue_row(
+            conn,
+            project_id=project_id,
+            statuses=("queued", "claimed", "in_progress", "blocked"),
+            columns="rq.recovery_queue_id",
+        )
+        if recovery_row is not None:
+            return 'recovery'
+        error_row = conn.execute(
+            "SELECT error_task_id FROM error_tasks WHERE project_id=? AND status IN ('open','triaged') ORDER BY created_at ASC LIMIT 1",
+            (project_id,),
+        ).fetchone()
+        if error_row is not None:
+            return 'error'
+        scope_row = conn.execute(
+            "SELECT scope_queue_id FROM scope_queue WHERE project_id=? AND status='queued' ORDER BY created_at ASC LIMIT 1",
+            (project_id,),
+        ).fetchone()
+        if scope_row is not None:
+            return 'scope'
+        idea_creation_row = conn.execute(
+            "SELECT idea_creation_queue_id FROM idea_creation_queue WHERE project_id=? AND status='queued' ORDER BY created_at ASC LIMIT 1",
+            (project_id,),
+        ).fetchone()
+        if idea_creation_row is not None:
+            return 'idea_creation'
+        idea_row = conn.execute(
+            "SELECT idea_queue_id FROM idea_queue WHERE project_id=? AND status='queued' ORDER BY created_at ASC LIMIT 1",
+            (project_id,),
+        ).fetchone()
+        if idea_row is not None:
+            return 'idea'
+        source_doc_mutation_row = conn.execute(
+            "SELECT source_doc_mutation_queue_id FROM source_doc_mutation_queue WHERE project_id=? AND status='queued' ORDER BY created_at ASC LIMIT 1",
+            (project_id,),
+        ).fetchone()
+        if source_doc_mutation_row is not None:
+            return 'source_doc_mutation'
+        story_row = conn.execute(
+            "SELECT story_queue_id FROM story_queue WHERE project_id=? AND status='queued' ORDER BY created_at ASC LIMIT 1",
+            (project_id,),
+        ).fetchone()
+        if story_row is not None:
+            return 'story'
+        integration_row = conn.execute(
+            "SELECT integration_queue_id FROM integration_queue WHERE project_id=? AND status='queued' ORDER BY created_at ASC LIMIT 1",
+            (project_id,),
+        ).fetchone()
+        if integration_row is not None:
+            return 'integration'
+    return None
+
+
+def _cleanup_failed_claim(
+    *,
+    project_id: str,
+    repo_root: Path,
+    store: ExecutionStore,
+    worker_id: str,
+    claimed: dict[str, Any] | None,
+    active_run_id: str | None,
+    exc: BaseException,
+) -> None:
+    if active_run_id is not None:
+        store.mark_run_finished(run_id=active_run_id, status="failed")
+    if claimed is not None:
+        message = str(exc) or repr(exc)
+        if isinstance(exc, (SystemExit, KeyboardInterrupt)):
+            message = exc.__class__.__name__ if not str(exc) else f"{exc.__class__.__name__}: {exc}"
+        failure_context: dict[str, Any] = {"error_type": exc.__class__.__name__, "error": message}
+        if isinstance(getattr(exc, "failure_context", None), dict):
+            failure_context.update(getattr(exc, "failure_context"))
+        if claimed["queue_type"] == "idea_creation":
+            try:
+                item = _load_idea_creation_payload(store=store, idea_creation_queue_id=str(claimed["item_id"]))
+                failure_context.update({"idea_id": str(item.get("idea_id") or ""), "idea_creation_queue_id": str(item.get("idea_creation_queue_id") or "")})
+            except Exception:
+                pass
+        if claimed["queue_type"] == "idea":
+            try:
+                item = _load_idea_payload(store=store, idea_queue_id=str(claimed["item_id"]))
+                failure_context.update({"idea_id": str(item.get("idea_id") or ""), "idea_queue_id": str(item.get("idea_queue_id") or "")})
+                idea_payload = Path(str(item.get("idea_payload_path") or ""))
+                if idea_payload.exists():
+                    idea_json = json.loads(idea_payload.read_text(encoding="utf-8"))
+                    ideation_output = idea_json.get("ideation_output") if isinstance(idea_json.get("ideation_output"), dict) else {}
+                    story_generation = ideation_output.get("story_generation") if isinstance(ideation_output.get("story_generation"), dict) else {}
+                    resume_cursor = story_generation.get("resume_cursor")
+                    if isinstance(resume_cursor, dict):
+                        failure_context["resume_cursor"] = resume_cursor
+            except Exception:
+                pass
+        if claimed["queue_type"] == "integration":
+            try:
+                item = _load_integration_payload(store=store, integration_queue_id=str(claimed["item_id"]))
+                failure_context.update({"idea_id": str(item.get("idea_id") or ""), "integration_queue_id": str(item.get("integration_queue_id") or "")})
+            except Exception:
+                pass
+        store.complete_project_queue_item(
+            project_id=project_id,
+            worker_id=worker_id,
+            queue_type=str(claimed["queue_type"]),
+            item_id=str(claimed["item_id"]),
+            status="failed",
+            current_run_id=active_run_id,
+            failure_message=message,
+            failure_context=failure_context,
+        )
+        if str(claimed["queue_type"]) in {"scope", "idea_creation", "idea", "story", "integration"}:
+            item_title = str(claimed["item_id"])
+            try:
+                if str(claimed["queue_type"]) == "scope":
+                    source_item = _load_scope_payload(store=store, scope_queue_id=str(claimed["item_id"]))
+                    item_title = str(source_item.get("title") or source_item.get("scope_id") or claimed["item_id"])
+                elif str(claimed["queue_type"]) == "idea_creation":
+                    source_item = _load_idea_creation_payload(store=store, idea_creation_queue_id=str(claimed["item_id"]))
+                    item_title = str(source_item.get("title") or source_item.get("idea_id") or claimed["item_id"])
+                elif str(claimed["queue_type"]) == "idea":
+                    source_item = _load_idea_payload(store=store, idea_queue_id=str(claimed["item_id"]))
+                    item_title = str(source_item.get("title") or source_item.get("idea_id") or claimed["item_id"])
+                elif str(claimed["queue_type"]) == "integration":
+                    source_item = _load_integration_payload(store=store, integration_queue_id=str(claimed["item_id"]))
+                    item_title = str(source_item.get("title") or source_item.get("idea_id") or claimed["item_id"])
+                else:
+                    source_item, _story = _load_story_payload(store=store, story_queue_id=str(claimed["item_id"]))
+                    item_title = str(source_item.get("title") or source_item.get("story_id") or claimed["item_id"])
+            except Exception:
+                pass
+            _enqueue_recovery_request(
+                project_id=project_id,
+                repo_root=repo_root,
+                store=store,
+                source_queue_type=str(claimed["queue_type"]),
+                source_item_id=str(claimed["item_id"]),
+                enqueue_run_id=str(active_run_id or claimed.get("run_id") or ""),
+                title=item_title,
+                failure_message=message,
+                failure_context=failure_context,
+            )
+    store.stop_project_worker(project_id=project_id)
+
+
+
+
+def reconcile_stale_worker(*, project_id: str, repo_root: Path, store: ExecutionStore, reason: str = "worker process died before cleanup") -> dict[str, Any] | None:
+    report = store.get_project_worker_report(project_id=project_id)
+    worker_id = str(report.get("worker_id") or "").strip()
+    queue_type = str(report.get("active_queue_type") or "").strip() or None
+    item_id = str(report.get("active_item_id") or "").strip() or None
+    run_id = str(report.get("current_run_id") or "").strip() or None
+    status = str(report.get("status") or "").strip()
+    if not worker_id or not queue_type or not item_id or status not in {"running", "starting", "idle", "stopping"}:
+        return None
+
+    failure_context: dict[str, Any] = {
+        "error_type": "WorkerAbandonedError",
+        "error": reason,
+        "project_id": project_id,
+        "repo_root": str(repo_root),
+        "reconciled": True,
+    }
+    if run_id:
+        store.mark_run_finished(run_id=run_id, status="failed")
+    if queue_type == "idea_creation":
+        try:
+            item = _load_idea_creation_payload(store=store, idea_creation_queue_id=item_id)
+            failure_context.update({"idea_id": str(item.get("idea_id") or ""), "idea_creation_queue_id": str(item.get("idea_creation_queue_id") or "")})
+        except Exception:
+            pass
+    if queue_type == "idea":
+        try:
+            item = _load_idea_payload(store=store, idea_queue_id=item_id)
+            failure_context.update({"idea_id": str(item.get("idea_id") or ""), "idea_queue_id": str(item.get("idea_queue_id") or "")})
+            idea_payload = Path(str(item.get("idea_payload_path") or ""))
+            if idea_payload.exists():
+                idea_json = json.loads(idea_payload.read_text(encoding="utf-8"))
+                ideation_output = idea_json.get("ideation_output") if isinstance(idea_json.get("ideation_output"), dict) else {}
+                story_generation = ideation_output.get("story_generation") if isinstance(ideation_output.get("story_generation"), dict) else {}
+                failed_stage = story_generation.get("failed_stage")
+                if isinstance(failed_stage, str) and failed_stage.strip():
+                    failure_context["failed_stage"] = failed_stage
+                resume_cursor = story_generation.get("resume_cursor")
+                if isinstance(resume_cursor, dict):
+                    failure_context["resume_cursor"] = resume_cursor
+        except Exception:
+            pass
+    if queue_type == "integration":
+        try:
+            item = _load_integration_payload(store=store, integration_queue_id=item_id)
+            failure_context.update({"idea_id": str(item.get("idea_id") or ""), "integration_queue_id": str(item.get("integration_queue_id") or "")})
+        except Exception:
+            pass
+    store.complete_project_queue_item(
+        project_id=project_id,
+        worker_id=worker_id,
+        queue_type=queue_type,
+        item_id=item_id,
+        status="failed",
+        current_run_id=run_id,
+        failure_message=reason,
+        failure_context=failure_context,
+    )
+    if queue_type in {"scope", "idea_creation", "idea", "story", "integration"}:
+        item_title = item_id
+        try:
+            if queue_type == "scope":
+                source_item = _load_scope_payload(store=store, scope_queue_id=item_id)
+                item_title = str(source_item.get("title") or source_item.get("scope_id") or item_id)
+            elif queue_type == "idea_creation":
+                source_item = _load_idea_creation_payload(store=store, idea_creation_queue_id=item_id)
+                item_title = str(source_item.get("title") or source_item.get("idea_id") or item_id)
+            elif queue_type == "idea":
+                source_item = _load_idea_payload(store=store, idea_queue_id=item_id)
+                item_title = str(source_item.get("title") or source_item.get("idea_id") or item_id)
+            elif queue_type == "integration":
+                source_item = _load_integration_payload(store=store, integration_queue_id=item_id)
+                item_title = str(source_item.get("title") or source_item.get("idea_id") or item_id)
+            else:
+                source_item, _story = _load_story_payload(store=store, story_queue_id=item_id)
+                item_title = str(source_item.get("title") or source_item.get("story_id") or item_id)
+        except Exception:
+            pass
+        _enqueue_recovery_request(
+            project_id=project_id,
+            repo_root=repo_root,
+            store=store,
+            source_queue_type=queue_type,
+            source_item_id=item_id,
+            enqueue_run_id=str(run_id or worker_id),
+            title=item_title,
+            failure_message=reason,
+            failure_context=failure_context,
+        )
+    final_report = store.stop_project_worker(project_id=project_id)
+    publish_devflow_state(
+        project_id=project_id,
+        run_id=run_id,
+        current_state="failed",
+        current_status="failed",
+        run_summary=reason,
+        display="project",
+        display_path=f"queue:{queue_type}",
+    )
+    return {"reconciled": True, "queue_type": queue_type, "item_id": item_id, "run_id": run_id, "worker_id": worker_id, "report": final_report}
+
+def process_once(*, project_id: str, repo_root: Path, store: ExecutionStore) -> WorkerStartResult:
+    ensure_worker_start_allowed(repo_root=repo_root)
+    source_doc_tick = process_source_doc_update_once(store=store, repo_root=repo_root)
+    if source_doc_tick.task is not None:
+        return WorkerStartResult(
+            project_id=project_id,
+            worker_id=source_doc_tick.worker_id,
+            processed={"queue_type": "source_docs", "item_id": source_doc_tick.task["task_id"]},
+            report=store.get_project_worker_report(project_id=project_id),
+        )
+
+    worker_id = store.start_project_worker(project_id=project_id, repo_root=repo_root)
+    processed: dict[str, Any] | None = None
+    claimed: dict[str, Any] | None = None
+    active_run_id: str | None = None
+
+    try:
+        claimed = store.claim_next_project_queue_item(project_id=project_id, worker_id=worker_id)
+        if claimed is None:
+            report = store.stop_project_worker(project_id=project_id)
+            return WorkerStartResult(project_id=project_id, worker_id=worker_id, processed=None, report=report)
+
+        processed = {"queue_type": claimed["queue_type"], "item_id": claimed["item_id"]}
+        if claimed["queue_type"] == "recovery":
+            item = _load_recovery_payload(store=store, recovery_queue_id=str(claimed["item_id"]))
+            run = store.start_run(
+                kind="worker.recovery",
+                repo_root=repo_root,
+                args={
+                    "project_id": project_id,
+                    "recovery_queue_id": item["recovery_queue_id"],
+                    "source_queue_type": item["source_queue_type"],
+                    "source_item_id": item["source_item_id"],
+                },
+            )
+            active_run_id = run.run_id
+            store.update_project_worker_context(worker_id=worker_id, current_run_id=run.run_id, current_node_exec_id=None)
+            with store._connect() as conn:
+                conn.execute("UPDATE recovery_queue SET status='in_progress', started_run_id=?, updated_at=? WHERE recovery_queue_id=?", (run.run_id, int(time.time()), item["recovery_queue_id"]))
+            result = run_failure_recovery_dag(
+                repo_root=repo_root,
+                store=store,
+                project_id=project_id,
+                queue_type=str(item["source_queue_type"]),
+                item_id=str(item["source_item_id"]),
+                run_id=run.run_id,
+            )
+            final_status = "completed" if result.exit_code == 0 else "failed"
+            store.mark_run_finished(run_id=run.run_id, status="succeeded" if result.exit_code == 0 else "failed")
+            store.complete_project_queue_item(
+                project_id=project_id,
+                worker_id=worker_id,
+                queue_type="recovery",
+                item_id=str(claimed["item_id"]),
+                status=final_status,
+                current_run_id=run.run_id,
+                failure_message=None if result.exit_code == 0 else result.message,
+            )
+        elif claimed["queue_type"] == "error":
+            result = store.run_error_solver_dag(
+                error_task_id=str(claimed["item_id"]),
+                repo_root=repo_root,
+                build_observability_fn=lambda ctx: build_runtime_observability(
+                    repo_root=repo_root,
+                    runtime_context=ctx.get("runtimeContext") if isinstance(ctx.get("runtimeContext"), dict) else None,
+                    repro_command=None,
+                    project_id=project_id,
+                    store=store,
+                ),
+                first_pass_fix_fn=lambda _ctx: {"fixed": False},
+                iterative_fix_fn=lambda _ctx: {"fixed": False},
+                needs_user_input_fn=lambda _ctx: True,
+                runtime_context=None,
+            )
+            outcome = str(result.get("outcome") or "blocked")
+            store.complete_project_queue_item(project_id=project_id, worker_id=worker_id, queue_type="error", item_id=str(claimed["item_id"]), status=outcome, current_run_id=str(claimed.get("run_id") or "") or None)
+        elif claimed["queue_type"] == "scope":
+            item = _load_scope_payload(store=store, scope_queue_id=str(claimed["item_id"]))
+            run = store.start_run(kind="worker.scope_to_idea", repo_root=repo_root, args={"project_id": project_id, "scope_queue_id": item["scope_queue_id"], "scope_id": item["scope_id"]})
+            active_run_id = run.run_id
+            store.update_project_worker_context(worker_id=worker_id, current_run_id=run.run_id, current_node_exec_id=None)
+            with store._connect() as conn:
+                conn.execute("UPDATE scope_queue SET status='in_progress', started_run_id=?, updated_at=? WHERE scope_queue_id=?", (run.run_id, int(time.time()), item["scope_queue_id"]))
+            result = run_scope_to_idea_dag(repo_root=repo_root, store=store, project_id=project_id, scope_set_id=str(item.get("scope_set_id") or ""), scope_id=str(item["scope_id"]), scope_payload_path=Path(str(item["scope_payload_path"])))
+            final_status = "completed" if result.exit_code == 0 else "failed"
+            store.mark_run_finished(run_id=run.run_id, status="succeeded" if result.exit_code == 0 else "failed")
+            store.complete_project_queue_item(project_id=project_id, worker_id=worker_id, queue_type="scope", item_id=str(claimed["item_id"]), status=final_status, current_run_id=run.run_id, failure_message=None if result.exit_code == 0 else result.message)
+            if final_status == "failed":
+                _enqueue_recovery_request(
+                    project_id=project_id,
+                    repo_root=repo_root,
+                    store=store,
+                    source_queue_type="scope",
+                    source_item_id=str(claimed["item_id"]),
+                    enqueue_run_id=run.run_id,
+                    title=str(item.get("title") or item.get("scope_id") or claimed["item_id"]),
+                    failure_message=result.message,
+                    failure_context=None,
+                )
+        elif claimed["queue_type"] == "idea_creation":
+            item = _load_idea_creation_payload(store=store, idea_creation_queue_id=str(claimed["item_id"]))
+            run = store.start_run(kind="worker.idea_creation", repo_root=repo_root, args={"project_id": project_id, "idea_creation_queue_id": item["idea_creation_queue_id"], "idea_id": item["idea_id"]})
+            active_run_id = run.run_id
+            store.update_project_worker_context(worker_id=worker_id, current_run_id=run.run_id, current_node_exec_id=None)
+            with store._connect() as conn:
+                conn.execute("UPDATE idea_creation_queue SET status='in_progress', started_run_id=?, updated_at=? WHERE idea_creation_queue_id=?", (run.run_id, int(time.time()), item["idea_creation_queue_id"]))
+            result = run_idea_creation_dag(repo_root=repo_root, store=store, idea_id=str(item["idea_id"]), project_id=project_id, text=None, source_path=Path(str(item["idea_payload_path"])))
+            final_status = "completed" if result.exit_code == 0 else "failed"
+            store.mark_run_finished(run_id=run.run_id, status="succeeded" if result.exit_code == 0 else "failed")
+            store.complete_project_queue_item(project_id=project_id, worker_id=worker_id, queue_type="idea_creation", item_id=str(claimed["item_id"]), status=final_status, current_run_id=run.run_id, failure_message=None if result.exit_code == 0 else result.message)
+            if final_status == "failed":
+                _enqueue_recovery_request(
+                    project_id=project_id,
+                    repo_root=repo_root,
+                    store=store,
+                    source_queue_type="idea_creation",
+                    source_item_id=str(claimed["item_id"]),
+                    enqueue_run_id=run.run_id,
+                    title=str(item.get("title") or item.get("idea_id") or claimed["item_id"]),
+                    failure_message=result.message,
+                    failure_context=None,
+                )
+        elif claimed["queue_type"] == "idea":
+            item = _load_idea_payload(store=store, idea_queue_id=str(claimed["item_id"]))
+            run = store.start_run(kind="worker.idea_to_story", repo_root=repo_root, args={"project_id": project_id, "idea_queue_id": item["idea_queue_id"], "idea_id": item["idea_id"]})
+            active_run_id = run.run_id
+            store.update_project_worker_context(worker_id=worker_id, current_run_id=run.run_id, current_node_exec_id=None)
+            with store._connect() as conn:
+                conn.execute("UPDATE idea_queue SET status='in_progress', started_run_id=?, updated_at=? WHERE idea_queue_id=?", (run.run_id, int(time.time()), item["idea_queue_id"]))
+            result = run_idea_to_devflow_stories_dag(
+                repo_root=repo_root,
+                store=store,
+                idea_id=str(item["idea_id"]),
+                text=None,
+                source_path=Path(str(item["idea_payload_path"])),
+                max_stories=0,
+                planes=list(item.get("candidate_planes") or []),
+            )
+            final_status = "completed" if result.exit_code == 0 else "failed"
+            store.mark_run_finished(run_id=run.run_id, status="succeeded" if result.exit_code == 0 else "failed")
+            store.complete_project_queue_item(project_id=project_id, worker_id=worker_id, queue_type="idea", item_id=str(claimed["item_id"]), status=final_status, current_run_id=run.run_id, failure_message=None if result.exit_code == 0 else result.message)
+            if final_status == "failed":
+                _enqueue_recovery_request(
+                    project_id=project_id,
+                    repo_root=repo_root,
+                    store=store,
+                    source_queue_type="idea",
+                    source_item_id=str(claimed["item_id"]),
+                    enqueue_run_id=run.run_id,
+                    title=str(item.get("title") or item.get("idea_id") or claimed["item_id"]),
+                    failure_message=result.message,
+                    failure_context=None,
+                )
+        elif claimed["queue_type"] == "source_doc_mutation":
+            item = _load_source_doc_mutation_payload(store=store, source_doc_mutation_queue_id=str(claimed["item_id"]))
+            run = store.start_run(kind="worker.source_doc_mutation", repo_root=repo_root, args={"project_id": project_id, "source_doc_mutation_queue_id": item["source_doc_mutation_queue_id"], "idea_id": item.get("idea_id")})
+            active_run_id = run.run_id
+            store.update_project_worker_context(worker_id=worker_id, current_run_id=run.run_id, current_node_exec_id=None)
+            with store._connect() as conn:
+                conn.execute("UPDATE source_doc_mutation_queue SET status='in_progress', started_run_id=?, updated_at=? WHERE source_doc_mutation_queue_id=?", (run.run_id, int(time.time()), item["source_doc_mutation_queue_id"]))
+            result = run_source_doc_mutation_dag(request=dict(item.get("mutation_request") or {}))
+            final_status = "completed"
+            store.mark_run_finished(run_id=run.run_id, status="succeeded")
+            store.complete_project_queue_item(project_id=project_id, worker_id=worker_id, queue_type="source_doc_mutation", item_id=str(claimed["item_id"]), status=final_status, current_run_id=run.run_id)
+        elif claimed["queue_type"] == "integration":
+            item = _load_integration_payload(store=store, integration_queue_id=str(claimed["item_id"]))
+            from .integration.dag import prepare_integration_payload
+            run = store.start_run(
+                kind="worker.integration",
+                repo_root=repo_root,
+                args={
+                    "project_id": project_id,
+                    "integration_queue_id": item["integration_queue_id"],
+                    "idea_id": item["idea_id"],
+                },
+            )
+            active_run_id = run.run_id
+            store.update_project_worker_context(worker_id=worker_id, current_run_id=run.run_id, current_node_exec_id=None)
+            with store._connect() as conn:
+                conn.execute(
+                    "UPDATE integration_queue SET status='in_progress', started_run_id=?, updated_at=? WHERE integration_queue_id=?",
+                    (run.run_id, int(time.time()), item["integration_queue_id"]),
+                )
+            publish_devflow_state(
+                project_id=project_id,
+                run_id=run.run_id,
+                current_state="running",
+                current_status="processing",
+                run_summary="Integrating",
+                display="project",
+                display_path=f'idea:{item["idea_id"]}',
+            )
+            from .integration.dag import _sync_integration_to_supabase, run_integration_dag
+            _sync_integration_to_supabase(
+                idea_id=str(item["idea_id"]),
+                project_id=project_id,
+                run_id=run.run_id,
+                repo_root=repo_root,
+                result=None,
+                status="running",
+            )
+            payload_path, payload = _resolve_integration_payload_path(repo_root=repo_root, item=item)
+            if payload_path is None or payload is None:
+                payload_path = prepare_integration_payload(repo_root=repo_root, idea_id=str(item["idea_id"]))
+                payload = json.loads(payload_path.read_text(encoding="utf-8"))
+            result = run_integration_dag(
+                repo_root=repo_root,
+                idea_id=str(item["idea_id"]),
+                implemented_idea=payload.get("implemented_idea", {}),
+                implemented_stories=payload.get("implemented_stories", []),
+                code_evidence=payload.get("code_evidence"),
+                source_docs=payload.get("source_docs"),
+                project_id=project_id,
+                dag_run_id=run.run_id,
+            )
+            final_status = "completed" if result.exit_code == 0 else "failed"
+            store.mark_run_finished(run_id=run.run_id, status="succeeded" if result.exit_code == 0 else "failed")
+            publish_devflow_state(
+                project_id=project_id,
+                run_id=run.run_id,
+                current_state="idle",
+                current_status="completed" if result.exit_code == 0 else "failed",
+                run_summary="Integration complete" if result.exit_code == 0 else "Integration failed",
+                error_message=result.message if result.exit_code != 0 else None,
+                display="project",
+                display_path=f'idea:{item["idea_id"]}',
+            )
+            _sync_integration_to_supabase(
+                idea_id=str(item["idea_id"]),
+                project_id=project_id,
+                run_id=run.run_id,
+                repo_root=repo_root,
+                result=result,
+                status=final_status,
+            )
+            store.complete_project_queue_item(
+                project_id=project_id,
+                worker_id=worker_id,
+                queue_type="integration",
+                item_id=str(claimed["item_id"]),
+                status=final_status,
+                current_run_id=run.run_id,
+                failure_message=None if result.exit_code == 0 else result.message,
+            )
+            if final_status == "failed":
+                _enqueue_recovery_request(
+                    project_id=project_id,
+                    repo_root=repo_root,
+                    store=store,
+                    source_queue_type="integration",
+                    source_item_id=str(claimed["item_id"]),
+                    enqueue_run_id=run.run_id,
+                    title=str(item.get("title") or item.get("idea_id") or claimed["item_id"]),
+                    failure_message=result.message,
+                    failure_context={"idea_id": str(item.get("idea_id") or ""), "integration_queue_id": str(item.get("integration_queue_id") or "")},
+                )
+        else:
+            item, story = _load_story_payload(store=store, story_queue_id=str(claimed["item_id"]))
+            run = store.start_run(
+                kind="worker.story.execute",
+                repo_root=repo_root,
+                args={"project_id": project_id, "story_queue_id": item["story_queue_id"], "story_id": item["story_id"]},
+            )
+            active_run_id = run.run_id
+            store.update_project_worker_context(worker_id=worker_id, current_run_id=run.run_id, current_node_exec_id=None)
+            with store._connect() as conn:
+                conn.execute(
+                    "UPDATE story_queue SET status='in_progress', started_run_id=?, updated_at=? WHERE story_queue_id=?",
+                    (run.run_id, int(time.time()), item["story_queue_id"]),
+                )
+            publish_devflow_state(
+                project_id=project_id,
+                run_id=run.run_id,
+                current_state="running",
+                current_status="processing",
+                run_summary="Implementing",
+                display="project",
+                display_path=f'story:{item["story_id"]}',
+            )
+
+            result = process_dag.run_process_dag(repo_root=repo_root, store=store, story=story)
+            final_status = "completed" if result.exit_code == 0 else "failed"
+            failure_context = None
+            if result.exit_code == 0:
+                store.update_story_status(story_id=str(item["story_id"]), status="implemented")
+                from devflow_engine.devflow_state import update_story_status_in_supabase
+                update_story_status_in_supabase(story_id=str(item["story_id"]), status="implemented")
+            else:
+                result_failure_context = getattr(result, "failure_context", None)
+                failure_context = dict(result_failure_context) if isinstance(result_failure_context, dict) else {}
+                prior_failure_context = item.get("failure_context") if isinstance(item.get("failure_context"), dict) else {}
+                failed_stage = getattr(result, "failed_stage", None)
+                implementation_run_id = getattr(result, "implementation_run_id", None)
+                if isinstance(failed_stage, str) and failed_stage.strip():
+                    failure_context.setdefault("failed_stage", failed_stage)
+                if isinstance(implementation_run_id, str) and implementation_run_id.strip():
+                    failure_context.setdefault("implementation_run_id", implementation_run_id)
+                actual_failed_node = _story_actual_failed_node(failure_context=failure_context, failed_stage=failed_stage)
+                if actual_failed_node:
+                    failure_context["actual_failed_node"] = actual_failed_node
+                story_id = str(item.get("story_id") or story.get("story_id") or "").strip()
+                if story_id:
+                    failure_context["churn_state"] = _build_story_churn_state(
+                        story_id=story_id,
+                        prior_failure_context=prior_failure_context,
+                        failure_context=failure_context,
+                        failed_stage=failed_stage,
+                    )
+                if isinstance(story.get("replay"), dict):
+                    failure_context["prior_replay"] = dict(story["replay"])
+                churn_state = failure_context.get("churn_state") if isinstance(failure_context.get("churn_state"), dict) else {}
+                if bool(churn_state.get("blocked_for_churn")):
+                    failure_context["queue_status"] = "blocked"
+                    failure_context["recovery_disposition"] = "blocked_for_churn"
+                    final_status = "blocked"
+                elif str(failure_context.get("recovery_disposition") or "") == "blocked_at_redreview":
+                    final_status = "blocked"
+                elif _implementation_failure_routes_to_error_solving(failure_context):
+                    error_task_id = _create_error_task_for_implementation_failure(
+                        store=store,
+                        project_id=project_id,
+                        run_id=run.run_id,
+                        story_queue_id=str(claimed["item_id"]),
+                        story_id=str(item.get("story_id") or story.get("story_id") or ""),
+                        failure_message=result.message,
+                        failure_context=failure_context,
+                    )
+                    failure_context["error_task_id"] = error_task_id
+                    failure_context["handoff_lane"] = "error_solving"
+                    failure_context["recovery_disposition"] = "delegated_to_error_solving"
+            store.mark_run_finished(run_id=run.run_id, status="succeeded" if result.exit_code == 0 else "failed")
+            publish_devflow_state(
+                project_id=project_id,
+                run_id=run.run_id,
+                current_state="idle",
+                current_status="completed" if result.exit_code == 0 else ("blocked" if final_status == "blocked" else "failed"),
+                run_summary="Implementation complete" if result.exit_code == 0 else ("Implementation blocked" if final_status == "blocked" else "Implementation failed"),
+                error_message=result.message if result.exit_code != 0 else None,
+                display="project",
+                display_path=f'story:{item["story_id"]}',
+            )
+            store.complete_project_queue_item(
+                project_id=project_id,
+                worker_id=worker_id,
+                queue_type="story",
+                item_id=str(claimed["item_id"]),
+                status=final_status,
+                current_run_id=run.run_id,
+                failure_message=None if result.exit_code == 0 else result.message,
+                failure_context=failure_context,
+            )
+            if final_status == "failed" and not _implementation_failure_routes_to_error_solving(failure_context):
+                _enqueue_recovery_request(
+                    project_id=project_id,
+                    repo_root=repo_root,
+                    store=store,
+                    source_queue_type="story",
+                    source_item_id=str(claimed["item_id"]),
+                    enqueue_run_id=run.run_id,
+                    title=str(item.get("title") or item.get("story_id") or claimed["item_id"]),
+                    failure_message=result.message,
+                    failure_context=failure_context,
+                )
+
+        report = store.stop_project_worker(project_id=project_id)
+        return WorkerStartResult(project_id=project_id, worker_id=worker_id, processed=processed, report=report)
+    except BaseException as exc:
+        _cleanup_failed_claim(
+            project_id=project_id,
+            repo_root=repo_root,
+            store=store,
+            worker_id=worker_id,
+            claimed=claimed,
+            active_run_id=active_run_id,
+            exc=exc,
+        )
+        raise
+
+
+def drain_project_queue(
+    *,
+    project_id: str,
+    repo_root: Path,
+    store: ExecutionStore,
+    queue_type: str | None = None,
+    max_items: int | None = None,
+) -> WorkerDrainResult:
+    ensure_worker_start_allowed(repo_root=repo_root)
+    processed: list[dict[str, Any]] = []
+    final_report = store.get_project_worker_report(project_id=project_id)
+    last_run_id: str | None = None
+
+    while True:
+        next_queue_type = _peek_next_queue_type(project_id=project_id, store=store)
+        if next_queue_type is None:
+            last_run_id = _maybe_run_post_integration_playwright_workflow(
+                project_id=project_id,
+                repo_root=repo_root,
+                store=store,
+                processed=processed,
+                last_run_id=last_run_id,
+            )
+            _publish_queue_boundary_state(project_id=project_id, run_id=last_run_id, queue_type=queue_type, next_queue_type=None)
+            return WorkerDrainResult(
+                project_id=project_id,
+                processed_count=len(processed),
+                processed=processed,
+                stopped_reason='empty',
+                final_report=final_report,
+            )
+        if queue_type is not None and next_queue_type != queue_type:
+            _publish_queue_boundary_state(project_id=project_id, run_id=last_run_id, queue_type=queue_type, next_queue_type=next_queue_type)
+            return WorkerDrainResult(
+                project_id=project_id,
+                processed_count=len(processed),
+                processed=processed,
+                stopped_reason=f'next_queue_type:{next_queue_type}',
+                final_report=final_report,
+            )
+        if max_items is not None and len(processed) >= max_items:
+            return WorkerDrainResult(
+                project_id=project_id,
+                processed_count=len(processed),
+                processed=processed,
+                stopped_reason='max_items',
+                final_report=final_report,
+            )
+
+        # Clear any stale idle/stuck worker before claiming so process_once can
+        # register a fresh worker. This handles the case where a previous iteration
+        # left the worker in 'idle' state (claim returned None without stopping).
+        store.stop_project_worker(project_id=project_id)
+
+        try:
+            result = process_once(project_id=project_id, repo_root=repo_root, store=store)
+            final_report = result.report
+            last_run_id = str(final_report.get('current_run_id') or last_run_id or '') or last_run_id
+            if result.processed is None:
+                return WorkerDrainResult(
+                    project_id=project_id,
+                    processed_count=len(processed),
+                    processed=processed,
+                    stopped_reason='no_claim',
+                    final_report=final_report,
+                )
+            processed.append(result.processed)
+        except Exception as exc:
+            # Stop any stale worker so the next iteration can register cleanly.
+            store.stop_project_worker(project_id=project_id)
+            final_report = store.get_project_worker_report(project_id=project_id)
+            last_run_id = str(final_report.get('current_run_id') or last_run_id or '') or last_run_id
+            next_queue_type_after_failure = _peek_next_queue_type(project_id=project_id, store=store)
+            failure_record: dict[str, Any] = {
+                'queue_type': queue_type or str(final_report.get('active_queue_type') or 'unknown'),
+                'error': str(exc),
+            }
+            processed.append(failure_record)
+            if next_queue_type_after_failure is None:
+                last_run_id = _maybe_run_post_integration_playwright_workflow(
+                    project_id=project_id,
+                    repo_root=repo_root,
+                    store=store,
+                    processed=processed,
+                    last_run_id=last_run_id,
+                )
+                return WorkerDrainResult(
+                    project_id=project_id,
+                    processed_count=len(processed),
+                    processed=processed,
+                    stopped_reason='empty_after_failure',
+                    final_report=final_report,
+                )
+            if queue_type is not None and next_queue_type_after_failure != queue_type:
+                _publish_queue_boundary_state(project_id=project_id, run_id=last_run_id, queue_type=queue_type, next_queue_type=next_queue_type_after_failure)
+                return WorkerDrainResult(
+                    project_id=project_id,
+                    processed_count=len(processed),
+                    processed=processed,
+                    stopped_reason=f'next_queue_type:{next_queue_type_after_failure}',
+                    final_report=final_report,
+                )
+            import time as _time
+            _time.sleep(1)  # brief backoff before retrying to avoid tight CPU loops
+            continue