devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/scope_idea/dag.py

@@ -0,0 +1,1069 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import subprocess
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote
+from urllib.request import Request, urlopen
+
+from pydantic import BaseModel
+
+from ..devflow_state import publish_devflow_state
+from ..idea.paths import get_idea_paths
+from ..stores.execution_store import ExecutionStore
+from ..vendor.datalumina_genai.core.nodes.agent import AgentConfig, AgentNode
+from ..vendor.datalumina_genai.core.nodes.base import Node
+from ..vendor.datalumina_genai.core.nodes.router import BaseRouter, RouterNode
+from ..vendor.datalumina_genai.core.schema import NodeConfig, WorkflowSchema
+from ..vendor.datalumina_genai.core.task import TaskContext
+from ..vendor.datalumina_genai.core.workflow import Workflow
+from . import agentic as scope_idea_agentic
+from .models import (
+    ArtifactLineage,
+    GoldilocksAssessmentArtifact,
+    GoldilocksDecisionArtifact,
+    IdeaCandidateArtifact,
+    IdeaCandidateSetArtifact,
+    IdeaRegistryRecordArtifact,
+    IdeaResolutionPackageArtifact,
+    IdeaSplitPlanArtifact,
+    NarrowScopeReviewArtifact,
+    RegisteredIdeaArtifact,
+    ScopeContextArtifact,
+    ScopeIdeaDagSummary,
+    SourceEvidenceRef,
+)
+
+DAG_ID = "scope_to_idea_dag"
+
+_CURRENT_STORE: ExecutionStore | None = None
+_CURRENT_RUN_ID: str | None = None
+
+
+def _dfs_running(*, project_id: str, run_id: str, scope_id: str, summary: str = "Generating ideas from scopes") -> None:
+    publish_devflow_state(
+        project_id=project_id,
+        run_id=run_id,
+        current_state="running",
+        current_status="processing",
+        run_summary=summary,
+        display="project",
+        display_path=f"scope:{scope_id}",
+    )
+
+
+def _dfs_terminal(
+    *,
+    project_id: str,
+    run_id: str,
+    scope_id: str,
+    current_state: str,
+    current_status: str,
+    summary: str,
+    error_message: str | None = None,
+) -> None:
+    publish_devflow_state(
+        project_id=project_id,
+        run_id=run_id,
+        current_state=current_state,
+        current_status=current_status,
+        run_summary=summary,
+        error_message=error_message,
+        display="project",
+        display_path=f"scope:{scope_id}",
+    )
+
+
+def _keychain_get(service: str, account: str) -> str | None:
+    try:
+        proc = subprocess.run(
+            ["security", "find-generic-password", "-s", service, "-a", account, "-w"],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=10,
+        )
+    except Exception:
+        return None
+    if proc.returncode != 0:
+        return None
+    value = proc.stdout.strip()
+    return value or None
+
+
+def _resolve_supabase_rest_config() -> tuple[str, str] | None:
+    url = (
+        os.environ.get("DEVFLOW_SUPABASE_URL")
+        or os.environ.get("SUPABASE_URL")
+        or _keychain_get("Supabase URL", "Clarity")
+    )
+    key = (
+        os.environ.get("DEVFLOW_SUPABASE_SERVICE_KEY")
+        or os.environ.get("SUPABASE_SERVICE_ROLE_KEY")
+        or os.environ.get("SUPABASE_SERVICE_KEY")
+        or _keychain_get("Supabase Service Key", "Clarity")
+    )
+    if not url or not key:
+        return None
+    return url.rstrip("/"), key
+
+
+def _postgrest_request(*, method: str, url: str, key: str, body: Any | None = None, prefer: str | None = None) -> Any:
+    payload = None if body is None else json.dumps(body).encode("utf-8")
+    req = Request(url, data=payload, method=method)
+    req.add_header("apikey", key)
+    req.add_header("Authorization", f"Bearer {key}")
+    if body is not None:
+        req.add_header("Content-Type", "application/json")
+    if prefer:
+        req.add_header("Prefer", prefer)
+    with urlopen(req, timeout=30) as resp:
+        raw = resp.read().decode("utf-8")
+        return json.loads(raw) if raw else None
+
+
+def _sync_registered_ideas_to_supabase(*, project_id: str, scope_id: str, run_id: str, pipeline_dir: Path) -> None:
+    config = _resolve_supabase_rest_config()
+    if config is None:
+        return
+    url, key = config
+    registry_path = pipeline_dir / "idea_registry_record.json"
+    if not registry_path.exists():
+        return
+    registry = json.loads(registry_path.read_text(encoding="utf-8"))
+    ideas = registry.get("registered_ideas") or []
+    rows: list[dict[str, Any]] = []
+    for idea in ideas:
+        if not isinstance(idea, dict) or not idea.get("idea_id"):
+            continue
+        persisted: dict[str, Any] = {}
+        idea_ref = idea.get("idea_ref")
+        if isinstance(idea_ref, str) and Path(idea_ref).exists():
+            persisted = json.loads(Path(idea_ref).read_text(encoding="utf-8"))
+        payload = persisted.get("idea") if isinstance(persisted.get("idea"), dict) else {}
+        rows.append(
+            {
+                "idea_id": str(idea.get("idea_id")),
+                "project_id": project_id,
+                "scope_set_id": registry.get("scope_set_id") or persisted.get("scope_set_id"),
+                "scope_id": scope_id,
+                "run_id": run_id,
+                "title": str(
+                    idea.get("title")
+                    or persisted.get("title")
+                    or payload.get("title")
+                    or payload.get("goal")
+                    or payload.get("problem")
+                    or idea.get("idea_id")
+                ),
+                "summary": (
+                    idea.get("summary")
+                    or persisted.get("summary")
+                    or payload.get("summary")
+                    or payload.get("goal")
+                    or payload.get("problem")
+                ),
+                "status": idea.get("status") or persisted.get("status"),
+                "shape": idea.get("shape") or registry.get("scope_shape"),
+                "resolution_status": registry.get("resolution_status"),
+                "origin": "scope_to_idea",
+                "artifact_path": str(idea_ref or registry_path),
+                "updated_at": datetime.now(UTC).isoformat(),
+            }
+        )
+    delete_url = f"{url}/rest/v1/devflow_project_ideas?project_id=eq.{quote(project_id)}&scope_id=eq.{quote(scope_id)}"
+    _postgrest_request(method="DELETE", url=delete_url, key=key)
+    if rows:
+        upsert_url = f"{url}/rest/v1/devflow_project_ideas?on_conflict=idea_id"
+        _postgrest_request(method="POST", url=upsert_url, key=key, body=rows, prefer="resolution=merge-duplicates")
+
+
+@dataclass(frozen=True)
+class ScopeToIdeaDagResult:
+    exit_code: int
+    run_id: str
+    pipeline_dir: Path
+    message: str
+    outcome: dict[str, Any]
+
+
+class ScopeToIdeaDagEvent(BaseModel):
+    repo_root: str
+    project_id: str
+    scope_set_id: str
+    scope_id: str
+    scope_payload_path: str | None = None
+    scope_payload_inline: dict[str, Any] | None = None
+    pipeline_key: str
+
+
+def _store_run() -> tuple[ExecutionStore, str]:
+    if _CURRENT_STORE is None or _CURRENT_RUN_ID is None:
+        raise RuntimeError("scope->idea dag missing runtime store/run_id")
+    return _CURRENT_STORE, _CURRENT_RUN_ID
+
+
+def _stable_hash(payload: Any) -> str:
+    return hashlib.sha256(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest()
+
+
+def _stable_id(prefix: str, payload: Any, *, size: int = 12) -> str:
+    return f"{prefix}{_stable_hash(payload)[:size]}"
+
+
+def _write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _append_node_trace(*, pipeline_root: Path, node: str, event: str, detail: dict[str, Any] | None = None) -> None:
+    path = pipeline_root / "node_trace.jsonl"
+    record = {
+        "ts": datetime.now(UTC).isoformat(),
+        "node": node,
+        "event": event,
+    }
+    if detail:
+        record["detail"] = detail
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(record, sort_keys=True) + "\n")
+
+
+def _agent_timeout_seconds() -> int:
+    raw = os.environ.get("DEVFLOW_SCOPE_IDEA_AGENT_TIMEOUT")
+    if not raw:
+        return 300
+    try:
+        value = int(raw)
+    except ValueError:
+        return 300
+    return value if value > 0 else 300
+
+
+def _relative_to_repo(repo_root: Path, path: Path) -> str:
+    try:
+        return str(path.relative_to(repo_root))
+    except ValueError:
+        return str(path)
+
+
+def _lineage_entry(
+    *,
+    stage: str,
+    origin: str,
+    mode: str,
+    repo_root: Path,
+    artifact_path: Path | None = None,
+    agent_run_path: Path | None = None,
+    generated_from: list[str] | None = None,
+    notes: list[str] | None = None,
+    metadata: dict[str, Any] | None = None,
+) -> ArtifactLineage:
+    return ArtifactLineage(
+        stage=stage,
+        origin=origin,  # type: ignore[arg-type]
+        mode=mode,
+        artifact_path=_relative_to_repo(repo_root, artifact_path) if artifact_path else None,
+        agent_run_ref=_relative_to_repo(repo_root, agent_run_path) if agent_run_path else None,
+        generated_from=generated_from or [],
+        notes=notes or [],
+        metadata=metadata or {},
+    )
+
+
+def _run_scope_idea_agent(
+    *,
+    repo_root: Path,
+    pipeline_root: Path,
+    stage_name: str,
+    output_model: type[BaseModel],
+    context_payload: dict[str, Any],
+    guidance: list[str],
+) -> tuple[BaseModel, ArtifactLineage]:
+    artifact, envelope = scope_idea_agentic.run_scope_idea_agent_step(
+        repo_root=repo_root,
+        stage_name=stage_name,
+        output_model=output_model,
+        context_payload=context_payload,
+        guidance=guidance,
+        timeout_seconds=_agent_timeout_seconds(),
+    )
+    agent_run_path = scope_idea_agentic.persist_agent_run(pipeline_root=pipeline_root, node_id=stage_name, envelope=envelope)
+    return artifact, _lineage_entry(
+        stage=stage_name,
+        origin="model",
+        mode="agent",
+        repo_root=repo_root,
+        agent_run_path=agent_run_path,
+        notes=["Artifact was produced from the model-backed execution path."],
+        metadata={"model_backed": True},
+    )
+
+
+def _pipeline_root(repo_root: Path, *, scope_id: str, pipeline_key: str) -> Path:
+    return repo_root / ".devflow" / "scopes" / scope_id / "pipelines" / DAG_ID / pipeline_key
+
+
+def build_pipeline_key(*, repo_root: Path, project_id: str, scope_set_id: str, scope_id: str, scope_payload: dict[str, Any]) -> str:
+    return _stable_id(
+        "run_",
+        {
+            "repo_root": str(repo_root),
+            "project_id": project_id,
+            "scope_set_id": scope_set_id,
+            "scope_id": scope_id,
+            "scope_payload": scope_payload,
+        },
+    )
+
+
+def _load_scope_payload(*, inline_payload: dict[str, Any] | None, payload_path: Path | None) -> dict[str, Any]:
+    if inline_payload and payload_path:
+        raise ValueError("Provide exactly one of scope_payload_inline or scope_payload_path")
+    if payload_path:
+        return json.loads(payload_path.read_text(encoding="utf-8"))
+    if inline_payload:
+        return dict(inline_payload)
+    raise ValueError("Missing scope payload for scope->idea DAG")
+
+
+def _scope_payload_item(payload: dict[str, Any]) -> dict[str, Any]:
+    scope_item = payload.get("scope_item")
+    if isinstance(scope_item, dict):
+        return scope_item
+    return payload
+
+
+def _coerce_source_support(items: Any) -> list[SourceEvidenceRef]:
+    if not isinstance(items, list):
+        return []
+    refs: list[SourceEvidenceRef] = []
+    for item in items:
+        if isinstance(item, dict):
+            refs.append(SourceEvidenceRef.model_validate(item))
+        elif isinstance(item, str) and item.strip():
+            refs.append(SourceEvidenceRef(ref=item.strip()))
+    return refs
+
+
+def _scope_context_from_payload(*, payload: dict[str, Any], project_id: str, scope_set_id: str, scope_id: str) -> ScopeContextArtifact:
+    scope_item = _scope_payload_item(payload)
+    scope_title = str(scope_item.get("title") or payload.get("scope_title") or payload.get("title") or scope_id).strip()
+    scope_description = str(scope_item.get("description") or payload.get("scope_description") or payload.get("description") or "").strip()
+    assumptions = scope_item.get("assumptions")
+    if not isinstance(assumptions, list):
+        assumptions = payload.get("assumptions")
+    constraints = scope_item.get("cross_cutting_constraints")
+    if not isinstance(constraints, list):
+        constraints = payload.get("cross_cutting_constraints")
+    neighbor_scope_refs = scope_item.get("neighbor_scope_refs")
+    if not isinstance(neighbor_scope_refs, list):
+        neighbor_scope_refs = payload.get("neighbor_scope_refs")
+    source_support = scope_item.get("source_support")
+    if not isinstance(source_support, list):
+        source_support = payload.get("source_support")
+
+    artifact = ScopeContextArtifact(
+        project_id=project_id,
+        scope_set_id=scope_set_id,
+        scope_id=scope_id,
+        scope_title=scope_title,
+        scope_description=scope_description,
+        source_support=_coerce_source_support(source_support),
+        assumptions=[str(item) for item in assumptions or []],
+        cross_cutting_constraints=[str(item) for item in constraints or []],
+        neighbor_scope_refs=[str(item) for item in neighbor_scope_refs or []],
+        approval_status=str(scope_item.get("approval_status") or payload.get("approval_status") or payload.get("status") or "approved"),
+    )
+    if not artifact.scope_title:
+        raise ValueError(f"Scope payload for {scope_id} is missing scope_item.title/title")
+    if not artifact.scope_description:
+        raise ValueError(f"Scope payload for {scope_id} is missing scope_item.description/description")
+    return artifact
+
+
+def _build_constraints(scope: ScopeContextArtifact, *, default_constraint: str) -> list[str]:
+    constraints = [item for item in scope.cross_cutting_constraints if item]
+    if default_constraint not in constraints:
+        constraints.append(default_constraint)
+    return constraints
+
+
+def _build_traceability(scope: ScopeContextArtifact, *, extras: list[str] | None = None) -> list[str]:
+    traceability = [
+        f"project:{scope.project_id}",
+        f"scope_set:{scope.scope_set_id}",
+        f"scope:{scope.scope_id}",
+    ]
+    for source_ref in scope.source_support:
+        traceability.append(f"source:{source_ref.ref}")
+    if extras:
+        traceability.extend(extras)
+    return traceability
+
+
+class LoadApprovedScopeItemNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        repo_root = Path(event.repo_root)
+        _dfs_running(project_id=event.project_id, run_id=_store_run()[1], scope_id=event.scope_id, summary="Loading approved scope")
+        payload = _load_scope_payload(
+            inline_payload=event.scope_payload_inline,
+            payload_path=Path(event.scope_payload_path) if event.scope_payload_path else None,
+        )
+        artifact = _scope_context_from_payload(
+            payload=payload,
+            project_id=event.project_id,
+            scope_set_id=event.scope_set_id,
+            scope_id=event.scope_id,
+        )
+        stage_path = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "scope_context.json"
+        _write_json(stage_path, artifact.model_dump())
+        task_context.metadata["scope_context"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class AssessScopeGoldilocksNode(AgentNode):
+    def get_agent_config(self) -> AgentConfig:
+        return AgentConfig(
+            instructions="Assess whether the approved scope is too broad, just right, or too narrow.",
+            output_type=GoldilocksAssessmentArtifact,
+        )
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        repo_root = Path(event.repo_root)
+        _dfs_running(project_id=event.project_id, run_id=_store_run()[1], scope_id=event.scope_id, summary="Assessing scope goldilocks")
+        pipeline_root = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key)
+        scope = task_context.metadata["scope_context"]
+        artifact, execution_lineage = _run_scope_idea_agent(
+            repo_root=repo_root,
+            pipeline_root=pipeline_root,
+            stage_name="assess_scope_goldilocks",
+            output_model=GoldilocksAssessmentArtifact,
+            context_payload={
+                "project_id": scope.project_id,
+                "scope_set_id": scope.scope_set_id,
+                "scope_id": scope.scope_id,
+                "approved_scope": scope.model_dump(),
+            },
+            guidance=[
+                "Assess whether the approved scope is too broad, just right, or too narrow for one idea candidate.",
+                "Treat the approved scope as the anchor. Do not rewrite or broaden its intent.",
+                "Set split_recommended true only when the approved scope clearly bundles multiple distinct business outcomes that a human would recognize as separate planning units.",
+                "Do not split merely because the scope contains multiple sub-workflows, metrics, linked entities, lifecycle steps, or supporting artifact types.",
+                "Should-not-split examples: customer management, reporting, documents and photos, and job management when they still describe one shared business slice.",
+                "Should-split example: quotes when internal composition and customer delivery/approval are clearly separate outcome surfaces.",
+                "Keep canonical idea sufficiency downstream; this node judges boundary size only.",
+            ],
+        )
+        artifact_data = artifact.model_dump()
+        notes = list(artifact_data.get("notes") or [])
+        notes.extend(execution_lineage.notes)
+        artifact_data["notes"] = list(dict.fromkeys(notes))
+        artifact = GoldilocksAssessmentArtifact.model_validate(artifact_data)
+        stage_path = pipeline_root / "goldilocks_assessment.json"
+        artifact.execution_lineage = execution_lineage.model_copy(
+            update={
+                "artifact_path": _relative_to_repo(repo_root, stage_path),
+                "generated_from": [_relative_to_repo(repo_root, pipeline_root / "scope_context.json")],
+            }
+        )
+        _write_json(stage_path, artifact.model_dump())
+        task_context.metadata["goldilocks_assessment"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class GoldilocksDecisionNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        repo_root = Path(event.repo_root)
+        _dfs_running(project_id=event.project_id, run_id=_store_run()[1], scope_id=event.scope_id, summary="Routing scope shape")
+        assessment = task_context.metadata["goldilocks_assessment"]
+        next_node = {
+            "too_broad": "SplitScopeIntoIdeaCandidatesNode",
+            "just_right": "DraftIdeaFromScopeNode",
+            "too_narrow": "NarrowScopeReviewPackageNode",
+        }[assessment.scope_shape]
+        artifact = GoldilocksDecisionArtifact(
+            decision=assessment.scope_shape,
+            reason=assessment.reasoning,
+            next_node=next_node,
+        )
+        stage_path = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "goldilocks_decision.json"
+        _write_json(stage_path, artifact.model_dump())
+        task_context.metadata["goldilocks_decision"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class _RouteTooBroad(RouterNode):
+    def determine_next_node(self, task_context: TaskContext) -> Node | None:
+        decision = task_context.metadata["goldilocks_decision"]
+        if decision.decision == "too_broad":
+            return SplitScopeIntoIdeaCandidatesNode(task_context=task_context)
+        return None
+
+
+class _RouteJustRight(RouterNode):
+    def determine_next_node(self, task_context: TaskContext) -> Node | None:
+        decision = task_context.metadata["goldilocks_decision"]
+        if decision.decision == "just_right":
+            return DraftIdeaFromScopeNode(task_context=task_context)
+        return None
+
+
+class _RouteTooNarrow(RouterNode):
+    def determine_next_node(self, task_context: TaskContext) -> Node | None:
+        decision = task_context.metadata["goldilocks_decision"]
+        if decision.decision == "too_narrow":
+            return NarrowScopeReviewPackageNode(task_context=task_context)
+        return None
+
+
+class GoldilocksDecisionRouter(BaseRouter):
+    def __init__(self) -> None:
+        self.routes = [_RouteTooBroad(), _RouteJustRight(), _RouteTooNarrow()]
+        self.fallback = NarrowScopeReviewPackageNode()
+
+
+class SplitScopeIntoIdeaCandidatesNode(AgentNode):
+    def get_agent_config(self) -> AgentConfig:
+        return AgentConfig(
+            instructions="Split broad approved scope into child idea candidates while preserving parent traceability.",
+            output_type=IdeaSplitPlanArtifact,
+        )
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        repo_root = Path(event.repo_root)
+        _dfs_running(project_id=event.project_id, run_id=_store_run()[1], scope_id=event.scope_id, summary="Splitting broad scope into ideas")
+        pipeline_root = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key)
+        scope = task_context.metadata["scope_context"]
+        assessment = task_context.metadata["goldilocks_assessment"]
+        artifact, execution_lineage = _run_scope_idea_agent(
+            repo_root=repo_root,
+            pipeline_root=pipeline_root,
+            stage_name="split_scope_into_idea_candidates",
+            output_model=IdeaSplitPlanArtifact,
+            context_payload={
+                "project_id": scope.project_id,
+                "scope_set_id": scope.scope_set_id,
+                "scope_id": scope.scope_id,
+                "approved_scope": scope.model_dump(),
+                "goldilocks_assessment": assessment.model_dump(),
+            },
+            guidance=[
+                "Split the approved parent scope into 2-3 coherent idea candidates only when the goldilocks assessment already says the scope is too broad.",
+                "Each child idea must stay fully inside the approved parent scope and preserve parent_scope_id plus traceability.",
+                "Do not silently create story-ready slices; produce idea candidates suitable for downstream idea sufficiency review.",
+                "Coverage_of_parent_scope should account for the parent scope bullets without drifting beyond them.",
+                "Preserve the assessment split rationale unless stronger evidence from the approved scope justifies a clearer phrasing of the same reason.",
+                "Do not split into foundation-vs-linkage ideas, metric-family ideas, or artifact-type ideas unless the approved scope itself clearly frames those as separate business outcomes.",
+                "If one child idea mostly exists to support another child idea, keep them together instead of splitting.",
+            ],
+        )
+        artifact_data = artifact.model_dump()
+        risks = list(artifact_data.get("remaining_risks") or [])
+        risks.extend(execution_lineage.notes)
+        artifact_data["remaining_risks"] = list(dict.fromkeys(risks))
+        artifact = IdeaSplitPlanArtifact.model_validate(artifact_data)
+        stage_path = pipeline_root / "idea_split_plan.json"
+        artifact.execution_lineage = execution_lineage.model_copy(
+            update={
+                "artifact_path": _relative_to_repo(repo_root, stage_path),
+                "generated_from": [
+                    _relative_to_repo(repo_root, pipeline_root / "scope_context.json"),
+                    _relative_to_repo(repo_root, pipeline_root / "goldilocks_assessment.json"),
+                ],
+            }
+        )
+        for child in artifact.child_ideas:
+            child.execution_lineage = artifact.execution_lineage
+        _write_json(stage_path, artifact.model_dump())
+        task_context.metadata["idea_split_plan"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class DraftIdeaFromScopeNode(AgentNode):
+    def get_agent_config(self) -> AgentConfig:
+        return AgentConfig(
+            instructions="Draft one idea from a Goldilocks-sized approved scope.",
+            output_type=IdeaCandidateArtifact,
+        )
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        repo_root = Path(event.repo_root)
+        _dfs_running(project_id=event.project_id, run_id=_store_run()[1], scope_id=event.scope_id, summary="Drafting idea from scope")
+        pipeline_root = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key)
+        _append_node_trace(pipeline_root=pipeline_root, node="DraftIdeaFromScopeNode", event="enter")
+        scope = task_context.metadata["scope_context"]
+        assessment = task_context.metadata["goldilocks_assessment"]
+        try:
+            artifact, execution_lineage = _run_scope_idea_agent(
+                repo_root=repo_root,
+                pipeline_root=pipeline_root,
+                stage_name="draft_idea_from_scope",
+                output_model=IdeaCandidateArtifact,
+                context_payload={
+                    "project_id": scope.project_id,
+                    "scope_set_id": scope.scope_set_id,
+                    "scope_id": scope.scope_id,
+                    "approved_scope": scope.model_dump(),
+                    "goldilocks_assessment": assessment.model_dump(),
+                },
+                guidance=[
+                    "Draft exactly one idea candidate from the approved scope when the goldilocks assessment says the scope is just_right.",
+                    "Keep the approved scope as the anchor; do not broaden or compress it into a different intent.",
+                    "Return a real idea candidate artifact with explicit problem, users, goal, scope bullets, constraints, acceptance criteria, assumptions, and traceability.",
+                    "Do not treat the result as canonically sufficient for stories; downstream idea sufficiency still decides that.",
+                    "Preserve refs and traceability back to the approved scope and source evidence.",
+                ],
+            )
+        except Exception as exc:
+            _append_node_trace(pipeline_root=pipeline_root, node="DraftIdeaFromScopeNode", event="failure", detail={"error": str(exc)})
+            raise
+        artifact = IdeaCandidateArtifact.model_validate(artifact.model_dump())
+        stage_path = pipeline_root / "idea_candidate.json"
+        artifact.execution_lineage = execution_lineage.model_copy(
+            update={
+                "artifact_path": _relative_to_repo(repo_root, stage_path),
+                "generated_from": [
+                    _relative_to_repo(repo_root, pipeline_root / "scope_context.json"),
+                    _relative_to_repo(repo_root, pipeline_root / "goldilocks_assessment.json"),
+                ],
+            }
+        )
+        _write_json(stage_path, artifact.model_dump())
+        _append_node_trace(pipeline_root=pipeline_root, node="DraftIdeaFromScopeNode", event="success", detail={"artifact": _relative_to_repo(repo_root, stage_path)})
+        task_context.metadata["idea_candidate"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class NarrowScopeReviewPackageNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        repo_root = Path(event.repo_root)
+        _dfs_running(project_id=event.project_id, run_id=_store_run()[1], scope_id=event.scope_id, summary="Packaging narrow-scope review")
+        scope = task_context.metadata["scope_context"]
+        assessment = task_context.metadata["goldilocks_assessment"]
+        stage_path = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "narrow_scope_review.json"
+        artifact = NarrowScopeReviewArtifact(
+            project_id=scope.project_id,
+            scope_set_id=scope.scope_set_id,
+            scope_id=scope.scope_id,
+            parent_scope_id=scope.scope_id,
+            refs=_build_traceability(scope),
+            reason_too_narrow=assessment.reasoning,
+            suggested_merge_targets=scope.neighbor_scope_refs,
+            recommended_next_action="Request human review or explicit merge recommendation; do not auto-merge.",
+            execution_lineage=_lineage_entry(
+                stage="narrow_scope_review",
+                origin="deterministic",
+                mode="deterministic",
+                repo_root=repo_root,
+                artifact_path=stage_path,
+                generated_from=[
+                    _relative_to_repo(repo_root, _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "scope_context.json"),
+                    _relative_to_repo(repo_root, _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "goldilocks_assessment.json"),
+                ],
+                notes=["Review package is deterministic packaging of a too-narrow boundary decision."],
+                metadata={"model_backed": False},
+            ),
+        )
+        _write_json(stage_path, artifact.model_dump())
+        task_context.metadata["narrow_scope_review"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class NormalizeIdeaCandidateSetNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        repo_root = Path(event.repo_root)
+        _dfs_running(project_id=event.project_id, run_id=_store_run()[1], scope_id=event.scope_id, summary="Normalizing idea candidate set")
+        pipeline_root = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key)
+        _append_node_trace(pipeline_root=pipeline_root, node="NormalizeIdeaCandidateSetNode", event="enter")
+        scope = task_context.metadata["scope_context"]
+        split_plan = task_context.metadata.get("idea_split_plan")
+        direct_candidate = task_context.metadata.get("idea_candidate")
+        pipeline_root = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key)
+        if split_plan is not None:
+            artifact = IdeaCandidateSetArtifact(
+                project_id=scope.project_id,
+                scope_set_id=scope.scope_set_id,
+                scope_id=scope.scope_id,
+                parent_scope_id=scope.scope_id,
+                refs=_build_traceability(scope),
+                idea_candidates=split_plan.child_ideas,
+                candidate_count=len(split_plan.child_ideas),
+                split_applied=True,
+                split_rationale=split_plan.split_rationale,
+                lineage=[split_plan.execution_lineage] if split_plan.execution_lineage else [],
+            )
+        else:
+            candidates = [direct_candidate] if direct_candidate is not None else []
+            artifact = IdeaCandidateSetArtifact(
+                project_id=scope.project_id,
+                scope_set_id=scope.scope_set_id,
+                scope_id=scope.scope_id,
+                parent_scope_id=scope.scope_id,
+                refs=_build_traceability(scope),
+                idea_candidates=candidates,
+                candidate_count=len(candidates),
+                split_applied=False,
+                split_rationale=None,
+                lineage=[direct_candidate.execution_lineage] if direct_candidate is not None and direct_candidate.execution_lineage else [],
+            )
+        stage_path = pipeline_root / "idea_candidate_set.json"
+        artifact.lineage.append(
+            _lineage_entry(
+                stage="normalize_idea_candidate_set",
+                origin="deterministic",
+                mode="deterministic",
+                repo_root=repo_root,
+                artifact_path=stage_path,
+                generated_from=[line.artifact_path for line in artifact.lineage if line and line.artifact_path],
+                notes=["Candidate set normalizes one-vs-many shaping results without changing their origin."],
+                metadata={"model_backed": False},
+            )
+        )
+        _write_json(stage_path, artifact.model_dump())
+        _append_node_trace(pipeline_root=pipeline_root, node="NormalizeIdeaCandidateSetNode", event="success", detail={"artifact": _relative_to_repo(repo_root, stage_path), "candidate_count": artifact.candidate_count})
+        task_context.metadata["idea_candidate_set"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class RegisterIdeasNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        repo_root = Path(event.repo_root)
+        _dfs_running(project_id=event.project_id, run_id=_store_run()[1], scope_id=event.scope_id, summary="Registering ideas")
+        pipeline_root = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key)
+        _append_node_trace(pipeline_root=pipeline_root, node="RegisterIdeasNode", event="enter")
+        scope = task_context.metadata["scope_context"]
+        candidate_set = task_context.metadata["idea_candidate_set"]
+        registry_root = repo_root / ".devflow" / "ideas"
+        registry_root.mkdir(parents=True, exist_ok=True)
+        registered: list[RegisteredIdeaArtifact] = []
+        assessment = task_context.metadata["goldilocks_assessment"]
+        for candidate in candidate_set.idea_candidates:
+            idea_id = _stable_id("idea_", {"scope_id": scope.scope_id, "candidate": candidate.model_dump()})
+            idea_paths = get_idea_paths(repo_root, idea_id=idea_id)
+            idea_paths.idea_dir.mkdir(parents=True, exist_ok=True)
+            registry_path = idea_paths.idea_dir / "idea.json"
+            candidate_lineage = candidate.execution_lineage
+            if candidate_lineage is None:
+                raise ValueError(f"Idea candidate {candidate.idea_candidate_id} is missing execution lineage")
+            summary = candidate.goal.strip() or candidate.problem.strip() or candidate.title.strip() or idea_id
+            persisted = {
+                "idea_id": idea_id,
+                "project_id": scope.project_id,
+                "scope_set_id": scope.scope_set_id,
+                "source_scope_id": scope.scope_id,
+                "parent_scope_id": candidate.parent_scope_id,
+                "status": "ready_for_story_sufficiency",
+                "title": candidate.title,
+                "summary": summary,
+                "idea": candidate.model_dump(),
+                "traceability": {
+                    "scope_id": scope.scope_id,
+                    "scope_artifact": str((_pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "scope_context.json").relative_to(repo_root)),
+                    "candidate_artifact": str((_pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "idea_candidate_set.json").relative_to(repo_root)),
+                },
+                "lineage": {
+                    "scope_to_idea_origin": candidate_lineage.origin,
+                    "scope_to_idea_mode": candidate_lineage.mode,
+                    "scope_to_idea_stage": candidate_lineage.stage,
+                    "scope_to_idea_artifact": candidate_lineage.artifact_path,
+                    "scope_to_idea_agent_run_ref": candidate_lineage.agent_run_ref,
+                    "model_backed": bool(candidate_lineage.metadata.get("model_backed")),
+                },
+            }
+            _write_json(registry_path, persisted)
+            registered.append(
+                RegisteredIdeaArtifact(
+                    idea_id=idea_id,
+                    source_scope_id=scope.scope_id,
+                    parent_scope_id=candidate.parent_scope_id,
+                    idea_candidate_id=candidate.idea_candidate_id,
+                    title=candidate.title,
+                    summary=summary,
+                    registry_ref=str(registry_path),
+                    idea_ref=str(registry_path),
+                    status="ready_for_story_sufficiency",
+                    execution_lineage=candidate_lineage,
+                )
+            )
+        stage_path = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "idea_registry_record.json"
+        artifact = IdeaRegistryRecordArtifact(
+            project_id=scope.project_id,
+            scope_set_id=scope.scope_set_id,
+            scope_id=scope.scope_id,
+            parent_scope_id=scope.scope_id,
+            refs=_build_traceability(scope),
+            scope_shape=assessment.scope_shape,
+            registered_ideas=registered,
+            registration_timestamp=datetime.now(UTC).isoformat(),
+            registry_root=str(registry_root),
+            lineage=list(candidate_set.lineage) + [
+                _lineage_entry(
+                    stage="register_ideas",
+                    origin="deterministic",
+                    mode="deterministic",
+                    repo_root=repo_root,
+                    artifact_path=stage_path,
+                    generated_from=[_relative_to_repo(repo_root, _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "idea_candidate_set.json")],
+                    notes=["Registry write preserves prior shaping lineage and exposes persisted idea records."],
+                    metadata={"model_backed": False},
+                )
+            ],
+        )
+        _write_json(stage_path, artifact.model_dump())
+        _append_node_trace(pipeline_root=pipeline_root, node="RegisterIdeasNode", event="success", detail={"artifact": _relative_to_repo(repo_root, stage_path), "registered_idea_count": len(artifact.registered_ideas)})
+        task_context.metadata["idea_registry_record"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class IdeaResolutionPackageNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        repo_root = Path(event.repo_root)
+        _dfs_running(project_id=event.project_id, run_id=_store_run()[1], scope_id=event.scope_id, summary="Packaging scope-to-idea resolution")
+        pipeline_root = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key)
+        _append_node_trace(pipeline_root=pipeline_root, node="IdeaResolutionPackageNode", event="enter")
+        scope = task_context.metadata["scope_context"]
+        assessment = task_context.metadata["goldilocks_assessment"]
+        registry = task_context.metadata.get("idea_registry_record")
+        narrow = task_context.metadata.get("narrow_scope_review")
+
+        if registry is not None:
+            artifact = IdeaResolutionPackageArtifact(
+                project_id=scope.project_id,
+                scope_set_id=scope.scope_set_id,
+                scope_id=scope.scope_id,
+                parent_scope_id=scope.scope_id,
+                refs=_build_traceability(scope),
+                scope_shape=assessment.scope_shape,
+                split_applied=bool(task_context.metadata.get("idea_split_plan")),
+                resolution_status="ideas_registered",
+                registered_idea_count=len(registry.registered_ideas),
+                registered_ideas=registry.registered_ideas,
+                assumptions_added=scope.assumptions,
+                remaining_risks=["Idea sufficiency is intentionally deferred to idea->DevFlow stories DAG."],
+                recommended_next_action="Submit registered idea(s) to the idea->story sufficiency DAG.",
+                lineage=list(registry.lineage),
+            )
+        else:
+            review_path = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "narrow_scope_review.json"
+            artifact = IdeaResolutionPackageArtifact(
+                project_id=scope.project_id,
+                scope_set_id=scope.scope_set_id,
+                scope_id=scope.scope_id,
+                parent_scope_id=scope.scope_id,
+                refs=_build_traceability(scope),
+                scope_shape=assessment.scope_shape,
+                split_applied=False,
+                resolution_status="narrow_scope_review_required",
+                registered_idea_count=0,
+                assumptions_added=scope.assumptions,
+                remaining_risks=[narrow.reason_too_narrow] if narrow is not None else [],
+                recommended_next_action="Human review required before any merge or rescoping.",
+                review_required=True,
+                review_package_ref=str(review_path) if review_path.exists() else None,
+                lineage=[narrow.execution_lineage] if narrow is not None and narrow.execution_lineage else [],
+            )
+        stage_path = _pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "idea_resolution_package.json"
+        artifact.lineage.append(
+            _lineage_entry(
+                stage="idea_resolution_package",
+                origin="deterministic",
+                mode="deterministic",
+                repo_root=repo_root,
+                artifact_path=stage_path,
+                generated_from=[line.artifact_path for line in artifact.lineage if line and line.artifact_path],
+                notes=["Resolution package summarizes the run without changing the shaped idea lineage."],
+                metadata={"model_backed": False},
+            )
+        )
+        _write_json(stage_path, artifact.model_dump())
+        _append_node_trace(pipeline_root=pipeline_root, node="IdeaResolutionPackageNode", event="success", detail={"artifact": _relative_to_repo(repo_root, stage_path), "resolution_status": artifact.resolution_status})
+        summary = ScopeIdeaDagSummary(
+            exit_code=0,
+            run_id=_store_run()[1],
+            pipeline_dir=str(_pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key)),
+            message="scope->idea run complete",
+            outcome={
+                "project_id": scope.project_id,
+                "scope_id": scope.scope_id,
+                "scope_shape": assessment.scope_shape,
+                "registered_idea_count": artifact.registered_idea_count,
+                "resolution_status": artifact.resolution_status,
+            },
+        )
+        _write_json(_pipeline_root(repo_root, scope_id=event.scope_id, pipeline_key=event.pipeline_key) / "summary.json", summary.model_dump())
+        task_context.metadata["outcome"] = dict(summary.outcome)
+        task_context.metadata["message"] = json.dumps({**summary.outcome, "run_id": summary.run_id, "pipeline_dir": summary.pipeline_dir}, sort_keys=True) + "\n"
+        task_context.metadata["exit_code"] = 0
+        self.save_output(artifact)
+        return task_context
+
+
+class ScopeToIdeaWorkflow(Workflow):
+    workflow_schema = WorkflowSchema(
+        description="Approved scope -> idea DAG (load -> goldilocks -> route -> shape/register -> resolution)",
+        event_schema=ScopeToIdeaDagEvent,
+        start=LoadApprovedScopeItemNode,
+        nodes=[
+            NodeConfig(node=LoadApprovedScopeItemNode, connections=[AssessScopeGoldilocksNode]),
+            NodeConfig(node=AssessScopeGoldilocksNode, connections=[GoldilocksDecisionNode]),
+            NodeConfig(node=GoldilocksDecisionNode, connections=[GoldilocksDecisionRouter]),
+            NodeConfig(node=GoldilocksDecisionRouter, connections=[SplitScopeIntoIdeaCandidatesNode, DraftIdeaFromScopeNode, NarrowScopeReviewPackageNode], is_router=True),
+            NodeConfig(node=SplitScopeIntoIdeaCandidatesNode, connections=[NormalizeIdeaCandidateSetNode]),
+            NodeConfig(node=DraftIdeaFromScopeNode, connections=[NormalizeIdeaCandidateSetNode]),
+            NodeConfig(node=NarrowScopeReviewPackageNode, connections=[IdeaResolutionPackageNode]),
+            NodeConfig(node=NormalizeIdeaCandidateSetNode, connections=[RegisterIdeasNode]),
+            NodeConfig(node=RegisterIdeasNode, connections=[IdeaResolutionPackageNode]),
+            NodeConfig(node=IdeaResolutionPackageNode, connections=[]),
+        ],
+    )
+
+
+def run_scope_to_idea_dag(
+    *,
+    repo_root: Path,
+    store: ExecutionStore,
+    project_id: str,
+    scope_set_id: str,
+    scope_id: str,
+    scope_payload_inline: dict[str, Any] | None = None,
+    scope_payload_path: Path | None = None,
+) -> ScopeToIdeaDagResult:
+    payload = _load_scope_payload(inline_payload=scope_payload_inline, payload_path=scope_payload_path)
+    _scope_context_from_payload(
+        payload=payload,
+        project_id=project_id,
+        scope_set_id=scope_set_id,
+        scope_id=scope_id,
+    )
+    pipeline_key = build_pipeline_key(
+        repo_root=repo_root,
+        project_id=project_id,
+        scope_set_id=scope_set_id,
+        scope_id=scope_id,
+        scope_payload=payload,
+    )
+    pipeline_dir = _pipeline_root(repo_root, scope_id=scope_id, pipeline_key=pipeline_key)
+    pipeline_dir.mkdir(parents=True, exist_ok=True)
+
+    run_id = store.create_run(
+        dag_id=DAG_ID,
+        dag_version="v1_scaffold",
+        root_correlation_id=f"corr_{pipeline_key}",
+        config={
+            "project_id": project_id,
+            "scope_set_id": scope_set_id,
+            "scope_id": scope_id,
+            "pipeline_key": pipeline_key,
+        },
+    )
+    store.mark_run_started(run_id=run_id)
+    _dfs_running(project_id=project_id, run_id=run_id, scope_id=scope_id)
+
+    wf = ScopeToIdeaWorkflow()
+    global _CURRENT_STORE, _CURRENT_RUN_ID
+    _CURRENT_STORE = store
+    _CURRENT_RUN_ID = run_id
+    try:
+        ctx = wf.run(
+            {
+                "repo_root": str(repo_root),
+                "project_id": project_id,
+                "scope_set_id": scope_set_id,
+                "scope_id": scope_id,
+                "scope_payload_inline": None if scope_payload_path else payload,
+                "scope_payload_path": str(scope_payload_path) if scope_payload_path else None,
+                "pipeline_key": pipeline_key,
+            }
+        )
+    except Exception as exc:
+        store.mark_run_finished(run_id=run_id, status="failed")
+        _dfs_terminal(
+            project_id=project_id,
+            run_id=run_id,
+            scope_id=scope_id,
+            current_state="error",
+            current_status="failed",
+            summary="Scope to idea failed",
+            error_message=str(exc),
+        )
+        raise
+    finally:
+        _CURRENT_STORE = None
+        _CURRENT_RUN_ID = None
+
+    exit_code = int(ctx.metadata.get("exit_code") or 0)
+    store.mark_run_finished(run_id=run_id, status="succeeded" if exit_code == 0 else "failed")
+    if exit_code == 0:
+        _dfs_terminal(
+            project_id=project_id,
+            run_id=run_id,
+            scope_id=scope_id,
+            current_state="idle",
+            current_status="completed",
+            summary="Ideas generated from scope",
+        )
+        _sync_registered_ideas_to_supabase(
+            project_id=project_id,
+            scope_id=scope_id,
+            run_id=run_id,
+            pipeline_dir=pipeline_dir,
+        )
+        registry_path = pipeline_dir / "idea_registry_record.json"
+        if registry_path.exists():
+            registry = json.loads(registry_path.read_text(encoding="utf-8"))
+            for idea in registry.get("registered_ideas") or []:
+                if not isinstance(idea, dict):
+                    continue
+                idea_id = str(idea.get("idea_id") or "").strip()
+                idea_ref = str(idea.get("idea_ref") or "").strip()
+                if not idea_id or not idea_ref:
+                    continue
+                store.enqueue_idea_task(
+                    project_id=project_id,
+                    enqueue_run_id=run_id,
+                    idea_id=idea_id,
+                    title=str(idea.get("title") or idea_id or "idea"),
+                    idea_payload_path=idea_ref,
+                    candidate_planes=[],
+                )
+    else:
+        _dfs_terminal(
+            project_id=project_id,
+            run_id=run_id,
+            scope_id=scope_id,
+            current_state="error",
+            current_status="failed",
+            summary="Scope to idea failed",
+            error_message=str(ctx.metadata.get("message") or "scope_to_idea failed"),
+        )
+    return ScopeToIdeaDagResult(
+        exit_code=exit_code,
+        run_id=run_id,
+        pipeline_dir=pipeline_dir,
+        message=str(ctx.metadata.get("message") or ""),
+        outcome=dict(ctx.metadata.get("outcome") or {}),
+    )