devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/ui_grounding/dag.py

@@ -0,0 +1,874 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel
+
+from ..devflow_state import publish_devflow_state
+from ..stores.execution_store import ExecutionStore
+from ..vendor.datalumina_genai.core.nodes.agent import AgentConfig, AgentNode
+from ..vendor.datalumina_genai.core.nodes.base import Node
+from ..vendor.datalumina_genai.core.schema import NodeConfig, WorkflowSchema
+from ..vendor.datalumina_genai.core.task import TaskContext
+from ..vendor.datalumina_genai.core.workflow import Workflow
+from . import agentic as ui_grounding_agentic
+from .models import (
+    CodeDesignGeneratedReference,
+    CodeDesignInventoryArtifact,
+    GroundedStoryReferencePatchArtifact,
+    UIGapReportArtifact,
+    UIGenerationManifestArtifact,
+    UIGroundingContextArtifact,
+    UIGroundingDagSummary,
+    UIGroundingReportArtifact,
+    UIGroundingSurfaceResult,
+    UIFlowMapArtifact,
+    UIReferenceInventoryArtifact,
+    UIReferenceInventoryItem,
+    UIScreenRelationshipsArtifact,
+    UIStructureContractArtifact,
+    UIStructureScreenItem,
+    UISurfaceDerivationArtifact,
+    DerivedUISurface,
+)
+from .pencil_bridge import run_pencil_preflight
+
+DAG_ID = "ui_grounding_dag"
+_CURRENT_STORE: ExecutionStore | None = None
+_CURRENT_RUN_ID: str | None = None
+
+
+@dataclass(frozen=True)
+class UIGroundingDagResult:
+    exit_code: int
+    run_id: str
+    pipeline_dir: Path
+    message: str
+    outcome: dict[str, Any]
+
+
+class UIGroundingDagEvent(BaseModel):
+    repo_root: str
+    project_id: str
+    idea_id: str
+    idea_path: str | None = None
+    idea_inline: dict[str, Any] | None = None
+    story_paths: list[str] = []
+    design_paths: list[str] = []
+    pipeline_key: str
+    allow_code_design_enrichment: bool = False
+    pencil_app_path: str | None = None
+
+
+def _store_run() -> tuple[ExecutionStore, str]:
+    if _CURRENT_STORE is None or _CURRENT_RUN_ID is None:
+        raise RuntimeError("ui grounding dag missing runtime store/run_id")
+    return _CURRENT_STORE, _CURRENT_RUN_ID
+
+
+def _stable_id(prefix: str, payload: Any, *, size: int = 12) -> str:
+    raw = json.dumps(payload, sort_keys=True).encode("utf-8")
+    return f"{prefix}{hashlib.sha256(raw).hexdigest()[:size]}"
+
+
+def _write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _dfs_running(*, project_id: str, run_id: str, summary: str, idea_id: str) -> None:
+    publish_devflow_state(
+        project_id=project_id,
+        run_id=run_id,
+        current_state="running",
+        current_status="processing",
+        run_summary=summary,
+        display="project",
+        display_path=f"idea:{idea_id}",
+    )
+
+
+def _pipeline_root(repo_root: Path, *, idea_id: str, pipeline_key: str) -> Path:
+    return repo_root / ".devflow" / "ideas" / idea_id / "pipelines" / DAG_ID / pipeline_key
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _normalize_text(text: str) -> str:
+    return re.sub(r"[^a-z0-9\s]+", " ", text.lower())
+
+
+def _tokens(text: str) -> list[str]:
+    stop = {"the", "and", "with", "from", "that", "this", "into", "for", "your", "their", "then", "when", "must", "should"}
+    out: list[str] = []
+    for tok in _normalize_text(text).split():
+        if len(tok) >= 4 and tok not in stop and tok not in out:
+            out.append(tok)
+    return out
+
+
+def _sentence_chunks(text: str) -> list[str]:
+    text = re.sub(r"\s+", " ", text).strip()
+    return [part.strip() for part in re.split(r"(?<=[.!?])\s+", text) if part.strip()]
+
+
+def _surface_name(chunk: str) -> str:
+    lowered = chunk.lower()
+    mapping = [
+        ("Manager dashboard", ["manager dashboard", "dashboard", "reporting", "score trends"]),
+        ("Need Help flow", ["need help", "crisis", "help request"]),
+        ("Vibe Check flow", ["vibe check", "mood"]),
+        ("Participant survey", ["participant", "survey", "question", "form"]),
+        ("Quote approval screen", ["quote approval", "approve quote", "secure link"]),
+    ]
+    for title, needles in mapping:
+        if any(n in lowered for n in needles):
+            return title
+    words = chunk.split()
+    return " ".join(words[:5]).strip().capitalize() or "UI surface"
+
+
+def _surface_actor(chunk: str) -> str:
+    lowered = chunk.lower()
+    if "manager" in lowered:
+        return "manager"
+    if "participant" in lowered or "customer" in lowered:
+        return "participant" if "participant" in lowered else "customer"
+    if "admin" in lowered:
+        return "admin"
+    return "user"
+
+
+
+
+def _classify_reference_type(path: Path) -> str:
+    lowered = path.as_posix().lower()
+    if path.suffix == ".pen":
+        return "pencil_artifact"
+    if any(part in lowered for part in ["screenshot", "mock", "wireframe"]):
+        return "screenshot_spec" if "screenshot" in lowered else "wireframe_package"
+    if any(part in lowered for part in ["design", "spec", "figma", "notes"]):
+        return "design_doc"
+    return "implemented_ui"
+
+
+def _candidate_ui_files(repo_root: Path) -> list[Path]:
+    roots = [repo_root / "app", repo_root / "src", repo_root / "components", repo_root / "pages", repo_root / "ai_docs"]
+    out: list[Path] = []
+    for root in roots:
+        if not root.exists():
+            continue
+        for path in root.rglob("*"):
+            if not path.is_file():
+                continue
+            if path.suffix.lower() not in {".tsx", ".ts", ".jsx", ".js", ".md", ".mdx", ".pen", ".json", ".png", ".jpg", ".jpeg", ".webp"}:
+                continue
+            rel = path.relative_to(repo_root).as_posix().lower()
+            if any(marker in rel for marker in ["dashboard", "screen", "page", "survey", "participant", "manager", "vibe", "help", "quote", "design", "wireframe", "mock"]):
+                out.append(path)
+    return sorted(out)[:80]
+
+
+def _labels_for_path(path: Path) -> list[str]:
+    stem = path.stem.replace("_", " ").replace("-", " ")
+    parent = path.parent.name.replace("_", " ").replace("-", " ")
+    return [item.strip() for item in [stem, parent] if item.strip()]
+
+
+def _materialize_report(derivation: UISurfaceDerivationArtifact, inventory: UIReferenceInventoryArtifact, *, prefer_generated: bool = False) -> tuple[UIGroundingReportArtifact, UIGapReportArtifact | None]:
+    by_id = {item.reference_id: item for item in inventory.references}
+    ref_index = [(item, set(_tokens(" ".join([*item.surface_labels, item.path])))) for item in inventory.references]
+    rows: list[UIGroundingSurfaceResult] = []
+    missing: list[str] = []
+    assumptions: list[str] = []
+    generated_hits = 0
+    for surface in derivation.surfaces:
+        if not surface.ui_bearing:
+            rows.append(UIGroundingSurfaceResult(
+                surface_id=surface.surface_id,
+                name=surface.name,
+                classification="non_ui_or_backend_only",
+                next_required_action="No UI grounding required for this slice.",
+            ))
+            continue
+        stoks = set(_tokens(f"{surface.name} {' '.join(surface.jobs)}"))
+        matches = [item for item, toks in ref_index if stoks and len(stoks & toks) >= 2]
+        if matches:
+            top_matches = matches[:3]
+            if prefer_generated and any(m.reference_type == "pencil_artifact" for m in top_matches):
+                classification = "grounded_with_generated_artifacts"
+                generated_hits += 1
+                next_action = "Attach strongest code-derived design anchors to grounded story/idea artifacts."
+            else:
+                classification = "grounded_existing"
+                next_action = "Attach existing UI anchors to grounded story/idea artifacts."
+            rows.append(UIGroundingSurfaceResult(
+                surface_id=surface.surface_id,
+                name=surface.name,
+                classification=classification,
+                matched_reference_ids=[m.reference_id for m in top_matches],
+                supporting_paths=[m.path for m in top_matches],
+                next_required_action=next_action,
+            ))
+        else:
+            missing.append(surface.name)
+            assumptions.append(f"No existing UI reference was found for {surface.name}; DFUS should not improvise this surface.")
+            rows.append(UIGroundingSurfaceResult(
+                surface_id=surface.surface_id,
+                name=surface.name,
+                classification="missing_reference",
+                unresolved_questions=[f"Which approved screen/spec should ground {surface.name}?"],
+                assumptions=[assumptions[-1]],
+                next_required_action="Create a UI gap artifact and route to design review or wireframe generation.",
+            ))
+    if missing:
+        status = "blocked_missing_ui_grounding"
+        gap = UIGapReportArtifact(
+            project_id=derivation.project_id,
+            idea_id=derivation.idea_id,
+            missing_surfaces=missing,
+            reasons=[f"No matching existing UI/design reference for {name}." for name in missing],
+            recommendation="Needs UI review before DFUS or implementation proceeds for the affected slices.",
+            blocked=True,
+        )
+        summary = f"UI grounding blocked: {len(missing)} required surface(s) are missing durable UI references."
+        next_actions = [gap.recommendation]
+    else:
+        status = "grounded_with_generated_artifacts" if generated_hits else "grounded_ready"
+        gap = None
+        summary = "All derived UI-bearing surfaces were matched to existing UI/design references."
+        if generated_hits:
+            summary = f"All derived UI-bearing surfaces were grounded after code-to-design enrichment; {generated_hits} surface(s) now use generated design anchors."
+        next_actions = ["Patch the idea/provisional story artifacts with explicit UI anchors."]
+    return UIGroundingReportArtifact(
+        project_id=derivation.project_id,
+        idea_id=derivation.idea_id,
+        status=status,
+        surfaces=rows,
+        summary=summary,
+        assumptions=assumptions,
+        next_actions=next_actions,
+    ), gap
+
+
+def _grounding_report_from_inventory(derivation: UISurfaceDerivationArtifact, inventory: UIReferenceInventoryArtifact, *, prefer_generated: bool) -> tuple[UIGroundingReportArtifact, UIGapReportArtifact | None]:
+    return _materialize_report(derivation, inventory, prefer_generated=prefer_generated)
+
+
+def _slug(text: str) -> str:
+    pieces = [part for part in re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_").split("_") if part]
+    return "_".join(pieces[:8]) or "screen"
+
+
+def _surface_class(surface: DerivedUISurface) -> str:
+    lowered = f"{surface.name} {' '.join(surface.jobs)}".lower()
+    if any(token in lowered for token in ["secure link", "approve quote", "quote approval", "payment", "checkout"]):
+        return "customer_secure_link"
+    if any(token in lowered for token in ["portal", "account area"]):
+        return "customer_portal"
+    if any(token in lowered for token in ["technician", "field", "crew", "job site", "tablet"]):
+        return "field_tablet"
+    if any(token in lowered for token in ["manager", "admin", "dashboard", "reporting", "back office"]):
+        return "admin_web"
+    if any(token in lowered for token in ["support", "need help", "csr"]):
+        return "internal_support_tool"
+    if any(token in lowered for token in ["participant", "survey", "form", "vibe check", "page", "flow"]):
+        return "general_web"
+    if not surface.ui_bearing:
+        return "non_ui_or_backend_only"
+    return "general_web"
+
+
+def _surface_platform(surface_class: str) -> str:
+    if surface_class == "field_tablet":
+        return "tablet"
+    if surface_class in {"customer_secure_link", "customer_portal"}:
+        return "web_mobile_responsive"
+    if surface_class == "non_ui_or_backend_only":
+        return "not_applicable"
+    return "web_desktop"
+
+
+def _target_audience(surface_class: str) -> str:
+    if surface_class == "non_ui_or_backend_only":
+        return "internal_exploration"
+    return "customer_review"
+
+
+def _target_fidelity(surface_class: str, audience: str) -> str:
+    if audience == "customer_review" and surface_class in {"customer_secure_link", "customer_portal", "admin_web", "field_tablet", "general_web"}:
+        return "medium"
+    return "low"
+
+
+def _journey_stage(surface: DerivedUISurface) -> str:
+    lowered = f"{surface.name} {' '.join(surface.jobs)}".lower()
+    if "approve" in lowered or "approval" in lowered:
+        return "review_and_approve"
+    if "dashboard" in lowered or "report" in lowered:
+        return "monitor_and_review"
+    if "survey" in lowered or "form" in lowered or "check" in lowered:
+        return "capture_input"
+    if "need help" in lowered or "support" in lowered:
+        return "request_support"
+    return "primary_flow"
+
+
+def _shared_shell_id(surface_class: str) -> str | None:
+    return {
+        "admin_web": "admin_shell",
+        "customer_secure_link": "secure_link_shell",
+        "customer_portal": "customer_portal_shell",
+        "field_tablet": "field_ops_shell",
+        "internal_support_tool": "support_shell",
+        "general_web": "default_web_shell",
+    }.get(surface_class)
+
+
+def _build_ui_structure_contract(*, context: UIGroundingContextArtifact, derivation: UISurfaceDerivationArtifact, report: UIGroundingReportArtifact) -> tuple[UIStructureContractArtifact, UIFlowMapArtifact, UIScreenRelationshipsArtifact, UIGenerationManifestArtifact]:
+    by_surface = {surface.surface_id: surface for surface in derivation.surfaces}
+    screens: list[UIStructureScreenItem] = []
+    flows: dict[str, list[str]] = {}
+    relationships: list[dict[str, Any]] = []
+    unresolved: list[str] = []
+
+    for row in report.surfaces:
+        surface = by_surface.get(row.surface_id)
+        if surface is None:
+            continue
+        surface_class = _surface_class(surface)
+        audience = _target_audience(surface_class)
+        fidelity = _target_fidelity(surface_class, audience)
+        flow_id = f"flow_{_journey_stage(surface)}"
+        screen_id = _slug(surface.name)
+        confidence = surface.confidence
+        confidence_reason = "Grounded to approved references." if row.classification in {"grounded_existing", "grounded_with_generated_artifacts"} else "Still missing approved grounding references."
+        assumption_tags = [
+            "assume_customer_facing_review" if audience == "customer_review" else "assume_internal_exploration",
+            "assume_medium_fidelity_minimum" if fidelity == "medium" else "assume_low_fidelity_allowed",
+        ]
+        notes = []
+        if fidelity == "medium":
+            notes.append("Use realistic labels, hierarchy, and CTA placement rather than skeletal placeholder boxes.")
+        if row.classification == "grounded_with_generated_artifacts":
+            notes.append("Generated design anchors should still receive review before implementation handoff.")
+        grounding_refs = []
+        for path in row.supporting_paths:
+            grounding_refs.append(path if path.startswith("pencil:") else f"ui:file:{path}")
+        screen = UIStructureScreenItem(
+            surface_id=row.surface_id,
+            screen_id=screen_id,
+            name=surface.name,
+            surface_class=surface_class,
+            platform=_surface_platform(surface_class),
+            actor=surface.actor,
+            journey_stage=_journey_stage(surface),
+            required=surface.ui_bearing,
+            source_refs=surface.source_refs,
+            grounding_refs=grounding_refs,
+            target_fidelity=fidelity,
+            target_audience=audience,
+            confidence=confidence,
+            confidence_reason=confidence_reason,
+            open_questions=list(row.unresolved_questions),
+            assumption_tags=assumption_tags,
+            notes=notes,
+            flow_id=flow_id,
+            shell_id=_shared_shell_id(surface_class),
+        )
+        screens.append(screen)
+        flows.setdefault(flow_id, []).append(screen_id)
+        if row.classification == "missing_reference":
+            unresolved.extend(row.unresolved_questions)
+        if screen.shell_id:
+            relationships.append({"type": "shared_shell", "from": screen_id, "to": screen.shell_id})
+
+    ordered_flow_ids = sorted(flows)
+    screen_index = {screen.screen_id: screen for screen in screens}
+    for flow_id, ids in flows.items():
+        for current, nxt in zip(ids, ids[1:]):
+            screen_index[current].dependency_screen_ids.append(nxt)
+            relationships.append({"type": "flow_transition", "from": current, "to": nxt, "flow_id": flow_id})
+
+    status = "non_ui_only" if not any(screen.required for screen in screens) else ("needs_ui_review" if report.status == "blocked_missing_ui_grounding" else "ready_for_generation")
+    shared_shells = [
+        {"shell_id": shell_id, "screen_ids": [screen.screen_id for screen in screens if screen.shell_id == shell_id]}
+        for shell_id in sorted({screen.shell_id for screen in screens if screen.shell_id})
+    ]
+    structure = UIStructureContractArtifact(
+        project_id=context.project_id,
+        idea_id=context.idea_id,
+        status=status,
+        default_minimum_fidelity="medium",
+        primary_audience="customer_review",
+        screens=screens,
+        flow_order=ordered_flow_ids,
+        shared_shells=shared_shells,
+        structural_notes=[
+            "Planning/discovery decides what surfaces exist; structure contract decides how the approved screen set coheres before generation.",
+            "Customer-facing and stakeholder-review surfaces default to medium fidelity minimum.",
+        ],
+        unresolved_questions=sorted(dict.fromkeys(unresolved)),
+    )
+    flow_map = UIFlowMapArtifact(
+        project_id=context.project_id,
+        idea_id=context.idea_id,
+        flows=[{"flow_id": flow_id, "screen_ids": ids} for flow_id, ids in flows.items()],
+    )
+    screen_relationships = UIScreenRelationshipsArtifact(
+        project_id=context.project_id,
+        idea_id=context.idea_id,
+        relationships=relationships,
+    )
+    generation_manifest = UIGenerationManifestArtifact(
+        project_id=context.project_id,
+        idea_id=context.idea_id,
+        primary_audience="customer_review",
+        default_minimum_fidelity="medium",
+        review_packet_outputs=[
+            "wireframe_index.md",
+            "customer_review_packet.json",
+            "screen_comment_matrix.json",
+            "screen_inventory_snapshot.json",
+        ],
+        screen_generation_items=[
+            {
+                "screen_id": screen.screen_id,
+                "surface_id": screen.surface_id,
+                "target_fidelity": screen.target_fidelity,
+                "target_audience": screen.target_audience,
+                "grounding_refs": screen.grounding_refs,
+                "flow_id": screen.flow_id,
+            }
+            for screen in screens if screen.required
+        ],
+    )
+    return structure, flow_map, screen_relationships, generation_manifest
+
+
+def _write_report_and_gap(pipeline_root: Path, artifact: UIGroundingReportArtifact, gap: UIGapReportArtifact | None) -> None:
+    report_path = pipeline_root / "ui_grounding_report.json"
+    _write_json(report_path, artifact.model_dump())
+    gap_path = pipeline_root / "ui_gap_report.json"
+    if gap is not None:
+        _write_json(gap_path, gap.model_dump())
+    elif gap_path.exists():
+        gap_path.unlink()
+
+
+class LoadUIGroundingContextNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        _dfs_running(project_id=event.project_id, run_id=_store_run()[1], summary="Reviewing existing UI", idea_id=event.idea_id)
+        repo_root = Path(event.repo_root)
+        idea_payload = dict(event.idea_inline or {})
+        idea_path = None
+        if event.idea_path:
+            idea_path = repo_root / event.idea_path if not Path(event.idea_path).is_absolute() else Path(event.idea_path)
+            idea_payload = _load_json(idea_path)
+        story_paths: list[str] = []
+        story_summaries: list[str] = []
+        for raw in event.story_paths:
+            path = repo_root / raw if not Path(raw).is_absolute() else Path(raw)
+            story_paths.append(str(path))
+            try:
+                payload = _load_json(path)
+                story_summaries.append(str(payload.get("title") or payload.get("user_value_statement") or path.name))
+            except Exception:
+                story_summaries.append(path.name)
+        design_paths = [str((repo_root / p if not Path(p).is_absolute() else Path(p))) for p in event.design_paths]
+        context = UIGroundingContextArtifact(
+            project_id=event.project_id,
+            idea_id=event.idea_id,
+            idea_title=str(idea_payload.get("title") or event.idea_id),
+            idea_summary=str(idea_payload.get("summary") or idea_payload.get("goal") or idea_payload.get("problem") or idea_payload),
+            idea_path=str(idea_path) if idea_path else None,
+            story_paths=story_paths,
+            story_summaries=story_summaries,
+            design_paths=design_paths,
+            repo_root=str(repo_root),
+            upstream_refs=[event.idea_path] if event.idea_path else [f"idea:{event.idea_id}"],
+        )
+        pipeline_root = _pipeline_root(repo_root, idea_id=event.idea_id, pipeline_key=event.pipeline_key)
+        _write_json(pipeline_root / "ui_grounding_context.json", context.model_dump())
+        task_context.metadata["pipeline_root"] = str(pipeline_root)
+        task_context.metadata["ui_grounding_context"] = context
+        task_context.metadata["allow_code_design_enrichment"] = bool(getattr(event, "allow_code_design_enrichment", False))
+        task_context.metadata["pencil_app_path"] = getattr(event, "pencil_app_path", None)
+        self.save_output(context)
+        return task_context
+
+
+class DeriveUISurfacesFromIdeaNode(AgentNode):
+    def get_agent_config(self) -> AgentConfig:
+        return AgentConfig(instructions="Derive the bounded set of UI-bearing surfaces implied by the approved idea/stories.", output_type=UISurfaceDerivationArtifact)
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        _dfs_running(project_id=task_context.event.project_id, run_id=_store_run()[1], summary="Mapping screens and flows", idea_id=task_context.event.idea_id)
+        context = task_context.metadata["ui_grounding_context"]
+        repo_root = Path(task_context.event.repo_root)
+        pipeline_root = Path(task_context.metadata["pipeline_root"])
+        artifact, envelope = ui_grounding_agentic.run_ui_grounding_agent_step(
+            repo_root=repo_root,
+            stage_name="derive_ui_surfaces",
+            output_model=UISurfaceDerivationArtifact,
+            context_payload=context.model_dump(),
+            guidance=["Return only surfaces that are explicitly or strongly implied by the approved idea/story inputs.", "Mark non-UI slices explicitly rather than pretending they need screens."],
+            timeout_seconds=1800,
+        )
+        ui_grounding_agentic.persist_agent_run(pipeline_root=pipeline_root, node_id="derive_ui_surfaces", envelope=envelope)
+        artifact = UISurfaceDerivationArtifact.model_validate(artifact.model_dump())
+        _write_json(pipeline_root / "ui_surface_derivation.json", artifact.model_dump())
+        task_context.metadata["ui_surface_derivation"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class InventoryExistingUIReferencesNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        context = task_context.metadata["ui_grounding_context"]
+        repo_root = Path(task_context.event.repo_root)
+        refs: list[UIReferenceInventoryItem] = []
+        seen: set[str] = set()
+        for path in [*(_candidate_ui_files(repo_root)), *(Path(p) for p in context.design_paths if Path(p).exists())]:
+            rel = path.relative_to(repo_root).as_posix() if path.is_relative_to(repo_root) else str(path)
+            if rel in seen:
+                continue
+            seen.add(rel)
+            refs.append(UIReferenceInventoryItem(
+                reference_id=_stable_id("uiref_", {"path": rel}),
+                reference_type=_classify_reference_type(path),
+                path=rel,
+                surface_labels=_labels_for_path(path),
+                provenance=["repo_scan" if str(path).startswith(str(repo_root)) else "design_input"],
+                approval_status="approved" if path.suffix == ".pen" else "unknown",
+            ))
+        artifact = UIReferenceInventoryArtifact(
+            project_id=context.project_id,
+            idea_id=context.idea_id,
+            references=refs,
+            inventory_notes=[f"Inventoried {len(refs)} UI/design reference candidate(s)."],
+        )
+        _write_json(Path(task_context.metadata["pipeline_root"]) / "ui_reference_inventory.json", artifact.model_dump())
+        task_context.metadata["ui_reference_inventory"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class EnsurePencilAvailableNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        _dfs_running(project_id=task_context.event.project_id, run_id=_store_run()[1], summary="Preparing UI generation", idea_id=task_context.event.idea_id)
+        pipeline_root = Path(task_context.metadata["pipeline_root"])
+        allow = bool(task_context.metadata.get("allow_code_design_enrichment"))
+        artifact_path = pipeline_root / "pencil_preflight.json"
+        if not allow:
+            payload = {
+                "ok": False,
+                "status": "not_requested",
+                "repo_root": str(task_context.event.repo_root),
+                "artifact_path": str(artifact_path),
+                "errors": [],
+            }
+            _write_json(artifact_path, payload)
+            task_context.metadata["pencil_preflight"] = payload
+            self.save_output(payload)
+            return task_context
+        result = run_pencil_preflight(
+            repo_root=Path(task_context.event.repo_root),
+            store=None,
+            app_path=Path(task_context.metadata["pencil_app_path"]) if task_context.metadata.get("pencil_app_path") else None,
+            artifact_path=artifact_path,
+        )
+        task_context.metadata["pencil_preflight"] = result.artifact.model_dump()
+        self.save_output(result.artifact)
+        return task_context
+
+
+class MatchUISurfacesToReferencesNode(AgentNode):
+    def get_agent_config(self) -> AgentConfig:
+        return AgentConfig(instructions="Match derived UI surfaces to the best existing UI/design references.", output_type=UIGroundingReportArtifact)
+
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        _dfs_running(project_id=task_context.event.project_id, run_id=_store_run()[1], summary="Matching UI references", idea_id=task_context.event.idea_id)
+        repo_root = Path(task_context.event.repo_root)
+        pipeline_root = Path(task_context.metadata["pipeline_root"])
+        derivation = task_context.metadata["ui_surface_derivation"]
+        inventory = task_context.metadata["ui_reference_inventory"]
+        deterministic_report, gap = _grounding_report_from_inventory(derivation, inventory, prefer_generated=False)
+        artifact, envelope = ui_grounding_agentic.run_ui_grounding_agent_step(
+            repo_root=repo_root,
+            stage_name="match_ui_surfaces",
+            output_model=UIGroundingReportArtifact,
+            context_payload={"ui_surface_derivation": derivation.model_dump(), "ui_reference_inventory": inventory.model_dump(), "deterministic_hints": deterministic_report.model_dump()},
+            guidance=["Classify each derived surface as grounded_existing, grounded_partial, missing_reference, or non_ui_or_backend_only.", "If a required UI surface has no durable reference, do not mark the slice grounded."],
+            timeout_seconds=1800,
+        )
+        ui_grounding_agentic.persist_agent_run(pipeline_root=pipeline_root, node_id="match_ui_surfaces", envelope=envelope)
+        artifact = UIGroundingReportArtifact.model_validate(artifact.model_dump())
+        _write_report_and_gap(pipeline_root, artifact, gap)
+        task_context.metadata["ui_grounding_report_first_pass"] = artifact
+        task_context.metadata["ui_gap_report"] = gap
+        task_context.metadata["ui_grounding_report"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class EnrichCodeDesignInventoryNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        pipeline_root = Path(task_context.metadata["pipeline_root"])
+        report: UIGroundingReportArtifact = task_context.metadata["ui_grounding_report_first_pass"]
+        inventory: UIReferenceInventoryArtifact = task_context.metadata["ui_reference_inventory"]
+        preflight = dict(task_context.metadata.get("pencil_preflight") or {})
+        missing_surface_ids = [row.surface_id for row in report.surfaces if row.classification in {"missing_reference", "grounded_partial"}]
+        if not bool(task_context.metadata.get("allow_code_design_enrichment")):
+            artifact = CodeDesignInventoryArtifact(project_id=report.project_id, idea_id=report.idea_id, status="not_requested", notes=["Code-to-design enrichment not requested for this run."])
+        elif preflight.get("status") != "ready":
+            artifact = CodeDesignInventoryArtifact(project_id=report.project_id, idea_id=report.idea_id, status="skipped_preflight_not_ready", target_surface_ids=missing_surface_ids, notes=[f"Pencil preflight status {preflight.get('status', 'unknown')} prevented enrichment."])
+        elif not missing_surface_ids:
+            artifact = CodeDesignInventoryArtifact(project_id=report.project_id, idea_id=report.idea_id, status="skipped_no_missing_surfaces", notes=["First-pass matching already grounded all required UI surfaces."])
+        else:
+            derivation: UISurfaceDerivationArtifact = task_context.metadata["ui_surface_derivation"]
+            generated: list[CodeDesignGeneratedReference] = []
+            for surface in derivation.surfaces:
+                if surface.surface_id not in missing_surface_ids:
+                    continue
+                stoks = set(_tokens(f"{surface.name} {' '.join(surface.jobs)}"))
+                candidates = [
+                    ref for ref in inventory.references
+                    if ref.reference_type == "implemented_ui" and stoks and len(stoks & set(_tokens(' '.join([*ref.surface_labels, ref.path])))) >= 1
+                ]
+                if not candidates:
+                    continue
+                source_paths = [ref.path for ref in candidates[:3]]
+                generated_anchor = f"pencil:code_design_inventory.json#{surface.surface_id}"
+                normalized = UIReferenceInventoryItem(
+                    reference_id=_stable_id("pencilref_", {"surface_id": surface.surface_id, "source_paths": source_paths}),
+                    reference_type="pencil_artifact",
+                    path=generated_anchor,
+                    surface_labels=[surface.name, *surface.jobs[:1]],
+                    provenance=[*source_paths, "code_to_design"],
+                    approval_status="generated_from_code",
+                )
+                generated.append(CodeDesignGeneratedReference(
+                    surface_id=surface.surface_id,
+                    source_paths=source_paths,
+                    generated_reference_id=normalized.reference_id,
+                    generated_anchor=generated_anchor,
+                    generation_kind="full_screen",
+                    review_needed=False,
+                    normalized_reference=normalized,
+                ))
+            if generated:
+                artifact = CodeDesignInventoryArtifact(
+                    project_id=report.project_id,
+                    idea_id=report.idea_id,
+                    status="generated",
+                    target_surface_ids=missing_surface_ids,
+                    generated_references=generated,
+                    notes=[f"Generated {len(generated)} code-derived design reference(s) from implemented UI evidence."],
+                )
+            else:
+                artifact = CodeDesignInventoryArtifact(
+                    project_id=report.project_id,
+                    idea_id=report.idea_id,
+                    status="skipped_no_relevant_implemented_ui",
+                    target_surface_ids=missing_surface_ids,
+                    notes=["No relevant implemented_ui references were available for code-to-design enrichment."],
+                )
+        _write_json(pipeline_root / "code_design_inventory.json", artifact.model_dump())
+        task_context.metadata["code_design_inventory"] = artifact
+        self.save_output(artifact)
+        return task_context
+
+
+class RerunMatchingAfterCodeDesignNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        pipeline_root = Path(task_context.metadata["pipeline_root"])
+        derivation: UISurfaceDerivationArtifact = task_context.metadata["ui_surface_derivation"]
+        inventory: UIReferenceInventoryArtifact = task_context.metadata["ui_reference_inventory"]
+        first_report: UIGroundingReportArtifact = task_context.metadata["ui_grounding_report_first_pass"]
+        code_design: CodeDesignInventoryArtifact = task_context.metadata["code_design_inventory"]
+        if code_design.status != "generated":
+            artifact = first_report
+            gap = task_context.metadata.get("ui_gap_report")
+            effective_inventory = inventory
+        else:
+            effective_inventory = UIReferenceInventoryArtifact(
+                project_id=inventory.project_id,
+                idea_id=inventory.idea_id,
+                references=[*inventory.references, *(item.normalized_reference for item in code_design.generated_references)],
+                inventory_notes=[*inventory.inventory_notes, f"Merged {len(code_design.generated_references)} code-derived design reference(s) for second-pass matching."],
+            )
+            artifact, gap = _materialize_report(derivation, effective_inventory, prefer_generated=True)
+            _write_json(pipeline_root / "effective_ui_reference_inventory.json", effective_inventory.model_dump())
+        _write_report_and_gap(pipeline_root, artifact, gap)
+        task_context.metadata["effective_ui_reference_inventory"] = effective_inventory
+        task_context.metadata["ui_grounding_report"] = artifact
+        task_context.metadata["ui_gap_report"] = gap
+        self.save_output(artifact)
+        return task_context
+
+
+class IntegrateUIScreenStructureNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        pipeline_root = Path(task_context.metadata["pipeline_root"])
+        context: UIGroundingContextArtifact = task_context.metadata["ui_grounding_context"]
+        derivation: UISurfaceDerivationArtifact = task_context.metadata["ui_surface_derivation"]
+        report: UIGroundingReportArtifact = task_context.metadata["ui_grounding_report"]
+        structure, flow_map, relationships, manifest = _build_ui_structure_contract(
+            context=context,
+            derivation=derivation,
+            report=report,
+        )
+        _write_json(pipeline_root / "ui_structure_contract.json", structure.model_dump())
+        _write_json(pipeline_root / "ui_flow_map.json", flow_map.model_dump())
+        _write_json(pipeline_root / "ui_screen_relationships.json", relationships.model_dump())
+        _write_json(pipeline_root / "ui_generation_manifest.json", manifest.model_dump())
+        task_context.metadata["ui_structure_contract"] = structure
+        task_context.metadata["ui_flow_map"] = flow_map
+        task_context.metadata["ui_screen_relationships"] = relationships
+        task_context.metadata["ui_generation_manifest"] = manifest
+        self.save_output(structure)
+        return task_context
+
+
+class PatchStoryAndIdeaUIReferencesNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        report: UIGroundingReportArtifact = task_context.metadata["ui_grounding_report"]
+        effective_inventory: UIReferenceInventoryArtifact = task_context.metadata.get("effective_ui_reference_inventory") or task_context.metadata["ui_reference_inventory"]
+        inventory_by_id = {item.reference_id: item for item in effective_inventory.references}
+        anchors: list[dict[str, Any]] = []
+        evidence: list[str] = []
+        anchor_kinds: set[str] = set()
+        for row in report.surfaces:
+            for ref_id, path in zip(row.matched_reference_ids, row.supporting_paths):
+                item = inventory_by_id.get(ref_id)
+                if item is not None and item.reference_type == "pencil_artifact":
+                    anchor = path
+                    anchor_type = "pencil"
+                else:
+                    anchor = f"ui:file:{path}"
+                    anchor_type = "ui:file"
+                anchors.append({"surface_id": row.surface_id, "surface": row.name, "anchor": anchor, "anchor_type": anchor_type, "reference_id": ref_id})
+                evidence.append(ref_id)
+                anchor_kinds.add(anchor_type)
+        if anchor_kinds == {"pencil"}:
+            origin = "generated_ui"
+        elif "pencil" in anchor_kinds and "ui:file" in anchor_kinds:
+            origin = "mixed"
+        else:
+            origin = "existing_ui"
+        patch = GroundedStoryReferencePatchArtifact(
+            project_id=report.project_id,
+            idea_id=report.idea_id,
+            target_refs=[task_context.event.idea_path or f"idea:{report.idea_id}", *task_context.event.story_paths],
+            attached_ui_anchors=anchors,
+            supporting_evidence_refs=sorted(set(evidence)),
+            origin=origin,
+        )
+        _write_json(Path(task_context.metadata["pipeline_root"]) / "grounded_story_reference_patch.json", patch.model_dump())
+        task_context.metadata["grounded_story_reference_patch"] = patch
+        self.save_output(patch)
+        return task_context
+
+
+class FinalizeUIGroundingOutcomeNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        report: UIGroundingReportArtifact = task_context.metadata["ui_grounding_report"]
+        patch: GroundedStoryReferencePatchArtifact = task_context.metadata["grounded_story_reference_patch"]
+        outcome = {
+            "project_id": report.project_id,
+            "idea_id": report.idea_id,
+            "status": report.status,
+            "surface_count": len(report.surfaces),
+            "grounded_surface_count": sum(1 for row in report.surfaces if row.classification in {"grounded_existing", "grounded_with_generated_artifacts"}),
+            "missing_surface_count": sum(1 for row in report.surfaces if row.classification == "missing_reference"),
+            "anchor_count": len(patch.attached_ui_anchors),
+        }
+        exit_code = 0 if report.status in {"grounded_ready", "grounded_with_generated_artifacts", "pass_with_assumptions"} else 2
+        summary = UIGroundingDagSummary(
+            exit_code=exit_code,
+            run_id=_store_run()[1],
+            pipeline_dir=str(task_context.metadata["pipeline_root"]),
+            message="ui grounding run complete",
+            outcome=outcome,
+        )
+        _write_json(Path(task_context.metadata["pipeline_root"]) / "summary.json", summary.model_dump())
+        task_context.metadata["message"] = json.dumps({**outcome, "run_id": summary.run_id, "pipeline_dir": summary.pipeline_dir}, sort_keys=True) + "\n"
+        task_context.metadata["outcome"] = outcome
+        task_context.metadata["exit_code"] = exit_code
+        self.save_output(summary)
+        return task_context
+
+
+class UIGroundingWorkflow(Workflow):
+    workflow_schema = WorkflowSchema(
+        description="UI grounding DAG Phase 2 (load context -> derive surfaces -> inventory refs -> Pencil gate -> first pass -> code-design enrichment -> second pass -> integrate screen structure -> patch -> finalize)",
+        event_schema=UIGroundingDagEvent,
+        start=LoadUIGroundingContextNode,
+        nodes=[
+            NodeConfig(node=LoadUIGroundingContextNode, connections=[DeriveUISurfacesFromIdeaNode]),
+            NodeConfig(node=DeriveUISurfacesFromIdeaNode, connections=[InventoryExistingUIReferencesNode]),
+            NodeConfig(node=InventoryExistingUIReferencesNode, connections=[EnsurePencilAvailableNode]),
+            NodeConfig(node=EnsurePencilAvailableNode, connections=[MatchUISurfacesToReferencesNode]),
+            NodeConfig(node=MatchUISurfacesToReferencesNode, connections=[EnrichCodeDesignInventoryNode]),
+            NodeConfig(node=EnrichCodeDesignInventoryNode, connections=[RerunMatchingAfterCodeDesignNode]),
+            NodeConfig(node=RerunMatchingAfterCodeDesignNode, connections=[IntegrateUIScreenStructureNode]),
+            NodeConfig(node=IntegrateUIScreenStructureNode, connections=[PatchStoryAndIdeaUIReferencesNode]),
+            NodeConfig(node=PatchStoryAndIdeaUIReferencesNode, connections=[FinalizeUIGroundingOutcomeNode]),
+            NodeConfig(node=FinalizeUIGroundingOutcomeNode, connections=[]),
+        ],
+    )
+
+
+def build_pipeline_key(*, repo_root: Path, project_id: str, idea_id: str, idea_payload: dict[str, Any], story_paths: list[str], design_paths: list[str], allow_code_design_enrichment: bool = False) -> str:
+    return _stable_id("run_", {"repo_root": str(repo_root), "project_id": project_id, "idea_id": idea_id, "idea_payload": idea_payload, "story_paths": story_paths, "design_paths": design_paths, "allow_code_design_enrichment": allow_code_design_enrichment})
+
+
+def run_ui_grounding_dag(*, repo_root: Path, store: ExecutionStore, project_id: str, idea_id: str, idea_path: Path | None = None, idea_inline: dict[str, Any] | None = None, story_paths: list[Path] | None = None, design_paths: list[Path] | None = None, allow_code_design_enrichment: bool = False, pencil_app_path: Path | None = None) -> UIGroundingDagResult:
+    payload = dict(idea_inline or {})
+    if idea_path is not None:
+        payload = _load_json(idea_path)
+    pipeline_key = build_pipeline_key(
+        repo_root=repo_root,
+        project_id=project_id,
+        idea_id=idea_id,
+        idea_payload=payload,
+        story_paths=[str(p) for p in (story_paths or [])],
+        design_paths=[str(p) for p in (design_paths or [])],
+        allow_code_design_enrichment=allow_code_design_enrichment,
+    )
+    pipeline_dir = _pipeline_root(repo_root, idea_id=idea_id, pipeline_key=pipeline_key)
+    pipeline_dir.mkdir(parents=True, exist_ok=True)
+    run_id = store.create_run(dag_id=DAG_ID, dag_version="v1_phase1", root_correlation_id=f"corr_{pipeline_key}", config={"project_id": project_id, "idea_id": idea_id, "pipeline_key": pipeline_key, "allow_code_design_enrichment": allow_code_design_enrichment})
+    store.mark_run_started(run_id=run_id)
+    wf = UIGroundingWorkflow()
+    global _CURRENT_STORE, _CURRENT_RUN_ID
+    _CURRENT_STORE = store
+    _CURRENT_RUN_ID = run_id
+    try:
+        ctx = wf.run({
+            "repo_root": str(repo_root),
+            "project_id": project_id,
+            "idea_id": idea_id,
+            "idea_path": str(idea_path.relative_to(repo_root)) if idea_path is not None and idea_path.is_relative_to(repo_root) else (str(idea_path) if idea_path else None),
+            "idea_inline": payload if idea_path is None else None,
+            "story_paths": [str(p.relative_to(repo_root)) if p.is_relative_to(repo_root) else str(p) for p in (story_paths or [])],
+            "design_paths": [str(p.relative_to(repo_root)) if p.is_relative_to(repo_root) else str(p) for p in (design_paths or [])],
+            "pipeline_key": pipeline_key,
+            "allow_code_design_enrichment": allow_code_design_enrichment,
+            "pencil_app_path": str(pencil_app_path) if pencil_app_path is not None else None,
+        })
+    finally:
+        _CURRENT_STORE = None
+        _CURRENT_RUN_ID = None
+    exit_code = int(ctx.metadata.get("exit_code") or 0)
+    store.mark_run_finished(run_id=run_id, status="succeeded" if exit_code == 0 else "failed")
+    return UIGroundingDagResult(exit_code=exit_code, run_id=run_id, pipeline_dir=Path(str(ctx.metadata.get("pipeline_root") or pipeline_dir)), message=str(ctx.metadata.get("message") or ""), outcome=dict(ctx.metadata.get("outcome") or {}))