devflow-engine 1.0.0__py3-none-any.whl

devin/devin_eval.py

@@ -0,0 +1,677 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import signal
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+from devflow_engine.agentic_prompts import load_agentic_prompt_lines
+from devflow_engine.agentic_runtime import run_agent_step
+from devin.dag import run_devin_chat_dag
+from devflow_engine.stores.execution_store import ExecutionStore
+
+
+class DevinEvalScenario(BaseModel):
+    scenario_id: str = Field(min_length=1)
+    scenario_family: Literal[
+        "basic_new_idea",
+        "extend_existing_codebase",
+        "drive_devflow_commands",
+        "inform_current_codebase",
+        "diagnose_devflow_issue",
+    ]
+    persona: str | None = None
+    mode: Literal["single_turn"] = "single_turn"
+    doctrine_tags: list[str] = Field(default_factory=list)
+    expected_judgment_focus: list[str] = Field(default_factory=list)
+    user_input: str = Field(min_length=1)
+    idea_id: str = Field(min_length=1)
+    max_stories: int = 0
+    planes: list[str] = Field(default_factory=list)
+    preloaded_history: list[dict[str, Any]] = Field(default_factory=list)
+    contract: dict[str, Any] = Field(default_factory=dict)
+
+
+class DevinResponseAssessment(BaseModel):
+    usefulness: float = Field(ge=0.0, le=1.0)
+    supportiveness: float = Field(ge=0.0, le=1.0)
+    forward_progress: float = Field(ge=0.0, le=1.0)
+    request_relevance: float = Field(ge=0.0, le=1.0)
+    human_style: float = Field(ge=0.0, le=1.0)
+    conciseness: float = Field(ge=0.0, le=1.0)
+    non_overload: float = Field(ge=0.0, le=1.0)
+    command_accuracy: float = Field(ge=0.0, le=1.0, default=0.0)
+    codebase_accuracy: float = Field(ge=0.0, le=1.0, default=0.0)
+    process_diagnosis_accuracy: float = Field(ge=0.0, le=1.0, default=0.0)
+    must_mention_coverage: float = Field(ge=0.0, le=1.0)
+    must_not_violation_free: float = Field(ge=0.0, le=1.0)
+    contract_satisfaction: float = Field(ge=0.0, le=1.0)
+    notes: list[str] = Field(default_factory=list)
+    strengths: list[str] = Field(default_factory=list)
+    weaknesses: list[str] = Field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class DevinEvalResult:
+    run_id: str
+    started_at: str
+    eval_root: Path
+    payload: dict[str, Any]
+
+
+class DevinChatEvalCatalog(BaseModel):
+    schema_version: str = Field(min_length=1)
+    doctrine_sources: list[str] = Field(default_factory=list)
+    design_notes: list[str] = Field(default_factory=list)
+    single_turn: list[DevinEvalScenario] = Field(default_factory=list)
+    multi_turn: list["DevinMultiTurnEvalScenario"] = Field(default_factory=list)
+
+
+DEVIN_CHAT_SCENARIO_CATALOG_PATH = Path(__file__).with_name("devin_chat_scenario_catalog.json")
+
+
+def load_devin_chat_eval_catalog(path: Path | None = None) -> DevinChatEvalCatalog:
+    catalog_path = (path or DEVIN_CHAT_SCENARIO_CATALOG_PATH).expanduser().resolve()
+    payload = json.loads(catalog_path.read_text(encoding="utf-8"))
+    return DevinChatEvalCatalog.model_validate(payload)
+
+
+def _require_real_eval_runtime() -> None:
+    if os.environ.get("DEVFLOW_REAL_LLM_EVAL") != "1":
+        raise RuntimeError("Devin eval requires DEVFLOW_REAL_LLM_EVAL=1. Mocked/stub eval runs are forbidden.")
+
+
+def _now_run_id() -> tuple[str, str]:
+    started_at = datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+    run_id = started_at.replace("-", "").replace(":", "")
+    return run_id, started_at
+
+
+def _write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _copy_if_exists(src: Path, dst: Path) -> bool:
+    if not src.exists():
+        return False
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(src, dst)
+    return True
+
+
+def _mean(values: list[float]) -> float:
+    if not values:
+        return 0.0
+    return round(sum(values) / len(values), 6)
+
+
+def _seed_history(*, store: ExecutionStore, repo_root: Path, project_id: str, scenario: DevinEvalScenario) -> None:
+    if not scenario.preloaded_history:
+        return
+    run = store.start_run(kind="devin.eval.history_seed", repo_root=repo_root, args={"project_id": project_id, "scenario_id": scenario.scenario_id})
+    for idx, row in enumerate(scenario.preloaded_history, start=1):
+        store.add_event(run_id=run.run_id, level="INFO", message=f"seed_history_{idx}", payload=row)
+    store.mark_run_finished(run_id=run.run_id, status="succeeded")
+
+
+ASSESSMENT_GUIDANCE = load_agentic_prompt_lines("devin_eval_assessment")
+
+
+def _assess_response(*, repo_root: Path, scenario: DevinEvalScenario, response_payload: dict[str, Any], context_payload: dict[str, Any]) -> DevinResponseAssessment:
+    model, _ = run_agent_step(
+        repo_root=repo_root,
+        stage_name=f"devin_eval_assess_{scenario.scenario_id}",
+        output_model=DevinResponseAssessment,
+        context_payload={
+            "scenario": scenario.model_dump(),
+            "response_payload": response_payload,
+            "context_payload": context_payload,
+        },
+        guidance=ASSESSMENT_GUIDANCE,
+        timeout_seconds=90,
+    )
+    return model
+
+
+def run_devin_response_eval(
+    *,
+    repo_root: Path,
+    scenario: DevinEvalScenario,
+    evals_root: Path | None = None,
+    project_id: str = "devin_eval_project",
+) -> DevinEvalResult:
+    _require_real_eval_runtime()
+    run_id, started_at = _now_run_id()
+    eval_root = (evals_root or (repo_root / "evals" / "devin")) / run_id / scenario.scenario_id
+    store = ExecutionStore(repo_root / ".devflow" / "execution.sqlite")
+    _seed_history(store=store, repo_root=repo_root, project_id=project_id, scenario=scenario)
+
+    dag_result = run_devin_chat_dag(
+        repo_root=repo_root,
+        store=store,
+        idea_id=scenario.idea_id,
+        text=scenario.user_input,
+        source_path=None,
+        max_stories=scenario.max_stories,
+        planes=list(scenario.planes),
+    )
+    pipeline_dir = dag_result.pipeline_dir
+    response_payload = _load_json(pipeline_dir / "devin_response.json")
+    context_payload = _load_json(pipeline_dir / "idea_context_resolution.json") if (pipeline_dir / "idea_context_resolution.json").exists() else {}
+    intake_payload = _load_json(pipeline_dir / "devin_intake.json") if (pipeline_dir / "devin_intake.json").exists() else {}
+    terminal_payload = _load_json(pipeline_dir / "devin_agent_loop_terminal.json") if (pipeline_dir / "devin_agent_loop_terminal.json").exists() else {}
+    response_post = _load_json(pipeline_dir / "devin_response_post.json") if (pipeline_dir / "devin_response_post.json").exists() else {}
+
+    assessment = _assess_response(
+        repo_root=repo_root,
+        scenario=scenario,
+        response_payload=response_payload,
+        context_payload={
+            "idea_context_resolution": context_payload,
+            "devin_intake": intake_payload,
+            "terminal_payload": terminal_payload,
+            "response_post": response_post,
+        },
+    )
+
+    artifact_map = {
+        "devin_response": "artifacts/devin_response.json",
+        "idea_context_resolution": "artifacts/idea_context_resolution.json",
+        "devin_intake": "artifacts/devin_intake.json",
+        "devin_terminal": "artifacts/devin_agent_loop_terminal.json",
+        "devin_response_post": "artifacts/devin_response_post.json",
+    }
+    _copy_if_exists(pipeline_dir / "devin_response.json", eval_root / artifact_map["devin_response"])
+    _copy_if_exists(pipeline_dir / "idea_context_resolution.json", eval_root / artifact_map["idea_context_resolution"])
+    _copy_if_exists(pipeline_dir / "devin_intake.json", eval_root / artifact_map["devin_intake"])
+    _copy_if_exists(pipeline_dir / "devin_agent_loop_terminal.json", eval_root / artifact_map["devin_terminal"])
+    _copy_if_exists(pipeline_dir / "devin_response_post.json", eval_root / artifact_map["devin_response_post"])
+
+    score_payload = assessment.model_dump()
+    score_payload.update(
+        {
+            "assistant_quality": _mean([
+                assessment.usefulness,
+                assessment.supportiveness,
+                assessment.forward_progress,
+                assessment.request_relevance,
+                assessment.human_style,
+            ]),
+            "style_quality": _mean([
+                assessment.human_style,
+                assessment.conciseness,
+                assessment.non_overload,
+            ]),
+        }
+    )
+    _write_json(eval_root / "metrics" / "scores.json", score_payload)
+
+    manifest = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "scenario": scenario.model_dump(),
+        "repo_root": str(repo_root),
+        "eval_root": str(eval_root),
+        "dag_run_id": dag_result.run_id,
+        "dag_pipeline_dir": str(dag_result.pipeline_dir),
+        "dag_exit_code": dag_result.exit_code,
+        "response_kind": response_payload.get("response_kind"),
+        "real_llm_eval": True,
+        "artifacts": artifact_map,
+        "metrics": {
+            "scores": "metrics/scores.json",
+        },
+    }
+    _write_json(eval_root / "eval_manifest.json", manifest)
+    return DevinEvalResult(run_id=run_id, started_at=started_at, eval_root=eval_root, payload=manifest)
+
+
+def run_devin_eval_suite(
+    *,
+    repo_root: Path,
+    scenarios: list[DevinEvalScenario] | None = None,
+    evals_root: Path | None = None,
+    project_id: str = "devin_eval_project",
+) -> DevinEvalResult:
+    _require_real_eval_runtime()
+    run_id, started_at = _now_run_id()
+    suite_root = (evals_root or (repo_root / "evals" / "devin")) / run_id
+    results: list[dict[str, Any]] = []
+    chosen = scenarios or list(DEFAULT_SCENARIOS)
+    for scenario in chosen:
+        result = run_devin_response_eval(
+            repo_root=repo_root,
+            scenario=scenario,
+            evals_root=suite_root,
+            project_id=project_id,
+        )
+        scores = _load_json(result.eval_root / "metrics" / "scores.json")
+        results.append(
+            {
+                "scenario_id": scenario.scenario_id,
+                "scenario_family": scenario.scenario_family,
+                "eval_root": str(result.eval_root),
+                "assistant_quality": scores.get("assistant_quality", 0.0),
+                "style_quality": scores.get("style_quality", 0.0),
+                "contract_satisfaction": scores.get("contract_satisfaction", 0.0),
+                "response_kind": result.payload.get("response_kind"),
+            }
+        )
+    summary = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "scenario_count": len(results),
+        "assistant_quality": _mean([float(item["assistant_quality"]) for item in results]),
+        "style_quality": _mean([float(item["style_quality"]) for item in results]),
+        "contract_satisfaction": _mean([float(item["contract_satisfaction"]) for item in results]),
+        "scenarios": results,
+        "real_llm_eval": True,
+    }
+    _write_json(suite_root / "suite_summary.json", summary)
+    _write_json(suite_root / "eval_manifest.json", summary)
+    return DevinEvalResult(run_id=run_id, started_at=started_at, eval_root=suite_root, payload=summary)
+
+
+
+class DevinMultiTurnEvalTurn(BaseModel):
+    turn_id: str = Field(min_length=1)
+    user_input: str = Field(min_length=1)
+    max_stories: int = 0
+    planes: list[str] = Field(default_factory=list)
+    expect_route_arm: Literal["ideation", "insight", "neither"] | None = None
+    expect_status: str | None = None
+    expect_response_kind: str | None = None
+    expect_activation: bool | None = None
+    must_mention: list[str] = Field(default_factory=list)
+    must_not: list[str] = Field(default_factory=list)
+    judgment_focus: list[str] = Field(default_factory=list)
+    decision_point_label: str | None = None
+    max_follow_up_questions: int | None = None
+    require_suggested_next_step: bool | None = None
+    forbid_internal_orchestration_terms: bool = False
+
+
+class DevinMultiTurnEvalScenario(BaseModel):
+    scenario_id: str = Field(min_length=1)
+    persona: str | None = None
+    mode: Literal["multi_turn"] = "multi_turn"
+    doctrine_tags: list[str] = Field(default_factory=list)
+    expected_judgment_focus: list[str] = Field(default_factory=list)
+    description: str = Field(min_length=1)
+    idea_id_seed: str = Field(min_length=1)
+    turns: list[DevinMultiTurnEvalTurn] = Field(min_length=1)
+
+
+DevinChatEvalCatalog.model_rebuild()
+_CHAT_EVAL_CATALOG = load_devin_chat_eval_catalog()
+DEFAULT_SCENARIOS: tuple[DevinEvalScenario, ...] = tuple(_CHAT_EVAL_CATALOG.single_turn)
+DEFAULT_MULTI_TURN_SCENARIOS: tuple[DevinMultiTurnEvalScenario, ...] = tuple(_CHAT_EVAL_CATALOG.multi_turn)
+
+
+def _normalize_text(value: Any) -> str:
+    return str(value or "").strip()
+
+
+_INTERNAL_ORCHESTRATION_TERMS = (
+    "dag",
+    "routernode",
+    "agentnode",
+    "dag handoff",
+    "pipeline handoff",
+)
+
+
+def _text_contains_all(text: str, needles: list[str]) -> bool:
+    hay = text.lower()
+    return all(str(item).strip().lower() in hay for item in needles if str(item).strip())
+
+
+def _text_contains_any(text: str, needles: list[str]) -> bool:
+    hay = text.lower()
+    return any(str(item).strip().lower() in hay for item in needles if str(item).strip())
+
+
+def _load_optional_json(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {}
+    return _load_json(path)
+
+
+def _copy_turn_artifacts(*, pipeline_dir: Path, turn_root: Path) -> dict[str, str]:
+    artifact_map = {
+        "route": "artifacts/route.json",
+        "idea_readiness_contract": "artifacts/idea_readiness_contract.json",
+        "idea_assumption_completion": "artifacts/idea_assumption_completion.json",
+        "idea_activation_decision": "artifacts/idea_activation_decision.json",
+        "devin_response": "artifacts/devin_response.json",
+        "devin_response_post": "artifacts/devin_response_post.json",
+        "devin_intake": "artifacts/devin_intake.json",
+        "idea_context_resolution": "artifacts/idea_context_resolution.json",
+        "devin_agent_loop_terminal": "artifacts/devin_agent_loop_terminal.json",
+    }
+    copied: dict[str, str] = {}
+    for kind, rel_path in artifact_map.items():
+        if _copy_if_exists(pipeline_dir / f"{kind}.json", turn_root / rel_path):
+            copied[kind] = rel_path
+    return copied
+
+
+def _response_message_from_payloads(*, response_payload: dict[str, Any], response_post: dict[str, Any], dag_result: Any) -> str:
+    candidates = [
+        response_payload.get("response_message"),
+        ((response_payload.get("response_guidance") or {}).get("response_message") if isinstance(response_payload.get("response_guidance"), dict) else None),
+        ((response_post.get("request") or {}).get("message") if isinstance(response_post.get("request"), dict) else None),
+        getattr(dag_result, "message", None),
+    ]
+    for item in candidates:
+        text = _normalize_text(item)
+        if text:
+            return text
+    return ""
+
+
+def _follow_up_questions_from_payloads(*, response_payload: dict[str, Any], response_post: dict[str, Any]) -> list[str]:
+    candidates = [
+        response_payload.get("follow_up_questions"),
+        ((response_payload.get("response_guidance") or {}).get("follow_up_questions") if isinstance(response_payload.get("response_guidance"), dict) else None),
+        (((response_post.get("request") or {}).get("metadata") or {}).get("follow_up_questions") if isinstance((response_post.get("request") or {}).get("metadata"), dict) else None),
+    ]
+    for item in candidates:
+        if isinstance(item, list):
+            return [str(entry).strip() for entry in item if str(entry).strip()]
+    return []
+
+
+def _suggested_next_step_from_payloads(*, response_payload: dict[str, Any], response_post: dict[str, Any]) -> str:
+    candidates = [
+        response_payload.get("suggested_next_step"),
+        ((response_payload.get("response_guidance") or {}).get("suggested_next_step") if isinstance(response_payload.get("response_guidance"), dict) else None),
+        (((response_post.get("request") or {}).get("metadata") or {}).get("suggested_next_step") if isinstance((response_post.get("request") or {}).get("metadata"), dict) else None),
+    ]
+    for item in candidates:
+        text = _normalize_text(item)
+        if text:
+            return text
+    return ""
+
+
+def _evaluate_turn_expectations(*, turn: DevinMultiTurnEvalTurn, observed: dict[str, Any]) -> dict[str, Any]:
+    message = _normalize_text(observed.get("message"))
+    follow_up_count = len(list(observed.get("follow_up_questions") or []))
+    suggested_next_step = _normalize_text(observed.get("suggested_next_step"))
+    checks = {
+        "route_arm": turn.expect_route_arm is None or observed.get("route_arm") == turn.expect_route_arm,
+        "status": turn.expect_status is None or observed.get("status") == turn.expect_status,
+        "response_kind": turn.expect_response_kind is None or observed.get("response_kind") == turn.expect_response_kind,
+        "activation": turn.expect_activation is None or bool(observed.get("activation_detected")) is bool(turn.expect_activation),
+        "must_mention": _text_contains_all(message, turn.must_mention),
+        "must_not": not _text_contains_any(message, turn.must_not),
+        "follow_up_question_count": turn.max_follow_up_questions is None or follow_up_count <= turn.max_follow_up_questions,
+        "suggested_next_step": turn.require_suggested_next_step is None or bool(suggested_next_step) is bool(turn.require_suggested_next_step),
+        "internal_orchestration": (not turn.forbid_internal_orchestration_terms) or (not _text_contains_any(message, list(_INTERNAL_ORCHESTRATION_TERMS))),
+    }
+    return {
+        "passed": all(checks.values()),
+        "checks": checks,
+        "follow_up_question_count": follow_up_count,
+    }
+
+
+def _summarize_behavior(results: list[dict[str, Any]]) -> dict[str, Any]:
+    ideation_turns = [item for item in results if item.get("route_arm") == "ideation"]
+    preapproval_turns = [
+        item
+        for item in ideation_turns
+        if item.get("expected_activation") is False
+    ]
+    approval_turns = [
+        item
+        for item in ideation_turns
+        if item.get("expected_activation") is True
+    ]
+    detour_seen = any(item.get("route_arm") in {"insight", "neither"} for item in results)
+    later_ideation_after_detour = False
+    saw_detour = False
+    for item in results:
+        if item.get("route_arm") in {"insight", "neither"}:
+            saw_detour = True
+            continue
+        if saw_detour and item.get("route_arm") == "ideation":
+            later_ideation_after_detour = True
+            break
+    ideation_without_premature_handoff = bool(preapproval_turns) and all(
+        item.get("status") == "ideation_contract_response"
+        and item.get("response_kind") == "ideation_contract_response"
+        and not bool(item.get("activation_detected"))
+        for item in preapproval_turns
+    )
+    approval_handoff_ready = (not approval_turns) or all(
+        item.get("status") == "queued"
+        and item.get("response_kind") == "ready_for_downstream"
+        and bool(item.get("activation_detected"))
+        for item in approval_turns
+    )
+    single_question_discipline = all(
+        bool((item.get("expectation") or {}).get("checks", {}).get("follow_up_question_count", True))
+        for item in results
+    )
+    abstracted_orchestration_visibility = all(
+        bool((item.get("expectation") or {}).get("checks", {}).get("internal_orchestration", True))
+        for item in results
+    )
+    momentum_preserved = all(
+        bool(item.get("activation_detected"))
+        or bool(_normalize_text(item.get("suggested_next_step")))
+        for item in ideation_turns
+    ) if ideation_turns else False
+    attention_discipline = (not detour_seen) or later_ideation_after_detour
+    return {
+        "ideation_without_premature_handoff": ideation_without_premature_handoff,
+        "approval_handoff_ready": approval_handoff_ready,
+        "ideation_insight_ideation_continuity": attention_discipline,
+        "all_three_arms_interplay": {"ideation", "insight", "neither"}.issubset({str(item.get("route_arm") or "") for item in results}),
+        "single_question_discipline": single_question_discipline,
+        "abstracted_orchestration_visibility": abstracted_orchestration_visibility,
+        "momentum_preserved": momentum_preserved,
+    }
+
+
+class DevinLiveTurnTimeout(RuntimeError):
+    pass
+
+
+def _latest_pipeline_dir_for_idea(*, repo_root: Path, idea_id: str) -> Path | None:
+    root = repo_root / ".devflow" / "ideas" / idea_id / "pipelines" / "devin_chat_dag"
+    if not root.exists():
+        return None
+    candidates = [path for path in root.iterdir() if path.is_dir()]
+    if not candidates:
+        return None
+    return max(candidates, key=lambda path: path.stat().st_mtime)
+
+
+def _run_turn_with_timeout(*, timeout_seconds: int, func):
+    if timeout_seconds <= 0 or not hasattr(signal, "SIGALRM"):
+        return func()
+    previous = signal.getsignal(signal.SIGALRM)
+
+    def _raise_timeout(signum, frame):
+        raise DevinLiveTurnTimeout(f"live turn exceeded timeout_seconds={timeout_seconds}")
+
+    signal.signal(signal.SIGALRM, _raise_timeout)
+    signal.setitimer(signal.ITIMER_REAL, timeout_seconds)
+    try:
+        return func()
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+        signal.signal(signal.SIGALRM, previous)
+
+
+def run_devin_multi_turn_eval(
+    *,
+    codebase_repo_root: Path,
+    target_repo_root: Path,
+    scenario: DevinMultiTurnEvalScenario,
+    evals_root: Path | None = None,
+    idea_id_prefix: str = "devin_live_eval",
+    turn_timeout_seconds: int = 180,
+) -> DevinEvalResult:
+    _require_real_eval_runtime()
+    run_id, started_at = _now_run_id()
+    target_repo_root = target_repo_root.expanduser().resolve()
+    suite_root = (evals_root or (target_repo_root / "evals" / "devin_multi_turn")) / run_id / scenario.scenario_id
+    idea_id = f"{idea_id_prefix}_{scenario.idea_id_seed}_{run_id[-6:].lower()}"
+    store = ExecutionStore(target_repo_root / ".devflow" / "execution.sqlite")
+    llm_config_path = Path(os.environ.get("DEVFLOW_HOME") or Path.home()).expanduser() / ".devflow" / "config.toml"
+    turn_results: list[dict[str, Any]] = []
+
+    for idx, turn in enumerate(scenario.turns, start=1):
+        dag_result = None
+        turn_error = None
+        try:
+            dag_result = _run_turn_with_timeout(
+                timeout_seconds=turn_timeout_seconds,
+                func=lambda: run_devin_chat_dag(
+                    repo_root=target_repo_root,
+                    store=store,
+                    idea_id=idea_id,
+                    text=turn.user_input,
+                    source_path=None,
+                    max_stories=turn.max_stories,
+                    planes=list(turn.planes),
+                ),
+            )
+            pipeline_dir = dag_result.pipeline_dir
+        except Exception as exc:
+            turn_error = f"{type(exc).__name__}: {exc}"
+            pipeline_dir = _latest_pipeline_dir_for_idea(repo_root=target_repo_root, idea_id=idea_id) or (suite_root / f"turn_{idx:02d}_{turn.turn_id}" / "missing_pipeline")
+        route_payload = _load_optional_json(pipeline_dir / "route.json")
+        contract_payload = _load_optional_json(pipeline_dir / "idea_readiness_contract.json")
+        activation_payload = _load_optional_json(pipeline_dir / "idea_activation_decision.json")
+        terminal_payload = _load_optional_json(pipeline_dir / "devin_agent_loop_terminal.json")
+        response_payload = _load_optional_json(pipeline_dir / "devin_response.json")
+        response_post = _load_optional_json(pipeline_dir / "devin_response_post.json")
+        idea_state = _load_optional_json(target_repo_root / ".devflow" / "ideas" / idea_id / "idea.json")
+        response_message = turn_error or _response_message_from_payloads(
+            response_payload=response_payload,
+            response_post=response_post,
+            dag_result=dag_result,
+        )
+        follow_up_questions = [] if turn_error else _follow_up_questions_from_payloads(
+            response_payload=response_payload,
+            response_post=response_post,
+        )
+        suggested_next_step = "" if turn_error else _suggested_next_step_from_payloads(
+            response_payload=response_payload,
+            response_post=response_post,
+        )
+        observed = {
+            "turn_index": idx,
+            "turn_id": turn.turn_id,
+            "user_input": turn.user_input,
+            "pipeline_dir": str(pipeline_dir),
+            "dag_run_id": None if dag_result is None else dag_result.run_id,
+            "exit_code": 1 if dag_result is None else dag_result.exit_code,
+            "route_arm": route_payload.get("route_arm"),
+            "status": turn_error or (dag_result.outcome.get("status") if dag_result is not None else None),
+            "response_kind": None if dag_result is None else (dag_result.outcome.get("response_kind") or ((dag_result.outcome.get("response_guidance") or {}).get("response_kind"))),
+            "message": response_message,
+            "follow_up_questions": follow_up_questions,
+            "suggested_next_step": suggested_next_step,
+            "expected_activation": turn.expect_activation,
+            "contract_status": contract_payload.get("status"),
+            "activation_detected": bool(activation_payload.get("activation_detected")),
+            "history_message_count": ((terminal_payload.get("history_grounding") or {}).get("history_message_count")),
+            "included_history_message_count": ((terminal_payload.get("history_grounding") or {}).get("included_history_message_count")),
+            "latest_message": idea_state.get("latest_message"),
+            "message_log_count": len(list(idea_state.get("message_log") or [])),
+            "response_payload": response_payload,
+            "response_post": response_post,
+        }
+        expectation = _evaluate_turn_expectations(turn=turn, observed=observed)
+        turn_root = suite_root / f"turn_{idx:02d}_{turn.turn_id}"
+        copied = _copy_turn_artifacts(pipeline_dir=pipeline_dir, turn_root=turn_root)
+        turn_record = {**observed, "expectation": expectation, "artifacts": copied}
+        turn_results.append(turn_record)
+        _write_json(turn_root / "turn_result.json", turn_record)
+
+    behavior = _summarize_behavior(turn_results)
+    summary = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "scenario": scenario.model_dump(),
+        "real_llm_eval": True,
+        "live_workflow": True,
+        "codebase_repo_root": str(codebase_repo_root.expanduser().resolve()),
+        "target_repo_root": str(target_repo_root),
+        "idea_id": idea_id,
+        "llm_config_path": str(llm_config_path),
+        "llm_cli_configured": llm_config_path.exists(),
+        "behavior_summary": behavior,
+        "turn_timeout_seconds": turn_timeout_seconds,
+        "all_turn_expectations_passed": all(bool(item.get("expectation", {}).get("passed")) for item in turn_results),
+        "turns": turn_results,
+    }
+    _write_json(suite_root / "eval_manifest.json", summary)
+    return DevinEvalResult(run_id=run_id, started_at=started_at, eval_root=suite_root, payload=summary)
+
+
+def run_devin_multi_turn_eval_suite(
+    *,
+    codebase_repo_root: Path,
+    target_repo_root: Path,
+    scenarios: list[DevinMultiTurnEvalScenario] | None = None,
+    evals_root: Path | None = None,
+    idea_id_prefix: str = "devin_live_eval",
+    turn_timeout_seconds: int = 180,
+) -> DevinEvalResult:
+    _require_real_eval_runtime()
+    run_id, started_at = _now_run_id()
+    suite_root = (evals_root or (target_repo_root.expanduser().resolve() / "evals" / "devin_multi_turn")) / run_id
+    chosen = scenarios or list(DEFAULT_MULTI_TURN_SCENARIOS)
+    scenario_results: list[dict[str, Any]] = []
+    for scenario in chosen:
+        result = run_devin_multi_turn_eval(
+            codebase_repo_root=codebase_repo_root,
+            target_repo_root=target_repo_root,
+            scenario=scenario,
+            evals_root=suite_root,
+            idea_id_prefix=idea_id_prefix,
+            turn_timeout_seconds=turn_timeout_seconds,
+        )
+        manifest = _load_json(result.eval_root / "eval_manifest.json")
+        scenario_results.append(
+            {
+                "scenario_id": scenario.scenario_id,
+                "persona": scenario.persona,
+                "eval_root": str(result.eval_root),
+                "all_turn_expectations_passed": bool(manifest.get("all_turn_expectations_passed")),
+                "behavior_summary": manifest.get("behavior_summary") or {},
+            }
+        )
+    summary = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "scenario_count": len(scenario_results),
+        "real_llm_eval": True,
+        "live_workflow": True,
+        "codebase_repo_root": str(codebase_repo_root.expanduser().resolve()),
+        "target_repo_root": str(target_repo_root.expanduser().resolve()),
+        "scenarios": scenario_results,
+        "turn_timeout_seconds": turn_timeout_seconds,
+        "all_scenarios_passed": all(bool(item.get("all_turn_expectations_passed")) for item in scenario_results),
+    }
+    _write_json(suite_root / "suite_summary.json", summary)
+    _write_json(suite_root / "eval_manifest.json", summary)
+    return DevinEvalResult(run_id=run_id, started_at=started_at, eval_root=suite_root, payload=summary)