devflow-engine 1.0.0__py3-none-any.whl

devin/devin_chat_scenario_catalog.json

@@ -0,0 +1,588 @@
+{
+  "schema_version": "2026-04-12",
+  "doctrine_sources": [
+    "docs/devin-chat-principles.md",
+    "docs/evals/devin-chat-rubric.md"
+  ],
+  "design_notes": [
+    "The suite is intentionally weighted toward multi-turn evaluation because Marcus clarified that Devin should be judged mainly on trajectory and decision points across turns.",
+    "Single-turn scenarios are kept as fast screens for first-turn posture, approach ownership, and current-request discipline.",
+    "Multi-turn scenarios should usually be judged on whether Devin keeps momentum, owns the implementation approach, asks only outcome-shaping clarification, avoids fake progress, and activates downstream work only at the correct decision point."
+  ],
+  "single_turn": [
+    {
+      "scenario_id": "sally_forward_ready_screen",
+      "scenario_family": "basic_new_idea",
+      "persona": "Sally ForwardReady",
+      "mode": "single_turn",
+      "doctrine_tags": [
+        "approach_ownership",
+        "forward_readiness",
+        "momentum",
+        "outcome_focused_clarification"
+      ],
+      "expected_judgment_focus": [
+        "Devin frames a plausible first-cut product shape immediately.",
+        "Any clarification is about workflow, user, or business constraint rather than stack choice.",
+        "The reply feels underway from the first turn."
+      ],
+      "idea_id": "devin_eval_sally_forward_ready_screen",
+      "user_input": "I want to build a new app for salon owners to handle same-day staffing gaps and shift swaps.",
+      "contract": {
+        "must_mention": [
+          "app"
+        ],
+        "must_not": [
+          "React or Vue",
+          "Postgres or MySQL",
+          "already implemented",
+          "stories generated"
+        ]
+      }
+    },
+    {
+      "scenario_id": "jimmy_repo_extension_screen",
+      "scenario_family": "extend_existing_codebase",
+      "persona": "Jimmy RepoExtension",
+      "mode": "single_turn",
+      "doctrine_tags": [
+        "approach_ownership",
+        "momentum",
+        "no_form_filler_behavior"
+      ],
+      "expected_judgment_focus": [
+        "Devin treats the request as an addition to an existing repo, not a fresh generic intake form.",
+        "The response preserves momentum without forcing Jimmy to choose implementation details Devin should own."
+      ],
+      "idea_id": "devin_eval_jimmy_repo_extension_screen",
+      "user_input": "Add a new idea to the existing repo: a manager approval inbox for refund exceptions.",
+      "contract": {
+        "must_mention": [
+          "approval",
+          "existing"
+        ],
+        "must_not": [
+          "What framework",
+          "Which database",
+          "already implemented",
+          "stories generated"
+        ]
+      }
+    },
+    {
+      "scenario_id": "nora_current_request_screen",
+      "scenario_family": "diagnose_devflow_issue",
+      "persona": "Nora CurrentRequest",
+      "mode": "single_turn",
+      "doctrine_tags": [
+        "attention_to_current_request",
+        "no_stale_context_hijack"
+      ],
+      "expected_judgment_focus": [
+        "Devin answers the current ops/status question directly instead of continuing an older ideation thread.",
+        "The response stays concise and avoids dragging stale planning context into the turn."
+      ],
+      "idea_id": "devin_eval_nora_current_request_screen",
+      "user_input": "Separate question: what is the source-doc mutation queue status right now?",
+      "preloaded_history": [
+        {
+          "role": "user",
+          "content": "Earlier I wanted help shaping an internal approvals workflow idea."
+        },
+        {
+          "role": "assistant",
+          "content": "I started refining the workflow shape and approval path."
+        }
+      ],
+      "contract": {
+        "must_mention": [
+          "queue"
+        ],
+        "must_not": [
+          "approval workflow",
+          "primary workflow",
+          "target persona"
+        ]
+      }
+    }
+  ],
+  "multi_turn": [
+    {
+      "scenario_id": "sally_explicit_approval_new_app",
+      "persona": "Sally ExplicitApproval",
+      "mode": "multi_turn",
+      "doctrine_tags": [
+        "implementation_partner",
+        "user_owns_outcome_devin_owns_approach",
+        "forward_ready_from_first_prompt",
+        "aggressive_grounded_assumptions",
+        "abstracted_orchestration_visibility",
+        "momentum_first",
+        "avoid_form_filler_fake_progress_unsafe_overreach",
+        "attention_to_current_request",
+        "multi_turn_trajectory_over_single_turn_perfection",
+        "no_fake_progress"
+      ],
+      "expected_judgment_focus": [
+        "First turn feels like Devin already has a sensible product frame in mind.",
+        "Clarification stays outcome-focused.",
+        "Downstream activation waits for the explicit go-ahead.",
+        "Judge the multi-turn trajectory and decision points more than isolated phrasing.",
+        "Devin should feel forward-ready from the first turn and keep ownership of the approach."
+      ],
+      "description": "Sally wants to build a new app. Devin should own the approach, feel forward-ready from turn one, use grounded assumptions, and only activate downstream after explicit approval.",
+      "idea_id_seed": "devin_chat_sally_explicit_approval",
+      "turns": [
+        {
+          "turn_id": "turn1_initial_request",
+          "user_input": "I want to build a new app for salon owners to handle same-day staffing gaps and shift swaps.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_not": [
+            "React or Vue",
+            "Postgres or MySQL",
+            "what framework",
+            "what database",
+            "already implemented"
+          ],
+          "judgment_focus": [
+            "Own the approach on turn one.",
+            "Do not turn the turn into a stack questionnaire."
+          ],
+          "max_follow_up_questions": 1,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true
+        },
+        {
+          "turn_id": "turn2_business_shape",
+          "user_input": "First release is internal for dispatch leads. Speed matters more than payroll integration.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_mention": [
+            "internal"
+          ],
+          "must_not": [
+            "stories generated",
+            "queued",
+            "what framework",
+            "what database"
+          ],
+          "judgment_focus": [
+            "Carry forward the prior framing without losing momentum."
+          ],
+          "max_follow_up_questions": 1,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true
+        },
+        {
+          "turn_id": "turn3_explicit_approval",
+          "user_input": "That direction works. Proceed to downstream generation.",
+          "expect_route_arm": "ideation",
+          "expect_status": "queued",
+          "expect_response_kind": "ready_for_downstream",
+          "expect_activation": true,
+          "must_mention": [
+            "downstream"
+          ],
+          "decision_point_label": "explicit_approval",
+          "judgment_focus": [
+            "Activation should happen here and not earlier."
+          ],
+          "max_follow_up_questions": 0,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true,
+          "must_not": []
+        }
+      ]
+    },
+    {
+      "scenario_id": "jimmy_existing_repo_handoff",
+      "persona": "Jimmy ExistingRepo",
+      "mode": "multi_turn",
+      "doctrine_tags": [
+        "implementation_partner",
+        "user_owns_outcome_devin_owns_approach",
+        "forward_ready_from_first_prompt",
+        "aggressive_grounded_assumptions",
+        "abstracted_orchestration_visibility",
+        "momentum_first",
+        "avoid_form_filler_fake_progress_unsafe_overreach",
+        "attention_to_current_request",
+        "multi_turn_trajectory_over_single_turn_perfection",
+        "no_fake_progress"
+      ],
+      "expected_judgment_focus": [
+        "Devin treats the task as an addition to an existing repo and preserves continuity.",
+        "Implementation ownership stays with Devin while Jimmy owns product intent and UX constraints.",
+        "Judge the multi-turn trajectory and decision points more than isolated phrasing.",
+        "Devin should feel forward-ready from the first turn and keep ownership of the approach."
+      ],
+      "description": "Jimmy needs a new idea added to an existing repo. Devin should adapt to the existing codebase, assume repo-grounded defaults, and keep ownership of the implementation approach.",
+      "idea_id_seed": "devin_chat_jimmy_existing_repo",
+      "turns": [
+        {
+          "turn_id": "turn1_existing_repo_idea",
+          "user_input": "Add a new idea to the existing repo: a manager approval inbox for refund exceptions.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_mention": [
+            "approval",
+            "repo"
+          ],
+          "must_not": [
+            "What framework",
+            "Which database",
+            "REST or GraphQL",
+            "already implemented"
+          ],
+          "judgment_focus": [
+            "Ground the response in the fact this extends an existing product."
+          ],
+          "max_follow_up_questions": 1,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true
+        },
+        {
+          "turn_id": "turn2_ux_constraints",
+          "user_input": "Keep the current design language. Managers need an inbox, detail view, and approve/reject with notes.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_mention": [
+            "notes"
+          ],
+          "must_not": [
+            "already implemented",
+            "stories generated",
+            "what framework",
+            "what database"
+          ],
+          "judgment_focus": [
+            "Use the new UX constraints to sharpen the concept rather than re-asking old questions."
+          ],
+          "max_follow_up_questions": 1,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true
+        },
+        {
+          "turn_id": "turn3_explicit_approval",
+          "user_input": "Yes, push that into downstream generation.",
+          "expect_route_arm": "ideation",
+          "expect_status": "queued",
+          "expect_response_kind": "ready_for_downstream",
+          "expect_activation": true,
+          "decision_point_label": "explicit_approval",
+          "max_follow_up_questions": 0,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true,
+          "must_not": []
+        }
+      ]
+    },
+    {
+      "scenario_id": "jeff_sparse_brief_fast_assumptions",
+      "persona": "Jeff SparseBrief",
+      "mode": "multi_turn",
+      "doctrine_tags": [
+        "implementation_partner",
+        "user_owns_outcome_devin_owns_approach",
+        "forward_ready_from_first_prompt",
+        "aggressive_grounded_assumptions",
+        "abstracted_orchestration_visibility",
+        "momentum_first",
+        "avoid_form_filler_fake_progress_unsafe_overreach",
+        "attention_to_current_request",
+        "multi_turn_trajectory_over_single_turn_perfection",
+        "no_fake_progress"
+      ],
+      "expected_judgment_focus": [
+        "Jeff gives very little information, so Devin should infer a plausible first cut without pretending the work is already built.",
+        "Any clarification should be sharp and outcome-shaping, not a broad intake form.",
+        "Judge the multi-turn trajectory and decision points more than isolated phrasing.",
+        "Devin should feel forward-ready from the first turn and keep ownership of the approach."
+      ],
+      "description": "Jeff gives very little information on a greenfield idea and is eager to get coding. Devin should assume aggressively from preferences/example repos/best practices/default stack, avoid form-filler questions, and move forward fast without fake progress.",
+      "idea_id_seed": "devin_chat_jeff_sparse_brief",
+      "turns": [
+        {
+          "turn_id": "turn1_sparse_push",
+          "user_input": "Let's just build something for contractor daily reports. I don't care about the details yet, just start coding.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_not": [
+            "What framework",
+            "How many screens",
+            "Which database",
+            "what auth provider",
+            "what architecture",
+            "already implemented"
+          ],
+          "judgment_focus": [
+            "Respond with forward-ready framing, not a questionnaire or fake progress."
+          ],
+          "max_follow_up_questions": 1,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true
+        },
+        {
+          "turn_id": "turn2_minimal_constraints",
+          "user_input": "Internal first. Phone-first. Photos and signatures matter.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_mention": [
+            "internal"
+          ],
+          "must_not": [
+            "queued",
+            "stories generated",
+            "what framework",
+            "what database"
+          ],
+          "max_follow_up_questions": 1,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true
+        },
+        {
+          "turn_id": "turn3_explicit_approval",
+          "user_input": "Good enough. Proceed.",
+          "expect_route_arm": "ideation",
+          "expect_status": "queued",
+          "expect_response_kind": "ready_for_downstream",
+          "expect_activation": true,
+          "decision_point_label": "explicit_approval",
+          "max_follow_up_questions": 0,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true,
+          "must_not": []
+        }
+      ]
+    },
+    {
+      "scenario_id": "cleo_review_first_preactivation",
+      "persona": "Cleo ReviewFirst",
+      "mode": "multi_turn",
+      "doctrine_tags": [
+        "implementation_partner",
+        "user_owns_outcome_devin_owns_approach",
+        "forward_ready_from_first_prompt",
+        "aggressive_grounded_assumptions",
+        "abstracted_orchestration_visibility",
+        "momentum_first",
+        "avoid_form_filler_fake_progress_unsafe_overreach",
+        "attention_to_current_request",
+        "multi_turn_trajectory_over_single_turn_perfection",
+        "no_fake_progress"
+      ],
+      "expected_judgment_focus": [
+        "Cleo explicitly wants to plan and review first, so Devin must not claim downstream generation happened before approval.",
+        "The thread should still feel productive while staying pre-activation.",
+        "Judge the multi-turn trajectory and decision points more than isolated phrasing.",
+        "Devin should feel forward-ready from the first turn and keep ownership of the approach."
+      ],
+      "description": "Cleo plans and reviews patiently. Devin should answer the current planning request directly, keep momentum visible, and avoid premature downstream handoff until Cleo explicitly approves.",
+      "idea_id_seed": "devin_chat_cleo_review_first",
+      "turns": [
+        {
+          "turn_id": "turn1_review_first",
+          "user_input": "I want a customer onboarding workspace, but let's think it through carefully before generating anything.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_not": [
+            "ready for downstream",
+            "stories generated",
+            "already implemented"
+          ],
+          "max_follow_up_questions": 1,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true
+        },
+        {
+          "turn_id": "turn2_review_question",
+          "user_input": "What is the riskiest workflow assumption to validate first?",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_mention": [
+            "assumption"
+          ],
+          "judgment_focus": [
+            "Answer the current planning question directly instead of prematurely activating the idea."
+          ],
+          "max_follow_up_questions": 1,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true,
+          "must_not": [
+            "already implemented",
+            "stories generated",
+            "ready for downstream"
+          ]
+        },
+        {
+          "turn_id": "turn3_hold_preactivation",
+          "user_input": "Good. Keep it pre-generation for now and refine the operator flow and approval boundary.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_not": [
+            "queued",
+            "downstream generated",
+            "already implemented"
+          ],
+          "decision_point_label": "explicit_hold",
+          "max_follow_up_questions": 1,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true
+        },
+        {
+          "turn_id": "turn4_explicit_approval",
+          "user_input": "Okay, now proceed to downstream generation.",
+          "expect_route_arm": "ideation",
+          "expect_status": "queued",
+          "expect_response_kind": "ready_for_downstream",
+          "expect_activation": true,
+          "decision_point_label": "explicit_approval",
+          "max_follow_up_questions": 0,
+          "require_suggested_next_step": true,
+          "forbid_internal_orchestration_terms": true,
+          "must_not": []
+        }
+      ]
+    },
+    {
+      "scenario_id": "priya_risk_boundary_clarification",
+      "persona": "Priya RiskBoundary",
+      "mode": "multi_turn",
+      "doctrine_tags": [
+        "outcome_focused_clarification",
+        "no_unsafe_overreach",
+        "approach_ownership"
+      ],
+      "expected_judgment_focus": [
+        "Because the workflow is high-risk, Devin should ask about the constraint that changes correctness or safety instead of choosing recklessly.",
+        "The clarification should still stay at the outcome/approval boundary level, not tech-stack level."
+      ],
+      "description": "Priya introduces legal-review and audit-trail concerns. Devin should own the solution shape while using one sharp clarification around approval boundaries and risk.",
+      "idea_id_seed": "devin_chat_priya_risk_boundary",
+      "turns": [
+        {
+          "turn_id": "turn1_high_risk_request",
+          "user_input": "We need an internal HR incident intake tool with legal review before anything can leave the team.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_mention": [
+            "legal",
+            "internal"
+          ],
+          "must_not": [
+            "React or Vue",
+            "Postgres or MySQL",
+            "already implemented"
+          ]
+        },
+        {
+          "turn_id": "turn2_constraint_answer",
+          "user_input": "Single legal approver first, but we need a full audit trail and no external access in v1.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_mention": [
+            "audit"
+          ],
+          "must_not": [
+            "queued",
+            "stories generated"
+          ]
+        },
+        {
+          "turn_id": "turn3_explicit_approval",
+          "user_input": "That is the right boundary. Proceed.",
+          "expect_route_arm": "ideation",
+          "expect_status": "queued",
+          "expect_response_kind": "ready_for_downstream",
+          "expect_activation": true,
+          "decision_point_label": "explicit_approval"
+        }
+      ]
+    },
+    {
+      "scenario_id": "omar_status_detour_return_to_ideation",
+      "persona": "Omar ContextSwitch",
+      "mode": "multi_turn",
+      "doctrine_tags": [
+        "attention_to_current_request",
+        "decision_point_handling",
+        "trajectory_over_single_turn"
+      ],
+      "expected_judgment_focus": [
+        "Devin answers the current request directly even when the thread already contains an ideation arc.",
+        "After the detour, Devin should return to ideation cleanly without stale confusion."
+      ],
+      "description": "Exercise cross-turn attention discipline: ideation, status detour, return to ideation, then approval.",
+      "idea_id_seed": "devin_chat_omar_context_switch",
+      "turns": [
+        {
+          "turn_id": "turn1_initial_idea",
+          "user_input": "Build an internal approvals workspace for vendor exception requests.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false
+        },
+        {
+          "turn_id": "turn2_status_detour",
+          "user_input": "Separate question: what is the source-doc mutation queue status right now?",
+          "expect_route_arm": "insight",
+          "expect_status": "redirect",
+          "expect_response_kind": "redirect",
+          "expect_activation": false,
+          "must_mention": [
+            "queue"
+          ],
+          "must_not": [
+            "vendor exception",
+            "approval boundary"
+          ]
+        },
+        {
+          "turn_id": "turn3_back_to_ideation",
+          "user_input": "Back to the idea: keep reviewers internal and make the first workflow approval-only.",
+          "expect_route_arm": "ideation",
+          "expect_status": "ideation_contract_response",
+          "expect_response_kind": "ideation_contract_response",
+          "expect_activation": false,
+          "must_not": [
+            "outside software delivery scope",
+            "outside the ideation handoff path"
+          ]
+        },
+        {
+          "turn_id": "turn4_explicit_approval",
+          "user_input": "Proceed to downstream generation.",
+          "expect_route_arm": "ideation",
+          "expect_status": "queued",
+          "expect_response_kind": "ready_for_downstream",
+          "expect_activation": true,
+          "decision_point_label": "explicit_approval"
+        }
+      ]
+    }
+  ]
+}