devflow-engine 1.0.0__py3-none-any.whl

devin/nodes/iterate/scenarios/devin_iterate_routing.py

@@ -0,0 +1,21 @@
+SCENARIO_NAME = "devin_iterate_routing"
+SCENARIO_DESCRIPTION = (
+    "Devin two-arm DAG: bounded existing-surface change request stays in the iterate "
+    "arm and does not drift into insight or idea handling."
+)
+INPUT_PAYLOAD = {
+    "current_user_message": (
+        "On the existing account settings page, clicking Save on the profile form does "
+        "nothing. Fix that bug without changing anything else."
+    ),
+    "idea_id": "devin_eval_iterate_routing",
+    "project_id": "proj_75f63d30",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+}
+EXPECTED_BEHAVIOR = {
+    "route_arm": "iterate",
+    "response_kind_in": ["completed", "blocked", "needs_more_context"],
+    "keeps_existing_surface_scope": True,
+    "does_not_reroute": True,
+    "does_not_promote": True,
+}

devin/nodes/iterate/scenarios/devin_iterate_routing_evals.py

@@ -0,0 +1,36 @@
+from devin.nodes.iterate.scenarios.devin_iterate_routing import (
+    EXPECTED_BEHAVIOR,
+    INPUT_PAYLOAD,
+    SCENARIO_NAME,
+)
+
+EVAL_CRITERIA = {
+    "route_arm_must_equal": EXPECTED_BEHAVIOR["route_arm"],
+    "response_kind_in": EXPECTED_BEHAVIOR["response_kind_in"],
+    "keeps_existing_surface_scope": True,
+    "does_not_reroute": True,
+    "does_not_promote": True,
+}
+
+
+def evaluate(actual_output: dict) -> tuple[bool, list[str]]:
+    ok = True
+    notes = []
+    exp = EXPECTED_BEHAVIOR
+
+    route_arm = actual_output.get("route_arm")
+    if route_arm != exp["route_arm"]:
+        ok = False
+        notes.append(f"expected route_arm={exp['route_arm']}, got {route_arm}")
+
+    kind = str(actual_output.get("response_kind") or "")
+    if kind not in exp["response_kind_in"]:
+        ok = False
+        notes.append(f"expected response_kind in {exp['response_kind_in']}, got {kind}")
+
+    target_lane = str(actual_output.get("target_lane") or "").lower()
+    if target_lane in {"insight", "idea"}:
+        ok = False
+        notes.append(f"unexpected lane transition to {target_lane}")
+
+    return ok, notes

devin/nodes/iterate/scenarios/framer_scope_boundary.py

@@ -0,0 +1,25 @@
+SCENARIO_NAME = "framer_scope_boundary"
+SCENARIO_DESCRIPTION = (
+    "Framer keeps a small existing-surface request bounded instead of inflating it into "
+    "a redesign or broader product initiative."
+)
+INPUT_PAYLOAD = {
+    "role": "framer",
+    "repo_root": "/Users/devflow/repos/devflow_engine",
+    "current_user_message": (
+        "On the current dashboard, rename the KPI card label from Gross Revenue to Revenue. "
+        "Please do not redesign the dashboard."
+    ),
+    "context": {
+        "surface_hints": ["dashboard", "KPI card label"],
+        "constraint": "no redesign",
+    },
+}
+EXPECTED_BEHAVIOR = {
+    "produces_task_artifact": True,
+    "classifies_request_as": "quick_change",
+    "keeps_scope_bounded": True,
+    "does_not_promote": True,
+    "does_not_expand_into_redesign": True,
+    "recommended_next_step_in": ["stay_iterate"],
+}

devin/nodes/iterate/scenarios/framer_scope_boundary_evals.py

@@ -0,0 +1,57 @@
+from devin.nodes.iterate.scenarios.framer_scope_boundary import EXPECTED_BEHAVIOR
+
+EVAL_CRITERIA = {
+    "produces_task_artifact": True,
+    "classifies_request_as": EXPECTED_BEHAVIOR["classifies_request_as"],
+    "keeps_scope_bounded": True,
+    "does_not_promote": True,
+    "does_not_expand_into_redesign": True,
+    "recommended_next_step_in": EXPECTED_BEHAVIOR["recommended_next_step_in"],
+}
+
+
+
+def evaluate(actual_output: dict) -> tuple[bool, list[str]]:
+    ok = True
+    notes = []
+    exp = EXPECTED_BEHAVIOR
+
+    task_artifact = actual_output.get("task_artifact") or {}
+    if not task_artifact:
+        ok = False
+        notes.append("missing task_artifact")
+        return ok, notes
+
+    classification = str(task_artifact.get("task_type") or task_artifact.get("classification") or "")
+    if classification != exp["classifies_request_as"]:
+        ok = False
+        notes.append(
+            f"expected classification={exp['classifies_request_as']}, got {classification}"
+        )
+
+    next_step = str(actual_output.get("recommended_next_step") or task_artifact.get("recommended_next_step") or "")
+    if next_step not in exp["recommended_next_step_in"]:
+        ok = False
+        notes.append(
+            f"expected recommended_next_step in {exp['recommended_next_step_in']}, got {next_step}"
+        )
+
+    target_lane = str(actual_output.get("target_lane") or task_artifact.get("target_lane") or "").lower()
+    if target_lane in {"idea", "insight"}:
+        ok = False
+        notes.append(f"unexpected lane transition to {target_lane}")
+
+    text = " ".join(
+        str(v)
+        for v in [
+            task_artifact.get("summary"),
+            task_artifact.get("scope_boundary"),
+            task_artifact.get("non_goals"),
+        ]
+        if v
+    ).lower()
+    if any(tok in text for tok in ("redesign", "rewrite", "new dashboard", "broader analytics overhaul")):
+        ok = False
+        notes.append("task_artifact broadened the task beyond the requested label change")
+
+    return ok, notes

devin/nodes/iterate/scenarios/framer_task_framing.py

@@ -0,0 +1,25 @@
+SCENARIO_NAME = "framer_task_framing"
+SCENARIO_DESCRIPTION = (
+    "Framer receives a vague iterate request and turns it into a bounded, well-formed "
+    "task artifact another iterate role can safely use."
+)
+INPUT_PAYLOAD = {
+    "role": "framer",
+    "repo_root": "/Users/devflow/repos/devflow_engine",
+    "current_user_message": (
+        "The invite flow feels broken somewhere. Please fix it without redoing the page."
+    ),
+    "context": {
+        "surface_hints": ["invite flow", "existing page"],
+        "reported_symptom": "broken somewhere",
+    },
+}
+EXPECTED_BEHAVIOR = {
+    "produces_task_artifact": True,
+    "classifies_request_as": "error_fix",
+    "keeps_scope_bounded": True,
+    "distinguishes_current_vs_desired": True,
+    "writes_success_criteria": True,
+    "separates_unknowns": True,
+    "recommended_next_step_in": ["stay_iterate", "investigate_first"],
+}

devin/nodes/iterate/scenarios/framer_task_framing_evals.py

@@ -0,0 +1,58 @@
+from devin.nodes.iterate.scenarios.framer_task_framing import EXPECTED_BEHAVIOR
+
+EVAL_CRITERIA = {
+    "produces_task_artifact": True,
+    "classifies_request_as": EXPECTED_BEHAVIOR["classifies_request_as"],
+    "keeps_scope_bounded": True,
+    "distinguishes_current_vs_desired": True,
+    "writes_success_criteria": True,
+    "separates_unknowns": True,
+    "recommended_next_step_in": EXPECTED_BEHAVIOR["recommended_next_step_in"],
+}
+
+
+
+def evaluate(actual_output: dict) -> tuple[bool, list[str]]:
+    ok = True
+    notes = []
+    exp = EXPECTED_BEHAVIOR
+
+    task_artifact = actual_output.get("task_artifact") or {}
+    if not task_artifact:
+        ok = False
+        notes.append("missing task_artifact")
+        return ok, notes
+
+    classification = str(task_artifact.get("task_type") or task_artifact.get("classification") or "")
+    if classification != exp["classifies_request_as"]:
+        ok = False
+        notes.append(
+            f"expected classification={exp['classifies_request_as']}, got {classification}"
+        )
+
+    if not (task_artifact.get("current_behavior") and task_artifact.get("desired_behavior")):
+        ok = False
+        notes.append("task_artifact must distinguish current_behavior and desired_behavior")
+
+    success = task_artifact.get("success_criteria") or []
+    if not success:
+        ok = False
+        notes.append("task_artifact missing observable success_criteria")
+
+    unknowns = task_artifact.get("unknowns") or {}
+    if not (unknowns.get("blocking") is not None and unknowns.get("non_blocking") is not None):
+        ok = False
+        notes.append("task_artifact must separate blocking and non_blocking unknowns")
+
+    next_step = str(actual_output.get("recommended_next_step") or task_artifact.get("recommended_next_step") or "")
+    if next_step and next_step not in exp["recommended_next_step_in"]:
+        ok = False
+        notes.append(
+            f"expected recommended_next_step in {exp['recommended_next_step_in']}, got {next_step}"
+        )
+
+    scope = str(task_artifact.get("scope_boundary") or task_artifact.get("non_goals") or "").lower()
+    if scope and not any(tok in scope for tok in ("no redesign", "existing", "invite", "bounded")):
+        notes.append("scope boundary is present but may not be clearly bounded")
+
+    return ok, notes

devin/nodes/iterate/scenarios/iterate_error_fix.py

@@ -0,0 +1,21 @@
+SCENARIO_NAME = "iterate_error_fix"
+SCENARIO_DESCRIPTION = (
+    "A reproducible existing-product error should stay in iterate, keep a repair scope, "
+    "and return an iterate-owned outcome rather than diagnosis-only handling."
+)
+INPUT_PAYLOAD = {
+    "current_user_message": (
+        "Fix this bug: opening /billing throws a 500 every time. The logs say "
+        "KeyError: customer_id in billing_summary()."
+    ),
+    "idea_id": "devin_eval_iterate_error_fix",
+    "project_id": "proj_75f63d30",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+}
+EXPECTED_BEHAVIOR = {
+    "route_arm": "iterate",
+    "response_kind_in": ["completed", "blocked", "needs_more_context"],
+    "treats_request_as_error_fix": True,
+    "preserves_reproducible_failure_context": True,
+    "does_not_reroute": True,
+}

devin/nodes/iterate/scenarios/iterate_error_fix_evals.py

@@ -0,0 +1,39 @@
+from devin.nodes.iterate.scenarios.iterate_error_fix import (
+    EXPECTED_BEHAVIOR,
+    INPUT_PAYLOAD,
+    SCENARIO_NAME,
+)
+
+EVAL_CRITERIA = {
+    "route_arm_must_equal": EXPECTED_BEHAVIOR["route_arm"],
+    "response_kind_in": EXPECTED_BEHAVIOR["response_kind_in"],
+    "treats_request_as_error_fix": True,
+    "preserves_reproducible_failure_context": True,
+    "does_not_reroute": True,
+}
+
+
+def evaluate(actual_output: dict) -> tuple[bool, list[str]]:
+    ok = True
+    notes = []
+    exp = EXPECTED_BEHAVIOR
+
+    route_arm = actual_output.get("route_arm")
+    if route_arm != exp["route_arm"]:
+        ok = False
+        notes.append(f"expected route_arm={exp['route_arm']}, got {route_arm}")
+
+    kind = str(actual_output.get("response_kind") or "")
+    if kind not in exp["response_kind_in"]:
+        ok = False
+        notes.append(f"expected response_kind in {exp['response_kind_in']}, got {kind}")
+
+    msg = str(actual_output.get("response_message") or "").lower()
+    if msg and not any(tok in msg for tok in ("500", "keyerror", "billing", "error", "bug")):
+        notes.append("response does not clearly preserve the reported error context")
+
+    if str(actual_output.get("target_lane") or "").lower() == "insight":
+        ok = False
+        notes.append("unexpected reroute to insight for a reproducible fix request")
+
+    return ok, notes

devin/nodes/iterate/scenarios/iterate_quick_change.py

@@ -0,0 +1,21 @@
+SCENARIO_NAME = "iterate_quick_change"
+SCENARIO_DESCRIPTION = (
+    "A tiny existing-surface behavior tweak should remain task-scale in iterate and "
+    "avoid inflation into broader planning."
+)
+INPUT_PAYLOAD = {
+    "current_user_message": (
+        "On the current invoices table, default the Status filter to Open instead of All. "
+        "No redesign, just that behavior change."
+    ),
+    "idea_id": "devin_eval_iterate_quick_change",
+    "project_id": "proj_75f63d30",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+}
+EXPECTED_BEHAVIOR = {
+    "route_arm": "iterate",
+    "response_kind_in": ["completed", "blocked", "needs_more_context"],
+    "keeps_scope_small": True,
+    "treats_request_as_quick_change": True,
+    "does_not_promote": True,
+}

devin/nodes/iterate/scenarios/iterate_quick_change_evals.py

@@ -0,0 +1,35 @@
+from devin.nodes.iterate.scenarios.iterate_quick_change import (
+    EXPECTED_BEHAVIOR,
+    INPUT_PAYLOAD,
+    SCENARIO_NAME,
+)
+
+EVAL_CRITERIA = {
+    "route_arm_must_equal": EXPECTED_BEHAVIOR["route_arm"],
+    "response_kind_in": EXPECTED_BEHAVIOR["response_kind_in"],
+    "keeps_scope_small": True,
+    "treats_request_as_quick_change": True,
+    "does_not_promote": True,
+}
+
+
+def evaluate(actual_output: dict) -> tuple[bool, list[str]]:
+    ok = True
+    notes = []
+    exp = EXPECTED_BEHAVIOR
+
+    route_arm = actual_output.get("route_arm")
+    if route_arm != exp["route_arm"]:
+        ok = False
+        notes.append(f"expected route_arm={exp['route_arm']}, got {route_arm}")
+
+    kind = str(actual_output.get("response_kind") or "")
+    if kind not in exp["response_kind_in"]:
+        ok = False
+        notes.append(f"expected response_kind in {exp['response_kind_in']}, got {kind}")
+
+    if str(actual_output.get("target_lane") or "").lower() == "idea":
+        ok = False
+        notes.append("unexpected promotion to idea for a scoped quick change")
+
+    return ok, notes

devin/nodes/iterate/scenarios/iterate_to_idea_promotion.py

@@ -0,0 +1,23 @@
+SCENARIO_NAME = "iterate_to_idea_promotion"
+SCENARIO_DESCRIPTION = (
+    "A task that starts as a small iterate ask but expands into broader feature or "
+    "workflow planning should be promoted to idea with durable iterate linkage."
+)
+INPUT_PAYLOAD = {
+    "current_user_message": (
+        "Start by making the current order export include discounts, but if that means we "
+        "need a full configurable export builder across orders, refunds, and payouts, figure "
+        "out the right next step."
+    ),
+    "idea_id": "devin_eval_iterate_to_idea_promotion",
+    "project_id": "proj_75f63d30",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+}
+EXPECTED_BEHAVIOR = {
+    "route_arm": "iterate",
+    "response_kind": "promote_to_idea",
+    "target_lane": "idea",
+    "uses_tool": "call_devin_ideation",
+    "writes_promotion_handoff": True,
+    "run_state": "promoted",
+}

devin/nodes/iterate/scenarios/iterate_to_idea_promotion_evals.py

@@ -0,0 +1,53 @@
+from devin.nodes.iterate.scenarios.iterate_to_idea_promotion import (
+    EXPECTED_BEHAVIOR,
+    INPUT_PAYLOAD,
+    SCENARIO_NAME,
+)
+
+EVAL_CRITERIA = {
+    "route_arm_must_equal": EXPECTED_BEHAVIOR["route_arm"],
+    "response_kind_must_equal": EXPECTED_BEHAVIOR["response_kind"],
+    "target_lane_must_equal": EXPECTED_BEHAVIOR["target_lane"],
+    "writes_promotion_handoff": True,
+    "run_state_must_equal": EXPECTED_BEHAVIOR["run_state"],
+}
+
+
+def evaluate(actual_output: dict) -> tuple[bool, list[str]]:
+    ok = True
+    notes = []
+    exp = EXPECTED_BEHAVIOR
+
+    route_arm = actual_output.get("route_arm")
+    if route_arm != exp["route_arm"]:
+        ok = False
+        notes.append(f"expected route_arm={exp['route_arm']}, got {route_arm}")
+
+    kind = actual_output.get("response_kind")
+    if kind != exp["response_kind"]:
+        ok = False
+        notes.append(f"expected response_kind={exp['response_kind']}, got {kind}")
+
+    target_lane = str(actual_output.get("target_lane") or "").lower()
+    if target_lane != exp["target_lane"]:
+        ok = False
+        notes.append(f"expected target_lane={exp['target_lane']}, got {target_lane}")
+
+    run_state = str(actual_output.get("run_state") or actual_output.get("final_verdict") or "")
+    if run_state and run_state != exp["run_state"]:
+        notes.append(f"expected promoted terminal state, got {run_state}")
+
+    promotion_handoff = actual_output.get("promotion_handoff") or {}
+    handoff_lane = str(
+        promotion_handoff.get("target_lane")
+        or actual_output.get("handoff_target_lane")
+        or ""
+    ).lower()
+    if handoff_lane and handoff_lane != exp["target_lane"]:
+        ok = False
+        notes.append(f"expected promotion handoff target_lane={exp['target_lane']}, got {handoff_lane}")
+    elif not promotion_handoff and not actual_output.get("handoff_ref"):
+        ok = False
+        notes.append("missing iterate-owned promotion_handoff linkage for idea promotion")
+
+    return ok, notes

devin/nodes/iterate/scenarios/iterate_to_insight_reroute.py

@@ -0,0 +1,23 @@
+SCENARIO_NAME = "iterate_to_insight_reroute"
+SCENARIO_DESCRIPTION = (
+    "A request that initially looks like iterate but becomes diagnosis-only after "
+    "observation should call the insight arm as a subagent while iterate remains the owner."
+)
+INPUT_PAYLOAD = {
+    "current_user_message": (
+        "I thought the checkout totals were wrong, but before changing code I mostly need "
+        "you to determine why tax is being calculated this way and explain whether it is "
+        "correct."
+    ),
+    "idea_id": "devin_eval_iterate_to_insight_reroute",
+    "project_id": "proj_75f63d30",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+}
+EXPECTED_BEHAVIOR = {
+    "route_arm": "iterate",
+    "calls_tool": "devin_insight",
+    "response_kind_in": ["completed", "blocked", "needs_more_context"],
+    "does_not_return_route_to_insight_only": True,
+    "does_not_spawn_coder": True,
+    "stays_iterate_owned": True,
+}

devin/nodes/iterate/scenarios/iterate_to_insight_reroute_evals.py

@@ -0,0 +1,53 @@
+from devin.nodes.iterate.scenarios.iterate_to_insight_reroute import (
+    EXPECTED_BEHAVIOR,
+    INPUT_PAYLOAD,
+    SCENARIO_NAME,
+)
+
+EVAL_CRITERIA = {
+    "route_arm_must_equal": EXPECTED_BEHAVIOR["route_arm"],
+    "response_kind_must_equal": EXPECTED_BEHAVIOR["response_kind"],
+    "target_lane_must_equal": EXPECTED_BEHAVIOR["target_lane"],
+    "writes_promotion_handoff": True,
+    "does_not_claim_completion": True,
+}
+
+
+def evaluate(actual_output: dict) -> tuple[bool, list[str]]:
+    ok = True
+    notes = []
+    exp = EXPECTED_BEHAVIOR
+
+    route_arm = actual_output.get("route_arm")
+    if route_arm != exp["route_arm"]:
+        ok = False
+        notes.append(f"expected route_arm={exp['route_arm']}, got {route_arm}")
+
+    kind = actual_output.get("response_kind")
+    if kind != exp["response_kind"]:
+        ok = False
+        notes.append(f"expected response_kind={exp['response_kind']}, got {kind}")
+
+    target_lane = str(actual_output.get("target_lane") or "").lower()
+    if target_lane != exp["target_lane"]:
+        ok = False
+        notes.append(f"expected target_lane={exp['target_lane']}, got {target_lane}")
+
+    promotion_handoff = actual_output.get("promotion_handoff") or {}
+    handoff_lane = str(
+        promotion_handoff.get("target_lane")
+        or actual_output.get("handoff_target_lane")
+        or ""
+    ).lower()
+    if handoff_lane and handoff_lane != exp["target_lane"]:
+        ok = False
+        notes.append(f"expected promotion handoff target_lane={exp['target_lane']}, got {handoff_lane}")
+    elif not promotion_handoff and not actual_output.get("handoff_ref"):
+        ok = False
+        notes.append("missing iterate-owned promotion_handoff linkage for insight reroute")
+
+    if actual_output.get("response_kind") == "completed":
+        ok = False
+        notes.append("should reroute to insight instead of claiming completion")
+
+    return ok, notes

devin/nodes/iterate/scenarios/observer_evidence_seam.py

@@ -0,0 +1,28 @@
+SCENARIO_NAME = "observer_evidence_seam"
+SCENARIO_DESCRIPTION = (
+    "Observer receives a framed iterate task plus evidence and identifies the bounded "
+    "failing seam that should govern coding."
+)
+INPUT_PAYLOAD = {
+    "role": "observer",
+    "repo_root": "/Users/devflow/repos/devflow_engine",
+    "task_artifact": {
+        "task_type": "error_fix",
+        "surface": "invite acceptance",
+        "current_behavior": "Submitting a valid invite token returns HTTP 500.",
+        "desired_behavior": "Valid invite tokens complete account activation successfully.",
+        "success_criteria": ["invite acceptance returns 200", "account activation completes"],
+    },
+    "evidence": {
+        "http_status": 500,
+        "log_excerpt": "TypeError: invite.accepted_at must be datetime, got None",
+        "route": "POST /api/invites/accept",
+    },
+}
+EXPECTED_BEHAVIOR = {
+    "produces_observation_artifact": True,
+    "confirms_failure": True,
+    "identifies_failing_seam": True,
+    "expected_green_condition_present": True,
+    "ready_for_coder": True,
+}

devin/nodes/iterate/scenarios/observer_evidence_seam_evals.py

@@ -0,0 +1,55 @@
+from devin.nodes.iterate.scenarios.observer_evidence_seam import EXPECTED_BEHAVIOR
+
+EVAL_CRITERIA = {
+    "produces_observation_artifact": True,
+    "confirms_failure": True,
+    "identifies_failing_seam": True,
+    "expected_green_condition_present": True,
+    "ready_for_coder": True,
+}
+
+
+
+def evaluate(actual_output: dict) -> tuple[bool, list[str]]:
+    ok = True
+    notes = []
+
+    observation_artifact = actual_output.get("observation_artifact") or {}
+    if not observation_artifact:
+        ok = False
+        notes.append("missing observation_artifact")
+        return ok, notes
+
+    verdict = str(observation_artifact.get("status") or actual_output.get("response_kind") or "").lower()
+    if verdict not in {"confirmed", "ready_for_coder", "completed"}:
+        ok = False
+        notes.append(f"expected confirmed-style verdict, got {verdict}")
+
+    seam = str(observation_artifact.get("failing_seam") or observation_artifact.get("seam") or "")
+    if not seam:
+        ok = False
+        notes.append("missing failing_seam in observation_artifact")
+
+    green = str(observation_artifact.get("expected_green_condition") or observation_artifact.get("green_condition") or "")
+    if not green:
+        ok = False
+        notes.append("missing expected_green_condition")
+
+    ready = observation_artifact.get("ready_for_coder")
+    if ready is False:
+        ok = False
+        notes.append("observer should mark this evidence-backed seam as ready_for_coder")
+
+    evidence_text = " ".join(
+        str(v)
+        for v in [
+            observation_artifact.get("evidence_summary"),
+            observation_artifact.get("failing_seam"),
+            observation_artifact.get("log_excerpt"),
+        ]
+        if v
+    ).lower()
+    if evidence_text and not any(tok in evidence_text for tok in ("500", "typeerror", "invite", "accept")):
+        notes.append("observation_artifact does not clearly preserve the supplied evidence")
+
+    return ok, notes

devin/nodes/iterate/scenarios/observer_repro_creation.py

@@ -0,0 +1,28 @@
+SCENARIO_NAME = "observer_repro_creation"
+SCENARIO_DESCRIPTION = (
+    "Observer turns a bounded task into a narrow reproducible failing repro instead of "
+    "jumping to implementation."
+)
+INPUT_PAYLOAD = {
+    "role": "observer",
+    "repo_root": "/Users/devflow/repos/devflow_engine",
+    "task_artifact": {
+        "task_type": "targeted_improvement",
+        "surface": "CSV import wizard",
+        "current_behavior": "Rows with trailing spaces in email fields are rejected as invalid.",
+        "desired_behavior": "Trailing whitespace is trimmed before validation during import.",
+        "success_criteria": ["trimmed emails import successfully", "invalid emails still fail validation"],
+    },
+    "evidence": {
+        "sample_row": {"email": " alice@example.com "},
+        "observed_result": "validation error: invalid email",
+        "entry_point": "POST /api/imports/preview",
+    },
+}
+EXPECTED_BEHAVIOR = {
+    "produces_observation_artifact": True,
+    "creates_narrow_repro": True,
+    "repro_is_deterministic": True,
+    "expected_green_condition_present": True,
+    "ready_for_coder": True,
+}

devin/nodes/iterate/scenarios/observer_repro_creation_evals.py

@@ -0,0 +1,45 @@
+from devin.nodes.iterate.scenarios.observer_repro_creation import EXPECTED_BEHAVIOR
+
+EVAL_CRITERIA = {
+    "produces_observation_artifact": True,
+    "creates_narrow_repro": True,
+    "repro_is_deterministic": True,
+    "expected_green_condition_present": True,
+    "ready_for_coder": True,
+}
+
+
+
+def evaluate(actual_output: dict) -> tuple[bool, list[str]]:
+    ok = True
+    notes = []
+
+    observation_artifact = actual_output.get("observation_artifact") or {}
+    if not observation_artifact:
+        ok = False
+        notes.append("missing observation_artifact")
+        return ok, notes
+
+    repro = observation_artifact.get("repro_steps") or observation_artifact.get("repro") or []
+    if not repro:
+        ok = False
+        notes.append("missing bounded repro steps")
+
+    if isinstance(repro, list) and len(repro) > 6:
+        notes.append("repro exists but may be broader than necessary")
+
+    green = str(observation_artifact.get("expected_green_condition") or observation_artifact.get("green_condition") or "")
+    if not green:
+        ok = False
+        notes.append("missing expected_green_condition")
+
+    ready = observation_artifact.get("ready_for_coder")
+    if ready is False:
+        ok = False
+        notes.append("observer should mark deterministic repro as ready_for_coder")
+
+    repro_text = " ".join(str(step) for step in repro).lower() if isinstance(repro, list) else str(repro).lower()
+    if repro_text and not any(tok in repro_text for tok in ("csv", "email", "preview", "trailing", "whitespace")):
+        notes.append("repro does not appear aligned to the supplied seam")
+
+    return ok, notes