devflow-engine 1.0.0__py3-none-any.whl

devin/nodes/ideation/scenarios/user_says_create_it.py

@@ -0,0 +1,30 @@
+SCENARIO_NAME = 'user_says_create_it'
+SCENARIO_DESCRIPTION = 'User and agent have iterated on an idea for several turns. User now says "create it" — agent should commit the idea to ready_for_downstream immediately, no questions.'
+INPUT_PAYLOAD = {
+    'current_user_message': 'Create it. Make it happen.',
+    'idea_id': 'proj_75f63d30_ideas_createit_001',
+    'project_id': 'proj_75f63d30',
+    'repo_root': '/Users/devflow/repos/Spicy-Server',
+    # Prior agent turn calls devflow_init_idea to persist the artifact before user says "create it".
+    # This mirrors real multi-turn behavior where ideas are persisted before commitment.
+    'prior_messages': [
+        {'from': 'user', 'text': 'I want to improve how our team handles onboarding new clients'},
+        {'from': 'agent', 'text': 'Let me understand your context better — what type of clients are you working with?'},
+        {'from': 'user', 'text': 'Small professional services firms — lawyers, accountants, consultants'},
+        {'from': 'agent', 'text': 'Got it. And roughly how many new clients per month are you onboarding?'},
+        {'from': 'user', 'text': 'About 15-20 per month across all our advisors'},
+        {'from': 'agent', 'text': 'Devflow_Init_Idea called — idea persisted in .devflow/ideas/proj_75f63d30_ideas_createit_001/.\n\nProblem: Small professional services firms lose revenue and reputation when client onboarding is slow and inconsistent.\nTarget users: Internal client-facing staff (paralegals, assistants, admin) who manage the onboarding workflow.\nUser outcomes: Clients feel welcomed promptly, staff spend less time chasing documents, no new client starts with confusion.\nScope: Onboarding workflow system with document collection, task tracking, client portal, and status notifications.'},
+        {'from': 'user', 'text': 'Create it. Make it happen.'},
+    ],
+    # Pre-create the idea artifact so it exists for devflow_commit_idea to commit.
+    # In real usage, devflow_init_idea is called in the prior turn and the artifact is created.
+    # The eval harness mirrors this by ensuring the artifact exists before the "create it" turn.
+    '_precreate_artifact': True,
+}
+EXPECTED_BEHAVIOR = {
+    'response_kind': 'ready_for_downstream',
+    'idea_artifact_exists': True,
+    'idea_artifact_contains_all_fields': True,
+    'no_clarifying_questions': True,
+    'devflow_commit_called': True,
+}

devin/nodes/ideation/scenarios/user_says_create_it_evals.py

@@ -0,0 +1,23 @@
+EVAL_CRITERIA = {
+    'response_kind_must_be': 'ready_for_downstream',
+    'no_clarifying_questions': True,
+    'devflow_commit_called': True,  # agent returned ready_for_downstream = committed
+}
+
+
+def evaluate(actual_output: dict) -> tuple[bool, list[str]]:
+    ok = True
+    notes = []
+
+    # Must be ready_for_downstream (agent committed when user said create it)
+    if actual_output.get('response_kind') != 'ready_for_downstream':
+        ok = False
+        notes.append(f"expected ready_for_downstream, got {actual_output.get('response_kind')}")
+
+    # Must not ask clarifying questions — user said create it, agent must commit
+    fqs = actual_output.get('follow_up_questions') or []
+    if fqs:
+        ok = False
+        notes.append(f'should not ask questions but got: {fqs}')
+
+    return ok, notes

devin/nodes/ideation/scenarios/vague_idea.py

@@ -0,0 +1,16 @@
+SCENARIO_NAME = 'vague_idea'
+SCENARIO_DESCRIPTION = 'User has a vague idea and needs the agent to move it forward.'
+INPUT_PAYLOAD = {
+    'current_user_message': 'I want to improve how our team handles onboarding new clients.',
+    'idea_id': 'proj_75f63d30_ideas_vague_001',
+    'project_id': 'proj_75f63d30',
+    'repo_root': '/Users/devflow/repos/Spicy-Server',
+}
+EXPECTED_BEHAVIOR = {
+    'response_kind': 'ideation_contract_response',
+    'produces_full_idea_artifact': True,
+    'assumptions_traced': True,
+    'asks_one_targeted_question': True,
+    'no_premature_stories': True,
+    'idea_artifact_exists': True,
+}

devin/nodes/ideation/scenarios/vague_idea_evals.py

@@ -0,0 +1,47 @@
+EVAL_CRITERIA = {
+    'response_kind_must_be': 'ideation_contract_response',
+    'produces_full_idea_artifact': True,  # vague input → agent FABRICATES complete idea
+    'assumptions_traced': True,
+    'asks_one_targeted_question': True,  # fabricates AND asks one question to verify assumptions
+    'no_premature_stories': True,
+    'idea_artifact_exists': True,
+}
+
+
+def evaluate(actual_output: dict) -> tuple[bool, list[str]]:
+    ok = True
+    notes = []
+
+    # Response kind check
+    if actual_output.get('response_kind') != 'ideation_contract_response':
+        ok = False
+        notes.append(f"expected ideation_contract_response, got {actual_output.get('response_kind')}")
+
+    # Must ask exactly one targeted question (the tool fabricates AND asks to validate assumptions)
+    fqs = actual_output.get('follow_up_questions') or []
+    if len(fqs) != 1:
+        ok = False
+        notes.append(f'expected exactly one follow-up question, got {len(fqs)}')
+
+    # Must produce a FABRICATED complete idea
+    msg = (actual_output.get('response_message') or '').lower()
+    has_user_mention = any(w in msg for w in ['client', 'team', 'staff', 'user', 'onboarding', 'person'])
+    has_outcome_mention = any(w in msg for w in ['outcome', 'result', 'benefit', 'improve', 'faster', 'reduce', 'consistent'])
+    has_scope_mention = any(w in msg for w in ['scope', 'cover', 'include', 'system', 'workflow', 'process', 'portal', 'track'])
+
+    if not has_user_mention:
+        ok = False
+        notes.append('fabricated idea does not mention who the target users are')
+    if not has_outcome_mention:
+        ok = False
+        notes.append('fabricated idea does not mention intended outcomes or benefits')
+    if not has_scope_mention:
+        ok = False
+        notes.append('fabricated idea does not describe the scope of what to build')
+
+    # Assumptions must be mentioned in response_message (case-insensitive check for assumption mentions)
+    assumption_phrases = ['assumption', 'assuming', "i'm assuming", "i assume", 'assumptions traced', 'assumptions:']
+    if not any(phrase in msg for phrase in assumption_phrases):
+        notes.append('no assumptions traced in fabricated idea')
+
+    return ok, notes

devin/nodes/ideation/tools.json

@@ -0,0 +1,312 @@
+{
+  "tools": [
+    {
+      "name": "devflow_read_project_config",
+      "description": "Read the DevFlow project configuration for a given project_id. Returns project metadata: name, id, repo location (local path and upstream URL), start commands, preview URL, and production URL.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "project_id": {
+            "type": "string",
+            "description": "The DevFlow project ID (e.g. proj_75f63d30)"
+          }
+        },
+        "required": [
+          "project_id"
+        ]
+      }
+    },
+    {
+      "name": "devflow_read_queue_summary",
+      "description": "Read a summary of all DevFlow queues for a project. Returns queue counts by status for: idea, story, implementation, integration, recovery, and error queues.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "project_id": {
+            "type": "string",
+            "description": "The DevFlow project ID"
+          }
+        },
+        "required": [
+          "project_id"
+        ]
+      }
+    },
+    {
+      "name": "devflow_read_story_queue",
+      "description": "Read the DevFlow story queue for a project, optionally filtered by status.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "project_id": {
+            "type": "string",
+            "description": "The DevFlow project ID"
+          },
+          "status": {
+            "type": "string",
+            "description": "Filter by status: queued, in_progress, completed, failed",
+            "enum": [
+              "queued",
+              "in_progress",
+              "completed",
+              "failed"
+            ]
+          }
+        },
+        "required": [
+          "project_id"
+        ]
+      }
+    },
+    {
+      "name": "devflow_read_worker_state",
+      "description": "Read the current DevFlow worker state for a project. Returns active workers and what they are working on, plus a summary of current or last active run logs. If the last run was a recovery, includes recovery and failed run logs.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "project_id": {
+            "type": "string",
+            "description": "The DevFlow project ID"
+          }
+        },
+        "required": [
+          "project_id"
+        ]
+      }
+    },
+    {
+      "name": "devflow_init_idea",
+      "description": "Initialize a new idea artifact in the DevFlow pipeline. Before calling this tool, call devin_insight to explore the codebase and understand existing patterns so the idea aligns with current architecture. The idea title should reflect the user's intended outcome, not a technical implementation. Args: idea_id (string, required), title (string, required), project_id (string, required), text (string, required - full idea text from the user's message)",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "idea_id": {
+            "type": "string",
+            "description": "Unique idea identifier (e.g. proj_xxx_idea_yyy)"
+          },
+          "title": {
+            "type": "string",
+            "description": "Human-readable title for the idea"
+          },
+          "project_id": {
+            "type": "string",
+            "description": "DevFlow project ID this idea belongs to"
+          },
+          "text": {
+            "type": "string",
+            "description": "Full idea text \u2014 the user's complete description of what they want to build or change. Include all context, constraints, and intent."
+          }
+        },
+        "required": [
+          "idea_id",
+          "title",
+          "project_id",
+          "text"
+        ]
+      }
+    },
+    {
+      "name": "devflow_amend_idea",
+      "description": "Amend an existing idea artifact with refined or additional text. Updates the idea shape and sufficiency fields in place. Args: idea_id (string, required), refined_text (string, required)",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "idea_id": {
+            "type": "string",
+            "description": "The idea ID to amend"
+          },
+          "refined_text": {
+            "type": "string",
+            "description": "The user's refined or additional text for this idea"
+          }
+        },
+        "required": [
+          "idea_id",
+          "refined_text"
+        ]
+      }
+    },
+    {
+      "name": "devflow_commit_idea",
+      "description": "Promote an approved idea to ready_for_downstream state. Commits the idea artifact and marks it ready for story generation.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "idea_id": {
+            "type": "string",
+            "description": "The idea ID to commit/promote"
+          },
+          "draft_set": {
+            "type": "string",
+            "description": "Draft set name to promote from"
+          },
+          "project_id": {
+            "type": "string",
+            "description": "DevFlow project ID"
+          }
+        },
+        "required": [
+          "idea_id",
+          "draft_set",
+          "project_id"
+        ]
+      }
+    },
+    {
+      "name": "emit_start_working",
+      "description": "Start real-time working feedback for the user. Registers a single row on start and updates it every 2 seconds with rotating verbs until emit_stop_working is called. Multiple sessions can be active simultaneously. Args: activity (string, optional - initial verb; auto-selected if omitted), session_id (string, required)",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "activity": {
+            "type": "string",
+            "description": "Initial activity verb (e.g. running, reviewing, shaping). Auto-selected from 50 verbs if omitted. Available verbs: running, reviewing, shaping, crafting, exploring, analyzing, mapping, building, checking, preparing, processing, designing, configuring, organizing, sequencing, implementing, validating, verifying, tracing, scoping, estimating, integrating, testing, deploying, monitoring, optimizing, documenting, routing, forwarding, translating, encoding, decoding, partitioning, distributing, collecting, filtering, transforming, loading, saving, retrieving, computing, calculating, scheduling, coordinating, synthesizing, extracting, compiling, resolving, confirming, assembling, orchestrating"
+          },
+          "session_id": {
+            "type": "string",
+            "description": "Session ID for the agent_devin_messages row"
+          }
+        },
+        "required": [
+          "session_id"
+        ]
+      }
+    },
+    {
+      "name": "emit_stop_working",
+      "description": "Stop the real-time working feedback for a session. Updates the existing start_working row with the stop signal and clears the rotating verb timer. Args: session_id (string, required)",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "session_id": {
+            "type": "string",
+            "description": "Session ID for the feedback stream to stop"
+          }
+        },
+        "required": [
+          "session_id"
+        ]
+      }
+    },
+    {
+      "name": "emit_response",
+      "description": "Emit a generic real-time feedback message to the agent_devin_messages table. Used for mid-turn progress updates that are not start/stop.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "message": {
+            "type": "string",
+            "description": "The feedback message to display to the user"
+          },
+          "emit_type": {
+            "type": "string",
+            "description": "Type of emit: start_working, stop_working, progress, info, conclude_node"
+          },
+          "session_id": {
+            "type": "string",
+            "description": "Session ID for the agent_devin_messages row"
+          }
+        },
+        "required": [
+          "message",
+          "emit_type",
+          "session_id"
+        ]
+      }
+    },
+    {
+      "name": "devin_insight",
+      "description": "Call the Devin InsightAgent as a subagent to explore the codebase and provide grounded context. Use when you need to understand existing patterns before shaping an idea.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "current_user_message": {
+            "type": "string",
+            "description": "The user's current message to pass to the InsightAgent for exploration"
+          },
+          "context": {
+            "type": "object",
+            "description": "Context dict containing idea_id, project_id, repo_root",
+            "properties": {
+              "idea_id": {
+                "type": "string"
+              },
+              "project_id": {
+                "type": "string"
+              },
+              "repo_root": {
+                "type": "string"
+              }
+            }
+          }
+        },
+        "required": [
+          "current_user_message",
+          "context"
+        ]
+      }
+    },
+    {
+      "name": "idea_compliance_check",
+      "description": "Validate a shaped idea against existing stories in queue, committed ideas, and codebase structure. Checks for duplicate work, architectural fit, and naming collisions before the idea is committed. Returns COMPLIANT, NON_COMPLIANT, or NEEDS_REVIEW with specific issues. Args: idea_id (string, required), idea_text (string, required - the full shaped idea text), repo_root (string, required), project_id (string, required)",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "idea_id": {
+            "type": "string",
+            "description": "The idea ID being validated"
+          },
+          "idea_text": {
+            "type": "string",
+            "description": "The full shaped idea text to validate against existing work"
+          },
+          "repo_root": {
+            "type": "string",
+            "description": "The project repository root path"
+          },
+          "project_id": {
+            "type": "string",
+            "description": "The DevFlow project ID"
+          }
+        },
+        "required": [
+          "idea_id",
+          "idea_text",
+          "repo_root",
+          "project_id"
+        ]
+      }
+    },
+    {
+      "name": "goldilocks_check",
+      "description": "Evaluate whether a shaped idea is well-scoped for story generation. Returns verdict: just_right (2-20 stories), split (>20 stories), or enrichment_needed (<2 stories). Also returns story estimate, cohesive area, and split suggestions if verdict is split. Args: idea_id (string, required), idea_text (string, required - the full shaped idea text), repo_root (string, required), project_id (string, required)",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "idea_id": {
+            "type": "string",
+            "description": "The idea ID being evaluated"
+          },
+          "idea_text": {
+            "type": "string",
+            "description": "The full shaped idea text to evaluate for scoping"
+          },
+          "repo_root": {
+            "type": "string",
+            "description": "The project repository root path"
+          },
+          "project_id": {
+            "type": "string",
+            "description": "The DevFlow project ID"
+          }
+        },
+        "required": [
+          "idea_id",
+          "idea_text",
+          "repo_root",
+          "project_id"
+        ]
+      }
+    }
+  ]
+}

devin/nodes/insight/node.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from devflow_engine.devin2.pi_runner import run_devin2_pi_agent
+from devflow_engine.vendor.datalumina_genai.core.nodes.base import Node
+from devflow_engine.vendor.datalumina_genai.core.task import TaskContext
+from devin.nodes.shared.helpers import dfs_node_running, load_node_prompt_lines, pipeline_root, resolve_project_id, store_run, write_json
+from devin.nodes.shared.models import DevinAgentResponse
+
+class InsightAgentNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        repo_root = Path(event.repo_root)
+        store, run_id = store_run()
+        node_exec_id = store.create_node_attempt(run_id=run_id, node_id='insight_agent', node_name='InsightAgent', attempt=1)
+        project_id = str(task_context.metadata.get('project_id') or resolve_project_id(repo_root, idea_id=event.idea_id))
+        dfs_node_running(project_id=project_id, run_id=run_id, node_id='insight_agent', summary='Running Devin insight agent', idea_id=event.idea_id)
+        guidance = load_node_prompt_lines(__file__) + [
+            "Return response_kind='redirect'.",
+            'Answer the current project-specific question directly.',
+            'Keep the reply concise and grounded in provided context.',
+            'Use Emit_Start_Working before running a devflow inspection call.',
+            'Use Emit_Stop_Working after the call completes.',
+            'Use Emit_Response for mid-turn progress or info messages.',
+        ]
+
+        # Build session_id for emit tools (matches pattern used in ideation)
+        session_id = f"insight:{project_id}:{event.idea_id}"
+
+        context_payload = {
+            'idea_id': event.idea_id,
+            'current_user_message': str(task_context.metadata.get('raw_text') or event.raw_text or ''),
+            'route': task_context.metadata.get('route') or {},
+            'expected_status': 'redirect',
+            'project_id': project_id,
+            'repo_root': str(repo_root),
+            'session_id': session_id,
+        }
+        result = run_devin2_pi_agent(repo_root=repo_root, stage_name='devin_insight_response', route_arm='insight', context_payload=context_payload, operational_guidance=guidance, output_model=DevinAgentResponse, timeout_seconds=90)
+        model = DevinAgentResponse.model_validate(result.response_model.model_dump())
+        response_payload = {'idea_id': event.idea_id, 'pipeline_dir': str(pipeline_root(repo_root, idea_id=event.idea_id, pipeline_key=event.pipeline_key)), 'response_message': model.response_message, 'response_kind': model.response_kind, 'suggested_next_step': model.suggested_next_step, 'follow_up_questions': model.follow_up_questions, 'response_style_notes': model.style_notes}
+        out_path = pipeline_root(repo_root, idea_id=event.idea_id, pipeline_key=event.pipeline_key) / 'insight_response.json'
+        write_json(out_path, response_payload)
+        store.add_artifact(run_id=run_id, node_exec_id=node_exec_id, kind='devin_insight_response', uri=str(out_path), metadata={'response_kind': model.response_kind})
+        store.mark_node_finished(node_exec_id=node_exec_id, status='succeeded', output=response_payload)
+        task_context.metadata['response_guidance'] = response_payload
+        task_context.metadata['agent_loop_terminal'] = {'status': model.response_kind, **response_payload}
+        return task_context

devin/nodes/insight/playground.py

@@ -0,0 +1,154 @@
+"""InsightAgent playground — unit test InsightAgentNode against scenario inputs."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import importlib.util
+import json
+import uuid
+from pathlib import Path
+
+from devin.nodes.insight.node import InsightAgentNode
+from devin.nodes.shared.helpers import set_runtime_store
+from devin.nodes.shared.models import DevinAgentResponse, DevinChatDagEvent, ScenarioResult
+from devflow_engine.stores.execution_store import ExecutionStore
+from devflow_engine.vendor.datalumina_genai.core.task import TaskContext
+
+SCENARIO_DIR = Path(__file__).with_name('scenarios')
+
+
+def _load_module(path: Path):
+    spec = importlib.util.spec_from_file_location(path.stem, path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec and spec.loader
+    spec.loader.exec_module(module)
+    return module
+
+
+def _collect_scenarios():
+    pairs = []
+    for scenario_path in sorted(SCENARIO_DIR.glob('*.py')):
+        if scenario_path.name.endswith('_evals.py'):
+            continue
+        eval_path = scenario_path.with_name(f'{scenario_path.stem}_evals.py')
+        if eval_path.exists():
+            pairs.append((_load_module(scenario_path), _load_module(eval_path)))
+    return pairs
+
+
+async def _run_scenario(scenario_module, eval_module, repo_root: Path, project_id: str) -> ScenarioResult:
+    """Run a single scenario against the real InsightAgentNode with proper runtime store."""
+    input_payload = dict(scenario_module.INPUT_PAYLOAD)
+    expected = dict(scenario_module.EXPECTED_BEHAVIOR)
+
+    idea_id = input_payload.get('idea_id', f'{project_id}_scenario_{scenario_module.SCENARIO_NAME}')
+    raw_text = input_payload.get('current_user_message', '')
+
+    event = DevinChatDagEvent(
+        repo_root=str(repo_root),
+        idea_id=idea_id,
+        raw_text=raw_text,
+        pipeline_key=f'insight-scenario-{scenario_module.SCENARIO_NAME}',
+    )
+
+    metadata = {
+        'raw_text': raw_text,
+        'route': {'route_arm': 'insight'},
+        'project_id': project_id,
+    }
+
+    task_context = TaskContext(event=event, metadata=metadata)
+
+    # Set up runtime store for this scenario (matching ideation playground pattern)
+    db_path = repo_root / '.devflow' / 'execution.sqlite'
+    store = ExecutionStore(db_path=db_path)
+    run_id = store.create_run(
+        dag_id='devin_chat_two_arm_dag',
+        dag_version='1.0',
+        root_correlation_id=str(uuid.uuid4()),
+        config={'project_id': project_id, 'scenario': scenario_module.SCENARIO_NAME},
+    )
+    set_runtime_store(store, run_id)
+
+    node = InsightAgentNode(task_context=None)
+    try:
+        result_ctx = await node.process(task_context)
+        response_guidance = result_ctx.metadata.get('response_guidance', {})
+        agent_terminal = result_ctx.metadata.get('agent_loop_terminal', {})
+    except Exception as exc:
+        set_runtime_store(None, None)
+        return ScenarioResult(
+            scenario_name=scenario_module.SCENARIO_NAME,
+            passed=False,
+            actual_output={'error': str(exc)},
+            expected_behavior=expected,
+            notes=[f'Node raised exception: {exc}'],
+        )
+
+    set_runtime_store(None, None)
+
+    actual_output = {
+        'response_message': response_guidance.get('response_message', ''),
+        'response_kind': response_guidance.get('response_kind', agent_terminal.get('status', '')),
+        'suggested_next_step': response_guidance.get('suggested_next_step', ''),
+        'follow_up_questions': response_guidance.get('follow_up_questions', []),
+        'style_notes': response_guidance.get('response_style_notes', []),
+    }
+
+    try:
+        passed, notes = eval_module.evaluate(actual_output)
+    except Exception as exc:
+        passed = False
+        notes = [f'Eval raised exception: {exc}']
+
+    return ScenarioResult(
+        scenario_name=scenario_module.SCENARIO_NAME,
+        passed=passed,
+        actual_output=actual_output,
+        expected_behavior=expected,
+        notes=notes,
+    )
+
+
+def run_all() -> list[ScenarioResult]:
+    """Run all insight scenarios against the InsightAgentNode."""
+    results = []
+    for scenario_mod, eval_mod in _collect_scenarios():
+        result = asyncio.run(_run_scenario(scenario_mod, eval_mod, Path.cwd(), 'proj_75f63d30'))
+        results.append(result)
+    return results
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='InsightAgent playground')
+    parser.add_argument('--real-pi', action='store_true', help='Run with real PI harness (slower, uses API budget)')
+    parser.add_argument('--project', type=str, default='proj_75f63d30', help='DevFlow project ID')
+    parser.add_argument('--repo-root', type=str, default='/Users/devflow/repos/Spicy-Server', help='Repo root path')
+    args = parser.parse_args()
+
+    repo_root = Path(args.repo_root)
+    project_id = args.project
+
+    print(f"Running InsightAgent eval against Spicy-Server ({project_id})")
+    print(f"Repo root: {repo_root}")
+    print("-" * 60)
+
+    results = []
+    for scenario_mod, eval_mod in _collect_scenarios():
+        print(f"Running scenario: {scenario_mod.SCENARIO_NAME} ...", flush=True)
+        result = asyncio.run(_run_scenario(scenario_mod, eval_mod, repo_root, project_id))
+        results.append(result)
+        status = "PASS" if result.passed else "FAIL"
+        print(f"  → {status}", flush=True)
+
+    print("-" * 60)
+    print(f"\nResults: {sum(1 for r in results if r.passed)}/{len(results)} passed\n")
+
+    for result in results:
+        status = "PASS" if result.passed else "FAIL"
+        print(f"  [{status}] {result.scenario_name}")
+        for note in result.notes:
+            print(f"        note: {note}")
+
+    print("\n" + json.dumps([item.model_dump() for item in results], indent=2, sort_keys=True))