jupyter-agent 2025.6.103__py3-none-any.whl → 2025.6.105__py3-none-any.whl

jupyter_agent/bot_flows/base.py

@@ -5,6 +5,7 @@ This software is released under the MIT License.
 https://opensource.org/licenses/MIT
 """
 
+import time
 import traceback
 
 from pydantic import BaseModel
@@ -12,11 +13,16 @@ from enum import Enum
 from typing import List, Dict, Optional, Type
 from IPython.display import Markdown
 from ..bot_agents.base import BaseAgent
-from ..bot_outputs import _D, _I, _W, _E, _F, _M, _B, set_stage, flush_output
+from ..bot_evaluators.dummy_global import DummyGlobalEvaluator
+from ..bot_evaluators.flow_task_executor import FlowTaskExecEvaluator
+from ..bot_outputs import _D, _I, _W, _E, _F, _M, _B
+from ..bot_outputs import set_stage, flush_output, output_evaluation
+from ..bot_evaluation import FlowEvaluationRecord, StageEvaluationRecord, NotebookEvaluationRecord
 
 TASK_AGENT_STATE_ERROR = "_AGENT_STATE_ERROR_32534526_"
 TASK_STAGE_START = "start"
 TASK_STAGE_COMPLETED = "completed"
+TASK_STAGE_GLOBAL_FINISHED = "global_finished"
 
 
 class TaskAction(str, Enum):
@@ -47,11 +53,14 @@ class BaseTaskFlow:
 
     STAGE_TRANSITIONS: List[StageTransition] = []
     START_STAGE = TASK_STAGE_START
-    STOP_STAGES = [TASK_STAGE_COMPLETED]
+    STOP_STAGES = [TASK_STAGE_COMPLETED, TASK_STAGE_GLOBAL_FINISHED]
+    FLOW_EVALUATOR = FlowTaskExecEvaluator
+    GLOBAL_EVALUATOR = DummyGlobalEvaluator
 
-    def __init__(self, notebook_context, agent_factory):
+    def __init__(self, notebook_context, agent_factory, evaluator_factory=None):
         self.notebook_context = notebook_context
         self.agent_factory = agent_factory
+        self.evaluator_factory = evaluator_factory
         self.stage_transitions = {}
         self.prepare_stage_transitions()
 
@@ -146,11 +155,15 @@ class BaseTaskFlow:
         ns = self._get_next_stage_trans(stage, state, action)
         return ns.stage
 
-    def __call__(self, stage, max_tries=3, stage_continue=True, stage_confirm=True):
+    def __call__(self, stage, max_tries=5, stage_continue=True, stage_confirm=True):
 
         n_tries = 0
+        flow_duration = 0.0
+        stage_count = 0
+        # Initialize the task stage
         stage = stage or self.START_STAGE
         while n_tries <= max_tries:
+            stage_st = time.time()
             try:
                 stage_name = stage.value if isinstance(stage, Enum) else stage
                 stage_name = stage_name.replace(".", "-").capitalize()
@@ -163,6 +176,46 @@ class BaseTaskFlow:
                 _M(f"```python\n{traceback.format_exc()}\n```")
                 state = TASK_AGENT_STATE_ERROR
                 failed = True
+            stage_count += 1
+            stage_duration = time.time() - stage_st
+            flow_duration += stage_duration
+            _M(f"Stage `{stage}` completed in {stage_duration:.2f} seconds with state `{state}` and failed `{failed}`")
+            if (
+                self.evaluator_factory is not None
+                and not failed
+                and hasattr(agent, "EVALUATORS")
+                and state in agent.EVALUATORS
+            ):
+                # If the agent has evaluators, run them
+                evaluator = self.evaluator_factory(agent.EVALUATORS[state])
+                try:
+                    _M(f"**Evaluating** stage `{stage}` with evaluator `{type(evaluator).__name__}` ...")
+                    evaluation_result = evaluator()
+                    evaluation_result.timestamp = evaluation_result.timestamp or time.time()
+                    evaluation_result.evaluator = evaluation_result.evaluator or type(evaluator).__name__
+                    evaluation_result.cell_index = self.task.cell_idx
+                    evaluation_result.flow = type(self).__name__
+                    evaluation_result.stage = str(stage)
+                    evaluation_result.agent = type(agent).__name__
+                    evaluation_result.execution_duration = stage_duration
+                    evaluation_result.is_success = not failed
+                    output_evaluation(evaluation_result)
+                except Exception as e:
+                    _M(f"**Error** during task evaluation stage `{stage}`: `{type(e)}`: `{e}`")
+                    _M(f"```python\n{traceback.format_exc()}\n```")
+            else:
+                output_evaluation(
+                    StageEvaluationRecord(
+                        timestamp=time.time(),
+                        evaluator="default",
+                        cell_index=self.task.cell_idx,
+                        flow=type(self).__name__,
+                        stage=str(stage),
+                        agent=type(agent).__name__,
+                        execution_duration=stage_duration,
+                        is_success=not failed,
+                    )
+                )
 
             if state != TASK_AGENT_STATE_ERROR:
                 # Agent did not fail, check if we have reached the final stage
@@ -171,14 +224,18 @@ class BaseTaskFlow:
                 self.task.update_cell()
                 if next_stage in self.STOP_STAGES:
                     _M(f"Task execution **Stopped** at stage `{next_stage}`")
+                    stage = next_stage
                     break
 
             if failed:
                 # Agent failed
                 n_tries += 1
+                if n_tries > max_tries:
+                    _M(f"**Max flow tries reached** during task execution stage `{stage}`, **Stop!**")
+                    break
 
-            if failed or stage_confirm:
-                # Agent failed or we need to confirm
+            if stage_confirm:
+                # We need to confirm
                 message = self.get_prompt_message(stage, state, failed)
                 _M("**Confirm**: " + message)
                 flush_output()
@@ -188,15 +245,13 @@ class BaseTaskFlow:
                 self.task.update_cell()
                 if action == TaskAction.STOP:
                     _M(f"Task execution **Stopped**, and set next stage to `{next_stage}`")
-                    break
-                elif n_tries > max_tries:
-                    _M(f"**Max tries reached** during task execution stage `{stage}`, **Stop!**")
+                    stage = next_stage
                     break
                 else:
                     _M(f"**Action**: `{action}` transits stage to `{next_stage}`")
                     stage = next_stage
             else:
-                # Agent succeeded, transit to the next stage without confirmation
+                # transit to the next stage without confirmation
                 next_stage = self.get_next_stage(stage, state, TaskAction.CONTINUE)
                 self.task.agent_stage = next_stage
                 self.task.update_cell()
@@ -205,5 +260,54 @@ class BaseTaskFlow:
 
             if not stage_continue:
                 break
+        # Finalize the task execution
+        stage_name = stage.value if isinstance(stage, Enum) else stage
+        if stage_name == TASK_STAGE_GLOBAL_FINISHED:
+            _M("Task execution **finished** globally.")
+            if self.evaluator_factory is not None and hasattr(self, "GLOBAL_EVALUATOR") and self.GLOBAL_EVALUATOR:
+                evaluator = self.evaluator_factory(self.GLOBAL_EVALUATOR)
+                _M(f"**Evaluating** notebook with evaluator `{type(evaluator).__name__}` ...")
+                evaluation_result = evaluator()
+                evaluation_result.timestamp = evaluation_result.timestamp or time.time()
+                evaluation_result.evaluator = evaluation_result.evaluator or type(evaluator).__name__
+                evaluation_result.cell_index = self.task.cell_idx
+                evaluation_result.is_success = True
+                output_evaluation(evaluation_result)
+            else:
+                output_evaluation(
+                    NotebookEvaluationRecord(
+                        timestamp=time.time(),
+                        evaluator="default",
+                        cell_index=self.task.cell_idx,
+                        is_success=True,
+                    )
+                )
+        elif stage_name == TASK_STAGE_COMPLETED:
+            _M(f"Task execution **completed** in {flow_duration:.2f} seconds with {stage_count} stages.")
+            if self.evaluator_factory is not None and hasattr(self, "FLOW_EVALUATOR") and self.FLOW_EVALUATOR:
+                evaluator = self.evaluator_factory(self.FLOW_EVALUATOR)
+                _M(f"**Evaluating** flow `{type(self).__name__}` with evaluator `{type(evaluator).__name__}` ...")
+                evaluation_result = evaluator()
+                evaluation_result.timestamp = evaluation_result.timestamp or time.time()
+                evaluation_result.evaluator = evaluation_result.evaluator or type(evaluator).__name__
+                evaluation_result.cell_index = self.task.cell_idx
+                evaluation_result.flow = type(self).__name__
+                evaluation_result.stage_count = stage_count
+                evaluation_result.execution_duration = flow_duration
+                evaluation_result.is_success = True
+                output_evaluation(evaluation_result)
+            else:
+                # If no evaluator, just output the evaluation record
+                output_evaluation(
+                    FlowEvaluationRecord(
+                        timestamp=time.time(),
+                        evaluator="default",
+                        cell_index=self.task.cell_idx,
+                        flow=type(self).__name__,
+                        stage_count=stage_count,
+                        execution_duration=flow_duration,
+                        is_success=True,
+                    )
+                )
         flush_output()
         return stage

jupyter_agent/bot_flows/master_planner.py

@@ -6,11 +6,16 @@ https://opensource.org/licenses/MIT
 """
 
 from .base import BaseTaskFlow, StageTransition, TASK_STAGE_START, TASK_STAGE_COMPLETED
+from ..bot_evaluators.flow_global_planning import FlowGlobalPlanningEvaluator
+from ..bot_agents.master_planner import MasterPlannerAgent
+from ..bot_agents.output_task_result import OutputTaskResult
 
 
 class MasterPlannerFlow(BaseTaskFlow):
 
     STAGE_TRANSITIONS = [
-        StageTransition(stage=TASK_STAGE_START, agent="MasterPlannerAgent", next_stage=TASK_STAGE_START)
+        StageTransition(stage=TASK_STAGE_START, agent=MasterPlannerAgent, next_stage=TASK_STAGE_COMPLETED),
+        StageTransition(stage=TASK_STAGE_COMPLETED, agent=OutputTaskResult, next_stage=TASK_STAGE_COMPLETED),
     ]
-    STOP_STAGES = [TASK_STAGE_START]
+    STOP_STAGES = [TASK_STAGE_COMPLETED]
+    FLOW_EVALUATOR = FlowGlobalPlanningEvaluator

jupyter_agent/bot_flows/task_executor_v3.py

@@ -6,35 +6,43 @@ https://opensource.org/licenses/MIT
 """
 
 from enum import Enum
-from .base import BaseTaskFlow, StageTransition, StageNext, TaskAction
-from ..bot_agents import (
-    TaskPlannerAgentV3,
-    TaskCodingAgent,
-    CodeDebugerAgent,
-    CodeExecutor,
-    TaskStructureSummaryAgent,
-    TaskStructureReasoningAgent,
-    OutputTaskResult,
+from .base import (
+    BaseTaskFlow,
+    StageTransition,
+    StageNext,
+    TaskAction,
+    TASK_STAGE_COMPLETED,
+    TASK_STAGE_GLOBAL_FINISHED,
 )
-from ..bot_agents.task_planner_v3 import TaskPlannerState
+from ..bot_agents.task_planner_v3 import TaskPlannerAgentV3, TaskPlannerState
+from ..bot_agents.task_coder import TaskCodingAgent
+from ..bot_agents.task_debuger import CodeDebugerAgent
+from ..bot_agents.task_code_executor import CodeExecutor
+from ..bot_agents.task_structrue_summarier import TaskStructureSummaryAgent, TaskStructureSummaryState
+from ..bot_agents.task_structrue_reasoner import TaskStructureReasoningAgent, TaskStructureReasonState
+from ..bot_agents.output_task_result import OutputTaskResult
+from ..bot_agents.request_user_supply import RequestAboveUserSupplyAgent, RequestBelowUserSupplyAgent
 
 
 class TaskStage(str, Enum):
     PLANNING = "planning"
     PLANNING_PAUSED = "planning_paused"
+    REQUEST_INFO_ABOVE = "request_info_above"
+    REQUEST_INFO_BELOW = "request_info_below"
     CODING = "coding"
     EXECUTING = "executing"
     DEBUGGING = "debugging"
     REASONING = "reasoning"
     SUMMARY = "summary"
-    COMPLETED = "completed"
     OUTPUT_RESULT = "output_result"
+    COMPLETED = TASK_STAGE_COMPLETED
+    GLOBAL_FINISHED = TASK_STAGE_GLOBAL_FINISHED
 
 
 class TaskExecutorFlowV3(BaseTaskFlow):
 
     START_STAGE = TaskStage.PLANNING
-    STOP_STAGES = [TaskStage.COMPLETED, TaskStage.PLANNING_PAUSED]
+    STOP_STAGES = [TaskStage.COMPLETED, TaskStage.PLANNING_PAUSED, TaskStage.GLOBAL_FINISHED]
     STAGE_TRANSITIONS = [
         StageTransition[TaskStage, TaskPlannerState](
             stage=TaskStage.PLANNING,
@@ -42,10 +50,13 @@ class TaskExecutorFlowV3(BaseTaskFlow):
             states={
                 TaskPlannerState.CODING_PLANNED: TaskStage.CODING,
                 TaskPlannerState.REASONING_PLANNED: TaskStage.REASONING,
-                TaskPlannerState.REQUEST_INFO: TaskStage.PLANNING_PAUSED,
-                TaskPlannerState.GLOBAL_FINISHED: TaskStage.COMPLETED,
+                TaskPlannerState.REQUEST_INFO: TaskStage.REQUEST_INFO_ABOVE,
+                TaskPlannerState.GLOBAL_FINISHED: TaskStage.GLOBAL_FINISHED,
             },
         ),
+        StageTransition[TaskStage, None](
+            stage=TaskStage.REQUEST_INFO_ABOVE, agent=RequestAboveUserSupplyAgent, next_stage=TaskStage.PLANNING_PAUSED
+        ),
         StageTransition[TaskStage, TaskPlannerState](
             stage=TaskStage.PLANNING_PAUSED,
             agent=TaskPlannerAgentV3,
@@ -67,17 +78,28 @@ class TaskExecutorFlowV3(BaseTaskFlow):
         StageTransition[TaskStage, None](
             stage=TaskStage.DEBUGGING, agent=CodeDebugerAgent, next_stage=TaskStage.EXECUTING
         ),
-        StageTransition[TaskStage, None](
-            stage=TaskStage.REASONING, agent=TaskStructureReasoningAgent, next_stage=TaskStage.COMPLETED
+        StageTransition[TaskStage, TaskStructureReasonState](
+            stage=TaskStage.REASONING,
+            agent=TaskStructureReasoningAgent,
+            states={
+                TaskStructureReasonState.DONE: TaskStage.COMPLETED,
+                TaskStructureReasonState.REQUEST_INFO: TaskStage.REQUEST_INFO_BELOW,
+            },
         ),
-        StageTransition[TaskStage, None](
+        StageTransition[TaskStage, TaskStructureSummaryState](
             stage=TaskStage.SUMMARY,
             agent=TaskStructureSummaryAgent,
-            next_stage={
-                TaskAction.DEFAULT: StageNext(stage=TaskStage.COMPLETED),
-                TaskAction.STOP: StageNext(stage=TaskStage.EXECUTING),
+            states={
+                TaskStructureSummaryState.DONE: {
+                    TaskAction.DEFAULT: StageNext(stage=TaskStage.COMPLETED),
+                    TaskAction.STOP: StageNext(stage=TaskStage.EXECUTING),
+                },
+                TaskStructureSummaryState.REQUEST_INFO: TaskStage.REQUEST_INFO_BELOW,
             },
         ),
+        StageTransition[TaskStage, None](
+            stage=TaskStage.REQUEST_INFO_BELOW, agent=RequestBelowUserSupplyAgent, next_stage=TaskStage.COMPLETED
+        ),
         StageTransition[TaskStage, bool](
             stage=TaskStage.COMPLETED,
             agent=CodeExecutor,
@@ -86,4 +108,7 @@ class TaskExecutorFlowV3(BaseTaskFlow):
         StageTransition[TaskStage, None](
             stage=TaskStage.OUTPUT_RESULT, agent=OutputTaskResult, next_stage=TaskStage.COMPLETED
         ),
+        StageTransition[TaskStage, None](
+            stage=TaskStage.GLOBAL_FINISHED, agent=OutputTaskResult, next_stage=TaskStage.GLOBAL_FINISHED
+        ),
     ]

jupyter_agent/bot_magics.py

@@ -15,11 +15,14 @@ from IPython.display import Markdown
 from IPython.core.magic import Magics, magics_class, cell_magic
 from traitlets import Unicode, Int, Bool
 from traitlets.config.configurable import Configurable
-from .bot_contexts import NotebookContext, AgentCellContext
-from .bot_agents import AgentFactory
-from .bot_agents.base import AgentModelType
-from .bot_flows import MasterPlannerFlow, TaskExecutorFlowV1, TaskExecutorFlowV2, TaskExecutorFlowV3
-from .bot_outputs import _D, _I, _W, _E, _F, _M, _B, _O, reset_output, set_logging_level
+from .bot_contexts import NotebookContext
+from .bot_agents.base import AgentModelType, AgentFactory
+from .bot_agents.request_user_supply import RequestUserSupplyAgent
+from .bot_evaluators.base import EvaluatorFactory
+from .bot_flows import MasterPlannerFlow, TaskExecutorFlowV3
+from .bot_outputs import _D, _I, _W, _E, _F, _M, _B, _O, reset_output, set_logging_level, flush_output
+from .bot_actions import close_action_dispatcher
+from .utils import get_env_capbilities
 
 
 @magics_class
@@ -43,9 +46,17 @@ class BotMagics(Magics, Configurable):
     display_message = Bool(False, help="Display chat message").tag(config=True)
     display_think = Bool(True, help="Display chatthink response").tag(config=True)
     display_response = Bool(False, help="Display chat full response").tag(config=True)
+    support_save_meta = Bool(False, help="Support save metadata to cell").tag(config=True)
+    support_user_confirm = Bool(False, help="Support user confirm").tag(config=True)
+    support_user_supply_info = Bool(False, help="Support user supply info").tag(config=True)
+    support_set_cell_content = Bool(False, help="Support set cell content").tag(config=True)
+    enable_evaluating = Bool(False, help="Enable evaluating task").tag(config=True)
+    enable_supply_mocking = Bool(False, help="Enable supply mocking").tag(config=True)
     notebook_path = Unicode(None, allow_none=True, help="Path to Notebook file").tag(config=True)
     default_task_flow = Unicode("v3", allow_none=True, help="Default task flow").tag(config=True)
-    support_save_meta = Bool(False, help="Support save metadata to cell").tag(config=True)
+    default_max_tries = Int(3, help="Default max tries for task execution").tag(config=True)
+    default_step_mode = Bool(False, help="Default step mode for task execution").tag(config=True)
+    default_auto_confirm = Bool(False, help="Default auto confirm for task execution").tag(config=True)
 
     def parse_args(self, line):
         """解析命令行参数"""
@@ -54,31 +65,32 @@ class BotMagics(Magics, Configurable):
         parser.add_argument("-P", "--planning", action="store_true", default=False, help="Run in planning mode")
         parser.add_argument("-s", "--stage", type=str, default=None, help="Task stage")
         parser.add_argument("-f", "--flow", type=str, default=self.default_task_flow, help="Flow name")
-        parser.add_argument("-m", "--max-tries", type=int, default=3, help="Max tries")
-        parser.add_argument("-S", "--step-mode", action="store_true", default=False, help="Run in single step mode")
-        parser.add_argument("-Y", "--auto-confirm", action="store_true", default=False, help="Run without confirm")
+        parser.add_argument("-m", "--max-tries", type=int, default=self.default_max_tries, help="Max tries")
+        parser.add_argument(
+            "-S",
+            "--step-mode",
+            action="store_true",
+            default=self.default_step_mode,
+            help="Run in single step mode",
+        )
+        parser.add_argument(
+            "-Y",
+            "--auto-confirm",
+            action="store_true",
+            default=self.default_auto_confirm,
+            help="Run without confirm",
+        )
         options, _ = parser.parse_known_args(shlex.split(line.strip()))
-
         return options
 
-    def ensure_notebook_path(self):
-        if self.notebook_path:
-            return self.notebook_path
-        result = self.shell and self.shell.run_cell("globals().get('__vsc_ipynb_file__')")
-        if result and result.success and result.result:
-            self.notebook_path = result.result
-            return self.notebook_path
-        try:
-            self.notebook_path = str(ipynbname.path())
-            return self.notebook_path
-        except Exception as e:
-            _F(f"Failed to get notebook path: {e}")
-            return None
-
     @cell_magic
     def bot(self, line, cell):
         """Jupyter cell magic: %%bot"""
         try:
+            reset_output(stage="Logging", logging_level=self.logging_level)
+            _I("Cell magic %%bot executing ...")
+            _D(f"Cell magic called with line: {line}")
+            _D(f"Cell magic called with cell: {repr(cell)[:50]} ...")
             if not self.ensure_notebook_path():
                 _O(
                     Markdown(
@@ -88,11 +100,6 @@ class BotMagics(Magics, Configurable):
                     )
                 )
                 return
-            AgentCellContext.SUPPORT_SAVE_META = self.support_save_meta
-            reset_output(stage="Logging", logging_level=self.logging_level)
-            _I("Cell magic %%bot executing ...")
-            _D(f"Cell magic called with line: {line}")
-            _D(f"Cell magic called with cell: {repr(cell)[:50]} ...")
             if not cell.strip():
                 _O(
                     Markdown(
@@ -105,42 +112,90 @@ class BotMagics(Magics, Configurable):
                         "%%bot {}\n\n# {}".format(line.strip(), time.strftime("%Y-%m-%d %H:%M:%S")), replace=True
                     )
                 return
+            get_env_capbilities().save_metadata = self.support_save_meta
+            get_env_capbilities().user_confirm = self.support_user_confirm
+            get_env_capbilities().user_supply_info = self.support_user_supply_info
+            get_env_capbilities().set_cell_content = self.support_set_cell_content
+            RequestUserSupplyAgent.MOCK_USER_SUPPLY = self.enable_supply_mocking
             options = self.parse_args(line)
-            _D(f"Cell magic called with options: {options}")
             set_logging_level(options.logging_level)
+            _D(f"Cell magic called with options: {options}")
             nb_context = NotebookContext(line, cell, notebook_path=self.notebook_path)
-            agent_factory = AgentFactory(
-                nb_context,
-                display_think=self.display_think,
-                display_message=self.display_message,
-                display_response=self.display_response,
+            agent_factory = self.get_agent_factory(nb_context)
+            evaluator_factory = self.get_evaluator_factory(nb_context)
+            if options.planning:
+                flow = MasterPlannerFlow(nb_context, agent_factory, evaluator_factory)
+            elif options.flow == "v3":
+                flow = TaskExecutorFlowV3(nb_context, agent_factory, evaluator_factory)
+            else:
+                raise ValueError(f"Unknown flow: {options.flow}")
+            flow(
+                options.stage,
+                options.max_tries,
+                not options.step_mode,
+                not options.auto_confirm,
             )
-            agent_factory.config_model(
+        except Exception as e:
+            traceback.print_exc()
+        finally:
+            close_action_dispatcher()
+            flush_output()
+
+    def ensure_notebook_path(self):
+        if self.notebook_path:
+            return self.notebook_path
+        result = self.shell and self.shell.run_cell(
+            "globals().get('__vsc_ipynb_file__') or globals().get('__evaluation_ipynb_file__')"
+        )
+        if result and result.success and result.result:
+            self.notebook_path = result.result
+            return self.notebook_path
+        try:
+            self.notebook_path = str(ipynbname.path())
+            return self.notebook_path
+        except Exception as e:
+            _F(f"Failed to get notebook path: {e}")
+            return None
+
+    def get_agent_factory(self, nb_context):
+        agent_factory = AgentFactory(
+            nb_context,
+            display_think=self.display_think,
+            display_message=self.display_message,
+            display_response=self.display_response,
+        )
+        agent_factory.config_model(
+            AgentModelType.DEFAULT, self.default_api_url, self.default_api_key, self.default_model_name
+        )
+        agent_factory.config_model(
+            AgentModelType.PLANNER, self.planner_api_url, self.planner_api_key, self.planner_model_name
+        )
+        agent_factory.config_model(
+            AgentModelType.CODING, self.coding_api_url, self.coding_api_key, self.coding_model_name
+        )
+        agent_factory.config_model(
+            AgentModelType.REASONING, self.reasoning_api_url, self.reasoning_api_key, self.reasoning_model_name
+        )
+        return agent_factory
+
+    def get_evaluator_factory(self, nb_context):
+        if self.enable_evaluating:
+            evaluator_factory = EvaluatorFactory(nb_context)
+            evaluator_factory.config_model(
                 AgentModelType.DEFAULT, self.default_api_url, self.default_api_key, self.default_model_name
             )
-            agent_factory.config_model(
+            evaluator_factory.config_model(
                 AgentModelType.PLANNER, self.planner_api_url, self.planner_api_key, self.planner_model_name
             )
-            agent_factory.config_model(
+            evaluator_factory.config_model(
                 AgentModelType.CODING, self.coding_api_url, self.coding_api_key, self.coding_model_name
             )
-            agent_factory.config_model(
+            evaluator_factory.config_model(
                 AgentModelType.REASONING, self.reasoning_api_url, self.reasoning_api_key, self.reasoning_model_name
             )
-            if options.planning:
-                flow = MasterPlannerFlow(nb_context, agent_factory)
-            else:
-                if options.flow == "v1":
-                    flow = TaskExecutorFlowV1(nb_context, agent_factory)
-                elif options.flow == "v2":
-                    flow = TaskExecutorFlowV2(nb_context, agent_factory)
-                elif options.flow == "v3":
-                    flow = TaskExecutorFlowV3(nb_context, agent_factory)
-                else:
-                    raise ValueError(f"Unknown flow: {options.flow}")
-            flow(options.stage, options.max_tries, not options.step_mode, not options.auto_confirm)
-        except Exception as e:
-            traceback.print_exc()
+        else:
+            evaluator_factory = None
+        return evaluator_factory
 
 
 def load_ipython_extension(ipython):