minitap-mobile-use 0.0.1.dev0__py3-none-any.whl

minitap/mobile_use/agents/contextor/contextor.py

@@ -0,0 +1,42 @@
+from minitap.mobile_use.agents.executor.utils import is_last_tool_message_take_screenshot
+from minitap.mobile_use.controllers.mobile_command_controller import get_screen_data
+from minitap.mobile_use.controllers.platform_specific_commands_controller import (
+    get_device_date,
+    get_focused_app_info,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.utils.decorators import wrap_with_callbacks
+from minitap.mobile_use.utils.logger import get_logger
+from minitap.mobile_use.context import MobileUseContext
+
+logger = get_logger(__name__)
+
+
+class ContextorNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    @wrap_with_callbacks(
+        before=lambda: logger.info("Starting Contextor Agent"),
+        on_success=lambda _: logger.success("Contextor Agent"),
+        on_failure=lambda _: logger.error("Contextor Agent"),
+    )
+    def __call__(self, state: State):
+        device_data = get_screen_data(self.ctx.screen_api_client)
+        focused_app_info = get_focused_app_info(self.ctx)
+        device_date = get_device_date(self.ctx)
+
+        should_add_screenshot_context = is_last_tool_message_take_screenshot(list(state.messages))
+
+        return state.sanitize_update(
+            ctx=self.ctx,
+            update={
+                "latest_screenshot_base64": device_data.base64
+                if should_add_screenshot_context
+                else None,
+                "latest_ui_hierarchy": device_data.elements,
+                "focused_app_info": focused_app_info,
+                "screen_size": (device_data.width, device_data.height),
+                "device_date": device_date,
+            },
+        )

minitap/mobile_use/agents/cortex/cortex.md

@@ -0,0 +1,93 @@
+## You are the **Cortex**
+
+Your job is to **analyze the current {{ platform }} mobile device state** and produce **structured decisions** to achieve the current subgoal.
+
+You must act like a human brain, responsible for giving instructions to your hands (the **Executor** agent). Therefore, you must act with the same imprecision and uncertainty as a human when performing swipe actions: humans don't know where exactly they are swiping (always prefer percentages of width and height instead of absolute coordinates), they just know they are swiping up or down, left or right, and with how much force (usually amplified compared to what's truly needed - go overboard of sliders for instance).
+
+### Context You Receive:
+
+You are provided with:
+
+- 📱 **Device state**:
+
+  - Latest **UI hierarchy**
+  - (Optional) Latest **screenshot (base64)**. You can query one if you need it by calling the take_screenshot tool. Often, the UI hierarchy is enough to understand what is happening on the screen.
+  - Current **focused app info**
+  - **Screen size** and **device date**
+
+- 🧭 **Task context**:
+
+  - The user's **initial goal**
+  - The **subgoal plan** with their statuses
+  - The **current subgoal** to act on (the one in `PENDING` in the plan)
+  - A list of **agent thoughts** (previous reasoning, observations about the environment)
+  - **Executor agent feedback** on the latest UI decisions
+
+### Your Mission:
+
+Focus on the **current subgoal**.
+
+1. **Analyze the UI** and environment to understand what action is required.
+   2.1. If the **subgoal is completed**, set the `complete_subgoal` field to `True`. To justify your conclusion, you will fill in the `agent_thought` field based on:
+
+- The current UI state
+- Past agent thoughts
+- Recent tool effects
+  2.2. Otherwise, output a **stringified structured set of instructions** that an **Executor agent** can perform on a real mobile device:
+
+- These must be **concrete low-level actions**: back,tap, swipe, launch app, list packages, close app, input text, paste, erase, text, copy, etc.
+- If you refer to a UI element or coordinates, specify it clearly (e.g., `resource-id: com.whatsapp:id/search`, `text: "Alice"`, `x: 100, y: 200`).
+- **The structure is up to you**, but it must be valid **JSON stringified output**. You will accompany this output with a **natural-language summary** of your reasoning and approach in your agent thought.
+- When you want to launch/stop an app, prefer using its package name.
+- **Only reference UI element IDs or visible texts that are explicitly present in the provided UI hierarchy or screenshot. Do not invent, infer, or guess any IDs or texts that are not directly observed**.
+- **For text clearing**: When you need to completely clear text from an input field, always use **LONG PRESS** first to select the text field, then erase. Do NOT use tap + erase as this only clears from cursor position.
+
+### Output
+
+- **Structured Decisions**:
+  A **valid stringified JSON** describing what should be executed **right now** to advance the current subgoal **IF THE SUBGOAL IS NOT COMPLETED**.
+
+- **Agent Thought** _(1-2 sentences)_:
+  If there is any information you need to remember for later steps, you must include it here, because only the agent thoughts will be used to produce the final structured output.
+
+  This also helps other agents understand your decision and learn from future failures.
+  You must also use this field to mention checkpoints when you perform actions without definite ending: for instance "Swiping up to reveal more recipes - last seen recipe was <ID or NAME>, stop when no more".
+
+- **Subgoal Completion** _(boolean)_:
+  Set to true if the current subgoal has been successfully completed - you **cannot set it to true and provide structured decisions at the same time**. You must base your decision ONLY on what you have as input (device state, agent thoughts, executor feedback, etc) - NEVER based on the decisions you have produced.
+
+---
+
+### Example
+
+#### Current Subgoal:
+
+> "Search for Alice in WhatsApp"
+
+#### Structured Decisions:
+
+```text
+"{\"action\": \"tap\", \"target\": {\"resource_id\": \"com.whatsapp:id/menuitem_search\", \"text\": \"Search\"}}"
+```
+
+#### Agent Thought:
+
+> I will tap the search icon at the top of the WhatsApp interface to begin searching for Alice.
+
+### Input
+
+**Initial Goal:**
+{{ initial_goal }}
+
+**Subgoal Plan:**
+{{ subgoal_plan }}
+
+**Current Subgoal (what needs to be done right now):**
+{{ current_subgoal }}
+
+**Agent thoughts (previous reasoning, observations about the environment):**
+{{ agents_thoughts }}
+
+**Executor agent feedback on latest UI decisions:**
+
+{{ executor_feedback }}

minitap/mobile_use/agents/cortex/cortex.py

@@ -0,0 +1,107 @@
+import json
+from pathlib import Path
+
+from jinja2 import Template
+from langchain_core.messages import (
+    AIMessage,
+    HumanMessage,
+    RemoveMessage,
+    SystemMessage,
+    ToolMessage,
+)
+from langgraph.graph.message import REMOVE_ALL_MESSAGES
+from minitap.mobile_use.agents.cortex.types import CortexOutput
+from minitap.mobile_use.agents.planner.utils import get_current_subgoal
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.services.llm import get_llm, with_fallback
+from minitap.mobile_use.utils.conversations import get_screenshot_message_for_llm
+from minitap.mobile_use.utils.decorators import wrap_with_callbacks
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class CortexNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    @wrap_with_callbacks(
+        before=lambda: logger.info("Starting Cortex Agent..."),
+        on_success=lambda _: logger.success("Cortex Agent"),
+        on_failure=lambda _: logger.error("Cortex Agent"),
+    )
+    async def __call__(self, state: State):
+        executor_feedback = get_executor_agent_feedback(state)
+
+        system_message = Template(
+            Path(__file__).parent.joinpath("cortex.md").read_text(encoding="utf-8")
+        ).render(
+            platform=self.ctx.device.mobile_platform.value,
+            initial_goal=state.initial_goal,
+            subgoal_plan=state.subgoal_plan,
+            current_subgoal=get_current_subgoal(state.subgoal_plan),
+            agents_thoughts=state.agents_thoughts,
+            executor_feedback=executor_feedback,
+        )
+        messages = [
+            SystemMessage(content=system_message),
+            HumanMessage(
+                content="Here are my device info:\n"
+                + self.ctx.device.to_str()
+                + f"Device date: {state.device_date}\n"
+                if state.device_date
+                else "" + f"Focused app info: {state.focused_app_info}\n"
+                if state.focused_app_info
+                else ""
+            ),
+        ]
+        for thought in state.agents_thoughts:
+            messages.append(AIMessage(content=thought))
+
+        if state.latest_screenshot_base64:
+            messages.append(get_screenshot_message_for_llm(state.latest_screenshot_base64))
+            logger.info("Added screenshot to context")
+
+        if state.latest_ui_hierarchy:
+            ui_hierarchy_dict: list[dict] = state.latest_ui_hierarchy
+            ui_hierarchy_str = json.dumps(ui_hierarchy_dict, indent=2, ensure_ascii=False)
+            messages.append(HumanMessage(content="Here is the UI hierarchy:\n" + ui_hierarchy_str))
+
+        llm = get_llm(ctx=self.ctx, name="cortex", temperature=1).with_structured_output(
+            CortexOutput
+        )
+        llm_fallback = get_llm(
+            ctx=self.ctx, name="cortex", use_fallback=True, temperature=1
+        ).with_structured_output(CortexOutput)
+        response: CortexOutput = await with_fallback(
+            main_call=lambda: llm.ainvoke(messages),
+            fallback_call=lambda: llm_fallback.ainvoke(messages),
+        )  # type: ignore
+
+        is_subgoal_completed = response.complete_current_subgoal
+        return state.sanitize_update(
+            ctx=self.ctx,
+            update={
+                "agents_thoughts": [response.agent_thought],
+                "structured_decisions": response.decisions if not is_subgoal_completed else None,
+                "latest_screenshot_base64": None,
+                "latest_ui_hierarchy": None,
+                "focused_app_info": None,
+                "device_date": None,
+                # Executor related fields
+                "executor_messages": [RemoveMessage(id=REMOVE_ALL_MESSAGES)],
+                "cortex_last_thought": response.agent_thought,
+            },
+        )
+
+
+def get_executor_agent_feedback(state: State) -> str:
+    if state.structured_decisions is None:
+        return "None."
+    executor_tool_messages = [m for m in state.executor_messages if isinstance(m, ToolMessage)]
+    return (
+        f"Latest UI decisions:\n{state.structured_decisions}"
+        + "\n\n"
+        + f"Executor feedback:\n{executor_tool_messages}"
+    )

minitap/mobile_use/agents/cortex/types.py

@@ -0,0 +1,11 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class CortexOutput(BaseModel):
+    decisions: str = Field(..., description="The decisions to be made. A stringified JSON object")
+    agent_thought: str = Field(..., description="The agent's thought")
+    complete_current_subgoal: Optional[bool] = Field(
+        False, description="Whether the current subgoal is complete"
+    )

minitap/mobile_use/agents/executor/executor.md

@@ -0,0 +1,73 @@
+## You are the **Executor**
+
+Your job is to **interpret the structured decisions** provided by the **Cortex** agent and use the appropriate tools to act on a **{{ platform }} mobile device**.
+
+### 🎯 Your Objective:
+
+Given the `structured_decisions` (a stringified object) from the **Cortex** agent
+and the previous tool calls, you must:
+
+1. **Parse the structured decisions** into usable Python objects.
+2. **Determine the most appropriate tool** to execute the intended action - **you can ONLY USE ONE**
+3. **Invoke tool accurately**, passing the required parameters.
+4. For **the tool you invoke**, always provide a clear `agent_thought` argument:
+
+   - This is a natural-language sentence (or two) **explaining why** this tool is being invoked.
+   - Keep it short but informative.
+   - This is essential for debugging, traceability, and adaptation by other agents.
+
+5. For **the tool you invoke**, always provide the `executor_metadata` argument:
+
+   - If you know you won't be able to achieve all Cortex decisions using the tool call you've chosen, set `retrigger` to `true` - otherwise set it to `false`
+
+---
+
+### 🧠 Example
+
+**Structured Decisions from the **Cortex** agent**:
+
+"I'm tapping on the chat item labeled 'Alice' to open the conversation."
+
+```json
+{
+  "action": "tap",
+  "target": {
+    "text": "Alice",
+    "resource_id": "com.whatsapp:id/conversation_item"
+  }
+}
+```
+
+**→ Executor Action**:
+
+Call the `tap_on_element` tool with:
+
+- `resource_id = "com.whatsapp:id/conversation_item"`
+- `text = "Alice"`
+- `agent_thought = "I'm tapping on the chat item labeled 'Alice' to open the conversation."`
+- `executor_metadata = {"retrigger": false}`
+
+---
+
+### ⚙️ Tools
+
+- Tools may include actions like: `tap`, `swipe`, `start_app`, `stop_app`, `list_packages`, `get_current_focus`, etc.
+- You **must not hardcode tool definitions** here.
+- Just use the right tool based on what the `structured_decisions` requires.
+- The tools are provided dynamically via LangGraph's tool binding mechanism.
+
+#### 🔄 Text Clearing Best Practice
+
+When you need to completely clear text from an input field, **DO NOT** simply use `erase_text` alone, as it only erases from the cursor position, backward. Instead:
+
+1. **Use `long_press_on` first** to select the text field and bring up selection options
+2. **Then use `erase_text`** to clear the selected content
+
+This approach ensures the **entire text content** is removed, not just the portion before the cursor position. The long press will typically select all text in the field, making the subsequent erase operation more effective.
+
+### 🔁 Final Notes
+
+- **You do not need to reason or decide strategy** — that's the Cortex's job.
+- You simply interpret and execute — like hands following the brain.
+- The `agent_thought` must always clearly reflect _why_ the action is being performed.
+- Be precise. Avoid vague or generic `agent_thought`s.

minitap/mobile_use/agents/executor/executor.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+
+from jinja2 import Template
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_core.messages.ai import AIMessage
+from langchain_google_genai import ChatGoogleGenerativeAI
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.services.llm import get_llm
+from minitap.mobile_use.tools.index import EXECUTOR_WRAPPERS_TOOLS, get_tools_from_wrappers
+from minitap.mobile_use.utils.decorators import wrap_with_callbacks
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ExecutorNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    @wrap_with_callbacks(
+        before=lambda: logger.info("Starting Executor Agent..."),
+        on_success=lambda _: logger.success("Executor Agent"),
+        on_failure=lambda _: logger.error("Executor Agent"),
+    )
+    async def __call__(self, state: State):
+        structured_decisions = state.structured_decisions
+        if not structured_decisions:
+            logger.warning("No structured decisions found.")
+            return state.sanitize_update(
+                ctx=self.ctx,
+                update={
+                    "agents_thoughts": [
+                        "No structured decisions found, I cannot execute anything."
+                    ],
+                },
+            )
+
+        if len(state.executor_messages) > 0 and isinstance(state.executor_messages[-1], AIMessage):
+            if len(state.executor_messages[-1].tool_calls) > 0:  # type: ignore
+                # A previous tool call raised an uncaught exception while retrigerring the executor
+                return state.sanitize_update(
+                    ctx=self.ctx,
+                    update={
+                        "executor_retrigger": False,
+                        "executor_failed": True,
+                        "executor_messages": [state.messages[-1]],
+                    },
+                )
+
+        system_message = Template(
+            Path(__file__).parent.joinpath("executor.md").read_text(encoding="utf-8")
+        ).render(platform=self.ctx.device.mobile_platform.value)
+        cortex_last_thought = (
+            state.cortex_last_thought if state.cortex_last_thought else state.agents_thoughts[-1]
+        )
+        messages = [
+            SystemMessage(content=system_message),
+            HumanMessage(content=cortex_last_thought),
+            HumanMessage(content=structured_decisions),
+            *state.executor_messages,
+        ]
+
+        llm = get_llm(ctx=self.ctx, name="executor")
+        llm_bind_tools_kwargs = {
+            "tools": get_tools_from_wrappers(self.ctx, EXECUTOR_WRAPPERS_TOOLS),
+            "tool_choice": "auto",  # automatically select a tool call or none
+        }
+
+        # ChatGoogleGenerativeAI does not support the "parallel_tool_calls" keyword
+        if not isinstance(llm, ChatGoogleGenerativeAI):
+            llm_bind_tools_kwargs["parallel_tool_calls"] = False
+
+        llm = llm.bind_tools(**llm_bind_tools_kwargs)
+        response = await llm.ainvoke(messages)
+
+        return state.sanitize_update(
+            ctx=self.ctx,
+            update={
+                "cortex_last_thought": cortex_last_thought,
+                "executor_messages": [response],
+                "messages": [response],
+            },
+        )

minitap/mobile_use/agents/executor/executor_context_cleaner.py

@@ -0,0 +1,27 @@
+from langchain_core.messages.ai import AIMessage
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.utils.decorators import wrap_with_callbacks
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+@wrap_with_callbacks(
+    before=lambda: logger.info("Starting Executor Context Cleaner..."),
+    on_success=lambda _: logger.success("Executor Context Cleaner"),
+    on_failure=lambda _: logger.error("Executor Context Cleaner"),
+)
+async def executor_context_cleaner_node(state: State):
+    """Clears the executor context."""
+    update: dict = {
+        "executor_failed": False,
+        "executor_retrigger": False,
+    }
+    if len(state.executor_messages) > 0 and isinstance(state.executor_messages[-1], AIMessage):
+        last_executor_message = state.executor_messages[-1]
+        if len(last_executor_message.tool_calls) > 0:
+            # A previous tool call raised an uncaught exception -> sanitize the executor messages
+            tool_error_message = state.messages[-1]
+            logger.error("Tool call failed with error: " + str(tool_error_message.content))
+            update["executor_messages"] = [tool_error_message]
+    return update

minitap/mobile_use/agents/executor/utils.py

@@ -0,0 +1,11 @@
+from langchain_core.messages import BaseMessage
+from minitap.mobile_use.utils.conversations import is_tool_message
+
+
+def is_last_tool_message_take_screenshot(messages: list[BaseMessage]) -> bool:
+    if not messages:
+        return False
+    for msg in messages[::-1]:
+        if is_tool_message(msg):
+            return msg.name == "take_screenshot"
+    return False

minitap/mobile_use/agents/hopper/hopper.md

@@ -0,0 +1,13 @@
+## Hopper
+
+Your goal is to analyze the input data and to pick only the most relevant information based on the current steps. We aim to reach the goal defined by the user as : {{ initial_goal }}
+
+### Input
+
+You have the list of steps we've done so far. We use those steps to track our progress to reach our goal. Here they are : {{ messages }}
+
+Finally, here is the data that we receive form executing the last task. We will dig this data to pick only the most relevant information to reach our goal. Keep the information as is, do not modify it since we will trigger actions based on it. Output this information in the output field, and you will describe what you did in the step field.
+
+Here is the data you must dig :
+
+{{ data }}

minitap/mobile_use/agents/hopper/hopper.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+from typing import Sequence
+
+from jinja2 import Template
+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.services.llm import get_llm
+from pydantic import BaseModel, Field
+
+
+class HopperOutput(BaseModel):
+    step: str = Field(
+        description=(
+            "The step that has been done, must be a valid one following the "
+            "current steps and the current goal to achieve."
+        )
+    )
+    output: str = Field(description="The interesting data extracted from the input data.")
+
+
+async def hopper(
+    ctx: MobileUseContext,
+    initial_goal: str,
+    messages: Sequence[BaseMessage],
+    data: str,
+) -> HopperOutput:
+    print("Starting Hopper Agent", flush=True)
+    system_message = Template(
+        Path(__file__).parent.joinpath("hopper.md").read_text(encoding="utf-8")
+    ).render(
+        initial_goal=initial_goal,
+        messages=messages,
+    )
+    messages = [
+        SystemMessage(content=system_message),
+        HumanMessage(content=data),
+    ]
+
+    llm = get_llm(ctx=ctx, name="hopper", is_utils=True, temperature=0)
+    structured_llm = llm.with_structured_output(HopperOutput)
+    response: HopperOutput = await structured_llm.ainvoke(messages)  # type: ignore
+    return HopperOutput(
+        step=response.step,
+        output=response.output,
+    )

minitap/mobile_use/agents/orchestrator/human.md

@@ -0,0 +1,13 @@
+Here is your input.
+
+---
+
+**Initial goal** : {{ initial_goal }}
+
+**Subgoal plan**
+{{ subgoal_plan }}
+
+**Current subgoal** : {{ current_subgoal }}
+
+**Agent thoughts**
+{{ agent_thoughts }}

minitap/mobile_use/agents/orchestrator/orchestrator.md

@@ -0,0 +1,18 @@
+You are the **Orchestrator**.
+
+Your role is to **decide what to do next**, based on the current execution state of a plan running on an **{{ platform }} mobile device**. You must assess the situation and choose between resuming, continuing, or replanning.
+
+### Responsibilities
+
+You're given:
+
+- The current **subgoal plan**
+- The current **subgoal** (which is marked as **PENDING** in the plan, but repeated here for your convenience)
+- A list of **agent thoughts** (insights, obstacles, or reasoning gathered during execution)
+- The original **initial goal**
+
+You must then **choose what to do next**:
+
+- `"resume"`: The current subgoal is clearly not finished, let's resume it. The status of the current subgoal will stay as `PENDING`.
+- `"continue"`: Move to the next subgoal in the list. The current subgoal will be marked as `SUCCESS`. If the current subgoal is the final step of the plan: The "reason" field must contain the final answer to the user’s initial goal. If the current subgoal is not the final step: The "reason" field must explain why this subgoal is now considered complete before moving on.
+- `"replan"`: The current plan no longer fits : the current subgoal will be marked as `FAILURE`. we need to define a new plan.

minitap/mobile_use/agents/orchestrator/orchestrator.py

@@ -0,0 +1,114 @@
+from pathlib import Path
+
+from jinja2 import Template
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from minitap.mobile_use.agents.orchestrator.types import OrchestratorOutput, OrchestratorStatus
+from minitap.mobile_use.agents.planner.utils import (
+    all_completed,
+    complete_current_subgoal,
+    fail_current_subgoal,
+    get_current_subgoal,
+    nothing_started,
+    start_next_subgoal,
+)
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.services.llm import get_llm
+from minitap.mobile_use.utils.decorators import wrap_with_callbacks
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class OrchestratorNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    @wrap_with_callbacks(
+        before=lambda: logger.info("Starting Orchestrator Agent..."),
+        on_success=lambda _: logger.success("Orchestrator Agent"),
+        on_failure=lambda _: logger.error("Orchestrator Agent"),
+    )
+    async def __call__(self, state: State):
+        if nothing_started(state.subgoal_plan):
+            state.subgoal_plan = start_next_subgoal(state.subgoal_plan)
+            new_subgoal = get_current_subgoal(state.subgoal_plan)
+            return state.sanitize_update(
+                ctx=self.ctx,
+                update={
+                    "agents_thoughts": [f"Starting the first subgoal: {new_subgoal}"],
+                    "subgoal_plan": state.subgoal_plan,
+                },
+            )
+
+        current_subgoal = get_current_subgoal(state.subgoal_plan)
+
+        if not current_subgoal:
+            return state.sanitize_update(
+                ctx=self.ctx,
+                update={"agents_thoughts": ["No subgoal to go for."]},
+            )
+
+        system_message = Template(
+            Path(__file__).parent.joinpath("orchestrator.md").read_text(encoding="utf-8")
+        ).render(platform=self.ctx.device.mobile_platform.value)
+        human_message = Template(
+            Path(__file__).parent.joinpath("human.md").read_text(encoding="utf-8")
+        ).render(
+            initial_goal=state.initial_goal,
+            subgoal_plan="\n".join(str(s) for s in state.subgoal_plan),
+            current_subgoal=str(current_subgoal),
+            agent_thoughts="\n".join(state.agents_thoughts),
+        )
+        messages = [
+            SystemMessage(content=system_message),
+            HumanMessage(content=human_message),
+        ]
+
+        llm = get_llm(ctx=self.ctx, name="orchestrator", temperature=1)
+        llm = llm.with_structured_output(OrchestratorOutput)
+        response: OrchestratorOutput = await llm.ainvoke(messages)  # type: ignore
+
+        if response.status == OrchestratorStatus.CONTINUE:
+            state.subgoal_plan = complete_current_subgoal(state.subgoal_plan)
+            thoughts = [response.reason]
+
+            if all_completed(state.subgoal_plan):
+                logger.success("All the subgoals have been completed successfully.")
+                return state.sanitize_update(
+                    ctx=self.ctx,
+                    update={
+                        "subgoal_plan": state.subgoal_plan,
+                        "agents_thoughts": thoughts,
+                    },
+                )
+            state.subgoal_plan = start_next_subgoal(state.subgoal_plan)
+            new_subgoal = get_current_subgoal(state.subgoal_plan)
+            thoughts.append(f"==== NEXT SUBGOAL: {new_subgoal} ====")
+            return state.sanitize_update(
+                ctx=self.ctx,
+                update={
+                    "agents_thoughts": thoughts,
+                    "subgoal_plan": state.subgoal_plan,
+                },
+            )
+
+        elif response.status == OrchestratorStatus.REPLAN:
+            thoughts = [response.reason]
+            state.subgoal_plan = fail_current_subgoal(state.subgoal_plan)
+            thoughts.append("==== END OF PLAN, REPLANNING ====")
+            return state.sanitize_update(
+                ctx=self.ctx,
+                update={
+                    "agents_thoughts": thoughts,
+                    "subgoal_plan": state.subgoal_plan,
+                },
+            )
+
+        return state.sanitize_update(
+            ctx=self.ctx,
+            update={
+                "agents_thoughts": [response.reason],
+            },
+        )