notte-agent 0.0.dev0__py3-none-any.whl

notte_agent/falco/agent.py

@@ -0,0 +1,343 @@
+import time
+import traceback
+import typing
+from collections.abc import Callable
+from enum import StrEnum
+
+import notte_core
+from litellm import AllMessageValues, override
+from loguru import logger
+from notte_browser.dom.locate import locate_element
+from notte_browser.resolution import NodeResolutionPipe
+from notte_browser.session import NotteSession, NotteSessionConfig
+from notte_browser.vault import VaultScreetsScreenshotMask
+from notte_browser.window import BrowserWindow
+from notte_core.browser.observation import Observation
+from notte_core.common.tracer import LlmUsageDictTracer
+from notte_core.controller.actions import (
+    BaseAction,
+    CompletionAction,
+    FallbackObserveAction,
+    InteractionAction,
+)
+from notte_core.credentials.base import BaseVault, LocatorAttributes
+from notte_core.llms.engine import LLMEngine
+from patchright.async_api import Locator
+
+from notte_agent.common.base import BaseAgent
+from notte_agent.common.captcha_detector import CaptchaDetector
+from notte_agent.common.config import AgentConfig, RaiseCondition
+from notte_agent.common.conversation import Conversation
+from notte_agent.common.safe_executor import ExecutionStatus, SafeActionExecutor
+from notte_agent.common.types import AgentResponse
+from notte_agent.common.validator import CompletionValidator
+from notte_agent.falco.perception import FalcoPerception
+from notte_agent.falco.prompt import FalcoPrompt
+from notte_agent.falco.trajectory_history import FalcoTrajectoryHistory
+from notte_agent.falco.types import StepAgentOutput
+
+# TODO: list
+# handle tooling calling methods for different providers (if not supported by litellm)
+# Handle control flags
+# Done callback
+# Setup telemetry
+# Setup memory
+# Handle custom functions, e.g. `Upload file to element`ç
+# Remove base 64 images from current state
+# TODO: add fault tolerance LLM parsing
+# TODO: only display modal actions when modal is open (same as before)
+# TODO: handle prevent default click JS events
+# TODO: add some tree structure for menu elements (like we had in notte before. Ex. Menu in Arxiv)
+
+
+class HistoryType(StrEnum):
+    FULL_CONVERSATION = "full_conversation"
+    SHORT_OBSERVATIONS = "short_observations"
+    SHORT_OBSERVATIONS_WITH_RAW_DATA = "short_observations_with_raw_data"
+    SHORT_OBSERVATIONS_WITH_SHORT_DATA = "short_observations_with_short_data"
+    COMPRESSED = "compressed"
+
+
+class FalcoAgentConfig(AgentConfig):
+    max_actions_per_step: int = 1
+    history_type: HistoryType = HistoryType.SHORT_OBSERVATIONS_WITH_SHORT_DATA
+
+    @classmethod
+    @override
+    def default_session(cls) -> NotteSessionConfig:
+        return NotteSessionConfig().disable_perception()
+
+
+class FalcoAgent(BaseAgent):
+    def __init__(
+        self,
+        config: FalcoAgentConfig,
+        window: BrowserWindow | None = None,
+        vault: BaseVault | None = None,
+        step_callback: Callable[[str, StepAgentOutput], None] | None = None,
+    ):
+        super().__init__(session=NotteSession(config=config.session, window=window))
+        self.config: FalcoAgentConfig = config
+        self.vault: BaseVault | None = vault
+
+        self.tracer: LlmUsageDictTracer = LlmUsageDictTracer()
+        self.llm: LLMEngine = LLMEngine(
+            model=config.reasoning_model,
+            tracer=self.tracer,
+            structured_output_retries=config.session.structured_output_retries,
+            verbose=self.config.verbose,
+        )
+
+        self.step_callback: Callable[[str, StepAgentOutput], None] | None = step_callback
+        # Users should implement their own parser to customize how observations
+        # and actions are formatted for their specific LLM and use case
+
+        if self.vault is not None:
+            # hide vault leaked credentials within llm inputs
+            self.llm.structured_completion = self.vault.patch_structured_completion(0, self.vault.get_replacement_map)(
+                self.llm.structured_completion
+            )
+
+        self.perception: FalcoPerception = FalcoPerception()
+        self.validator: CompletionValidator = CompletionValidator(llm=self.llm, perception=self.perception)
+        self.captcha_detector: CaptchaDetector = CaptchaDetector(llm=self.llm, perception=self.perception)
+        self.prompt: FalcoPrompt = FalcoPrompt(max_actions_per_step=config.max_actions_per_step)
+        self.conv: Conversation = Conversation(
+            max_tokens=config.max_history_tokens,
+            convert_tools_to_assistant=True,
+            autosize=True,
+            model=config.reasoning_model,
+        )
+        self.history_type: HistoryType = config.history_type
+        self.trajectory: FalcoTrajectoryHistory = FalcoTrajectoryHistory(max_error_length=config.max_error_length)
+
+        async def execute_action(action: BaseAction) -> Observation:
+            if self.vault is not None and self.vault.contains_credentials(action):
+                action_with_selector = await NodeResolutionPipe.forward(action, self.session.snapshot)
+                if isinstance(action_with_selector, InteractionAction) and action_with_selector.selector is not None:
+                    locator: Locator = await locate_element(self.session.window.page, action_with_selector.selector)
+                    assert (
+                        isinstance(action_with_selector, InteractionAction)
+                        and action_with_selector.selector is not None
+                    )
+
+                    attrs = await FalcoAgent.compute_locator_attributes(locator)
+                    action = self.vault.replace_credentials(
+                        action,
+                        attrs,
+                        self.session.snapshot,
+                    )
+            return await self.session.act(action)
+
+        self.step_executor: SafeActionExecutor[BaseAction, Observation] = SafeActionExecutor(
+            func=execute_action,
+            raise_on_failure=(self.config.raise_condition is RaiseCondition.IMMEDIATELY),
+            max_consecutive_failures=config.max_consecutive_failures,
+        )
+
+    @staticmethod
+    async def compute_locator_attributes(locator: Locator) -> LocatorAttributes:
+        attr_type = await locator.get_attribute("type")
+        autocomplete = await locator.get_attribute("autocomplete")
+        outer_html = await locator.evaluate("el => el.outerHTML")
+        return LocatorAttributes(type=attr_type, autocomplete=autocomplete, outerHTML=outer_html)
+
+    async def reset(self) -> None:
+        self.conv.reset()
+        self.trajectory.reset()
+        self.step_executor.reset()
+        await self.session.reset()
+
+    def output(self, answer: str, success: bool) -> AgentResponse:
+        return AgentResponse(
+            answer=answer,
+            success=success,
+            session_trajectory=self.session.trajectory,
+            agent_trajectory=self.trajectory.steps,  # type: ignore[reportArgumentType]
+            messages=self.conv.messages(),
+            duration_in_s=time.time() - self.start_time,
+            llm_usage=self.tracer.usage,
+        )
+
+    async def get_messages(self, task: str) -> list[AllMessageValues]:
+        self.conv.reset()
+        system_msg, task_msg = self.prompt.system(), self.prompt.task(task)
+        if self.vault is not None:
+            system_msg += "\n" + self.vault.instructions()
+        self.conv.add_system_message(content=system_msg)
+        self.conv.add_user_message(content=task_msg)
+        # just for logging
+        traj_msg = self.trajectory.perceive()
+        if self.config.verbose:
+            logger.info(f"🔍 Trajectory history:\n{traj_msg}")
+        # add trajectory to the conversation
+        match self.history_type:
+            case HistoryType.COMPRESSED:
+                self.conv.add_user_message(content=traj_msg)
+            case _:
+                if len(self.trajectory.steps) == 0:
+                    self.conv.add_user_message(content=self.trajectory.start_rules())
+                for step in self.trajectory.steps:
+                    # TODO: choose if we want this to be an assistant message or a tool message
+                    # self.conv.add_tool_message(step.agent_response, tool_id="step")
+                    self.conv.add_assistant_message(step.agent_response.model_dump_json(exclude_none=True))
+                    for result in step.results:
+                        short_step_msg = self.trajectory.perceive_step_result(result, include_ids=True)
+                        self.conv.add_user_message(content=short_step_msg)
+                        if not result.success:
+                            continue
+                        # add observation data to the conversation
+                        obs = result.get()
+                        match (self.history_type, obs.has_data()):
+                            case (HistoryType.FULL_CONVERSATION, _):
+                                self.conv.add_user_message(
+                                    content=self.perception.perceive(obs),
+                                    image=(obs.screenshot if self.config.include_screenshot else None),
+                                )
+                            case (HistoryType.SHORT_OBSERVATIONS_WITH_RAW_DATA, True):
+                                # add data if data was scraped
+                                self.conv.add_user_message(content=self.perception.perceive_data(obs, raw=True))
+
+                            case (HistoryType.SHORT_OBSERVATIONS_WITH_SHORT_DATA, True):
+                                self.conv.add_user_message(content=self.perception.perceive_data(obs, raw=False))
+                            case _:
+                                pass
+
+        last_valid_obs = self.trajectory.last_obs()
+        if last_valid_obs is not None and self.history_type is not HistoryType.FULL_CONVERSATION:
+            self.conv.add_user_message(
+                content=self.perception.perceive(last_valid_obs),
+                image=(last_valid_obs.screenshot if self.config.include_screenshot else None),
+            )
+
+        if len(self.trajectory.steps) > 0:
+            self.conv.add_user_message(self.prompt.action_message())
+
+        return self.conv.messages()
+
+    async def step(self, task: str) -> CompletionAction | None:
+        """Execute a single step of the agent"""
+        messages = await self.get_messages(task)
+        response: StepAgentOutput = self.llm.structured_completion(messages, response_format=StepAgentOutput)
+        if self.step_callback is not None:
+            self.step_callback(task, response)
+
+        if self.config.verbose:
+            logger.info(f"🔍 LLM response:\n{response}")
+
+        for line in response.pretty_string().split("\n"):
+            logger.opt(colors=True).info(line)
+
+        self.trajectory.add_output(response)
+        # check for completion
+        if response.output is not None:
+            return response.output
+        # Execute the actions
+        for action in response.get_actions(self.config.max_actions_per_step):
+            result = await self.step_executor.execute(action)
+
+            self.trajectory.add_step(result)
+            step_msg = self.trajectory.perceive_step_result(result, include_ids=True)
+            logger.info(f"{step_msg}\n\n")
+            if not result.success:
+                # observe again
+                obs = await self.session.observe()
+
+                # cast is necessary because we cant have covariance
+                # in ExecutionStatus
+                ex_status = ExecutionStatus(
+                    input=typing.cast(BaseAction, FallbackObserveAction()),
+                    output=obs,
+                    success=True,
+                    message="Observed",
+                )
+                self.trajectory.add_output(response)
+                self.trajectory.add_step(ex_status)
+
+                # stop the loop
+                break
+            # Successfully executed the action
+        return None
+
+    @override
+    async def run(self, task: str, url: str | None = None) -> AgentResponse:
+        logger.info(f"Running task: {task}")
+        self.start_time: float = time.time()
+        try:
+            return await self._run(task, url=url)
+
+        except Exception as e:
+            if self.config.raise_condition is RaiseCondition.NEVER:
+                return self.output(f"Failed due to {e}: {traceback.format_exc()}", False)
+            raise e
+
+    async def _human_in_the_loop(self) -> None:
+        # Check for captcha if human-in-the-loop is enabled
+        captcha_result = self.captcha_detector.detect(self.session.trajectory[-1])
+        if captcha_result.has_captcha:
+            logger.warning(f"⚠️ Captcha detected: {captcha_result.description}")
+            logger.info("🔄 Waiting for human intervention...")
+            _ = input("Press Enter to continue after solving the captcha...")
+            # Observe again after human intervention
+            obs = await self.session.observe()
+            self.trajectory.add_step(
+                ExecutionStatus(
+                    input=typing.cast(BaseAction, FallbackObserveAction()),
+                    output=obs,
+                    success=True,
+                    message="Observed after human intervention",
+                )
+            )
+
+    async def _run(self, task: str, url: str | None = None) -> AgentResponse:
+        """Execute the task with maximum number of steps"""
+        # change this to DEV if you want more explicit error messages
+        # when you are developing your own agent
+        notte_core.set_error_mode("agent")
+        if url is not None:
+            task = f"Start on '{url}' and {task}"
+
+        # Loop through the steps
+        async with self.session:
+            # hide vault leaked credentials within screenshots
+            if self.vault is not None:
+                self.session.window.screenshot_mask = VaultScreetsScreenshotMask(vault=self.vault)
+
+            for step in range(self.session.config.max_steps):
+                logger.info(f"💡 Step {step}")
+                output: CompletionAction | None = await self.step(task)
+                # Check for captcha if human-in-the-loop is enabled
+                if self.config.human_in_the_loop:
+                    await self._human_in_the_loop()
+                if output is None:
+                    continue
+                # validate the output
+                if not output.success:
+                    logger.error(f"🚨 Agent terminated early with failure: {output.answer}")
+                    return self.output(output.answer, False)
+                # Sucessful execution and LLM output is not None
+                # Need to validate the output
+                logger.info(f"🔥 Validating agent output:\n{output.model_dump_json()}")
+                val = self.validator.validate(task, output, self.session.trajectory[-1])
+                if val.is_valid:
+                    # Successfully validated the output
+                    logger.info("✅ Task completed successfully")
+                    return self.output(output.answer, output.success)
+                else:
+                    # TODO handle that differently
+                    failed_val_msg = f"Final validation failed: {val.reason}. Continuing..."
+                    logger.error(failed_val_msg)
+                    # add the validation result to the trajectory and continue
+                    self.trajectory.add_step(
+                        ExecutionStatus(
+                            input=output,
+                            output=None,
+                            success=False,
+                            message=failed_val_msg,
+                        )
+                    )
+
+        error_msg = f"Failed to solve task in {self.session.config.max_steps} steps"
+        logger.info(f"🚨 {error_msg}")
+        notte_core.set_error_mode("developer")
+        return self.output(error_msg, False)

notte_agent/falco/perception.py

@@ -0,0 +1,83 @@
+from typing import final
+
+from notte_core.browser.observation import Observation
+from typing_extensions import override
+
+from notte_agent.common.perception import BasePerception
+
+
+@final
+class FalcoPerception(BasePerception):
+    def __init__(
+        self,
+        include_step_info: bool = True,
+        include_attributes: list[str] | None = None,
+    ):
+        self.include_attributes = include_attributes
+        self.include_step_info = include_step_info
+
+    @override
+    def perceive_metadata(self, obs: Observation) -> str:
+        if obs.progress is None:
+            raise ValueError("Observation has no progress")
+        return f"""
+You will see the following only once. If you need to remember it and you dont know it yet, write it down in the memory.
+
+* Current url: {obs.metadata.url}
+* Current page title: {obs.metadata.title}
+* Current date and time: {obs.metadata.timestamp.strftime("%Y-%m-%d %H:%M:%S")}
+* Available tabs:
+{obs.metadata.tabs}
+* Current step: {obs.progress.current_step + 1}/{obs.progress.max_steps}'
+"""
+
+    @override
+    def perceive(self, obs: Observation) -> str:
+        return f"""
+[Relevant metadata]
+{self.perceive_metadata(obs)}
+
+[Interaction elements and context]
+{self.perceive_actions(obs)}
+
+[Data found in the page]
+{self.perceive_data(obs)}
+"""
+
+    @override
+    def perceive_actions(self, obs: Observation) -> str:
+        px_above = obs.metadata.viewport.pixels_above
+        px_below = obs.metadata.viewport.pixels_below
+
+        more_above = f"... {px_above} pixels above - scroll or scrape content to see more ..."
+        more_below = f"... {px_below} pixels below - scroll or scrape content to see more ..."
+
+        space_description = obs.space.description
+
+        return f"""
+[Start of page]
+{more_above if px_above > 0 else ""}
+{space_description or "No content to display"}
+{more_below if px_below > 0 else ""}
+[End of page]
+
+"""
+
+    @override
+    def perceive_data(self, obs: Observation, raw: bool = True) -> str:
+        if not obs.has_data() or obs.data is None:
+            return ""
+        if raw:
+            percieved_data = obs.data.markdown
+        else:
+            structured_data = obs.data.structured
+            if structured_data is None or not structured_data.success or structured_data.data is None:
+                error_msg = f" with error: {structured_data.error}" if structured_data is not None else ""
+                return f"Scraping failed{error_msg}. Please try again with different instructions."
+            percieved_data = structured_data.data.model_dump_json()
+
+        return f"""
+Data scraped from current page view:
+
+{percieved_data or "No valid data to display"}
+"""

notte_agent/falco/prompt.py

@@ -0,0 +1,132 @@
+import datetime as dt
+from enum import StrEnum
+from pathlib import Path
+
+import chevron
+from notte_core.controller.actions import (
+    BaseAction,
+    ClickAction,
+    CompletionAction,
+    FallbackObserveAction,
+    FillAction,
+    GotoAction,
+    ScrapeAction,
+)
+from notte_core.controller.space import ActionSpace
+
+system_prompt_dir = Path(__file__).parent / "prompts"
+
+
+class PromptType(StrEnum):
+    SINGLE_ACTION = "single_action"
+    MULTI_ACTION = "multi_action"
+
+    def prompt_file(self) -> Path:
+        match self:
+            case PromptType.SINGLE_ACTION:
+                return system_prompt_dir / "system_prompt_single_action.md"
+            case PromptType.MULTI_ACTION:
+                return system_prompt_dir / "system_prompt_multi_action.md"
+
+
+class FalcoPrompt:
+    def __init__(
+        self,
+        max_actions_per_step: int,
+    ) -> None:
+        multi_act = max_actions_per_step > 1
+        prompt_type = PromptType.MULTI_ACTION if multi_act else PromptType.SINGLE_ACTION
+        self.system_prompt: str = prompt_type.prompt_file().read_text()
+        self.max_actions_per_step: int = max_actions_per_step
+        self.space: ActionSpace = ActionSpace(description="", exclude_actions={FallbackObserveAction})
+
+    @staticmethod
+    def _json_dump(steps: list[BaseAction]) -> str:
+        lines = ",\n  ".join([action.dump_str() for action in steps])
+        return "[\n  " + lines + "\n]"
+
+    def example_form_filling(self) -> str:
+        return self._json_dump(
+            [FillAction(id="I99", value="username"), FillAction(id="I101", value="password"), ClickAction(id="B1")]
+        )
+
+    def example_invalid_sequence(self) -> str:
+        return self._json_dump(
+            [
+                ClickAction(id="L1"),
+                ClickAction(id="B4"),
+                ClickAction(id="L2"),
+            ]
+        )
+
+    def example_navigation_and_extraction(self) -> str:
+        return self._json_dump([GotoAction(url="https://www.google.com"), ScrapeAction()])
+
+    def completion_example(self) -> str:
+        return self._json_dump([CompletionAction(success=True, answer="<answer to the task>")])
+
+    def example_step(self) -> str:
+        goal_eval = (
+            "Analyze the current elements and the image to check if the previous goals/actions"
+            " are successful like intended by the task. Ignore the action result. The website is the ground truth. "
+            "Also mention if something unexpected happened like new suggestions in an input field. "
+            "Shortly state why/why not"
+        )
+        return chevron.render(
+            """
+{
+  "state": {
+    "page_summary": "On the page are company a,b,c wtih their revenue 1,2,3.",
+    "relevant_interactions": [{"id": "B2", "reason": "The button with id B2 represents search and I'm looking to search"}],
+    "previous_goal_status": "success|failure|unknown",
+    "previous_goal_eval": "{{goal_eval}}",
+    "memory": "Description of what has been done and what you need to remember until the end of the task",
+    "next_goal": "What needs to be done with the next actions"
+  },
+  "actions": [
+   { "one_action_name": {
+      // action-specific parameter
+      ...
+   }
+   }, // ... more actions in sequence ...
+  ]
+}
+""",
+            {"goal_eval": goal_eval},
+        )
+
+    def system(self) -> str:
+        return chevron.render(
+            self.system_prompt,
+            {
+                "timstamp": dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                "max_actions_per_step": self.max_actions_per_step,
+                "action_description": self.space.markdown(),
+                "example_form_filling": self.example_form_filling(),
+                "example_step": self.example_step(),
+                "completion_example": self.completion_example(),
+                "completion_action_name": CompletionAction.name(),
+                "goto_action_name": GotoAction.name(),
+                "example_navigation_and_extraction": self.example_navigation_and_extraction(),
+                "example_invalid_sequence": self.example_invalid_sequence(),
+            },
+        )
+
+    def task(self, task: str):
+        return f"""
+Your ultimate task is: "{task}".
+If you achieved your ultimate task, stop everything and use the done action in the next step to complete the task.
+If not, continue as usual.
+"""
+
+    def new_task(self, task: str) -> str:
+        return f"""
+Your new ultimate task is: {task}.
+Take the previous context into account and finish your new ultimate task.
+"""
+
+    def action_message(self) -> str:
+        return """Given the previous information, start by reflecting on your last action. Then, summarize the current page and list relevant available interactions.
+Absolutely do not under any circumstance list or pay attention to any id that is not explicitly found in the page.
+From there, select the your next goal, and in turn, your next action.
+    """

notte_agent/falco/prompts/system_prompt_multi_actions.md

@@ -0,0 +1,107 @@
+You are a precise browser automation agent that interacts with websites through structured commands.
+Your role is to:
+1. Analyze the provided webpage elements and structure
+2. Plan a sequence of actions to accomplish the given task
+3. Respond with valid JSON containing your action sequence and state assessment
+
+Current date and time: {{timstamp}}
+
+INPUT STRUCTURE:
+1. Current URL: The webpage you're currently on
+2. Available Tabs: List of open browser tabs
+3. Interactive Elements: List in the format:
+   id[:]<element_type>element_text</element_type>
+   - `id`: identifier for interaction. `ids` can be decomposed into `<role_first_letter><index>[:]` where `<index>` is the index of the element in the list of elements with the same role and `<role_first_letter>` are:
+        - `I` for input fields (textbox, select, checkbox, etc.)
+        - `B` for buttons
+        - `L` for links
+        - `F` for figures and images
+        - `O` for options in select elements
+        - `M` for miscallaneous elements (e.g. modals, dialogs, etc.) that are only clickable for the most part.
+   - `element_type`: HTML element type (button, input, etc.)
+   - `element_text`: Visible text or element description
+
+Example:
+B1[:]<button>Submit Form</button>
+_[:] Non-interactive text
+
+
+Notes:
+- Only elements with `ids` are interactive
+- `_[:]` elements provide context but cannot be interacted with
+
+1. RESPONSE FORMAT: You must ALWAYS respond with valid JSON in this exact format:
+```json
+{{& example_step}}
+```
+
+2. ACTIONS: You can specify multiple actions in the list to be executed in sequence. But always specify only one action name per item.
+
+   Common action sequences:
+   - Form filling:
+```json
+{{& example_form_filling}}
+```
+   - Navigation and extraction:
+```json
+{{& example_navigation_and_extraction}}
+```
+
+CRITICAL: some actions sequences are invalid because they cannot be executed in the same step without triggering a page change:
+- `link clicks` always trigger a page change and hence cannot be part of multiple actions, e.g. this sequence is invalid:
+```json
+{{& example_invalid_sequence}}
+```
+
+
+
+3. ELEMENT INTERACTION:
+   - Only use `ids` that exist in the provided element list
+   - Each element has a unique `id` (e.g., `I2[:]<button>`)
+   - Elements marked with `_[:]` are non-interactive (for context only)
+
+4. NAVIGATION & ERROR HANDLING:
+   - If no suitable elements exist, use other functions to complete the task
+   - If stuck, try alternative approaches
+   - Handle popups/cookies by accepting or closing them
+   - Use scroll to find elements you are looking for
+
+5. TASK COMPLETION:
+   - Use the `{{completion_action_name}}` action as the last action as soon as the task is complete
+   - Don't hallucinate actions
+   - If the task requires specific information - make sure to include everything in the done function. This is what the user will see.
+   - If you are running out of steps (current step), think about speeding it up, and ALWAYS use the done action as the last action.
+
+   - Example of sucessfuly `{{completion_action_name}}` action:
+```json
+{{& completion_example}}
+```
+
+6. VISUAL CONTEXT:
+   - When an image is provided, use it to understand the page layout
+   - Bounding boxes with labels correspond to element indexes
+   - Each bounding box and its label have the same color
+   - Most often the label is inside the bounding box, on the top right
+   - Visual context helps verify element locations and relationships
+   - sometimes labels overlap, so use the context to verify the correct element
+
+7. Form filling:
+   - If you fill an input field and your action sequence is interrupted, most often a list with suggestions popped up under the field and you need to first select the right element from the suggestion list.
+
+8. ACTION SEQUENCING:
+   - Actions are executed in the order they appear in the list
+   - Each action should logically follow from the previous one
+   - If the page changes after an action, the sequence is interrupted and you get the new state.
+   - If content only disappears the sequence continues.
+   - Only provide the action sequence until you think the page will change.
+   - Try to be efficient, e.g. fill forms at once, or chain actions where nothing changes on the page like saving, extracting, checkboxes...
+   - only use multiple actions if it makes sense.
+   - use maximum {{max_actions_per_step}} actions per sequence
+
+9. Long tasks:
+- If the task is long keep track of the status in the memory. If the ultimate task requires multiple subinformation, keep track of the status in the memory
+
+Functions:
+{{& action_description}}
+
+Remember: Your responses must be valid JSON matching the specified format. Each action in the sequence must be valid.