autoglm-gui 1.5.0__py3-none-any.whl → 1.5.1__py3-none-any.whl

mai_agent/mai_naivigation_agent.py

@@ -1,526 +0,0 @@
-# Copyright (c) 2025, Alibaba Cloud and its affiliates;
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-MAI Mobile Agent - A GUI automation agent for mobile devices.
-
-This module provides the MAIMobileAgent class that uses vision-language models
-to interact with mobile device interfaces based on natural language instructions.
-"""
-
-import copy
-import json
-import re
-import traceback
-from io import BytesIO
-from typing import Any, Dict, List, Optional, Tuple
-
-import numpy as np
-from openai import OpenAI
-from PIL import Image
-
-from base import BaseAgent
-from prompt import MAI_MOBILE_SYS_PROMPT, MAI_MOBILE_SYS_PROMPT_ASK_USER_MCP
-from unified_memory import TrajStep
-from utils import pil_to_base64, safe_pil_to_bytes
-
-# Constants
-SCALE_FACTOR = 999
-
-
-def mask_image_urls_for_logging(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    """
-    Create a copy of messages with image URLs masked for logging.
-
-    Args:
-        messages: List of message dictionaries that may contain image URLs.
-
-    Returns:
-        Deep copy of messages with image URLs replaced by "[IMAGE_DATA]".
-    """
-    messages_masked = copy.deepcopy(messages)
-    for message in messages_masked:
-        content = message.get("content", [])
-        if content and isinstance(content, list):
-            for item in content:
-                if isinstance(item, dict) and "image_url" in item:
-                    item["image_url"]["url"] = "[IMAGE_DATA]"
-    return messages_masked
-
-
-def parse_tagged_text(text: str) -> Dict[str, Any]:
-    """
-    Parse text containing XML-style tags to extract thinking and tool_call content.
-
-    Args:
-        text: Text containing <thinking> and <tool_call> tags.
-
-    Returns:
-        Dictionary with keys:
-            - "thinking": Content inside <thinking> tags (str or None)
-            - "tool_call": Parsed JSON content inside <tool_call> tags (dict or None)
-
-    Raises:
-        ValueError: If tool_call content is not valid JSON.
-    """
-    # Handle thinking model output format (uses </think> instead of </thinking>)
-    if "</think>" in text and "</thinking>" not in text:
-        text = text.replace("</think>", "</thinking>")
-        text = "<thinking>" + text
-
-    # Define regex pattern with non-greedy matching
-    pattern = r"<thinking>(.*?)</thinking>.*?<tool_call>(.*?)</tool_call>"
-
-    result: Dict[str, Any] = {
-        "thinking": None,
-        "tool_call": None,
-    }
-
-    # Use re.DOTALL to match newlines
-    match = re.search(pattern, text, re.DOTALL)
-    if match:
-        result = {
-            "thinking": match.group(1).strip().strip('"'),
-            "tool_call": match.group(2).strip().strip('"'),
-        }
-
-    # Parse tool_call as JSON
-    if result["tool_call"]:
-        try:
-            result["tool_call"] = json.loads(result["tool_call"])
-        except json.JSONDecodeError as e:
-            raise ValueError(f"Invalid JSON in tool_call: {e}")
-
-    return result
-
-
-def parse_action_to_structure_output(text: str) -> Dict[str, Any]:
-    """
-    Parse model output text into structured action format.
-
-    Args:
-        text: Raw model output containing thinking and tool_call tags.
-
-    Returns:
-        Dictionary with keys:
-            - "thinking": The model's reasoning process
-            - "action_json": Parsed action with normalized coordinates
-
-    Note:
-        Coordinates are normalized to [0, 1] range by dividing by SCALE_FACTOR.
-    """
-    text = text.strip()
-
-    results = parse_tagged_text(text)
-    thinking = results["thinking"]
-    tool_call = results["tool_call"]
-    action = tool_call["arguments"]
-
-    # Normalize coordinates from SCALE_FACTOR range to [0, 1]
-    if "coordinate" in action:
-        coordinates = action["coordinate"]
-        if len(coordinates) == 2:
-            point_x, point_y = coordinates
-        elif len(coordinates) == 4:
-            x1, y1, x2, y2 = coordinates
-            point_x = (x1 + x2) / 2
-            point_y = (y1 + y2) / 2
-        else:
-            raise ValueError(
-                f"Invalid coordinate format: expected 2 or 4 values, got {len(coordinates)}"
-            )
-        point_x = point_x / SCALE_FACTOR
-        point_y = point_y / SCALE_FACTOR
-        action["coordinate"] = [point_x, point_y]
-
-    return {
-        "thinking": thinking,
-        "action_json": action,
-    }
-
-
-class MAIUINaivigationAgent(BaseAgent):
-    """
-    Mobile automation agent using vision-language models.
-
-    This agent processes screenshots and natural language instructions to
-    generate GUI actions for mobile device automation.
-
-    Attributes:
-        llm_base_url: Base URL for the LLM API endpoint.
-        model_name: Name of the model to use for predictions.
-        runtime_conf: Configuration dictionary for runtime parameters.
-        history_n: Number of history steps to include in context.
-    """
-
-    def __init__(
-        self,
-        llm_base_url: str,
-        model_name: str,
-        runtime_conf: Optional[Dict[str, Any]] = None,
-        tools: Optional[List[Dict[str, Any]]] = None,
-    ):
-        """
-        Initialize the MAIMobileAgent.
-
-        Args:
-            llm_base_url: Base URL for the LLM API endpoint.
-            model_name: Name of the model to use.
-            runtime_conf: Optional configuration dictionary with keys:
-                - history_n: Number of history images to include (default: 3)
-                - max_pixels: Maximum pixels for image processing
-                - min_pixels: Minimum pixels for image processing
-                - temperature: Sampling temperature (default: 0.0)
-                - top_k: Top-k sampling parameter (default: -1)
-                - top_p: Top-p sampling parameter (default: 1.0)
-                - max_tokens: Maximum tokens in response (default: 2048)
-            tools: Optional list of MCP tool definitions. Each tool should be a dict
-                with 'name', 'description', and 'parameters' keys.
-        """
-        super().__init__()
-
-        # Store MCP tools
-        self.tools = tools or []
-
-        # Set default configuration
-        default_conf = {
-            "history_n": 3,
-            "temperature": 0.0,
-            "top_k": -1,
-            "top_p": 1.0,
-            "max_tokens": 2048,
-        }
-        self.runtime_conf = {**default_conf, **(runtime_conf or {})}
-
-        self.llm_base_url = llm_base_url
-        self.model_name = model_name
-        self.llm = OpenAI(
-            base_url=self.llm_base_url,
-            api_key="empty",
-        )
-
-        # Extract frequently used config values
-        self.temperature = self.runtime_conf["temperature"]
-        self.top_k = self.runtime_conf["top_k"]
-        self.top_p = self.runtime_conf["top_p"]
-        self.max_tokens = self.runtime_conf["max_tokens"]
-        self.history_n = self.runtime_conf["history_n"]
-
-    @property
-    def system_prompt(self) -> str:
-        """
-        Generate the system prompt based on available MCP tools.
-
-        Returns:
-            System prompt string, with MCP tools section if tools are configured.
-        """
-        if self.tools:
-            tools_str = "\n".join(
-                [json.dumps(tool, ensure_ascii=False) for tool in self.tools]
-            )
-            return MAI_MOBILE_SYS_PROMPT_ASK_USER_MCP.render(tools=tools_str)
-        return MAI_MOBILE_SYS_PROMPT
-
-    @property
-    def history_responses(self) -> List[str]:
-        """
-        Generate formatted history responses for context.
-
-        Returns:
-            List of formatted response strings with thinking and tool_call tags.
-        """
-        history_responses = []
-
-        for step in self.traj_memory.steps:
-            thinking = step.thought
-            structured_action = step.structured_action
-
-            if not structured_action:
-                continue
-
-            action_json = copy.deepcopy(structured_action.get("action_json", {}))
-
-            # Convert normalized coordinates back to SCALE_FACTOR range
-            if "coordinate" in action_json:
-                coordinates = action_json.get("coordinate", [])
-                if len(coordinates) == 2:
-                    point_x, point_y = coordinates
-                elif len(coordinates) == 4:
-                    x1, y1, x2, y2 = coordinates
-                    point_x = (x1 + x2) / 2
-                    point_y = (y1 + y2) / 2
-                else:
-                    continue
-                action_json["coordinate"] = [
-                    int(point_x * SCALE_FACTOR),
-                    int(point_y * SCALE_FACTOR),
-                ]
-
-            tool_call_dict = {
-                "name": "mobile_use",
-                "arguments": action_json,
-            }
-            tool_call_json = json.dumps(tool_call_dict, separators=(",", ":"))
-            history_responses.append(
-                f"<thinking>\n{thinking}\n</thinking>\n<tool_call>\n{tool_call_json}\n</tool_call>"
-            )
-
-        return history_responses
-
-    def _prepare_images(self, screenshot_bytes: bytes) -> List[Image.Image]:
-        """
-        Prepare image list including history and current screenshot.
-
-        Args:
-            screenshot_bytes: Current screenshot as bytes.
-
-        Returns:
-            List of PIL Images (history + current).
-        """
-        # Calculate how many history images to include
-        if len(self.history_images) > 0:
-            max_history = min(len(self.history_images), self.history_n - 1)
-            recent_history = (
-                self.history_images[-max_history:] if max_history > 0 else []
-            )
-        else:
-            recent_history = []
-
-        # Add current image bytes
-        recent_history.append(screenshot_bytes)
-
-        # Normalize input type
-        if isinstance(recent_history, bytes):
-            recent_history = [recent_history]
-        elif isinstance(recent_history, np.ndarray):
-            recent_history = list(recent_history)
-        elif not isinstance(recent_history, list):
-            raise TypeError(f"Unidentified images type: {type(recent_history)}")
-
-        # Convert all images to PIL format
-        images = []
-        for image in recent_history:
-            if isinstance(image, bytes):
-                image = Image.open(BytesIO(image))
-            elif isinstance(image, Image.Image):
-                pass
-            else:
-                raise TypeError(f"Expected bytes or PIL Image, got {type(image)}")
-
-            if image.mode != "RGB":
-                image = image.convert("RGB")
-
-            images.append(image)
-
-        return images
-
-    def _build_messages(
-        self,
-        instruction: str,
-        images: List[Image.Image],
-    ) -> List[Dict[str, Any]]:
-        """
-        Build the message list for the LLM API call.
-
-        Args:
-            instruction: Task instruction from user.
-            images: List of prepared images.
-
-        Returns:
-            List of message dictionaries for the API.
-        """
-        messages = [
-            {
-                "role": "system",
-                "content": [{"type": "text", "text": self.system_prompt}],
-            },
-            {
-                "role": "user",
-                "content": [{"type": "text", "text": instruction}],
-            },
-        ]
-
-        image_num = 0
-        history_responses = self.history_responses
-
-        if len(history_responses) > 0:
-            for history_idx, history_response in enumerate(history_responses):
-                # Only include images for recent history (last history_n responses)
-                if history_idx + self.history_n >= len(history_responses):
-                    # Add image before the assistant response
-                    if image_num < len(images) - 1:
-                        cur_image = images[image_num]
-                        encoded_string = pil_to_base64(cur_image)
-                        messages.append(
-                            {
-                                "role": "user",
-                                "content": [
-                                    {
-                                        "type": "image_url",
-                                        "image_url": {
-                                            "url": f"data:image/png;base64,{encoded_string}"
-                                        },
-                                    }
-                                ],
-                            }
-                        )
-                        image_num += 1
-
-                messages.append(
-                    {
-                        "role": "assistant",
-                        "content": [{"type": "text", "text": history_response}],
-                    }
-                )
-
-            # Add current image (last one in images list)
-            if image_num < len(images):
-                cur_image = images[image_num]
-                encoded_string = pil_to_base64(cur_image)
-                messages.append(
-                    {
-                        "role": "user",
-                        "content": [
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": f"data:image/png;base64,{encoded_string}"
-                                },
-                            }
-                        ],
-                    }
-                )
-        else:
-            # No history, just add the current image
-            cur_image = images[0]
-            encoded_string = pil_to_base64(cur_image)
-            messages.append(
-                {
-                    "role": "user",
-                    "content": [
-                        {
-                            "type": "image_url",
-                            "image_url": {
-                                "url": f"data:image/png;base64,{encoded_string}"
-                            },
-                        }
-                    ],
-                }
-            )
-
-        return messages
-
-    def predict(
-        self,
-        instruction: str,
-        obs: Dict[str, Any],
-        **kwargs: Any,
-    ) -> Tuple[str, Dict[str, Any]]:
-        """
-        Predict the next action based on the current observation.
-
-        Args:
-            instruction: Task instruction/goal.
-            obs: Current observation containing:
-                - screenshot: PIL Image or bytes of current screen
-                - accessibility_tree: Optional accessibility tree data
-            **kwargs: Additional arguments including:
-                - extra_info: Optional extra context string
-
-        Returns:
-            Tuple of (prediction_text, action_dict) where:
-                - prediction_text: Raw model response or error message
-                - action_dict: Parsed action dictionary
-        """
-        # Set task goal if not already set
-        if not self.traj_memory.task_goal:
-            self.traj_memory.task_goal = instruction
-
-        # Process screenshot
-        screenshot_pil = obs["screenshot"]
-        screenshot_bytes = safe_pil_to_bytes(screenshot_pil)
-
-        # Prepare images
-        images = self._prepare_images(screenshot_bytes)
-
-        # Build messages
-        messages = self._build_messages(instruction, images)
-
-        # Make API call with retry logic
-        max_retries = 3
-        prediction = None
-        action_json = None
-
-        for attempt in range(max_retries):
-            try:
-                messages_print = mask_image_urls_for_logging(messages)
-                print(f"Messages (attempt {attempt + 1}):\n{messages_print}")
-
-                response = self.llm.chat.completions.create(
-                    model=self.model_name,
-                    messages=messages,
-                    max_tokens=self.max_tokens,
-                    temperature=self.temperature,
-                    top_p=self.top_p,
-                    frequency_penalty=0.0,
-                    presence_penalty=0.0,
-                    extra_body={"repetition_penalty": 1.0, "top_k": self.top_k},
-                    seed=42,
-                )
-                prediction = response.choices[0].message.content.strip()
-                print(f"Raw response:\n{prediction}")
-
-                # Parse response
-                parsed_response = parse_action_to_structure_output(prediction)
-                thinking = parsed_response["thinking"]
-                action_json = parsed_response["action_json"]
-                print(f"Parsed response:\n{parsed_response}")
-                break
-
-            except Exception as e:
-                print(f"Error on attempt {attempt + 1}: {e}")
-                traceback.print_exc()
-                prediction = None
-                action_json = None
-
-        # Return error if all retries failed
-        if prediction is None or action_json is None:
-            print("Max retry attempts reached, returning error flag.")
-            return "llm client error", {"action": None}
-
-        # Create and store trajectory step
-        traj_step = TrajStep(
-            screenshot=screenshot_pil,
-            accessibility_tree=obs.get("accessibility_tree"),
-            prediction=prediction,
-            action=action_json,
-            conclusion="",
-            thought=thinking,
-            step_index=len(self.traj_memory.steps),
-            agent_type="MAIMobileAgent",
-            model_name=self.model_name,
-            screenshot_bytes=screenshot_bytes,
-            structured_action={"action_json": action_json},
-        )
-        self.traj_memory.steps.append(traj_step)
-
-        return prediction, action_json
-
-    def reset(self, runtime_logger: Any = None) -> None:
-        """
-        Reset the trajectory memory for a new task.
-
-        Args:
-            runtime_logger: Optional logger (unused, kept for API compatibility).
-        """
-        super().reset()

mai_agent/prompt.py

@@ -1,148 +0,0 @@
-# Copyright (c) 2025, Alibaba Cloud and its affiliates;
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""System prompts for MAI Mobile Agent."""
-
-from jinja2 import Template
-
-MAI_MOBILE_SYS_PROMPT = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
-
-## Output Format
-For each function call, return the thinking process in <thinking> </thinking> tags, and a json object with function name and arguments within <tool_call></tool_call> XML tags:
-```
-<thinking>
-...
-</thinking>
-<tool_call>
-{"name": "mobile_use", "arguments": <args-json-object>}
-</tool_call>
-```
-
-## Action Space
-
-{"action": "click", "coordinate": [x, y]}
-{"action": "long_press", "coordinate": [x, y]}
-{"action": "type", "text": ""}
-{"action": "swipe", "direction": "up or down or left or right", "coordinate": [x, y]} # "coordinate" is optional. Use the "coordinate" if you want to swipe a specific UI element.
-{"action": "open", "text": "app_name"}
-{"action": "drag", "start_coordinate": [x1, y1], "end_coordinate": [x2, y2]}
-{"action": "system_button", "button": "button_name"} # Options: back, home, menu, enter
-{"action": "wait"}
-{"action": "terminate", "status": "success or fail"}
-{"action": "answer", "text": "xxx"} # Use escape characters \\', \\", and \\n in text part to ensure we can parse the text in normal python string format.
-
-
-## Note
-- Write a small plan and finally summarize your next action (with its target element) in one sentence in <thinking></thinking> part.
-- Available Apps: `["Camera","Chrome","Clock","Contacts","Dialer","Files","Settings","Markor","Tasks","Simple Draw Pro","Simple Gallery Pro","Simple SMS Messenger","Audio Recorder","Pro Expense","Broccoli APP","OSMand","VLC","Joplin","Retro Music","OpenTracks","Simple Calendar Pro"]`.
-You should use the `open` action to open the app as possible as you can, because it is the fast way to open the app.
-- You must follow the Action Space strictly, and return the correct json object within <thinking> </thinking> and <tool_call></tool_call> XML tags.
-""".strip()
-
-
-MAI_MOBILE_SYS_PROMPT_NO_THINKING = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
-
-## Output Format
-For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
-```
-<tool_call>
-{"name": "mobile_use", "arguments": <args-json-object>}
-</tool_call>
-```
-
-## Action Space
-
-{"action": "click", "coordinate": [x, y]}
-{"action": "long_press", "coordinate": [x, y]}
-{"action": "type", "text": ""}
-{"action": "swipe", "direction": "up or down or left or right", "coordinate": [x, y]} # "coordinate" is optional. Use the "coordinate" if you want to swipe a specific UI element.
-{"action": "open", "text": "app_name"}
-{"action": "drag", "start_coordinate": [x1, y1], "end_coordinate": [x2, y2]}
-{"action": "system_button", "button": "button_name"} # Options: back, home, menu, enter
-{"action": "wait"}
-{"action": "terminate", "status": "success or fail"}
-{"action": "answer", "text": "xxx"} # Use escape characters \\', \\", and \\n in text part to ensure we can parse the text in normal python string format.
-
-
-## Note
-- Available Apps: `["Camera","Chrome","Clock","Contacts","Dialer","Files","Settings","Markor","Tasks","Simple Draw Pro","Simple Gallery Pro","Simple SMS Messenger","Audio Recorder","Pro Expense","Broccoli APP","OSMand","VLC","Joplin","Retro Music","OpenTracks","Simple Calendar Pro"]`.
-You should use the `open` action to open the app as possible as you can, because it is the fast way to open the app.
-- You must follow the Action Space strictly, and return the correct json object within <thinking> </thinking> and <tool_call></tool_call> XML tags.
-""".strip()
-
-
-# Placeholder prompts for future features
-MAI_MOBILE_SYS_PROMPT_ASK_USER_MCP = Template(
-    """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
-
-## Output Format
-For each function call, return the thinking process in <thinking> </thinking> tags, and a json object with function name and arguments within <tool_call></tool_call> XML tags:
-```
-<thinking>
-...
-</thinking>
-<tool_call>
-{"name": "mobile_use", "arguments": <args-json-object>}
-</tool_call>
-```
-
-## Action Space
-
-{"action": "click", "coordinate": [x, y]}
-{"action": "long_press", "coordinate": [x, y]}
-{"action": "type", "text": ""}
-{"action": "swipe", "direction": "up or down or left or right", "coordinate": [x, y]} # "coordinate" is optional. Use the "coordinate" if you want to swipe a specific UI element.
-{"action": "open", "text": "app_name"}
-{"action": "drag", "start_coordinate": [x1, y1], "end_coordinate": [x2, y2]}
-{"action": "system_button", "button": "button_name"} # Options: back, home, menu, enter 
-{"action": "wait"}
-{"action": "terminate", "status": "success or fail"} 
-{"action": "answer", "text": "xxx"} # Use escape characters \\', \\", and \\n in text part to ensure we can parse the text in normal python string format.
-{"action": "ask_user", "text": "xxx"} # you can ask user for more information to complete the task.
-{"action": "double_click", "coordinate": [x, y]}
-
-{% if tools -%}
-## MCP Tools
-You are also provided with MCP tools, you can use them to complete the task.
-{{ tools }}
-
-If you want to use MCP tools, you must output as the following format:
-```
-<thinking>
-...
-</thinking>
-<tool_call>
-{"name": <function-name>, "arguments": <args-json-object>}
-</tool_call>
-```
-{% endif -%}
-
-
-## Note
-- Available Apps: `["Contacts", "Settings", "Clock", "Maps", "Chrome", "Calendar", "files", "Gallery", "Taodian", "Mattermost", "Mastodon", "Mail", "SMS", "Camera"]`.
-- Write a small plan and finally summarize your next action (with its target element) in one sentence in <thinking></thinking> part.
-""".strip()
-)
-
-MAI_MOBILE_SYS_PROMPT_GROUNDING = """
-You are a GUI grounding agent. 
-## Task
-Given a screenshot and the user's grounding instruction. Your task is to accurately locate a UI element based on the user's instructions.
-First, you should carefully examine the screenshot and analyze the user's instructions,  translate the user's instruction into a effective reasoning process, and then provide the final coordinate.
-## Output Format
-Return a json object with a reasoning process in <grounding_think></grounding_think> tags, a [x,y] format coordinate within <answer></answer> XML tags:
-<grounding_think>...</grounding_think>
-<answer>
-{"coordinate": [x,y]}
-</answer>
-""".strip()