autoglm-gui 1.4.1__py3-none-any.whl → 1.5.1__py3-none-any.whl

AutoGLM_GUI/agents/factory.py

@@ -6,20 +6,14 @@ making it easy to add new agent types without modifying existing code.
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Callable, Dict
+from typing import Callable, Dict
 
+from AutoGLM_GUI.config import AgentConfig, ModelConfig
 from AutoGLM_GUI.logger import logger
 from AutoGLM_GUI.types import AgentSpecificConfig
 
 from .protocols import BaseAgent
 
-if TYPE_CHECKING:
-    from phone_agent import PhoneAgent
-    from phone_agent.agent import AgentConfig
-    from phone_agent.model import ModelConfig
-
-    from .mai_adapter import MAIAgentAdapter
-
 
 # Agent registry: agent_type -> (creator_function, config_schema)
 AGENT_REGISTRY: Dict[str, Callable] = {}
@@ -52,12 +46,13 @@ def register_agent(
 
 def create_agent(
     agent_type: str,
-    model_config: "ModelConfig",
-    agent_config: "AgentConfig",
+    model_config: ModelConfig,
+    agent_config: AgentConfig,
     agent_specific_config: AgentSpecificConfig,
+    device,
     takeover_callback: Callable | None = None,
     confirmation_callback: Callable | None = None,
-) -> "BaseAgent":
+) -> BaseAgent:
     """
     Create an agent instance using the factory pattern.
 
@@ -66,6 +61,7 @@ def create_agent(
         model_config: Model configuration
         agent_config: Agent configuration
         agent_specific_config: Agent-specific configuration (e.g., MAIConfig fields)
+        device: DeviceProtocol instance (provided by PhoneAgentManager)
         takeover_callback: Takeover callback
         confirmation_callback: Confirmation callback
 
@@ -88,6 +84,7 @@ def create_agent(
             model_config=model_config,
             agent_config=agent_config,
             agent_specific_config=agent_specific_config,
+            device=device,
             takeover_callback=takeover_callback,
             confirmation_callback=confirmation_callback,
         )
@@ -111,50 +108,46 @@ def is_agent_type_registered(agent_type: str) -> bool:
 # ==================== Built-in Agent Creators ====================
 
 
-def _create_phone_agent(
-    model_config: "ModelConfig",
-    agent_config: "AgentConfig",
+def _create_glm_agent_v2(
+    model_config: ModelConfig,
+    agent_config: AgentConfig,
     agent_specific_config: AgentSpecificConfig,
+    device,
     takeover_callback: Callable | None = None,
     confirmation_callback: Callable | None = None,
-) -> "PhoneAgent":
-    from phone_agent import PhoneAgent
+) -> BaseAgent:
+    from .glm.agent import GLMAgent
 
-    return PhoneAgent(
+    return GLMAgent(
         model_config=model_config,
         agent_config=agent_config,
-        takeover_callback=takeover_callback,
+        device=device,
         confirmation_callback=confirmation_callback,
+        takeover_callback=takeover_callback,
     )
 
 
-def _create_mai_agent(
-    model_config: "ModelConfig",
-    agent_config: "AgentConfig",
+def _create_internal_mai_agent(
+    model_config: ModelConfig,
+    agent_config: AgentConfig,
     agent_specific_config: AgentSpecificConfig,
+    device,
     takeover_callback: Callable | None = None,
     confirmation_callback: Callable | None = None,
-) -> "MAIAgentAdapter":
-    from .mai_adapter import MAIAgentAdapter, MAIAgentConfig
-
-    # Build MAI config from dict
-    mai_config = MAIAgentConfig(
-        history_n=agent_specific_config.get("history_n", 3),
-        max_pixels=agent_specific_config.get("max_pixels"),
-        min_pixels=agent_specific_config.get("min_pixels"),
-        tools=agent_specific_config.get("tools"),
-        use_mai_prompt=agent_specific_config.get("use_mai_prompt", False),
-    )
+) -> BaseAgent:
+    from .mai.agent import InternalMAIAgent
+
+    history_n = agent_specific_config.get("history_n", 3)
 
-    return MAIAgentAdapter(
+    return InternalMAIAgent(
         model_config=model_config,
         agent_config=agent_config,
-        mai_config=mai_config,
-        takeover_callback=takeover_callback,
+        device=device,
+        history_n=history_n,
         confirmation_callback=confirmation_callback,
+        takeover_callback=takeover_callback,
     )
 
 
-# Register built-in agents
-register_agent("glm", _create_phone_agent)
-register_agent("mai", _create_mai_agent)
+register_agent("glm", _create_glm_agent_v2)
+register_agent("mai", _create_internal_mai_agent)

AutoGLM_GUI/agents/glm/__init__.py

@@ -0,0 +1,7 @@
+from .prompts_en import SYSTEM_PROMPT as SYSTEM_PROMPT_EN
+from .prompts_zh import SYSTEM_PROMPT as SYSTEM_PROMPT_ZH
+
+__all__ = [
+    "SYSTEM_PROMPT_EN",
+    "SYSTEM_PROMPT_ZH",
+]

AutoGLM_GUI/agents/glm/agent.py

@@ -0,0 +1,297 @@
+import json
+import traceback
+from typing import Any, Callable, cast
+
+from openai import OpenAI
+
+from AutoGLM_GUI.actions import ActionHandler, ActionResult
+from AutoGLM_GUI.config import AgentConfig, ModelConfig, StepResult
+from AutoGLM_GUI.device_protocol import DeviceProtocol
+from AutoGLM_GUI.logger import logger
+from AutoGLM_GUI.prompt_config import get_messages, get_system_prompt
+
+from .message_builder import MessageBuilder
+from .parser import GLMParser
+
+
+class GLMAgent:
+    def __init__(
+        self,
+        model_config: ModelConfig,
+        agent_config: AgentConfig,
+        device: DeviceProtocol,
+        confirmation_callback: Callable[[str], bool] | None = None,
+        takeover_callback: Callable[[str], None] | None = None,
+        thinking_callback: Callable[[str], None] | None = None,
+    ):
+        self.model_config = model_config
+        self.agent_config = agent_config
+
+        self.openai_client = OpenAI(
+            base_url=model_config.base_url,
+            api_key=model_config.api_key,
+            timeout=120,
+        )
+        self.parser = GLMParser()
+
+        self.device = device
+        self.action_handler = ActionHandler(
+            device=self.device,
+            confirmation_callback=confirmation_callback,
+            takeover_callback=takeover_callback,
+        )
+
+        self._context: list[dict[str, Any]] = []
+        self._step_count = 0
+        self._is_running = False
+        self._thinking_callback = thinking_callback
+
+    def run(self, task: str) -> str:
+        self._context = []
+        self._step_count = 0
+        self._is_running = True
+
+        try:
+            result = self._execute_step(task, is_first=True)
+
+            if result.finished:
+                return result.message or "Task completed"
+
+            while self._step_count < self.agent_config.max_steps and self._is_running:
+                result = self._execute_step(is_first=False)
+
+                if result.finished:
+                    return result.message or "Task completed"
+
+            return "Max steps reached"
+        finally:
+            self._is_running = False
+
+    def step(self, task: str | None = None) -> StepResult:
+        is_first = len(self._context) == 0
+
+        if is_first and not task:
+            raise ValueError("Task is required for the first step")
+
+        return self._execute_step(task, is_first)
+
+    def reset(self) -> None:
+        self._context = []
+        self._step_count = 0
+        self._is_running = False
+
+    def abort(self) -> None:
+        self._is_running = False
+        logger.info("Agent aborted by user")
+
+    def _stream_request(
+        self,
+        messages: list[dict[str, Any]],
+        on_thinking_chunk: Callable[[str], None] | None = None,
+    ) -> tuple[str, str, str]:
+        stream = self.openai_client.chat.completions.create(
+            messages=cast(Any, messages),
+            model=self.model_config.model_name,
+            max_tokens=self.model_config.max_tokens,
+            temperature=self.model_config.temperature,
+            top_p=self.model_config.top_p,
+            frequency_penalty=self.model_config.frequency_penalty,
+            extra_body=self.model_config.extra_body,
+            stream=True,
+        )
+
+        raw_content = ""
+        buffer = ""
+        action_markers = ["finish(message=", "do(action="]
+        in_action_phase = False
+
+        for chunk in stream:
+            if len(chunk.choices) == 0:
+                continue
+            if chunk.choices[0].delta.content is not None:
+                content = chunk.choices[0].delta.content
+                raw_content += content
+
+                if in_action_phase:
+                    continue
+
+                buffer += content
+
+                marker_found = False
+                for marker in action_markers:
+                    if marker in buffer:
+                        thinking_part = buffer.split(marker, 1)[0]
+                        if on_thinking_chunk:
+                            on_thinking_chunk(thinking_part)
+                        in_action_phase = True
+                        marker_found = True
+                        break
+
+                if marker_found:
+                    continue
+
+                is_potential_marker = False
+                for marker in action_markers:
+                    for i in range(1, len(marker)):
+                        if buffer.endswith(marker[:i]):
+                            is_potential_marker = True
+                            break
+                    if is_potential_marker:
+                        break
+
+                if not is_potential_marker:
+                    if on_thinking_chunk:
+                        on_thinking_chunk(buffer)
+                    buffer = ""
+
+        thinking, action = self._parse_raw_response(raw_content)
+        return thinking, action, raw_content
+
+    def _parse_raw_response(self, content: str) -> tuple[str, str]:
+        if "finish(message=" in content:
+            parts = content.split("finish(message=", 1)
+            thinking = parts[0].strip()
+            action = "finish(message=" + parts[1]
+            return thinking, action
+
+        if "do(action=" in content:
+            parts = content.split("do(action=", 1)
+            thinking = parts[0].strip()
+            action = "do(action=" + parts[1]
+            return thinking, action
+
+        if "<answer>" in content:
+            parts = content.split("<answer>", 1)
+            thinking = parts[0].replace("<think>", "").replace("</think>", "").strip()
+            action = parts[1].replace("</answer>", "").strip()
+            return thinking, action
+
+        return "", content
+
+    def _execute_step(
+        self, user_prompt: str | None = None, is_first: bool = False
+    ) -> StepResult:
+        self._step_count += 1
+
+        screenshot = self.device.get_screenshot()
+        current_app = self.device.get_current_app()
+
+        if is_first:
+            system_prompt = self.agent_config.system_prompt
+            if system_prompt is None:
+                system_prompt = get_system_prompt(self.agent_config.lang)
+
+            self._context.append(MessageBuilder.create_system_message(system_prompt))
+
+            screen_info = MessageBuilder.build_screen_info(current_app)
+            text_content = f"{user_prompt}\n\n{screen_info}"
+
+            self._context.append(
+                MessageBuilder.create_user_message(
+                    text=text_content, image_base64=screenshot.base64_data
+                )
+            )
+        else:
+            screen_info = MessageBuilder.build_screen_info(current_app)
+            # 如果有新的用户消息（多轮对话场景），把它加入消息中
+            if user_prompt:
+                text_content = f"{user_prompt}\n\n** Screen Info **\n\n{screen_info}"
+            else:
+                # 继续执行当前任务，只需要屏幕信息
+                text_content = f"** Screen Info **\n\n{screen_info}"
+
+            self._context.append(
+                MessageBuilder.create_user_message(
+                    text=text_content, image_base64=screenshot.base64_data
+                )
+            )
+
+        try:
+            msgs = get_messages(self.agent_config.lang)
+            if self.agent_config.verbose:
+                print("\n" + "=" * 50)
+                print(f"💭 {msgs['thinking']}:")
+                print("-" * 50)
+
+            callback = self._thinking_callback
+            if callback is None and self.agent_config.verbose:
+
+                def print_chunk(chunk: str) -> None:
+                    print(chunk, end="", flush=True)
+
+                callback = print_chunk
+
+            thinking, action_str, raw_content = self._stream_request(
+                self._context, on_thinking_chunk=callback
+            )
+        except Exception as e:
+            if self.agent_config.verbose:
+                traceback.print_exc()
+            return StepResult(
+                success=False,
+                finished=True,
+                action=None,
+                thinking="",
+                message=f"Model error: {e}",
+            )
+
+        try:
+            action = self.parser.parse(action_str)
+        except ValueError as e:
+            if self.agent_config.verbose:
+                logger.warning(f"Failed to parse action: {e}, treating as finish")
+            action = {"_metadata": "finish", "message": action_str}
+
+        if self.agent_config.verbose:
+            print()
+            print("-" * 50)
+            print(f"🎯 {msgs['action']}:")
+            print(json.dumps(action, ensure_ascii=False, indent=2))
+            print("=" * 50 + "\n")
+
+        self._context[-1] = MessageBuilder.remove_images_from_message(self._context[-1])
+
+        try:
+            result = self.action_handler.execute(
+                action, screenshot.width, screenshot.height
+            )
+        except Exception as e:
+            if self.agent_config.verbose:
+                traceback.print_exc()
+            result = ActionResult(success=False, should_finish=True, message=str(e))
+
+        self._context.append(
+            MessageBuilder.create_assistant_message(
+                f"<think>{thinking}</think><answer>{action_str}</answer>"
+            )
+        )
+
+        finished = action.get("_metadata") == "finish" or result.should_finish
+
+        if finished and self.agent_config.verbose:
+            msgs = get_messages(self.agent_config.lang)
+            print("\n" + "🎉 " + "=" * 48)
+            print(
+                f"✅ {msgs['task_completed']}: {result.message or action.get('message', msgs['done'])}"
+            )
+            print("=" * 50 + "\n")
+
+        return StepResult(
+            success=result.success,
+            finished=finished,
+            action=action,
+            thinking=thinking,
+            message=result.message or action.get("message"),
+        )
+
+    @property
+    def context(self) -> list[dict[str, Any]]:
+        return self._context.copy()
+
+    @property
+    def step_count(self) -> int:
+        return self._step_count
+
+    @property
+    def is_running(self) -> bool:
+        return self._is_running

AutoGLM_GUI/agents/glm/message_builder.py

@@ -0,0 +1,81 @@
+"""Message builder for GLM agent - copied from phone_agent.model.client.
+
+This is an exact copy of the upstream MessageBuilder to ensure consistent behavior.
+"""
+
+import json
+from typing import Any
+
+
+class MessageBuilder:
+    """Helper class for building conversation messages."""
+
+    @staticmethod
+    def create_system_message(content: str) -> dict[str, Any]:
+        """Create a system message."""
+        return {"role": "system", "content": content}
+
+    @staticmethod
+    def create_user_message(
+        text: str, image_base64: str | None = None
+    ) -> dict[str, Any]:
+        """
+        Create a user message with optional image.
+
+        Args:
+            text: Text content.
+            image_base64: Optional base64-encoded image.
+
+        Returns:
+            Message dictionary.
+        """
+        content = []
+
+        if image_base64:
+            content.append(
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/png;base64,{image_base64}"},
+                }
+            )
+
+        content.append({"type": "text", "text": text})
+
+        return {"role": "user", "content": content}
+
+    @staticmethod
+    def create_assistant_message(content: str) -> dict[str, Any]:
+        """Create an assistant message."""
+        return {"role": "assistant", "content": content}
+
+    @staticmethod
+    def remove_images_from_message(message: dict[str, Any]) -> dict[str, Any]:
+        """
+        Remove image content from a message to save context space.
+
+        Args:
+            message: Message dictionary.
+
+        Returns:
+            Message with images removed.
+        """
+        if isinstance(message.get("content"), list):
+            message["content"] = [
+                item for item in message["content"] if item.get("type") == "text"
+            ]
+        return message
+
+    @staticmethod
+    def build_screen_info(current_app: str, **extra_info) -> str:
+        """
+        Build screen info string for the model.
+
+        Args:
+            current_app: Current app name.
+            **extra_info: Additional info to include.
+
+        Returns:
+            JSON string with screen info.
+        """
+        info = {"current_app": current_app, **extra_info}
+        return json.dumps(info, ensure_ascii=False)

AutoGLM_GUI/agents/glm/parser.py

@@ -0,0 +1,110 @@
+import ast
+from typing import Any
+
+
+class GLMParser:
+    @property
+    def coordinate_scale(self) -> int:
+        return 1000
+
+    def parse(self, raw_response: str) -> dict[str, Any]:
+        action_str = raw_response.strip()
+
+        if action_str.startswith("finish("):
+            return self._parse_finish(action_str)
+        if action_str.startswith("do("):
+            return self._parse_do(action_str)
+        raise ValueError(f"Unknown action format: {action_str}")
+
+    def _parse_finish(self, action_str: str) -> dict[str, Any]:
+        try:
+            params = self._extract_params(action_str, "finish")
+            return {
+                "_metadata": "finish",
+                "message": params.get("message", "Task completed"),
+            }
+        except Exception as e:
+            raise ValueError(f"Failed to parse finish action: {e}") from e
+
+    def _parse_do(self, action_str: str) -> dict[str, Any]:
+        try:
+            params = self._extract_params(action_str, "do")
+            action_name = params.get("action", "")
+
+            result = {
+                "_metadata": "do",
+                "action": action_name,
+            }
+
+            for key, value in params.items():
+                if key != "action":
+                    result[key] = value
+
+            return result
+        except Exception as e:
+            raise ValueError(f"Failed to parse do action: {e}") from e
+
+    def _extract_params(self, action_str: str, function_name: str) -> dict[str, Any]:
+        prefix = f"{function_name}("
+        if not action_str.startswith(prefix):
+            raise ValueError(f"Action does not start with {prefix}")
+
+        params_str = action_str[len(prefix) : -1]
+
+        params: dict[str, Any] = {}
+        current_key = None
+        current_value = ""
+        in_quotes = False
+        quote_char = None
+        bracket_depth = 0
+        i = 0
+
+        while i < len(params_str):
+            char = params_str[i]
+
+            if char in ('"', "'") and (i == 0 or params_str[i - 1] != "\\"):
+                if not in_quotes:
+                    in_quotes = True
+                    quote_char = char
+                elif char == quote_char:
+                    in_quotes = False
+                    quote_char = None
+
+            if not in_quotes:
+                if char in ("[", "{"):
+                    bracket_depth += 1
+                elif char in ("]", "}"):
+                    bracket_depth -= 1
+
+                if char == "=" and bracket_depth == 0:
+                    current_key = current_value.strip()
+                    current_value = ""
+                    i += 1
+                    continue
+
+                if char == "," and bracket_depth == 0:
+                    if current_key:
+                        params[current_key] = self._parse_value(current_value.strip())
+                        current_key = None
+                        current_value = ""
+                    i += 1
+                    continue
+
+            current_value += char
+            i += 1
+
+        if current_key:
+            params[current_key] = self._parse_value(current_value.strip())
+
+        return params
+
+    def _parse_value(self, value_str: str) -> Any:
+        value_str = value_str.strip()
+
+        if not value_str:
+            return ""
+
+        try:
+            return ast.literal_eval(value_str)
+        except (ValueError, SyntaxError):
+            return value_str

{phone_agent/config → AutoGLM_GUI/agents/glm}/prompts_en.py

@@ -1,5 +1,3 @@
-"""System prompts for the AI agent."""
-
 from datetime import datetime
 
 today = datetime.today()
@@ -30,44 +28,44 @@ Your output should STRICTLY follow the format:
   Perform a tap action on a specified screen area. The element is a list of 2 integers, representing the coordinates of the tap point.
   **Example**:
   <answer>
-  do(action="Tap", element=[x,y])
+  do(action=\"Tap\", element=[x,y])
   </answer>
 - **Type**
   Enter text into the currently focused input field.
   **Example**:
   <answer>
-  do(action="Type", text="Hello World")
+  do(action=\"Type\", text=\"Hello World\")
   </answer>
 - **Swipe**
   Perform a swipe action with start point and end point.
   **Examples**:
   <answer>
-  do(action="Swipe", start=[x1,y1], end=[x2,y2])
+  do(action=\"Swipe\", start=[x1,y1], end=[x2,y2])
   </answer>
 - **Long Press**
   Perform a long press action on a specified screen area.
   You can add the element to the action to specify the long press area. The element is a list of 2 integers, representing the coordinates of the long press point.
   **Example**:
   <answer>
-  do(action="Long Press", element=[x,y])
+  do(action=\"Long Press\", element=[x,y])
   </answer>
 - **Launch**
   Launch an app. Try to use launch action when you need to launch an app. Check the instruction to choose the right app before you use this action.
   **Example**:
   <answer>
-  do(action="Launch", app="Settings")
+  do(action=\"Launch\", app=\"Settings\")
   </answer>
 - **Back**
   Press the Back button to navigate to the previous screen.
   **Example**:
   <answer>
-  do(action="Back")
+  do(action=\"Back\")
   </answer>
 - **Finish**
   Terminate the program and optionally print a message.
   **Example**:
   <answer>
-  finish(message="Task completed.")
+  finish(message=\"Task completed.\")
   </answer>