autoglm-gui 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

AutoGLM_GUI/api/__init__.py

@@ -19,6 +19,7 @@ from . import (
     control,
     devices,
     dual_model,
+    layered_agent,
     mcp,
     media,
     metrics,
@@ -89,6 +90,7 @@ def create_app() -> FastAPI:
     )
 
     app.include_router(agents.router)
+    app.include_router(layered_agent.router)
     app.include_router(devices.router)
     app.include_router(control.router)
     app.include_router(media.router)
@@ -97,9 +99,9 @@ def create_app() -> FastAPI:
     app.include_router(workflows.router)
     app.include_router(dual_model.router)
 
-    # Mount MCP server at root (mcp_app already has /mcp path prefix)
-    app.mount("/", mcp_app)
-
+    # Mount static files BEFORE MCP to ensure they have priority
+    # This is critical: FastAPI processes mounts in order, so static files
+    # must be mounted before the catch-all MCP mount
     static_dir = _get_static_dir()
     if static_dir is not None and static_dir.exists():
         assets_dir = static_dir / "assets"
@@ -113,11 +115,15 @@ def create_app() -> FastAPI:
                 return FileResponse(file_path)
             return FileResponse(static_dir / "index.html")
 
-        # Add catch-all route AFTER all mounts to ensure lower priority
+        # Add catch-all route for SPA (handles all non-API routes)
         app.add_api_route(
             "/{full_path:path}", serve_spa, methods=["GET"], include_in_schema=False
         )
 
+    # Mount MCP server at root (mcp_app already has /mcp path prefix)
+    # This must be AFTER static files to avoid intercepting them
+    app.mount("/", mcp_app)
+
     return app
 
 

AutoGLM_GUI/api/agents.py

@@ -460,7 +460,6 @@ def get_config_endpoint() -> ConfigResponse:
         decision_api_key=effective_config.decision_api_key
         if effective_config.decision_api_key
         else "",
-        thinking_mode=effective_config.thinking_mode,
         conflicts=[
             {
                 "field": c.field,
@@ -497,7 +496,6 @@ def save_config_endpoint(request: ConfigSaveRequest) -> dict:
             decision_base_url=request.decision_base_url,
             decision_model_name=request.decision_model_name,
             decision_api_key=request.decision_api_key,
-            thinking_mode=request.thinking_mode,
             merge_mode=True,
         )
 

AutoGLM_GUI/api/dual_model.py

@@ -30,19 +30,17 @@ class DualModelInitRequest(BaseModel):
     device_id: str
 
     # 决策大模型配置
-    decision_base_url: str = "https://api-inference.modelscope.cn/v1"
+    decision_base_url: str
     decision_api_key: str
-    decision_model_name: str = "ZhipuAI/GLM-4.7"
+    decision_model_name: str
 
     # 视觉小模型配置(复用现有配置)
     vision_base_url: Optional[str] = None
     vision_api_key: Optional[str] = None
     vision_model_name: Optional[str] = None
 
-    # 思考模式: fast 或 deep
-    thinking_mode: str = "deep"
-
     max_steps: int = 50
+    thinking_mode: str = "deep"  # fast, deep, turbo
 
 
 class DualModelChatRequest(BaseModel):
@@ -73,9 +71,12 @@ def init_dual_model(request: DualModelInitRequest) -> dict:
     from AutoGLM_GUI.phone_agent_manager import PhoneAgentManager
 
     device_id = request.device_id
-    thinking_mode = (
-        ThinkingMode.FAST if request.thinking_mode == "fast" else ThinkingMode.DEEP
-    )
+    thinking_mode_map = {
+        "fast": ThinkingMode.FAST,
+        "deep": ThinkingMode.DEEP,
+        "turbo": ThinkingMode.TURBO,
+    }
+    thinking_mode = thinking_mode_map.get(request.thinking_mode, ThinkingMode.DEEP)
     logger.info(f"初始化双模型Agent: {device_id}, 模式: {thinking_mode.value}")
 
     # 检查设备是否已有单模型Agent初始化

AutoGLM_GUI/api/layered_agent.py

@@ -0,0 +1,586 @@
+"""Layered agent API for hierarchical task execution.
+
+This module provides the layered agent API endpoint that uses
+a decision model for planning and autoglm-phone for execution.
+"""
+
+import asyncio
+import json
+from typing import Any
+
+from agents import Agent, Runner, SQLiteSession, function_tool
+from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
+from fastapi import APIRouter
+from fastapi.responses import StreamingResponse
+from openai import AsyncOpenAI
+from pydantic import BaseModel
+
+from AutoGLM_GUI.config_manager import config_manager
+from AutoGLM_GUI.logger import logger
+
+router = APIRouter()
+
+# ==================== Session 管理 ====================
+# 存储每个 session_id 对应的 SQLiteSession（内存模式）
+_sessions: dict[str, SQLiteSession] = {}
+
+
+def _get_or_create_session(session_id: str) -> SQLiteSession:
+    """获取或创建指定 session_id 的内存 session."""
+    if session_id not in _sessions:
+        # 使用 session_id 作为会话名称创建 session
+        _sessions[session_id] = SQLiteSession(session_id)
+        logger.info(f"[LayeredAgent] Created new session: {session_id}")
+    return _sessions[session_id]
+
+
+def _clear_session(session_id: str) -> bool:
+    """清除指定 session_id 的 session."""
+    if session_id in _sessions:
+        del _sessions[session_id]
+        logger.info(f"[LayeredAgent] Cleared session: {session_id}")
+        return True
+    return False
+
+
+def get_planner_model() -> str:
+    """获取规划层使用的模型名称，从配置读取."""
+    config = config_manager.get_effective_config()
+    return config.decision_model_name or "glm-4.7"
+
+
+PLANNER_INSTRUCTIONS = """## 核心目标
+你是一个负责操控手机的高级智能中枢。你的任务是将用户的意图转化为**视觉模型（Vision Model）**可以执行的原子操作。
+
+## ⚠️ 极其重要的限制：视觉模型的能力边界 (Must Read)
+你的下级（Vision Model）是一个**纯粹的执行者和观察者**。
+1. **无"记忆/笔记"功能**：它没有 `Note` 功能，无法为你保存数据。
+2. **无"系统级"权限**：它不能复制源代码，不能直接提取文本，不能读取剪贴板。
+3. **唯一的输出**：它只能通过**对话**告诉你它看到了什么，或者去**点击/滑动**屏幕。
+
+## 交互策略 (Interaction Strategy)
+
+### 1. 如果你需要"操作手机" (To Act)
+下达明确的 UI 动作指令。
+- ✅ "点击'设置'图标。"
+- ✅ "向下滑动屏幕。"
+- ✅ "打开微信。"
+
+### 2. 如果你需要"获取信息" (To Read/Extract)
+你必须通过**提问**的方式，让视觉模型在对话中把信息"念"给你听。
+- ❌ **错误**: "把验证码保存下来。" (它做不到)
+- ❌ **错误**: "使用 Note 功能记录价格。" (它没有这个功能)
+- ✅ **正确**: 调用 `chat` 询问："请看屏幕，告诉我现在的订单总金额是多少？"
+  - *结果*: 视觉模型会回复 "25.5元"。你需要自己处理这个文本信息。
+
+### 3. 如果用户要求"复制/粘贴"
+必须通过模拟手指操作来实现，不能直接操作剪贴板。
+- ✅ **正确**: "长按这段文字，等待弹出菜单，然后点击'复制'按钮。"
+
+## 任务拆解原则 (Decomposition Rules)
+
+1. **原子化**: 每次只给一个动作。
+2. **可视化**: 指令必须基于屏幕上**看得见**的元素。不要说"点击确认"，如果屏幕上显示的按钮叫"OK"，请说"点击'OK'按钮"。
+3. **Fail Fast**: 如果视觉模型回复 `ELEMENT_NOT_FOUND`，不要死循环。询问它："那现在屏幕上有什么？"或者尝试滑动寻找。
+
+## 核心工作流 (The Loop)
+1. **Observe (看)**: 调用 `chat` 询问当前状态。
+   - "现在屏幕上显示什么？" / "刚才的点击生效了吗？"
+2. **Think (想)**:
+   - 用户的目标是什么？
+   - 我需要让视觉模型**做什么动作**，还是**回答什么问题**？
+3. **Act (做)**:
+   - **Case A (动作)**: 发送指令 `点击[坐标]...`
+   - **Case B (询问)**: 发送问题 `请读取...`
+
+## 内部思维链示例 (Inner Monologue)
+
+**场景 1: 用户让你"把这篇笔记的标题发给我"**
+> **Current State**: 笔记详情页。
+> **Goal**: 获取标题文本。
+> **Constraint**: 视觉模型无法直接提取变量，我必须问它。
+> **Strategy**: 问视觉模型标题是什么，它回答后，我再反馈给用户。
+> **Next Action**: 提问。
+**Output**: `chat(id, "请读取并告诉我屏幕上这篇笔记的标题文字内容是什么？")`
+
+**场景 2: 用户让你"复制链接"**
+> **Current State**: 详情页。
+> **Goal**: 把链接复制到系统剪贴板。
+> **Constraint**: 不能直接 Get Link。必须找"分享"或"复制"按钮。
+> **Strategy**: 先点右上角菜单，再找复制链接。
+> **Next Action**: 点击菜单。
+**Output**: `chat(id, "点击屏幕右上角的'...'（三个点）菜单按钮。")`
+
+## 工具集 (Tools)
+1. `list_devices()`
+2. `chat(device_id, message)`: 
+   - 发送操作指令（如"点击红色按钮"）。
+   - 发送查询问题（如"那个验证码是多少？"）。
+
+"""
+
+
+# ==================== 工具定义 ====================
+
+
+def _sync_list_devices() -> str:
+    """同步实现：获取所有连接的 ADB 设备列表。"""
+    from AutoGLM_GUI.api.devices import _build_device_response_with_agent
+    from AutoGLM_GUI.device_manager import DeviceManager
+    from AutoGLM_GUI.phone_agent_manager import PhoneAgentManager
+
+    logger.info("[LayeredAgent] list_devices tool called")
+
+    device_manager = DeviceManager.get_instance()
+    agent_manager = PhoneAgentManager.get_instance()
+
+    # 如果轮询未启动，执行同步刷新
+    if not device_manager._poll_thread or not device_manager._poll_thread.is_alive():
+        logger.warning("Polling not started, performing sync refresh")
+        device_manager.force_refresh()
+
+    managed_devices = device_manager.get_devices()
+
+    # 构建设备响应
+    devices_with_agents = [
+        _build_device_response_with_agent(d, agent_manager) for d in managed_devices
+    ]
+
+    return json.dumps(devices_with_agents, ensure_ascii=False, indent=2)
+
+
+@function_tool
+async def list_devices() -> str:
+    """
+    获取所有连接的 ADB 设备列表。
+
+    返回设备信息包括:
+    - id: 设备标识符，用于 chat 工具调用
+    - model: 设备型号
+    - status: 连接状态
+    - connection_type: 连接类型 (usb/wifi/remote)
+
+    Returns:
+        JSON 格式的设备列表
+    """
+    return await asyncio.to_thread(_sync_list_devices)
+
+
+def _sync_chat(device_id: str, message: str) -> str:
+    """同步实现：向指定设备的 Phone Agent 发送子任务指令。"""
+    from AutoGLM_GUI.exceptions import DeviceBusyError
+    from AutoGLM_GUI.phone_agent_manager import PhoneAgentManager
+    from AutoGLM_GUI.prompts import MCP_SYSTEM_PROMPT_ZH
+
+    MCP_MAX_STEPS = 5
+
+    logger.info(
+        f"[LayeredAgent] chat tool called: device_id={device_id}, message={message}"
+    )
+
+    manager = PhoneAgentManager.get_instance()
+
+    try:
+        # use_agent 现在会自动初始化 agent（auto_initialize=True）
+        with manager.use_agent(device_id, timeout=None) as agent:
+            # 临时覆盖配置
+            original_max_steps = agent.agent_config.max_steps
+            original_system_prompt = agent.agent_config.system_prompt
+
+            agent.agent_config.max_steps = MCP_MAX_STEPS
+            agent.agent_config.system_prompt = MCP_SYSTEM_PROMPT_ZH
+
+            try:
+                # 重置 agent 确保干净状态
+                agent.reset()
+
+                result = agent.run(message)
+                steps = agent.step_count
+
+                # 检查是否达到步数限制
+                if steps >= MCP_MAX_STEPS and result == "Max steps reached":
+                    context_json = json.dumps(
+                        agent.context, ensure_ascii=False, indent=2
+                    )
+                    return json.dumps(
+                        {
+                            "result": f"⚠️ 已达到最大步数限制（{MCP_MAX_STEPS}步）。视觉模型可能遇到了困难，任务未完成。\n\n执行历史:\n{context_json}\n\n建议: 请重新规划任务或将其拆分为更小的子任务。",
+                            "steps": MCP_MAX_STEPS,
+                            "success": False,
+                        },
+                        ensure_ascii=False,
+                    )
+
+                return json.dumps(
+                    {
+                        "result": result,
+                        "steps": steps,
+                        "success": True,
+                    },
+                    ensure_ascii=False,
+                )
+
+            finally:
+                # 恢复原始配置
+                agent.agent_config.max_steps = original_max_steps
+                agent.agent_config.system_prompt = original_system_prompt
+
+    except DeviceBusyError:
+        return json.dumps(
+            {
+                "result": f"设备 {device_id} 正忙，请稍后再试。",
+                "steps": 0,
+                "success": False,
+            },
+            ensure_ascii=False,
+        )
+    except Exception as e:
+        logger.error(f"[LayeredAgent] chat tool error: {e}")
+        return json.dumps(
+            {
+                "result": str(e),
+                "steps": 0,
+                "success": False,
+            },
+            ensure_ascii=False,
+        )
+
+
+@function_tool
+async def chat(device_id: str, message: str) -> str:
+    """
+    向指定设备的 Phone Agent 发送子任务指令。
+
+    Phone Agent 是一个视觉模型，能够看到手机屏幕并执行操作。
+    每次调用会执行一个原子化的子任务（最多 5 步操作）。
+
+    Args:
+        device_id: 设备标识符，从 list_devices 获取
+        message: 子任务指令，例如 "打开微信"、"点击搜索按钮"
+
+    Returns:
+        JSON 格式的执行结果，包含:
+        - result: 执行结果描述
+        - steps: 执行的步数
+        - success: 是否成功
+    """
+    return await asyncio.to_thread(_sync_chat, device_id, message)
+
+
+# ==================== Agent 初始化 ====================
+
+
+def _setup_openai_client() -> AsyncOpenAI:
+    """设置 OpenAI 客户端，使用 AutoGLM 的配置"""
+    config_manager.load_file_config()
+    effective_config = config_manager.get_effective_config()
+
+    if not effective_config.base_url:
+        raise ValueError("base_url not configured")
+
+    planner_model = get_planner_model()
+    logger.info(f"[LayeredAgent] API Base URL: {effective_config.base_url}")
+    logger.info(f"[LayeredAgent] Planner Model: {planner_model}")
+
+    return AsyncOpenAI(
+        base_url=effective_config.base_url,
+        api_key=effective_config.api_key,
+    )
+
+
+def _create_planner_agent(client: AsyncOpenAI) -> Agent[Any]:
+    """创建规划 Agent，使用 Chat Completions API"""
+    planner_model = get_planner_model()
+    model = OpenAIChatCompletionsModel(
+        model=planner_model,
+        openai_client=client,
+    )
+
+    return Agent(
+        name="Planner",
+        instructions=PLANNER_INSTRUCTIONS,
+        model=model,
+        tools=[list_devices, chat],
+    )
+
+
+# Global agent instance (lazy initialized)
+_client: AsyncOpenAI | None = None
+_agent: Agent[Any] | None = None
+
+
+def _ensure_agent() -> Agent[Any]:
+    """Ensure the planner agent is initialized."""
+    global _client, _agent
+    if _agent is None:
+        _client = _setup_openai_client()
+        _agent = _create_planner_agent(_client)
+    return _agent
+
+
+# ==================== API 路由 ====================
+
+
+class LayeredAgentRequest(BaseModel):
+    """Request for layered agent chat."""
+
+    message: str
+    device_id: str | None = None
+    session_id: str | None = None  # 用于保持对话上下文，前端可传入 deviceId
+
+
+@router.post("/api/layered-agent/chat")
+async def layered_agent_chat(request: LayeredAgentRequest):
+    """
+    Layered agent chat API with streaming execution steps.
+
+    Uses a decision model for planning and autoglm-phone for execution.
+
+    Returns SSE stream with events:
+    - tool_call: Agent is calling a tool (with tool_name and tool_args)
+    - tool_result: Tool execution result
+    - message: Intermediate message from agent
+    - done: Final response
+    - error: Error occurred
+    """
+    from agents.stream_events import (
+        RawResponsesStreamEvent,
+        RunItemStreamEvent,
+    )
+
+    async def event_generator():
+        try:
+            # Ensure agent is initialized
+            agent = _ensure_agent()
+
+            # 获取或创建 session 以保持对话上下文
+            # 优先使用 session_id，其次使用 device_id，最后使用默认值
+            session_id = request.session_id or request.device_id or "default"
+            session = _get_or_create_session(session_id)
+
+            # Run the agent with streaming and session for memory
+            result = Runner.run_streamed(
+                agent,
+                request.message,
+                max_turns=50,
+                session=session,
+            )
+
+            current_tool_call: dict[str, Any] | None = None
+
+            async for event in result.stream_events():
+                if isinstance(event, RawResponsesStreamEvent):
+                    # Raw response chunk - could contain thinking
+                    pass
+
+                elif isinstance(event, RunItemStreamEvent):
+                    item = event.item
+
+                    # Handle different item types
+                    item_type = getattr(item, "type", None)
+
+                    if item_type == "tool_call_item":
+                        # Tool call started - extract name from raw_item
+                        tool_name = "unknown"
+                        tool_args: dict[str, Any] = {}
+
+                        # Try to get from raw_item
+                        if hasattr(item, "raw_item") and item.raw_item:
+                            raw = item.raw_item
+
+                            # Handle dict format (sometimes returned as dict)
+                            if isinstance(raw, dict):
+                                tool_name = raw.get(
+                                    "name",
+                                    raw.get("function", {}).get("name", "unknown"),
+                                )
+                                args_str = raw.get(
+                                    "arguments",
+                                    raw.get("function", {}).get("arguments", "{}"),
+                                )
+                                try:
+                                    tool_args = (
+                                        json.loads(args_str)
+                                        if isinstance(args_str, str)
+                                        else args_str
+                                    )
+                                except Exception:
+                                    tool_args = {"raw": str(args_str)}
+                            else:
+                                # Chat Completions API format: raw_item.function.name
+                                if hasattr(raw, "function") and raw.function:
+                                    func = raw.function
+                                    if hasattr(func, "name"):
+                                        tool_name = func.name
+                                    if hasattr(func, "arguments"):
+                                        try:
+                                            tool_args = (
+                                                json.loads(func.arguments)
+                                                if isinstance(func.arguments, str)
+                                                else func.arguments
+                                            )
+                                        except Exception:
+                                            tool_args = {"raw": str(func.arguments)}
+                                # Responses API format: raw_item.name directly
+                                elif hasattr(raw, "name") and raw.name:
+                                    tool_name = raw.name
+                                    if hasattr(raw, "arguments"):
+                                        try:
+                                            tool_args = (
+                                                json.loads(raw.arguments)
+                                                if isinstance(raw.arguments, str)
+                                                else raw.arguments
+                                            )
+                                        except Exception:
+                                            tool_args = {"raw": str(raw.arguments)}
+
+                        # Fallback to direct item attributes
+                        if tool_name == "unknown":
+                            if hasattr(item, "name") and item.name:
+                                tool_name = item.name
+                            elif hasattr(item, "call") and item.call:
+                                call = item.call
+                                if hasattr(call, "function") and call.function:
+                                    if hasattr(call.function, "name"):
+                                        tool_name = call.function.name
+                                    if hasattr(call.function, "arguments"):
+                                        try:
+                                            tool_args = (
+                                                json.loads(call.function.arguments)
+                                                if isinstance(
+                                                    call.function.arguments, str
+                                                )
+                                                else call.function.arguments
+                                            )
+                                        except Exception:
+                                            tool_args = {
+                                                "raw": str(call.function.arguments)
+                                            }
+                                elif hasattr(call, "name"):
+                                    tool_name = call.name
+                                    if hasattr(call, "arguments"):
+                                        try:
+                                            tool_args = (
+                                                json.loads(call.arguments)
+                                                if isinstance(call.arguments, str)
+                                                else call.arguments
+                                            )
+                                        except Exception:
+                                            tool_args = {"raw": str(call.arguments)}
+
+                        logger.info(
+                            f"[LayeredAgent] Tool call: {tool_name}, args keys: {list(tool_args.keys()) if isinstance(tool_args, dict) else 'not dict'}"
+                        )
+
+                        current_tool_call = {
+                            "name": tool_name,
+                            "args": tool_args,
+                        }
+
+                        event_data = {
+                            "type": "tool_call",
+                            "tool_name": tool_name,
+                            "tool_args": tool_args,
+                        }
+                        yield f"data: {json.dumps(event_data, ensure_ascii=False)}\n\n"
+
+                    elif item_type == "tool_call_output_item":
+                        # Tool call result
+                        output = getattr(item, "output", "")
+
+                        # Get tool name from current_tool_call or try to extract from item
+                        tool_name = (
+                            current_tool_call["name"]
+                            if current_tool_call
+                            else "unknown"
+                        )
+
+                        # Try to get tool name from raw_item if available
+                        if (
+                            tool_name == "unknown"
+                            and hasattr(item, "raw_item")
+                            and item.raw_item
+                        ):
+                            if hasattr(item.raw_item, "name"):
+                                tool_name = item.raw_item.name
+
+                        logger.info(
+                            f"[LayeredAgent] Tool result for {tool_name}: {str(output)[:100] if output else 'empty'}..."
+                        )
+
+                        event_data = {
+                            "type": "tool_result",
+                            "tool_name": tool_name,
+                            "result": output,
+                        }
+                        yield f"data: {json.dumps(event_data, ensure_ascii=False)}\n\n"
+                        current_tool_call = None
+
+                    elif item_type == "message_output_item":
+                        # Final message
+                        content = ""
+                        if hasattr(item, "raw_item") and item.raw_item:
+                            if (
+                                hasattr(item.raw_item, "content")
+                                and item.raw_item.content
+                            ):
+                                for c in item.raw_item.content:
+                                    if hasattr(c, "text"):
+                                        content += c.text
+
+                        if content:
+                            event_data = {
+                                "type": "message",
+                                "content": content,
+                            }
+                            yield f"data: {json.dumps(event_data, ensure_ascii=False)}\n\n"
+
+            # Final result
+            final_output = (
+                result.final_output if hasattr(result, "final_output") else ""
+            )
+            event_data = {
+                "type": "done",
+                "content": final_output,
+                "success": True,
+            }
+            yield f"data: {json.dumps(event_data, ensure_ascii=False)}\n\n"
+
+        except Exception as e:
+            logger.exception(f"[LayeredAgent] Error: {e}")
+            event_data = {
+                "type": "error",
+                "message": str(e),
+            }
+            yield f"data: {json.dumps(event_data, ensure_ascii=False)}\n\n"
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",
+        },
+    )
+
+
+class ResetSessionRequest(BaseModel):
+    """Request for resetting a session."""
+
+    session_id: str
+
+
+@router.post("/api/layered-agent/reset")
+def reset_session(request: ResetSessionRequest):
+    """
+    Reset/clear a session to forget conversation history.
+
+    This should be called when the user clicks "reset" button
+    or refreshes the page.
+    """
+    cleared = _clear_session(request.session_id)
+    return {
+        "success": True,
+        "message": f"Session {request.session_id} {'cleared' if cleared else 'not found (already empty)'}",
+    }