autoglm-gui 0.3.2__py3-none-any.whl → 0.4.1__py3-none-any.whl

AutoGLM_GUI/__main__.py

@@ -12,7 +12,9 @@ import webbrowser
 DEFAULT_MODEL_NAME = "autoglm-phone-9b"
 
 
-def find_available_port(start_port: int = 8000, max_attempts: int = 100, host: str = "127.0.0.1") -> int:
+def find_available_port(
+    start_port: int = 8000, max_attempts: int = 100, host: str = "127.0.0.1"
+) -> int:
     """Find an available port starting from start_port.
 
     Args:
@@ -52,7 +54,9 @@ def open_browser(host: str, port: int, delay: float = 1.5) -> None:
 
     def _open():
         time.sleep(delay)
-        url = f"http://127.0.0.1:{port}" if host == "0.0.0.0" else f"http://{host}:{port}"
+        url = (
+            f"http://127.0.0.1:{port}" if host == "0.0.0.0" else f"http://{host}:{port}"
+        )
         try:
             webbrowser.open(url)
         except Exception as e:

AutoGLM_GUI/adb_plus/screenshot.py

@@ -10,7 +10,6 @@ import base64
 import subprocess
 from dataclasses import dataclass
 from io import BytesIO
-from typing import Iterable
 
 from PIL import Image
 
@@ -72,9 +71,7 @@ def capture_screenshot(
     return _fallback_screenshot()
 
 
-def _try_capture(
-    device_id: str | None, adb_path: str, timeout: int
-) -> bytes | None:
+def _try_capture(device_id: str | None, adb_path: str, timeout: int) -> bytes | None:
     """Run exec-out screencap and return raw bytes or None on failure."""
     cmd: list[str | bytes] = [adb_path]
     if device_id:

AutoGLM_GUI/api/__init__.py

@@ -0,0 +1,66 @@
+"""FastAPI application factory and route registration."""
+
+from importlib.resources import files
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+
+from AutoGLM_GUI.version import APP_VERSION
+
+from . import agents, control, devices, media
+
+
+def _get_static_dir() -> Path | None:
+    """Locate packaged static assets."""
+    try:
+        static_dir = files("AutoGLM_GUI").joinpath("static")
+        if hasattr(static_dir, "_path"):
+            path = Path(str(static_dir))
+            if path.exists():
+                return path
+        path = Path(str(static_dir))
+        if path.exists():
+            return path
+    except (TypeError, FileNotFoundError):
+        pass
+
+    return None
+
+
+def create_app() -> FastAPI:
+    """Build the FastAPI app with routers and static assets."""
+    app = FastAPI(title="AutoGLM-GUI API", version=APP_VERSION)
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["http://localhost:3000"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    app.include_router(agents.router)
+    app.include_router(devices.router)
+    app.include_router(control.router)
+    app.include_router(media.router)
+
+    static_dir = _get_static_dir()
+    if static_dir is not None and static_dir.exists():
+        assets_dir = static_dir / "assets"
+        if assets_dir.exists():
+            app.mount("/assets", StaticFiles(directory=assets_dir), name="assets")
+
+        @app.get("/{full_path:path}")
+        async def serve_spa(full_path: str) -> FileResponse:
+            file_path = static_dir / full_path
+            if file_path.is_file():
+                return FileResponse(file_path)
+            return FileResponse(static_dir / "index.html")
+
+    return app
+
+
+app = create_app()

AutoGLM_GUI/api/agents.py

@@ -0,0 +1,231 @@
+"""Agent lifecycle and chat routes."""
+
+import json
+
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import StreamingResponse
+from phone_agent import PhoneAgent
+from phone_agent.agent import AgentConfig
+from phone_agent.model import ModelConfig
+
+from AutoGLM_GUI.schemas import (
+    APIAgentConfig,
+    APIModelConfig,
+    ChatRequest,
+    ChatResponse,
+    InitRequest,
+    ResetRequest,
+    StatusResponse,
+)
+from AutoGLM_GUI.state import (
+    DEFAULT_API_KEY,
+    DEFAULT_BASE_URL,
+    DEFAULT_MODEL_NAME,
+    agent_configs,
+    agents,
+    non_blocking_takeover,
+)
+from AutoGLM_GUI.version import APP_VERSION
+
+router = APIRouter()
+
+
+@router.post("/api/init")
+def init_agent(request: InitRequest) -> dict:
+    """初始化 PhoneAgent（多设备支持）。"""
+    req_model_config = request.model or APIModelConfig()
+    req_agent_config = request.agent or APIAgentConfig()
+
+    device_id = req_agent_config.device_id
+    if not device_id:
+        raise HTTPException(
+            status_code=400, detail="device_id is required in agent_config"
+        )
+
+    base_url = req_model_config.base_url or DEFAULT_BASE_URL
+    api_key = req_model_config.api_key or DEFAULT_API_KEY
+    model_name = req_model_config.model_name or DEFAULT_MODEL_NAME
+
+    if not base_url:
+        raise HTTPException(
+            status_code=400, detail="base_url is required (in model_config or env)"
+        )
+
+    model_config = ModelConfig(
+        base_url=base_url,
+        api_key=api_key,
+        model_name=model_name,
+        max_tokens=req_model_config.max_tokens,
+        temperature=req_model_config.temperature,
+        top_p=req_model_config.top_p,
+        frequency_penalty=req_model_config.frequency_penalty,
+    )
+
+    agent_config = AgentConfig(
+        max_steps=req_agent_config.max_steps,
+        device_id=device_id,
+        lang=req_agent_config.lang,
+        system_prompt=req_agent_config.system_prompt,
+        verbose=req_agent_config.verbose,
+    )
+
+    agents[device_id] = PhoneAgent(
+        model_config=model_config,
+        agent_config=agent_config,
+        takeover_callback=non_blocking_takeover,
+    )
+
+    agent_configs[device_id] = (model_config, agent_config)
+
+    return {
+        "success": True,
+        "device_id": device_id,
+        "message": f"Agent initialized for device {device_id}",
+    }
+
+
+@router.post("/api/chat", response_model=ChatResponse)
+def chat(request: ChatRequest) -> ChatResponse:
+    """发送任务给 Agent 并执行。"""
+    device_id = request.device_id
+    if device_id not in agents:
+        raise HTTPException(
+            status_code=400, detail="Agent not initialized. Call /api/init first."
+        )
+
+    agent = agents[device_id]
+
+    try:
+        result = agent.run(request.message)
+        steps = agent.step_count
+        agent.reset()
+
+        return ChatResponse(result=result, steps=steps, success=True)
+    except Exception as e:
+        return ChatResponse(result=str(e), steps=0, success=False)
+
+
+@router.post("/api/chat/stream")
+def chat_stream(request: ChatRequest):
+    """发送任务给 Agent 并实时推送执行进度（SSE，多设备支持）。"""
+    device_id = request.device_id
+
+    if device_id not in agents:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Device {device_id} not initialized. Call /api/init first.",
+        )
+
+    agent = agents[device_id]
+
+    def event_generator():
+        """SSE 事件生成器"""
+        try:
+            step_result = agent.step(request.message)
+            while True:
+                event_data = {
+                    "type": "step",
+                    "step": agent.step_count,
+                    "thinking": step_result.thinking,
+                    "action": step_result.action,
+                    "success": step_result.success,
+                    "finished": step_result.finished,
+                }
+
+                yield "event: step\n"
+                yield f"data: {json.dumps(event_data, ensure_ascii=False)}\n\n"
+
+                if step_result.finished:
+                    done_data = {
+                        "type": "done",
+                        "message": step_result.message,
+                        "steps": agent.step_count,
+                        "success": step_result.success,
+                    }
+                    yield "event: done\n"
+                    yield f"data: {json.dumps(done_data, ensure_ascii=False)}\n\n"
+                    break
+
+                if agent.step_count >= agent.agent_config.max_steps:
+                    done_data = {
+                        "type": "done",
+                        "message": "Max steps reached",
+                        "steps": agent.step_count,
+                        "success": step_result.success,
+                    }
+                    yield "event: done\n"
+                    yield f"data: {json.dumps(done_data, ensure_ascii=False)}\n\n"
+                    break
+
+                step_result = agent.step()
+
+            agent.reset()
+
+        except Exception as e:
+            error_data = {
+                "type": "error",
+                "message": str(e),
+            }
+            yield "event: error\n"
+            yield f"data: {json.dumps(error_data, ensure_ascii=False)}\n\n"
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",
+        },
+    )
+
+
+@router.get("/api/status", response_model=StatusResponse)
+def get_status(device_id: str | None = None) -> StatusResponse:
+    """获取 Agent 状态和版本信息（多设备支持）。"""
+    if device_id is None:
+        return StatusResponse(
+            version=APP_VERSION,
+            initialized=len(agents) > 0,
+            step_count=0,
+        )
+
+    if device_id not in agents:
+        return StatusResponse(
+            version=APP_VERSION,
+            initialized=False,
+            step_count=0,
+        )
+
+    agent = agents[device_id]
+    return StatusResponse(
+        version=APP_VERSION,
+        initialized=True,
+        step_count=agent.step_count,
+    )
+
+
+@router.post("/api/reset")
+def reset_agent(request: ResetRequest) -> dict:
+    """重置 Agent 状态（多设备支持）。"""
+    device_id = request.device_id
+
+    if device_id not in agents:
+        raise HTTPException(status_code=404, detail=f"Device {device_id} not found")
+
+    agent = agents[device_id]
+    agent.reset()
+
+    if device_id in agent_configs:
+        model_config, agent_config = agent_configs[device_id]
+        agents[device_id] = PhoneAgent(
+            model_config=model_config,
+            agent_config=agent_config,
+            takeover_callback=non_blocking_takeover,
+        )
+
+    return {
+        "success": True,
+        "device_id": device_id,
+        "message": f"Agent reset for device {device_id}",
+    }

AutoGLM_GUI/api/control.py

@@ -0,0 +1,111 @@
+"""Device control routes (tap/swipe/touch)."""
+
+from fastapi import APIRouter
+
+from AutoGLM_GUI.schemas import (
+    SwipeRequest,
+    SwipeResponse,
+    TapRequest,
+    TapResponse,
+    TouchDownRequest,
+    TouchDownResponse,
+    TouchMoveRequest,
+    TouchMoveResponse,
+    TouchUpRequest,
+    TouchUpResponse,
+)
+
+router = APIRouter()
+
+
+@router.post("/api/control/tap", response_model=TapResponse)
+def control_tap(request: TapRequest) -> TapResponse:
+    """Execute tap at specified device coordinates."""
+    try:
+        from phone_agent.adb import tap
+
+        tap(
+            x=request.x,
+            y=request.y,
+            device_id=request.device_id,
+            delay=request.delay,
+        )
+
+        return TapResponse(success=True)
+    except Exception as e:
+        return TapResponse(success=False, error=str(e))
+
+
+@router.post("/api/control/swipe", response_model=SwipeResponse)
+def control_swipe(request: SwipeRequest) -> SwipeResponse:
+    """Execute swipe from start to end coordinates."""
+    try:
+        from phone_agent.adb import swipe
+
+        swipe(
+            start_x=request.start_x,
+            start_y=request.start_y,
+            end_x=request.end_x,
+            end_y=request.end_y,
+            duration_ms=request.duration_ms,
+            device_id=request.device_id,
+            delay=request.delay,
+        )
+
+        return SwipeResponse(success=True)
+    except Exception as e:
+        return SwipeResponse(success=False, error=str(e))
+
+
+@router.post("/api/control/touch/down", response_model=TouchDownResponse)
+def control_touch_down(request: TouchDownRequest) -> TouchDownResponse:
+    """Send touch DOWN event at specified device coordinates."""
+    try:
+        from AutoGLM_GUI.adb_plus import touch_down
+
+        touch_down(
+            x=request.x,
+            y=request.y,
+            device_id=request.device_id,
+            delay=request.delay,
+        )
+
+        return TouchDownResponse(success=True)
+    except Exception as e:
+        return TouchDownResponse(success=False, error=str(e))
+
+
+@router.post("/api/control/touch/move", response_model=TouchMoveResponse)
+def control_touch_move(request: TouchMoveRequest) -> TouchMoveResponse:
+    """Send touch MOVE event at specified device coordinates."""
+    try:
+        from AutoGLM_GUI.adb_plus import touch_move
+
+        touch_move(
+            x=request.x,
+            y=request.y,
+            device_id=request.device_id,
+            delay=request.delay,
+        )
+
+        return TouchMoveResponse(success=True)
+    except Exception as e:
+        return TouchMoveResponse(success=False, error=str(e))
+
+
+@router.post("/api/control/touch/up", response_model=TouchUpResponse)
+def control_touch_up(request: TouchUpRequest) -> TouchUpResponse:
+    """Send touch UP event at specified device coordinates."""
+    try:
+        from AutoGLM_GUI.adb_plus import touch_up
+
+        touch_up(
+            x=request.x,
+            y=request.y,
+            device_id=request.device_id,
+            delay=request.delay,
+        )
+
+        return TouchUpResponse(success=True)
+    except Exception as e:
+        return TouchUpResponse(success=False, error=str(e))

AutoGLM_GUI/api/devices.py

@@ -0,0 +1,29 @@
+"""Device discovery routes."""
+
+from fastapi import APIRouter
+
+from AutoGLM_GUI.schemas import DeviceListResponse
+from AutoGLM_GUI.state import agents
+
+router = APIRouter()
+
+
+@router.get("/api/devices", response_model=DeviceListResponse)
+def list_devices() -> DeviceListResponse:
+    """列出所有 ADB 设备。"""
+    from phone_agent.adb import list_devices as adb_list
+
+    adb_devices = adb_list()
+
+    return DeviceListResponse(
+        devices=[
+            {
+                "id": d.device_id,
+                "model": d.model or "Unknown",
+                "status": d.status,
+                "connection_type": d.connection_type.value,
+                "is_initialized": d.device_id in agents,
+            }
+            for d in adb_devices
+        ]
+    )

AutoGLM_GUI/api/media.py

@@ -0,0 +1,163 @@
+"""Media routes: screenshot, video stream, stream reset."""
+
+import asyncio
+
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect
+
+from AutoGLM_GUI.adb_plus import capture_screenshot
+from AutoGLM_GUI.schemas import ScreenshotRequest, ScreenshotResponse
+from AutoGLM_GUI.scrcpy_stream import ScrcpyStreamer
+from AutoGLM_GUI.state import scrcpy_locks, scrcpy_streamers
+
+router = APIRouter()
+
+
+@router.post("/api/video/reset")
+async def reset_video_stream(device_id: str | None = None) -> dict:
+    """Reset video stream (cleanup scrcpy server，多设备支持)."""
+    if device_id:
+        if device_id in scrcpy_locks:
+            async with scrcpy_locks[device_id]:
+                if device_id in scrcpy_streamers:
+                    print(f"[video/reset] Stopping streamer for device {device_id}")
+                    scrcpy_streamers[device_id].stop()
+                    del scrcpy_streamers[device_id]
+                    print(f"[video/reset] Streamer reset for device {device_id}")
+                    return {
+                        "success": True,
+                        "message": f"Video stream reset for device {device_id}",
+                    }
+                return {
+                    "success": True,
+                    "message": f"No active video stream for device {device_id}",
+                }
+        return {"success": True, "message": f"No video stream for device {device_id}"}
+
+    device_ids = list(scrcpy_streamers.keys())
+    for dev_id in device_ids:
+        if dev_id in scrcpy_locks:
+            async with scrcpy_locks[dev_id]:
+                if dev_id in scrcpy_streamers:
+                    scrcpy_streamers[dev_id].stop()
+                    del scrcpy_streamers[dev_id]
+    print("[video/reset] All streamers reset")
+    return {"success": True, "message": "All video streams reset"}
+
+
+@router.post("/api/screenshot", response_model=ScreenshotResponse)
+def take_screenshot(request: ScreenshotRequest) -> ScreenshotResponse:
+    """获取设备截图。此操作无副作用，不影响 PhoneAgent 运行。"""
+    try:
+        screenshot = capture_screenshot(device_id=request.device_id)
+        return ScreenshotResponse(
+            success=True,
+            image=screenshot.base64_data,
+            width=screenshot.width,
+            height=screenshot.height,
+            is_sensitive=screenshot.is_sensitive,
+        )
+    except Exception as e:
+        return ScreenshotResponse(
+            success=False,
+            image="",
+            width=0,
+            height=0,
+            is_sensitive=False,
+            error=str(e),
+        )
+
+
+@router.websocket("/api/video/stream")
+async def video_stream_ws(websocket: WebSocket, device_id: str | None = None):
+    """Stream real-time H.264 video from scrcpy server via WebSocket（多设备支持）."""
+    await websocket.accept()
+
+    if not device_id:
+        await websocket.send_json({"error": "device_id is required"})
+        return
+
+    print(f"[video/stream] WebSocket connection for device {device_id}")
+
+    if device_id not in scrcpy_locks:
+        scrcpy_locks[device_id] = asyncio.Lock()
+
+    async with scrcpy_locks[device_id]:
+        if device_id not in scrcpy_streamers:
+            print(f"[video/stream] Creating streamer for device {device_id}")
+            scrcpy_streamers[device_id] = ScrcpyStreamer(
+                device_id=device_id, max_size=1280, bit_rate=4_000_000
+            )
+
+            try:
+                print(f"[video/stream] Starting scrcpy server for device {device_id}")
+                await scrcpy_streamers[device_id].start()
+                print(f"[video/stream] Scrcpy server started for device {device_id}")
+            except Exception as e:
+                import traceback
+
+                print(f"[video/stream] Failed to start streamer: {e}")
+                print(f"[video/stream] Traceback:\n{traceback.format_exc()}")
+                scrcpy_streamers[device_id].stop()
+                del scrcpy_streamers[device_id]
+                try:
+                    await websocket.send_json({"error": str(e)})
+                except Exception:
+                    pass
+                return
+        else:
+            print(f"[video/stream] Reusing streamer for device {device_id}")
+
+            streamer = scrcpy_streamers[device_id]
+            if streamer.cached_sps and streamer.cached_pps:
+                init_data = streamer.cached_sps + streamer.cached_pps
+                await websocket.send_bytes(init_data)
+                print(f"[video/stream] Sent SPS/PPS for device {device_id}")
+            else:
+                print(
+                    f"[video/stream] Warning: No cached SPS/PPS for device {device_id}"
+                )
+
+    streamer = scrcpy_streamers[device_id]
+
+    stream_failed = False
+    try:
+        chunk_count = 0
+        while True:
+            try:
+                h264_chunk = await streamer.read_h264_chunk()
+                await websocket.send_bytes(h264_chunk)
+                chunk_count += 1
+                if chunk_count % 100 == 0:
+                    print(
+                        f"[video/stream] Device {device_id}: Sent {chunk_count} chunks"
+                    )
+            except ConnectionError as e:
+                print(f"[video/stream] Device {device_id}: Connection error: {e}")
+                stream_failed = True
+                try:
+                    await websocket.send_json({"error": f"Stream error: {str(e)}"})
+                except Exception:
+                    pass
+                break
+
+    except WebSocketDisconnect:
+        print(f"[video/stream] Device {device_id}: Client disconnected")
+    except Exception as e:
+        import traceback
+
+        print(f"[video/stream] Device {device_id}: Error: {e}")
+        print(f"[video/stream] Traceback:\n{traceback.format_exc()}")
+        stream_failed = True
+        try:
+            await websocket.send_json({"error": str(e)})
+        except Exception:
+            pass
+
+    if stream_failed:
+        async with scrcpy_locks[device_id]:
+            if device_id in scrcpy_streamers:
+                print(f"[video/stream] Resetting streamer for device {device_id}")
+                scrcpy_streamers[device_id].stop()
+                del scrcpy_streamers[device_id]
+
+    print(f"[video/stream] Device {device_id}: Stream ended")