autoglm-gui 0.3.1__py3-none-any.whl → 0.4.1__py3-none-any.whl

AutoGLM_GUI/api/media.py

@@ -0,0 +1,163 @@
+"""Media routes: screenshot, video stream, stream reset."""
+
+import asyncio
+
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect
+
+from AutoGLM_GUI.adb_plus import capture_screenshot
+from AutoGLM_GUI.schemas import ScreenshotRequest, ScreenshotResponse
+from AutoGLM_GUI.scrcpy_stream import ScrcpyStreamer
+from AutoGLM_GUI.state import scrcpy_locks, scrcpy_streamers
+
+router = APIRouter()
+
+
+@router.post("/api/video/reset")
+async def reset_video_stream(device_id: str | None = None) -> dict:
+    """Reset video stream (cleanup scrcpy server，多设备支持)."""
+    if device_id:
+        if device_id in scrcpy_locks:
+            async with scrcpy_locks[device_id]:
+                if device_id in scrcpy_streamers:
+                    print(f"[video/reset] Stopping streamer for device {device_id}")
+                    scrcpy_streamers[device_id].stop()
+                    del scrcpy_streamers[device_id]
+                    print(f"[video/reset] Streamer reset for device {device_id}")
+                    return {
+                        "success": True,
+                        "message": f"Video stream reset for device {device_id}",
+                    }
+                return {
+                    "success": True,
+                    "message": f"No active video stream for device {device_id}",
+                }
+        return {"success": True, "message": f"No video stream for device {device_id}"}
+
+    device_ids = list(scrcpy_streamers.keys())
+    for dev_id in device_ids:
+        if dev_id in scrcpy_locks:
+            async with scrcpy_locks[dev_id]:
+                if dev_id in scrcpy_streamers:
+                    scrcpy_streamers[dev_id].stop()
+                    del scrcpy_streamers[dev_id]
+    print("[video/reset] All streamers reset")
+    return {"success": True, "message": "All video streams reset"}
+
+
+@router.post("/api/screenshot", response_model=ScreenshotResponse)
+def take_screenshot(request: ScreenshotRequest) -> ScreenshotResponse:
+    """获取设备截图。此操作无副作用，不影响 PhoneAgent 运行。"""
+    try:
+        screenshot = capture_screenshot(device_id=request.device_id)
+        return ScreenshotResponse(
+            success=True,
+            image=screenshot.base64_data,
+            width=screenshot.width,
+            height=screenshot.height,
+            is_sensitive=screenshot.is_sensitive,
+        )
+    except Exception as e:
+        return ScreenshotResponse(
+            success=False,
+            image="",
+            width=0,
+            height=0,
+            is_sensitive=False,
+            error=str(e),
+        )
+
+
+@router.websocket("/api/video/stream")
+async def video_stream_ws(websocket: WebSocket, device_id: str | None = None):
+    """Stream real-time H.264 video from scrcpy server via WebSocket（多设备支持）."""
+    await websocket.accept()
+
+    if not device_id:
+        await websocket.send_json({"error": "device_id is required"})
+        return
+
+    print(f"[video/stream] WebSocket connection for device {device_id}")
+
+    if device_id not in scrcpy_locks:
+        scrcpy_locks[device_id] = asyncio.Lock()
+
+    async with scrcpy_locks[device_id]:
+        if device_id not in scrcpy_streamers:
+            print(f"[video/stream] Creating streamer for device {device_id}")
+            scrcpy_streamers[device_id] = ScrcpyStreamer(
+                device_id=device_id, max_size=1280, bit_rate=4_000_000
+            )
+
+            try:
+                print(f"[video/stream] Starting scrcpy server for device {device_id}")
+                await scrcpy_streamers[device_id].start()
+                print(f"[video/stream] Scrcpy server started for device {device_id}")
+            except Exception as e:
+                import traceback
+
+                print(f"[video/stream] Failed to start streamer: {e}")
+                print(f"[video/stream] Traceback:\n{traceback.format_exc()}")
+                scrcpy_streamers[device_id].stop()
+                del scrcpy_streamers[device_id]
+                try:
+                    await websocket.send_json({"error": str(e)})
+                except Exception:
+                    pass
+                return
+        else:
+            print(f"[video/stream] Reusing streamer for device {device_id}")
+
+            streamer = scrcpy_streamers[device_id]
+            if streamer.cached_sps and streamer.cached_pps:
+                init_data = streamer.cached_sps + streamer.cached_pps
+                await websocket.send_bytes(init_data)
+                print(f"[video/stream] Sent SPS/PPS for device {device_id}")
+            else:
+                print(
+                    f"[video/stream] Warning: No cached SPS/PPS for device {device_id}"
+                )
+
+    streamer = scrcpy_streamers[device_id]
+
+    stream_failed = False
+    try:
+        chunk_count = 0
+        while True:
+            try:
+                h264_chunk = await streamer.read_h264_chunk()
+                await websocket.send_bytes(h264_chunk)
+                chunk_count += 1
+                if chunk_count % 100 == 0:
+                    print(
+                        f"[video/stream] Device {device_id}: Sent {chunk_count} chunks"
+                    )
+            except ConnectionError as e:
+                print(f"[video/stream] Device {device_id}: Connection error: {e}")
+                stream_failed = True
+                try:
+                    await websocket.send_json({"error": f"Stream error: {str(e)}"})
+                except Exception:
+                    pass
+                break
+
+    except WebSocketDisconnect:
+        print(f"[video/stream] Device {device_id}: Client disconnected")
+    except Exception as e:
+        import traceback
+
+        print(f"[video/stream] Device {device_id}: Error: {e}")
+        print(f"[video/stream] Traceback:\n{traceback.format_exc()}")
+        stream_failed = True
+        try:
+            await websocket.send_json({"error": str(e)})
+        except Exception:
+            pass
+
+    if stream_failed:
+        async with scrcpy_locks[device_id]:
+            if device_id in scrcpy_streamers:
+                print(f"[video/stream] Resetting streamer for device {device_id}")
+                scrcpy_streamers[device_id].stop()
+                del scrcpy_streamers[device_id]
+
+    print(f"[video/stream] Device {device_id}: Stream ended")

AutoGLM_GUI/schemas.py

@@ -0,0 +1,127 @@
+"""Shared Pydantic models for the AutoGLM-GUI API."""
+
+from pydantic import BaseModel, Field
+
+
+class APIModelConfig(BaseModel):
+    base_url: str | None = None
+    api_key: str | None = None
+    model_name: str | None = None
+    max_tokens: int = 3000
+    temperature: float = 0.0
+    top_p: float = 0.85
+    frequency_penalty: float = 0.2
+
+
+class APIAgentConfig(BaseModel):
+    max_steps: int = 100
+    device_id: str | None = None
+    lang: str = "cn"
+    system_prompt: str | None = None
+    verbose: bool = True
+
+
+class InitRequest(BaseModel):
+    model: APIModelConfig | None = Field(default=None, alias="model_config")
+    agent: APIAgentConfig | None = Field(default=None, alias="agent_config")
+
+
+class ChatRequest(BaseModel):
+    message: str
+    device_id: str  # 设备 ID（必填）
+
+
+class ChatResponse(BaseModel):
+    result: str
+    steps: int
+    success: bool
+
+
+class StatusResponse(BaseModel):
+    version: str
+    initialized: bool
+    step_count: int
+
+
+class ResetRequest(BaseModel):
+    device_id: str  # 设备 ID（必填）
+
+
+class ScreenshotRequest(BaseModel):
+    device_id: str | None = None
+
+
+class ScreenshotResponse(BaseModel):
+    success: bool
+    image: str  # base64 encoded PNG
+    width: int
+    height: int
+    is_sensitive: bool
+    error: str | None = None
+
+
+class TapRequest(BaseModel):
+    x: int
+    y: int
+    device_id: str | None = None
+    delay: float = 0.0
+
+
+class TapResponse(BaseModel):
+    success: bool
+    error: str | None = None
+
+
+class SwipeRequest(BaseModel):
+    start_x: int
+    start_y: int
+    end_x: int
+    end_y: int
+    duration_ms: int | None = None
+    device_id: str | None = None
+    delay: float = 0.0
+
+
+class SwipeResponse(BaseModel):
+    success: bool
+    error: str | None = None
+
+
+class TouchDownRequest(BaseModel):
+    x: int
+    y: int
+    device_id: str | None = None
+    delay: float = 0.0
+
+
+class TouchDownResponse(BaseModel):
+    success: bool
+    error: str | None = None
+
+
+class TouchMoveRequest(BaseModel):
+    x: int
+    y: int
+    device_id: str | None = None
+    delay: float = 0.0
+
+
+class TouchMoveResponse(BaseModel):
+    success: bool
+    error: str | None = None
+
+
+class TouchUpRequest(BaseModel):
+    x: int
+    y: int
+    device_id: str | None = None
+    delay: float = 0.0
+
+
+class TouchUpResponse(BaseModel):
+    success: bool
+    error: str | None = None
+
+
+class DeviceListResponse(BaseModel):
+    devices: list[dict]

AutoGLM_GUI/scrcpy_stream.py

@@ -104,6 +104,7 @@ class ScrcpyStreamer:
         except Exception as e:
             print(f"[ScrcpyStreamer] Failed to start: {e}")
             import traceback
+
             traceback.print_exc()
             self.stop()
             raise RuntimeError(f"Failed to start scrcpy server: {e}") from e
@@ -124,7 +125,7 @@ class ScrcpyStreamer:
         # Method 2: Find and kill by PID (more reliable)
         cmd = cmd_base + [
             "shell",
-            "ps -ef | grep 'app_process.*scrcpy' | grep -v grep | awk '{print $2}' | xargs kill -9"
+            "ps -ef | grep 'app_process.*scrcpy' | grep -v grep | awk '{print $2}' | xargs kill -9",
         ]
         process = await asyncio.create_subprocess_exec(
             *cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
@@ -200,9 +201,7 @@ class ScrcpyStreamer:
 
             # Capture stderr to see error messages
             self.scrcpy_process = await asyncio.create_subprocess_exec(
-                *cmd,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE
+                *cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
             )
 
             # Wait for server to start
@@ -217,12 +216,16 @@ class ScrcpyStreamer:
                 # Check if it's an "Address already in use" error
                 if "Address already in use" in error_msg:
                     if attempt < max_retries - 1:
-                        print(f"[ScrcpyStreamer] Address in use, retrying in {retry_delay}s (attempt {attempt + 1}/{max_retries})...")
+                        print(
+                            f"[ScrcpyStreamer] Address in use, retrying in {retry_delay}s (attempt {attempt + 1}/{max_retries})..."
+                        )
                         await self._cleanup_existing_server()
                         await asyncio.sleep(retry_delay)
                         continue
                     else:
-                        raise RuntimeError(f"scrcpy server failed after {max_retries} attempts: {error_msg}")
+                        raise RuntimeError(
+                            f"scrcpy server failed after {max_retries} attempts: {error_msg}"
+                        )
                 else:
                     raise RuntimeError(f"scrcpy server exited immediately: {error_msg}")
 
@@ -239,7 +242,9 @@ class ScrcpyStreamer:
         # Increase socket buffer size for high-resolution video
         # Default is often 64KB, but complex frames can be 200-500KB
         try:
-            self.tcp_socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 2 * 1024 * 1024)  # 2MB
+            self.tcp_socket.setsockopt(
+                socket.SOL_SOCKET, socket.SO_RCVBUF, 2 * 1024 * 1024
+            )  # 2MB
             print("[ScrcpyStreamer] Set socket receive buffer to 2MB")
         except OSError as e:
             print(f"[ScrcpyStreamer] Warning: Failed to set socket buffer size: {e}")
@@ -267,9 +272,9 @@ class ScrcpyStreamer:
 
         while i < data_len - 4:
             # Look for start codes: 0x00 0x00 0x00 0x01 or 0x00 0x00 0x01
-            if data[i:i+4] == b'\x00\x00\x00\x01':
+            if data[i : i + 4] == b"\x00\x00\x00\x01":
                 start_code_len = 4
-            elif data[i:i+3] == b'\x00\x00\x01':
+            elif data[i : i + 3] == b"\x00\x00\x01":
                 start_code_len = 3
             else:
                 i += 1
@@ -285,8 +290,10 @@ class ScrcpyStreamer:
             # Find next start code to determine NAL unit size
             next_start = nal_start + 1
             while next_start < data_len - 3:
-                if (data[next_start:next_start+4] == b'\x00\x00\x00\x01' or
-                    data[next_start:next_start+3] == b'\x00\x00\x01'):
+                if (
+                    data[next_start : next_start + 4] == b"\x00\x00\x00\x01"
+                    or data[next_start : next_start + 3] == b"\x00\x00\x01"
+                ):
                     break
                 next_start += 1
             else:
@@ -309,7 +316,7 @@ class ScrcpyStreamer:
         nal_units = self._find_nal_units(data)
 
         for start, nal_type, size in nal_units:
-            nal_data = data[start:start+size]
+            nal_data = data[start : start + size]
 
             if nal_type == 7:  # SPS
                 # Only cache SPS if not yet locked
@@ -317,10 +324,16 @@ class ScrcpyStreamer:
                     # Validate: SPS should be at least 10 bytes
                     if size >= 10 and not self.cached_sps:
                         self.cached_sps = nal_data
-                        hex_preview = ' '.join(f'{b:02x}' for b in nal_data[:min(12, len(nal_data))])
-                        print(f"[ScrcpyStreamer] ✓ Cached complete SPS ({size} bytes): {hex_preview}...")
+                        hex_preview = " ".join(
+                            f"{b:02x}" for b in nal_data[: min(12, len(nal_data))]
+                        )
+                        print(
+                            f"[ScrcpyStreamer] ✓ Cached complete SPS ({size} bytes): {hex_preview}..."
+                        )
                     elif size < 10:
-                        print(f"[ScrcpyStreamer] ✗ Skipped truncated SPS ({size} bytes, too short)")
+                        print(
+                            f"[ScrcpyStreamer] ✗ Skipped truncated SPS ({size} bytes, too short)"
+                        )
 
             elif nal_type == 8:  # PPS
                 # Only cache PPS if not yet locked
@@ -328,10 +341,16 @@ class ScrcpyStreamer:
                     # Validate: PPS should be at least 6 bytes
                     if size >= 6 and not self.cached_pps:
                         self.cached_pps = nal_data
-                        hex_preview = ' '.join(f'{b:02x}' for b in nal_data[:min(12, len(nal_data))])
-                        print(f"[ScrcpyStreamer] ✓ Cached complete PPS ({size} bytes): {hex_preview}...")
+                        hex_preview = " ".join(
+                            f"{b:02x}" for b in nal_data[: min(12, len(nal_data))]
+                        )
+                        print(
+                            f"[ScrcpyStreamer] ✓ Cached complete PPS ({size} bytes): {hex_preview}..."
+                        )
                     elif size < 6:
-                        print(f"[ScrcpyStreamer] ✗ Skipped truncated PPS ({size} bytes, too short)")
+                        print(
+                            f"[ScrcpyStreamer] ✗ Skipped truncated PPS ({size} bytes, too short)"
+                        )
 
             elif nal_type == 5:  # IDR frame
                 # ✅ ALWAYS update IDR to keep the LATEST frame
@@ -340,7 +359,9 @@ class ScrcpyStreamer:
                     is_first = self.cached_idr is None
                     self.cached_idr = nal_data
                     if is_first:
-                        print(f"[ScrcpyStreamer] ✓ Cached initial IDR frame ({size} bytes)")
+                        print(
+                            f"[ScrcpyStreamer] ✓ Cached initial IDR frame ({size} bytes)"
+                        )
                     # Don't log every IDR update (too verbose)
 
         # Lock SPS/PPS once we have complete initial parameters
@@ -366,7 +387,9 @@ class ScrcpyStreamer:
             return data
 
         # Find all IDR frames
-        idr_positions = [(start, size) for start, nal_type, size in nal_units if nal_type == 5]
+        idr_positions = [
+            (start, size) for start, nal_type, size in nal_units if nal_type == 5
+        ]
 
         if not idr_positions:
             return data
@@ -386,7 +409,9 @@ class ScrcpyStreamer:
             if data[prepend_offset:idr_start] != sps_pps:
                 # Prepend SPS/PPS before this IDR
                 result.extend(sps_pps)
-                print(f"[ScrcpyStreamer] Prepended SPS/PPS before IDR at position {idr_start}")
+                print(
+                    f"[ScrcpyStreamer] Prepended SPS/PPS before IDR at position {idr_start}"
+                )
 
             # Update position to start of IDR
             last_pos = idr_start
@@ -409,11 +434,17 @@ class ScrcpyStreamer:
                 init_data += self.cached_idr
 
             # Validate data integrity
-            print(f"[ScrcpyStreamer] Returning init data:")
-            print(f"  - SPS: {len(self.cached_sps)} bytes, starts with {' '.join(f'{b:02x}' for b in self.cached_sps[:8])}")
-            print(f"  - PPS: {len(self.cached_pps)} bytes, starts with {' '.join(f'{b:02x}' for b in self.cached_pps[:8])}")
+            print("[ScrcpyStreamer] Returning init data:")
+            print(
+                f"  - SPS: {len(self.cached_sps)} bytes, starts with {' '.join(f'{b:02x}' for b in self.cached_sps[:8])}"
+            )
+            print(
+                f"  - PPS: {len(self.cached_pps)} bytes, starts with {' '.join(f'{b:02x}' for b in self.cached_pps[:8])}"
+            )
             if self.cached_idr:
-                print(f"  - IDR: {len(self.cached_idr)} bytes, starts with {' '.join(f'{b:02x}' for b in self.cached_idr[:8])}")
+                print(
+                    f"  - IDR: {len(self.cached_idr)} bytes, starts with {' '.join(f'{b:02x}' for b in self.cached_idr[:8])}"
+                )
             print(f"  - Total: {len(init_data)} bytes")
 
             return init_data
@@ -442,7 +473,9 @@ class ScrcpyStreamer:
 
             # Log large chunks (might indicate complex frames)
             if len(data) > 200 * 1024:  # > 200KB
-                print(f"[ScrcpyStreamer] Large chunk received: {len(data) / 1024:.1f} KB")
+                print(
+                    f"[ScrcpyStreamer] Large chunk received: {len(data) / 1024:.1f} KB"
+                )
 
             # Cache INITIAL complete SPS/PPS/IDR for future use
             # (Later chunks may have truncated NAL units, so we only cache once)
@@ -457,7 +490,9 @@ class ScrcpyStreamer:
         except ConnectionError:
             raise
         except Exception as e:
-            print(f"[ScrcpyStreamer] Unexpected error in read_h264_chunk: {type(e).__name__}: {e}")
+            print(
+                f"[ScrcpyStreamer] Unexpected error in read_h264_chunk: {type(e).__name__}: {e}"
+            )
             raise ConnectionError(f"Failed to read from socket: {e}") from e
 
     def stop(self) -> None:
@@ -489,7 +524,9 @@ class ScrcpyStreamer:
                 if self.device_id:
                     cmd.extend(["-s", self.device_id])
                 cmd.extend(["forward", "--remove", f"tcp:{self.port}"])
-                subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2)
+                subprocess.run(
+                    cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2
+                )
             except Exception:
                 pass
             self.forward_cleanup_needed = False