autoglm-gui 0.4.6__tar.gz → 0.4.8__tar.gz

{autoglm_gui-0.4.6 → autoglm_gui-0.4.8}/AutoGLM_GUI/api/media.py

@@ -1,6 +1,8 @@
 """Media routes: screenshot, video stream, stream reset."""
 
 import asyncio
+import os
+from pathlib import Path
 
 from fastapi import APIRouter, WebSocket, WebSocketDisconnect
 
@@ -11,6 +13,9 @@ from AutoGLM_GUI.state import scrcpy_locks, scrcpy_streamers
 
 router = APIRouter()
 
+# Debug configuration: Set DEBUG_SAVE_VIDEO_STREAM=1 to save streams to debug_streams/
+DEBUG_SAVE_STREAM = os.getenv("DEBUG_SAVE_VIDEO_STREAM", "0") == "1"
+
 
 @router.post("/api/video/reset")
 async def reset_video_stream(device_id: str | None = None) -> dict:
@@ -68,7 +73,10 @@ def take_screenshot(request: ScreenshotRequest) -> ScreenshotResponse:
 
 
 @router.websocket("/api/video/stream")
-async def video_stream_ws(websocket: WebSocket, device_id: str | None = None):
+async def video_stream_ws(
+    websocket: WebSocket,
+    device_id: str | None = None,
+):
     """Stream real-time H.264 video from scrcpy server via WebSocket（多设备支持）."""
     await websocket.accept()
 
@@ -78,6 +86,15 @@ async def video_stream_ws(websocket: WebSocket, device_id: str | None = None):
 
     print(f"[video/stream] WebSocket connection for device {device_id}")
 
+    # Debug: Save stream to file for analysis (controlled by DEBUG_SAVE_VIDEO_STREAM env var)
+    debug_file = None
+    if DEBUG_SAVE_STREAM:
+        debug_dir = Path("debug_streams")
+        debug_dir.mkdir(exist_ok=True)
+        debug_file_path = debug_dir / f"{device_id}_{int(__import__('time').time())}.h264"
+        debug_file = open(debug_file_path, "wb")
+        print(f"[video/stream] DEBUG: Saving stream to {debug_file_path}")
+
     if device_id not in scrcpy_locks:
         scrcpy_locks[device_id] = asyncio.Lock()
 
@@ -92,6 +109,53 @@ async def video_stream_ws(websocket: WebSocket, device_id: str | None = None):
                 print(f"[video/stream] Starting scrcpy server for device {device_id}")
                 await scrcpy_streamers[device_id].start()
                 print(f"[video/stream] Scrcpy server started for device {device_id}")
+
+                # Read NAL units until we have SPS, PPS, and IDR
+                streamer = scrcpy_streamers[device_id]
+
+                print("[video/stream] Reading NAL units for initialization...")
+                for attempt in range(20):  # Max 20 NAL units for initialization
+                    try:
+                        nal_unit = await streamer.read_nal_unit(auto_cache=True)
+                        nal_type = nal_unit[4] & 0x1F if len(nal_unit) > 4 else -1
+                        nal_type_names = {5: "IDR", 7: "SPS", 8: "PPS"}
+                        print(
+                            f"[video/stream] Read NAL unit: type={nal_type_names.get(nal_type, nal_type)}, size={len(nal_unit)} bytes"
+                        )
+
+                        # Check if we have all required parameter sets
+                        if (
+                            streamer.cached_sps
+                            and streamer.cached_pps
+                            and streamer.cached_idr
+                        ):
+                            print(
+                                f"[video/stream] ✓ Initialization complete: SPS={len(streamer.cached_sps)}B, PPS={len(streamer.cached_pps)}B, IDR={len(streamer.cached_idr)}B"
+                            )
+                            break
+                    except Exception as e:
+                        print(f"[video/stream] Failed to read NAL unit: {e}")
+                        await asyncio.sleep(0.5)
+                        continue
+
+                # Get initialization data (SPS + PPS + IDR)
+                init_data = streamer.get_initialization_data()
+                if not init_data:
+                    raise RuntimeError(
+                        "Failed to get initialization data (missing SPS/PPS/IDR)"
+                    )
+
+                # Send initialization data as ONE message (SPS+PPS+IDR combined)
+                await websocket.send_bytes(init_data)
+                print(
+                    f"[video/stream] ✓ Sent initialization data to first client: {len(init_data)} bytes total"
+                )
+
+                # Debug: Save to file
+                if debug_file:
+                    debug_file.write(init_data)
+                    debug_file.flush()
+
             except Exception as e:
                 import traceback
 
@@ -108,28 +172,73 @@ async def video_stream_ws(websocket: WebSocket, device_id: str | None = None):
             print(f"[video/stream] Reusing streamer for device {device_id}")
 
             streamer = scrcpy_streamers[device_id]
-            if streamer.cached_sps and streamer.cached_pps:
-                init_data = streamer.cached_sps + streamer.cached_pps
-                await websocket.send_bytes(init_data)
-                print(f"[video/stream] Sent SPS/PPS for device {device_id}")
-            else:
+            # CRITICAL: Send complete initialization data (SPS+PPS+IDR)
+            # Without IDR frame, decoder cannot start and will show black screen
+
+            # Wait for initialization data to be ready (max 5 seconds)
+            init_data = None
+            for attempt in range(10):
+                init_data = streamer.get_initialization_data()
+                if init_data:
+                    break
+                print(
+                    f"[video/stream] Waiting for initialization data (attempt {attempt + 1}/10)..."
+                )
+                await asyncio.sleep(0.5)
+
+            if init_data:
+                # Log what we're sending
+                print(
+                    f"[video/stream] ✓ Sending cached initialization data for device {device_id}:"
+                )
+                print(
+                    f"  - SPS: {len(streamer.cached_sps) if streamer.cached_sps else 0}B"
+                )
                 print(
-                    f"[video/stream] Warning: No cached SPS/PPS for device {device_id}"
+                    f"  - PPS: {len(streamer.cached_pps) if streamer.cached_pps else 0}B"
                 )
+                print(
+                    f"  - IDR: {len(streamer.cached_idr) if streamer.cached_idr else 0}B"
+                )
+                print(f"  - Total: {len(init_data)} bytes")
+
+                await websocket.send_bytes(init_data)
+                print("[video/stream] ✓ Initialization data sent successfully")
+
+                # Debug: Save to file
+                if debug_file:
+                    debug_file.write(init_data)
+                    debug_file.flush()
+            else:
+                error_msg = f"Initialization data not ready for device {device_id} after 5 seconds"
+                print(f"[video/stream] ERROR: {error_msg}")
+                try:
+                    await websocket.send_json({"error": error_msg})
+                except Exception:
+                    pass
+                return
 
     streamer = scrcpy_streamers[device_id]
 
     stream_failed = False
     try:
-        chunk_count = 0
+        nal_count = 0
         while True:
             try:
-                h264_chunk = await streamer.read_h264_chunk()
-                await websocket.send_bytes(h264_chunk)
-                chunk_count += 1
-                if chunk_count % 100 == 0:
+                # Read one complete NAL unit
+                # Each WebSocket message = one complete NAL unit (clear semantic boundary)
+                nal_unit = await streamer.read_nal_unit(auto_cache=True)
+                await websocket.send_bytes(nal_unit)
+
+                # Debug: Save to file
+                if debug_file:
+                    debug_file.write(nal_unit)
+                    debug_file.flush()
+
+                nal_count += 1
+                if nal_count % 100 == 0:
                     print(
-                        f"[video/stream] Device {device_id}: Sent {chunk_count} chunks"
+                        f"[video/stream] Device {device_id}: Sent {nal_count} NAL units"
                     )
             except ConnectionError as e:
                 print(f"[video/stream] Device {device_id}: Connection error: {e}")
@@ -160,4 +269,9 @@ async def video_stream_ws(websocket: WebSocket, device_id: str | None = None):
                 scrcpy_streamers[device_id].stop()
                 del scrcpy_streamers[device_id]
 
+    # Debug: Close file
+    if debug_file:
+        debug_file.close()
+        print("[video/stream] DEBUG: Closed debug file")
+
     print(f"[video/stream] Device {device_id}: Stream ended")

{autoglm_gui-0.4.6 → autoglm_gui-0.4.8}/AutoGLM_GUI/scrcpy_stream.py

@@ -47,6 +47,9 @@ class ScrcpyStreamer:
         self.sps_pps_locked = False  # Lock SPS/PPS after initial complete capture
         # Note: IDR is NOT locked - we keep updating to the latest frame
 
+        # NAL unit reading buffer (for read_nal_unit method)
+        self._nal_read_buffer = bytearray()
+
         # Find scrcpy-server location
         self.scrcpy_server_path = self._find_scrcpy_server()
 
@@ -83,6 +86,10 @@ class ScrcpyStreamer:
 
     async def start(self) -> None:
         """Start scrcpy server and establish connection."""
+        # Clear NAL reading buffer to ensure clean state
+        self._nal_read_buffer.clear()
+        print("[ScrcpyStreamer] Cleared NAL read buffer")
+
         try:
             # 0. Kill existing scrcpy server processes on device
             print("[ScrcpyStreamer] Cleaning up existing scrcpy processes...")
@@ -307,11 +314,14 @@ class ScrcpyStreamer:
 
         raise ConnectionError("Failed to connect to scrcpy server")
 
-    def _find_nal_units(self, data: bytes) -> list[tuple[int, int, int]]:
+    def _find_nal_units(
+        self, data: bytes
+    ) -> list[tuple[int, int, int, bool]]:
         """Find NAL units in H.264 data.
 
         Returns:
-            List of (start_pos, nal_type, nal_size) tuples
+            List of (start_pos, nal_type, nal_size, is_complete) tuples
+            is_complete=False if NAL unit extends to chunk boundary (may be truncated)
         """
         nal_units = []
         i = 0
@@ -336,18 +346,22 @@ class ScrcpyStreamer:
 
             # Find next start code to determine NAL unit size
             next_start = nal_start + 1
+            found_next = False
             while next_start < data_len - 3:
                 if (
                     data[next_start : next_start + 4] == b"\x00\x00\x00\x01"
                     or data[next_start : next_start + 3] == b"\x00\x00\x01"
                 ):
+                    found_next = True
                     break
                 next_start += 1
             else:
                 next_start = data_len
 
             nal_size = next_start - i
-            nal_units.append((i, nal_type, nal_size))
+            # NAL unit is complete only if we found the next start code
+            is_complete = found_next
+            nal_units.append((i, nal_type, nal_size, is_complete))
 
             i = next_start
 
@@ -356,13 +370,13 @@ class ScrcpyStreamer:
     def _cache_nal_units(self, data: bytes) -> None:
         """Parse and cache INITIAL complete NAL units (SPS, PPS, IDR).
 
-        IMPORTANT: Only caches complete SPS/PPS from stream start.
-        NAL units may be truncated across chunks, so we validate minimum sizes
-        and lock the cache after getting complete initial parameters.
+        IMPORTANT: Caches NAL units with size validation.
+        For small NAL units (SPS/PPS), we cache even if at chunk boundary.
+        For large NAL units (IDR), we require minimum size to ensure completeness.
         """
         nal_units = self._find_nal_units(data)
 
-        for start, nal_type, size in nal_units:
+        for start, nal_type, size, is_complete in nal_units:
             nal_data = data[start : start + size]
 
             if nal_type == 7:  # SPS
@@ -375,11 +389,11 @@ class ScrcpyStreamer:
                             f"{b:02x}" for b in nal_data[: min(12, len(nal_data))]
                         )
                         print(
-                            f"[ScrcpyStreamer] ✓ Cached complete SPS ({size} bytes): {hex_preview}..."
+                            f"[ScrcpyStreamer] ✓ Cached SPS ({size} bytes, complete={is_complete}): {hex_preview}..."
                         )
                     elif size < 10:
                         print(
-                            f"[ScrcpyStreamer] ✗ Skipped truncated SPS ({size} bytes, too short)"
+                            f"[ScrcpyStreamer] ✗ Skipped short SPS ({size} bytes)"
                         )
 
             elif nal_type == 8:  # PPS
@@ -392,82 +406,36 @@ class ScrcpyStreamer:
                             f"{b:02x}" for b in nal_data[: min(12, len(nal_data))]
                         )
                         print(
-                            f"[ScrcpyStreamer] ✓ Cached complete PPS ({size} bytes): {hex_preview}..."
+                            f"[ScrcpyStreamer] ✓ Cached PPS ({size} bytes, complete={is_complete}): {hex_preview}..."
                         )
                     elif size < 6:
                         print(
-                            f"[ScrcpyStreamer] ✗ Skipped truncated PPS ({size} bytes, too short)"
+                            f"[ScrcpyStreamer] ✗ Skipped short PPS ({size} bytes)"
                         )
 
             elif nal_type == 5:  # IDR frame
-                # ✅ ALWAYS update IDR to keep the LATEST frame
-                # This gives better UX on reconnect (recent content, not stale startup frame)
-                if self.cached_sps and self.cached_pps:
+                # Cache IDR if it's large enough (size check is sufficient)
+                # Note: When called from read_nal_unit(), the NAL is guaranteed complete
+                # because we extract it between two start codes. The is_complete flag
+                # is only False because the NAL is isolated (no next start code in buffer).
+                if self.cached_sps and self.cached_pps and size >= 1024:
                     is_first = self.cached_idr is None
                     self.cached_idr = nal_data
                     if is_first:
                         print(
-                            f"[ScrcpyStreamer] ✓ Cached initial IDR frame ({size} bytes)"
+                            f"[ScrcpyStreamer] ✓ Cached IDR frame ({size} bytes)"
                         )
                     # Don't log every IDR update (too verbose)
+                elif size < 1024:
+                    print(
+                        f"[ScrcpyStreamer] ✗ Skipped small IDR ({size} bytes, likely incomplete)"
+                    )
 
         # Lock SPS/PPS once we have complete initial parameters
         if self.cached_sps and self.cached_pps and not self.sps_pps_locked:
             self.sps_pps_locked = True
             print("[ScrcpyStreamer] 🔒 SPS/PPS locked (IDR will continue updating)")
 
-    def _prepend_sps_pps_to_idr(self, data: bytes) -> bytes:
-        """Prepend SPS/PPS before EVERY IDR frame unconditionally.
-
-        This ensures that clients can start decoding from any IDR frame,
-        even if they join mid-stream. We always prepend to guarantee
-        that every IDR is self-contained.
-
-        Returns:
-            Modified data with SPS/PPS prepended to all IDR frames
-        """
-        if not self.cached_sps or not self.cached_pps:
-            return data
-
-        nal_units = self._find_nal_units(data)
-        if not nal_units:
-            return data
-
-        # Find all IDR frames
-        idr_positions = [
-            (start, size) for start, nal_type, size in nal_units if nal_type == 5
-        ]
-
-        if not idr_positions:
-            return data
-
-        # Build modified data by prepending SPS/PPS before each IDR
-        result = bytearray()
-        last_pos = 0
-        sps_pps = self.cached_sps + self.cached_pps
-
-        for idr_start, idr_size in idr_positions:
-            # Add data before this IDR
-            result.extend(data[last_pos:idr_start])
-
-            # Check if SPS/PPS already exists right before this IDR
-            # (to avoid duplicating if scrcpy already sent them)
-            prepend_offset = max(0, idr_start - len(sps_pps))
-            if data[prepend_offset:idr_start] != sps_pps:
-                # Prepend SPS/PPS before this IDR
-                result.extend(sps_pps)
-                print(
-                    f"[ScrcpyStreamer] Prepended SPS/PPS before IDR at position {idr_start}"
-                )
-
-            # Update position to start of IDR
-            last_pos = idr_start
-
-        # Add remaining data (including all IDR frames and data after)
-        result.extend(data[last_pos:])
-
-        return bytes(result)
-
     def get_initialization_data(self) -> bytes | None:
         """Get cached SPS/PPS/IDR for initializing new connections.
 
@@ -497,11 +465,14 @@ class ScrcpyStreamer:
             return init_data
         return None
 
-    async def read_h264_chunk(self) -> bytes:
+    async def read_h264_chunk(self, auto_cache: bool = True) -> bytes:
         """Read H.264 data chunk from socket.
 
+        Args:
+            auto_cache: If True, automatically cache SPS/PPS/IDR from this chunk
+
         Returns:
-            bytes: Raw H.264 data with SPS/PPS prepended to IDR frames
+            bytes: Raw H.264 data
 
         Raises:
             ConnectionError: If socket is closed or error occurs
@@ -524,9 +495,9 @@ class ScrcpyStreamer:
                     f"[ScrcpyStreamer] Large chunk received: {len(data) / 1024:.1f} KB"
                 )
 
-            # Cache INITIAL complete SPS/PPS/IDR for future use
-            # (Later chunks may have truncated NAL units, so we only cache once)
-            self._cache_nal_units(data)
+            # Optionally cache SPS/PPS/IDR from this chunk
+            if auto_cache:
+                self._cache_nal_units(data)
 
             # NOTE: We don't automatically prepend SPS/PPS here because:
             # 1. NAL units may be truncated across chunks
@@ -542,6 +513,86 @@ class ScrcpyStreamer:
             )
             raise ConnectionError(f"Failed to read from socket: {e}") from e
 
+    async def read_nal_unit(self, auto_cache: bool = True) -> bytes:
+        """Read one complete NAL unit from socket.
+
+        This method ensures each returned chunk is a complete, self-contained NAL unit.
+        WebSocket messages will have clear semantic boundaries (one message = one NAL unit).
+
+        Args:
+            auto_cache: If True, automatically cache SPS/PPS/IDR from this NAL unit
+
+        Returns:
+            bytes: Complete NAL unit (including start code)
+
+        Raises:
+            ConnectionError: If socket is closed or error occurs
+        """
+        if not self.tcp_socket:
+            raise ConnectionError("Socket not connected")
+
+        while True:
+            # Look for start codes in buffer
+            buffer = bytes(self._nal_read_buffer)
+            start_positions = []
+
+            # Find all start codes (0x00 0x00 0x00 0x01 or 0x00 0x00 0x01)
+            i = 0
+            while i < len(buffer) - 3:
+                if buffer[i] == 0x00 and buffer[i + 1] == 0x00:
+                    if buffer[i + 2] == 0x00 and buffer[i + 3] == 0x01:
+                        start_positions.append(i)
+                        i += 4
+                    elif buffer[i + 2] == 0x01:
+                        start_positions.append(i)
+                        i += 3
+                    else:
+                        i += 1
+                else:
+                    i += 1
+
+            # If we have at least 2 start codes, we can extract the first NAL unit
+            if len(start_positions) >= 2:
+                # Extract first complete NAL unit (from first start code to second start code)
+                nal_unit = buffer[start_positions[0] : start_positions[1]]
+
+                # Remove extracted NAL unit from buffer
+                self._nal_read_buffer = bytearray(buffer[start_positions[1] :])
+
+                # Cache parameter sets if enabled
+                if auto_cache:
+                    self._cache_nal_units(nal_unit)
+
+                return nal_unit
+
+            # Need more data - read from socket
+            try:
+                loop = asyncio.get_event_loop()
+                chunk = await loop.run_in_executor(
+                    None, self.tcp_socket.recv, 512 * 1024
+                )
+
+                if not chunk:
+                    # Socket closed - return any remaining buffered data as final NAL unit
+                    if len(self._nal_read_buffer) > 0:
+                        final_nal = bytes(self._nal_read_buffer)
+                        self._nal_read_buffer.clear()
+                        if auto_cache:
+                            self._cache_nal_units(final_nal)
+                        return final_nal
+                    raise ConnectionError("Socket closed by remote")
+
+                # Append new data to buffer
+                self._nal_read_buffer.extend(chunk)
+
+            except ConnectionError:
+                raise
+            except Exception as e:
+                print(
+                    f"[ScrcpyStreamer] Unexpected error in read_nal_unit: {type(e).__name__}: {e}"
+                )
+                raise ConnectionError(f"Failed to read from socket: {e}") from e
+
     def stop(self) -> None:
         """Stop scrcpy server and cleanup resources."""
         # Close socket

autoglm_gui-0.4.6/AutoGLM_GUI/static/assets/about-BgmYMjz1.js → autoglm_gui-0.4.8/AutoGLM_GUI/static/assets/about-BI6OV6gm.js

@@ -1 +1 @@
-import{j as o}from"./index-sfTWLHvP.js";function t(){return o.jsx("div",{className:"p-2",children:o.jsx("h3",{children:"About"})})}export{t as component};
+import{j as o}from"./index-Dn3vR6uV.js";function t(){return o.jsx("div",{className:"p-2",children:o.jsx("h3",{children:"About"})})}export{t as component};