webtap-tool 0.7.1__py3-none-any.whl → 0.8.1__py3-none-any.whl

webtap/services/body.py

@@ -19,17 +19,25 @@ class BodyService:
         self.cdp: CDPSession | None = None
         self._body_cache: dict[str, dict] = {}
 
-    def get_response_body(self, rowid: int, use_cache: bool = True) -> dict:
-        """Fetch response body for a response.
+    def get_body(self, rowid: int, use_cache: bool = True) -> dict:
+        """Fetch request or response body for an event.
+
+        Automatically detects event type and fetches appropriate body:
+        - Network.requestWillBeSent: request body (POST data)
+        - Network.responseReceived: response body
+        - Fetch.requestPaused: request or response body based on stage
 
         Args:
-            rowid: Row ID from events table (Network or Fetch response)
+            rowid: Row ID from events table
             use_cache: Whether to use cached body if available
+
+        Returns:
+            Dict with 'body' (str), 'base64Encoded' (bool), and 'event' (dict), or 'error' (str)
         """
         if not self.cdp:
             return {"error": "No CDP session"}
 
-        # Get event from DB to extract requestId
+        # Get event from DB to extract requestId and method
         result = self.cdp.query("SELECT event FROM events WHERE rowid = ?", [rowid])
 
         if not result:
@@ -42,45 +50,89 @@ class BodyService:
 
         method = event_data.get("method", "")
         params = event_data.get("params", {})
+        request_id = params.get("requestId")
+
+        if not request_id:
+            return {"error": "No requestId in event"}
+
+        # Check cache first (cache includes event_data)
+        cache_key = f"{request_id}:{method}"
+        if use_cache and cache_key in self._body_cache:
+            logger.debug(f"Using cached body for {cache_key}")
+            return self._body_cache[cache_key]
+
+        # Handle request body (POST data)
+        if method == "Network.requestWillBeSent":
+            request = params.get("request", {})
+
+            # Check inline postData first (may be present for small bodies)
+            if request.get("postData"):
+                logger.debug(f"Using inline postData for {request_id}")
+                body_data = {"body": request["postData"], "base64Encoded": False, "event": event_data}
+                if use_cache:
+                    self._body_cache[cache_key] = body_data
+                return body_data
+
+            # Check if request has POST data
+            if not request.get("hasPostData"):
+                return {"error": "No POST data in this request (GET or no body)"}
+
+            # Try to fetch POST data via CDP
+            try:
+                logger.debug(f"Fetching POST data for {request_id} using Network.getRequestPostData")
+                result = self.cdp.execute("Network.getRequestPostData", {"requestId": request_id})
+                body_data = {"body": result.get("postData", ""), "base64Encoded": False, "event": event_data}
+
+                if use_cache:
+                    self._body_cache[cache_key] = body_data
+                    logger.debug(f"Cached POST data for {request_id}")
+
+                return body_data
 
-        # Handle both Fetch and Network events
-        if method == "Fetch.requestPaused":
+            except Exception as e:
+                error_msg = str(e)
+                logger.error(f"Failed to fetch POST data for {request_id}: {error_msg}")
+                # Provide helpful error message
+                if "No resource with given identifier found" in error_msg:
+                    return {"error": "POST data not available (possibly too large or not captured by CDP)"}
+                return {"error": f"Failed to fetch POST data: {error_msg}"}
+
+        # Handle response body
+        elif method == "Fetch.requestPaused":
             # Fetch interception - verify it's response stage
             if "responseStatusCode" not in params:
                 return {"error": "Not a response stage event (no responseStatusCode)"}
-            request_id = params.get("requestId")
             domain = "Fetch"
+
         elif method == "Network.responseReceived":
             # Regular network response
-            request_id = params.get("requestId")
             domain = "Network"
-        else:
-            return {"error": f"Not a response event (method: {method})"}
 
-        if not request_id:
-            return {"error": "No requestId in event"}
-
-        # Check cache
-        if use_cache and request_id in self._body_cache:
-            logger.debug(f"Using cached body for {request_id}")
-            return self._body_cache[request_id]
+        else:
+            # Unknown event type - return empty body but include event for expr access
+            logger.debug(f"Unknown event type {method} - returning empty body with event data")
+            return {"body": "", "base64Encoded": False, "event": event_data}
 
+        # Fetch response body from CDP
         try:
-            # Fetch body from CDP using appropriate domain
-            logger.debug(f"Fetching body for {request_id} using {domain}.getResponseBody")
+            logger.debug(f"Fetching response body for {request_id} using {domain}.getResponseBody")
             result = self.cdp.execute(f"{domain}.getResponseBody", {"requestId": request_id})
 
-            body_data = {"body": result.get("body", ""), "base64Encoded": result.get("base64Encoded", False)}
+            body_data = {
+                "body": result.get("body", ""),
+                "base64Encoded": result.get("base64Encoded", False),
+                "event": event_data,
+            }
 
-            # Cache it for this request
+            # Cache it
             if use_cache:
-                self._body_cache[request_id] = body_data
-                logger.debug(f"Cached body for {request_id}")
+                self._body_cache[cache_key] = body_data
+                logger.debug(f"Cached response body for {request_id}")
 
             return body_data
 
         except Exception as e:
-            logger.error(f"Failed to fetch body for {request_id}: {e}")
+            logger.error(f"Failed to fetch response body for {request_id}: {e}")
             return {"error": str(e)}
 
     def clear_cache(self):
@@ -111,3 +163,136 @@ class BodyService:
         except Exception as e:
             logger.error(f"Failed to decode base64 body: {e}")
             return body_content  # Return original if decode fails
+
+    def prepare_for_generation(
+        self,
+        event: int,
+        json_path: str = None,  # pyright: ignore[reportArgumentType]
+        expr: str = None,  # pyright: ignore[reportArgumentType]
+    ) -> dict:
+        """Prepare HTTP body for code generation.
+
+        Orchestrates the complete pipeline:
+        1. Fetch body + event from CDP
+        2. Decode base64 if needed
+        3. Transform via expression OR validate and parse JSON
+        4. Extract nested data via json_path
+        5. Validate data structure (dict/list)
+
+        Args:
+            event: Event row ID from network() or events()
+            json_path: Optional JSON path for nested extraction (e.g., "data[0]")
+            expr: Optional Python expression with 'body' and 'event' variables
+
+        Returns:
+            Dict with 'data' (dict|list) on success, or 'error' (str) on failure.
+            May include 'suggestions' (list[str]) for error guidance.
+
+        Examples:
+            result = body_service.prepare_for_generation(123, json_path="data[0]")
+            if result.get("error"):
+                return error_response(result["error"], suggestions=result.get("suggestions"))
+            data = result["data"]
+        """
+        # 1. Fetch body + event from CDP
+        result = self.get_body(event, use_cache=True)
+        if "error" in result:
+            return {"error": result["error"], "suggestions": [], "data": None}
+
+        body_content = result["body"]
+        is_base64 = result["base64Encoded"]
+        event_data = result["event"]
+
+        # 2. Decode if base64
+        if is_base64:
+            decoded = self.decode_body(body_content, is_base64)
+            if isinstance(decoded, bytes):
+                return {
+                    "error": "Body is binary content",
+                    "suggestions": [
+                        "Only text/JSON can be converted to code",
+                        "Try a different event with text content",
+                    ],
+                    "data": None,
+                }
+            body_content = decoded
+
+        # 3. Transform via expression OR validate and parse JSON
+        if expr:
+            # Use expression evaluation from _utils
+            from webtap.commands._utils import evaluate_expression
+
+            try:
+                namespace = {"body": body_content, "event": event_data}
+                data, _ = evaluate_expression(expr, namespace)
+            except Exception as e:
+                return {
+                    "error": f"Expression evaluation failed: {e}",
+                    "suggestions": [
+                        "Check your expression syntax",
+                        "Variables available: 'body' (str), 'event' (dict)",
+                        "Example: dict(urllib.parse.parse_qsl(body))",
+                        "Example: json.loads(body)['data'][0]",
+                    ],
+                    "data": None,
+                }
+        else:
+            # Validate body is not empty before parsing
+            if not body_content.strip():
+                return {
+                    "error": "Body is empty",
+                    "suggestions": [
+                        "Use expr to extract data from event for non-HTTP events",
+                        "Example: expr=\"json.loads(event['params']['response']['payloadData'])\"",
+                        "Check the event structure with inspect() first",
+                    ],
+                    "data": None,
+                }
+
+            # Parse as JSON
+            from webtap.commands._code_generation import parse_json
+
+            data, error = parse_json(body_content)
+            if error:
+                return {
+                    "error": error,
+                    "suggestions": [
+                        "Body must be valid JSON or use expr to transform it",
+                        'For form data: expr="dict(urllib.parse.parse_qsl(body))"',
+                        "Check the body with body() command first",
+                    ],
+                    "data": None,
+                }
+
+        # 4. Extract nested path if specified
+        if json_path:
+            from webtap.commands._code_generation import extract_json_path
+
+            data, error = extract_json_path(data, json_path)
+            if error:
+                return {
+                    "error": error,
+                    "suggestions": [
+                        f"Path '{json_path}' not found in body",
+                        "Check the body structure with body() command",
+                        'Try a simpler path like "data" or "data[0]"',
+                    ],
+                    "data": None,
+                }
+
+        # 5. Validate structure
+        from webtap.commands._code_generation import validate_generation_data
+
+        is_valid, error = validate_generation_data(data)
+        if not is_valid:
+            return {
+                "error": error,
+                "suggestions": [
+                    "Code generation requires dict or list structure",
+                    "Adjust json_path to extract a complex object",
+                    "Or use expr to transform data into dict/list",
+                ],
+                "data": None,
+            }
+
+        return {"data": data, "error": None, "suggestions": []}

webtap/services/dom.py

@@ -46,7 +46,7 @@ class DOMService:
         self.state = state
         self._inspection_active = False
         self._next_id = 1
-        self._broadcast_queue: "Any | None" = None  # asyncio.Queue for thread-safe broadcasts
+        self._broadcast_callback: "Any | None" = None  # Callback to service._trigger_broadcast()
         self._state_lock = threading.Lock()  # Protect state mutations
         self._pending_selections = 0  # Track in-flight selection processing
         self._executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="dom-worker")
@@ -59,13 +59,13 @@ class DOMService:
         """Set state after initialization."""
         self.state = state
 
-    def set_broadcast_queue(self, queue: "Any") -> None:
-        """Set queue for broadcasting state changes.
+    def set_broadcast_callback(self, callback: "Any") -> None:
+        """Set callback for broadcasting state changes.
 
         Args:
-            queue: asyncio.Queue for thread-safe signaling
+            callback: Function to call when state changes (service._trigger_broadcast)
         """
-        self._broadcast_queue = queue
+        self._broadcast_callback = callback
 
     def start_inspect(self) -> dict[str, Any]:
         """Enable CDP element inspection mode.
@@ -116,6 +116,7 @@ class DOMService:
             self._inspection_active = True
             logger.info("Element inspection mode enabled")
 
+            self._trigger_broadcast()
             return {"success": True, "inspect_active": True}
 
         except Exception as e:
@@ -143,6 +144,7 @@ class DOMService:
             self._inspection_active = False
             logger.info("Element inspection mode disabled")
 
+            self._trigger_broadcast()
             return {"success": True, "inspect_active": False}
 
         except Exception as e:
@@ -244,12 +246,12 @@ class DOMService:
             self._trigger_broadcast()
 
     def _trigger_broadcast(self) -> None:
-        """Trigger SSE broadcast via queue (thread-safe helper)."""
-        if self._broadcast_queue:
+        """Trigger SSE broadcast via service callback (ensures snapshot update)."""
+        if self._broadcast_callback:
             try:
-                self._broadcast_queue.put_nowait({"type": "dom_update"})
+                self._broadcast_callback()
             except Exception as e:
-                logger.debug(f"Failed to queue broadcast: {e}")
+                logger.debug(f"Failed to trigger broadcast: {e}")
 
     def _extract_node_data(self, backend_node_id: int) -> dict[str, Any]:
         """Extract complete element data via CDP.
@@ -486,14 +488,16 @@ class DOMService:
                 self.state.browser_data["selections"] = {}
             self._next_id = 1
         logger.info("Selections cleared")
+        self._trigger_broadcast()
 
     def cleanup(self) -> None:
         """Cleanup resources (executor, callbacks).
 
         Call this before disconnect or app exit.
+        Safe to call multiple times.
         """
         # Shutdown executor - wait=False to avoid blocking on stuck tasks
-        # cancel_futures=True prevents hanging on incomplete selections (Python 3.9+)
+        # cancel_futures=True prevents hanging on incomplete selections
         if hasattr(self, "_executor"):
             try:
                 self._executor.shutdown(wait=False, cancel_futures=True)
@@ -501,12 +505,15 @@ class DOMService:
             except Exception as e:
                 logger.debug(f"Executor shutdown error (non-fatal): {e}")
 
-        # Clear inspection state
-        if self._inspection_active:
+        # Clear inspection state (only if connected)
+        if self._inspection_active and self.cdp and self.cdp.is_connected:
             try:
                 self.stop_inspect()
             except Exception as e:
                 logger.debug(f"Failed to stop inspect on cleanup: {e}")
 
+        # Force clear inspection flag even if CDP call failed
+        self._inspection_active = False
+
 
 __all__ = ["DOMService"]

webtap/services/fetch.py

@@ -21,6 +21,23 @@ class FetchService:
         self.enable_response_stage = False  # Config option for future
         self.cdp: CDPSession | None = None
         self.body_service: BodyService | None = None
+        self._broadcast_callback: "Any | None" = None  # Callback to service._trigger_broadcast()
+
+    def set_broadcast_callback(self, callback: "Any") -> None:
+        """Set callback for broadcasting state changes.
+
+        Args:
+            callback: Function to call when state changes (service._trigger_broadcast)
+        """
+        self._broadcast_callback = callback
+
+    def _trigger_broadcast(self) -> None:
+        """Trigger SSE broadcast via service callback (ensures snapshot update)."""
+        if self._broadcast_callback:
+            try:
+                self._broadcast_callback()
+            except Exception as e:
+                logger.debug(f"Failed to trigger broadcast: {e}")
 
     # ============= Core State Queries =============
 
@@ -147,6 +164,7 @@ class FetchService:
             stage_msg = "Request and Response stages" if response_stage else "Request stage only"
             logger.info(f"Fetch interception enabled ({stage_msg})")
 
+            self._trigger_broadcast()  # Update snapshot
             return {"enabled": True, "stages": stage_msg, "paused": self.paused_count}
 
         except Exception as e:
@@ -174,6 +192,7 @@ class FetchService:
                 self.body_service.clear_cache()
 
             logger.info("Fetch interception disabled")
+            self._trigger_broadcast()  # Update snapshot
             return {"enabled": False}
 
         except Exception as e:

webtap/services/main.py

@@ -12,6 +12,7 @@ from webtap.services.network import NetworkService
 from webtap.services.console import ConsoleService
 from webtap.services.body import BodyService
 from webtap.services.dom import DOMService
+from webtap.services.state_snapshot import StateSnapshot
 
 
 REQUIRED_DOMAINS = [
@@ -47,8 +48,11 @@ class WebTapService:
         Args:
             state: WebTapState instance from app.py
         """
+        import threading
+
         self.state = state
         self.cdp = state.cdp
+        self._state_lock = threading.RLock()  # Reentrant lock - safe to acquire multiple times by same thread
 
         self.enabled_domains: set[str] = set()
         self.filters = FilterManager()
@@ -65,8 +69,10 @@ class WebTapService:
         self.body.cdp = self.cdp
         self.dom.set_cdp(self.cdp)
         self.dom.set_state(self.state)
+        self.dom.set_broadcast_callback(self._trigger_broadcast)  # DOM calls back for snapshot updates
 
         self.fetch.body_service = self.body
+        self.fetch.set_broadcast_callback(self._trigger_broadcast)  # Fetch calls back for snapshot updates
 
         # Legacy wiring for CDP event handler
         self.cdp.fetch_service = self.fetch
@@ -75,6 +81,127 @@ class WebTapService:
         self.cdp.register_event_callback("Overlay.inspectNodeRequested", self.dom.handle_inspect_node_requested)
         self.cdp.register_event_callback("Page.frameNavigated", self.dom.handle_frame_navigated)
 
+        # Register disconnect callback for unexpected disconnects
+        self.cdp.set_disconnect_callback(self._handle_unexpected_disconnect)
+
+        # Broadcast queue for SSE state updates (set by API server)
+        self._broadcast_queue: "Any | None" = None
+
+        # Immutable state snapshot for thread-safe SSE reads
+        # Updated atomically on every state change, read without locks
+        self._state_snapshot: StateSnapshot = StateSnapshot.create_empty()
+
+    def set_broadcast_queue(self, queue: "Any") -> None:
+        """Set queue for broadcasting state changes.
+
+        Args:
+            queue: asyncio.Queue for thread-safe signaling
+        """
+        self._broadcast_queue = queue
+
+    def _create_snapshot(self) -> StateSnapshot:
+        """Create immutable state snapshot from current state.
+
+        MUST be called with self._state_lock held to ensure atomic read.
+
+        Returns:
+            Frozen StateSnapshot with current state
+        """
+        # Connection state (read page_info first to avoid race with disconnect)
+        page_info = self.cdp.page_info
+        connected = self.cdp.is_connected and page_info is not None
+        page_id = page_info.get("id", "") if page_info else ""
+        page_title = page_info.get("title", "") if page_info else ""
+        page_url = page_info.get("url", "") if page_info else ""
+
+        # Event count
+        event_count = self.event_count
+
+        # Fetch state
+        fetch_enabled = self.fetch.enabled
+        paused_count = self.fetch.paused_count if fetch_enabled else 0
+
+        # Filter state (convert to immutable tuples)
+        fm = self.filters
+        filter_categories = list(fm.filters.keys())
+        enabled_filters = tuple(fm.enabled_categories)
+        disabled_filters = tuple(cat for cat in filter_categories if cat not in enabled_filters)
+
+        # Browser/DOM state (get_state() is already thread-safe internally)
+        browser_state = self.dom.get_state()
+
+        # Error state
+        error = self.state.error_state
+        error_message = error.get("message") if error else None
+        error_timestamp = error.get("timestamp") if error else None
+
+        # Deep copy selections to ensure true immutability
+        import copy
+
+        selections = copy.deepcopy(browser_state["selections"])
+
+        return StateSnapshot(
+            connected=connected,
+            page_id=page_id,
+            page_title=page_title,
+            page_url=page_url,
+            event_count=event_count,
+            fetch_enabled=fetch_enabled,
+            paused_count=paused_count,
+            enabled_filters=enabled_filters,
+            disabled_filters=disabled_filters,
+            inspect_active=browser_state["inspect_active"],
+            selections=selections,  # Deep copy ensures nested dicts are immutable
+            prompt=browser_state["prompt"],
+            pending_count=browser_state["pending_count"],
+            error_message=error_message,
+            error_timestamp=error_timestamp,
+        )
+
+    def _trigger_broadcast(self) -> None:
+        """Trigger SSE broadcast with updated state snapshot (thread-safe).
+
+        Called after service mutations to:
+        1. Create fresh immutable snapshot (atomic replacement)
+        2. Signal SSE clients to broadcast
+
+        Uses RLock so same thread can call multiple times safely.
+        asyncio.Queue.put_nowait() is thread-safe for cross-thread communication.
+        """
+        import logging
+
+        logger = logging.getLogger(__name__)
+
+        # Update snapshot atomically
+        # RLock allows same thread to acquire multiple times, blocks other threads
+        try:
+            with self._state_lock:
+                self._state_snapshot = self._create_snapshot()
+        except (TypeError, AttributeError) as e:
+            # Programming errors should propagate for debugging
+            logger.error(f"Programming error in snapshot creation: {e}")
+            raise
+        except Exception as e:
+            # Unexpected errors logged but don't crash the app
+            logger.error(f"Failed to create state snapshot: {e}", exc_info=True)
+            return  # Don't signal broadcast if snapshot creation failed
+
+        # Signal broadcast (store reference to avoid TOCTOU race)
+        queue = self._broadcast_queue
+        if queue:
+            try:
+                queue.put_nowait({"type": "state_change"})
+            except Exception as e:
+                logger.warning(f"Failed to queue broadcast: {e}")
+
+    def get_state_snapshot(self) -> StateSnapshot:
+        """Get current immutable state snapshot (thread-safe, no locks).
+
+        Returns:
+            Current StateSnapshot - immutable, safe to read from any thread
+        """
+        return self._state_snapshot
+
     @property
     def event_count(self) -> int:
         """Total count of all CDP events stored."""
@@ -105,6 +232,7 @@ class WebTapService:
             self.filters.load()
 
             page_info = self.cdp.page_info or {}
+            self._trigger_broadcast()
             return {"connected": True, "title": page_info.get("title", "Untitled"), "url": page_info.get("url", "")}
         except Exception as e:
             return {"error": str(e)}
@@ -118,6 +246,7 @@ class WebTapService:
 
         self.body.clear_cache()
         self.dom.clear_selections()
+        self.dom.cleanup()  # Shutdown executor properly
 
         # Clear error state on disconnect
         if self.state.error_state:
@@ -126,6 +255,7 @@ class WebTapService:
         self.cdp.disconnect()
         self.enabled_domains.clear()
 
+        self._trigger_broadcast()
         return {"disconnected": True, "was_connected": was_connected}
 
     def enable_domains(self, domains: list[str]) -> dict[str, str]:
@@ -175,6 +305,7 @@ class WebTapService:
     def clear_events(self) -> dict[str, Any]:
         """Clear all stored CDP events."""
         self.cdp.clear_events()
+        self._trigger_broadcast()
         return {"cleared": True, "events": 0}
 
     def list_pages(self) -> dict[str, Any]:
@@ -187,3 +318,64 @@ class WebTapService:
             return {"pages": pages}
         except Exception as e:
             return {"error": str(e), "pages": []}
+
+    def _handle_unexpected_disconnect(self, code: int, reason: str) -> None:
+        """Handle unexpected WebSocket disconnect (tab closed, crashed, etc).
+
+        Called from background thread by CDPSession._on_close.
+        Performs service-level cleanup and notifies SSE clients.
+        Events are preserved for debugging.
+
+        Args:
+            code: WebSocket close code (e.g., 1006 = abnormal closure)
+            reason: Human-readable close reason
+        """
+        import logging
+        import time
+
+        logger = logging.getLogger(__name__)
+
+        # Map WebSocket close codes to user-friendly messages
+        reason_map = {
+            1000: "Page closed normally",
+            1001: "Browser tab closed",
+            1006: "Connection lost (tab crashed or browser closed)",
+            1011: "Chrome internal error",
+        }
+
+        # Handle None code (abnormal closure with no code)
+        if code is None:
+            user_reason = "Connection lost (page closed or crashed)"
+        else:
+            user_reason = reason_map.get(code, f"Connection closed unexpectedly (code {code})")
+
+        logger.warning(f"Unexpected disconnect: {user_reason}")
+
+        try:
+            # Thread-safe state cleanup (called from background thread)
+            with self._state_lock:
+                # Clean up service state (no CDP calls - connection already gone)
+                if self.fetch.enabled:
+                    self.fetch.enabled = False  # Direct state update, no CDP disable
+
+                self.body.clear_cache()
+                self.dom.clear_selections()
+
+                # Events preserved for debugging - use Clear button to remove explicitly
+                # DB thread and field_paths persist for reconnection
+
+                # Set error state with disconnect info
+                self.state.error_state = {"message": user_reason, "timestamp": time.time()}
+
+                self.enabled_domains.clear()
+
+            # Cleanup outside lock (safe to call multiple times, has internal protection)
+            self.dom.cleanup()  # Shutdown executor
+
+            # Notify SSE clients
+            self._trigger_broadcast()
+
+            logger.info("Unexpected disconnect cleanup completed")
+
+        except Exception as e:
+            logger.error(f"Error during unexpected disconnect cleanup: {e}")

webtap/services/setup/extension.py

@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
 
 # GitHub URLs for extension files
 EXTENSION_BASE_URL = "https://raw.githubusercontent.com/angelsen/tap-tools/main/packages/webtap/extension"
-EXTENSION_FILES = ["manifest.json", "content.js", "sidepanel.html", "sidepanel.js"]
+EXTENSION_FILES = ["manifest.json", "background.js", "content.js", "sidepanel.html", "sidepanel.js"]
 
 
 class ExtensionSetupService: