webtap-tool 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

webtap/__init__.py

@@ -9,10 +9,14 @@ PUBLIC API:
   - main: Entry point function for CLI
 """
 
+import atexit
 import sys
 
 from webtap.app import app
 
+# Register cleanup on exit to shutdown DB thread
+atexit.register(lambda: app.state.cleanup() if hasattr(app, "state") and app.state else None)
+
 
 def main():
     """Entry point for the WebTap REPL.

webtap/api.py

@@ -119,9 +119,6 @@ async def connect(request: ConnectRequest) -> Dict[str, Any]:
     # Wrap blocking CDP calls (connect + enable domains) in thread
     result = await asyncio.to_thread(app_state.service.connect_to_page, page_id=request.page_id)
 
-    # Broadcast state change
-    await broadcast_state()
-
     return result
 
 
@@ -134,9 +131,6 @@ async def disconnect() -> Dict[str, Any]:
     # Wrap blocking CDP calls (fetch.disable + disconnect) in thread
     result = await asyncio.to_thread(app_state.service.disconnect)
 
-    # Broadcast state change
-    await broadcast_state()
-
     return result
 
 
@@ -146,7 +140,10 @@ async def clear_events() -> Dict[str, Any]:
     if not app_state:
         return {"error": "WebTap not initialized"}
 
-    return app_state.service.clear_events()
+    # Wrap blocking DB operation in thread
+    result = await asyncio.to_thread(app_state.service.clear_events)
+
+    return result
 
 
 @api.post("/fetch")
@@ -164,7 +161,7 @@ async def set_fetch_interception(request: FetchRequest) -> Dict[str, Any]:
         result = await asyncio.to_thread(app_state.service.fetch.disable)
 
     # Broadcast state change
-    await broadcast_state()
+    app_state.service._trigger_broadcast()
 
     return result
 
@@ -200,7 +197,7 @@ async def toggle_filter_category(category: str) -> Dict[str, Any]:
     fm.save()
 
     # Broadcast state change
-    await broadcast_state()
+    app_state.service._trigger_broadcast()
 
     return {"category": category, "enabled": enabled, "total_enabled": len(fm.enabled_categories)}
 
@@ -216,7 +213,7 @@ async def enable_all_filters() -> Dict[str, Any]:
     fm.save()
 
     # Broadcast state change
-    await broadcast_state()
+    app_state.service._trigger_broadcast()
 
     return {"enabled": list(fm.enabled_categories), "total": len(fm.enabled_categories)}
 
@@ -232,7 +229,7 @@ async def disable_all_filters() -> Dict[str, Any]:
     fm.save()
 
     # Broadcast state change
-    await broadcast_state()
+    app_state.service._trigger_broadcast()
 
     return {"enabled": [], "total": 0}
 
@@ -249,9 +246,6 @@ async def start_inspect() -> Dict[str, Any]:
     # Wrap blocking CDP calls (DOM.enable, CSS.enable, Overlay.enable, setInspectMode) in thread
     result = await asyncio.to_thread(app_state.service.dom.start_inspect)
 
-    # Broadcast state change
-    await broadcast_state()
-
     return result
 
 
@@ -264,9 +258,6 @@ async def stop_inspect() -> Dict[str, Any]:
     # Wrap blocking CDP call (Overlay.setInspectMode) in thread
     result = await asyncio.to_thread(app_state.service.dom.stop_inspect)
 
-    # Broadcast state change
-    await broadcast_state()
-
     return result
 
 
@@ -278,9 +269,6 @@ async def clear_selections() -> Dict[str, Any]:
 
     app_state.service.dom.clear_selections()
 
-    # Broadcast state change
-    await broadcast_state()
-
     return {"success": True, "selections": {}}
 
 
@@ -293,7 +281,7 @@ async def dismiss_error() -> Dict[str, Any]:
     app_state.error_state = None
 
     # Broadcast state change
-    await broadcast_state()
+    app_state.service._trigger_broadcast()
 
     return {"success": True}
 
@@ -363,11 +351,11 @@ async def stream_events():
 def get_full_state() -> Dict[str, Any]:
     """Get complete WebTap state for broadcasting.
 
-    Returns real-time state only (no blocking I/O).
-    Page list excluded - fetch via /info endpoint on-demand.
+    Thread-safe, zero-lock reads from immutable snapshot.
+    No blocking I/O - returns cached snapshot immediately.
 
     Returns:
-        Dictionary with all state information
+        Dictionary with all state information for SSE clients
     """
     if not app_state:
         return {
@@ -375,40 +363,35 @@ def get_full_state() -> Dict[str, Any]:
             "events": {"total": 0},
             "fetch": {"enabled": False, "paused_count": 0},
             "filters": {"enabled": [], "disabled": []},
-            "browser": {"inspect_active": False, "selections": {}, "prompt": ""},
+            "browser": {"inspect_active": False, "selections": {}, "prompt": "", "pending_count": 0},
             "error": None,
         }
 
-    # Get connection status
-    connected = app_state.cdp.is_connected
-    page_info = app_state.cdp.page_info or {}
-
-    # Get event counts
-    event_count = app_state.service.event_count
-
-    # Get fetch status
-    fetch_enabled = app_state.service.fetch.enabled
-    paused_count = app_state.service.fetch.paused_count if fetch_enabled else 0
-
-    # Get filter status
-    fm = app_state.service.filters
-    filter_categories = list(fm.filters.keys())
-    enabled_filters = list(fm.enabled_categories)
-    disabled_filters = [cat for cat in filter_categories if cat not in enabled_filters]
-
-    # Get browser/DOM state (includes pending_count for progress indicator)
-    browser_state = app_state.service.dom.get_state()
+    # Get immutable snapshot (NO LOCKS NEEDED - inherently thread-safe)
+    snapshot = app_state.service.get_state_snapshot()
 
+    # Convert snapshot to frontend format
     return {
-        "connected": connected,
-        "page": {"id": page_info.get("id", ""), "title": page_info.get("title", ""), "url": page_info.get("url", "")}
-        if connected
+        "connected": snapshot.connected,
+        "page": {
+            "id": snapshot.page_id,
+            "title": snapshot.page_title,
+            "url": snapshot.page_url,
+        }
+        if snapshot.connected
+        else None,
+        "events": {"total": snapshot.event_count},
+        "fetch": {"enabled": snapshot.fetch_enabled, "paused_count": snapshot.paused_count},
+        "filters": {"enabled": list(snapshot.enabled_filters), "disabled": list(snapshot.disabled_filters)},
+        "browser": {
+            "inspect_active": snapshot.inspect_active,
+            "selections": snapshot.selections,
+            "prompt": snapshot.prompt,
+            "pending_count": snapshot.pending_count,
+        },
+        "error": {"message": snapshot.error_message, "timestamp": snapshot.error_timestamp}
+        if snapshot.error_message
         else None,
-        "events": {"total": event_count},
-        "fetch": {"enabled": fetch_enabled, "paused_count": paused_count},
-        "filters": {"enabled": enabled_filters, "disabled": disabled_filters},
-        "browser": browser_state,  # Contains inspect_active, selections, prompt, pending_count
-        "error": app_state.error_state,  # Current error or None
     }
 
 
@@ -429,7 +412,14 @@ async def broadcast_state():
         try:
             queue.put_nowait(state)
         except asyncio.QueueFull:
-            logger.warning("SSE client queue full, skipping broadcast")
+            # Client is falling behind - discard oldest state and retry with latest
+            logger.warning(f"SSE client queue full ({queue.qsize()}/{queue.maxsize}), discarding oldest state")
+            try:
+                queue.get_nowait()  # Discard oldest
+                queue.put_nowait(state)  # Retry with latest
+            except Exception as retry_err:
+                logger.debug(f"Failed to recover full queue: {retry_err}")
+                dead_queues.add(queue)
         except Exception as e:
             logger.debug(f"Failed to broadcast to client: {e}")
             dead_queues.add(queue)
@@ -470,7 +460,9 @@ async def broadcast_processor():
     async with _sse_clients_lock:
         for queue in list(_sse_clients):
             try:
-                await queue.put(None)  # Signal shutdown to client
+                queue.put_nowait(None)  # Non-blocking shutdown signal
+            except asyncio.QueueFull:
+                pass  # Client is hung, skip
             except Exception:
                 pass
         _sse_clients.clear()
@@ -521,11 +513,12 @@ def start_api_server(state, host: str = "127.0.0.1", port: int = 8765) -> thread
         logger.error("Broadcast queue initialization timed out")
         return thread
 
-    # Wire queue to DOM service and CDP session after event loop starts
+    # Wire queue to service and CDP session after event loop starts
+    # Note: DOMService uses callback to service._trigger_broadcast instead of direct queue access
     if _broadcast_queue and app_state:
-        app_state.service.dom.set_broadcast_queue(_broadcast_queue)
+        app_state.service.set_broadcast_queue(_broadcast_queue)
         app_state.cdp.set_broadcast_queue(_broadcast_queue)
-        logger.info("Broadcast queue wired to DOMService and CDPSession")
+        logger.info("Broadcast queue wired to WebTapService and CDPSession")
 
     logger.info(f"API server started on http://{host}:{port}")
     return thread

webtap/app.py

@@ -57,6 +57,10 @@ class WebTapState:
             # Give server 1.5s to close SSE connections and shutdown gracefully
             self.api_thread.join(timeout=1.5)
 
+        # Shutdown DB thread (this is the only place where DB thread should stop)
+        if hasattr(self, "cdp") and self.cdp:
+            self.cdp.cleanup()
+
 
 # Must be created before command imports for decorator registration
 app = App(
@@ -90,6 +94,7 @@ else:
     from webtap.commands import fetch  # noqa: E402, F401
     from webtap.commands import body  # noqa: E402, F401
     from webtap.commands import to_model  # noqa: E402, F401
+    from webtap.commands import quicktype  # noqa: E402, F401
     from webtap.commands import selections  # noqa: E402, F401
     from webtap.commands import server  # noqa: E402, F401
     from webtap.commands import setup  # noqa: E402, F401

webtap/cdp/session.py

@@ -6,6 +6,7 @@ PUBLIC API:
 
 import json
 import logging
+import queue
 import threading
 from concurrent.futures import Future, TimeoutError
 from typing import Any
@@ -54,9 +55,18 @@ class CDPSession:
         self._lock = threading.Lock()
 
         # DuckDB storage - store events AS-IS
+        # DuckDB connections are NOT thread-safe - use dedicated DB thread
         self.db = duckdb.connect(":memory:")
+        self._db_work_queue: queue.Queue = queue.Queue()
+        self._db_result_queues: dict[int, queue.Queue] = {}
+        self._db_running = True
 
-        self.db.execute("CREATE TABLE events (event JSON)")
+        # Start dedicated database thread
+        self._db_thread = threading.Thread(target=self._db_worker, daemon=True)
+        self._db_thread.start()
+
+        # Initialize schema via queue
+        self._db_execute("CREATE TABLE events (event JSON)", wait_result=False)
 
         # Live field path lookup for fast discovery
         # Maps lowercase field names to their full paths with original case
@@ -68,8 +78,81 @@ class CDPSession:
 
         # Broadcast queue for SSE state updates (set by API server)
         self._broadcast_queue: "Any | None" = None
-        self._last_broadcast_time = 0.0
-        self._broadcast_debounce = 1.0  # 1 second debounce
+
+        # Disconnect callback for service-level cleanup
+        self._disconnect_callback: "Any | None" = None
+
+    def _db_worker(self) -> None:
+        """Dedicated thread for all database operations.
+
+        Ensures thread safety by serializing all DuckDB access through one thread.
+        DuckDB connections are not thread-safe - sharing them causes malloc corruption.
+        """
+        while self._db_running:
+            try:
+                task = self._db_work_queue.get(timeout=1)
+
+                if task is None:  # Shutdown signal
+                    break
+
+                operation_type, sql, params, result_queue_id = task
+
+                try:
+                    if operation_type == "execute":
+                        result = self.db.execute(sql, params or [])
+                        data = result.fetchall() if result else []
+                    elif operation_type == "delete":
+                        self.db.execute(sql, params or [])
+                        data = None
+                    else:
+                        data = None
+
+                    # Send result back if requested
+                    if result_queue_id and result_queue_id in self._db_result_queues:
+                        self._db_result_queues[result_queue_id].put(("success", data))
+
+                except Exception as e:
+                    logger.error(f"Database error: {e}")
+                    if result_queue_id and result_queue_id in self._db_result_queues:
+                        self._db_result_queues[result_queue_id].put(("error", str(e)))
+
+                finally:
+                    self._db_work_queue.task_done()
+
+            except queue.Empty:
+                continue
+
+    def _db_execute(self, sql: str, params: list | None = None, wait_result: bool = True) -> Any:
+        """Submit database operation to dedicated thread.
+
+        Args:
+            sql: SQL query or command
+            params: Optional query parameters
+            wait_result: Block until operation completes and return result
+
+        Returns:
+            Query results if wait_result=True, None otherwise
+        """
+        result_queue_id = None
+        result_queue = None
+
+        if wait_result:
+            result_queue_id = id(threading.current_thread())
+            result_queue = queue.Queue()
+            self._db_result_queues[result_queue_id] = result_queue
+
+        # Submit to work queue
+        self._db_work_queue.put(("execute", sql, params, result_queue_id))
+
+        if wait_result and result_queue and result_queue_id:
+            status, data = result_queue.get()
+            del self._db_result_queues[result_queue_id]
+
+            if status == "error":
+                raise RuntimeError(f"Database error: {data}")
+            return data
+
+        return None
 
     def list_pages(self) -> list[dict]:
         """List available Chrome pages via HTTP API.
@@ -136,7 +219,7 @@ class CDPSession:
             kwargs={
                 "ping_interval": 30,  # Ping every 30s
                 "ping_timeout": 20,  # Wait 20s for pong (increased from 10s for heavy CDP load)
-                "reconnect": 5,  # Auto-reconnect with max 5s delay
+                # No auto-reconnect - make disconnects explicit
                 "skip_utf8_validation": True,  # Faster
             },
         )
@@ -149,18 +232,46 @@ class CDPSession:
             raise TimeoutError("Failed to connect to Chrome")
 
     def disconnect(self) -> None:
-        """Disconnect WebSocket and clean up resources."""
-        if self.ws_app:
-            self.ws_app.close()
+        """Disconnect WebSocket while preserving events and DB thread.
+
+        Events and DB thread persist across connection cycles.
+        Use cleanup() on app exit to shutdown DB thread.
+        """
+        # Atomically clear ws_app to signal manual disconnect
+        # This prevents _on_close from triggering service callback
+        with self._lock:
+            ws_app = self.ws_app
             self.ws_app = None
 
+        if ws_app:
+            ws_app.close()
+
         if self.ws_thread and self.ws_thread.is_alive():
             self.ws_thread.join(timeout=2)
             self.ws_thread = None
 
+        # Keep DB thread running - events preserved for reconnection
+        # DB cleanup happens in cleanup() on app exit only
+
         self.connected.clear()
         self.page_info = None
 
+    def cleanup(self) -> None:
+        """Shutdown DB thread and disconnect (call on app exit only).
+
+        This is the only place where DB thread should be stopped.
+        Events are lost when DB thread stops (in-memory database).
+        """
+        # Disconnect WebSocket if connected
+        if self.ws_app:
+            self.disconnect()
+
+        # Shutdown database thread
+        self._db_running = False
+        self._db_work_queue.put(None)  # Signal shutdown
+        if self._db_thread.is_alive():
+            self._db_thread.join(timeout=2)
+
     def send(self, method: str, params: dict | None = None) -> Future:
         """Send CDP command asynchronously.
 
@@ -245,7 +356,7 @@ class CDPSession:
 
             # CDP event - store AS-IS in DuckDB and update field lookup
             elif "method" in data:
-                self.db.execute("INSERT INTO events VALUES (?)", [json.dumps(data)])
+                self._db_execute("INSERT INTO events VALUES (?)", [json.dumps(data)], wait_result=False)
                 self._update_field_lookup(data)
 
                 # Call registered event callbacks
@@ -263,15 +374,39 @@ class CDPSession:
 
     def _on_close(self, ws, code, reason):
         """Handle WebSocket closure and cleanup."""
-        logger.info(f"WebSocket closed: {code} {reason}")
+        logger.info(f"WebSocket closed: code={code} reason={reason}")
+
+        # Mark as disconnected
+        was_connected = self.connected.is_set()
         self.connected.clear()
 
-        # Fail pending commands
+        # Fail pending commands and check if this is unexpected disconnect
+        is_unexpected = False
         with self._lock:
             for future in self._pending.values():
-                future.set_exception(RuntimeError("Connection closed"))
+                future.set_exception(RuntimeError(f"Connection closed: {reason or 'Unknown'}"))
             self._pending.clear()
 
+            # Unexpected disconnect: was connected and ws_app still set (not manual disconnect)
+            is_unexpected = was_connected and self.ws_app is not None
+
+            # Clear state to allow reconnection (DB thread and events preserved)
+            self.ws_app = None
+            self.page_info = None
+
+        # Trigger service-level cleanup if this was unexpected
+        if is_unexpected and self._disconnect_callback:
+            try:
+                # Call in background to avoid blocking WebSocket thread
+                threading.Thread(
+                    target=self._disconnect_callback, args=(code, reason), daemon=True, name="cdp-disconnect-handler"
+                ).start()
+            except Exception as e:
+                logger.error(f"Error calling disconnect callback: {e}")
+
+        # Trigger SSE broadcast immediately
+        self._trigger_state_broadcast()
+
     def _extract_paths(self, obj, parent_key=""):
         """Extract all JSON paths from nested dict structure.
 
@@ -332,7 +467,7 @@ class CDPSession:
 
     def clear_events(self) -> None:
         """Clear all stored events and reset field lookup."""
-        self.db.execute("DELETE FROM events")
+        self._db_execute("DELETE FROM events", wait_result=False)
         self.field_paths.clear()
 
     def query(self, sql: str, params: list | None = None) -> list:
@@ -352,8 +487,7 @@ class CDPSession:
             query("SELECT * FROM events WHERE json_extract_string(event, '$.method') = 'Network.responseReceived'")
             query("SELECT json_extract_string(event, '$.params.request.url') as url FROM events")
         """
-        result = self.db.execute(sql, params or [])
-        return result.fetchall() if result else []
+        return self._db_execute(sql, params)
 
     def fetch_body(self, request_id: str) -> dict | None:
         """Fetch response body via Network.getResponseBody CDP call.
@@ -379,6 +513,18 @@ class CDPSession:
         """
         return self.connected.is_set()
 
+    def set_disconnect_callback(self, callback) -> None:
+        """Register callback for unexpected disconnect events.
+
+        Called when WebSocket closes externally (tab close, crash, etc).
+        NOT called on manual disconnect() to avoid duplicate cleanup.
+
+        Args:
+            callback: Function called with (code: int, reason: str)
+        """
+        self._disconnect_callback = callback
+        logger.debug("Disconnect callback registered")
+
     def register_event_callback(self, method: str, callback) -> None:
         """Register callback for specific CDP event.
 
@@ -445,21 +591,14 @@ class CDPSession:
         logger.debug("Broadcast queue set on CDPSession")
 
     def _trigger_state_broadcast(self) -> None:
-        """Trigger SSE broadcast with 1s debounce.
+        """Trigger SSE broadcast immediately.
 
-        Called after CDP events are stored. Debounces rapid-fire events
-        to avoid overwhelming SSE clients during heavy network activity.
+        Called after CDP events are stored. Queue naturally buffers rapid-fire events.
         """
         if not self._broadcast_queue:
             return
 
-        import time
-
-        now = time.time()
-        if now - self._last_broadcast_time > self._broadcast_debounce:
-            self._last_broadcast_time = now
-            try:
-                self._broadcast_queue.put_nowait({"type": "cdp_event"})
-                logger.debug("State broadcast triggered")
-            except Exception as e:
-                logger.debug(f"Failed to queue broadcast: {e}")
+        try:
+            self._broadcast_queue.put_nowait({"type": "cdp_event"})
+        except Exception as e:
+            logger.debug(f"Failed to queue broadcast: {e}")