webtap-tool 0.11.0__py3-none-any.whl

webtap/api/sse.py

@@ -0,0 +1,182 @@
+"""SSE streaming and broadcast management.
+
+PUBLIC API:
+  - router: FastAPI router with SSE endpoints
+  - get_broadcast_queue: Get broadcast queue for service
+  - set_broadcast_ready_event: Set ready event signal
+  - broadcast_processor: Background task for coalesced broadcasts
+"""
+
+import asyncio
+import json as json_module
+import logging
+from typing import Any, Dict
+
+from fastapi import APIRouter
+from fastapi.responses import StreamingResponse
+
+import webtap.api.app as app_module
+from webtap.api.state import get_full_state
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+# SSE client management
+_sse_clients: set[asyncio.Queue] = set()
+_sse_clients_lock = asyncio.Lock()
+_broadcast_queue: asyncio.Queue[Dict[str, Any]] | None = None
+_broadcast_ready_event: asyncio.Event | None = None
+
+
+def set_broadcast_ready_event(event: asyncio.Event) -> None:
+    """Set the event to signal when broadcast processor is ready."""
+    global _broadcast_ready_event
+    _broadcast_ready_event = event
+
+
+@router.get("/events/stream")
+async def stream_events():
+    """Server-Sent Events stream for real-time WebTap state updates.
+
+    Streams full state object on every change. Extension receives:
+    - Connection status
+    - Event counts
+    - Fetch interception status
+    - Filter status
+    - Element selection state (inspect_active, selections)
+
+    Returns:
+        StreamingResponse with text/event-stream content type
+    """
+
+    async def event_generator():
+        """Generate SSE events with full state."""
+        queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue(maxsize=100)
+
+        async with _sse_clients_lock:
+            _sse_clients.add(queue)
+
+        try:
+            # Send initial state on connect
+            initial_state = get_full_state()
+            yield f"data: {json_module.dumps(initial_state)}\n\n"
+
+            # Stream state updates with keepalive
+            while True:
+                try:
+                    state = await asyncio.wait_for(queue.get(), timeout=30.0)
+                    if state is None:  # Shutdown signal
+                        break
+                    yield f"data: {json_module.dumps(state)}\n\n"
+                except asyncio.TimeoutError:
+                    # Send keepalive comment
+                    yield ": keepalive\n\n"
+
+        except asyncio.CancelledError:
+            # Expected during shutdown
+            pass
+        except Exception as e:
+            logger.debug(f"SSE stream error: {e}")
+        finally:
+            async with _sse_clients_lock:
+                _sse_clients.discard(queue)
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",  # Disable nginx buffering
+            "Connection": "keep-alive",
+        },
+    )
+
+
+async def broadcast_state():
+    """Broadcast current state to all SSE clients."""
+    global _sse_clients
+
+    async with _sse_clients_lock:
+        if not _sse_clients:
+            return
+        clients = list(_sse_clients)
+
+    state = get_full_state()
+    dead_queues = set()
+
+    # Send to all connected clients
+    for queue in clients:
+        try:
+            queue.put_nowait(state)
+        except asyncio.QueueFull:
+            # Client is falling behind - discard oldest state and retry with latest
+            logger.warning(f"SSE client queue full ({queue.qsize()}/{queue.maxsize}), discarding oldest state")
+            try:
+                queue.get_nowait()  # Discard oldest
+                queue.put_nowait(state)  # Retry with latest
+            except Exception as retry_err:
+                logger.debug(f"Failed to recover full queue: {retry_err}")
+                dead_queues.add(queue)
+        except Exception as e:
+            logger.debug(f"Failed to broadcast to client: {e}")
+            dead_queues.add(queue)
+
+    # Remove dead queues
+    if dead_queues:
+        async with _sse_clients_lock:
+            _sse_clients -= dead_queues
+
+
+async def broadcast_processor():
+    """Background task that processes broadcast queue.
+
+    This runs in the FastAPI event loop and watches for signals
+    from WebSocket thread (via asyncio.Queue).
+    """
+    global _broadcast_queue
+    _broadcast_queue = asyncio.Queue()
+
+    # Signal that processor is ready
+    if _broadcast_ready_event:
+        _broadcast_ready_event.set()
+
+    logger.debug("Broadcast processor started")
+
+    try:
+        while True:
+            try:
+                # Wait for broadcast signal (with timeout for keepalive)
+                signal = await asyncio.wait_for(_broadcast_queue.get(), timeout=1.0)
+                logger.debug(f"Broadcast signal received: {signal}")
+
+                # Broadcast to all SSE clients
+                await broadcast_state()
+
+                # Clear pending flag to allow next broadcast (service owns coalescing)
+                if app_module.app_state and app_module.app_state.service:
+                    app_module.app_state.service.clear_broadcast_pending()
+            except asyncio.TimeoutError:
+                continue  # Normal timeout, continue loop
+            except asyncio.CancelledError:
+                raise  # Propagate cancellation
+            except Exception as e:
+                logger.error(f"Error in broadcast processor: {e}")
+    finally:
+        # Graceful shutdown: close all SSE clients
+        async with _sse_clients_lock:
+            for queue in list(_sse_clients):
+                try:
+                    queue.put_nowait(None)  # Non-blocking shutdown signal
+                except asyncio.QueueFull:
+                    pass  # Client is hung, skip
+                except Exception:
+                    pass
+            _sse_clients.clear()
+
+        logger.debug("Broadcast processor stopped")
+
+
+def get_broadcast_queue() -> asyncio.Queue | None:
+    """Get broadcast queue for wiring to service."""
+    return _broadcast_queue

webtap/api/state.py

@@ -0,0 +1,89 @@
+"""State management for SSE broadcasting."""
+
+import hashlib
+from typing import Any, Dict
+
+import webtap.api.app as app_module
+
+
+def _stable_hash(data: str) -> str:
+    """Generate deterministic hash for frontend change detection.
+
+    Uses MD5 for speed (not security). Returns 16-char hex digest.
+    Ensures hashes remain stable across process restarts (unlike Python's hash()).
+    """
+    return hashlib.md5(data.encode()).hexdigest()[:16]
+
+
+def get_full_state() -> Dict[str, Any]:
+    """Get complete WebTap state for broadcasting.
+
+    Thread-safe, zero-lock reads from immutable snapshot.
+    No blocking I/O - returns cached snapshot immediately.
+
+    Returns:
+        Dictionary with all state information for SSE clients
+    """
+    if not app_module.app_state:
+        return {
+            "connectionState": "disconnected",
+            "epoch": 0,
+            "connected": False,
+            "events": {"total": 0},
+            "fetch": {"enabled": False, "paused_count": 0},
+            "filters": {"enabled": [], "disabled": []},
+            "browser": {"inspect_active": False, "selections": {}, "prompt": "", "pending_count": 0},
+            "error": None,
+        }
+
+    # Get immutable snapshot (NO LOCKS NEEDED - inherently thread-safe)
+    snapshot = app_module.app_state.service.get_state_snapshot()
+
+    # Get connection state and epoch from RPC machine
+    machine = app_module.app_state.service.rpc.machine if app_module.app_state.service.rpc else None
+    connection_state = machine.state if machine else "disconnected"
+    epoch = machine.epoch if machine else 0
+
+    # Compute content hashes for frontend change detection
+    # Only computed here when building SSE response (not on every state change)
+    selections_hash = _stable_hash(str(sorted(snapshot.selections.keys())))
+    filters_hash = _stable_hash(f"{sorted(snapshot.enabled_filters)}")
+    fetch_hash = _stable_hash(f"{snapshot.fetch_enabled}:{snapshot.response_stage}:{snapshot.paused_count}")
+    page_hash = _stable_hash(f"{snapshot.connected}:{snapshot.page_id}")
+    error_hash = _stable_hash(snapshot.error_message) if snapshot.error_message else ""
+
+    # Convert snapshot to frontend format
+    return {
+        "connectionState": connection_state,
+        "epoch": epoch,
+        "connected": snapshot.connected,
+        "page": {
+            "id": snapshot.page_id,
+            "title": snapshot.page_title,
+            "url": snapshot.page_url,
+        }
+        if snapshot.connected
+        else None,
+        "events": {"total": snapshot.event_count},
+        "fetch": {
+            "enabled": snapshot.fetch_enabled,
+            "response_stage": snapshot.response_stage,
+            "paused_count": snapshot.paused_count,
+        },
+        "filters": {"enabled": list(snapshot.enabled_filters), "disabled": list(snapshot.disabled_filters)},
+        "browser": {
+            "inspect_active": snapshot.inspect_active,
+            "selections": snapshot.selections,
+            "prompt": snapshot.prompt,
+            "pending_count": snapshot.pending_count,
+        },
+        "error": {"message": snapshot.error_message, "timestamp": snapshot.error_timestamp}
+        if snapshot.error_message
+        else None,
+        # Content hashes for efficient change detection
+        "selections_hash": selections_hash,
+        "filters_hash": filters_hash,
+        "fetch_hash": fetch_hash,
+        "page_hash": page_hash,
+        "error_hash": error_hash,
+    }

webtap/app.py

@@ -0,0 +1,79 @@
+"""Main application entry point for WebTap browser debugger.
+
+PUBLIC API:
+  - WebTapState: Application state class with daemon client
+  - app: Main ReplKit2 App instance (imported by commands and __init__)
+"""
+
+import sys
+from dataclasses import dataclass, field
+
+from replkit2 import App
+
+from webtap.client import RPCClient
+
+
+@dataclass
+class WebTapState:
+    """Application state for WebTap browser debugging.
+
+    Client-side state that communicates with the daemon via HTTP.
+    All CDP operations and data storage happen in the daemon.
+
+    Attributes:
+        client: RPCClient for JSON-RPC communication with daemon
+    """
+
+    client: RPCClient = field(init=False)
+
+    def __post_init__(self):
+        """Initialize RPC client after dataclass init."""
+        self.client = RPCClient()
+
+    def cleanup(self):
+        """Cleanup resources on exit."""
+        if hasattr(self, "client") and self.client:
+            self.client.close()
+
+
+# Must be created before command imports for decorator registration
+app = App(
+    "webtap",
+    WebTapState,
+    mcp_config={
+        "uri_scheme": "webtap",
+        "instructions": "Chrome DevTools Protocol debugger",
+    },
+    typer_config={
+        "add_completion": False,  # Hide shell completion options
+        "help": "WebTap - Chrome DevTools Protocol CLI",
+    },
+)
+
+# Command imports trigger @app.command decorator registration
+if "--cli" in sys.argv:
+    # Only import CLI-compatible commands (no dict/list parameters)
+    from webtap.commands import setup  # noqa: E402, F401
+    from webtap.commands import launch  # noqa: E402, F401
+else:
+    # Import all commands for REPL/MCP mode
+    from webtap.commands import connection  # noqa: E402, F401
+    from webtap.commands import navigation  # noqa: E402, F401
+    from webtap.commands import javascript  # noqa: E402, F401
+    from webtap.commands import network  # noqa: E402, F401
+    from webtap.commands import request  # noqa: E402, F401
+    from webtap.commands import console  # noqa: E402, F401
+    from webtap.commands import filters  # noqa: E402, F401
+    from webtap.commands import fetch  # noqa: E402, F401
+    from webtap.commands import to_model  # noqa: E402, F401
+    from webtap.commands import quicktype  # noqa: E402, F401
+    from webtap.commands import js_export  # noqa: E402, F401
+    from webtap.commands import selections  # noqa: E402, F401
+    from webtap.commands import setup  # noqa: E402, F401
+    from webtap.commands import launch  # noqa: E402, F401
+
+
+# Entry point is in __init__.py:main() as specified in pyproject.toml
+
+
+__all__ = ["WebTapState", "app"]

webtap/cdp/README.md

@@ -0,0 +1,275 @@
+# Chrome DevTools Protocol (CDP) Integration
+
+This module handles the core CDP connection and event management for WebTap.
+
+## Overview
+
+The CDP module provides:
+- WebSocket connection to Chrome's debugging port
+- Event capture and storage in DuckDB
+- Dynamic field discovery for flexible querying
+- Native event storage (no transformation)
+
+## Architecture
+
+```
+Chrome Browser
+    ↓ (WebSocket)
+CDPSession (session.py)
+    ├── WebSocketApp (connection management)
+    ├── DuckDB (event storage + HAR views)
+    └── Method-indexed events for O(1) filtering
+         ↓
+WebTap Commands (via RPCClient JSON-RPC 2.0)
+```
+
+## Core Components
+
+### session.py
+The main CDP session manager:
+- Establishes WebSocket connection
+- Stores events as-is in DuckDB
+- Discovers field paths dynamically
+- Handles CDP command execution
+
+### har.py
+HAR view aggregation:
+- Pre-aggregated network request views (`har_entries`, `har_summary`)
+- Efficient querying without JSON extraction at query time
+- Request/response correlation by requestId
+
+### schema/
+CDP protocol reference:
+- Protocol version information
+- Domain definitions (future)
+
+## Philosophy: Native Storage
+
+We store CDP events exactly as received:
+
+```python
+# CDP sends this
+{
+    "method": "Network.responseReceived",
+    "params": {
+        "requestId": "123.456",
+        "response": {
+            "status": 200,
+            "headers": {...}
+        }
+    }
+}
+
+# We store it as-is in DuckDB
+# No transformation, no data loss
+```
+
+## Event Domains
+
+Currently capturing events from:
+
+### Network Domain
+- `Network.requestWillBeSent`
+- `Network.responseReceived`
+- `Network.loadingFinished`
+- `Network.loadingFailed`
+
+### Page Domain
+- `Page.frameNavigated`
+- `Page.domContentEventFired`
+- `Page.loadEventFired`
+
+### Runtime Domain
+- `Runtime.consoleAPICalled`
+- `Runtime.exceptionThrown`
+
+### Fetch Domain
+- `Fetch.requestPaused`
+- `Fetch.authRequired`
+
+### Storage Domain
+- `Storage.cookiesChanged`
+- `Storage.cacheStorageContentUpdated`
+
+## Database Schema
+
+### events table
+```sql
+CREATE TABLE events (
+    rowid INTEGER PRIMARY KEY,
+    method VARCHAR,           -- Indexed for O(1) event type filtering
+    event JSON,
+    timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+)
+CREATE INDEX idx_events_method ON events(method)
+```
+
+### HAR views (pre-aggregated)
+```sql
+-- har_entries: Full request/response pairs
+-- har_summary: Minimal data for table display (id, method, url, status, type, size)
+```
+
+### Query Examples
+
+```sql
+-- Find all 404 responses
+SELECT * FROM events 
+WHERE json_extract_string(event, '$.params.response.status') = '404'
+
+-- Get request/response pairs
+SELECT 
+    e1.rowid as request_row,
+    e2.rowid as response_row,
+    json_extract_string(e1.event, '$.params.request.url') as url
+FROM events e1
+JOIN events e2 ON 
+    json_extract_string(e1.event, '$.params.requestId') = 
+    json_extract_string(e2.event, '$.params.requestId')
+WHERE 
+    json_extract_string(e1.event, '$.method') = 'Network.requestWillBeSent'
+    AND json_extract_string(e2.event, '$.method') = 'Network.responseReceived'
+```
+
+## Field Discovery
+
+The system automatically discovers all field paths:
+
+```python
+# When we see this event:
+{
+    "method": "Network.responseReceived",
+    "params": {
+        "response": {
+            "status": 200,
+            "url": "https://example.com"
+        }
+    }
+}
+
+# We discover these paths:
+# - method
+# - params.response.status
+# - params.response.url
+
+# Users can query via commands:
+network(status=200)  # Filter by HTTP status
+network(url="*example*")  # Filter by URL pattern
+request(123, ["response.*"])  # Get specific request details
+```
+
+## Connection Management
+
+### Initialization
+```python
+cdp = CDPSession()
+await cdp.connect("localhost", 9222, page_id)
+```
+
+### Event Flow
+1. Chrome sends event over WebSocket
+2. CDPSession receives in `on_message()`
+3. Event stored in DuckDB immediately
+4. Field paths extracted for discovery
+5. Event available for querying
+
+## CDP Command Execution
+
+Direct command execution:
+```python
+# Get response body
+result = cdp.execute("Network.getResponseBody", {
+    "requestId": "123.456"
+})
+
+# Evaluate JavaScript
+result = cdp.execute("Runtime.evaluate", {
+    "expression": "document.title"
+})
+```
+
+## Performance Considerations
+
+- **Minimal Processing**: Events stored as-is
+- **Lazy Evaluation**: Field discovery on-demand
+- **Efficient Storage**: DuckDB's columnar format
+- **Fast Queries**: JSON functions optimized in DuckDB
+
+## Extension Points
+
+### Adding New Domains
+To capture events from additional CDP domains:
+
+1. Enable the domain:
+```python
+cdp.execute("DOMStorage.enable")
+```
+
+2. Events automatically captured and stored with indexed `method` column
+
+3. Query via SQL or implement a new command:
+```python
+# Events are stored with method for fast filtering
+SELECT * FROM events WHERE method LIKE 'DOMStorage.%'
+```
+
+### Custom Event Processing
+While we store events as-is, you can add custom processors:
+
+```python
+def process_network_event(event):
+    # Custom logic here
+    pass
+
+# Register processor
+cdp.register_processor("Network.*", process_network_event)
+```
+
+## Integration with SDP
+
+The CDP module will work alongside the future SDP (Svelte Debug Protocol) module:
+
+```
+CDP Events (Network, DOM, Console)
+    +
+SDP Events (State, Components, Reactivity)
+    ↓
+Unified Event Stream in DuckDB
+    ↓
+Correlated Analysis
+```
+
+## Best Practices
+
+1. **Don't Transform**: Store CDP data as-is
+2. **Query Don't Parse**: Use SQL for extraction
+3. **Discover Don't Define**: Let field paths emerge
+4. **Correlate Don't Duplicate**: Link events by IDs
+
+## Debugging
+
+### Enable verbose logging
+```python
+import logging
+logging.basicConfig(level=logging.DEBUG)
+```
+
+### Check connection
+```python
+cdp.connected  # Should be True
+cdp.ws.sock.connected  # WebSocket status
+```
+
+### Inspect stored events
+```python
+cdp.query("SELECT COUNT(*) FROM events")
+cdp.query("SELECT * FROM events ORDER BY rowid DESC LIMIT 5")
+```
+
+## Future Enhancements
+
+- [ ] Event compression for long sessions
+- [ ] Streaming to external storage
+- [ ] Real-time event subscriptions
+- [ ] Custom domain definitions
+- [ ] Event replay functionality

webtap/cdp/__init__.py

@@ -0,0 +1,12 @@
+"""Chrome DevTools Protocol client with native event storage.
+
+Native CDP approach - store events as-is, query on-demand.
+Built on WebSocketApp + DuckDB for minimal overhead.
+
+PUBLIC API:
+  - CDPSession: Main CDP client with WebSocket connection and event storage
+"""
+
+from webtap.cdp.session import CDPSession
+
+__all__ = ["CDPSession"]