webtap-tool 0.1.1__py3-none-any.whl

webtap/services/console.py

@@ -0,0 +1,116 @@
+"""Console monitoring service for browser messages."""
+
+import logging
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from webtap.cdp import CDPSession
+
+logger = logging.getLogger(__name__)
+
+
+class ConsoleService:
+    """Internal service for console event queries and monitoring."""
+
+    def __init__(self):
+        """Initialize console service."""
+        self.cdp: CDPSession | None = None
+
+    @property
+    def message_count(self) -> int:
+        """Count of all console messages."""
+        if not self.cdp:
+            return 0
+        result = self.cdp.query(
+            "SELECT COUNT(*) FROM events WHERE json_extract_string(event, '$.method') IN ('Runtime.consoleAPICalled', 'Log.entryAdded')"
+        )
+        return result[0][0] if result else 0
+
+    @property
+    def error_count(self) -> int:
+        """Count of console errors."""
+        if not self.cdp:
+            return 0
+        result = self.cdp.query("""
+            SELECT COUNT(*) FROM events 
+            WHERE json_extract_string(event, '$.method') IN ('Runtime.consoleAPICalled', 'Log.entryAdded')
+            AND (
+                json_extract_string(event, '$.params.type') = 'error' 
+                OR json_extract_string(event, '$.params.entry.level') = 'error'
+            )
+        """)
+        return result[0][0] if result else 0
+
+    def get_recent_messages(self, limit: int = 50, level: str | None = None) -> list[tuple]:
+        """Get recent console messages with common fields extracted.
+
+        Args:
+            limit: Maximum results
+            level: Optional filter by level (error, warning, log, info)
+        """
+        if not self.cdp:
+            return []
+
+        sql = """
+        SELECT 
+            rowid,
+            COALESCE(
+                json_extract_string(event, '$.params.type'),
+                json_extract_string(event, '$.params.entry.level')
+            ) as Level,
+            COALESCE(
+                json_extract_string(event, '$.params.source'),
+                json_extract_string(event, '$.params.entry.source'),
+                'console'
+            ) as Source,
+            COALESCE(
+                json_extract_string(event, '$.params.args[0].value'),
+                json_extract_string(event, '$.params.entry.text')
+            ) as Message,
+            COALESCE(
+                json_extract_string(event, '$.params.timestamp'),
+                json_extract_string(event, '$.params.entry.timestamp')
+            ) as Time
+        FROM events 
+        WHERE json_extract_string(event, '$.method') IN ('Runtime.consoleAPICalled', 'Log.entryAdded')
+        """
+
+        if level:
+            sql += f"""
+            AND (
+                json_extract_string(event, '$.params.type') = '{level.lower()}'
+                OR json_extract_string(event, '$.params.entry.level') = '{level.lower()}'
+            )
+            """
+
+        sql += f" ORDER BY rowid DESC LIMIT {limit}"
+
+        return self.cdp.query(sql)
+
+    def get_errors(self, limit: int = 20) -> list[tuple]:
+        """Get console errors only.
+
+        Args:
+            limit: Maximum results
+        """
+        return self.get_recent_messages(limit=limit, level="error")
+
+    def get_warnings(self, limit: int = 20) -> list[tuple]:
+        """Get console warnings only.
+
+        Args:
+            limit: Maximum results
+        """
+        return self.get_recent_messages(limit=limit, level="warning")
+
+    def clear_browser_console(self) -> bool:
+        """Clear console in the browser (CDP command)."""
+        if not self.cdp:
+            return False
+
+        try:
+            self.cdp.execute("Runtime.discardConsoleEntries")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to clear browser console: {e}")
+            return False

webtap/services/fetch.py

@@ -0,0 +1,397 @@
+"""Fetch interception service for request/response debugging."""
+
+import json
+import logging
+import time
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from webtap.cdp import CDPSession
+    from webtap.services.body import BodyService
+
+logger = logging.getLogger(__name__)
+
+
+class FetchService:
+    """Internal service for fetch interception with explicit actions."""
+
+    def __init__(self):
+        """Initialize fetch service."""
+        self.enabled = False
+        self.enable_response_stage = False  # Config option for future
+        self.cdp: CDPSession | None = None
+        self.body_service: BodyService | None = None
+
+    # ============= Core State Queries =============
+
+    def get_paused_list(self) -> list[dict[str, Any]]:
+        """Get list of currently paused requests for display.
+
+        Returns:
+            List with ID, Stage, Method, Status, URL for each paused request
+        """
+        if not self.cdp:
+            return []
+
+        results = self.cdp.query(
+            """
+            WITH fetch_events AS (
+                SELECT 
+                    rowid,
+                    json_extract_string(event, '$.params.requestId') as request_id,
+                    json_extract_string(event, '$.params.networkId') as network_id,
+                    json_extract_string(event, '$.params.responseStatusCode') as response_status,
+                    json_extract_string(event, '$.params.request.url') as url,
+                    json_extract_string(event, '$.params.request.method') as method,
+                    CASE WHEN json_extract_string(event, '$.params.responseStatusCode') IS NOT NULL 
+                         THEN 'Response' ELSE 'Request' END as stage
+                FROM events
+                WHERE json_extract_string(event, '$.method') = 'Fetch.requestPaused'
+            ),
+            completed_networks AS (
+                SELECT DISTINCT json_extract_string(event, '$.params.requestId') as network_id
+                FROM events
+                WHERE json_extract_string(event, '$.method') = 'Network.loadingFinished'
+            ),
+            latest_per_request AS (
+                SELECT request_id, MAX(rowid) as max_rowid
+                FROM fetch_events
+                GROUP BY request_id
+            )
+            SELECT 
+                f.rowid,
+                f.stage,
+                f.method,
+                f.response_status,
+                f.url,
+                f.network_id,
+                f.request_id
+            FROM fetch_events f
+            INNER JOIN latest_per_request l ON f.rowid = l.max_rowid
+            WHERE f.network_id NOT IN (SELECT network_id FROM completed_networks)
+            ORDER BY f.rowid DESC
+        """
+        )
+
+        return [
+            {
+                "ID": row[0],
+                "Stage": row[1],
+                "Method": row[2] or "GET",
+                "Status": row[3] or "-",
+                "URL": row[4][:60] if row[4] else "-",
+                "_network_id": row[5],
+                "_request_id": row[6],
+            }
+            for row in results
+        ]
+
+    @property
+    def paused_count(self) -> int:
+        """Count of actually paused requests (not completed)."""
+        return len(self.get_paused_list())
+
+    def get_paused_event(self, rowid: int) -> dict | None:
+        """Get full event data for a paused request.
+
+        Args:
+            rowid: Row ID from the database
+
+        Returns:
+            Full CDP event data or None if not found
+        """
+        if not self.cdp:
+            return None
+
+        result = self.cdp.query(
+            """
+            SELECT event 
+            FROM events 
+            WHERE rowid = ? 
+              AND json_extract_string(event, '$.method') = 'Fetch.requestPaused'
+        """,
+            [rowid],
+        )
+
+        if result:
+            return json.loads(result[0][0])
+        return None
+
+    # ============= Enable/Disable =============
+
+    def enable(self, cdp: "CDPSession", response_stage: bool = False) -> dict[str, Any]:
+        """Enable fetch interception.
+
+        Args:
+            cdp: CDP session for executing commands
+            response_stage: Whether to also pause at Response stage
+
+        Returns:
+            Status dict with enabled state and paused count
+        """
+        if self.enabled:
+            return {"enabled": True, "message": "Already enabled"}
+
+        self.cdp = cdp
+        self.enable_response_stage = response_stage
+
+        try:
+            patterns = [{"urlPattern": "*", "requestStage": "Request"}]
+
+            if response_stage:
+                patterns.append({"urlPattern": "*", "requestStage": "Response"})
+
+            cdp.execute("Fetch.enable", {"patterns": patterns})
+
+            self.enabled = True
+            stage_msg = "Request and Response stages" if response_stage else "Request stage only"
+            logger.info(f"Fetch interception enabled ({stage_msg})")
+
+            return {"enabled": True, "stages": stage_msg, "paused": self.paused_count}
+
+        except Exception as e:
+            logger.error(f"Failed to enable fetch: {e}")
+            return {"enabled": False, "error": str(e)}
+
+    def disable(self) -> dict[str, Any]:
+        """Disable fetch interception.
+
+        Returns:
+            Status dict with disabled state
+        """
+        if not self.enabled:
+            return {"enabled": False, "message": "Already disabled"}
+
+        if not self.cdp:
+            return {"enabled": False, "error": "No CDP session"}
+
+        try:
+            self.cdp.execute("Fetch.disable")
+            self.enabled = False
+
+            # Clear body cache when fetch is disabled
+            if self.body_service:
+                self.body_service.clear_cache()
+
+            logger.info("Fetch interception disabled")
+            return {"enabled": False}
+
+        except Exception as e:
+            logger.error(f"Failed to disable fetch: {e}")
+            return {"enabled": self.enabled, "error": str(e)}
+
+    # ============= Explicit Actions =============
+
+    def continue_request(
+        self, rowid: int, modifications: dict[str, Any] | None = None, wait_for_next: float = 0.5
+    ) -> dict[str, Any]:
+        """Continue a specific paused request.
+
+        Args:
+            rowid: Row ID from requests() table
+            modifications: Optional modifications to apply
+            wait_for_next: Time to wait for follow-up events (0 to disable)
+
+        Returns:
+            Dict with continuation status and optional next event info
+        """
+        if not self.enabled or not self.cdp:
+            return {"error": "Fetch not enabled"}
+
+        # Get the event
+        event = self.get_paused_event(rowid)
+        if not event:
+            return {"error": f"Event {rowid} not found"}
+
+        params = event["params"]
+        request_id = params["requestId"]
+        network_id = params.get("networkId")
+
+        # Determine stage and continue
+        if params.get("responseStatusCode"):
+            # Response stage
+            cdp_params = {"requestId": request_id}
+            if modifications:
+                cdp_params.update(modifications)
+            self.cdp.execute("Fetch.continueResponse", cdp_params)
+            stage = "response"
+        else:
+            # Request stage
+            cdp_params = {"requestId": request_id}
+            if modifications:
+                cdp_params.update(modifications)
+            self.cdp.execute("Fetch.continueRequest", cdp_params)
+            stage = "request"
+
+        result = {"continued": rowid, "stage": stage, "request_id": request_id}
+
+        # Wait for follow-up if requested
+        if wait_for_next > 0 and network_id:
+            next_event = self._wait_for_next_event(request_id, network_id, rowid, wait_for_next)
+            if next_event:
+                result["next_event"] = next_event
+
+        # Add remaining count
+        result["remaining"] = self.paused_count
+
+        return result
+
+    def _wait_for_next_event(
+        self, request_id: str, network_id: str, after_rowid: int, timeout: float
+    ) -> dict[str, Any] | None:
+        """Wait for the next event in the chain (response stage or redirect).
+
+        Args:
+            request_id: The request ID that was just continued
+            network_id: The network ID for tracking redirects
+            after_rowid: Row ID to search after
+            timeout: Maximum time to wait
+
+        Returns:
+            Dict with next event info or None if nothing found
+        """
+        if not self.cdp:
+            return None
+
+        start = time.time()
+
+        while time.time() - start < timeout:
+            try:
+                # Check for response stage (same requestId)
+                response = self.cdp.query(
+                    """
+                    SELECT 
+                        rowid,
+                        json_extract_string(event, '$.params.responseStatusCode') as status
+                    FROM events
+                    WHERE json_extract_string(event, '$.method') = 'Fetch.requestPaused'
+                      AND json_extract_string(event, '$.params.requestId') = ?
+                      AND json_extract_string(event, '$.params.responseStatusCode') IS NOT NULL
+                      AND rowid > ?
+                    LIMIT 1
+                """,
+                    [request_id, after_rowid],
+                )
+
+                if response and len(response) > 0:
+                    return {
+                        "rowid": response[0][0],
+                        "type": "response",
+                        "status": response[0][1],
+                        "description": f"Response stage ready (status {response[0][1]})",
+                    }
+
+                # Check for redirect (new requestId, same networkId)
+                redirect = self.cdp.query(
+                    """
+                    SELECT 
+                        rowid,
+                        json_extract_string(event, '$.params.requestId') as new_request_id,
+                        json_extract_string(event, '$.params.request.url') as url
+                    FROM events
+                    WHERE json_extract_string(event, '$.method') = 'Fetch.requestPaused'
+                      AND json_extract_string(event, '$.params.networkId') = ?
+                      AND json_extract_string(event, '$.params.redirectedRequestId') = ?
+                      AND rowid > ?
+                    LIMIT 1
+                """,
+                    [network_id, request_id, after_rowid],
+                )
+
+                if redirect and len(redirect) > 0:
+                    url = redirect[0][2]
+                    return {
+                        "rowid": redirect[0][0],
+                        "type": "redirect",
+                        "request_id": redirect[0][1],
+                        "url": url[:60] if url else None,
+                        "description": f"Redirected to {url[:40]}..." if url else "Redirected",
+                    }
+            except Exception as e:
+                logger.debug(f"Error during polling: {e}")
+                # Continue polling on transient errors
+
+            time.sleep(0.05)  # 50ms polling
+
+        return None
+
+    def fail_request(self, rowid: int, reason: str = "BlockedByClient") -> dict[str, Any]:
+        """Explicitly fail a request.
+
+        Args:
+            rowid: Row ID from requests() table
+            reason: CDP error reason
+
+        Returns:
+            Dict with failure status
+        """
+        if not self.enabled or not self.cdp:
+            return {"error": "Fetch not enabled"}
+
+        event = self.get_paused_event(rowid)
+        if not event:
+            return {"error": f"Event {rowid} not found"}
+
+        request_id = event["params"]["requestId"]
+
+        try:
+            self.cdp.execute("Fetch.failRequest", {"requestId": request_id, "errorReason": reason})
+
+            return {"failed": rowid, "reason": reason, "remaining": self.paused_count - 1}
+
+        except Exception as e:
+            logger.error(f"Failed to fail request {rowid}: {e}")
+            return {"error": str(e)}
+
+    def fulfill_request(
+        self,
+        rowid: int,
+        response_code: int = 200,
+        response_headers: list[dict[str, str]] | None = None,
+        body: str = "",
+    ) -> dict[str, Any]:
+        """Fulfill a request with a custom response.
+
+        Args:
+            rowid: Row ID from requests() table
+            response_code: HTTP response code
+            response_headers: Response headers
+            body: Response body
+
+        Returns:
+            Dict with fulfillment status
+        """
+        if not self.enabled or not self.cdp:
+            return {"error": "Fetch not enabled"}
+
+        event = self.get_paused_event(rowid)
+        if not event:
+            return {"error": f"Event {rowid} not found"}
+
+        request_id = event["params"]["requestId"]
+
+        try:
+            import base64
+
+            # Encode body to base64
+            body_base64 = base64.b64encode(body.encode()).decode()
+
+            params = {
+                "requestId": request_id,
+                "responseCode": response_code,
+                "body": body_base64,
+            }
+
+            if response_headers:
+                params["responseHeaders"] = response_headers
+
+            self.cdp.execute("Fetch.fulfillRequest", params)
+
+            return {"fulfilled": rowid, "response_code": response_code, "remaining": self.paused_count - 1}
+
+        except Exception as e:
+            logger.error(f"Failed to fulfill request {rowid}: {e}")
+            return {"error": str(e)}
+
+
+# No exports - internal service only

webtap/services/main.py

@@ -0,0 +1,175 @@
+"""Main service orchestrator for WebTap business logic.
+
+PUBLIC API:
+  - WebTapService: Main service orchestrating all domain services
+"""
+
+from typing import Any
+
+from webtap.filters import FilterManager
+from webtap.services.fetch import FetchService
+from webtap.services.network import NetworkService
+from webtap.services.console import ConsoleService
+from webtap.services.body import BodyService
+
+
+REQUIRED_DOMAINS = [
+    "Page",
+    "Network",
+    "Runtime",
+    "Log",
+    "DOMStorage",
+]
+
+
+class WebTapService:
+    """Main service orchestrating all WebTap domain services.
+
+    Coordinates CDP session management, domain services, and filter management.
+    Shared between REPL commands and API endpoints for consistent state.
+
+    Attributes:
+        state: WebTap application state instance.
+        cdp: CDP session for browser communication.
+        enabled_domains: Set of currently enabled CDP domains.
+        filters: Filter manager for event filtering.
+        fetch: Fetch interception service.
+        network: Network monitoring service.
+        console: Console message service.
+        body: Response body fetching service.
+    """
+
+    def __init__(self, state):
+        """Initialize with WebTapState instance.
+
+        Args:
+            state: WebTapState instance from app.py
+        """
+        self.state = state
+        self.cdp = state.cdp
+
+        self.enabled_domains: set[str] = set()
+        self.filters = FilterManager()
+
+        self.fetch = FetchService()
+        self.network = NetworkService()
+        self.console = ConsoleService()
+        self.body = BodyService()
+
+        self.fetch.cdp = self.cdp
+        self.network.cdp = self.cdp
+        self.console.cdp = self.cdp
+        self.body.cdp = self.cdp
+
+        self.fetch.body_service = self.body
+
+        # Legacy wiring for CDP event handler
+        self.cdp.fetch_service = self.fetch
+
+    @property
+    def event_count(self) -> int:
+        """Total count of all CDP events stored."""
+        if not self.cdp or not self.cdp.is_connected:
+            return 0
+        try:
+            result = self.cdp.db.execute("SELECT COUNT(*) FROM events").fetchone()
+            return result[0] if result else 0
+        except Exception:
+            return 0
+
+    def connect_to_page(self, page_index: int | None = None, page_id: str | None = None) -> dict[str, Any]:
+        """Connect to Chrome page and enable required domains.
+
+        Args:
+            page_index: Index of page to connect to (for REPL)
+            page_id: ID of page to connect to (for extension)
+        """
+        try:
+            self.cdp.connect(page_index=page_index, page_id=page_id)
+
+            failures = self.enable_domains(REQUIRED_DOMAINS)
+
+            if failures:
+                self.cdp.disconnect()
+                return {"error": f"Failed to enable domains: {failures}"}
+
+            self.filters.load()
+
+            page_info = self.cdp.page_info or {}
+            return {"connected": True, "title": page_info.get("title", "Untitled"), "url": page_info.get("url", "")}
+        except Exception as e:
+            return {"error": str(e)}
+
+    def disconnect(self) -> dict[str, Any]:
+        """Disconnect from Chrome."""
+        was_connected = self.cdp.is_connected
+
+        if self.fetch.enabled:
+            self.fetch.disable()
+
+        self.body.clear_cache()
+
+        self.cdp.disconnect()
+        self.enabled_domains.clear()
+
+        return {"disconnected": True, "was_connected": was_connected}
+
+    def enable_domains(self, domains: list[str]) -> dict[str, str]:
+        """Enable CDP domains.
+
+        Args:
+            domains: List of domain names to enable
+        """
+        failures = {}
+        for domain in domains:
+            try:
+                self.cdp.execute(f"{domain}.enable")
+                self.enabled_domains.add(domain)
+            except Exception as e:
+                failures[domain] = str(e)
+        return failures
+
+    def get_status(self) -> dict[str, Any]:
+        """Get current connection and state status."""
+        if not self.cdp.is_connected:
+            return {
+                "connected": False,
+                "events": 0,
+                "fetch_enabled": self.fetch.enabled,
+                "paused_requests": 0,
+                "network_requests": 0,
+                "console_messages": 0,
+                "console_errors": 0,
+            }
+
+        page_info = self.cdp.page_info or {}
+
+        return {
+            "connected": True,
+            "connected_page_id": page_info.get("id"),  # Stable page ID
+            "url": page_info.get("url"),
+            "title": page_info.get("title"),
+            "events": self.event_count,
+            "fetch_enabled": self.fetch.enabled,
+            "paused_requests": self.fetch.paused_count if self.fetch.enabled else 0,
+            "network_requests": self.network.request_count,
+            "console_messages": self.console.message_count,
+            "console_errors": self.console.error_count,
+            "enabled_domains": list(self.enabled_domains),
+        }
+
+    def clear_events(self) -> dict[str, Any]:
+        """Clear all stored CDP events."""
+        self.cdp.clear_events()
+        return {"cleared": True, "events": 0}
+
+    def list_pages(self) -> dict[str, Any]:
+        """List available Chrome pages."""
+        try:
+            pages = self.cdp.list_pages()
+            connected_id = self.cdp.page_info.get("id") if self.cdp.page_info else None
+            for page in pages:
+                page["is_connected"] = page.get("id") == connected_id
+            return {"pages": pages}
+        except Exception as e:
+            return {"error": str(e), "pages": []}