ghost-api-cli 0.2.0__py3-none-any.whl

ghost_api/browser/driver.py

@@ -0,0 +1,273 @@
+"""Control a real Chromium browser with Playwright (FR-7).
+
+Every action ends with `settle()`, which waits for the network to go quiet and then
+flushes the traffic recorder so response bodies are read before the next action.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from collections.abc import Callable
+from pathlib import Path
+from types import TracebackType
+from typing import Any
+from urllib.parse import urlsplit
+
+from playwright.sync_api import Browser, BrowserContext, Page, Playwright, sync_playwright
+from playwright.sync_api import Error as PlaywrightError
+
+from ghost_api.browser.egress import EgressProxy
+from ghost_api.browser.gate import RequestGate
+from ghost_api.browser.snapshot import (
+    SNAPSHOT_ARGS,
+    SNAPSHOT_JS,
+    PageSnapshot,
+    build_snapshot,
+)
+from ghost_api.capture.models import CapturedRequest
+from ghost_api.capture.recorder import TrafficRecorder
+from ghost_api.config import tool_user_agent
+from ghost_api.net import RobotsPolicy
+from ghost_api.netguard import NetworkPolicy
+
+NAV_TIMEOUT_MS = 30_000
+SETTLE_TIMEOUT_MS = 8_000
+QUIET_MS = 600
+POLL_MS = 100
+ACTION_TIMEOUT_MS = 5_000
+
+
+class ActionError(Exception):
+    """An agent action could not be performed; the message is shown to the agent."""
+
+
+class BrowserSession:
+    def __init__(
+        self,
+        start_url: str,
+        recorder: TrafficRecorder,
+        headed: bool = False,
+        policy: NetworkPolicy | None = None,
+        gate: RequestGate | None = None,
+        on_block: Callable[[str], None] | None = None,
+        storage_state: dict[str, Any] | None = None,
+    ) -> None:
+        """`policy`: where the browser may connect (default: decided from `start_url`).
+        `gate`: robots.txt and rate limits for the browser's own requests."""
+        self.start_url = start_url
+        self.recorder = recorder
+        self.headed = headed
+        self.policy = policy or NetworkPolicy.for_target(start_url)
+        self.gate = gate
+        self.on_block = on_block
+        self.storage_state = storage_state  # a saved login (auth_state.py)
+        self._proxy: EgressProxy | None = None
+        self._pw: Playwright | None = None
+        self._browser: Browser | None = None
+        self._context: BrowserContext | None = None
+        self._page: Page | None = None
+        self._inflight = 0
+        self.popup_urls: list[str] = []
+
+    @property
+    def page(self) -> Page:
+        if self._page is None:
+            raise RuntimeError("BrowserSession is not open")
+        return self._page
+
+    @property
+    def host(self) -> str:
+        return urlsplit(self.start_url).hostname or ""
+
+    def open(self) -> BrowserSession:
+        launch: dict[str, Any] = {"headless": not self.headed}
+        if not self.policy.allow_private:
+            # Every connection goes through the egress proxy, loopback included (Chromium
+            # skips the proxy for localhost unless told "<-loopback>").
+            self._proxy = EgressProxy(self.policy, self._blocked).start()
+            launch["proxy"] = {"server": self._proxy.url, "bypass": "<-loopback>"}
+        self._pw = sync_playwright().start()
+        self._browser = self._pw.chromium.launch(**launch)
+        # Keep the real browser user-agent (so sites behave normally) and add an honest
+        # identifier for this tool (NFR-20).
+        probe = self._browser.new_page()
+        base_ua = probe.evaluate("navigator.userAgent")
+        probe.close()
+        self._context = self._browser.new_context(
+            user_agent=f"{base_ua} {tool_user_agent()}",
+            storage_state=self.storage_state,  # type: ignore[arg-type]
+        )
+        if self.gate is not None:
+            self._context.route("**/*", self.gate.handle)
+        self._page = self._context.new_page()
+        # Registered after the main tab exists, so only real popups are caught.
+        self._context.on("page", self._on_popup)
+        self._page.set_default_navigation_timeout(NAV_TIMEOUT_MS)
+        self._page.on("request", lambda _: self._track(+1))
+        self._page.on("requestfinished", lambda _: self._track(-1))
+        self._page.on("requestfailed", lambda _: self._track(-1))
+        self.recorder.attach(self._page)
+        return self
+
+    def close(self) -> None:
+        for closer in (self._context, self._browser):
+            if closer is not None:
+                with contextlib.suppress(Exception):  # best-effort cleanup
+                    closer.close()
+        if self._pw is not None:
+            self._pw.stop()
+        if self._proxy is not None:
+            self._proxy.stop()
+        self._page = self._context = self._browser = self._pw = None
+        self._proxy = None
+
+    def export_state(self) -> dict[str, Any]:
+        """Cookies and site storage now (after the user logged in), for auth_state.save."""
+        if self._context is None:
+            raise RuntimeError("BrowserSession is not open")
+        return dict(self._context.storage_state())
+
+    def _blocked(self, message: str) -> None:
+        self.recorder.dropped["blocked-network"] += 1
+        if self.on_block is not None:
+            self.on_block(message)
+
+    def __enter__(self) -> BrowserSession:
+        return self.open()
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    # --- actions ---
+
+    def settle(self) -> list[CapturedRequest]:
+        """Wait until no requests have been in flight for QUIET_MS, then collect traffic.
+
+        Playwright's "networkidle" load state only applies to the initial page load, so
+        requests triggered later (by a scroll or click) are tracked here instead. Sites
+        with long-polling never go quiet; SETTLE_TIMEOUT_MS caps the wait.
+        """
+        waited = quiet = 0
+        while waited < SETTLE_TIMEOUT_MS:
+            self.page.wait_for_timeout(POLL_MS)  # lets Playwright deliver events
+            waited += POLL_MS
+            quiet = quiet + POLL_MS if self._inflight == 0 else 0
+            if quiet >= QUIET_MS:
+                break
+        return self.recorder.flush()
+
+    def _track(self, delta: int) -> None:
+        self._inflight = max(0, self._inflight + delta)
+
+    def _on_popup(self, popup: Page) -> None:
+        """Keep everything in one tab: remember where a popup was going, then close it."""
+        if popup is self._page:
+            return
+        with contextlib.suppress(PlaywrightError):
+            popup.wait_for_load_state("domcontentloaded", timeout=ACTION_TIMEOUT_MS)
+            self.popup_urls.append(popup.url)
+            popup.close()
+
+    @property
+    def url(self) -> str:
+        return self.page.url
+
+    def snapshot(self, target_host: str, robots: RobotsPolicy | None = None) -> PageSnapshot:
+        raw = self.page.evaluate(SNAPSHOT_JS, SNAPSHOT_ARGS)
+        return build_snapshot(raw, target_host, robots)
+
+    def screenshot(self, path: Path) -> bool:
+        """Save a small JPEG of what the browser shows (for the dashboard's live view)."""
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            self.page.screenshot(path=str(path), type="jpeg", quality=55)
+        except PlaywrightError:
+            return False
+        return True
+
+    def element_box(self, element_id: int) -> dict[str, float] | None:
+        """Where an element is on the page (viewport pixels), before acting on it."""
+        try:
+            box = self.page.locator(f'[data-ghost-id="{element_id}"]').first.bounding_box(
+                timeout=1_000
+            )
+        except PlaywrightError:
+            return None
+        if box is None:
+            return None
+        viewport = self.page.viewport_size or {"width": 1280, "height": 720}
+        return {
+            "x": box["x"],
+            "y": box["y"],
+            "width": box["width"],
+            "height": box["height"],
+            "viewport_width": viewport["width"],
+            "viewport_height": viewport["height"],
+        }
+
+    def _element(self, element_id: int):  # type: ignore[no-untyped-def]
+        loc = self.page.locator(f'[data-ghost-id="{element_id}"]').first
+        if loc.count() == 0:
+            raise ActionError(f"element [{element_id}] is no longer on the page")
+        return loc
+
+    def _after_action(self) -> list[CapturedRequest]:
+        # A click may start a navigation; give the new document a moment to load.
+        with contextlib.suppress(PlaywrightError):
+            self.page.wait_for_load_state("domcontentloaded", timeout=ACTION_TIMEOUT_MS)
+        return self.settle()
+
+    def click(self, element_id: int) -> list[CapturedRequest]:
+        loc = self._element(element_id)
+        try:
+            loc.click(timeout=ACTION_TIMEOUT_MS)
+        except PlaywrightError:
+            # Covered by an overlay or animating: fall back to a DOM click.
+            try:
+                loc.evaluate("el => el.click()")
+            except PlaywrightError as exc:
+                raise ActionError(f"could not click [{element_id}]: {_first_line(exc)}") from exc
+        return self._after_action()
+
+    def type_text(self, element_id: int, text: str, submit: bool) -> list[CapturedRequest]:
+        loc = self._element(element_id)
+        try:
+            loc.fill(text, timeout=ACTION_TIMEOUT_MS)
+            if submit:
+                loc.press("Enter", timeout=ACTION_TIMEOUT_MS)
+        except PlaywrightError as exc:
+            raise ActionError(f"could not type into [{element_id}]: {_first_line(exc)}") from exc
+        return self._after_action()
+
+    def go_back(self) -> list[CapturedRequest]:
+        try:
+            self.page.go_back(wait_until="domcontentloaded")
+        except PlaywrightError as exc:
+            raise ActionError(f"could not go back: {_first_line(exc)}") from exc
+        return self.settle()
+
+    def goto(self, url: str) -> list[CapturedRequest]:
+        try:
+            self.page.goto(url, wait_until="domcontentloaded")
+        except PlaywrightError as exc:
+            raise ActionError(f"could not open {url}: {_first_line(exc)}") from exc
+        return self.settle()
+
+    def scroll(self, direction: str = "down") -> list[CapturedRequest]:
+        if direction == "bottom":
+            self.page.evaluate("window.scrollTo(0, document.documentElement.scrollHeight)")
+        else:
+            delta = -1 if direction == "up" else 1
+            viewport = self.page.viewport_size
+            height = viewport["height"] if viewport else 800
+            self.page.mouse.wheel(0, delta * height)
+        return self.settle()
+
+
+def _first_line(exc: Exception) -> str:
+    return str(exc).strip().splitlines()[0][:160] if str(exc).strip() else type(exc).__name__

ghost_api/browser/egress.py

@@ -0,0 +1,183 @@
+"""A tiny local proxy that every browser connection goes through (the network boundary).
+
+Playwright's request interception does not see redirect hops: when a page or an API call
+is redirected, the browser follows it on its own. A proxy sees every connection the
+browser opens — redirects, images, scripts, websockets — so each destination is checked
+with `NetworkPolicy` before anything is sent to it. The proxy then connects to the very
+address that passed the check (no second DNS lookup to switch to a private address).
+
+HTTPS goes through `CONNECT host:port` tunnels: the proxy sees the host, not the path or
+the content (paths and robots.txt are checked by the route gate in `gate.py`). Plain HTTP
+requests are forwarded one per connection.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import select
+import socket
+import socketserver
+import threading
+from collections.abc import Callable
+from urllib.parse import urlsplit
+
+from ghost_api.netguard import NetworkPolicy, PolicyDenied
+
+HEAD_LIMIT = 64 * 1024
+CONNECT_TIMEOUT_S = 15
+IDLE_TIMEOUT_S = 120
+
+
+class EgressProxy:
+    def __init__(
+        self, policy: NetworkPolicy, on_block: Callable[[str], None] | None = None
+    ) -> None:
+        self.policy = policy
+        self.on_block = on_block
+        self.blocked: list[str] = []
+        self._lock = threading.Lock()
+        proxy = self
+
+        class Handler(socketserver.BaseRequestHandler):
+            def handle(self) -> None:
+                proxy._handle(self.request)
+
+        class Server(socketserver.ThreadingTCPServer):
+            daemon_threads = True
+            allow_reuse_address = True
+
+        self._server = Server(("127.0.0.1", 0), Handler)
+        self._thread: threading.Thread | None = None
+
+    @property
+    def url(self) -> str:
+        return f"http://127.0.0.1:{self._server.server_address[1]}"
+
+    def start(self) -> EgressProxy:
+        self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
+        self._thread.start()
+        return self
+
+    def stop(self) -> None:
+        self._server.shutdown()
+        self._server.server_close()
+
+    def __enter__(self) -> EgressProxy:
+        return self.start()
+
+    def __exit__(self, *exc: object) -> None:
+        self.stop()
+
+    # --- one client connection ---
+
+    def _handle(self, client: socket.socket) -> None:
+        client.settimeout(CONNECT_TIMEOUT_S)
+        try:
+            head = _read_head(client)
+        except OSError:
+            return
+        if not head:
+            return
+        line, _, rest = head.partition(b"\r\n")
+        try:
+            method, target, version = line.decode("latin-1").split(" ", 2)
+        except ValueError:
+            _reply(client, 400, "Bad Request")
+            return
+        if method.upper() == "CONNECT":
+            host, _, port_text = target.rpartition(":")
+            port = int(port_text) if port_text.isdigit() else 443
+            upstream = self._connect(client, host, port)
+            if upstream is None:
+                return
+            _reply(client, 200, "Connection Established")
+            _pipe(client, upstream)
+            return
+        parts = urlsplit(target)
+        if parts.scheme != "http" or not parts.hostname:
+            _reply(client, 400, "Bad Request")
+            return
+        upstream = self._connect(client, parts.hostname, parts.port or 80)
+        if upstream is None:
+            return
+        path = parts.path or "/"
+        if parts.query:
+            path += f"?{parts.query}"
+        headers, _, body = rest.partition(b"\r\n\r\n")
+        kept = [
+            h
+            for h in headers.split(b"\r\n")
+            if h and not h.lower().startswith((b"proxy-connection:", b"connection:"))
+        ]
+        request = f"{method} {path} {version}\r\n".encode("latin-1")
+        request += b"\r\n".join([*kept, b"Connection: close"]) + b"\r\n\r\n" + body
+        upstream.sendall(request)
+        _pipe(client, upstream)
+
+    def _connect(self, client: socket.socket, host: str, port: int) -> socket.socket | None:
+        try:
+            addresses = self.policy.resolve(host, port)
+        except PolicyDenied as exc:
+            self._block(str(exc))
+            _reply(client, 403, "Blocked by Ghost API")
+            return None
+        for address in addresses:
+            try:
+                return socket.create_connection((address, port), timeout=CONNECT_TIMEOUT_S)
+            except OSError:
+                continue
+        _reply(client, 502, "Bad Gateway")
+        return None
+
+    def _block(self, message: str) -> None:
+        with self._lock:
+            self.blocked.append(message)
+        if self.on_block is not None:
+            with contextlib.suppress(Exception):
+                self.on_block(message)
+
+
+def _read_head(sock: socket.socket) -> bytes:
+    data = b""
+    while b"\r\n\r\n" not in data and len(data) < HEAD_LIMIT:
+        chunk = sock.recv(4096)
+        if not chunk:
+            break
+        data += chunk
+    return data
+
+
+def _reply(client: socket.socket, status: int, reason: str) -> None:
+    """Answer the browser; any answer but "200 Connection Established" ends the connection."""
+    with contextlib.suppress(OSError):
+        if status == 200:
+            client.sendall(f"HTTP/1.1 200 {reason}\r\n\r\n".encode("latin-1"))
+            return
+        client.sendall(
+            f"HTTP/1.1 {status} {reason}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+            .encode("latin-1")
+        )  # fmt: skip
+        client.close()
+
+
+def _pipe(a: socket.socket, b: socket.socket) -> None:
+    """Copy bytes both ways until either side closes or the connection idles out."""
+    sockets = [a, b]
+    try:
+        for s in sockets:
+            s.settimeout(None)
+        while True:
+            readable, _, broken = select.select(sockets, [], sockets, IDLE_TIMEOUT_S)
+            if broken or not readable:
+                return
+            for source in readable:
+                data = source.recv(65536)
+                if not data:
+                    return
+                (b if source is a else a).sendall(data)
+    except OSError:
+        return
+    finally:
+        for s in sockets:
+            with contextlib.suppress(OSError):
+                s.close()

ghost_api/browser/gate.py

@@ -0,0 +1,127 @@
+"""Rules for requests the browser makes, applied before they are sent.
+
+  - Page loads (navigations) anywhere must be allowed by that host's robots.txt.
+  - The site's own API calls (fetch / XHR to the site or its subdomains) must be allowed
+    by robots.txt and are spaced out per host by the run's rate limit, like Ghost API's
+    direct requests. The page's own images, scripts and styles load as in any browser.
+  - Third-party calls (analytics, maps, CDNs...) are not checked against robots.txt: the
+    page makes them, not Ghost API, and fetching every third party's robots.txt would add
+    traffic. They still go through the egress proxy (no private addresses).
+
+Every decision to block is reported once per host and path through `on_block`.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from typing import Protocol
+from urllib.parse import urlsplit
+
+from ghost_api.capture.filters import is_third_party
+from ghost_api.net import RateLimiter, RobotsPolicy
+
+API_TYPES = {"fetch", "xhr", "eventsource"}
+MAX_WAIT_S = 10.0
+
+
+class _Frame(Protocol):
+    @property
+    def parent_frame(self) -> object | None: ...
+
+
+class _Request(Protocol):
+    @property
+    def url(self) -> str: ...
+
+    @property
+    def resource_type(self) -> str: ...
+
+    @property
+    def frame(self) -> _Frame: ...
+
+    def is_navigation_request(self) -> bool: ...
+
+
+class _Route(Protocol):
+    @property
+    def request(self) -> _Request: ...
+
+    def continue_(self) -> None: ...
+
+    def abort(self, error_code: str | None = None) -> None: ...
+
+
+class RequestGate:
+    def __init__(
+        self,
+        target_host: str,
+        robots: RobotsPolicy | None,
+        limiter: RateLimiter | None,
+        on_block: Callable[[str], None] | None = None,
+        clock: Callable[[], float] = time.monotonic,
+        sleep: Callable[[float], None] = time.sleep,
+        third_party_robots: bool = False,
+    ) -> None:
+        """`third_party_robots`: also check the page's API calls to other sites (search
+        services, maps...) against those sites' own robots.txt."""
+        self.third_party_robots = third_party_robots
+        self.target_host = target_host
+        self.robots = robots
+        self.limiter = limiter
+        self.on_block = on_block
+        self._clock = clock
+        self._sleep = sleep
+        self._last: dict[str, float] = {}
+        self._reported: set[str] = set()
+        self.blocked = 0
+        self.delayed = 0
+
+    def decide(self, url: str, resource_type: str, navigation: bool) -> str | None:
+        """Why the request must not be sent, or None (after any rate-limit wait)."""
+        parts = urlsplit(url)
+        if parts.scheme not in ("http", "https"):
+            return None  # data:, blob: ... never reach the network
+        first_party = not is_third_party(url, self.target_host)
+        api_call = first_party and resource_type in API_TYPES
+        if (navigation or api_call) and self.robots is not None and not self.robots.allows(url):
+            return "disallowed by robots.txt"
+        third_party_api = not first_party and resource_type in API_TYPES
+        checked = third_party_api and self.third_party_robots and self.robots is not None
+        if checked and self.robots is not None and not self.robots.allows(url):
+            return "disallowed by that site's robots.txt"
+        if api_call or (navigation and first_party):
+            self._space_out((parts.hostname or "").lower())
+        return None
+
+    def handle(self, route: _Route) -> None:
+        request = route.request
+        try:  # only the main page's own loads, not embedded frames (ads, videos...)
+            navigation = request.is_navigation_request() and request.frame.parent_frame is None
+        except Exception:  # noqa: BLE001 - a service worker or detached frame
+            navigation = False
+        reason = self.decide(request.url, request.resource_type, navigation)
+        if reason is None:
+            route.continue_()
+            return
+        self.blocked += 1
+        self._report(request.url, reason)
+        route.abort("blockedbyclient")
+
+    def _space_out(self, host: str) -> None:
+        if self.limiter is None:
+            return
+        now = self._clock()
+        wait = min(self._last.get(host, float("-inf")) + self.limiter.interval - now, MAX_WAIT_S)
+        if wait > 0:
+            self.delayed += 1
+            self._sleep(wait)
+        self._last[host] = self._clock()
+
+    def _report(self, url: str, reason: str) -> None:
+        parts = urlsplit(url)
+        key = f"{parts.hostname}{parts.path}"
+        if key in self._reported or self.on_block is None:
+            return
+        self._reported.add(key)
+        self.on_block(f"Blocked {parts.hostname}{parts.path[:60]}: {reason}")

ghost_api/browser/install.py

@@ -0,0 +1,28 @@
+"""Is the browser Ghost API drives installed, and install it if not."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def chromium_path() -> Path | None:
+    """Where Playwright's Chromium is, or None when it isn't installed."""
+    from playwright.sync_api import Error as PlaywrightError
+    from playwright.sync_api import sync_playwright
+
+    try:
+        with sync_playwright() as pw:
+            path = Path(pw.chromium.executable_path)
+    except PlaywrightError:
+        return None
+    return path if path.exists() else None
+
+
+def install_chromium() -> bool:
+    """Download Chromium for Playwright (about 150 MB, once). True when it worked."""
+    proc = subprocess.run(
+        [sys.executable, "-m", "playwright", "install", "chromium"], check=False, timeout=1800
+    )
+    return proc.returncode == 0 and chromium_path() is not None