parsek-cdp-server 0.1.0__py3-none-any.whl

parsek_cdp_server/__init__.py

@@ -0,0 +1,46 @@
+"""parsek-cdp-server -- the server scope: launch, supervise and proxy a browser.
+
+Separate distribution from ``parsek-cdp`` (the client); installed via
+``pip install parsek-cdp[server]``.  It *depends on* ``parsek-cdp`` and reuses
+its shared layers wholesale -- the generated ``cdp`` protocol, the ``core``
+primitives (``CDPConnection``, ``Target``, ``DataType``) and the ``parsek``
+contract -- adding only what must live next to the browser:
+
+* :mod:`~parsek_cdp_server.launcher`   -- spawn Chrome, find its websocket;
+* :mod:`~parsek_cdp_server.supervisor` -- own the browser lifecycle: health,
+  crash detection, restart, idle shutdown, and ``Parsek.browserStateChanged``
+  broadcast;
+* :mod:`~parsek_cdp_server.proxy`      -- accept client websockets and bridge
+  each one 1:1 to a Chrome target (no ``sessionId`` multiplexing, no id remap),
+  hosting feature producers on page pipes and serving the ``Parsek.*`` surface;
+* :mod:`~parsek_cdp_server.metrics`    -- Prometheus exposition at ``/metrics``
+  (browsers, targets/types, nested targets, per-target CDP events, CPU/RAM);
+* :mod:`~parsek_cdp_server.reaper`     -- a separate subprocess that kills leaked
+  (zombie/orphaned) parsek-launched Chrome processes.
+
+Feature *producers* (raw CDP -> ``Parsek.*``) are shared code shipped in the
+client distribution (:mod:`parsek_cdp.features`); the server hosts them but does
+not define them.  ``ParsekServer`` registers none by default -- it is a pure
+passthrough until features are added via ``register_feature``.
+
+Endpoints (see :mod:`~parsek_cdp_server.proxy`)::
+
+    HTTP POST /browsers                            create task -> browser_uuid
+    HTTP GET  /metrics                             Prometheus metrics
+    ws        /cdp/{browser_uuid}/control          lifecycle + Parsek.*
+    ws        /cdp/{browser_uuid}/page/{target_id} CDP proxy (per-target) + Parsek.*
+"""
+
+from __future__ import annotations
+
+from .launcher import ChromeLauncher
+from .metrics import ServerMetrics
+from .proxy import ParsekServer
+from .supervisor import BrowserSupervisor
+
+__all__ = [
+    "ParsekServer",
+    "ChromeLauncher",
+    "BrowserSupervisor",
+    "ServerMetrics",
+]

parsek_cdp_server/launcher.py

@@ -0,0 +1,303 @@
+"""Launching a browser process and locating its DevTools websocket.
+
+The launcher is the lowest server layer: it knows how to start Chrome/Chromium
+with the right flags, wait until its DevTools endpoint is up, and read the
+browser-level websocket url from ``/json/version``.  Lifecycle decisions
+(restart on crash, etc.) belong to :class:`~parsek_cdp_server.supervisor`, which
+drives this class.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import random
+import shutil
+import signal
+import tempfile
+import urllib.error
+import urllib.request
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Optional
+
+import psutil
+
+#: Env var naming *directories* to search for browser binaries,
+#: ``os.pathsep``-separated (like ``PATH``), e.g. ``/opt/browsers/chrome:/opt/browsers/brave``.
+#: Each directory is walked recursively; every executable file whose name is a
+#: known Chromium browser (see :data:`_BROWSER_NAMES`) becomes a candidate, and
+#: each launch runs a randomly chosen one -- a cheap way to spread load (and
+#: fingerprints) across several installed browsers.  When unset,
+#: :data:`_KNOWN_PATHS` is searched as the default.
+CHROMES_PATH_ENV = "PARSEK_CHROMES_PATH"
+
+#: Prefix of the temp profile directories the launcher creates (``tempfile``
+#: prefix).  It doubles as an ownership *marker*: the reaper recognises a browser
+#: as parsek-launched by finding ``--user-data-dir=<...PROFILE_PREFIX...>`` in its
+#: argv, so it only ever kills processes we started.
+PROFILE_PREFIX = "parsek-cdp-"
+
+#: Recognised Chromium-browser executable names (matched case-insensitively, with
+#: any ``.exe`` suffix stripped).  Restricting the recursive search to these
+#: avoids picking up helper binaries (``chrome_crashpad_handler``, ``chrome_sandbox``).
+_BROWSER_NAMES = frozenset(
+    {
+        "chrome",
+        "google-chrome",
+        "google-chrome-stable",
+        "chromium",
+        "chromium-browser",
+        "msedge",
+        "microsoft-edge",
+        "microsoft-edge-stable",
+        "brave",
+        "brave-browser",
+        "vivaldi",
+        "vivaldi-bin",
+        "opera",
+    }
+)
+
+#: Default directories searched when :data:`CHROMES_PATH_ENV` is unset.  Any
+#: Chromium-based browser works -- they all speak the same DevTools Protocol.
+#: Non-existent dirs are skipped, so listing every platform here is harmless.
+_KNOWN_PATHS = (
+    # Linux
+    "/usr/bin",
+    "/usr/local/bin",
+    "/opt",
+    "/snap/bin",
+    # macOS
+    "/Applications/Google Chrome.app/Contents/MacOS",
+    "/Applications/Chromium.app/Contents/MacOS",
+    "/Applications/Microsoft Edge.app/Contents/MacOS",
+    "/Applications/Brave Browser.app/Contents/MacOS",
+    # Windows
+    r"C:\Program Files\Google\Chrome\Application",
+    r"C:\Program Files (x86)\Google\Chrome\Application",
+    r"C:\Program Files (x86)\Microsoft\Edge\Application",
+    r"C:\Program Files\BraveSoftware\Brave-Browser\Application",
+)
+
+#: Executable names tried on ``PATH`` as a last resort when neither
+#: :attr:`LaunchOptions.executable` nor a directory search yields a binary.
+_CANDIDATES = (
+    "google-chrome",
+    "google-chrome-stable",
+    "chromium",
+    "chromium-browser",
+    "microsoft-edge",
+    "microsoft-edge-stable",
+    "brave-browser",
+    "brave",
+    "chrome",
+)
+
+
+def _is_browser(filename: str) -> bool:
+    name = filename[:-4] if filename.lower().endswith(".exe") else filename
+    return name.lower() in _BROWSER_NAMES
+
+
+def _find_in_dirs(dirs: List[str]) -> List[str]:
+    """Recursively collect browser executables found under ``dirs``."""
+    found: List[str] = []
+    for directory in dirs:
+        directory = directory.strip()
+        if not directory or not os.path.isdir(directory):
+            continue
+        for root, _subdirs, files in os.walk(directory):
+            for filename in files:
+                if not _is_browser(filename):
+                    continue
+                full = os.path.join(root, filename)
+                if os.path.isfile(full) and os.access(full, os.X_OK):
+                    found.append(full)
+    return found
+
+
+def _detect_executable() -> str:
+    """Pick a Chromium-based binary, or raise if none is available.
+
+    Precedence: a random browser executable found by recursively searching the
+    directories in :data:`CHROMES_PATH_ENV` (defaulting to :data:`_KNOWN_PATHS`),
+    else the first :data:`_CANDIDATES` name resolvable on ``PATH``.
+    """
+    raw = os.environ.get(CHROMES_PATH_ENV)
+    search_dirs = raw.split(os.pathsep) if raw else list(_KNOWN_PATHS)
+    matches = _find_in_dirs(search_dirs)
+    if matches:
+        return random.choice(matches)
+    for name in _CANDIDATES:
+        resolved = shutil.which(name)
+        if resolved:
+            return resolved
+    raise FileNotFoundError(
+        f"no Chrome/Chromium executable found; set {CHROMES_PATH_ENV} "
+        "(os.pathsep-separated directories to search) or LaunchOptions.executable"
+    )
+
+
+@dataclass
+class LaunchOptions:
+    """How to start the browser."""
+
+    executable: Optional[str] = None  # autodetect if None
+    headless: bool = True
+    port: int = 0
+    user_data_dir: Optional[str] = None
+    extra_args: List[str] = field(default_factory=list)
+    default_args: bool = True
+
+
+class ChromeLauncher:
+    """Spawn a browser process and expose its browser-level websocket url.
+
+    Responsibilities:
+
+    * build the argv (``--remote-debugging-port``, ``--user-data-dir``,
+      ``--headless=new``, ``--no-first-run``, ...);
+    * start the subprocess; with ``port == 0`` Chrome writes the port it chose to
+      the ``DevToolsActivePort`` file in the profile dir, which we poll;
+    * poll ``/json/version`` until ``webSocketDebuggerUrl`` is available;
+    * expose :attr:`pid` and :attr:`ws_url`; provide :meth:`terminate`.
+    """
+
+    def __init__(self, options: Optional[LaunchOptions] = None) -> None:
+        self.options = options or LaunchOptions()
+        self.pid: Optional[int] = None
+        self.ws_url: Optional[str] = None
+        self._proc: Optional[asyncio.subprocess.Process] = None
+        #: Temp profile dir we created (and must clean up); None if caller supplied one.
+        self._owned_user_data_dir: Optional[str] = None
+
+    # -- argv -------------------------------------------------------------- #
+
+    def _resolve_user_data_dir(self) -> str:
+        if self.options.user_data_dir is not None:
+            return self.options.user_data_dir
+        path = tempfile.mkdtemp(prefix=PROFILE_PREFIX)
+        self._owned_user_data_dir = path
+        return path
+
+    def _build_argv(self, user_data_dir: str) -> List[str]:
+        executable = self.options.executable or _detect_executable()
+        argv = [executable, f"--remote-debugging-port={self.options.port}"]
+        if self.options.default_args:
+            argv += [
+                f"--user-data-dir={user_data_dir}",
+                "--no-first-run",
+                "--no-default-browser-check",
+                "--disable-background-networking",
+                "--disable-popup-blocking",
+                "--remote-allow-origins=*",
+            ]
+            if self.options.headless:
+                argv += ["--headless=new", "--disable-gpu"]
+        argv += self.options.extra_args
+        return argv
+
+    async def launch(self) -> str:
+        """Start the browser and return its browser-level websocket url."""
+        user_data_dir = self._resolve_user_data_dir()
+        argv = self._build_argv(user_data_dir)
+        self._proc = await asyncio.create_subprocess_exec(
+            *argv,
+            stdout=asyncio.subprocess.DEVNULL,
+            stderr=asyncio.subprocess.DEVNULL,
+        )
+        self.pid = self._proc.pid
+        try:
+            port = await self._await_devtools_port(Path(user_data_dir))
+            self.ws_url = await self._await_ws_url(port)
+        except Exception:
+            await self.terminate()
+            raise
+        return self.ws_url
+
+    async def _await_devtools_port(
+        self, user_data_dir: Path, *, timeout: float = 30.0
+    ) -> int:
+        """Wait for Chrome to write the chosen port to ``DevToolsActivePort``.
+
+        The file's first line is the port; it appears once the DevTools endpoint
+        is listening, so its presence doubles as a readiness signal.  When a
+        fixed port was requested we still wait for the file to confirm liveness.
+        """
+        active_port_file = user_data_dir / "DevToolsActivePort"
+        loop = asyncio.get_running_loop()
+        deadline = loop.time() + timeout
+        while True:
+            if self._proc is not None and self._proc.returncode is not None:
+                raise RuntimeError(
+                    f"browser exited during startup (code {self._proc.returncode})"
+                )
+            try:
+                first_line = active_port_file.read_text().splitlines()[0]
+                return int(first_line)
+            except (FileNotFoundError, IndexError, ValueError):
+                pass
+            if loop.time() >= deadline:
+                raise TimeoutError("browser did not open its DevTools port in time")
+            await asyncio.sleep(0.05)
+
+    async def _await_ws_url(self, port: int, *, timeout: float = 10.0) -> str:
+        """Poll ``/json/version`` until ``webSocketDebuggerUrl`` is served."""
+        endpoint = f"http://127.0.0.1:{port}/json/version"
+        loop = asyncio.get_running_loop()
+        deadline = loop.time() + timeout
+        while True:
+            try:
+                info = await asyncio.to_thread(self._read_json, endpoint)
+                return info["webSocketDebuggerUrl"]
+            except (urllib.error.URLError, KeyError, ConnectionError, OSError):
+                if loop.time() >= deadline:
+                    raise TimeoutError(f"{endpoint} did not become available")
+                await asyncio.sleep(0.05)
+
+    @staticmethod
+    def _read_json(url: str) -> dict:
+        with urllib.request.urlopen(url, timeout=2) as resp:
+            return json.loads(resp.read())
+
+    def is_alive(self) -> bool:
+        """Whether the browser process is still running (used by the supervisor)."""
+        if self._proc is not None and self._proc.returncode is not None:
+            return False
+        if self.pid is None:
+            return False
+        return psutil.pid_exists(self.pid)
+
+    async def terminate(self, *, timeout: float = 5.0) -> None:
+        """Terminate the browser process gracefully, then kill if it lingers."""
+        proc = self._proc
+        if proc is not None and proc.returncode is None:
+            try:
+                proc.send_signal(signal.SIGTERM)
+            except ProcessLookupError:
+                pass
+            else:
+                try:
+                    await asyncio.wait_for(proc.wait(), timeout)
+                except asyncio.TimeoutError:
+                    with _suppress_lookup():
+                        proc.kill()
+                    await proc.wait()
+        self._cleanup_profile()
+
+    def _cleanup_profile(self) -> None:
+        if self._owned_user_data_dir is not None:
+            shutil.rmtree(self._owned_user_data_dir, ignore_errors=True)
+            self._owned_user_data_dir = None
+
+
+class _suppress_lookup:
+    """Context manager swallowing ``ProcessLookupError`` (process already gone)."""
+
+    def __enter__(self) -> "_suppress_lookup":
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> bool:
+        return exc_type is not None and issubclass(exc_type, ProcessLookupError)

parsek_cdp_server/metrics.py

@@ -0,0 +1,239 @@
+"""Prometheus metrics for the server: browsers, targets, CDP events, CPU/RAM.
+
+Exposed at ``GET /metrics``.  The exposition is split in two so a scrape never
+blocks the event loop:
+
+* **live counters** -- :meth:`ServerMetrics.record_event` is called from the page
+  bridges on every CDP event, bumping an in-memory ``(browser, target, domain)``
+  counter synchronously;
+* **a periodic snapshot** -- a background task polls each browser's
+  ``/json/list`` (targets + their types + nesting via ``parentId``) and samples
+  the browser process tree with :mod:`psutil` (CPU%, RSS), off the loop in a
+  thread.  :meth:`collect` (called at scrape time) only reads these cached
+  values.
+
+Metrics:
+
+==================================  =====  ===========================================
+``parsek_browsers``                 gauge  supervised browsers
+``parsek_targets``                  gauge  targets, labelled ``browser``, ``type``
+``parsek_nested_targets``           gauge  targets with a ``parentId``, per ``browser``
+``parsek_browser_cpu_percent``      gauge  CPU% of the browser tree, per ``browser``
+``parsek_browser_memory_bytes``     gauge  RSS of the browser tree, per ``browser``
+``parsek_cdp_events_total``         count  CDP events, labelled ``browser``, ``target``, ``domain``
+==================================  =====  ===========================================
+
+Cardinality note: ``parsek_cdp_events_total`` is labelled by target id, which is
+unbounded over time.  Series for targets that disappear from ``/json/list`` are
+pruned on every snapshot, so the live set tracks the browser, not history.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import urllib.error
+import urllib.request
+from collections import defaultdict
+from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple
+
+import psutil
+from aiohttp import web
+from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, generate_latest
+from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily
+
+from parsek_cdp._logging import get_logger
+
+if TYPE_CHECKING:
+    from .proxy import ParsekServer
+
+logger = get_logger(__name__)
+
+#: (browser_uuid, target_id, domain) -> event count.
+EventKey = Tuple[str, str, str]
+
+
+class _BrowserSnapshot:
+    """Cached per-browser facts sampled by the refresh task."""
+
+    __slots__ = ("targets_by_type", "nested", "cpu_percent", "memory_bytes", "target_ids")
+
+    def __init__(self) -> None:
+        self.targets_by_type: Dict[str, int] = {}
+        self.nested: int = 0
+        self.cpu_percent: float = 0.0
+        self.memory_bytes: int = 0
+        self.target_ids: Set[str] = set()
+
+
+class ServerMetrics:
+    """Collects and exposes Prometheus metrics for a :class:`ParsekServer`."""
+
+    def __init__(self, server: "ParsekServer", *, refresh_interval: float = 5.0) -> None:
+        self._server = server
+        self._refresh_interval = refresh_interval
+        self._events: Dict[EventKey, int] = defaultdict(int)
+        self._snapshot: Dict[str, _BrowserSnapshot] = {}
+        #: psutil.Process cache so cpu_percent() deltas are meaningful across sweeps.
+        self._proc_cache: Dict[int, psutil.Process] = {}
+        self._refresh_task: Optional[asyncio.Task] = None
+        self._registry = CollectorRegistry()
+        self._registry.register(_Collector(self))
+
+    # -- live event counting ---------------------------------------------- #
+
+    def record_event(self, browser_uuid: str, target_id: str, domain: str) -> None:
+        """Count one CDP event (called from a page bridge per event frame)."""
+        self._events[(browser_uuid, target_id, domain)] += 1
+
+    # -- lifecycle --------------------------------------------------------- #
+
+    async def start(self) -> None:
+        self._refresh_task = asyncio.create_task(self._refresh_loop())
+
+    async def stop(self) -> None:
+        if self._refresh_task is not None:
+            self._refresh_task.cancel()
+            try:
+                await self._refresh_task
+            except asyncio.CancelledError:
+                pass
+            self._refresh_task = None
+
+    async def handle(self, request: web.Request) -> web.Response:
+        """``GET /metrics`` -- render the registry in the Prometheus text format.
+
+        Rendered synchronously on the event loop (not via ``to_thread``) so that
+        ``collect`` reading ``_events`` cannot race with :meth:`record_event`
+        mutating it from the same loop -- the registry is small, so this is cheap.
+        """
+        body = generate_latest(self._registry)
+        return web.Response(body=body, content_type=CONTENT_TYPE_LATEST.split(";")[0],
+                            charset="utf-8")
+
+    # -- snapshot refresh -------------------------------------------------- #
+
+    async def _refresh_loop(self) -> None:
+        while True:
+            try:
+                await self._refresh_once()
+            except Exception:
+                logger.exception("metrics refresh failed")
+            await asyncio.sleep(self._refresh_interval)
+
+    async def _refresh_once(self) -> None:
+        snapshot: Dict[str, _BrowserSnapshot] = {}
+        live_targets: Set[Tuple[str, str]] = set()
+        for browser_uuid, supervisor in list(self._server.supervisors.items()):
+            snap = _BrowserSnapshot()
+            origin = supervisor.http_origin
+            if origin is not None:
+                await self._sample_targets(origin, snap)
+            if supervisor.pid is not None:
+                await asyncio.to_thread(self._sample_resources, supervisor.pid, snap)
+            snapshot[browser_uuid] = snap
+            live_targets.update((browser_uuid, tid) for tid in snap.target_ids)
+        self._snapshot = snapshot
+        self._prune_events(live_targets)
+
+    async def _sample_targets(self, http_origin: str, snap: _BrowserSnapshot) -> None:
+        """Fill ``snap`` from ``{http_origin}/json/list`` (excludes the browser target)."""
+        try:
+            targets = await asyncio.to_thread(self._read_json, f"{http_origin}/json/list")
+        except (urllib.error.URLError, OSError, ValueError):
+            return
+        by_type: Dict[str, int] = defaultdict(int)
+        for target in targets:
+            by_type[target.get("type", "other")] += 1
+            tid = target.get("id")
+            if tid:
+                snap.target_ids.add(tid)
+            if target.get("parentId"):
+                snap.nested += 1
+        snap.targets_by_type = dict(by_type)
+
+    def _sample_resources(self, pid: int, snap: _BrowserSnapshot) -> None:
+        """Sum CPU% and RSS over the browser process tree (renderers, gpu, ...)."""
+        try:
+            root = self._proc(pid)
+            procs = [root] + root.children(recursive=True)
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            return
+        cpu = 0.0
+        rss = 0
+        for proc in procs:
+            try:
+                cpu += self._proc(proc.pid).cpu_percent(None)
+                rss += proc.memory_info().rss
+            except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+                continue
+        snap.cpu_percent = cpu
+        snap.memory_bytes = rss
+
+    def _proc(self, pid: int) -> psutil.Process:
+        """Return a cached :class:`psutil.Process` so cpu_percent() deltas persist."""
+        proc = self._proc_cache.get(pid)
+        if proc is None or not proc.is_running():
+            proc = psutil.Process(pid)
+            proc.cpu_percent(None)  # prime the delta baseline
+            self._proc_cache[pid] = proc
+        return proc
+
+    def _prune_events(self, live: Set[Tuple[str, str]]) -> None:
+        """Drop event series for targets no longer present (bounds cardinality)."""
+        stale = [k for k in self._events if (k[0], k[1]) not in live]
+        for key in stale:
+            del self._events[key]
+
+    @staticmethod
+    def _read_json(url: str) -> list:
+        with urllib.request.urlopen(url, timeout=2) as resp:
+            return json.loads(resp.read())
+
+
+class _Collector:
+    """Bridges :class:`ServerMetrics` state into Prometheus metric families."""
+
+    def __init__(self, metrics: ServerMetrics) -> None:
+        self._metrics = metrics
+
+    def collect(self):
+        m = self._metrics
+        snapshot = m._snapshot
+
+        browsers = GaugeMetricFamily("parsek_browsers", "Number of supervised browsers")
+        browsers.add_metric([], float(len(m._server.supervisors)))
+        yield browsers
+
+        targets = GaugeMetricFamily(
+            "parsek_targets", "Targets per browser by type", labels=["browser", "type"]
+        )
+        nested = GaugeMetricFamily(
+            "parsek_nested_targets", "Targets with a parent (nested)", labels=["browser"]
+        )
+        cpu = GaugeMetricFamily(
+            "parsek_browser_cpu_percent", "CPU%% of the browser process tree",
+            labels=["browser"],
+        )
+        mem = GaugeMetricFamily(
+            "parsek_browser_memory_bytes", "RSS of the browser process tree",
+            labels=["browser"],
+        )
+        for browser_uuid, snap in snapshot.items():
+            for type_, count in snap.targets_by_type.items():
+                targets.add_metric([browser_uuid, type_], float(count))
+            nested.add_metric([browser_uuid], float(snap.nested))
+            cpu.add_metric([browser_uuid], snap.cpu_percent)
+            mem.add_metric([browser_uuid], float(snap.memory_bytes))
+        yield targets
+        yield nested
+        yield cpu
+        yield mem
+
+        events = CounterMetricFamily(
+            "parsek_cdp_events", "CDP events seen, by target and domain",
+            labels=["browser", "target", "domain"],
+        )
+        for (browser_uuid, target_id, domain), count in list(m._events.items()):
+            events.add_metric([browser_uuid, target_id, domain], float(count))
+        yield events