funbrowser 0.1.0__py3-none-any.whl

funbrowser/browser.py

@@ -0,0 +1,281 @@
+"""Browser — the launched Chrome process plus the CDP control plane."""
+
+from __future__ import annotations
+
+import asyncio
+import dataclasses
+import shutil
+import time
+from collections import deque
+from collections.abc import Sequence
+from pathlib import Path
+from types import TracebackType
+from typing import TYPE_CHECKING, Any, Self
+
+from ._cdp import CDPConnection
+from ._flags import merge_flags, mini_flags
+from ._launcher import LaunchedBrowser, launch_chrome
+from .fingerprint import Fingerprint
+from .geo import GeoInfo, lookup_proxy_geo
+from .humanly import DEFAULT as DEFAULT_HUMANLY
+from .humanly import HumanBehavior
+from .proxy import Proxy
+from .proxy import parse as parse_proxy
+from .solver import FunSolverClient
+from .stealth import stealth_flags
+from .tab import Tab
+
+if TYPE_CHECKING:
+    from .context import BrowserContext as BrowserContextT
+
+
+def _enrich_with_geo(fp: Fingerprint | None, geo: GeoInfo) -> Fingerprint:
+    """Layer a geo lookup into a Fingerprint without clobbering caller values."""
+    if fp is None:
+        return Fingerprint(
+            timezone=geo.timezone or None,
+            locale=geo.locale or None,
+            accept_language=geo.accept_language or None,
+        )
+    return dataclasses.replace(
+        fp,
+        timezone=fp.timezone or geo.timezone or None,
+        locale=fp.locale or geo.locale or None,
+        accept_language=fp.accept_language or geo.accept_language or None,
+    )
+
+
+class Browser:
+    def __init__(
+        self,
+        launched: LaunchedBrowser,
+        cdp: CDPConnection,
+        *,
+        stealth: bool = True,
+        fingerprint: Fingerprint | None = None,
+        proxy: Proxy | None = None,
+        humanly: HumanBehavior | None = None,
+        solver_client: FunSolverClient | None = None,
+        geo: GeoInfo | None = None,
+    ) -> None:
+        self._launched = launched
+        self._cdp = cdp
+        self._stealth = stealth
+        self._fingerprint = fingerprint
+        self._proxy = proxy
+        self._humanly = humanly
+        self._solver_client = solver_client
+        self._geo = geo
+        self._tabs: dict[str, Tab] = {}
+        # Browser-scoped event log — populated by the solver bridge whenever
+        # a captcha is attempted. Panel and any other observer can read it.
+        self._events: deque[dict[str, Any]] = deque(maxlen=100)
+        # Browser contexts (CDP browserContextIds) currently alive on this
+        # process. Populated by funbrowser.context._create_context_on,
+        # drained by BrowserContext.close.
+        self._contexts: set[str] = set()
+
+    @property
+    def stealth_enabled(self) -> bool:
+        return self._stealth
+
+    @property
+    def fingerprint(self) -> Fingerprint | None:
+        return self._fingerprint
+
+    @property
+    def proxy(self) -> Proxy | None:
+        return self._proxy
+
+    @property
+    def auto_solve_enabled(self) -> bool:
+        return self._solver_client is not None
+
+    @property
+    def solver_client(self) -> FunSolverClient | None:
+        return self._solver_client
+
+    @property
+    def humanly(self) -> HumanBehavior | None:
+        return self._humanly
+
+    @property
+    def events(self) -> tuple[dict[str, Any], ...]:
+        """Recent browser-scoped events (most recent first)."""
+        return tuple(self._events)
+
+    def record_event(self, **fields: Any) -> None:
+        """Append an event to the browser's log. Timestamp added automatically."""
+        self._events.appendleft({"ts": time.time(), **fields})
+
+    async def create_context(
+        self,
+        *,
+        proxy: str | Proxy | None = None,
+        proxy_bypass: list[str] | None = None,
+    ) -> BrowserContextT:
+        """Create an isolated :class:`BrowserContext` inside this Browser.
+
+        Cheap (~5-15 MB) compared to launching a fresh Chrome. Each
+        context has its own cookies / storage / cache, and optionally
+        its own proxy. See :mod:`funbrowser.context` for the trade-offs.
+        """
+        from .context import _create_context_on
+
+        return await _create_context_on(self, proxy=proxy, proxy_bypass=proxy_bypass)
+
+    @property
+    def geo(self) -> GeoInfo | None:
+        return self._geo
+
+    @classmethod
+    async def start(
+        cls,
+        *,
+        executable: str | Path | None = None,
+        user_data_dir: str | Path | None = None,
+        headless: bool = False,
+        stealth: bool = True,
+        fingerprint: Fingerprint | None = None,
+        proxy: str | Proxy | None = None,
+        geo_autoconfigure: bool = True,
+        humanly: bool | HumanBehavior = False,
+        mini: bool = False,
+        api_key: str | None = None,
+        auto_solve: bool = True,
+        solver_base_url: str | None = None,
+        args: Sequence[str] = (),
+    ) -> Self:
+        parts: list[list[str]] = []
+        if stealth:
+            parts.append(stealth_flags())
+        if mini:
+            parts.append(mini_flags())
+
+        proxy_obj: Proxy | None = None
+        if proxy is not None:
+            proxy_obj = parse_proxy(proxy)
+            parts.append([f"--proxy-server={proxy_obj.chrome_arg()}"])
+
+        parts.append(list(args))
+        extra = merge_flags(*parts)
+
+        launched = await launch_chrome(
+            executable=Path(executable) if executable else None,
+            user_data_dir=Path(user_data_dir) if user_data_dir else None,
+            headless=headless,
+            extra_args=extra,
+        )
+        cdp = CDPConnection(launched.ws_url)
+        await cdp.connect()
+
+        solver_client: FunSolverClient | None = None
+        if api_key and auto_solve:
+            if solver_base_url:
+                solver_client = FunSolverClient(api_key, base_url=solver_base_url)
+            else:
+                solver_client = FunSolverClient(api_key)
+
+        humanly_profile: HumanBehavior | None
+        if humanly is True:
+            humanly_profile = DEFAULT_HUMANLY
+        elif humanly is False:
+            humanly_profile = None
+        elif isinstance(humanly, HumanBehavior):
+            humanly_profile = humanly
+        else:
+            humanly_profile = None
+
+        # Geo auto-coupling: ask ip-api.com (through the proxy) for the exit
+        # IP's timezone + locale, fill any matching fingerprint fields the
+        # caller didn't already set. Skip silently on any failure.
+        geo: GeoInfo | None = None
+        if proxy_obj is not None and geo_autoconfigure:
+            geo = await lookup_proxy_geo(proxy_obj)
+            if geo is not None:
+                fingerprint = _enrich_with_geo(fingerprint, geo)
+
+        return cls(
+            launched,
+            cdp,
+            stealth=stealth,
+            fingerprint=fingerprint,
+            proxy=proxy_obj,
+            humanly=humanly_profile,
+            solver_client=solver_client,
+            geo=geo,
+        )
+
+    @property
+    def tabs(self) -> list[Tab]:
+        return list(self._tabs.values())
+
+    async def new_tab(self, url: str = "about:blank") -> Tab:
+        res = await self._cdp.send("Target.createTarget", {"url": url})
+        target_id = res["targetId"]
+        attach = await self._cdp.send(
+            "Target.attachToTarget",
+            {"targetId": target_id, "flatten": True},
+        )
+        session_id = attach["sessionId"]
+        tab = Tab(self, target_id, session_id)
+        await tab._initialize()
+        self._tabs[target_id] = tab
+        return tab
+
+    async def get(self, url: str, *, wait_until: str = "load") -> Tab:
+        tab = await self.new_tab(url="about:blank")
+        await tab.goto(url, wait_until=wait_until)
+        return tab
+
+    def _on_tab_closed(self, tab: Tab) -> None:
+        self._tabs.pop(tab.target_id, None)
+
+    # ── cookie store (browser-wide) ───────────────────────────────────
+
+    async def cookies(self) -> list[dict[str, Any]]:
+        """Every cookie the browser is currently holding."""
+        result = await self._cdp.send("Storage.getCookies")
+        return list(result.get("cookies", []))
+
+    async def set_cookies(self, cookies: Sequence[dict[str, Any]]) -> None:
+        """Add or overwrite cookies. Same dict shape as :meth:`cookies` returns."""
+        await self._cdp.send("Storage.setCookies", {"cookies": list(cookies)})
+
+    async def clear_cookies(self) -> None:
+        """Wipe every cookie. Useful between tests."""
+        await self._cdp.send("Storage.clearCookies")
+
+    async def stop(self) -> None:
+        for tab in list(self._tabs.values()):
+            try:
+                await tab.close()
+            except Exception:
+                pass
+        await self._cdp.close()
+        if self._solver_client is not None:
+            try:
+                await self._solver_client.close()
+            except Exception:
+                pass
+        proc = self._launched.process
+        if proc.returncode is None:
+            try:
+                proc.terminate()
+                await asyncio.wait_for(proc.wait(), timeout=5.0)
+            except TimeoutError:
+                proc.kill()
+                await proc.wait()
+        if self._launched.user_data_dir_is_tmp:
+            shutil.rmtree(self._launched.user_data_dir, ignore_errors=True)
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        await self.stop()

funbrowser/context.py

@@ -0,0 +1,163 @@
+"""BrowserContext — an isolated browsing context inside a shared Browser.
+
+CDP exposes ``Target.createBrowserContext`` which carves a fresh, isolated
+browsing identity out of an existing Chrome process: separate cookies,
+localStorage, IndexedDB, cache, and (optionally) its own proxy. Each
+context behaves like its own browser to the page running inside, but a
+single Chrome process serves any number of them.
+
+For farm operators who would otherwise spawn 10-50 standalone Chrome
+processes, switching to one Chrome + N contexts cuts memory roughly
+**7-10x** (each context adds ~5-15 MB on top of the host process, vs
+~150 MB for a fresh Chrome).
+
+Trade-offs vs full :class:`Browser` per slot:
+
+- Crash in the host Chrome takes every context down with it
+- All contexts share one process — kernel-level isolation is weaker
+- Stealth + fingerprint patches still apply per-tab (each new tab gets
+  its own ``Page.addScriptToEvaluateOnNewDocument`` set)
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from .proxy import Proxy
+from .proxy import parse as parse_proxy
+from .tab import Tab
+
+if TYPE_CHECKING:
+    from .browser import Browser
+
+logger = logging.getLogger(__name__)
+
+
+class BrowserContext:
+    """One isolated browsing identity inside a shared :class:`Browser`."""
+
+    def __init__(
+        self,
+        browser: Browser,
+        context_id: str,
+        *,
+        proxy: Proxy | None = None,
+    ) -> None:
+        self._browser = browser
+        self._context_id = context_id
+        self._proxy = proxy
+        self._tabs: dict[str, Tab] = {}
+        self._closed = False
+
+    @property
+    def browser(self) -> Browser:
+        return self._browser
+
+    @property
+    def context_id(self) -> str:
+        return self._context_id
+
+    @property
+    def proxy(self) -> Proxy | None:
+        return self._proxy
+
+    @property
+    def tabs(self) -> list[Tab]:
+        return list(self._tabs.values())
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    async def new_tab(self, url: str = "about:blank") -> Tab:
+        if self._closed:
+            raise RuntimeError("context is closed")
+        res = await self._browser._cdp.send(
+            "Target.createTarget",
+            {"url": url, "browserContextId": self._context_id},
+        )
+        target_id = res["targetId"]
+        attach = await self._browser._cdp.send(
+            "Target.attachToTarget",
+            {"targetId": target_id, "flatten": True},
+        )
+        session_id = attach["sessionId"]
+        tab = Tab(self._browser, target_id, session_id)
+        await tab._initialize()
+        self._tabs[target_id] = tab
+        # Also register on the host browser so its tab counters stay accurate.
+        self._browser._tabs[target_id] = tab
+        return tab
+
+    async def get(self, url: str, *, wait_until: str = "load") -> Tab:
+        tab = await self.new_tab()
+        await tab.goto(url, wait_until=wait_until)
+        return tab
+
+    async def cookies(self) -> list[dict[str, Any]]:
+        """Cookies scoped to this context. No interference between contexts."""
+        result = await self._browser._cdp.send(
+            "Storage.getCookies",
+            {"browserContextId": self._context_id},
+        )
+        return list(result.get("cookies", []))
+
+    async def set_cookies(self, cookies: list[dict[str, Any]]) -> None:
+        await self._browser._cdp.send(
+            "Storage.setCookies",
+            {"cookies": list(cookies), "browserContextId": self._context_id},
+        )
+
+    async def clear_cookies(self) -> None:
+        await self._browser._cdp.send(
+            "Storage.clearCookies",
+            {"browserContextId": self._context_id},
+        )
+
+    async def close(self) -> None:
+        """Close every tab in this context, then dispose the context itself."""
+        if self._closed:
+            return
+        self._closed = True
+        for tab in list(self._tabs.values()):
+            try:
+                await tab.close()
+            except Exception:
+                pass
+        self._tabs.clear()
+        try:
+            await self._browser._cdp.send(
+                "Target.disposeBrowserContext",
+                {"browserContextId": self._context_id},
+            )
+        except Exception:
+            logger.exception("context: disposeBrowserContext failed for %s", self._context_id)
+        self._browser._contexts.discard(self._context_id)
+
+    async def __aenter__(self) -> BrowserContext:
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.close()
+
+
+async def _create_context_on(
+    browser: Browser,
+    *,
+    proxy: str | Proxy | None = None,
+    proxy_bypass: list[str] | None = None,
+) -> BrowserContext:
+    """Internal: create a browser context on a Browser instance."""
+    params: dict[str, Any] = {}
+    proxy_obj: Proxy | None = None
+    if proxy is not None:
+        proxy_obj = parse_proxy(proxy)
+        params["proxyServer"] = proxy_obj.chrome_arg()
+    if proxy_bypass:
+        params["proxyBypassList"] = ",".join(proxy_bypass)
+
+    res = await browser._cdp.send("Target.createBrowserContext", params)
+    context_id = str(res["browserContextId"])
+    browser._contexts.add(context_id)
+    return BrowserContext(browser, context_id, proxy=proxy_obj)

funbrowser/context_pool.py

@@ -0,0 +1,162 @@
+"""ContextPool — lightweight pool of BrowserContexts on a shared Chrome.
+
+API mirrors :class:`funbrowser.BrowserPool`: ``acquire()`` / ``run()`` /
+``run_all()`` / ``stop()`` / async-context-manager support — only the
+unit of isolation changes. Where ``BrowserPool`` keeps N Chrome
+processes alive, ``ContextPool`` keeps **one** Chrome alive with N
+isolated browser contexts.
+
+Memory comparison (typical example, headless + mini):
+
+- ``BrowserPool(size=10)``  -> ~10 x 100 MB  = ~1.0 GB
+- ``ContextPool(size=10)``  -> ~1 x 180 MB + 10 x 8 MB = ~260 MB
+
+You give up the process-level isolation: if the host Chrome crashes,
+every context dies with it. For farms where the workload is "many
+parallel lightweight scrapes" this is the right shape; for "long-lived
+high-value sessions where crash recovery matters", stick with
+``BrowserPool``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Sequence
+from contextlib import asynccontextmanager
+from types import TracebackType
+from typing import Any, Self, TypeVar
+
+from .browser import Browser
+from .context import BrowserContext
+from .proxy import Proxy
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class ContextPool:
+    def __init__(
+        self,
+        size: int = 5,
+        *,
+        proxies: Sequence[str | Proxy] | None = None,
+        **browser_kwargs: Any,
+    ) -> None:
+        if size < 1:
+            raise ValueError("pool size must be >= 1")
+        self._size = size
+        self._proxies = list(proxies) if proxies else None
+        self._browser_kwargs = browser_kwargs
+        self._lock = asyncio.Lock()
+        self._browser: Browser | None = None
+        self._contexts: list[BrowserContext] = []
+        self._available: asyncio.Queue[BrowserContext] = asyncio.Queue()
+        self._closed = False
+
+    @property
+    def size(self) -> int:
+        return self._size
+
+    @property
+    def created(self) -> int:
+        """Number of contexts actually spawned so far (lazy)."""
+        return len(self._contexts)
+
+    @property
+    def idle(self) -> int:
+        return self._available.qsize()
+
+    @property
+    def busy(self) -> int:
+        return len(self._contexts) - self._available.qsize()
+
+    @property
+    def contexts(self) -> tuple[BrowserContext, ...]:
+        return tuple(self._contexts)
+
+    @property
+    def browser(self) -> Browser | None:
+        """The shared host Browser. ``None`` until first acquire."""
+        return self._browser
+
+    async def _ensure_browser(self) -> Browser:
+        if self._browser is None:
+            self._browser = await Browser.start(**self._browser_kwargs)
+        return self._browser
+
+    async def _spawn(self, index: int) -> BrowserContext:
+        browser = await self._ensure_browser()
+        proxy = self._proxies[index % len(self._proxies)] if self._proxies else None
+        logger.debug("context-pool: spawning context %d/%d", index + 1, self._size)
+        return await browser.create_context(proxy=proxy)
+
+    @asynccontextmanager
+    async def acquire(self) -> AsyncIterator[BrowserContext]:
+        if self._closed:
+            raise RuntimeError("pool is closed")
+
+        ctx: BrowserContext | None = None
+        try:
+            ctx = self._available.get_nowait()
+        except asyncio.QueueEmpty:
+            ctx = None
+
+        if ctx is None:
+            async with self._lock:
+                try:
+                    ctx = self._available.get_nowait()
+                except asyncio.QueueEmpty:
+                    if len(self._contexts) < self._size:
+                        idx = len(self._contexts)
+                        ctx = await self._spawn(idx)
+                        self._contexts.append(ctx)
+            if ctx is None:
+                ctx = await self._available.get()
+
+        try:
+            yield ctx
+        finally:
+            if not self._closed:
+                self._available.put_nowait(ctx)
+
+    async def run(self, task: Callable[[BrowserContext], Awaitable[T]]) -> T:
+        async with self.acquire() as ctx:
+            return await task(ctx)
+
+    async def run_all(self, tasks: Iterable[Callable[[BrowserContext], Awaitable[T]]]) -> list[T]:
+        return list(await asyncio.gather(*(self.run(t) for t in tasks)))
+
+    async def stop(self) -> None:
+        if self._closed:
+            return
+        self._closed = True
+        for ctx in self._contexts:
+            try:
+                await ctx.close()
+            except Exception:
+                pass
+        self._contexts.clear()
+        while not self._available.empty():
+            try:
+                self._available.get_nowait()
+            except asyncio.QueueEmpty:
+                break
+        if self._browser is not None:
+            try:
+                await self._browser.stop()
+            except Exception:
+                pass
+            self._browser = None
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        await self.stop()