funbrowser 0.1.0__py3-none-any.whl

funbrowser/pool.py

@@ -0,0 +1,152 @@
+"""BrowserPool — run a farm of Browser instances with bounded concurrency.
+
+Each pool holds up to ``size`` :class:`Browser` instances, created lazily on
+first use and kept alive between tasks. Acquire one via the
+``acquire()`` async-context-manager, or submit a callable via ``run(fn)`` /
+``run_all([fn, ...])`` and the pool will dispatch + retrieve the result.
+
+If ``proxies`` is given, each browser in the pool gets the next proxy from
+the list (round-robin by creation order). Combined with
+``geo_autoconfigure=True`` (default), the result is a fleet of browsers
+each pinned to a different exit IP + timezone + locale.
+
+Single-process only — for multi-machine farms, run separate pools.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Sequence
+from contextlib import asynccontextmanager
+from types import TracebackType
+from typing import Any, Self, TypeVar
+
+from .browser import Browser
+from .proxy import Proxy
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class BrowserPool:
+    def __init__(
+        self,
+        size: int = 5,
+        *,
+        proxies: Sequence[str | Proxy] | None = None,
+        **browser_kwargs: Any,
+    ) -> None:
+        if size < 1:
+            raise ValueError("pool size must be >= 1")
+        self._size = size
+        self._proxies = list(proxies) if proxies else None
+        self._browser_kwargs = browser_kwargs
+        self._lock = asyncio.Lock()
+        self._created: list[Browser] = []
+        self._available: asyncio.Queue[Browser] = asyncio.Queue()
+        self._closed = False
+
+    @property
+    def size(self) -> int:
+        return self._size
+
+    @property
+    def created(self) -> int:
+        """Number of browsers actually spawned so far (lazy)."""
+        return len(self._created)
+
+    @property
+    def idle(self) -> int:
+        """How many of the created browsers are currently free."""
+        return self._available.qsize()
+
+    @property
+    def busy(self) -> int:
+        """How many of the created browsers are currently in-use."""
+        return len(self._created) - self._available.qsize()
+
+    @property
+    def browsers(self) -> tuple[Browser, ...]:
+        """Snapshot of all created browsers (busy or idle)."""
+        return tuple(self._created)
+
+    async def _spawn(self, index: int) -> Browser:
+        kwargs = dict(self._browser_kwargs)
+        if self._proxies:
+            kwargs["proxy"] = self._proxies[index % len(self._proxies)]
+        logger.debug("pool: spawning browser %d/%d", index + 1, self._size)
+        return await Browser.start(**kwargs)
+
+    @asynccontextmanager
+    async def acquire(self) -> AsyncIterator[Browser]:
+        if self._closed:
+            raise RuntimeError("pool is closed")
+
+        # Fast path — an idle browser is available.
+        browser: Browser | None = None
+        try:
+            browser = self._available.get_nowait()
+        except asyncio.QueueEmpty:
+            browser = None
+
+        if browser is None:
+            # Either lazy-spawn a fresh one (under cap) or wait for a busy
+            # browser to come back. The lock serialises the check.
+            async with self._lock:
+                try:
+                    browser = self._available.get_nowait()
+                except asyncio.QueueEmpty:
+                    if len(self._created) < self._size:
+                        idx = len(self._created)
+                        browser = await self._spawn(idx)
+                        self._created.append(browser)
+            if browser is None:
+                browser = await self._available.get()
+
+        try:
+            yield browser
+        finally:
+            if not self._closed:
+                self._available.put_nowait(browser)
+
+    async def run(self, task: Callable[[Browser], Awaitable[T]]) -> T:
+        """Acquire a browser, run ``task(browser)``, release. Returns the result."""
+        async with self.acquire() as browser:
+            return await task(browser)
+
+    async def run_all(self, tasks: Iterable[Callable[[Browser], Awaitable[T]]]) -> list[T]:
+        """Dispatch every task across the pool concurrently and gather results.
+
+        At most :attr:`size` tasks execute in parallel; the rest queue.
+        """
+        return list(await asyncio.gather(*(self.run(t) for t in tasks)))
+
+    async def stop(self) -> None:
+        """Tear down every created browser. The pool is then unusable."""
+        if self._closed:
+            return
+        self._closed = True
+        await asyncio.gather(
+            *(b.stop() for b in self._created),
+            return_exceptions=True,
+        )
+        self._created.clear()
+        # Drain the queue so nobody waits forever on a closed pool.
+        while not self._available.empty():
+            try:
+                self._available.get_nowait()
+            except asyncio.QueueEmpty:
+                break
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        await self.stop()

funbrowser/profile.py

@@ -0,0 +1,73 @@
+"""Persistent browser profiles.
+
+A profile is a directory Chrome uses for cookies, localStorage, IndexedDB,
+extensions, history, and login state. Two sessions sharing the same
+profile directory share that state; two sessions with different directories
+are isolated.
+
+Lifecycle:
+- ``Profile.ensure("alice")`` returns the path under ``./funbrowser_profiles/alice``,
+  creating it if missing.
+- Pass the path to ``funbrowser.start(user_data_dir=...)``.
+- ``Profile.delete("alice")`` wipes the directory (use to log out / reset).
+- ``Profile.list()`` enumerates profiles under the default root.
+
+Only one Chrome instance per profile directory can run at a time — Chrome
+holds a lock on the dir. Spawn a second instance against the same profile
+and Chrome will exit early.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import shutil
+from pathlib import Path
+
+_SAFE_NAME = re.compile(r"^[A-Za-z0-9_.\-]+$")
+
+
+def _default_root() -> Path:
+    env = os.environ.get("FUNBROWSER_PROFILES")
+    if env:
+        return Path(env)
+    return Path.cwd() / "funbrowser_profiles"
+
+
+class Profile:
+    @staticmethod
+    def root() -> Path:
+        return _default_root()
+
+    @staticmethod
+    def path(name: str, *, root: Path | None = None) -> Path:
+        if not _SAFE_NAME.match(name):
+            raise ValueError(f"profile name {name!r} must match {_SAFE_NAME.pattern!r}")
+        return (root or _default_root()) / name
+
+    @staticmethod
+    def ensure(name: str, *, root: Path | None = None) -> Path:
+        """Return the profile path, creating the directory if needed."""
+        p = Profile.path(name, root=root)
+        p.mkdir(parents=True, exist_ok=True)
+        return p
+
+    @staticmethod
+    def exists(name: str, *, root: Path | None = None) -> bool:
+        return Profile.path(name, root=root).is_dir()
+
+    @staticmethod
+    def delete(name: str, *, root: Path | None = None) -> bool:
+        """Remove the profile directory. Returns True if it existed."""
+        p = Profile.path(name, root=root)
+        if not p.exists():
+            return False
+        shutil.rmtree(p, ignore_errors=True)
+        return True
+
+    @staticmethod
+    def list(*, root: Path | None = None) -> list[str]:
+        r = root or _default_root()
+        if not r.is_dir():
+            return []
+        return sorted(p.name for p in r.iterdir() if p.is_dir())

funbrowser/proxy.py

@@ -0,0 +1,236 @@
+"""Proxy parsing for every format a proxy provider has ever invented.
+
+Accepted on the wire — pass any of these to ``funbrowser.start(proxy=...)``:
+
+- ``scheme://user:pass@host:port`` — RFC 3986
+- ``scheme://host:port``
+- ``user:pass@host:port``                — same with implicit ``http``
+- ``host:port@user:pass``                — some Bright Data exports
+- ``host:port``                          — no auth
+- ``host:port:user:pass``                — IPRoyal, Smartproxy, most lists
+- ``user:pass:host:port``                — some legacy lists
+- ``host:port:user``                     — port-then-user (no password)
+
+Schemes recognised: ``http``, ``https``, ``socks4``, ``socks5``, ``socks5h``.
+
+The format is auto-detected by inspecting which segment contains a valid
+TCP port and which side of an ``@`` looks like a ``host:port`` pair.
+
+HTTP/HTTPS authentication is plumbed through CDP automatically. SOCKS
+authentication isn't exposed by Chrome at the HTTP-auth layer; front it
+with a local HTTP proxy that adds credentials upstream.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .tab import Tab
+
+logger = logging.getLogger(__name__)
+
+VALID_SCHEMES = frozenset({"http", "https", "socks", "socks4", "socks5", "socks5h"})
+
+_IPV4 = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
+
+
+class ProxyParseError(ValueError):
+    """Raised when a proxy string cannot be parsed into a Proxy."""
+
+
+@dataclass(frozen=True, slots=True)
+class Proxy:
+    scheme: str
+    host: str
+    port: int
+    username: str | None = None
+    password: str | None = None
+
+    @property
+    def has_auth(self) -> bool:
+        return self.username is not None
+
+    @property
+    def is_socks(self) -> bool:
+        return self.scheme.startswith("socks")
+
+    def chrome_arg(self) -> str:
+        """``--proxy-server=`` value. Auth is excluded — Chrome ignores it in the URL."""
+        scheme = "socks5" if self.scheme == "socks5h" else self.scheme
+        return f"{scheme}://{self.host}:{self.port}"
+
+    def url(self) -> str:
+        """Full URL including auth, for libraries that take a single proxy URL."""
+        if self.has_auth:
+            return f"{self.scheme}://{self.username}:{self.password}@{self.host}:{self.port}"
+        return f"{self.scheme}://{self.host}:{self.port}"
+
+
+def parse(s: str | Proxy) -> Proxy:
+    """Parse a proxy string in any supported format. See module docstring."""
+    if isinstance(s, Proxy):
+        return s
+    if not isinstance(s, str):
+        raise ProxyParseError(f"expected str or Proxy, got {type(s).__name__}")
+    raw = s.strip()
+    if not raw:
+        raise ProxyParseError("empty proxy string")
+
+    scheme = "http"
+    if "://" in raw:
+        scheme_part, _, rest = raw.partition("://")
+        scheme = scheme_part.lower()
+        if scheme not in VALID_SCHEMES:
+            raise ProxyParseError(f"unknown scheme {scheme!r}")
+        raw = rest
+
+    if "@" in raw:
+        return _parse_with_at(scheme, raw)
+    return _parse_colon_only(scheme, raw)
+
+
+def _parse_with_at(scheme: str, s: str) -> Proxy:
+    left, _, right = s.rpartition("@")
+    if _looks_like_server(right) and not _looks_like_server(left):
+        auth, server = left, right
+    elif _looks_like_server(left) and not _looks_like_server(right):
+        auth, server = right, left
+    elif _looks_like_server(right) and _looks_like_server(left):
+        # Both sides look like host:port — prefer the standard interpretation
+        # (auth on the left, server on the right).
+        auth, server = left, right
+    else:
+        raise ProxyParseError(f"could not locate host:port on either side of '@' in {s!r}")
+    user, _, pwd = auth.partition(":")
+    if not user:
+        raise ProxyParseError(f"empty username in {s!r}")
+    host, port = _split_host_port(server)
+    return Proxy(scheme, host, port, user, pwd or None)
+
+
+def _parse_colon_only(scheme: str, s: str) -> Proxy:
+    parts = s.split(":")
+    if len(parts) == 2:
+        host, port = _split_host_port(s)
+        return Proxy(scheme, host, port)
+
+    if len(parts) == 3:
+        # host:port:user — port-then-user, no password
+        if _is_port(parts[1]) and _looks_like_host(parts[0]):
+            return Proxy(scheme, parts[0], int(parts[1]), parts[2])
+        raise ProxyParseError(f"ambiguous 3-segment proxy {s!r}")
+
+    if len(parts) == 4:
+        a, b, c, d = parts
+        b_is_port = _is_port(b)
+        d_is_port = _is_port(d)
+        if b_is_port and not d_is_port:
+            return Proxy(scheme, a, int(b), c, d)
+        if d_is_port and not b_is_port:
+            return Proxy(scheme, c, int(d), a, b)
+        if b_is_port and d_is_port:
+            # Both look like ports; disambiguate by the host slot.
+            if _looks_like_host(a) and not _looks_like_host(c):
+                return Proxy(scheme, a, int(b), c, d)
+            if _looks_like_host(c) and not _looks_like_host(a):
+                return Proxy(scheme, c, int(d), a, b)
+            # Final fallback: assume host:port:user:pass (the more common
+            # listing convention from proxy providers).
+            return Proxy(scheme, a, int(b), c, d)
+        raise ProxyParseError(f"could not find a port in 4-segment proxy {s!r}")
+
+    raise ProxyParseError(f"could not parse proxy {s!r}")
+
+
+def _is_port(s: str) -> bool:
+    return s.isdigit() and 1 <= int(s) <= 65535
+
+
+def _looks_like_host(s: str) -> bool:
+    if not s:
+        return False
+    if s == "localhost":
+        return True
+    if _IPV4.match(s):
+        return True
+    if "." in s and any(ch.isalpha() for ch in s):
+        return True
+    return False
+
+
+def _looks_like_server(s: str) -> bool:
+    if ":" not in s:
+        return False
+    host, _, port = s.rpartition(":")
+    return bool(host) and _is_port(port)
+
+
+def _split_host_port(s: str) -> tuple[str, int]:
+    host, _, port = s.rpartition(":")
+    if not host:
+        raise ProxyParseError(f"missing host in {s!r}")
+    if not _is_port(port):
+        raise ProxyParseError(f"invalid port in {s!r}")
+    return host, int(port)
+
+
+async def attach_auth(tab: Tab, proxy: Proxy) -> None:
+    """Install a per-tab CDP handler that satisfies HTTP/HTTPS proxy auth.
+
+    SOCKS proxies don't surface their auth as an HTTP challenge — Chrome
+    doesn't expose a public hook for it. SOCKS-with-auth callers should
+    front the SOCKS server with a local HTTP proxy that adds credentials
+    upstream (e.g. ``microsocks``, ``proxychains``, or ``proxy.py``).
+    """
+    if not proxy.has_auth:
+        return
+    if proxy.is_socks:
+        logger.warning(
+            "SOCKS proxy auth (%s) is not wired through CDP; Chrome will "
+            "fail the connection. Front with a local HTTP proxy that adds "
+            "credentials upstream.",
+            proxy.host,
+        )
+        return
+
+    # `handleAuthRequests=True` without `patterns` means only auth-required
+    # requests are paused — pass-through stays fast for everything else.
+    await tab._send("Fetch.enable", {"handleAuthRequests": True})
+
+    user = proxy.username or ""
+    pwd = proxy.password or ""
+
+    async def _on_auth_required(params: dict[str, object]) -> None:
+        try:
+            await tab._cdp.send(
+                "Fetch.continueWithAuth",
+                {
+                    "requestId": params["requestId"],
+                    "authChallengeResponse": {
+                        "response": "ProvideCredentials",
+                        "username": user,
+                        "password": pwd,
+                    },
+                },
+                session_id=tab.session_id,
+            )
+        except Exception:
+            logger.exception("proxy auth: continueWithAuth failed")
+
+    async def _on_request_paused(params: dict[str, object]) -> None:
+        # Paired with each auth-required request; just let it through.
+        try:
+            await tab._cdp.send(
+                "Fetch.continueRequest",
+                {"requestId": params["requestId"]},
+                session_id=tab.session_id,
+            )
+        except Exception:
+            pass  # request may already have been resolved by the auth callback
+
+    tab._cdp.on("Fetch.authRequired", _on_auth_required, session_id=tab.session_id)
+    tab._cdp.on("Fetch.requestPaused", _on_request_paused, session_id=tab.session_id)

funbrowser/solver/__init__.py

@@ -0,0 +1,12 @@
+"""Solver — auto-solve captchas through the funsolver.com API.
+
+Scope (M3): Cloudflare Turnstile. M4 adds the rest of the major captcha
+families.
+"""
+
+from __future__ import annotations
+
+from .bridge import apply_solver
+from .client import FunSolverClient, FunSolverError
+
+__all__ = ["FunSolverClient", "FunSolverError", "apply_solver"]

funbrowser/solver/bridge.py

@@ -0,0 +1,167 @@
+"""Wire funsolver.com calls onto a Tab via a CDP binding.
+
+Mechanism:
+- ``Runtime.addBinding`` registers ``window.__funbrowser_solve`` so any JS
+  call to it shows up on the CDP side as a ``Runtime.bindingCalled`` event.
+- The bootstrap script gives the page a Promise-based API
+  (``window.__funbrowser.solve(req)``) that wraps the binding call.
+- The Python handler reads the payload, talks to funsolver.com via the
+  ``FunSolverClient``, and pushes the result back with ``Runtime.evaluate``
+  calling ``window.__funbrowser_resolve(id, {ok, token|error})``.
+
+Note: this requires ``Runtime.enable``, which has a known minor antibot
+tell (CDP frames appear in error stacks). Moving the binding to an
+isolated world via ``Page.createIsolatedWorld`` is planned for a
+follow-up — it keeps the binding off the page's main world entirely.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from importlib.resources import files
+from typing import TYPE_CHECKING, Any
+
+from .client import FunSolverClient, FunSolverError
+
+if TYPE_CHECKING:
+    from ..tab import Tab
+
+logger = logging.getLogger(__name__)
+
+BINDING_NAME = "__funbrowser_solve"
+SCRIPTS = (
+    "_bootstrap.js",
+    "turnstile.js",
+    "recaptcha_v2.js",
+    "recaptcha_v3.js",
+    "hcaptcha.js",
+    "funcaptcha.js",
+    "geetest.js",
+)
+
+
+def _load_scripts() -> str:
+    pkg = files("funbrowser.solver.scripts")
+    return "\n".join(pkg.joinpath(s).read_text(encoding="utf-8") for s in SCRIPTS)
+
+
+_SOLVER_SOURCE = _load_scripts()
+
+
+async def _solve_dispatch(client: FunSolverClient, payload: dict[str, Any]) -> str:
+    cap_type = payload.get("type")
+    if cap_type == "turnstile":
+        return await client.solve_turnstile(
+            sitekey=payload["sitekey"],
+            page_url=payload["url"],
+            action=payload.get("action"),
+            cdata=payload.get("cdata"),
+        )
+    if cap_type == "recaptcha2":
+        return await client.solve_recaptcha_v2(
+            sitekey=payload["sitekey"],
+            page_url=payload["url"],
+            invisible=bool(payload.get("invisible", False)),
+            data_s=payload.get("dataS"),
+            is_enterprise=bool(payload.get("enterprise", False)),
+        )
+    if cap_type == "recaptcha3":
+        return await client.solve_recaptcha_v3(
+            sitekey=payload["sitekey"],
+            page_url=payload["url"],
+            action=payload.get("action", "verify"),
+            min_score=float(payload.get("minScore", 0.7)),
+            is_enterprise=bool(payload.get("enterprise", False)),
+        )
+    if cap_type == "hcaptcha":
+        return await client.solve_hcaptcha(
+            sitekey=payload["sitekey"],
+            page_url=payload["url"],
+            is_invisible=bool(payload.get("invisible", False)),
+        )
+    if cap_type == "funcaptcha":
+        return await client.solve_funcaptcha(
+            public_key=payload["sitekey"],
+            page_url=payload["url"],
+            surl=payload.get("surl"),
+            data=payload.get("blob"),
+        )
+    if cap_type == "geetest":
+        return await client.solve_geetest(
+            gt=payload["gt"],
+            challenge=payload["challenge"],
+            page_url=payload["url"],
+            api_server=payload.get("apiServer"),
+            version=int(payload.get("version", 3)),
+        )
+    raise FunSolverError(f"unsupported captcha type: {cap_type!r}")
+
+
+async def apply_solver(tab: Tab, client: FunSolverClient) -> None:
+    """Attach the funsolver bridge + per-captcha detectors to a Tab."""
+    await tab._send("Runtime.enable")
+    await tab._send("Runtime.addBinding", {"name": BINDING_NAME})
+
+    async def on_binding_called(params: dict[str, Any]) -> None:
+        if params.get("name") != BINDING_NAME:
+            return
+        try:
+            payload = json.loads(params["payload"])
+        except (KeyError, ValueError):
+            logger.exception("solver: malformed binding payload")
+            return
+
+        task_id = payload.get("id")
+        cap_type = str(payload.get("type", "?"))
+        t0 = time.monotonic()
+        try:
+            token = await _solve_dispatch(client, payload)
+            ms = (time.monotonic() - t0) * 1000.0
+            result: dict[str, Any] = {"ok": True, "token": token}
+            tab._browser.record_event(
+                kind="captcha",
+                captcha=cap_type,
+                ok=True,
+                ms=ms,
+                token_preview=(token[:24] + "…") if len(token) > 24 else token,
+                url=payload.get("url"),
+            )
+        except Exception as exc:
+            ms = (time.monotonic() - t0) * 1000.0
+            logger.warning("solver: solve failed: %s", exc)
+            result = {"ok": False, "error": str(exc)}
+            tab._browser.record_event(
+                kind="captcha",
+                captcha=cap_type,
+                ok=False,
+                ms=ms,
+                error=str(exc),
+                url=payload.get("url"),
+            )
+
+        try:
+            await tab._cdp.send(
+                "Runtime.evaluate",
+                {
+                    "expression": (
+                        f"window.__funbrowser_resolve({json.dumps(task_id)}, {json.dumps(result)})"
+                    ),
+                    "awaitPromise": False,
+                },
+                session_id=tab.session_id,
+            )
+        except Exception:
+            logger.exception("solver: failed to push result back to page")
+
+    tab._cdp.on(
+        "Runtime.bindingCalled",
+        on_binding_called,
+        session_id=tab.session_id,
+    )
+
+    await tab._send(
+        "Page.addScriptToEvaluateOnNewDocument",
+        {"source": _SOLVER_SOURCE, "runImmediately": True},
+    )