dataapp_crawler 0.1.0__py3-none-any.whl

dataapp_crawler/providers/pool.py

@@ -0,0 +1,451 @@
+"""Bounded native Playwright process pool with isolated per-open contexts."""
+
+from __future__ import annotations
+
+import asyncio
+import math
+import time
+from dataclasses import dataclass, field
+from typing import Any, TypeVar
+
+from ..errors import ConfigurationError
+from ..models import Identity
+from .browser import PlaywrightBrowser
+from .proxy import ProxyHandle
+
+_CAPABILITIES = frozenset(
+    {"page", "cookies", "origin_headers", "proxy", "storage_state", "browser_pool"}
+)
+T = TypeVar("T")
+
+
+async def _await_invariant(task: asyncio.Task[T]) -> tuple[T, asyncio.CancelledError | None]:
+    """Finish an ownership transition even if its caller is cancelled repeatedly."""
+
+    cancelled: asyncio.CancelledError | None = None
+    while not task.done():
+        try:
+            await asyncio.shield(task)
+        except asyncio.CancelledError as error:
+            cancelled = cancelled or error
+    return task.result(), cancelled
+
+
+def _positive_int(name: str, value: int) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        raise ConfigurationError(f"{name} must be a positive integer")
+    return value
+
+
+def _positive_timeout(name: str, value: float) -> float:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise ConfigurationError(f"{name} must be finite and positive")
+    result = float(value)
+    if not math.isfinite(result) or result <= 0:
+        raise ConfigurationError(f"{name} must be finite and positive")
+    return result
+
+
+@dataclass(eq=False)
+class _Process:
+    playwright: Any = field(repr=False)
+    browser: Any = field(repr=False)
+    contexts: int = 0
+    close_task: asyncio.Task[None] | None = field(default=None, repr=False)
+
+    def connected(self) -> bool:
+        check = getattr(self.browser, "is_connected", None)
+        if not callable(check):
+            return True
+        try:
+            return bool(check())
+        except Exception:
+            return False
+
+
+class PooledBrowserHandle:
+    """One isolated context leased from :class:`BrowserPool`."""
+
+    provider = "playwright_pool"
+    capabilities = _CAPABILITIES
+    owned = True
+
+    def __init__(
+        self,
+        pool: BrowserPool,
+        process: _Process,
+        context: Any,
+        *,
+        max_pages: int,
+        page_acquire_timeout: float,
+    ) -> None:
+        self.context = context
+        self._pool = pool
+        self._process = process
+        self._page_slots = asyncio.Semaphore(max_pages)
+        self._page_acquire_timeout = page_acquire_timeout
+        self._pages: list[Any] = []
+        self._state_lock = asyncio.Lock()
+        self._closed = False
+        self._close_task: asyncio.Task[None] | None = None
+
+    async def new_page(self) -> Any:
+        acquired = False
+        try:
+            try:
+                async with asyncio.timeout(self._page_acquire_timeout):
+                    await self._page_slots.acquire()
+            except TimeoutError:
+                raise ConfigurationError(
+                    "BrowserPool page capacity acquisition timed out"
+                ) from None
+            acquired = True
+
+            async with self._state_lock:
+                if self._closed:
+                    raise RuntimeError("Browser handle is closed")
+                page = await self.context.new_page()
+                self._pages.append(page)
+                acquired = False
+                return page
+        finally:
+            if acquired:
+                self._page_slots.release()
+
+    async def close_page(self, page: Any) -> None:
+        cancelled: asyncio.CancelledError | None = None
+        failure: BaseException | None = None
+        async with self._state_lock:
+            if page not in self._pages:
+                return
+            close_task = asyncio.create_task(asyncio.wait_for(page.close(), timeout=5))
+            try:
+                await asyncio.shield(close_task)
+            except asyncio.CancelledError as error:
+                cancelled = error
+                try:
+                    await asyncio.shield(close_task)
+                except BaseException as close_error:
+                    failure = close_error
+            except BaseException as error:
+                failure = error
+            if failure is None:
+                self._pages.remove(page)
+                self._page_slots.release()
+
+        if failure is not None:
+            # A page that cannot be closed makes this context unsafe to reuse.  The
+            # handle close task retries bounded page cleanup, then always closes the
+            # context and returns its pool reservation.
+            try:
+                await self.close()
+            except BaseException:
+                pass
+            if cancelled is not None:
+                raise cancelled
+            raise ConfigurationError("Browser page cleanup failed") from None
+        if cancelled is not None:
+            raise cancelled
+
+    async def close(self) -> None:
+        async with self._state_lock:
+            if self._close_task is None:
+                self._closed = True
+                pages = list(reversed(self._pages))
+                self._pages.clear()
+                self._close_task = asyncio.create_task(self._close_impl(pages))
+            task = self._close_task
+        await asyncio.shield(task)
+
+    async def _close_impl(self, pages: list[Any]) -> None:
+        first_error: BaseException | None = None
+        context_failed = False
+        for page in pages:
+            try:
+                await asyncio.wait_for(page.close(), timeout=5)
+            except BaseException as error:
+                first_error = first_error or error
+        try:
+            await asyncio.wait_for(self.context.close(), timeout=5)
+        except BaseException as error:
+            context_failed = True
+            first_error = first_error or error
+        finally:
+            await self._pool._release(self._process, self, retire=context_failed)
+        if first_error is not None:
+            if not isinstance(first_error, Exception):
+                raise first_error
+            raise ConfigurationError("Pooled browser context cleanup failed") from None
+
+
+class BrowserPool:
+    """Share bounded native processes while isolating each open in its own context.
+
+    The pool owns browser processes and Playwright clients.  Handles own only their
+    context and pages.  Close all handles (normally by closing the Crawler or
+    SessionPool) before closing this pool.  ``capacity`` is the maximum number of
+    simultaneous contexts; a SessionPool that keeps idle browser handles should not
+    have a larger ``max_size`` unless finite acquisition failures are acceptable.
+    """
+
+    def __init__(
+        self,
+        provider: PlaywrightBrowser,
+        *,
+        max_browsers: int = 2,
+        max_contexts_per_browser: int = 4,
+        max_pages_per_context: int = 1,
+        acquire_timeout: float = 30,
+        page_acquire_timeout: float = 30,
+    ) -> None:
+        if not isinstance(provider, PlaywrightBrowser):
+            raise ConfigurationError("BrowserPool requires a PlaywrightBrowser provider")
+        self.provider = provider
+        self.max_browsers = _positive_int("max_browsers", max_browsers)
+        self.max_contexts_per_browser = _positive_int(
+            "max_contexts_per_browser", max_contexts_per_browser
+        )
+        self.max_pages_per_context = _positive_int(
+            "max_pages_per_context", max_pages_per_context
+        )
+        self.acquire_timeout = _positive_timeout("acquire_timeout", acquire_timeout)
+        self.page_acquire_timeout = _positive_timeout(
+            "page_acquire_timeout", page_acquire_timeout
+        )
+        self._condition = asyncio.Condition()
+        self._processes: list[_Process] = []
+        self._retiring: set[_Process] = set()
+        self._handles: set[PooledBrowserHandle] = set()
+        self._starting = 0
+        self._closed = False
+        self._close_task: asyncio.Task[None] | None = None
+
+    @property
+    def capacity(self) -> int:
+        return self.max_browsers * self.max_contexts_per_browser
+
+    async def open(
+        self, *, identity: Identity | None = None, proxy: ProxyHandle | None = None
+    ) -> PooledBrowserHandle:
+        process = await self._reserve()
+        try:
+            context = await self.provider._open_context(
+                process.browser, identity=identity, proxy=proxy
+            )
+        except BaseException as error:
+            cleanup = asyncio.create_task(
+                self._release_reservation(process, retire=not process.connected())
+            )
+            await _await_invariant(cleanup)
+            raise error
+
+        handle = PooledBrowserHandle(
+            self,
+            process,
+            context,
+            max_pages=self.max_pages_per_context,
+            page_acquire_timeout=self.page_acquire_timeout,
+        )
+        try:
+            async with self._condition:
+                if not self._closed:
+                    self._handles.add(handle)
+                    return handle
+        except BaseException as error:
+            cleanup = asyncio.create_task(handle.close())
+            await _await_invariant(cleanup)
+            raise error
+        cleanup = asyncio.create_task(handle.close())
+        await _await_invariant(cleanup)
+        raise ConfigurationError("BrowserPool is closed")
+
+    async def _reserve(self) -> _Process:
+        deadline = time.monotonic() + self.acquire_timeout
+        while True:
+            stale: list[_Process] = []
+            selected: _Process | None = None
+            start = False
+            async with self._condition:
+                if self._closed:
+                    raise ConfigurationError("BrowserPool is closed")
+                for process in list(self._processes):
+                    if not process.connected():
+                        self._processes.remove(process)
+                        self._retiring.add(process)
+                        stale.append(process)
+                for process in self._processes:
+                    if process.contexts < self.max_contexts_per_browser:
+                        process.contexts += 1
+                        selected = process
+                        break
+                if selected is None:
+                    if len(self._processes) + self._starting < self.max_browsers:
+                        self._starting += 1
+                        start = True
+                    elif not stale:
+                        remaining = deadline - time.monotonic()
+                        if remaining <= 0:
+                            raise ConfigurationError(
+                                "BrowserPool context capacity acquisition timed out"
+                            )
+                        try:
+                            async with asyncio.timeout(remaining):
+                                await self._condition.wait()
+                        except TimeoutError:
+                            raise ConfigurationError(
+                                "BrowserPool context capacity acquisition timed out"
+                            ) from None
+            if stale:
+                try:
+                    cleanup = asyncio.create_task(self._close_stale_processes(stale))
+                    _, cancelled = await _await_invariant(cleanup)
+                except BaseException as error:
+                    rollback = asyncio.create_task(self._rollback_reserve(selected, start=start))
+                    await _await_invariant(rollback)
+                    raise error
+                if cancelled is not None:
+                    rollback = asyncio.create_task(self._rollback_reserve(selected, start=start))
+                    await _await_invariant(rollback)
+                    raise cancelled
+            if selected is not None:
+                return selected
+            if not start:
+                continue
+
+            try:
+                playwright, browser = await self.provider._launch()
+                process = _Process(playwright=playwright, browser=browser, contexts=1)
+            except BaseException as error:
+                cleanup = asyncio.create_task(self._finish_failed_start())
+                await _await_invariant(cleanup)
+                raise error
+            try:
+                async with self._condition:
+                    if not self._closed:
+                        self._starting -= 1
+                        self._processes.append(process)
+                        self._condition.notify_all()
+                        return process
+            except BaseException as error:
+                cleanup = asyncio.create_task(self._abandon_started_process(process))
+                await _await_invariant(cleanup)
+                raise error
+            cleanup = asyncio.create_task(self._abandon_started_process(process))
+            await _await_invariant(cleanup)
+            raise ConfigurationError("BrowserPool is closed")
+
+    async def _finish_failed_start(self) -> None:
+        async with self._condition:
+            self._starting -= 1
+            self._condition.notify_all()
+
+    async def _rollback_reserve(self, selected: _Process | None, *, start: bool) -> None:
+        async with self._condition:
+            if selected is not None:
+                selected.contexts = max(0, selected.contexts - 1)
+            if start:
+                self._starting -= 1
+            self._condition.notify_all()
+
+    async def _abandon_started_process(self, process: _Process) -> None:
+        try:
+            await self._close_process(process)
+        finally:
+            await self._finish_failed_start()
+
+    async def _close_stale_processes(self, processes: list[_Process]) -> None:
+        first_error: BaseException | None = None
+        for process in processes:
+            try:
+                await self._retire_process(process)
+            except BaseException as error:
+                first_error = first_error or error
+        if first_error is not None:
+            raise first_error
+
+    async def _release(
+        self,
+        process: _Process,
+        handle: PooledBrowserHandle,
+        *,
+        retire: bool,
+    ) -> None:
+        async with self._condition:
+            self._handles.discard(handle)
+        await self._release_reservation(process, retire=retire)
+
+    async def _release_reservation(self, process: _Process, *, retire: bool) -> None:
+        close = False
+        async with self._condition:
+            process.contexts = max(0, process.contexts - 1)
+            if (retire or not process.connected()) and process in self._processes:
+                self._processes.remove(process)
+                self._retiring.add(process)
+                close = True
+            self._condition.notify_all()
+        if close:
+            await self._retire_process(process)
+
+    async def close(self) -> None:
+        async with self._condition:
+            if self._close_task is None:
+                self._closed = True
+                self._condition.notify_all()
+                self._close_task = asyncio.create_task(self._close_impl())
+            task = self._close_task
+        await asyncio.shield(task)
+
+    async def _close_impl(self) -> None:
+        first_error: BaseException | None = None
+        while True:
+            async with self._condition:
+                handles = list(self._handles)
+            for handle in handles:
+                try:
+                    await handle.close()
+                except BaseException as error:
+                    first_error = first_error or error
+            async with self._condition:
+                if self._starting == 0 and not self._retiring:
+                    break
+                await self._condition.wait_for(
+                    lambda: self._starting == 0 and not self._retiring
+                )
+
+        async with self._condition:
+            processes = list(self._processes)
+            self._processes.clear()
+        for process in processes:
+            try:
+                await self._close_process(process)
+            except BaseException as error:
+                first_error = first_error or error
+        if first_error is not None:
+            if not isinstance(first_error, Exception):
+                raise first_error
+            raise ConfigurationError("BrowserPool resource cleanup failed") from None
+
+    async def _close_process(self, process: _Process) -> None:
+        if process.close_task is None:
+            process.close_task = asyncio.create_task(self._close_process_impl(process))
+        await asyncio.shield(process.close_task)
+
+    async def _retire_process(self, process: _Process) -> None:
+        try:
+            await self._close_process(process)
+        finally:
+            async with self._condition:
+                self._retiring.discard(process)
+                self._condition.notify_all()
+
+    async def _close_process_impl(self, process: _Process) -> None:
+        first_error: BaseException | None = None
+        for closer in (process.browser.close, process.playwright.stop):
+            try:
+                await asyncio.wait_for(closer(), timeout=5)
+            except BaseException as error:
+                first_error = first_error or error
+        if first_error is not None:
+            if not isinstance(first_error, Exception):
+                raise first_error
+            raise ConfigurationError("Pooled browser process cleanup failed") from None

dataapp_crawler/providers/proxy.py

@@ -0,0 +1,209 @@
+"""Proxy providers with deliberately small, capability-based lifecycles."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable, Mapping
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+from urllib.parse import quote, unquote, urlsplit, urlunsplit
+
+from ..errors import ConfigurationError, LimitReached
+
+Rotate = Callable[[], Awaitable[Any]]
+Release = Callable[[], Awaitable[str]]
+
+
+def _proxy_parts(url: str) -> tuple[str, str | None, str | None]:
+    parts = urlsplit(url)
+    if parts.scheme not in {"http", "https", "socks5"} or not parts.hostname:
+        raise ConfigurationError("Proxy URL must be an absolute HTTP(S) or SOCKS5 URL")
+    host = f"[{parts.hostname}]" if ":" in parts.hostname else parts.hostname
+    netloc = host + (f":{parts.port}" if parts.port is not None else "")
+    server = urlunsplit((parts.scheme, netloc, "", "", ""))
+    return (
+        server,
+        unquote(parts.username) if parts.username else None,
+        (unquote(parts.password) if parts.password else None),
+    )
+
+
+def _proxy_url(value: Any) -> tuple[str, str | None, int | None, str | None]:
+    """Normalize the public ProxyUrl shape without importing its package."""
+    if isinstance(value, Mapping):
+        get = value.get
+    else:
+
+        def get(name: str, default: Any = None) -> Any:
+            return getattr(value, name, default)
+
+    raw = get("url")
+    username = get("username")
+    password = get("password")
+    hostname = get("hostname")
+    port = get("port")
+    if raw:
+        parts = urlsplit(str(raw))
+        try:
+            raw_port = parts.port
+        except ValueError:
+            raise ConfigurationError("Proxy provider returned an invalid connection port") from None
+        if (
+            parts.scheme not in {"http", "https", "socks5"}
+            or not parts.hostname
+            or parts.path not in {"", "/"}
+            or parts.query
+            or parts.fragment
+        ):
+            raise ConfigurationError("Proxy provider returned an invalid connection URL")
+        raw_username = unquote(parts.username) if parts.username is not None else username
+        raw_password = unquote(parts.password) if parts.password is not None else password
+        raw = _build_proxy_url(
+            parts.scheme,
+            parts.hostname,
+            raw_port,
+            raw_username,
+            raw_password,
+        )
+        return (
+            raw,
+            get("tunnel_id") or get("tunnelId"),
+            get("version"),
+            raw_username,
+        )
+    if not hostname or not port:
+        raise ConfigurationError("Proxy provider returned no usable connection address")
+    return (
+        _build_proxy_url("http", str(hostname), port, username, password),
+        (get("tunnel_id") or get("tunnelId")),
+        get("version"),
+        username,
+    )
+
+
+def _build_proxy_url(
+    scheme: str,
+    hostname: str,
+    port: Any,
+    username: Any,
+    password: Any,
+) -> str:
+    if isinstance(port, bool) or (port is not None and not isinstance(port, int)):
+        raise ConfigurationError("Proxy provider returned an invalid connection port")
+    if port is not None and not 1 <= port <= 65535:
+        raise ConfigurationError("Proxy provider returned an invalid connection port")
+    if password is not None and username is None:
+        raise ConfigurationError("Proxy provider returned a password without a username")
+    host = f"[{hostname}]" if ":" in hostname else hostname
+    auth = quote(str(username), safe="") if username is not None else ""
+    if password is not None:
+        auth += ":" + quote(str(password), safe="")
+    netloc = (f"{auth}@" if auth else "") + host + (f":{port}" if port else "")
+    return urlunsplit((scheme, netloc, "", "", ""))
+
+
+@dataclass
+class ProxyHandle:
+    """One proxy connection. Closing only releases this local handle."""
+
+    provider: str
+    _url: str = field(repr=False)
+    session_id: str | None = None
+    tunnel_id: str | None = None
+    version: int | None = None
+    max_rotations: int = 3
+    release_status: str = field(default="active", init=False)
+    _rotate: Rotate | None = field(default=None, repr=False)
+    _release: Release | None = field(default=None, repr=False)
+    _closed: bool = field(default=False, init=False, repr=False)
+    _rotation_attempts: int = field(default=0, init=False, repr=False)
+    _rotation_lock: asyncio.Lock = field(default_factory=asyncio.Lock, init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        _proxy_parts(self._url)
+        if (
+            isinstance(self.max_rotations, bool)
+            or not isinstance(self.max_rotations, int)
+            or self.max_rotations < 0
+        ):
+            raise ConfigurationError("max_rotations must be a nonnegative integer")
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    def http_url(self) -> str:
+        if self._closed:
+            raise RuntimeError("Proxy handle is closed")
+        return self._url
+
+    def as_playwright(self) -> dict[str, str]:
+        if self._closed:
+            raise RuntimeError("Proxy handle is closed")
+        server, username, password = _proxy_parts(self._url)
+        result = {"server": server}
+        if username is not None:
+            result["username"] = username
+        if password is not None:
+            result["password"] = password
+        return result
+
+    async def rotate(self) -> None:
+        async with self._rotation_lock:
+            if self._closed:
+                raise RuntimeError("Proxy handle is closed")
+            if self._rotate is None:
+                raise ConfigurationError(
+                    f"Proxy provider {self.provider!r} does not support rotation"
+                )
+            if self._rotation_attempts >= self.max_rotations:
+                raise LimitReached("Proxy rotation attempt limit reached")
+            self._rotation_attempts += 1
+            value = await self._rotate()
+            try:
+                url, tunnel_id, version, _ = _proxy_url(value)
+                _proxy_parts(url)
+                new_version = int(version) if version is not None else self.version
+            except Exception:
+                raise ConfigurationError("Proxy rotation returned an invalid connection") from None
+            self._url = url
+            self.tunnel_id = tunnel_id or self.tunnel_id
+            self.version = new_version
+
+    async def close(self) -> None:
+        """Release local use; this does not claim to revoke upstream credentials."""
+        if self._closed:
+            return
+        self._closed = True
+        if self._release is None:
+            self.release_status = "not_applicable"
+            return
+        try:
+            self.release_status = await self._release()
+        except asyncio.CancelledError:
+            self.release_status = "failed"
+            raise
+        except Exception as error:
+            self.release_status = "failed"
+            raise ConfigurationError(
+                f"Proxy client release failed ({type(error).__name__})"
+            ) from None
+
+
+class ProxyProvider(Protocol):
+    """Acquire a connection lease from a developer-owned proxy service or pool.
+
+    Selection and renewal belong to the provider. A handle stays associated
+    with its browser/session until that session is retired; this interface
+    does not promise a new exit IP for every HTTP request.
+    """
+
+    async def acquire(self) -> ProxyHandle: ...
+
+
+@dataclass(frozen=True)
+class StaticProxy:
+    url: str = field(repr=False)
+
+    async def acquire(self) -> ProxyHandle:
+        return ProxyHandle(provider="static", _url=self.url)