dataapp_crawler 0.1.0__py3-none-any.whl

dataapp_crawler/context.py

@@ -0,0 +1,231 @@
+"""Cooperative cancellation and bounded execution, independent of platform internals."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import math
+import time
+from collections.abc import Awaitable, Callable, Iterable
+from contextvars import ContextVar, Token
+from fnmatch import fnmatchcase
+from itertools import islice
+from typing import Any, Literal, TypeVar
+from urllib.parse import urldefrag, urljoin
+
+from .errors import ConfigurationError, CrawlCancelled, LimitReached
+from .models import Limits, Request, Stats, origin
+
+T = TypeVar("T")
+
+_CURRENT_SESSION: ContextVar[tuple[Any, Any] | None] = ContextVar(
+    "dataapp_crawler_current_session", default=None
+)
+
+
+class CrawlContext:
+    def __init__(
+        self, limits: Limits, *, run_id: str | None = None, attempt_id: str | None = None
+    ) -> None:
+        self.limits = limits
+        self.run_id = run_id
+        self.attempt_id = attempt_id
+        self.stats = Stats()
+        self.cancelled = asyncio.Event()
+        self._started: float | None = None
+        self._enqueue: Callable[[Request], bool] | None = None
+        self._seen: set[str] = set()
+
+    @property
+    def session(self) -> Any | None:
+        """Safe metadata and health controls for the current exclusive pool lease."""
+        current = _CURRENT_SESSION.get()
+        if current is None or current[0] is not self:
+            return None
+        session = current[1]
+        return session if session.active else None
+
+    def _set_session(self, session: Any) -> Token[tuple[Any, Any] | None]:
+        return _CURRENT_SESSION.set((self, session))
+
+    def _reset_session(self, token: Token[tuple[Any, Any] | None]) -> None:
+        _CURRENT_SESSION.reset(token)
+
+    def start(self) -> None:
+        if self._started is None:
+            self._started = time.monotonic()
+
+    @property
+    def remaining(self) -> float:
+        self.start()
+        assert self._started is not None
+        return max(0, self.limits.max_duration - (time.monotonic() - self._started))
+
+    def check(self) -> None:
+        if self.cancelled.is_set():
+            raise CrawlCancelled("Acquisition cancelled")
+        if self.remaining <= 0:
+            self.stats.stop_reason = "max_duration"
+            raise LimitReached("Acquisition duration limit reached")
+
+    def cancel(self, reason: str = "cancelled") -> None:
+        self.stats.stop_reason = reason
+        self.cancelled.set()
+
+    def stop(self) -> None:
+        self.cancel("developer_stop")
+
+    def enqueue(self, request: Request | str) -> bool:
+        """Add follow-up work; false means duplicate. Full queues raise QueueFull."""
+        self.check()
+        if self._enqueue is None:
+            raise RuntimeError("enqueue is only available during crawl")
+        return self._enqueue(Request(request) if isinstance(request, str) else request)
+
+    def enqueue_links(
+        self,
+        links: Iterable[str],
+        *,
+        base_url: str,
+        label: str = "",
+        mode: Literal["http", "browser"] = "http",
+        same_origin: bool = True,
+        include: Iterable[str] = (),
+        exclude: Iterable[str] = (),
+        max_links: int = 100,
+    ) -> int:
+        """Resolve and enqueue bounded links from a developer's DOM/JSON/Page parser.
+
+        Patterns use case-sensitive URL globs. Inspect at most ``max_links``
+        candidates, strip fragments, and use the crawl queue's existing deduplication
+        and limits. Cross-origin links require an explicit ``same_origin=False``.
+        """
+        self.check()
+        if self._enqueue is None:
+            raise RuntimeError("enqueue_links is only available during crawl")
+        if isinstance(max_links, bool) or not isinstance(max_links, int) or max_links < 1:
+            raise ConfigurationError("max_links must be a positive integer")
+        if mode not in {"http", "browser"} or not isinstance(same_origin, bool):
+            raise ConfigurationError("Invalid link acquisition mode or origin policy")
+        source_origin = origin(base_url)
+        included, excluded = tuple(include), tuple(exclude)
+        added = 0
+        for href in islice(links, max_links):
+            self.check()
+            if not isinstance(href, str) or not href.strip() or href.lstrip().startswith("#"):
+                continue
+            try:
+                url = urldefrag(urljoin(base_url, href.strip()))[0]
+                target_origin = origin(url)
+            except (ConfigurationError, ValueError):
+                continue
+            if same_origin and target_origin != source_origin:
+                continue
+            if included and not any(fnmatchcase(url, pattern) for pattern in included):
+                continue
+            if any(fnmatchcase(url, pattern) for pattern in excluded):
+                continue
+            added += self.enqueue(Request(url, label=label, mode=mode))
+        return added
+
+    def seen(self, key: str) -> bool:
+        """Return whether a record/cursor key was already seen, then remember it."""
+        present = key in self._seen
+        if not present:
+            if len(self._seen) >= self.limits.max_results + self.limits.max_requests:
+                raise LimitReached("Seen-key memory limit reached")
+            self._seen.add(key)
+        return present
+
+    def reserve_request(self) -> None:
+        self.check()
+        if self.stats.requests >= self.limits.max_requests:
+            self.stats.stop_reason = "max_requests"
+            raise LimitReached("Request limit reached")
+        self.stats.requests += 1
+
+    async def run(self, awaitable: Awaitable[T], *, timeout: float | None = None) -> T:
+        """Run one operation within cancellation and time budgets.
+
+        ``timeout`` bounds this operation while the remaining ``max_duration``
+        budget still bounds the whole crawl. An operation timeout raises the
+        standard :class:`TimeoutError`; exhausting the crawl budget raises
+        :class:`LimitReached`. The owned task is cancelled and joined on every
+        timeout or cancellation path.
+        """
+        try:
+            if timeout is not None and (
+                isinstance(timeout, bool)
+                or not isinstance(timeout, (int, float))
+                or not math.isfinite(timeout)
+                or timeout <= 0
+            ):
+                raise ConfigurationError("timeout must be a finite positive number or None")
+            self.check()
+        except BaseException:
+            # Calling an async function creates a coroutine before this method can
+            # validate its keyword arguments. Close that unstarted coroutine so a
+            # configuration error never produces an unawaited-coroutine warning.
+            if inspect.iscoroutine(awaitable):
+                awaitable.close()
+            else:
+                rejected_task = asyncio.ensure_future(awaitable)
+                if not rejected_task.done():
+                    rejected_task.cancel()
+                await asyncio.gather(rejected_task, return_exceptions=True)
+            raise
+
+        remaining = self.remaining
+        operation_timeout = float(timeout) if timeout is not None else None
+        deadline = min(remaining, operation_timeout) if operation_timeout is not None else remaining
+        operation_deadline_first = operation_timeout is not None and operation_timeout < remaining
+        task = asyncio.ensure_future(awaitable)
+        abort = asyncio.create_task(self.cancelled.wait())
+        try:
+            done, _ = await asyncio.wait(
+                {task, abort}, timeout=deadline, return_when=asyncio.FIRST_COMPLETED
+            )
+            if abort in done:
+                raise CrawlCancelled("Acquisition cancelled")
+            if task in done:
+                return task.result()
+            if operation_deadline_first:
+                raise TimeoutError("Operation timeout reached")
+            self.stats.stop_reason = "max_duration"
+            raise LimitReached("Acquisition duration limit reached")
+        finally:
+            for pending_task in (task, abort):
+                if not pending_task.done():
+                    pending_task.cancel()
+            await asyncio.gather(task, abort, return_exceptions=True)
+
+    async def sleep(self, delay: float) -> None:
+        await self.run(asyncio.sleep(max(0, delay)))
+
+
+class SiteGate:
+    """Start-rate and concurrency limits shared by all requests/retries to one site."""
+
+    def __init__(self, context: CrawlContext) -> None:
+        self.context = context
+        self.semaphore = asyncio.Semaphore(context.limits.per_site_concurrency)
+        self._lock = asyncio.Lock()
+        self._next_start = 0.0
+
+    def defer(self, delay: float) -> None:
+        self._next_start = max(self._next_start, time.monotonic() + delay)
+
+    async def __aenter__(self) -> SiteGate:
+        await self.context.run(self.semaphore.acquire())
+        try:
+            async with self._lock:
+                while self._next_start > time.monotonic():
+                    await self.context.sleep(self._next_start - time.monotonic())
+                self._next_start = time.monotonic() + self.context.limits.min_interval
+        except BaseException:
+            self.semaphore.release()
+            raise
+        return self
+
+    async def __aexit__(self, *exc: Any) -> None:
+        self.semaphore.release()

dataapp_crawler/cookies.py

@@ -0,0 +1,207 @@
+"""Cookie conversion and reconciliation for opt-in cross-engine sessions."""
+
+from __future__ import annotations
+
+import http.cookiejar
+import time
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from typing import Any
+
+from .errors import ConfigurationError
+
+CookieKey = tuple[str, str, str]
+
+
+def _same_site(value: Any) -> str | None:
+    if not isinstance(value, str):
+        return None
+    normalized = value.lower()
+    return {"strict": "Strict", "lax": "Lax", "none": "None"}.get(normalized)
+
+
+def _rest_value(cookie: http.cookiejar.Cookie, name: str) -> Any:
+    for key, value in getattr(cookie, "_rest", {}).items():
+        if key.lower() == name.lower():
+            return value
+    return None
+
+
+def _has_rest(cookie: http.cookiejar.Cookie, name: str) -> bool:
+    return any(key.lower() == name.lower() for key in getattr(cookie, "_rest", {}))
+
+
+def _as_bool(value: Any) -> bool:
+    if isinstance(value, str):
+        return value.lower() not in {"", "0", "false", "no"}
+    return bool(value)
+
+
+def _is_partitioned_record(cookie: dict[str, Any]) -> bool:
+    return "partitionKey" in cookie or cookie.get("partitioned") is not None
+
+
+def _normalized(cookie: dict[str, Any]) -> dict[str, Any] | None:
+    if _is_partitioned_record(cookie):
+        raise ConfigurationError("Partitioned cookies cannot be shared across engines")
+    name = cookie.get("name")
+    domain = cookie.get("domain")
+    if not isinstance(name, str) or not name or not isinstance(domain, str) or not domain:
+        return None
+    expires = cookie.get("expires")
+    if not isinstance(expires, (int, float)) or expires < 0:
+        expires = None
+    if expires is not None and expires <= time.time():
+        return None
+    host_only = cookie.get("hostOnly")
+    if not isinstance(host_only, bool):
+        host_only = not domain.startswith(".")
+    if host_only:
+        domain = domain.lstrip(".")
+    elif not domain.startswith("."):
+        domain = f".{domain}"
+    result: dict[str, Any] = {
+        "name": name,
+        "value": str(cookie.get("value", "")),
+        "domain": domain.lower(),
+        "path": cookie.get("path") or "/",
+        "hostOnly": host_only,
+        "secure": bool(cookie.get("secure", False)),
+        "httpOnly": bool(cookie.get("httpOnly", False)),
+        "expires": expires,
+    }
+    same_site = _same_site(cookie.get("sameSite"))
+    if same_site is not None:
+        result["sameSite"] = same_site
+    return result
+
+
+def cookies_from_jar(jar: http.cookiejar.CookieJar) -> list[dict[str, Any]]:
+    """Export current cookies without exposing HTTP headers or other identity state."""
+    jar.clear_expired_cookies()
+    exported: list[dict[str, Any]] = []
+    for cookie in jar:
+        if _has_rest(cookie, "Partitioned") or _has_rest(cookie, "partitionKey"):
+            raise ConfigurationError("Partitioned cookies cannot be shared across engines")
+        normalized = _normalized(
+            {
+                "name": cookie.name,
+                "value": cookie.value,
+                "domain": cookie.domain,
+                "path": cookie.path,
+                "hostOnly": not cookie.domain_specified,
+                "secure": cookie.secure,
+                "httpOnly": _as_bool(_rest_value(cookie, "HttpOnly")),
+                "sameSite": _rest_value(cookie, "SameSite"),
+                "expires": cookie.expires,
+            }
+        )
+        if normalized is not None:
+            exported.append(normalized)
+    return exported
+
+
+def is_partitioned_jar_cookie(cookie: http.cookiejar.Cookie) -> bool:
+    return _has_rest(cookie, "Partitioned") or _has_rest(cookie, "partitionKey")
+
+
+def validate_shareable_cookies(cookies: Iterable[dict[str, Any]]) -> None:
+    """Fail before a bridge can silently downgrade partitioned browser state."""
+    for cookie in cookies:
+        _normalized(cookie)
+
+
+def cookie_to_jar(cookie: dict[str, Any]) -> http.cookiejar.Cookie | None:
+    normalized = _normalized(cookie)
+    if normalized is None:
+        return None
+    rest: dict[str, Any] = {}
+    if normalized["httpOnly"]:
+        rest["HttpOnly"] = True
+    if "sameSite" in normalized:
+        rest["SameSite"] = normalized["sameSite"]
+    domain = normalized["domain"]
+    host_only = normalized["hostOnly"]
+    expires = normalized["expires"]
+    return http.cookiejar.Cookie(
+        version=0,
+        name=normalized["name"],
+        value=normalized["value"],
+        port=None,
+        port_specified=False,
+        domain=domain,
+        domain_specified=not host_only,
+        domain_initial_dot=domain.startswith("."),
+        path=normalized["path"],
+        path_specified=True,
+        secure=normalized["secure"],
+        expires=expires,
+        discard=expires is None,
+        comment=None,
+        comment_url=None,
+        rest=rest,
+        rfc2109=False,
+    )
+
+
+def replace_jar(
+    jar: http.cookiejar.CookieJar,
+    cookies: Iterable[dict[str, Any]],
+    *,
+    ignore_partitioned: bool = False,
+) -> None:
+    jar.clear()
+    for item in cookies:
+        if ignore_partitioned and _is_partitioned_record(item):
+            continue
+        cookie = cookie_to_jar(item)
+        if cookie is not None:
+            jar.set_cookie(cookie)
+
+
+def cookies_for_browser(cookies: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    for item in cookies:
+        cookie = _normalized(item)
+        if cookie is None:
+            continue
+        browser_cookie: dict[str, Any] = {
+            "name": cookie["name"],
+            "value": cookie["value"],
+            "domain": cookie["domain"],
+            "path": cookie["path"],
+            "secure": cookie["secure"],
+            "httpOnly": cookie["httpOnly"],
+            "expires": cookie["expires"] if cookie["expires"] is not None else -1,
+        }
+        if "sameSite" in cookie:
+            browser_cookie["sameSite"] = cookie["sameSite"]
+        result.append(browser_cookie)
+    return result
+
+
+@dataclass
+class CookieState:
+    """A session-local canonical snapshot; callers serialize access through the lease."""
+
+    _cookies: dict[CookieKey, dict[str, Any]] = field(default_factory=dict, repr=False)
+
+    @classmethod
+    def from_jar(cls, jar: http.cookiejar.CookieJar) -> CookieState:
+        state = cls()
+        state.replace(cookies_from_jar(jar))
+        return state
+
+    def replace(self, cookies: Iterable[dict[str, Any]]) -> None:
+        replacement: dict[CookieKey, dict[str, Any]] = {}
+        for item in cookies:
+            cookie = _normalized(item)
+            if cookie is None:
+                continue
+            key = (cookie["domain"], cookie["path"], cookie["name"])
+            replacement[key] = cookie
+        self._cookies = replacement
+
+    def snapshot(self) -> list[dict[str, Any]]:
+        self.replace(self._cookies.values())
+        return [dict(self._cookies[key]) for key in sorted(self._cookies)]