dataapp_crawler 0.1.0__py3-none-any.whl

dataapp_crawler/local_auth.py

@@ -0,0 +1,77 @@
+"""Local-only helpers for starting a Crawler app with managed resources."""
+
+from __future__ import annotations
+
+import os
+import stat
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+
+from .errors import ConfigurationError
+
+_MAX_API_KEY_FILE_BYTES = 8 * 1024
+
+
+def read_api_key_file(path: str | Path) -> str:
+    """Read one developer API Key from a regular local file.
+
+    The file is consumed locally and the value is returned to the caller. It is
+    never sent as a file to Resource Proxy. Symlinks, directories, special files,
+    multiple lines, and whitespace inside the key are rejected.
+    """
+
+    source = Path(path).expanduser()
+    try:
+        info = source.lstat()
+    except OSError as error:
+        raise ConfigurationError(
+            f"cannot inspect API Key file ({type(error).__name__})"
+        ) from None
+    if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode):
+        raise ConfigurationError("API Key file must be a regular file, not a symlink or directory")
+    if info.st_size > _MAX_API_KEY_FILE_BYTES:
+        raise ConfigurationError("API Key file is too large")
+    try:
+        value = source.read_text(encoding="utf-8")
+    except (OSError, UnicodeError) as error:
+        raise ConfigurationError(
+            f"cannot read API Key file ({type(error).__name__})"
+        ) from None
+    lines = value.splitlines()
+    if len(lines) != 1 or not lines[0].strip() or lines[0].strip() != lines[0]:
+        raise ConfigurationError("API Key file must contain exactly one non-empty line")
+    key = lines[0]
+    if any(character.isspace() for character in key):
+        raise ConfigurationError("API Key file must contain one Key without whitespace")
+    return key
+
+
+def local_resource_environment(
+    api_key_file: str | Path,
+    environ: Mapping[str, str] | None = None,
+) -> dict[str, str]:
+    """Return a child-process environment with the local API Key injected.
+
+    This helper does not mutate the parent process and does not persist the Key.
+    It is intended for a local launcher that starts ``app.py``. Platform Runs
+    use their injected Run identity and should not call this helper.
+    """
+
+    result = dict(os.environ if environ is None else environ)
+    result["DATAAPP_API_KEY"] = read_api_key_file(api_key_file)
+    result.pop("DATAOS_API_KEY", None)
+    return result
+
+
+def run_local_app(
+    api_key_file: str | Path,
+    command: Sequence[str],
+) -> int:
+    """Run a local app command with a file-backed API Key in its environment."""
+
+    if not command:
+        raise ConfigurationError("a local app command is required")
+    import subprocess
+
+    completed = subprocess.run(command, env=local_resource_environment(api_key_file))
+    return completed.returncode

dataapp_crawler/models.py

@@ -0,0 +1,285 @@
+"""Public acquisition values. No platform Run, Dataset or workflow models live here."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any, Literal
+from urllib.parse import urlsplit, urlunsplit
+
+from .errors import ConfigurationError
+
+
+def origin(url: str) -> str:
+    parts = urlsplit(url)
+    if parts.scheme not in {"http", "https"} or not parts.hostname:
+        raise ConfigurationError("Only absolute HTTP(S) URLs are supported")
+    if parts.username is not None or parts.password is not None:
+        raise ConfigurationError("Target URLs must not contain credentials")
+    port = parts.port
+    suffix = f":{port}" if port and port != (443 if parts.scheme == "https" else 80) else ""
+    host = f"[{parts.hostname}]" if ":" in parts.hostname else parts.hostname
+    return f"{parts.scheme}://{host}{suffix}"
+
+
+def safe_url(url: str) -> str:
+    """Diagnostics omit query values, fragments and URL userinfo."""
+    parts = urlsplit(url)
+    host = parts.hostname or ""
+    if ":" in host:
+        host = f"[{host}]"
+    try:
+        port = parts.port
+    except ValueError:
+        port = None
+    return urlunsplit((parts.scheme, host + (f":{port}" if port else ""), parts.path, "", ""))
+
+
+@dataclass(frozen=True)
+class Request:
+    """One bounded acquisition request.
+
+    ``mode`` selects HTTP, browser, or adaptive acquisition. Browser requests
+    use ``wait_until`` only for navigation; site-specific readiness belongs in
+    ``process`` and may call native Page methods through ``CrawlContext.run``.
+    """
+
+    url: str = field(repr=False)
+    method: str = "GET"
+    headers: dict[str, str] = field(default_factory=dict, repr=False)
+    params: dict[str, Any] | None = field(default=None, repr=False)
+    content: bytes | str | None = field(default=None, repr=False)
+    json: Any = field(default=None, repr=False)
+    label: str = ""
+    user_data: dict[str, Any] = field(default_factory=dict, repr=False)
+    key: str | None = None
+    replayable: bool | None = None
+    mode: Literal["http", "browser", "auto"] = "http"
+    site_key: str | None = None
+    session_key: str | None = field(default=None, repr=False)
+    deduplicate: bool = True
+    wait_until: Literal["commit", "domcontentloaded", "load", "networkidle"] = "domcontentloaded"
+    snapshot: bool = True
+
+    def __post_init__(self) -> None:
+        origin(self.url)
+        object.__setattr__(self, "method", self.method.upper())
+        if self.mode not in {"http", "browser", "auto"}:
+            raise ConfigurationError("Unknown acquisition mode")
+        if self.wait_until not in {"commit", "domcontentloaded", "load", "networkidle"}:
+            raise ConfigurationError("Unknown browser navigation wait strategy")
+        if not isinstance(self.snapshot, bool):
+            raise ConfigurationError("snapshot must be a boolean")
+        if self.mode == "auto" and self.method not in {"GET", "HEAD"}:
+            raise ConfigurationError("Adaptive acquisition supports GET or HEAD only")
+        if self.mode == "auto" and (
+            self.headers or self.content is not None or self.json is not None
+        ):
+            raise ConfigurationError("Adaptive identity headers belong in origin-scoped Identity")
+        if self.mode == "auto" and self.method == "GET" and not self.can_retry:
+            raise ConfigurationError("Adaptive GET requests must be replayable")
+        if self.mode == "browser" and self.method != "GET":
+            raise ConfigurationError("Browser navigation supports GET; use native Page for actions")
+        if self.mode == "browser" and self.headers:
+            raise ConfigurationError("Browser headers must use origin-scoped Identity.headers")
+        if self.content is not None and self.json is not None:
+            raise ConfigurationError("Specify either content or json")
+        if any(k.lower() == "proxy-authorization" for k in self.headers):
+            raise ConfigurationError("Proxy authentication belongs in proxy configuration")
+        if self.session_key is not None and (
+            not isinstance(self.session_key, str) or not self.session_key
+        ):
+            raise ConfigurationError("session_key must be a non-empty string or None")
+
+    @property
+    def can_retry(self) -> bool:
+        return (
+            self.replayable
+            if self.replayable is not None
+            else self.method in {"GET", "HEAD", "OPTIONS"}
+        )
+
+    @property
+    def unique_key(self) -> str:
+        if self.key is not None:
+            return self.key
+        payload = [
+            self.method,
+            self.url,
+            self.params,
+            self.headers,
+            self.content,
+            self.json,
+            self.label,
+            self.user_data,
+            self.mode,
+        ]
+        if self.mode in {"browser", "auto"}:
+            payload.extend([self.wait_until, self.snapshot])
+        return hashlib.sha256(json.dumps(payload, default=str, sort_keys=True).encode()).hexdigest()
+
+
+@dataclass
+class Identity:
+    """Single-run website state. Header keys are exact HTTP(S) origins, not wildcards."""
+
+    cookies: list[dict[str, Any]] = field(default_factory=list, repr=False)
+    headers: dict[str, dict[str, str]] = field(default_factory=dict, repr=False)
+    storage_state: dict[str, Any] | None = field(default=None, repr=False)
+
+    def __post_init__(self) -> None:
+        for scope in self.headers:
+            parts = urlsplit(scope)
+            if parts.path not in {"", "/"} or parts.query or parts.fragment:
+                raise ConfigurationError("Identity header scope must be an exact origin")
+        self.headers = {origin(k): dict(v) for k, v in self.headers.items()}
+        cookies = [*(self.storage_state or {}).get("cookies", []), *self.cookies]
+        for cookie in cookies:
+            domain = cookie.get("domain")
+            if not isinstance(domain, str) or not domain or not cookie.get("name"):
+                raise ConfigurationError("Identity cookies require an explicit domain and name")
+            if domain.startswith("."):
+                raise ConfigurationError("Identity cookies must be host-only")
+        for headers in self.headers.values():
+            if any(k.lower() in {"cookie", "proxy-authorization", "host"} for k in headers):
+                raise ConfigurationError(
+                    "Use domain cookies and proxy configuration for authentication"
+                )
+
+    def headers_for(self, url: str) -> dict[str, str]:
+        return dict(self.headers.get(origin(url), {}))
+
+
+@dataclass(frozen=True)
+class Limits:
+    """Finite crawl, concurrency, response-size, retry, and timeout budgets.
+
+    ``navigation_timeout`` and ``snapshot_timeout`` default to
+    ``request_timeout``. These acquisition budgets are separate from any short
+    Page-operation timeout supplied to ``CrawlContext.run`` by a process.
+    """
+
+    concurrency: int = 4
+    per_site_concurrency: int = 2
+    max_requests: int = 1000
+    max_pages: int = 1000
+    max_results: int = 1000
+    max_pending: int = 1000
+    max_duration: float = 300
+    request_timeout: float = 30
+    min_interval: float = 0.25
+    retries: int = 2
+    backoff: float = 0.5
+    max_response_bytes: int = 10 * 1024 * 1024
+    max_redirects: int = 10
+    navigation_timeout: float | None = None
+    snapshot_timeout: float | None = None
+
+    def __post_init__(self) -> None:
+        for name in (
+            "concurrency",
+            "per_site_concurrency",
+            "max_requests",
+            "max_pages",
+            "max_results",
+            "max_pending",
+            "max_response_bytes",
+        ):
+            value = getattr(self, name)
+            if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+                raise ConfigurationError(f"{name} must be a positive integer")
+        for name in ("max_duration", "request_timeout", "min_interval", "backoff"):
+            value = getattr(self, name)
+            if not math.isfinite(value) or value < 0 or (name.endswith("timeout") and value == 0):
+                raise ConfigurationError(f"{name} must be finite and within its allowed range")
+        if self.max_duration == 0:
+            raise ConfigurationError("max_duration must be positive")
+        for name in ("navigation_timeout", "snapshot_timeout"):
+            value = getattr(self, name)
+            if value is not None and (
+                isinstance(value, bool)
+                or not isinstance(value, (int, float))
+                or not math.isfinite(value)
+                or value <= 0
+            ):
+                raise ConfigurationError(f"{name} must be a finite positive number or None")
+        for value in (self.retries, self.max_redirects):
+            if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+                raise ConfigurationError("Retry and redirect limits must be nonnegative integers")
+
+
+@dataclass
+class CaptureResult:
+    """Raw acquired content and safe diagnostics passed to ``process``.
+
+    A browser ``page`` remains live only while the process callback runs. It is
+    cleared when the attempt closes; callers that need browser actions after a
+    fetch should instead use ``Crawler.browser()`` explicitly.
+    """
+
+    request: Request = field(repr=False)
+    final_url: str = field(repr=False)
+    status: int | None
+    headers: dict[str, str] = field(default_factory=dict, repr=False)
+    body: bytes = field(default=b"", repr=False)
+    page: Any = field(default=None, repr=False)
+    elapsed: float = 0
+    attempts: int = 1
+    captured_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    diagnostics: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def text(self) -> str:
+        content_type = self.headers.get("content-type", "")
+        encoding = (
+            content_type.split("charset=")[-1].split(";")[0].strip()
+            if "charset=" in content_type
+            else "utf-8"
+        )
+        try:
+            return self.body.decode(encoding, errors="replace")
+        except LookupError:
+            return self.body.decode("utf-8", errors="replace")
+
+    def json(self) -> Any:
+        return json.loads(self.body)
+
+    def selector(self) -> Any:
+        from .parsing import selector
+
+        return selector(self.text)
+
+    @property
+    def access_issue(self) -> str | None:
+        if self.status is None:
+            return None
+        return {401: "authentication", 403: "access_denied", 429: "rate_limited"}.get(self.status)
+
+
+@dataclass
+class Stats:
+    requests: int = 0
+    pages: int = 0
+    captured: int = 0
+    results: int = 0
+    retries: int = 0
+    duplicates: int = 0
+    failed: int = 0
+    stop_reason: str | None = None
+
+
+@dataclass(frozen=True)
+class Failure:
+    request: Request = field(repr=False)
+    kind: str
+    message: str
+    diagnostics: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class CrawlItem:
+    data: Any = field(repr=False)
+    capture: CaptureResult = field(repr=False)

dataapp_crawler/pagination.py

@@ -0,0 +1,172 @@
+"""Bounded manual pagination for developer-owned site rules."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import replace
+from typing import Any
+from urllib.parse import urldefrag, urljoin
+
+import httpx
+
+from .context import CrawlContext
+from .errors import ConfigurationError
+from .models import CaptureResult, Request, origin
+
+_STATE_KEY = "__dataos_pagination__"
+_STATE_VERSION = 1
+
+
+def _digest(value: object) -> str:
+    encoded = json.dumps(value, ensure_ascii=True, separators=(",", ":")).encode()
+    return hashlib.sha256(encoded).hexdigest()
+
+
+def _effective_url(request: Request) -> str:
+    """Match HTTPX query handling while excluding fragments from loop identity."""
+    url = request.url
+    if request.params is not None:
+        url = str(httpx.URL(url, params=request.params))
+    return urldefrag(str(httpx.URL(url)))[0]
+
+
+class Pagination:
+    """Enqueue developer-discovered pages with a finite, carried chain state.
+
+    ``max_pages`` includes the current/seed page. URL pagination rejects a URL
+    already present in the chain, ignoring changing ``user_data``. Cursor-based
+    pagination permits the same URL but rejects a repeated explicit cursor. The
+    helper discovers no links or cursors itself and delegates queue deduplication
+    and pending/request budgets to ``CrawlContext.enqueue``.
+    """
+
+    def __init__(self, *, max_pages: int = 100, same_origin: bool = True) -> None:
+        if isinstance(max_pages, bool) or not isinstance(max_pages, int) or max_pages < 1:
+            raise ConfigurationError("Pagination max_pages must be a positive integer")
+        if not isinstance(same_origin, bool):
+            raise ConfigurationError("Pagination same_origin must be a boolean")
+        self.max_pages = max_pages
+        self.same_origin = same_origin
+
+    def enqueue(
+        self,
+        capture: CaptureResult,
+        context: CrawlContext,
+        next_page: Request | str | None,
+        *,
+        cursor: str | int | None = None,
+    ) -> bool:
+        """Enqueue one explicit next page; false means filtered, repeated, or full chain.
+
+        Queue exceptions are intentionally propagated. Chain state advances only
+        on a request accepted by the crawl queue.
+        """
+        state = self._read_state(capture, context)
+        if state["pages"] >= state["limit"] or next_page is None:
+            return False
+        if cursor is not None and (
+            isinstance(cursor, bool)
+            or not isinstance(cursor, (str, int))
+            or (isinstance(cursor, str) and not cursor)
+        ):
+            raise ConfigurationError("Pagination cursor must be a non-empty string or integer")
+
+        request = self._request(capture, next_page)
+        if request is None:
+            return False
+        try:
+            if self.same_origin and origin(request.url) != origin(capture.final_url):
+                return False
+        except (ConfigurationError, ValueError):
+            return False
+
+        url_token = _digest(["url", _effective_url(request)])
+        cursor_token = (
+            _digest(["cursor", type(cursor).__name__, cursor]) if cursor is not None else None
+        )
+        if cursor_token is not None:
+            if cursor_token in state["cursors"]:
+                return False
+        elif url_token in state["urls"]:
+            return False
+
+        next_state = {
+            "version": _STATE_VERSION,
+            "limit": state["limit"],
+            "pages": state["pages"] + 1,
+            "urls": [*state["urls"], url_token],
+            "cursors": [*state["cursors"], *([cursor_token] if cursor_token else [])],
+        }
+        # Always replace carried state, including when a developer derived the
+        # next Request from the current request's user_data.
+        request = replace(request, user_data={**request.user_data, _STATE_KEY: next_state})
+        return context.enqueue(request)
+
+    def _request(self, capture: CaptureResult, next_page: Request | str) -> Request | None:
+        if isinstance(next_page, Request):
+            return replace(next_page, url=urldefrag(next_page.url)[0])
+        if not isinstance(next_page, str):
+            raise ConfigurationError("Next page must be a Request, URL string, or None")
+        if not next_page.strip():
+            return None
+        candidate = next_page.strip()
+        if candidate.startswith("#"):
+            return None
+        try:
+            url = urldefrag(urljoin(capture.final_url, candidate))[0]
+            origin(url)
+            return Request(
+                url,
+                label=capture.request.label,
+                mode=capture.request.mode,
+                site_key=capture.request.site_key,
+                session_key=capture.request.session_key,
+                wait_until=capture.request.wait_until,
+                snapshot=capture.request.snapshot,
+            )
+        except (ConfigurationError, ValueError):
+            return None
+
+    def _read_state(self, capture: CaptureResult, context: CrawlContext) -> dict[str, Any]:
+        raw = capture.request.user_data.get(_STATE_KEY)
+        if raw is None:
+            initial_limit = min(self.max_pages, context.limits.max_pages)
+            return {
+                "version": _STATE_VERSION,
+                "limit": initial_limit,
+                "pages": 1,
+                "urls": [_digest(["url", _effective_url(capture.request)])],
+                "cursors": [],
+            }
+        if not isinstance(raw, dict):
+            raise ConfigurationError("Invalid carried pagination state")
+        version = raw.get("version")
+        limit = raw.get("limit")
+        pages = raw.get("pages")
+        urls = raw.get("urls")
+        cursors = raw.get("cursors")
+        if (
+            version != _STATE_VERSION
+            or isinstance(limit, bool)
+            or not isinstance(limit, int)
+            or limit < 1
+            or isinstance(pages, bool)
+            or not isinstance(pages, int)
+            or pages < 1
+            or pages > limit
+            or not isinstance(urls, list)
+            or not isinstance(cursors, list)
+            or len(urls) > pages
+            or len(cursors) > pages
+            or not all(isinstance(value, str) and len(value) == 64 for value in [*urls, *cursors])
+        ):
+            raise ConfigurationError("Invalid carried pagination state")
+        effective_limit = min(limit, self.max_pages, context.limits.max_pages)
+        return {
+            "version": _STATE_VERSION,
+            "limit": effective_limit,
+            "pages": pages,
+            "urls": list(urls),
+            "cursors": list(cursors),
+        }

dataapp_crawler/parsing.py

@@ -0,0 +1,13 @@
+"""Optional, ordinary parsing helpers for developer process functions."""
+
+from typing import Any
+
+from .errors import ConfigurationError
+
+
+def selector(text: str) -> Any:
+    try:
+        from parsel import Selector
+    except ImportError:
+        raise ConfigurationError("DOM helpers require dataapp_crawler[parse]") from None
+    return Selector(text=text)

dataapp_crawler/providers/__init__.py

@@ -0,0 +1,20 @@
+"""Generic browser and proxy provider contracts and implementations."""
+
+from .browser import BrowserHandle, BrowserProvider, CDPBrowser, PlaywrightBrowser
+from .camoufox import CamoufoxBrowser
+from .dataos import DataOSProxy
+from .pool import BrowserPool
+from .proxy import ProxyHandle, ProxyProvider, StaticProxy
+
+__all__ = [
+    "BrowserHandle",
+    "BrowserPool",
+    "BrowserProvider",
+    "CDPBrowser",
+    "CamoufoxBrowser",
+    "DataOSProxy",
+    "PlaywrightBrowser",
+    "ProxyHandle",
+    "ProxyProvider",
+    "StaticProxy",
+]