dataapp_crawler 0.1.0__py3-none-any.whl

dataapp_crawler/__init__.py

@@ -0,0 +1,87 @@
+"""Developer-friendly acquisition, without a dependency on the DataOS Runtime."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from .captcha import (
+    CaptchaConfidence,
+    CaptchaEvidence,
+    CaptchaFamily,
+    CaptchaObservation,
+    CaptchaOverride,
+    CaptchaPresence,
+    SolverComponent,
+    SolverMatch,
+    detect_captchas,
+    detect_captchas_on_page,
+)
+from .checkpoint import CrawlCheckpoint
+from .cloudflare import CloudflareChallengeCapture, CloudflareChallengeParameters
+from .context import CrawlContext
+from .crawler import Crawler
+from .errors import (
+    ConfigurationError,
+    CrawlCancelled,
+    CrawlerError,
+    FetchError,
+    LimitReached,
+    ProcessingError,
+    QueueFull,
+    ResponseTooLarge,
+)
+from .local_auth import local_resource_environment, read_api_key_file, run_local_app
+from .models import CaptureResult, CrawlItem, Failure, Identity, Limits, Request, Stats
+from .pagination import Pagination
+from .routing import RouteAlreadyRegistered, RouteNotFound, Router
+from .scheduling import AdaptivePolicy, AutoConcurrency, ResourceSample
+from .sessions import Session, SessionInfo, SessionPool
+
+__all__ = [
+    "AdaptivePolicy",
+    "AutoConcurrency",
+    "CaptchaConfidence",
+    "CaptchaEvidence",
+    "CaptchaFamily",
+    "CaptchaObservation",
+    "CaptchaOverride",
+    "CaptchaPresence",
+    "CaptureResult",
+    "CrawlCheckpoint",
+    "CloudflareChallengeCapture",
+    "CloudflareChallengeParameters",
+    "ResourceSample",
+    "ConfigurationError",
+    "CrawlCancelled",
+    "CrawlContext",
+    "Crawler",
+    "CrawlerError",
+    "CrawlItem",
+    "Failure",
+    "FetchError",
+    "Identity",
+    "LimitReached",
+    "local_resource_environment",
+    "Limits",
+    "ProcessingError",
+    "QueueFull",
+    "Request",
+    "ResponseTooLarge",
+    "read_api_key_file",
+    "RouteAlreadyRegistered",
+    "RouteNotFound",
+    "run_local_app",
+    "Router",
+    "Pagination",
+    "Stats",
+    "Session",
+    "SessionInfo",
+    "SessionPool",
+    "SolverComponent",
+    "SolverMatch",
+    "detect_captchas",
+    "detect_captchas_on_page",
+]
+
+try:
+    __version__ = version("dataapp_crawler")
+except PackageNotFoundError:
+    __version__ = "0.0.0"

dataapp_crawler/__main__.py

@@ -0,0 +1,35 @@
+"""Local Crawler SDK command helpers."""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Sequence
+
+from .errors import ConfigurationError
+from .local_auth import run_local_app
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m dataapp_crawler")
+    commands = parser.add_subparsers(dest="command", required=True)
+    run = commands.add_parser(
+        "run",
+        help="run a local DataApp with a file-backed platform API Key",
+    )
+    run.add_argument("--api-key-file", required=True, help="regular file containing one API Key")
+    run.add_argument("app_command", nargs=argparse.REMAINDER, help="application command after --")
+    options = parser.parse_args(argv)
+    if options.command != "run":
+        return 2
+    command = list(options.app_command)
+    if command[:1] == ["--"]:
+        command = command[1:]
+    try:
+        return run_local_app(options.api_key_file, command)
+    except ConfigurationError as error:
+        print(f"配置错误：{error}")
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dataapp_crawler/_logging.py

@@ -0,0 +1,148 @@
+"""Default stderr logging and bounded, credential-free diagnostic labels."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import math
+import re
+import sys
+from urllib.parse import urlsplit
+
+from .errors import (
+    CrawlCancelled,
+    FetchError,
+    LimitReached,
+    ProcessingError,
+    browser_error_code,
+    http_error_code,
+    process_error_code,
+)
+
+_PROVIDERS = {
+    "AWSBrowser": "AgentCore",
+    "AgentCoreBrowser": "AgentCore",
+    "BrowserEnvRelay": "Relay/BrowserEnv",
+    "RelayBrowser": "Relay",
+    "CDPBrowser": "CDP",
+    "PlaywrightBrowser": "Playwright",
+    "CamoufoxBrowser": "Camoufox",
+    "DataOSProxy": "ResourceProxy",
+    "StaticProxy": "custom-proxy",
+}
+_PHASES = frozenset({
+    "navigation", "before_navigation", "after_navigation", "headers", "snapshot",
+    "page_creation", "page_cleanup", "http_transport", "response", "process",
+    "browser_open", "proxy_acquire", "startup", "cleanup", "request",
+})
+_EXCEPTION_TYPES = frozenset({
+    "TimeoutError", "Error", "TargetClosedError", "ConnectionError", "ConnectionResetError",
+    "ConnectionAbortedError", "BrokenPipeError", "OSError", "RuntimeError", "ValueError",
+    "KeyError", "IndexError", "AttributeError", "TypeError", "AssertionError",
+    "FetchError", "ProcessingError", "ConfigurationError", "CrawlCancelled",
+    "CancelledError", "LimitReached", "Exception",
+})
+
+
+class _StderrHandler(logging.StreamHandler):
+    @property
+    def stream(self):  # type: ignore[no-untyped-def]
+        return sys.stderr
+
+    @stream.setter
+    def stream(self, value):  # type: ignore[no-untyped-def]
+        pass
+
+
+def get_logger() -> logging.Logger:
+    """Return the standalone INFO logger without configuring the root logger.
+
+    INFO is intentionally concise. Developers can set this logger, or a logger
+    supplied to :class:`Crawler`, to DEBUG for per-stage acquisition details.
+    """
+    logger = logging.getLogger("dataapp_crawler")
+    if not logger.handlers:
+        handler = _StderrHandler()
+        handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
+        logger.addHandler(handler)
+        logger.setLevel(logging.INFO)
+        logger.propagate = False
+    return logger
+
+
+def provider_name(provider: object | None) -> str:
+    return "none" if provider is None else _PROVIDERS.get(type(provider).__name__, "custom")
+
+
+def target_host(url: str) -> str:
+    """Keep only an ASCII hostname; omit paths, userinfo, query and fragments."""
+    host = urlsplit(url).hostname or ""
+    return host[:253] if re.fullmatch(r"[A-Za-z0-9.:\[\]-]+", host) else "redacted"
+
+
+def root_page_host(page: object) -> str:
+    """Read only the root Page's hostname; a closed page must not mask an error."""
+    try:
+        url = getattr(page, "url", None)
+        if not isinstance(url, str):
+            return "none"
+        return target_host(url) or "none"
+    except BaseException:
+        return "unavailable"
+
+
+def failure_details(error: BaseException, default_phase: str) -> tuple[str, str]:
+    phase = getattr(error, "phase", default_phase)
+    phase = phase if isinstance(phase, str) and phase in _PHASES else default_phase
+    if isinstance(error, ProcessingError):
+        return "process", process_error_code(error)
+    if isinstance(error, CrawlCancelled | asyncio.CancelledError):
+        return phase, "CRAWL_CANCELLED"
+    if isinstance(error, LimitReached):
+        return phase, "LIMIT_REACHED"
+    if isinstance(error, FetchError):
+        return phase, browser_error_code(error) or http_error_code(error) or "FETCH_ERROR"
+    return phase, "OPERATION_FAILED"
+
+
+def failure_context(error: BaseException) -> str:
+    """Append fixed exception labels and browser evidence, without raw error text."""
+    exception_type = type(error).__name__
+    exception_type = exception_type if exception_type in _EXCEPTION_TYPES else "Exception"
+    fields = [f"exception_type={exception_type}"]
+    if not isinstance(error, FetchError) or not isinstance(error.browser_diagnostics, dict):
+        return " " + " ".join(fields)
+    diagnostics = error.browser_diagnostics
+    if not diagnostics:
+        return " " + " ".join(fields)
+    diagnostic_type = diagnostics.get("exception_type")
+    exception_type = (
+        diagnostic_type if isinstance(diagnostic_type, str)
+        and diagnostic_type in _EXCEPTION_TYPES else "Exception"
+    )
+    fields = [f"exception_type={exception_type}"]
+    timeout = diagnostics.get("timeout_seconds")
+    if (
+        isinstance(timeout, int | float) and not isinstance(timeout, bool)
+        and 0 < timeout <= 86400 and math.isfinite(timeout)
+    ):
+        fields.append(f"timeout={timeout:.3f}s")
+    wait_until = diagnostics.get("wait_until")
+    if (
+        isinstance(wait_until, str)
+        and wait_until in {"commit", "domcontentloaded", "load", "networkidle"}
+    ):
+        fields.append(f"wait_until={wait_until}")
+    if isinstance(error.retryable, bool):
+        fields.append(f"retryable={error.retryable}")
+    status = error.main_document_status
+    valid_status = isinstance(status, int) and not isinstance(status, bool) and 100 <= status <= 599
+    fields.append(f"main_status={status if valid_status else 'none'}")
+    for key, label in (
+        ("main_document_request_seen", "request_seen"),
+        ("main_document_response_seen", "response_seen"),
+    ):
+        value = diagnostics.get(key)
+        if isinstance(value, bool):
+            fields.append(f"{label}={value}")
+    return " " + " ".join(fields)

dataapp_crawler/_network_diagnostics.py

@@ -0,0 +1,193 @@
+"""Bounded browser navigation telemetry containing no network identifiers or objects."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from typing import Any
+
+from .errors import NetworkDiagnosticsSummary, browser_error_code
+
+_EVENTS = ("request", "response", "requestfailed")
+_RESOURCE_TYPES = frozenset(
+    {
+        "document",
+        "stylesheet",
+        "image",
+        "media",
+        "font",
+        "script",
+        "texttrack",
+        "xhr",
+        "fetch",
+        "eventsource",
+        "websocket",
+        "manifest",
+        "other",
+    }
+)
+_MAX_EVENTS = 20
+
+
+class NetworkDiagnostics:
+    """Own exact Page listeners for one navigation/read attempt."""
+
+    def __init__(
+        self,
+        remove_listener: Callable[[str, Callable[[Any], None]], Any],
+        started: float,
+        main_frame: Any,
+        capture_events: bool,
+    ) -> None:
+        self._remove_listener: Callable[[str, Callable[[Any], None]], Any] | None = (
+            remove_listener
+        )
+        self._started = started
+        self._main_frame = main_frame
+        self._capture_events = capture_events
+        self._listeners: list[tuple[str, Callable[[Any], None]]] = []
+        self._records: list[dict[str, Any]] = []
+        self._events_dropped = 0
+        self._main_document_status: int | None = None
+
+    def add_listener(self, event: str, callback: Callable[[Any], None]) -> None:
+        self._listeners.append((event, callback))
+
+    def detach(self) -> None:
+        remove_listener = self._remove_listener
+        if remove_listener is None:
+            return
+        for event, callback in self._listeners:
+            try:
+                remove_listener(event, callback)
+            except Exception:
+                # Diagnostics must never change acquisition or cleanup semantics.
+                pass
+        self._listeners.clear()
+        # A bound remove_listener retains its Page; discard it after the window closes.
+        self._remove_listener = None
+        self._main_frame = None
+
+    def summary(self) -> NetworkDiagnosticsSummary:
+        return {
+            "events": [dict(record) for record in self._records],
+            "events_dropped": self._events_dropped,
+            "main_document_status": self._main_document_status,
+        }
+
+    @property
+    def main_document_status(self) -> int | None:
+        return self._main_document_status
+
+    def on_request(self, request: Any) -> None:
+        try:
+            self._append("request", request)
+        except Exception:
+            pass
+
+    def on_response(self, response: Any) -> None:
+        try:
+            request = response.request
+            status = response.status
+            self._append(
+                "response",
+                request,
+                status=status if isinstance(status, int) and not isinstance(status, bool) else None,
+            )
+        except Exception:
+            pass
+
+    def on_requestfailed(self, request: Any) -> None:
+        try:
+            failure = request.failure
+            code = browser_error_code(failure) if isinstance(failure, str) else None
+            self._append(
+                "requestfailed",
+                request,
+                browser_error_code=code,
+                unknown_failure=code is None,
+            )
+        except Exception:
+            pass
+
+    def _append(
+        self,
+        event: str,
+        request: Any,
+        *,
+        status: int | None = None,
+        browser_error_code: str | None = None,
+        unknown_failure: bool | None = None,
+    ) -> None:
+        try:
+            navigation = request.is_navigation_request()
+            frame = request.frame
+        except Exception:
+            return
+        main_document = (
+            navigation is True
+            and self._main_frame is not None
+            and frame is self._main_frame
+        )
+        valid_status = (
+            status
+            if isinstance(status, int) and not isinstance(status, bool) and 100 <= status <= 599
+            else None
+        )
+        if main_document and valid_status is not None:
+            # Keep the latest main-frame response even when the bounded event list is full.
+            self._main_document_status = valid_status
+        if not self._capture_events:
+            return
+        try:
+            resource_type = request.resource_type
+        except Exception:
+            return
+        if len(self._records) >= _MAX_EVENTS:
+            self._events_dropped += 1
+            return
+        record: dict[str, Any] = {
+            "event": event,
+            "resource_type": resource_type if resource_type in _RESOURCE_TYPES else "other",
+            "main_document": main_document,
+        }
+        if valid_status is not None:
+            record["status"] = valid_status
+        if browser_error_code is not None:
+            record["browser_error_code"] = browser_error_code
+        if unknown_failure is not None:
+            record["unknown_failure"] = unknown_failure
+        record["elapsed"] = max(0.0, round(time.monotonic() - self._started, 6))
+        self._records.append(record)
+
+
+def observe_network(
+    page: Any, started: float, *, capture_events: bool
+) -> NetworkDiagnostics | None:
+    """Attach supported listeners atomically, otherwise leave the Page untouched."""
+    try:
+        on = getattr(page, "on", None)
+        remove_listener = getattr(page, "remove_listener", None)
+        main_frame = getattr(page, "main_frame", None)
+    except Exception:
+        return None
+    if not callable(on) or not callable(remove_listener):
+        return None
+    if callable(main_frame):
+        main_frame = None
+    diagnostics = NetworkDiagnostics(remove_listener, started, main_frame, capture_events)
+    callbacks = {
+        "request": diagnostics.on_request,
+        "response": diagnostics.on_response,
+        "requestfailed": diagnostics.on_requestfailed,
+    }
+    events = _EVENTS if capture_events else ("response",)
+    try:
+        for event in events:
+            callback = callbacks[event]
+            diagnostics.add_listener(event, callback)
+            on(event, callback)
+    except Exception:
+        diagnostics.detach()
+        return None
+    return diagnostics

dataapp_crawler/browser_tools/__init__.py

@@ -0,0 +1,23 @@
+"""Small Playwright helpers for page-driven browser collection.
+
+The module imports no Playwright classes at runtime. Developers keep native
+``Page`` and ``Locator`` objects while these helpers add Crawler budgets,
+bounded readiness, deterministic navigation cleanup, and stable list keys.
+"""
+
+from .dom import attribute_of, text_of
+from .lists import ListItem, iterate_list
+from .navigation import new_page, open_url, popup_page, same_tab
+from .readiness import wait_until
+
+__all__ = [
+    "ListItem",
+    "attribute_of",
+    "iterate_list",
+    "new_page",
+    "open_url",
+    "popup_page",
+    "same_tab",
+    "text_of",
+    "wait_until",
+]

dataapp_crawler/browser_tools/_common.py

@@ -0,0 +1,66 @@
+"""Internal validation and cleanup helpers."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import math
+from collections.abc import Awaitable
+from typing import Any, TypeVar
+
+from ..errors import ConfigurationError
+
+T = TypeVar("T")
+
+
+def positive_seconds(value: float, name: str) -> float:
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, (int, float))
+        or not math.isfinite(value)
+        or value <= 0
+    ):
+        raise ConfigurationError(f"{name} must be a finite positive number")
+    return float(value)
+
+
+def positive_integer(value: int, name: str) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        raise ConfigurationError(f"{name} must be a positive integer")
+    return value
+
+
+async def maybe_await(value: T | Awaitable[T]) -> T:
+    return await value if inspect.isawaitable(value) else value
+
+
+async def cleanup(awaitable: Awaitable[T], *, timeout: float) -> T:
+    """Finish one bounded cleanup even after the CrawlContext was cancelled."""
+    task = asyncio.ensure_future(awaitable)
+    try:
+        return await asyncio.wait_for(asyncio.shield(task), timeout=timeout)
+    except BaseException:
+        if not task.done():
+            task.cancel()
+        await asyncio.gather(task, return_exceptions=True)
+        raise
+
+
+async def finish_cleanup(
+    awaitable: Awaitable[Any],
+    *,
+    timeout: float,
+    primary_error: BaseException | None,
+    failure_message: str,
+    failure_note: str,
+) -> None:
+    """Run sanitized cleanup without replacing an active body exception."""
+    try:
+        await cleanup(awaitable, timeout=timeout)
+    except BaseException as error:
+        if primary_error is not None:
+            primary_error.add_note(failure_note)
+            return
+        if not isinstance(error, Exception):
+            raise
+        raise ConfigurationError(failure_message) from None

dataapp_crawler/browser_tools/dom.py

@@ -0,0 +1,49 @@
+"""Lightweight extraction from native Playwright locators."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..context import CrawlContext
+from ..errors import ConfigurationError
+from ._common import positive_seconds
+
+
+async def text_of(
+    locator: Any,
+    context: CrawlContext,
+    *,
+    timeout: float = 10,
+    required: bool = False,
+) -> str | None:
+    """Return stripped inner text, or ``None`` when optional text is empty."""
+    timeout = positive_seconds(timeout, "timeout")
+    value = (await context.run(locator.inner_text(timeout=timeout * 1000), timeout=timeout)).strip()
+    if value:
+        return value
+    if required:
+        raise ValueError("Required element text is empty")
+    return None
+
+
+async def attribute_of(
+    locator: Any,
+    name: str,
+    context: CrawlContext,
+    *,
+    timeout: float = 10,
+    required: bool = False,
+) -> str | None:
+    """Return one locator attribute with optional required validation."""
+    timeout = positive_seconds(timeout, "timeout")
+    if not isinstance(name, str) or not name:
+        raise ConfigurationError("Attribute name must be a non-empty string")
+    value = await context.run(
+        locator.get_attribute(name, timeout=timeout * 1000),
+        timeout=timeout,
+    )
+    if value is not None and value != "":
+        return value
+    if required:
+        raise ValueError(f"Required element attribute {name!r} is empty")
+    return None