dataapp_crawler 0.1.0__py3-none-any.whl

dataapp_crawler/browser_tools/lists.py

@@ -0,0 +1,125 @@
+"""Bounded list discovery with stable keys and locator re-resolution."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from ..context import CrawlContext
+from ..errors import ConfigurationError
+from ._common import maybe_await, positive_integer, positive_seconds
+
+LocatorFactory = Callable[[], Any]
+KeyFunction = Callable[[Any], Awaitable[str] | str]
+AdvanceFunction = Callable[[int], Awaitable[bool | None] | bool | None]
+
+
+@dataclass(frozen=True)
+class ListItem:
+    """A stable business key whose current Locator is resolved on demand."""
+
+    key: str
+    round_number: int
+    discovered_index: int
+    _resolve: Callable[[], Awaitable[Any]] = field(repr=False, compare=False)
+
+    async def locator(self) -> Any:
+        """Find this key in the current DOM and return its current Locator."""
+        return await self._resolve()
+
+
+async def iterate_list(
+    items: LocatorFactory,
+    key: KeyFunction,
+    context: CrawlContext,
+    *,
+    advance: AdvanceFunction | None = None,
+    max_items: int = 100,
+    max_rounds: int = 20,
+    no_progress_rounds: int = 1,
+    operation_timeout: float = 10,
+) -> AsyncIterator[ListItem]:
+    """Yield newly discovered list keys across bounded scroll/page rounds.
+
+    ``items`` must return a fresh locator for the current list. ``key`` must
+    derive a stable, non-empty business key from one item locator. The helper
+    never retains a Locator between yields; ``ListItem.locator`` scans the
+    current DOM by key. ``advance(round_number)`` may scroll or paginate and
+    returns ``False`` when no later round exists.
+
+    The helper does not restore virtual-list position or page state. Keep those
+    site-specific responsibilities in the calling collection flow.
+    """
+    if not callable(items) or not callable(key):
+        raise ConfigurationError("items and key must be callable")
+    if advance is not None and not callable(advance):
+        raise ConfigurationError("advance must be callable or None")
+    max_items = positive_integer(max_items, "max_items")
+    max_rounds = positive_integer(max_rounds, "max_rounds")
+    no_progress_rounds = positive_integer(no_progress_rounds, "no_progress_rounds")
+    operation_timeout = positive_seconds(operation_timeout, "operation_timeout")
+    seen: set[str] = set()
+    stalled = 0
+
+    async def read_key(locator: Any) -> str:
+        context.check()
+        value = await context.run(maybe_await(key(locator)), timeout=operation_timeout)
+        if not isinstance(value, str) or not value.strip():
+            raise ValueError("List item key must be a non-empty string")
+        return value.strip()
+
+    async def resolve(item_key: str) -> Any:
+        context.check()
+        current = items()
+        count = await context.run(current.count(), timeout=operation_timeout)
+        for index in range(count):
+            candidate = current.nth(index)
+            if await read_key(candidate) == item_key:
+                return candidate
+        raise LookupError("List item is no longer present")
+
+    def resolver_for(item_key: str) -> Callable[[], Awaitable[Any]]:
+        async def current_locator() -> Any:
+            return await resolve(item_key)
+
+        return current_locator
+
+    for round_number in range(1, max_rounds + 1):
+        context.check()
+        locator = items()
+        count = await context.run(locator.count(), timeout=operation_timeout)
+        round_keys: list[str] = []
+        round_seen: set[str] = set()
+        for index in range(count):
+            context.check()
+            item_key = await read_key(locator.nth(index))
+            if item_key not in round_seen:
+                round_seen.add(item_key)
+                round_keys.append(item_key)
+        del locator
+        added = 0
+        for index, item_key in enumerate(round_keys):
+            context.check()
+            if item_key in seen:
+                continue
+            seen.add(item_key)
+            added += 1
+            yield ListItem(
+                key=item_key,
+                round_number=round_number,
+                discovered_index=index,
+                _resolve=resolver_for(item_key),
+            )
+            if len(seen) >= max_items:
+                return
+        stalled = stalled + 1 if added == 0 else 0
+        if stalled >= no_progress_rounds or advance is None or round_number >= max_rounds:
+            return
+        context.check()
+        should_continue = await context.run(
+            maybe_await(advance(round_number)),
+            timeout=operation_timeout,
+        )
+        if should_continue is False:
+            return

dataapp_crawler/browser_tools/navigation.py

@@ -0,0 +1,243 @@
+"""Bounded navigation scopes for a native Playwright Page."""
+
+from __future__ import annotations
+
+import logging
+import sys
+import time
+from collections.abc import AsyncIterator, Awaitable, Callable
+from contextlib import asynccontextmanager
+from typing import Any, Literal
+
+from .._logging import get_logger, target_host
+from .._network_diagnostics import observe_network
+from ..context import CrawlContext
+from ..errors import (
+    ConfigurationError,
+    CrawlCancelled,
+    FetchError,
+    LimitReached,
+    NetworkDiagnosticsSummary,
+    browser_error_code,
+)
+from ._common import cleanup, finish_cleanup, maybe_await, positive_seconds
+
+WaitUntil = Literal["commit", "domcontentloaded", "load", "networkidle"]
+
+
+def _navigation_evidence(summary: NetworkDiagnosticsSummary) -> tuple[bool, str | None]:
+    main_request_seen = False
+    failed_code = None
+    for event in summary["events"]:
+        if not event.get("main_document"):
+            continue
+        if event.get("event") == "request":
+            main_request_seen = True
+        if event.get("event") == "requestfailed":
+            failed_code = event.get("browser_error_code")
+    return main_request_seen, failed_code
+
+
+def _reserve_top_level(context: CrawlContext) -> None:
+    context.check()
+    if context.stats.pages >= context.limits.max_pages:
+        context.stats.stop_reason = "max_pages"
+        raise LimitReached("Page limit reached")
+    context.reserve_request()
+    context.stats.pages += 1
+
+
+async def open_url(
+    page: Any,
+    url: str,
+    context: CrawlContext,
+    *,
+    wait_until: WaitUntil = "domcontentloaded",
+    timeout: float = 30,
+) -> Any:
+    """Navigate and count one explicit top-level browser acquisition.
+
+    Page actions performed directly by developer code cannot be counted by the
+    Crawler. Use this helper for top-level navigations that should consume the
+    current ``max_requests`` and ``max_pages`` budgets.
+    """
+    timeout = positive_seconds(timeout, "timeout")
+    if not isinstance(url, str) or not url.strip():
+        raise ConfigurationError("url must be a non-empty string")
+    if wait_until not in {"commit", "domcontentloaded", "load", "networkidle"}:
+        raise ConfigurationError("Unknown browser navigation wait strategy")
+    _reserve_top_level(context)
+    started = time.monotonic()
+    network = observe_network(page, started, capture_events=True)
+    try:
+        return await context.run(
+            page.goto(url, wait_until=wait_until, timeout=timeout * 1000),
+            timeout=timeout,
+        )
+    except (CrawlCancelled, LimitReached):
+        raise
+    except Exception as exc:
+        code = browser_error_code(exc)
+        summary: NetworkDiagnosticsSummary = (
+            network.summary()
+            if network is not None
+            else {"events": [], "events_dropped": 0, "main_document_status": None}
+        )
+        main_request_seen, failed_code = _navigation_evidence(summary)
+        code = code or failed_code
+        main_status = summary["main_document_status"]
+        failure = FetchError(
+            "Browser navigation failed",
+            retryable=(
+                isinstance(exc, TimeoutError)
+                or type(exc).__name__ == "TimeoutError"
+                or code
+                in {"ERR_CONNECTION_RESET", "ERR_CONNECTION_CLOSED", "ERR_TIMED_OUT"}
+            ),
+            phase="navigation",
+            browser_error_code=code,
+            network_diagnostics=summary,
+            main_document_status=main_status,
+        )
+        failure.browser_diagnostics = {
+            "exception_type": type(exc).__name__,
+            "wait_until": wait_until,
+            "timeout_seconds": timeout,
+            "elapsed_seconds": round(time.monotonic() - started, 3),
+            "main_document_request_seen": main_request_seen,
+            "main_document_response_seen": main_status is not None,
+        }
+        get_logger().log(
+            logging.ERROR,
+            "[crawler] Navigation failed host=%s phase=navigation code=%s "
+            "main_status=%s request_seen=%s response_seen=%s elapsed=%.3fs",
+            target_host(url),
+            code or "NAVIGATION_FAILED",
+            main_status if main_status is not None else "none",
+            main_request_seen,
+            main_status is not None,
+            failure.browser_diagnostics["elapsed_seconds"],
+        )
+        raise failure from None
+    finally:
+        if network is not None:
+            network.detach()
+
+
+@asynccontextmanager
+async def new_page(
+    source_page: Any,
+    url: str,
+    context: CrawlContext,
+    *,
+    wait_until: WaitUntil = "domcontentloaded",
+    timeout: float = 30,
+    cleanup_timeout: float = 5,
+) -> AsyncIterator[Any]:
+    """Open an independently owned tab and always close it on exit."""
+    cleanup_timeout = positive_seconds(cleanup_timeout, "cleanup_timeout")
+    page = await context.run(source_page.context.new_page(), timeout=timeout)
+    try:
+        await open_url(page, url, context, wait_until=wait_until, timeout=timeout)
+        yield page
+    finally:
+        await finish_cleanup(
+            page.close(),
+            timeout=cleanup_timeout,
+            primary_error=sys.exc_info()[1],
+            failure_message="Browser helper page cleanup failed",
+            failure_note="Browser helper page cleanup also failed",
+        )
+
+
+@asynccontextmanager
+async def popup_page(
+    source_page: Any,
+    action: Callable[[], Awaitable[Any] | Any],
+    context: CrawlContext,
+    *,
+    timeout: float = 30,
+    cleanup_timeout: float = 5,
+) -> AsyncIterator[Any]:
+    """Run an action that opens a popup, then close the popup on exit."""
+    timeout = positive_seconds(timeout, "timeout")
+    cleanup_timeout = positive_seconds(cleanup_timeout, "cleanup_timeout")
+    if not callable(action):
+        raise ConfigurationError("action must be callable")
+    _reserve_top_level(context)
+    popup_info: Any | None = None
+    page: Any | None = None
+    try:
+        async with source_page.expect_popup(timeout=timeout * 1000) as popup_info:
+            context.check()
+            try:
+                await context.run(maybe_await(action()), timeout=timeout)
+            except BaseException:
+                try:
+                    page = await cleanup(popup_info.value, timeout=cleanup_timeout)
+                except BaseException:
+                    page = None
+                raise
+        page = await context.run(popup_info.value, timeout=timeout)
+        yield page
+    finally:
+        primary_error = sys.exc_info()[1]
+        if page is None and popup_info is not None:
+            try:
+                page = await cleanup(popup_info.value, timeout=cleanup_timeout)
+            except BaseException:
+                # 动作可能尚未创建弹窗；只在有限时间内观察未决事件。
+                page = None
+        if page is not None and page is not source_page:
+            await finish_cleanup(
+                page.close(),
+                timeout=cleanup_timeout,
+                primary_error=primary_error,
+                failure_message="Browser helper popup cleanup failed",
+                failure_note="Browser helper popup cleanup also failed",
+            )
+
+
+@asynccontextmanager
+async def same_tab(
+    page: Any,
+    url: str,
+    context: CrawlContext,
+    *,
+    wait_until: WaitUntil = "domcontentloaded",
+    timeout: float = 30,
+    restore_timeout: float = 10,
+) -> AsyncIterator[Any]:
+    """Navigate in place and restore the original URL on exit.
+
+    Browser history is tried first. If it does not restore the original URL,
+    the helper navigates directly to that URL. This restores location, not
+    arbitrary in-memory DOM, scroll, form, or virtual-list state.
+    """
+    restore_timeout = positive_seconds(restore_timeout, "restore_timeout")
+    original_url = page.url
+    try:
+        await open_url(page, url, context, wait_until=wait_until, timeout=timeout)
+        yield page
+    finally:
+        primary_error = sys.exc_info()[1]
+        if page.url != original_url:
+            try:
+                await cleanup(
+                    page.go_back(wait_until=wait_until, timeout=restore_timeout * 1000),
+                    timeout=restore_timeout,
+                )
+            except Exception:
+                pass
+        if page.url != original_url:
+            await finish_cleanup(
+                page.goto(
+                    original_url,
+                    wait_until=wait_until,
+                    timeout=restore_timeout * 1000,
+                ),
+                timeout=restore_timeout,
+                primary_error=primary_error,
+                failure_message="Browser helper tab restoration failed",
+                failure_note="Browser helper tab restoration also failed",
+            )

dataapp_crawler/browser_tools/readiness.py

@@ -0,0 +1,42 @@
+"""Cancellation-aware polling for site-specific readiness."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Awaitable, Callable
+from typing import TypeVar
+
+from ..context import CrawlContext
+from ._common import maybe_await, positive_seconds
+
+T = TypeVar("T")
+
+
+async def wait_until(
+    observe: Callable[[], Awaitable[T] | T],
+    predicate: Callable[[T], bool],
+    context: CrawlContext,
+    *,
+    timeout: float = 30,
+    poll_interval: float = 0.1,
+) -> T:
+    """Return the first observed value accepted by ``predicate``.
+
+    ``timeout`` bounds this readiness condition while ``CrawlContext`` still
+    enforces cancellation and the whole-crawl duration.
+    """
+    timeout = positive_seconds(timeout, "timeout")
+    poll_interval = positive_seconds(poll_interval, "poll_interval")
+    deadline = time.monotonic() + timeout
+    while True:
+        context.check()
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            raise TimeoutError("Readiness timeout reached")
+        value = await context.run(maybe_await(observe()), timeout=remaining)
+        if predicate(value):
+            return value
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            raise TimeoutError("Readiness timeout reached")
+        await context.sleep(min(poll_interval, remaining))