dataapp_crawler 0.1.0__py3-none-any.whl

dataapp_crawler/crawler.py

@@ -0,0 +1,1392 @@
+"""Three public entrances: fetch, browser session and a bounded in-memory crawl."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import logging
+import sys
+import time
+from collections import deque
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable
+from contextlib import asynccontextmanager
+from contextvars import ContextVar
+from datetime import UTC, datetime
+from email.utils import parsedate_to_datetime
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+import httpx
+
+from ._logging import (
+    failure_context,
+    failure_details,
+    get_logger,
+    provider_name,
+    root_page_host,
+    target_host,
+)
+from ._network_diagnostics import observe_network
+from .checkpoint import CrawlCheckpoint
+from .context import CrawlContext, SiteGate
+from .errors import (
+    ConfigurationError,
+    CrawlCancelled,
+    FetchError,
+    LimitReached,
+    ProcessingError,
+    QueueFull,
+    ResponseTooLarge,
+    browser_error_code,
+    http_error_code,
+    process_error_code,
+)
+from .http import HttpSession
+from .models import CaptureResult, CrawlItem, Failure, Identity, Limits, Request, Stats, origin
+from .scheduling import AdaptivePolicy, AutoConcurrency
+from .sessions import SessionLease, SessionPool
+
+Process = Callable[[CaptureResult, CrawlContext], Any]
+BrowserCollect = Callable[[Any, CrawlContext], Any]
+NavigationHook = Callable[[Request, Any, CrawlContext], Any]
+FailureHook = Callable[[Failure, CrawlContext], Any]
+_RETRYABLE_STATUSES = frozenset({429, 500, 502, 503, 504})
+
+
+def _safe_exception_type(error: Exception) -> str:
+    """Return a bounded class name without retaining exception text."""
+    name = type(error).__name__
+    if len(name) <= 80 and name.isascii() and name.isidentifier():
+        return name
+    return "Exception"
+
+
+class _AcquisitionScope:
+    __slots__ = ("crawler_id", "active")
+
+    def __init__(self, crawler_id: int) -> None:
+        self.crawler_id = crawler_id
+        self.active = True
+
+
+_ACQUISITION_SCOPES: ContextVar[tuple[_AcquisitionScope, ...]] = ContextVar(
+    "dataapp_crawler_acquisition_scopes", default=()
+)
+
+
+class _SessionFetchScope:
+    __slots__ = ("crawler_id", "active")
+
+    def __init__(self, crawler_id: int) -> None:
+        self.crawler_id = crawler_id
+        self.active = True
+
+
+_SESSION_FETCH_SCOPES: ContextVar[tuple[_SessionFetchScope, ...]] = ContextVar(
+    "dataapp_crawler_session_fetch_scopes", default=()
+)
+
+
+class _RecursivePooledFetch(ConfigurationError):
+    pass
+
+
+class Crawler:
+    """Acquire HTTP or browser content under shared limits and lifecycle control.
+
+    ``fetch`` handles one request, ``crawl`` owns a bounded in-memory queue, and
+    ``browser`` exposes an explicit manual browser session. ``run_browser``
+    passes a native Page to a developer flow under cancellation and time budgets.
+    Queue checkpoints are values only: callers or the Runtime remain responsible for persisting
+    them as private artifacts. Concise INFO lifecycle, result and failure logs go
+    to stderr by default; pass ``logger=`` to select a standard Python logger and
+    enable DEBUG for per-stage acquisition details. Logs omit request payloads,
+    URL paths/query strings, connection credentials and exception messages.
+    """
+
+    def __init__(
+        self,
+        *,
+        limits: Limits | None = None,
+        identity: Identity | None = None,
+        browser: Any = None,
+        proxy: Any = None,
+        http_transport: httpx.AsyncBaseTransport | None = None,
+        run_id: str | None = None,
+        attempt_id: str | None = None,
+        required_capabilities: Iterable[str] = (),
+        diagnostics_dir: str | Path | None = None,
+        network_diagnostics: bool = False,
+        session_pool: SessionPool | None = None,
+        before_navigation: NavigationHook | None = None,
+        after_navigation: NavigationHook | None = None,
+        on_error: FailureHook | None = None,
+        on_failure: FailureHook | None = None,
+        auto_concurrency: AutoConcurrency | None = None,
+        adaptive_policy: AdaptivePolicy | None = None,
+        logger: logging.Logger | None = None,
+    ) -> None:
+        if not isinstance(network_diagnostics, bool):
+            raise ConfigurationError("network_diagnostics must be a boolean")
+        self.context = CrawlContext(limits or Limits(), run_id=run_id, attempt_id=attempt_id)
+        self.log = logger if logger is not None else get_logger()
+        self._log_requests = 0
+        self._log_failed = 0
+        self._cleanup_incomplete = False
+        self.identity = identity or Identity()
+        self.browser_provider = browser
+        self.proxy_provider = proxy
+        self._transport = http_transport
+        self._http: HttpSession | None = None
+        self._proxy: Any = None
+        self._browser: Any = None
+        self._browser_lock = asyncio.Lock()
+        self._resource_lock = asyncio.Lock()
+        self._active_acquisitions = 0
+        self._acquisition_tasks: dict[asyncio.Task[Any], int] = {}
+        self._rotating = False
+        self._resource_failure: str | None = None
+        self._retired_resources: list[Any] = []
+        self._close_task: asyncio.Task[None] | None = None
+        self._global = (
+            auto_concurrency.controller(ceiling=self.context.limits.concurrency)
+            if auto_concurrency is not None
+            else asyncio.Semaphore(self.context.limits.concurrency)
+        )
+        self.adaptive_policy = adaptive_policy or AdaptivePolicy()
+        self._sites: dict[str, SiteGate] = {}
+        self._entered = False
+        self._closed = False
+        self._crawling = False
+        self._queue_state: dict[str, Any] | None = None
+        self._last_checkpoint: CrawlCheckpoint | None = None
+        self._checkpoint_unavailable: str | None = None
+        self._active: set[asyncio.Task[Any]] = set()
+        self._required_capabilities = frozenset(required_capabilities)
+        self._diagnostics_dir = Path(diagnostics_dir) if diagnostics_dir is not None else None
+        self._diagnostic_count = 0
+        self._network_diagnostics = network_diagnostics
+        self.session_pool = session_pool
+        self._owns_session_pool = False
+        self.before_navigation = before_navigation
+        self.after_navigation = after_navigation
+        self.on_error = on_error
+        self.on_failure = on_failure
+        if session_pool is not None and proxy is not None:
+            raise ConfigurationError(
+                "Pool proxy configuration belongs in SessionPool.proxy_factory"
+            )
+        self.failures: list[Failure] = []
+
+    @property
+    def stats(self) -> Stats:
+        return self.context.stats
+
+    def _log(self, level: int, message: str, *args: object) -> None:
+        # 日志接收端故障不能改变采集、取消或资源回收的行为。
+        try:
+            self.log.log(level, "[crawler] " + message, *args)
+        except Exception:
+            pass
+
+    def cancel(self, reason: str = "cancelled") -> None:
+        self.context.cancel(reason)
+
+    @property
+    def proxy(self) -> Any:
+        """Current proxy handle. Explicit rotation affects new upstream connections."""
+        self._require_open()
+        if self.session_pool is not None:
+            raise ConfigurationError("Proxy handles are session-scoped in SessionPool mode")
+        return self._proxy
+
+    async def rotate_proxy(self) -> None:
+        """Rotate at an idle boundary and start a fresh acquisition generation.
+
+        Existing HTTP connections and browser contexts are retired even when the
+        provider keeps the same tunnel URL. The new generation is seeded only from
+        the configured ``Identity``; cookies observed by the old exit are discarded.
+        """
+        self._require_open()
+        if self.session_pool is not None:
+            raise ConfigurationError(
+                "rotate_proxy is unavailable in pool mode; retire the current session instead"
+            )
+        if self._proxy is None:
+            raise ConfigurationError("A proxy Provider is required for rotation")
+        await self.context.run(self._rotate_proxy_generation())
+
+    async def retire_sessions(self) -> None:
+        """Close all idle pool generations; active sessions retire by lease control."""
+        self._require_open()
+        if self.session_pool is None:
+            raise ConfigurationError("A SessionPool is required to retire sessions")
+        await self.context.run(self.session_pool.retire_idle())
+
+    def _new_http(self) -> HttpSession:
+        proxy_url = self._proxy.http_url() if self._proxy is not None else None
+        return HttpSession(
+            self.context.limits,
+            self.identity,
+            proxy_url=proxy_url,
+            transport=self._transport,
+            close_transport=self._transport is None,
+        )
+
+    @asynccontextmanager
+    async def _acquisition(self) -> AsyncIterator[None]:
+        async with self._resource_lock:
+            if self._closed:
+                raise RuntimeError("Crawler closed before acquisition could start")
+            self.context.check()
+            if self._resource_failure is not None:
+                raise ConfigurationError(
+                    "Crawler acquisition resources are unavailable after proxy rotation failure"
+                )
+            if self._rotating:
+                raise ConfigurationError("Proxy rotation is already in progress")
+            self._active_acquisitions += 1
+            task = asyncio.current_task()
+            if task is not None:
+                self._acquisition_tasks[task] = self._acquisition_tasks.get(task, 0) + 1
+            marker = _AcquisitionScope(id(self))
+            token = _ACQUISITION_SCOPES.set((*_ACQUISITION_SCOPES.get(), marker))
+        try:
+            yield
+        finally:
+            marker.active = False
+            _ACQUISITION_SCOPES.reset(token)
+            async with self._resource_lock:
+                self._active_acquisitions -= 1
+                if task is not None:
+                    remaining = self._acquisition_tasks[task] - 1
+                    if remaining:
+                        self._acquisition_tasks[task] = remaining
+                    else:
+                        del self._acquisition_tasks[task]
+
+    async def _retire(self, resources: Iterable[Any]) -> list[BaseException]:
+        errors: list[BaseException] = []
+        for resource in resources:
+            if resource is None:
+                continue
+            self._retired_resources.append(resource)
+            try:
+                await self._close_resource(resource)
+            except asyncio.CancelledError as error:
+                errors.append(error)
+            except Exception as error:
+                errors.append(error)
+            else:
+                self._retired_resources.remove(resource)
+        return errors
+
+    async def _close_resource(self, resource: Any) -> None:
+        closer = getattr(resource, "close", None)
+        if not callable(closer):
+            closer = getattr(resource, "aclose", None)
+        if not callable(closer):
+            raise TypeError("Acquisition resource exposes no async close method")
+        result = closer()
+        if inspect.isawaitable(result):
+            await asyncio.wait_for(result, timeout=10)
+
+    async def _rotate_proxy_generation(self) -> None:
+        async with self._resource_lock:
+            if self._closed:
+                raise RuntimeError("Crawler closed before proxy rotation could start")
+            self.context.check()
+            if self._resource_failure is not None:
+                raise ConfigurationError(
+                    "Crawler acquisition resources are unavailable after proxy rotation failure"
+                )
+            if self._active_acquisitions or self._crawling:
+                raise ConfigurationError(
+                    "Proxy rotation requires an idle crawler with no active acquisition or crawl"
+                )
+            self._rotating = True
+            old_browser, old_http = self._browser, self._http
+            self._browser = None
+            self._http = None
+            try:
+                errors = await self._retire((old_browser, old_http))
+                cancellation = next(
+                    (error for error in errors if isinstance(error, asyncio.CancelledError)), None
+                )
+                if cancellation is not None:
+                    self._resource_failure = "rotation_cancelled"
+                    raise cancellation
+                if errors:
+                    self._resource_failure = "retirement_failed"
+                    raise FetchError(
+                        "Proxy rotation resource retirement failed", phase="proxy_rotation"
+                    ) from None
+                try:
+                    assert self._proxy is not None
+                    await self._proxy.rotate()
+                except asyncio.CancelledError:
+                    self._resource_failure = "rotation_cancelled"
+                    raise
+                except (ConfigurationError, LimitReached):
+                    self._resource_failure = "rotation_failed"
+                    raise
+                except Exception:
+                    self._resource_failure = "rotation_failed"
+                    raise ConfigurationError(
+                        "Proxy rotation failed; crawler resources are unavailable"
+                    ) from None
+                try:
+                    self._http = self._new_http()
+                except Exception:
+                    self._resource_failure = "http_rebuild_failed"
+                    raise ConfigurationError(
+                        "HTTP session rebuild failed after proxy rotation"
+                    ) from None
+            finally:
+                self._rotating = False
+
+    async def __aenter__(self) -> Crawler:
+        if self._entered or self._closed:
+            raise RuntimeError(
+                "A Crawler is a single-run context; create a new one for another run"
+            )
+        self._entered = True
+        self.context.start()
+        if self.proxy_provider is not None:
+            proxy_label = provider_name(self.proxy_provider)
+        elif self.session_pool is not None and self.session_pool.proxy_factory is not None:
+            proxy_label = "session-pool-policy"
+        elif self.browser_provider is not None:
+            proxy_label = "browser-provider-policy"
+        else:
+            proxy_label = "none"
+        self._log(
+            logging.INFO, "Started browser=%s proxy=%s session_pool=%s",
+            provider_name(self.browser_provider),
+            proxy_label,
+            self.session_pool is not None,
+        )
+        phase = "startup"
+        try:
+            if self.session_pool is not None:
+                self.session_pool.bind(
+                    limits=self.context.limits,
+                    identity=self.identity,
+                    transport=self._transport,
+                    browser_provider=self.browser_provider,
+                    close_resource=self._close_resource,
+                )
+                self._owns_session_pool = True
+                return self
+            if self.proxy_provider is not None:
+                phase = "proxy_acquire"
+                self._log(
+                    logging.DEBUG, "Proxy acquiring provider=%s",
+                    provider_name(self.proxy_provider),
+                )
+                self._proxy = await self.context.run(self.proxy_provider.acquire())
+                self._log(
+                    logging.INFO, "Proxy ready provider=%s", provider_name(self.proxy_provider),
+                )
+            phase = "startup"
+            self._http = self._new_http()
+            return self
+        except BaseException as error:
+            stage, code = failure_details(error, phase)
+            self._log(logging.ERROR, "Startup failed phase=%s code=%s", stage, code)
+            try:
+                await self.close()
+            except Exception:
+                error.add_note("Acquisition resource cleanup also failed")
+            raise
+
+    async def __aexit__(self, *exc: Any) -> None:
+        try:
+            await self.close()
+        except Exception:
+            if exc[1] is None:
+                raise
+            exc[1].add_note("Acquisition resource cleanup also failed")
+
+    async def close(self) -> None:
+        current = asyncio.current_task()
+        if any(
+            marker.active and marker.crawler_id == id(self) for marker in _ACQUISITION_SCOPES.get()
+        ) or (current is not None and current in self._acquisition_tasks):
+            raise ConfigurationError("Cannot close a Crawler from an active acquisition")
+        if self._close_task is None:
+            # Set the terminal state before scheduling cleanup so an in-progress
+            # rotation or acquisition cannot complete a new generation.
+            self._closed = True
+            self.context.cancelled.set()
+            self._close_task = asyncio.create_task(self._close_resources())
+        cleanup = self._close_task
+        try:
+            await asyncio.shield(cleanup)
+        except asyncio.CancelledError as cancellation:
+            # A caller cancellation must not strand shared acquisition resources.
+            try:
+                await asyncio.shield(cleanup)
+            except Exception:
+                cancellation.add_note("Crawler cleanup also failed")
+            raise
+
+    async def _close_resources(self) -> None:
+        self._log(logging.DEBUG, "Resource cleanup started")
+        async with self._resource_lock:
+            active = set(self._active) | set(self._acquisition_tasks)
+        for task in active:
+            task.cancel()
+        await asyncio.gather(*active, return_exceptions=True)
+        async with self._resource_lock:
+            if self._active_acquisitions:
+                raise FetchError("Active acquisitions did not stop during cleanup")
+        # Each resource is attempted even when another cleanup operation fails.
+        errors: list[BaseException] = []
+        resources = [
+            self.session_pool if self._owns_session_pool else None,
+            self._browser,
+            self._http,
+            *self._retired_resources,
+            self._proxy,
+        ]
+        if self._transport is not None:
+            resources.append(self._transport)
+        seen: set[int] = set()
+        for resource in resources:
+            if resource is not None:
+                if id(resource) in seen:
+                    continue
+                seen.add(id(resource))
+                try:
+                    await self._close_resource(resource)
+                except asyncio.CancelledError as exc:
+                    errors.append(exc)
+                except Exception as exc:
+                    errors.append(exc)
+        cancellation = next(
+            (error for error in errors if isinstance(error, asyncio.CancelledError)), None
+        )
+        if cancellation is not None:
+            self._log(logging.WARNING, "Resource cleanup interrupted code=CRAWL_CANCELLED")
+            raise cancellation
+        if errors:
+            self._log(logging.ERROR, "Resource cleanup failed code=CLEANUP_FAILED")
+            raise FetchError("One or more acquisition resources failed to close") from None
+        self._log(
+            logging.WARNING if self._cleanup_incomplete else logging.INFO,
+            "Resource cleanup finished status=%s; "
+            "requests=%d captured=%d failed=%d retries=%d elapsed=%.3fs",
+            "incomplete" if self._cleanup_incomplete else "complete",
+            self._log_requests, self.stats.captured, self._log_failed, self.stats.retries,
+            max(0, time.monotonic() - (self.context._started or time.monotonic())),
+        )
+
+    def _require_open(self) -> None:
+        if not self._entered or self._closed:
+            raise RuntimeError("Use async with Crawler(...) before acquiring data")
+        self.context.check()
+
+    def _gate(self, key: str) -> SiteGate:
+        if key not in self._sites:
+            if len(self._sites) >= self.context.limits.max_requests:
+                raise LimitReached("Site-state memory limit reached")
+            self._sites[key] = SiteGate(self.context)
+        return self._sites[key]
+
+    async def _get_browser(self, lease: SessionLease | None = None) -> Any:
+        if lease is not None:
+            self._log(logging.DEBUG, "Browser session acquiring source=session-pool")
+            handle = await lease.browser(required_capabilities=self._required_capabilities)
+            self._log(logging.DEBUG, "Browser session ready source=session-pool")
+            return handle
+        async with self._browser_lock:
+            if self._browser is None:
+                if self.browser_provider is None:
+                    raise ConfigurationError("A browser Provider is required for browser mode")
+                started = time.monotonic()
+                self._log(
+                    logging.DEBUG, "Browser session acquiring provider=%s",
+                    provider_name(self.browser_provider),
+                )
+                try:
+                    self._browser = await self.context.run(
+                        self.browser_provider.open(identity=self.identity, proxy=self._proxy)
+                    )
+                except BaseException as error:
+                    _, code = failure_details(error, "browser_open")
+                    self._log(
+                        logging.DEBUG, "Browser session failed phase=browser_open code=%s", code,
+                    )
+                    raise
+                self._log(
+                    logging.INFO, "Browser session ready elapsed=%.3fs", time.monotonic() - started,
+                )
+            self._check_capabilities(self._browser)
+            return self._browser
+
+    def _check_capabilities(self, handle: Any) -> None:
+        if self._required_capabilities - handle.capabilities:
+            raise ConfigurationError("Browser Provider does not support required capabilities")
+
+    @asynccontextmanager
+    async def browser(self) -> AsyncIterator[Any]:
+        """Yield a browser handle whose native Pages live until context exit.
+
+        The handle is exclusive to this context and is always closed on exit.
+        Prefer ``fetch`` when one process callback can finish all page work.
+        """
+        self._require_open()
+        if self.session_pool is not None:
+            raise ConfigurationError("Manual browser sessions are unavailable in SessionPool mode")
+        if self.browser_provider is None:
+            raise ConfigurationError("A browser Provider is required")
+        async with self._acquisition():
+            self._log(
+                logging.DEBUG, "Browser session acquiring provider=%s",
+                provider_name(self.browser_provider),
+            )
+            try:
+                handle = await self.context.run(
+                    self.browser_provider.open(identity=self.identity, proxy=self._proxy)
+                )
+            except BaseException as error:
+                _, code = failure_details(error, "browser_open")
+                self._log(logging.ERROR, "Browser session failed phase=browser_open code=%s", code)
+                raise
+            self._log(logging.INFO, "Browser session ready")
+            try:
+                self._check_capabilities(handle)
+                yield handle
+            finally:
+                primary_error = sys.exc_info()[1]
+                try:
+                    await asyncio.wait_for(handle.close(), timeout=10)
+                except BaseException as error:
+                    self._cleanup_incomplete = True
+                    self._log(logging.ERROR, "Browser session cleanup failed code=CLEANUP_FAILED")
+                    if not isinstance(error, Exception):
+                        raise
+                    if primary_error is None:
+                        raise FetchError("Browser session cleanup failed") from None
+                    primary_error.add_note("Browser session cleanup also failed")
+
+    async def run_browser(self, collect: BrowserCollect) -> Any:
+        """Run a page-driven flow with one native Page and this CrawlContext.
+
+        ``collect(page, context)`` owns navigation, extraction and business output.
+        Its return value is passed through; nothing is written to a Dataset here.
+        The whole flow observes cancellation and ``Limits.max_duration``. It is
+        never automatically retried because browser actions may have side effects.
+
+        Pages created through browser_tools navigation helpers are scoped there;
+        the root Page is closed with the manual browser handle on every exit.
+        Direct Playwright actions do not increment request/page/result counters
+        or apply per-site scheduling; use fetch/crawl for request-driven work.
+        Browser tools account for their explicit navigations, not every subresource
+        or navigation caused by native Playwright actions. SessionPool is unsupported,
+        as with the existing manual ``browser()`` context.
+
+        Failure logs retain a fixed exception type and the root Page's hostname,
+        never exception messages or URL paths. The root Page may differ from a
+        failing detail tab. Capture evidence inside that tab's scope, before its
+        cleanup, when private diagnostic files are explicitly needed.
+        """
+        self._require_open()
+        if not callable(collect):
+            raise ConfigurationError("Browser collect must be callable")
+        async with self.browser() as handle:
+            try:
+                page = await self.context.run(
+                    handle.new_page(), timeout=self.context.limits.request_timeout,
+                )
+            except BaseException as error:
+                phase, code = failure_details(error, "page_creation")
+                self._log(
+                    logging.ERROR, "Browser flow failed phase=%s code=%s%s",
+                    phase, code, failure_context(error),
+                )
+                raise
+
+            async def invoke() -> Any:
+                result = collect(page, self.context)
+                return await result if inspect.isawaitable(result) else result
+
+            started = time.monotonic()
+            self._log(logging.DEBUG, "Browser flow started")
+            try:
+                result = await self.context.run(invoke())
+            except BaseException as error:
+                phase, code = failure_details(error, "collect")
+                self._log(
+                    logging.ERROR, "Browser flow failed phase=%s code=%s%s root_host=%s",
+                    phase, code, failure_context(error), root_page_host(page),
+                )
+                raise
+            self._log(
+                logging.INFO, "Browser flow finished elapsed=%.3fs", time.monotonic() - started,
+            )
+            return result
+
+    async def fetch(self, request: Request | str, *, process: Process | None = None) -> Any:
+        """Acquire one request and optionally transform it with ``process``.
+
+        Without ``process`` this returns :class:`CaptureResult`. With a process
+        callback it returns that callback's value. A browser Page is valid only
+        during the callback; bound individual Page operations with
+        ``context.run(page_method(), timeout=...)`` when needed.
+        """
+        self._require_open()
+        req = Request(request) if isinstance(request, str) else request
+        self._log_requests += 1
+        request_number = self._log_requests
+        started = time.monotonic()
+        self._log(
+            logging.DEBUG, "Request started request=%d mode=%s host=%s",
+            request_number, req.mode, target_host(req.url),
+        )
+        try:
+            async with self._acquisition():
+                data = await self.context.run(self._fetch(req, process))
+        except BaseException as error:
+            self._log_failed += 1
+            phase, code = failure_details(error, "request")
+            self._log(
+                logging.ERROR,
+                "Request failed request=%d phase=%s code=%s mode=%s host=%s elapsed=%.3fs%s",
+                request_number, phase, code, req.mode, target_host(req.url),
+                time.monotonic() - started, failure_context(error),
+            )
+            raise
+        self._log(
+            logging.INFO, "Request finished request=%d mode=%s host=%s elapsed=%.3fs",
+            request_number, req.mode, target_host(req.url), time.monotonic() - started,
+        )
+        return data
+
+    async def _fetch(self, request: Request, process: Process | None) -> Any:
+        notified = False
+        try:
+            if self.session_pool is None:
+                return await self._fetch_with_resources(request, process, None)
+            if any(
+                marker.active and marker.crawler_id == id(self)
+                for marker in _SESSION_FETCH_SCOPES.get()
+            ):
+                raise _RecursivePooledFetch(
+                    "A pooled fetch cannot recursively await another fetch on the same Crawler"
+                )
+            target_origin = origin(request.url)
+            affinity = (
+                ("explicit", f"{target_origin}\0{request.session_key}")
+                if request.session_key is not None
+                else ("origin", target_origin)
+            )
+            async with self.session_pool.lease(affinity, origin=target_origin) as lease:
+                marker = _SessionFetchScope(id(self))
+                scope_token = _SESSION_FETCH_SCOPES.set((*_SESSION_FETCH_SCOPES.get(), marker))
+                session_token = self.context._set_session(lease.session)
+                try:
+                    return await self._fetch_with_resources(request, process, lease)
+                except (CrawlCancelled, LimitReached):
+                    raise
+                except Exception as exc:
+                    notified = True
+                    await self._notify(self.on_failure, self._failure(request, exc))
+                    raise
+                finally:
+                    marker.active = False
+                    self.context._reset_session(session_token)
+                    _SESSION_FETCH_SCOPES.reset(scope_token)
+        except (CrawlCancelled, LimitReached):
+            raise
+        except Exception as exc:
+            if not notified:
+                await self._notify(self.on_failure, self._failure(request, exc))
+            raise
+
+    async def _fetch_with_resources(
+        self, request: Request, process: Process | None, lease: SessionLease | None
+    ) -> Any:
+        async with self._global:
+            if request.mode != "auto":
+                return await self._fetch_attempts(request, process, lease)
+            initial = self.adaptive_policy.initial_request(request)
+            capture = await self._fetch_attempts(initial, None, lease)
+            decision = await self.adaptive_policy.decide(capture)
+            if decision.use_browser:
+                self._log(logging.INFO, "Adaptive acquisition switching HTTP to browser")
+                browser_request = self.adaptive_policy.browser_request(request)
+
+                async def selected(capture: CaptureResult, ctx: CrawlContext) -> Any:
+                    capture.diagnostics["adaptive"] = {
+                        "selected": "browser",
+                        "reason": decision.reason,
+                        "http_status": capture_status,
+                    }
+                    if process is None:
+                        return capture
+                    result = process(capture, ctx)
+                    return await result if inspect.isawaitable(result) else result
+
+                capture_status = capture.status
+                return await self._fetch_attempts(browser_request, selected, lease)
+            capture.diagnostics["adaptive"] = {
+                "selected": "http",
+                "reason": decision.reason,
+            }
+            return await self._process_capture(capture, process)
+
+    async def _process_capture(self, capture: CaptureResult, process: Process | None) -> Any:
+        if process is None:
+            return capture
+        started = time.monotonic()
+        self._log(logging.DEBUG, "Process started host=%s", target_host(capture.request.url))
+        try:
+            result = process(capture, self.context)
+            result = await result if inspect.isawaitable(result) else result
+            self._log(logging.DEBUG, "Process finished elapsed=%.3fs", time.monotonic() - started)
+            return result
+        except (CrawlCancelled, LimitReached, _RecursivePooledFetch):
+            raise
+        except Exception as exc:
+            code = process_error_code(exc)
+            self._log(logging.DEBUG, "Process failed phase=process code=%s", code)
+            capture.diagnostics["process_error_code"] = code
+            await self._diagnose(capture, code)
+            raise ProcessingError(
+                "Developer process failed; acquisition was not retried",
+                capture=capture,
+                process_error_code=code,
+            ) from None
+
+    async def _fetch_attempts(
+        self, request: Request, process: Process | None, lease: SessionLease | None
+    ) -> Any:
+        self.context.check()
+        if self.stats.pages >= self.context.limits.max_pages:
+            self.stats.stop_reason = "max_pages"
+            raise LimitReached("Page limit reached")
+        self.stats.pages += 1
+        retry_events: list[dict[str, Any]] = []
+        for attempt in range(self.context.limits.retries + 1):
+            self._log(
+                logging.DEBUG, "Acquisition attempt=%d/%d mode=%s host=%s",
+                attempt + 1, self.context.limits.retries + 1,
+                request.mode, target_host(request.url),
+            )
+            page = None
+            handle = None
+            capture = None
+            network = None
+            cookie_handle = None
+            http_used = False
+            try:
+                if request.mode == "browser":
+                    handle = await self._get_browser(lease)
+                    if lease is not None:
+                        await lease.sync_to_browser(handle)
+                        cookie_handle = handle
+                    try:
+                        page = await handle.new_page()
+                    except Exception:
+                        raise FetchError(
+                            "Browser page creation failed", phase="page_creation"
+                        ) from None
+                    async with self._gate(request.site_key or origin(request.url)):
+                        self.context.reserve_request()
+                        started = time.monotonic()
+                        url = str(httpx.URL(request.url).copy_merge_params(request.params or {}))
+                        phase = "navigation"
+                        try:
+                            network = observe_network(
+                                page,
+                                started,
+                                capture_events=self._network_diagnostics,
+                            )
+                            try:
+                                navigation_timeout = (
+                                    self.context.limits.navigation_timeout
+                                    or self.context.limits.request_timeout
+                                )
+                                snapshot_timeout = (
+                                    self.context.limits.snapshot_timeout
+                                    or self.context.limits.request_timeout
+                                )
+                                async with asyncio.timeout(navigation_timeout):
+                                    phase = "before_navigation"
+                                    await self._navigation_hook(
+                                        "before_navigation",
+                                        self.before_navigation,
+                                        request,
+                                        page,
+                                    )
+                                    phase = "navigation"
+                                    response = await page.goto(
+                                        url,
+                                        wait_until=request.wait_until,
+                                        timeout=navigation_timeout * 1000,
+                                    )
+                                    phase = "after_navigation"
+                                    await self._navigation_hook(
+                                        "after_navigation",
+                                        self.after_navigation,
+                                        request,
+                                        page,
+                                    )
+                                async with asyncio.timeout(snapshot_timeout):
+                                    phase = "headers"
+                                    headers = await response.all_headers() if response else {}
+                                    phase = "snapshot"
+                                    body = (
+                                        (await page.content()).encode() if request.snapshot else b""
+                                    )
+                            finally:
+                                if network is not None:
+                                    network.detach()
+                        except Exception as exc:
+                            if isinstance(exc, FetchError):
+                                raise
+                            code = browser_error_code(exc)
+                            main_document_status = (
+                                network.main_document_status if network is not None else None
+                            )
+                            retryable = phase == "navigation" and (
+                                isinstance(exc, TimeoutError)
+                                or type(exc).__name__ == "TimeoutError"
+                                or code
+                                in {
+                                    "ERR_CONNECTION_RESET",
+                                    "ERR_CONNECTION_CLOSED",
+                                    "ERR_TIMED_OUT",
+                                }
+                                or (
+                                    code == "ERR_HTTP_RESPONSE_CODE_FAILURE"
+                                    and main_document_status in _RETRYABLE_STATUSES
+                                )
+                            )
+                            failure = FetchError(
+                                f"Browser {phase} failed",
+                                retryable=retryable,
+                                phase=phase,
+                                browser_error_code=code,
+                                network_diagnostics=(
+                                    network.summary()
+                                    if self._network_diagnostics and network is not None
+                                    else None
+                                ),
+                                main_document_status=main_document_status,
+                            )
+                            timeout_seconds = (
+                                navigation_timeout
+                                if phase in {"before_navigation", "navigation", "after_navigation"}
+                                else snapshot_timeout
+                            )
+                            failure.browser_diagnostics = {
+                                "exception_type": _safe_exception_type(exc),
+                                "wait_until": request.wait_until,
+                                "timeout_seconds": timeout_seconds,
+                                "elapsed_seconds": round(time.monotonic() - started, 3),
+                            }
+                            raise failure from None
+                    if len(body) > self.context.limits.max_response_bytes:
+                        raise ResponseTooLarge(
+                            "DOM snapshot exceeds configured byte limit",
+                            network_diagnostics=(
+                                network.summary()
+                                if self._network_diagnostics and network is not None
+                                else None
+                            ),
+                            main_document_status=(
+                                network.main_document_status if network is not None else None
+                            ),
+                        )
+                    diagnostics: dict[str, Any] = {
+                        "mode": "browser",
+                        "wait_until": request.wait_until,
+                        "snapshot": "included" if request.snapshot else "skipped",
+                    }
+                    if self._network_diagnostics and network is not None:
+                        diagnostics["network"] = network.summary()
+                    capture = CaptureResult(
+                        request,
+                        page.url,
+                        response.status if response else None,
+                        headers,
+                        body,
+                        page=page,
+                        elapsed=time.monotonic() - started,
+                        diagnostics=diagnostics,
+                    )
+                else:
+                    http = lease.http if lease is not None else self._http
+                    assert http is not None
+                    if lease is not None:
+                        await lease.sync_to_http()
+                        http_used = True
+                    capture = await http.fetch(request, self.context, self._gate)
+                capture.attempts = attempt + 1
+                self._log(
+                    logging.DEBUG, "Page acquired mode=%s status=%s elapsed=%.3fs",
+                    request.mode, capture.status, capture.elapsed,
+                )
+                if retry_events:
+                    capture.diagnostics["retry_events"] = list(retry_events)
+                if (
+                    capture.status in _RETRYABLE_STATUSES
+                    and request.can_retry
+                    and attempt < self.context.limits.retries
+                ):
+                    await self._attempt_status_error(capture, attempt + 1)
+                    if lease is not None and capture.status in {401, 403, 429}:
+                        lease.session.mark_bad()
+                    delay = self._retry_delay(capture.headers.get("retry-after"), attempt)
+                    response_retry_event: dict[str, Any] = {
+                        "attempt": attempt + 1,
+                        "phase": "response",
+                        "status": capture.status,
+                    }
+                    if "network" in capture.diagnostics:
+                        response_retry_event["network"] = capture.diagnostics["network"]
+                    retry_events.append(response_retry_event)
+                    self._gate(request.site_key or origin(capture.final_url)).defer(delay)
+                    self.stats.retries += 1
+                    self._log(
+                        logging.WARNING,
+                        "Retry scheduled phase=response status=%s attempt=%d/%d delay=%.3fs",
+                        capture.status, attempt + 1, self.context.limits.retries + 1, delay,
+                    )
+                    await self.context.sleep(delay)
+                    continue
+                if capture.status in {401, 403, 429, 500, 502, 503, 504}:
+                    await self._attempt_status_error(capture, attempt + 1)
+                if lease is not None and capture.status in {401, 403, 429}:
+                    lease.session.mark_bad()
+                elif lease is not None and (capture.status is None or capture.status < 400):
+                    lease.session.mark_good()
+                self.stats.captured += 1
+                if capture.access_issue:
+                    self._log(
+                        logging.WARNING, "Page access issue detected; inspect capture diagnostics",
+                    )
+                    await self._diagnose(capture, capture.access_issue)
+                return await self._process_capture(capture, process)
+            except FetchError as exc:
+                if lease is not None and self._damages_session(exc):
+                    lease.session.mark_bad()
+                await self._notify(
+                    self.on_error,
+                    self._failure(request, exc, attempt=attempt + 1),
+                )
+                if (
+                    not exc.retryable
+                    or not request.can_retry
+                    or attempt >= self.context.limits.retries
+                ):
+                    exc.retry_events = list(retry_events)
+                    raise
+                event: dict[str, Any] = {"attempt": attempt + 1, "phase": exc.phase}
+                if exc.browser_error_code is not None:
+                    event["browser_error_code"] = exc.browser_error_code
+                if exc.main_document_status is not None:
+                    event["main_document_status"] = exc.main_document_status
+                if exc.network_diagnostics is not None:
+                    event["network"] = exc.network_diagnostics
+                retry_events.append(event)
+                self.stats.retries += 1
+                delay = self._retry_delay(None, attempt)
+                phase, code = failure_details(exc, "request")
+                self._log(
+                    logging.WARNING,
+                    "Retry scheduled phase=%s code=%s attempt=%d/%d delay=%.3fs",
+                    phase, code, attempt + 1, self.context.limits.retries + 1, delay,
+                )
+                await self.context.sleep(delay)
+            finally:
+                primary_error = sys.exc_info()[1]
+
+                async def finish_attempt(
+                    cookie_handle: Any = cookie_handle,
+                    http_used: bool = http_used,
+                    page: Any = page,
+                    handle: Any = handle,
+                    capture: CaptureResult | None = capture,
+                ) -> None:
+                    cleanup_error: BaseException | None = None
+                    try:
+                        if lease is not None:
+                            if cookie_handle is not None:
+                                await asyncio.wait_for(lease.sync_from_browser(cookie_handle), 5)
+                            elif http_used:
+                                await asyncio.wait_for(lease.sync_from_http(), 5)
+                    except BaseException as exc:
+                        cleanup_error = exc
+                        if lease is not None:
+                            lease.session.retire()
+                    finally:
+                        if page is not None:
+                            try:
+                                await handle.close_page(page)
+                            except BaseException as exc:
+                                if lease is not None:
+                                    lease.session.retire()
+                                if cleanup_error is None:
+                                    if isinstance(exc, Exception):
+                                        cleanup_error = FetchError(
+                                            "Browser page cleanup failed", phase="page_cleanup"
+                                        )
+                                    else:
+                                        cleanup_error = exc
+                                else:
+                                    cleanup_error.add_note("Browser page cleanup also failed")
+                            finally:
+                                if capture is not None and capture.page is page:
+                                    capture.page = None
+                    if cleanup_error is not None:
+                        raise cleanup_error
+
+                cleanup = asyncio.create_task(finish_attempt())
+                cancellation: asyncio.CancelledError | None = None
+                while not cleanup.done():
+                    try:
+                        await asyncio.shield(cleanup)
+                    except asyncio.CancelledError as exc:
+                        cancellation = cancellation or exc
+                    except Exception:
+                        break
+                try:
+                    cleanup.result()
+                except BaseException as cleanup_error:
+                    if cancellation is not None:
+                        cancellation.add_note("Attempt cleanup also failed")
+                    elif isinstance(cleanup_error, asyncio.CancelledError):
+                        raise
+                    elif primary_error is None:
+                        if isinstance(cleanup_error, FetchError):
+                            await self._notify(
+                                self.on_error,
+                                self._failure(request, cleanup_error, attempt=attempt + 1),
+                            )
+                        raise
+                    else:
+                        primary_error.add_note("Attempt resource or cookie cleanup also failed")
+                if cancellation is not None:
+                    raise cancellation
+        raise FetchError("Acquisition retry limit reached")
+
+    def _damages_session(self, error: FetchError) -> bool:
+        """Only transport/access failures affect reuse health, never hooks or parsing."""
+        return error.phase == "http_transport" or (
+            error.phase == "navigation"
+            and (
+                (error.retryable and error.main_document_status is None)
+                or error.browser_error_code is not None
+                or error.main_document_status in {401, 403, 429}
+            )
+        )
+
+    async def _navigation_hook(
+        self, name: str, hook: NavigationHook | None, request: Request, page: Any
+    ) -> None:
+        if hook is None:
+            return
+        try:
+            result = hook(request, page, self.context)
+            if inspect.isawaitable(result):
+                await result
+        except (CrawlCancelled, LimitReached):
+            raise
+        except Exception:
+            raise FetchError(f"{name} hook failed", phase=name) from None
+
+    async def _attempt_status_error(self, capture: CaptureResult, attempt: int) -> None:
+        await self._notify(
+            self.on_error,
+            Failure(
+                capture.request,
+                "network",
+                "Acquisition attempt returned an error status",
+                {"phase": "response", "status": capture.status, "attempt": attempt},
+            ),
+        )
+
+    async def _notify(self, hook: FailureHook | None, failure: Failure) -> None:
+        if hook is None:
+            return
+        try:
+            result = hook(failure, self.context)
+            if inspect.isawaitable(result):
+                timeout = min(self.context.limits.request_timeout, self.context.remaining)
+                await asyncio.wait_for(result, timeout=max(0.001, timeout))
+        except Exception as exc:
+            failure.diagnostics["hook_error_code"] = (
+                "HOOK_TIMEOUT" if isinstance(exc, TimeoutError) else "HOOK_ERROR"
+            )
+
+    def _failure(self, request: Request, exc: Exception, *, attempt: int | None = None) -> Failure:
+        diagnostics = dict(getattr(getattr(exc, "capture", None), "diagnostics", {}))
+        phase = getattr(exc, "phase", None)
+        if phase is not None:
+            diagnostics["phase"] = phase
+        if attempt is not None:
+            diagnostics["attempt"] = attempt
+        if isinstance(exc, ProcessingError):
+            diagnostics["process_error_code"] = process_error_code(exc)
+        code = browser_error_code(exc)
+        if code is not None:
+            diagnostics["browser_error_code"] = code
+        http_code = http_error_code(exc)
+        if http_code is not None:
+            diagnostics["http_error_code"] = http_code
+        if isinstance(exc, FetchError) and exc.retry_events:
+            diagnostics["retry_events"] = exc.retry_events
+        if isinstance(exc, FetchError) and exc.network_diagnostics is not None:
+            diagnostics["network"] = exc.network_diagnostics
+        if isinstance(exc, FetchError) and exc.main_document_status is not None:
+            diagnostics["main_document_status"] = exc.main_document_status
+        if isinstance(exc, FetchError):
+            diagnostics.update(exc.browser_diagnostics)
+        return Failure(
+            request,
+            getattr(exc, "kind", "browser"),
+            "Acquisition or process failed",
+            diagnostics,
+        )
+
+    async def _diagnose(self, capture: CaptureResult, kind: str) -> None:
+        capture.diagnostics["error_kind"] = kind
+        # Explicit opt-in only: snapshots can contain sensitive website content.
+        if self._diagnostics_dir is None or self._diagnostic_count >= 20:
+            return
+        self._diagnostic_count += 1
+        try:
+            self._diagnostics_dir.mkdir(parents=True, exist_ok=True)
+            stem = self._diagnostics_dir / uuid4().hex
+            snapshot = stem.with_suffix(".html" if capture.page is not None else ".body")
+            snapshot.write_bytes(capture.body[: min(self.context.limits.max_response_bytes, 2**21)])
+            capture.diagnostics["snapshot_path"] = str(snapshot)
+            if capture.page is not None:
+                shot = await asyncio.wait_for(capture.page.screenshot(), timeout=5)
+                if len(shot) <= 2**21:
+                    image = stem.with_suffix(".png")
+                    image.write_bytes(shot)
+                    capture.diagnostics["screenshot_path"] = str(image)
+        except Exception:
+            capture.diagnostics["snapshot_error"] = "diagnostic capture unavailable"
+
+    def _retry_delay(self, header: str | None, attempt: int) -> float:
+        if header:
+            try:
+                return max(0, float(header))
+            except ValueError:
+                try:
+                    return max(
+                        0, (parsedate_to_datetime(header) - datetime.now(UTC)).total_seconds()
+                    )
+                except (ValueError, TypeError, OverflowError):
+                    pass
+        return self.context.limits.backoff * 2**attempt
+
+    def export_checkpoint(self) -> CrawlCheckpoint:
+        """Snapshot queued work; Runtime/developer owns private artifact persistence.
+
+        Unread lazy seeds cannot be serialized. Inflight or partly yielded requests
+        are replayed on restore, so processors and outputs must tolerate duplicates.
+        Identity state, browser state and provider configuration are never exported.
+        """
+        state = self._queue_state
+        if state is not None:
+            if not state["exhausted"]:
+                raise ConfigurationError(
+                    "Checkpoint cannot include unread seeds; supply a bounded seed batch"
+                )
+            return CrawlCheckpoint.capture(
+                pending=state["pending"],
+                inflight=[*state["tasks"].values(), *state["delivering"]],
+                dedup_keys=state["keys"],
+                seen_keys=self.context._seen,
+                stats=self.stats,
+                elapsed_seconds=max(
+                    0, time.monotonic() - (self.context._started or time.monotonic())
+                ),
+            )
+        if self._last_checkpoint is not None:
+            return self._last_checkpoint
+        raise ConfigurationError(
+            self._checkpoint_unavailable or "No crawl queue is available to checkpoint"
+        )
+
+    async def crawl(
+        self,
+        seeds: Iterable[Request | str] | None = None,
+        *,
+        process: Process | None = None,
+        checkpoint: CrawlCheckpoint | None = None,
+    ) -> AsyncGenerator[CrawlItem, None]:
+        """Yield processed items from a bounded, deduplicated in-memory queue.
+
+        ``process`` may enqueue follow-up requests through the context. Restoring
+        a checkpoint requires a fresh Crawler, and checkpoint persistence remains
+        the caller's responsibility. Each yielded item's browser Page has already
+        been closed and cleared.
+        """
+        self._require_open()
+        if self._crawling:
+            raise RuntimeError("Only one crawl may own a Crawler queue")
+        if checkpoint is not None:
+            if seeds is not None and (not isinstance(seeds, (list, tuple)) or len(seeds) != 0):
+                raise ConfigurationError("Checkpoint restore requires empty seeds or None")
+            if (
+                any(
+                    getattr(self.stats, name)
+                    for name in (
+                        "requests",
+                        "pages",
+                        "captured",
+                        "results",
+                        "retries",
+                        "duplicates",
+                        "failed",
+                    )
+                )
+                or self.context._seen
+                or self._last_checkpoint is not None
+            ):
+                raise ConfigurationError("Restore requires a fresh unused Crawler")
+            queued = checkpoint.requeued_requests
+            if len(queued) > self.context.limits.max_pending:
+                raise ConfigurationError("Checkpoint exceeds this Crawler's pending capacity")
+            self.context.stats = checkpoint.stats
+            self.context._seen = set(checkpoint.seen_keys)
+            self.context._started = time.monotonic() - checkpoint.elapsed_seconds
+        self._crawling = True
+        pending: deque[Request] = deque(checkpoint.requeued_requests if checkpoint else ())
+        keys: set[str] = set(checkpoint.dedup_keys if checkpoint else ())
+        iterator = iter(seeds if seeds is not None else ())
+        exhausted = checkpoint is not None or seeds is None
+
+        def enqueue(req: Request) -> bool:
+            self.context.check()
+            if req.deduplicate and req.unique_key in keys:
+                self.stats.duplicates += 1
+                return False
+            if len(pending) >= self.context.limits.max_pending:
+                raise QueueFull("Pending request limit reached; finish a batch before expanding")
+            if len(keys) >= self.context.limits.max_pages:
+                raise LimitReached("Queued-page limit reached")
+            if req.deduplicate:
+                keys.add(req.unique_key)
+            pending.append(req)
+            return True
+
+        async def execute(req: Request) -> tuple[CaptureResult, Any]:
+            holder: list[CaptureResult] = []
+
+            async def callback(capture: CaptureResult, ctx: CrawlContext) -> Any:
+                holder.append(capture)
+                if process is None:
+                    return capture
+                result = process(capture, ctx)
+                return await result if inspect.isawaitable(result) else result
+
+            data = await self.fetch(req, process=callback)
+            return holder[0], data
+
+        self.context._enqueue = enqueue
+        tasks: dict[asyncio.Task[Any], Request] = {}
+        delivering: list[Request] = []
+        self._queue_state = {
+            "pending": pending,
+            "keys": keys,
+            "tasks": tasks,
+            "delivering": delivering,
+            "exhausted": exhausted,
+        }
+        active_sites: dict[str, int] = {}
+        try:
+            while True:
+                self.context.check()
+                if self.stats.results >= self.context.limits.max_results:
+                    self.stats.stop_reason = "max_results"
+                    break
+                while not exhausted and len(pending) < self.context.limits.max_pending:
+                    if len(keys) >= self.context.limits.max_pages:
+                        exhausted = True
+                        self.stats.stop_reason = "max_pages"
+                        break
+                    try:
+                        seed = next(iterator)
+                    except StopIteration:
+                        exhausted = True
+                        self._queue_state["exhausted"] = True
+                        break
+                    enqueue(Request(seed) if isinstance(seed, str) else seed)
+                candidates = len(pending)
+                while pending and len(tasks) < self.context.limits.concurrency and candidates:
+                    candidates -= 1
+                    req = pending.popleft()
+                    site = req.site_key or origin(req.url)
+                    if active_sites.get(site, 0) >= self.context.limits.per_site_concurrency:
+                        pending.append(req)
+                        continue
+                    task = asyncio.create_task(execute(req))
+                    tasks[task] = req
+                    active_sites[site] = active_sites.get(site, 0) + 1
+                    self._active.add(task)
+                if not tasks:
+                    break
+                done, _ = await self.context.run(
+                    asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+                )
+                for task in done:
+                    if self.stats.results >= self.context.limits.max_results:
+                        break
+                    req = tasks.pop(task)
+                    site = req.site_key or origin(req.url)
+                    active_sites[site] -= 1
+                    self._active.discard(task)
+                    try:
+                        capture, data = task.result()
+                    except (CrawlCancelled, LimitReached, asyncio.CancelledError):
+                        # Keep interrupted work inflight so checkpoint export
+                        # can validate replayability instead of silently losing it.
+                        tasks[task] = req
+                        raise
+                    except Exception as exc:
+                        self.stats.failed += 1
+                        self.failures.append(self._failure(req, exc))
+                        continue
+                    records = data if isinstance(data, (list, tuple)) else [data]
+                    delivering[:] = [req]
+                    for record in records:
+                        if record is None:
+                            continue
+                        if self.stats.results >= self.context.limits.max_results:
+                            break
+                        self.stats.results += 1
+                        yield CrawlItem(record, capture)
+                    else:
+                        delivering.clear()
+                    if delivering:
+                        break
+        except CrawlCancelled:
+            pass
+        except LimitReached:
+            self.stats.stop_reason = self.stats.stop_reason or "limit"
+        finally:
+            try:
+                self._last_checkpoint = self.export_checkpoint()
+                self._checkpoint_unavailable = None
+            except ConfigurationError:
+                self._last_checkpoint = None
+                self._checkpoint_unavailable = (
+                    "Queue snapshot unavailable: unread seeds or non-replayable inflight work"
+                )
+            self.context._enqueue = None
+            for task in tasks:
+                task.cancel()
+            await asyncio.gather(*tasks, return_exceptions=True)
+            self._active.difference_update(tasks)
+            self._queue_state = None
+            self._crawling = False