apify 4.0.2b2__tar.gz → 4.0.2b4__tar.gz

{apify-4.0.2b2 → apify-4.0.2b4}/CHANGELOG.md

@@ -9,6 +9,8 @@ All notable changes to this project will be documented in this file.
 
 - Prolong and track shared request queue locks to prevent duplicate processing ([#1062](https://github.com/apify/apify-sdk-python/pull/1062)) ([ccaad7b](https://github.com/apify/apify-sdk-python/commit/ccaad7b71cd6cd1ddc2c71e5e9d9780001f2688a)) by [@vdusek](https://github.com/vdusek)
 - Verify shared request queue is_finished against per-request reads ([#1087](https://github.com/apify/apify-sdk-python/pull/1087)) ([22613aa](https://github.com/apify/apify-sdk-python/commit/22613aac0ce4e6ef852d5d3daf8c1a25cb3731e3)) by [@vdusek](https://github.com/vdusek)
+- Keep non-forefront reclaimed requests pending in the single request queue client ([#1060](https://github.com/apify/apify-sdk-python/pull/1060)) ([b61890c](https://github.com/apify/apify-sdk-python/commit/b61890cf286c3a83faedac69315acff74b5fba5b)) by [@vdusek](https://github.com/vdusek)
+- **scrapy:** Stop silently dropping in-flight requests and redirects ([#1097](https://github.com/apify/apify-sdk-python/pull/1097)) ([5a0a752](https://github.com/apify/apify-sdk-python/commit/5a0a752523b95c3143e5058fe07785e69817f4a2)) by [@vdusek](https://github.com/vdusek)
 
 
 <!-- git-cliff-unreleased-end -->

{apify-4.0.2b2 → apify-4.0.2b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 4.0.2b2
+Version: 4.0.2b4
 Summary: Apify SDK for Python
 Keywords: apify,sdk,automation,chrome,crawlee,crawler,headless,scraper,scraping
 Author: Apify Technologies s.r.o.

{apify-4.0.2b2 → apify-4.0.2b4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "apify"
-version = "4.0.2b2"
+version = "4.0.2b4"
 description = "Apify SDK for Python"
 license = "Apache-2.0"
 license-files = ["LICENSE"]

{apify-4.0.2b2 → apify-4.0.2b4}/pyproject.toml.orig

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "apify"
-version = "4.0.2b2"
+version = "4.0.2b4"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = "Apache-2.0"

{apify-4.0.2b2 → apify-4.0.2b4}/src/apify/scrapy/_async_thread.py

@@ -12,6 +12,9 @@ if TYPE_CHECKING:
 
 logger = getLogger(__name__)
 
+SUBMITTED_PRUNE_THRESHOLD = 128
+"""How many `submit_coro` futures may pile up before the finished ones are dropped from the tracking list."""
+
 
 class AsyncThread:
     """Run an asyncio event loop in a dedicated background thread.
@@ -26,6 +29,9 @@ class AsyncThread:
         self._default_timeout = default_timeout
         self._eventloop = asyncio.new_event_loop()
 
+        self._submitted: list[futures.Future] = []
+        """Futures of the coroutines submitted via `submit_coro` that may still be running."""
+
         # Start the event loop in a dedicated daemon thread.
         self._thread = threading.Thread(
             target=self._start_event_loop,
@@ -74,6 +80,55 @@ class AsyncThread:
             future.cancel()
             raise
 
+    def submit_coro(self, coro: Coroutine) -> futures.Future:
+        """Schedule a coroutine on the event loop without waiting for its result.
+
+        Use this for work nothing depends on, so the calling thread is not blocked by the round trip. Failures
+        are logged, as there is no caller to propagate them to, and `close` cancels whatever is still pending -
+        call `wait_for_submitted` first if that matters.
+
+        Args:
+            coro: The coroutine to run.
+
+        Returns:
+            The future of the scheduled coroutine, for callers that want to inspect its outcome later.
+
+        Raises:
+            RuntimeError: If the event loop has been closed.
+        """
+        if self._eventloop.is_closed():
+            raise RuntimeError(f'The coroutine {coro} cannot be executed because the event loop is closed.')
+
+        # Callers may go a long time between `wait_for_submitted` calls, so without pruning the list would hold
+        # every coroutine ever submitted, with its result.
+        if len(self._submitted) >= SUBMITTED_PRUNE_THRESHOLD:
+            self._submitted = [submitted for submitted in self._submitted if not submitted.done()]
+
+        future = asyncio.run_coroutine_threadsafe(coro, self._eventloop)
+        future.add_done_callback(self._log_failure)
+        self._submitted.append(future)
+
+        return future
+
+    def wait_for_submitted(self, timeout: timedelta | None = None) -> None:
+        """Block until the coroutines submitted via `submit_coro` have finished.
+
+        Use this before anything that would observe their effects, or before `close`, which cancels whatever is
+        still running. Coroutines that do not finish within the timeout stay tracked for the next call.
+
+        Args:
+            timeout: The maximum time to wait for the submitted coroutines. Pass `None` to use the
+                `default_timeout` passed to the constructor.
+        """
+        if timeout is None:
+            timeout = self._default_timeout
+
+        self._submitted = list(futures.wait(self._submitted, timeout=timeout.total_seconds()).not_done)
+
+        # Callers rely on the effects having landed, so a timeout has to be visible.
+        if self._submitted:
+            logger.warning(f'{len(self._submitted)} submitted coroutines did not finish within the timeout.')
+
     def close(self, timeout: timedelta | None = None) -> None:
         """Close the event loop and its thread gracefully.
 
@@ -110,6 +165,15 @@ class AsyncThread:
                 logger.warning('Event loop thread did not exit cleanly! Forcing shutdown...')
                 self._force_exit_event_loop()
 
+    @staticmethod
+    def _log_failure(future: futures.Future) -> None:
+        """Log the failure of a coroutine submitted via `submit_coro`."""
+        if future.cancelled():
+            return
+
+        if (exc := future.exception()) is not None:
+            logger.error('A coroutine submitted to the event loop failed.', exc_info=exc)
+
     def _start_event_loop(self) -> None:
         """Set up and run the asyncio event loop in the dedicated thread."""
         asyncio.set_event_loop(self._eventloop)

{apify-4.0.2b2 → apify-4.0.2b4}/src/apify/scrapy/requests.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from copy import deepcopy
 from logging import getLogger
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, get_args
 
 from scrapy import Request as ScrapyRequest
 from scrapy import Spider
@@ -11,11 +11,15 @@ from scrapy.utils.misc import load_object
 from scrapy.utils.request import request_from_dict
 
 from crawlee._request import UserData
-from crawlee._types import HttpHeaders
+from crawlee._types import HttpHeaders, HttpMethod
+from crawlee._utils.requests import compute_unique_key
 
 from ._serialization import decode_from_json, encode_to_json
 from apify import Request as ApifyRequest
 
+if TYPE_CHECKING:
+    from typing_extensions import TypeIs
+
 logger = getLogger(__name__)
 
 
@@ -47,6 +51,38 @@ def _ensure_known_request_class(request_dict: dict[str, Any]) -> None:
         )
 
 
+def _is_http_method(method: str) -> TypeIs[HttpMethod]:
+    """Tell whether `method` is one of the HTTP methods the RQ accepts; Scrapy itself takes any string."""
+    return method in get_args(HttpMethod)
+
+
+def _compute_fingerprint(scrapy_request: ScrapyRequest) -> str:
+    """Identify the request an RQ unique key was minted for.
+
+    `to_scrapy_request` stamps this beside the unique key so `to_apify_request` can tell a request that came out
+    of the RQ from one Scrapy derived from it. It covers the URL, the method and the body, so a redirect, a
+    method switch and a changed body all read as a different request rather than the parent.
+
+    Headers are left out even though the unique key covers them: `DefaultHeadersMiddleware` and
+    `UserAgentMiddleware` call `headers.setdefault()` on the request in place before the scheduler sees it
+    again, so an otherwise untouched request would no longer match its own stamp.
+    """
+    method = scrapy_request.method
+    if not _is_http_method(method):
+        raise ValueError(
+            f'Unsupported HTTP method {method!r} in {scrapy_request}; '
+            f'the request queue accepts {", ".join(get_args(HttpMethod))}.'
+        )
+
+    return compute_unique_key(
+        url=scrapy_request.url,
+        method=method,
+        payload=scrapy_request.body,
+        keep_url_fragment=False,
+        use_extended_unique_key=True,
+    )
+
+
 def to_apify_request(scrapy_request: ScrapyRequest, spider: Spider) -> ApifyRequest | None:
     """Convert a Scrapy request to an Apify request.
 
@@ -76,8 +112,14 @@ def to_apify_request(scrapy_request: ScrapyRequest, spider: Spider) -> ApifyRequ
     try:
         if scrapy_request.dont_filter:
             request_kwargs['always_enqueue'] = True
-        elif scrapy_request.meta.get('apify_request_unique_key'):
-            request_kwargs['unique_key'] = scrapy_request.meta['apify_request_unique_key']
+        elif unique_key := scrapy_request.meta.get('apify_request_unique_key'):
+            # Reuse the RQ unique key only while this is still the request it was minted for. Redirects
+            # (`Request.replace()`) and spiders forwarding `meta` to another URL inherit the stamp, and reusing
+            # it there deduplicates the derived request against its parent. A stamp without a fingerprint was
+            # set by hand, so it is taken at face value.
+            fingerprint = _compute_fingerprint(scrapy_request)
+            if scrapy_request.meta.get('apify_request_fingerprint', fingerprint) == fingerprint:
+                request_kwargs['unique_key'] = unique_key
 
         # Serialize the Scrapy request now, before `Request.from_url()` runs below. `from_url()` mutates the
         # `user_data` dict it receives in place (it injects a live `CrawleeRequestData` under `__crawlee`), and that
@@ -113,8 +155,9 @@ def to_apify_request(scrapy_request: ScrapyRequest, spider: Spider) -> ApifyRequ
         # the old behavior, which dropped such requests entirely.
         if isinstance(scrapy_request.headers, Headers):
             try:
-                headers = cast('dict[str, str]', dict(scrapy_request.headers.to_unicode_dict()))
-                request_kwargs['headers'] = HttpHeaders(headers)
+                # `to_unicode_dict()` yields str keys and values, but Scrapy types it as `UserDict[str | bytes, Any]`,
+                # so the mapping goes through the model's validator instead of a cast.
+                request_kwargs['headers'] = HttpHeaders.model_validate(scrapy_request.headers.to_unicode_dict())
             except UnicodeDecodeError:
                 logger.warning(
                     'Could not represent Scrapy request headers as Apify request headers (non-UTF-8 values); '
@@ -165,7 +208,7 @@ def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequ
     Returns:
         The converted Scrapy request.
     """
-    if not isinstance(cast('Any', apify_request), ApifyRequest):
+    if not isinstance(apify_request, ApifyRequest):
         raise TypeError('apify_request must be an apify.Request instance')
 
     # If the apify_request comes from the Scrapy
@@ -187,21 +230,14 @@ def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequ
         if not isinstance(scrapy_request, ScrapyRequest):
             raise TypeError('scrapy_request must be an instance of the ScrapyRequest class')
 
-        # Update the meta field with the meta field from the apify_request
-        meta = scrapy_request.meta or {}
-        meta.update({'apify_request_unique_key': apify_request.unique_key})
-        # scrapy_request.meta is a property, so we have to set it like this
-        scrapy_request._meta = meta  # noqa: SLF001
-
     # If the apify_request comes directly from the Scrapy, typically start URLs.
     else:
-        scrapy_request = ScrapyRequest(
-            url=apify_request.url,
-            method=apify_request.method,
-            meta={
-                'apify_request_unique_key': apify_request.unique_key,
-            },
-        )
+        scrapy_request = ScrapyRequest(url=apify_request.url, method=apify_request.method)
+
+    # Stamp the unique key with a fingerprint of the request it belongs to, so `to_apify_request` can tell
+    # this request apart from the ones Scrapy derives from it.
+    scrapy_request.meta['apify_request_unique_key'] = apify_request.unique_key
+    scrapy_request.meta['apify_request_fingerprint'] = _compute_fingerprint(scrapy_request)
 
     # Add optional 'headers' field
     if apify_request.headers:

apify-4.0.2b4/src/apify/scrapy/scheduler.py

@@ -0,0 +1,416 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import timedelta
+from logging import getLogger
+from typing import TYPE_CHECKING, Any
+
+from scrapy import Spider
+from scrapy import __version__ as scrapy_version
+from scrapy.core.scheduler import BaseScheduler
+from scrapy.utils.reactor import is_asyncio_reactor_installed
+
+from ._async_thread import AsyncThread
+from .requests import to_apify_request, to_scrapy_request
+from apify import Configuration
+from apify.storage_clients import ApifyStorageClient
+from apify.storages import RequestQueue
+
+if TYPE_CHECKING:
+    from collections.abc import Coroutine, Iterable
+    from concurrent.futures import Future
+
+    from scrapy.crawler import Crawler
+    from scrapy.http.request import Request
+    from twisted.internet.defer import Deferred
+
+    from apify import Request as ApifyRequest
+
+    InFlightRequest = tuple[ApifyRequest, Request]
+    """A request handed over to Scrapy, paired with the RQ request it came from."""
+
+logger = getLogger(__name__)
+
+
+async def _gather_failures(operations: Iterable[Coroutine[Any, Any, Any]]) -> list[BaseException | None]:
+    """Run RQ updates concurrently, reporting each one's failure, or `None`, in the order given.
+
+    The whole batch costs one round trip, and one failing update neither raises nor stops the others.
+    """
+    outcomes = await asyncio.gather(*operations, return_exceptions=True)
+    return [outcome if isinstance(outcome, BaseException) else None for outcome in outcomes]
+
+
+class ApifyScheduler(BaseScheduler):
+    """A Scrapy scheduler that uses the Apify `RequestQueue` to manage requests.
+
+    This scheduler requires the asyncio Twisted reactor to be installed.
+    """
+
+    def __init__(
+        self,
+        async_thread_timeout: timedelta = timedelta(seconds=60),
+        crawler: Crawler | None = None,
+    ) -> None:
+        if not is_asyncio_reactor_installed():
+            raise ValueError(
+                f'{ApifyScheduler.__qualname__} requires the asyncio Twisted reactor. '
+                'Make sure you have it configured in the TWISTED_REACTOR setting. See the asyncio '
+                'documentation of Scrapy for more information.',
+            )
+        self._rq: RequestQueue | None = None
+        self.spider: Spider | None = None
+        self._crawler = crawler
+
+        self._requests_in_flight: list[InFlightRequest] = []
+        """Requests handed over to Scrapy and not resolved in the RQ yet."""
+
+        self._pending_marks: list[tuple[list[InFlightRequest], Future]] = []
+        """Batches of mark-as-handled updates dispatched off the hot path, whose outcome is not known yet."""
+
+        # A thread with the asyncio event loop to run coroutines on.
+        self._async_thread = AsyncThread(default_timeout=async_thread_timeout)
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> ApifyScheduler:
+        """Create the scheduler, reading the async-thread timeout from the Scrapy settings.
+
+        The `APIFY_ASYNC_THREAD_TIMEOUT_SECS` setting (in seconds) caps how long each coroutine run on the
+        background event loop may take before timing out; it defaults to 60 seconds.
+        """
+        timeout_secs = crawler.settings.getint('APIFY_ASYNC_THREAD_TIMEOUT_SECS', 60)
+        return cls(async_thread_timeout=timedelta(seconds=timeout_secs), crawler=crawler)
+
+    def open(self, spider: Spider) -> Deferred[None] | None:
+        """Open the scheduler.
+
+        Args:
+            spider: The spider that the scheduler is associated with.
+        """
+        self.spider = spider
+
+        if self._crawler is None:
+            logger.warning(
+                f'{ApifyScheduler.__qualname__} was built without a crawler, so it cannot see what Scrapy is '
+                'still working on. Every request is marked as handled the moment it is handed over, and an '
+                'interrupted run loses whatever was in flight. Build it with `from_crawler` to avoid that.'
+            )
+        else:
+            self._verify_engine_internals()
+
+        async def open_rq() -> RequestQueue:
+            configuration = Configuration.get_global_configuration()
+            if configuration.is_at_home:
+                storage_client = ApifyStorageClient()
+                return await RequestQueue.open(
+                    configuration=configuration,
+                    storage_client=storage_client,
+                )
+            return await RequestQueue.open()
+
+        try:
+            self._rq = self._async_thread.run_coro(open_rq())
+        except Exception:
+            logger.exception('Failed to open the request queue.')
+            # Close the freshly started async thread so a failed open does not leak its event-loop thread.
+            # Guard the close so a secondary failure here cannot mask the original error.
+            try:
+                self._async_thread.close()
+            except Exception:
+                logger.exception('Failed to close the async thread after a failed scheduler open.')
+            raise
+
+        return None
+
+    def close(self, reason: str) -> None:
+        """Close the scheduler.
+
+        Resolve the requests Scrapy still holds, then shut down the event loop and its thread gracefully.
+
+        Args:
+            reason: The reason for closing the spider.
+        """
+        logger.debug(f'Closing {self.__class__.__name__} due to {reason}...')
+
+        rq = self._rq
+        if isinstance(rq, RequestQueue):
+            try:
+                self._resolve_finished_requests(wait=True)
+            except Exception:
+                logger.exception('Failed to resolve the requests still in flight in the request queue.')
+
+            # Scrapy drains its in-progress requests before closing the scheduler, so this only runs if a future
+            # Scrapy closes it earlier; then they go back to the RQ as pending, in a single round trip. One failed
+            # reclaim must not strand the rest either.
+            if self._requests_in_flight:
+                reclaims = _gather_failures(
+                    rq.reclaim_request(apify_request) for apify_request, _ in self._requests_in_flight
+                )
+                try:
+                    outcomes = self._async_thread.run_coro(reclaims)
+                except Exception:
+                    logger.exception('Failed to reclaim the requests still in flight in the request queue.')
+                else:
+                    for (apify_request, _), outcome in zip(self._requests_in_flight, outcomes, strict=True):
+                        if outcome is not None:
+                            logger.error(
+                                f'Failed to reclaim the request {apify_request} in the request queue.',
+                                exc_info=outcome,
+                            )
+
+            self._requests_in_flight.clear()
+
+        # Closing the event loop would silently cancel the updates fired off on the hot path.
+        self._async_thread.wait_for_submitted()
+
+        try:
+            self._async_thread.close()
+
+        except KeyboardInterrupt:
+            logger.warning('Shutdown interrupted by KeyboardInterrupt!')
+
+        except Exception:
+            logger.exception('Exception occurred while shutting down.')
+
+        finally:
+            logger.debug(f'{self.__class__.__name__} closed successfully.')
+
+    def has_pending_requests(self) -> bool:
+        """Check if the scheduler has any pending requests.
+
+        Resolves the requests Scrapy has finished with first, as their outcome is what decides the answer.
+
+        Returns:
+            True if the scheduler has any pending requests, False otherwise.
+        """
+        if not isinstance(self._rq, RequestQueue):
+            raise TypeError('self._rq must be an instance of the RequestQueue class')
+
+        # Scrapy asks this only once both its downloader and its scraper are idle, so everything still tracked
+        # as in flight is provably finished.
+        self._resolve_finished_requests(wait=True)
+
+        # The RQ answers from its own bookkeeping, which a pending update has not reached yet.
+        self._async_thread.wait_for_submitted()
+
+        # Log here before re-raising: this coroutine ran on a separate event-loop thread, and the failure is
+        # otherwise easy to lose as it crosses that thread boundary back into Scrapy's synchronous machinery.
+        try:
+            is_finished = self._async_thread.run_coro(self._rq.is_finished())
+        except Exception:
+            logger.exception('Failed to check whether the request queue is finished.')
+            raise
+
+        return not is_finished
+
+    def enqueue_request(self, request: Request) -> bool:
+        """Add a request to the scheduler.
+
+        This could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).
+
+        Args:
+            request: The request to add to the scheduler.
+
+        Returns:
+            True if the request was successfully enqueued, False otherwise.
+        """
+        if not isinstance(self.spider, Spider):
+            raise TypeError('self.spider must be an instance of the Spider class')
+
+        apify_request = to_apify_request(request, spider=self.spider)
+        if apify_request is None:
+            logger.warning(f'Request {request} could not be converted to Apify request; skipping it.')
+            return False
+
+        if not isinstance(self._rq, RequestQueue):
+            raise TypeError('self._rq must be an instance of the RequestQueue class')
+
+        # Log here before re-raising: this coroutine ran on a separate event-loop thread, and the failure is
+        # otherwise easy to lose as it crosses that thread boundary back into Scrapy's synchronous machinery.
+        try:
+            result = self._async_thread.run_coro(self._rq.add_request(apify_request))
+        except Exception:
+            logger.exception('Failed to enqueue the request to the request queue.')
+            raise
+
+        return not bool(result.was_already_present)
+
+    def next_request(self) -> Request | None:
+        """Fetch the next request from the scheduler.
+
+        Returns:
+            The next request, or None if there are no more requests.
+        """
+        if not isinstance(self._rq, RequestQueue):
+            raise TypeError('self._rq must be an instance of the RequestQueue class')
+
+        # The engine polls this method throughout the crawl, so resolving here keeps the RQ current without
+        # blocking on the round trips.
+        self._resolve_finished_requests(wait=False)
+
+        # Log here before re-raising: this coroutine ran on a separate event-loop thread, and the failure is
+        # otherwise easy to lose as it crosses that thread boundary back into Scrapy's synchronous machinery.
+        try:
+            apify_request = self._async_thread.run_coro(self._rq.fetch_next_request())
+        except Exception:
+            logger.exception('Failed to fetch the next request from the request queue.')
+            raise
+
+        if apify_request is None:
+            return None
+
+        if not isinstance(self.spider, Spider):
+            raise TypeError('self.spider must be an instance of the Spider class')
+
+        # A corrupt or legacy payload must not crash the run, and is marked as handled right away, otherwise
+        # the RQ would keep handing it back forever.
+        try:
+            scrapy_request = to_scrapy_request(apify_request, spider=self.spider)
+        except Exception as exc:
+            logger.warning(f'Failed to convert Apify request {apify_request} to a Scrapy request; skipping it: {exc}')
+            try:
+                self._async_thread.run_coro(self._rq.mark_request_as_handled(apify_request))
+            except Exception:
+                logger.exception('Failed to mark the request as handled in the request queue.')
+                raise
+            return None
+
+        # The entry stays unresolved until Scrapy is done with the request, so a run interrupted mid-flight
+        # leaves it pending instead of silently handled.
+        self._requests_in_flight.append((apify_request, scrapy_request))
+
+        return scrapy_request
+
+    def _verify_engine_internals(self) -> None:
+        """Fail at open time if Scrapy's engine no longer exposes what the in-flight tracking reads.
+
+        The hot path deliberately does not guard those undocumented internals: swallowing an `AttributeError`
+        there would quietly go back to marking every request as handled the moment it is handed over. The
+        scraper's slot does not exist yet when the scheduler opens, so this only proves `downloader.active` and
+        `scraper.slot` are there; a missing `slot.active` surfaces from `next_request` on the first request.
+
+        Raises:
+            RuntimeError: If the engine internals the tracking relies on cannot be read.
+        """
+        try:
+            self._requests_busy_in_scrapy()
+        except AttributeError as exc:
+            raise RuntimeError(
+                f'{ApifyScheduler.__qualname__} cannot tell which requests Scrapy is working on: this Scrapy '
+                f'version ({scrapy_version}) does not expose the engine internals it reads. Please report this '
+                'at https://github.com/apify/apify-sdk-python/issues.'
+            ) from exc
+
+    def _requests_busy_in_scrapy(self) -> set[Request]:
+        """Return the requests Scrapy is still working on.
+
+        A request joins the downloader's active set before the middleware chain runs and leaves the scraper's
+        only once the callback and the item pipeline are done, so absence from both means Scrapy is done with
+        it - downloaded, dropped by a middleware or errored out alike.
+
+        Without a crawler there is nothing to ask and every request reads as finished; `open` warns about that.
+        """
+        engine = self._crawler.engine if self._crawler is not None else None
+        if engine is None:
+            return set()
+
+        scraper_slot = engine.scraper.slot
+        return engine.downloader.active | (scraper_slot.active if scraper_slot is not None else set())
+
+    def _resolve_finished_requests(self, *, wait: bool) -> None:
+        """Mark every request Scrapy has finished processing as handled in the RQ.
+
+        A whole pass reaches the RQ in a single round trip. A request whose update cannot be dispatched, or
+        turns out to have failed, stays tracked for the next call to retry.
+
+        Args:
+            wait: Whether to block until the RQ has been updated. Pass False on the crawl's hot path, where
+                nothing depends on the result and blocking would stall the Twisted reactor.
+        """
+        rq = self._rq
+        if not isinstance(rq, RequestQueue) or not (self._requests_in_flight or self._pending_marks):
+            return
+
+        # Updates dispatched by an earlier pass that did not land are marked again by this one.
+        finished = self._collect_failed_marks(wait=wait)
+        unresolved: list[InFlightRequest] = []
+
+        if self._requests_in_flight:
+            busy = self._requests_busy_in_scrapy()
+            for apify_request, scrapy_request in self._requests_in_flight:
+                if scrapy_request in busy:
+                    unresolved.append((apify_request, scrapy_request))
+                else:
+                    finished.append((apify_request, scrapy_request))
+
+        if finished:
+            marks = _gather_failures(rq.mark_request_as_handled(apify_request) for apify_request, _ in finished)
+            try:
+                if wait:
+                    unresolved.extend(self._failed_marks(finished, self._async_thread.run_coro(marks)))
+                else:
+                    self._pending_marks.append((finished, self._async_thread.submit_coro(marks)))
+            except Exception:
+                logger.exception(f'Failed to mark {len(finished)} request(s) as handled in the request queue.')
+                unresolved.extend(finished)
+
+        self._requests_in_flight = unresolved
+
+    def _collect_failed_marks(self, *, wait: bool) -> list[InFlightRequest]:
+        """Return the requests whose already dispatched mark-as-handled did not land, so it can be retried.
+
+        `submit_coro` reports nothing back to the reactor thread, so an update failing after dispatch would
+        otherwise drop its request from the tracking and leave it in progress in the RQ forever, which would
+        keep the RQ from ever reporting itself finished.
+
+        Args:
+            wait: Whether to block until every dispatched update has finished. Updates still running are kept
+                for the next call.
+        """
+        if not self._pending_marks:
+            return []
+
+        if wait:
+            self._async_thread.wait_for_submitted()
+
+        failed: list[InFlightRequest] = []
+        pending: list[tuple[list[InFlightRequest], Future]] = []
+
+        for requests, future in self._pending_marks:
+            if not future.done():
+                pending.append((requests, future))
+            elif future.cancelled():
+                logger.error(f'Marking {len(requests)} request(s) as handled was cancelled before it finished.')
+                failed.extend(requests)
+            else:
+                # `_gather_failures` reports a failed update in its result rather than raising, so an exception
+                # here means the dispatch itself did not survive the event loop being torn down under it.
+                try:
+                    outcomes = future.result()
+                except Exception:
+                    logger.exception(f'Failed to mark {len(requests)} request(s) as handled in the request queue.')
+                    failed.extend(requests)
+                else:
+                    failed.extend(self._failed_marks(requests, outcomes))
+
+        self._pending_marks = pending
+
+        return failed
+
+    @staticmethod
+    def _failed_marks(
+        requests: list[InFlightRequest],
+        outcomes: list[BaseException | None],
+    ) -> list[InFlightRequest]:
+        """Pair a batch of updates back with their requests, returning and logging the ones that failed."""
+        failed = []
+
+        for (apify_request, scrapy_request), outcome in zip(requests, outcomes, strict=True):
+            if outcome is not None:
+                logger.error(
+                    f'Failed to mark the request {apify_request} as handled in the request queue.',
+                    exc_info=outcome,
+                )
+                failed.append((apify_request, scrapy_request))
+
+        return failed

{apify-4.0.2b2 → apify-4.0.2b4}/src/apify/storage_clients/_apify/_request_queue_single_client.py

@@ -82,7 +82,10 @@ class ApifyRequestQueueSingleClient:
         """LRU cache storing unhandled request objects, keyed by request ID."""
 
         self._head_requests: deque[str] = deque()
-        """Ordered queue of request IDs representing the local estimate of the queue head."""
+        """Ordered queue of request IDs representing the local estimate of the queue head.
+
+        Served from the right end, so `append` places a request at the top of the head and `appendleft` at the bottom.
+        """
 
         self._requests_already_handled: set[str] = set()
         """Set of request IDs known to be already processed on the platform.
@@ -303,9 +306,12 @@ class ApifyRequestQueueSingleClient:
             # No longer handled
             self._requests_already_handled.discard(request_id)
 
+            # `is_empty` reads only the head estimation, so the reclaimed request has to go back into it until the
+            # platform head listing catches up. Placement mirrors `add_batch_of_requests`.
             if forefront:
-                # Append to top of the local head estimation
                 self._head_requests.append(request_id)
+            else:
+                self._head_requests.appendleft(request_id)
 
             processed_request = await self._update_request(request, forefront=forefront)
             processed_request.id = request_id

apify-4.0.2b2/src/apify/scrapy/scheduler.py

@@ -1,203 +0,0 @@
-from __future__ import annotations
-
-from datetime import timedelta
-from logging import getLogger
-from typing import TYPE_CHECKING
-
-from scrapy import Spider
-from scrapy.core.scheduler import BaseScheduler
-from scrapy.utils.reactor import is_asyncio_reactor_installed
-
-from ._async_thread import AsyncThread
-from .requests import to_apify_request, to_scrapy_request
-from apify import Configuration
-from apify.storage_clients import ApifyStorageClient
-from apify.storages import RequestQueue
-
-if TYPE_CHECKING:
-    from scrapy.crawler import Crawler
-    from scrapy.http.request import Request
-    from twisted.internet.defer import Deferred
-
-logger = getLogger(__name__)
-
-
-class ApifyScheduler(BaseScheduler):
-    """A Scrapy scheduler that uses the Apify `RequestQueue` to manage requests.
-
-    This scheduler requires the asyncio Twisted reactor to be installed.
-    """
-
-    def __init__(self, async_thread_timeout: timedelta = timedelta(seconds=60)) -> None:
-        if not is_asyncio_reactor_installed():
-            raise ValueError(
-                f'{ApifyScheduler.__qualname__} requires the asyncio Twisted reactor. '
-                'Make sure you have it configured in the TWISTED_REACTOR setting. See the asyncio '
-                'documentation of Scrapy for more information.',
-            )
-        self._rq: RequestQueue | None = None
-        self.spider: Spider | None = None
-
-        # A thread with the asyncio event loop to run coroutines on.
-        self._async_thread = AsyncThread(default_timeout=async_thread_timeout)
-
-    @classmethod
-    def from_crawler(cls, crawler: Crawler) -> ApifyScheduler:
-        """Create the scheduler, reading the async-thread timeout from the Scrapy settings.
-
-        The `APIFY_ASYNC_THREAD_TIMEOUT_SECS` setting (in seconds) caps how long each coroutine run on the
-        background event loop may take before timing out; it defaults to 60 seconds.
-        """
-        timeout_secs = crawler.settings.getint('APIFY_ASYNC_THREAD_TIMEOUT_SECS', 60)
-        return cls(async_thread_timeout=timedelta(seconds=timeout_secs))
-
-    def open(self, spider: Spider) -> Deferred[None] | None:
-        """Open the scheduler.
-
-        Args:
-            spider: The spider that the scheduler is associated with.
-        """
-        self.spider = spider
-
-        async def open_rq() -> RequestQueue:
-            configuration = Configuration.get_global_configuration()
-            if configuration.is_at_home:
-                storage_client = ApifyStorageClient()
-                return await RequestQueue.open(
-                    configuration=configuration,
-                    storage_client=storage_client,
-                )
-            return await RequestQueue.open()
-
-        try:
-            self._rq = self._async_thread.run_coro(open_rq())
-        except Exception:
-            logger.exception('Failed to open the request queue.')
-            # Close the freshly started async thread so a failed open does not leak its event-loop thread.
-            # Guard the close so a secondary failure here cannot mask the original error.
-            try:
-                self._async_thread.close()
-            except Exception:
-                logger.exception('Failed to close the async thread after a failed scheduler open.')
-            raise
-
-        return None
-
-    def close(self, reason: str) -> None:
-        """Close the scheduler.
-
-        Shut down the event loop and its thread gracefully.
-
-        Args:
-            reason: The reason for closing the spider.
-        """
-        logger.debug(f'Closing {self.__class__.__name__} due to {reason}...')
-        try:
-            self._async_thread.close()
-
-        except KeyboardInterrupt:
-            logger.warning('Shutdown interrupted by KeyboardInterrupt!')
-
-        except Exception:
-            logger.exception('Exception occurred while shutting down.')
-
-        finally:
-            logger.debug(f'{self.__class__.__name__} closed successfully.')
-
-    def has_pending_requests(self) -> bool:
-        """Check if the scheduler has any pending requests.
-
-        Returns:
-            True if the scheduler has any pending requests, False otherwise.
-        """
-        if not isinstance(self._rq, RequestQueue):
-            raise TypeError('self._rq must be an instance of the RequestQueue class')
-
-        # Log here before re-raising: this coroutine ran on a separate event-loop thread, and the failure is
-        # otherwise easy to lose as it crosses that thread boundary back into Scrapy's synchronous machinery.
-        try:
-            is_finished = self._async_thread.run_coro(self._rq.is_finished())
-        except Exception:
-            logger.exception('Failed to check whether the request queue is finished.')
-            raise
-
-        return not is_finished
-
-    def enqueue_request(self, request: Request) -> bool:
-        """Add a request to the scheduler.
-
-        This could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).
-
-        Args:
-            request: The request to add to the scheduler.
-
-        Returns:
-            True if the request was successfully enqueued, False otherwise.
-        """
-        if not isinstance(self.spider, Spider):
-            raise TypeError('self.spider must be an instance of the Spider class')
-
-        apify_request = to_apify_request(request, spider=self.spider)
-        if apify_request is None:
-            logger.warning(f'Request {request} could not be converted to Apify request; skipping it.')
-            return False
-
-        if not isinstance(self._rq, RequestQueue):
-            raise TypeError('self._rq must be an instance of the RequestQueue class')
-
-        # Log here before re-raising: this coroutine ran on a separate event-loop thread, and the failure is
-        # otherwise easy to lose as it crosses that thread boundary back into Scrapy's synchronous machinery.
-        try:
-            result = self._async_thread.run_coro(self._rq.add_request(apify_request))
-        except Exception:
-            logger.exception('Failed to enqueue the request to the request queue.')
-            raise
-
-        return not bool(result.was_already_present)
-
-    def next_request(self) -> Request | None:
-        """Fetch the next request from the scheduler.
-
-        Returns:
-            The next request, or None if there are no more requests.
-        """
-        if not isinstance(self._rq, RequestQueue):
-            raise TypeError('self._rq must be an instance of the RequestQueue class')
-
-        # Log here before re-raising: this coroutine ran on a separate event-loop thread, and the failure is
-        # otherwise easy to lose as it crosses that thread boundary back into Scrapy's synchronous machinery.
-        try:
-            apify_request = self._async_thread.run_coro(self._rq.fetch_next_request())
-        except Exception:
-            logger.exception('Failed to fetch the next request from the request queue.')
-            raise
-
-        if apify_request is None:
-            return None
-
-        if not isinstance(self.spider, Spider):
-            raise TypeError('self.spider must be an instance of the Spider class')
-
-        # Reconstruct the Scrapy request before consuming the queue entry. A malformed entry must not crash
-        # the whole run, so on failure it is logged and skipped (None) rather than propagating.
-        try:
-            scrapy_request = to_scrapy_request(apify_request, spider=self.spider)
-        except Exception as exc:
-            logger.warning(f'Failed to convert Apify request {apify_request} to a Scrapy request; skipping it: {exc}')
-            scrapy_request = None
-
-        # Mark the request as handled. This runs even when reconstruction failed above: an unrecoverable entry
-        # (a corrupt or legacy payload) must still be consumed, otherwise the queue would keep handing it back
-        # forever. Retrying genuine failures is the RetryMiddleware's job.
-        # Log here before re-raising: this coroutine ran on a separate event-loop thread, and the failure is
-        # otherwise easy to lose as it crosses that thread boundary back into Scrapy's synchronous machinery.
-        try:
-            self._async_thread.run_coro(self._rq.mark_request_as_handled(apify_request))
-        except Exception:
-            logger.exception('Failed to mark the request as handled in the request queue.')
-            raise
-
-        if scrapy_request is None:
-            return None
-
-        return scrapy_request