apify 4.0.1b4__tar.gz → 4.0.2__tar.gz

{apify-4.0.1b4 → apify-4.0.2}/CHANGELOG.md

@@ -2,13 +2,24 @@
 
 All notable changes to this project will be documented in this file.
 
-<!-- git-cliff-unreleased-start -->
-## 4.0.1 - **not yet released**
+## [4.0.2](https://github.com/apify/apify-sdk-python/releases/tag/v4.0.2) (2026-09-03)
+
+### 🐛 Bug Fixes
+
+- Prolong and track shared request queue locks to prevent duplicate processing ([#1062](https://github.com/apify/apify-sdk-python/pull/1062)) ([ccaad7b](https://github.com/apify/apify-sdk-python/commit/ccaad7b71cd6cd1ddc2c71e5e9d9780001f2688a)) by [@vdusek](https://github.com/vdusek)
+- Verify shared request queue is_finished against per-request reads ([#1087](https://github.com/apify/apify-sdk-python/pull/1087)) ([22613aa](https://github.com/apify/apify-sdk-python/commit/22613aac0ce4e6ef852d5d3daf8c1a25cb3731e3)) by [@vdusek](https://github.com/vdusek), closes [#808](https://github.com/apify/apify-sdk-python/issues/808)
+- Keep non-forefront reclaimed requests pending in the single request queue client ([#1060](https://github.com/apify/apify-sdk-python/pull/1060)) ([b61890c](https://github.com/apify/apify-sdk-python/commit/b61890cf286c3a83faedac69315acff74b5fba5b)) by [@vdusek](https://github.com/vdusek)
+- **scrapy:** Stop silently dropping in-flight requests and redirects ([#1097](https://github.com/apify/apify-sdk-python/pull/1097)) ([5a0a752](https://github.com/apify/apify-sdk-python/commit/5a0a752523b95c3143e5058fe07785e69817f4a2)) by [@vdusek](https://github.com/vdusek)
+- **scrapy:** Stop the crawl gracefully when the Actor run is migrated or aborted ([#1104](https://github.com/apify/apify-sdk-python/pull/1104)) ([d560af9](https://github.com/apify/apify-sdk-python/commit/d560af953f384cefd5a9c30e55a26dc3a9a49498)) by [@vdusek](https://github.com/vdusek), closes [#1103](https://github.com/apify/apify-sdk-python/issues/1103)
+
+
+## [4.0.1](https://github.com/apify/apify-sdk-python/releases/tag/v4.0.1) (2026-08-07)
 
 ### 🐛 Bug Fixes
 
 - Close the platform events websocket iterator on shutdown ([#1077](https://github.com/apify/apify-sdk-python/pull/1077)) ([cad8d7e](https://github.com/apify/apify-sdk-python/commit/cad8d7e9e95e91270348c6b9b2d57b2a7d98e733)) by [@vdusek](https://github.com/vdusek)
 - Do not abort Actor exit when the terminal status message fails ([#1078](https://github.com/apify/apify-sdk-python/pull/1078)) ([439ee7b](https://github.com/apify/apify-sdk-python/commit/439ee7b68fe71bc0965f3ed3f8b08d7666b6f657)) by [@vdusek](https://github.com/vdusek)
+- **events:** Open the platform websocket only once per context ([#1082](https://github.com/apify/apify-sdk-python/pull/1082)) ([c471dde](https://github.com/apify/apify-sdk-python/commit/c471dde0cd4d07cbf5ff3eb91347701da4d0fae5)) by [@vdusek](https://github.com/vdusek)
 
 ### ⚡ Performance
 
@@ -16,7 +27,6 @@ All notable changes to this project will be documented in this file.
 - Allow concurrent dataset pushes by narrowing the push-path locks ([#1079](https://github.com/apify/apify-sdk-python/pull/1079)) ([7d23ecd](https://github.com/apify/apify-sdk-python/commit/7d23ecdc656f36cc087a23e419ae77fde1adbad8)) by [@vdusek](https://github.com/vdusek)
 
 
-<!-- git-cliff-unreleased-end -->
 ## [4.0.0](https://github.com/apify/apify-sdk-python/releases/tag/v4.0.0) (2026-07-20)
 
 ### 🚀 Features
@@ -647,4 +657,4 @@ All notable changes to this project will be documented in this file.
 - Key error for storage name ([#28](https://github.com/apify/apify-sdk-python/pull/28)) ([83b30a9](https://github.com/apify/apify-sdk-python/commit/83b30a90df4d3b173302f1c6006b346091fced60)) by [@drobnikj](https://github.com/drobnikj)
 
 
-<!-- generated by git-cliff -->
+<!-- generated by git-cliff -->

{apify-4.0.1b4 → apify-4.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 4.0.1b4
+Version: 4.0.2
 Summary: Apify SDK for Python
 Keywords: apify,sdk,automation,chrome,crawlee,crawler,headless,scraper,scraping
 Author: Apify Technologies s.r.o.

{apify-4.0.1b4 → apify-4.0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "apify"
-version = "4.0.1b4"
+version = "4.0.2"
 description = "Apify SDK for Python"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
@@ -189,7 +189,7 @@ quote-style = "single"
 indent-style = "space"
 
 [tool.pytest.ini_options]
-addopts = "-r a --verbose"
+addopts = "-r a --verbose --dist worksteal"
 asyncio_default_fixture_loop_scope = "function"
 asyncio_mode = "auto"
 timeout = 1800

{apify-4.0.1b4 → apify-4.0.2}/pyproject.toml.orig

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "apify"
-version = "4.0.1b4"
+version = "4.0.2"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = "Apache-2.0"
@@ -198,7 +198,7 @@ known-first-party = ["apify_client", "crawlee"]
 max-branches = 18
 
 [tool.pytest.ini_options]
-addopts = "-r a --verbose"
+addopts = "-r a --verbose --dist worksteal"
 asyncio_default_fixture_loop_scope = "function"
 asyncio_mode = "auto"
 timeout = 1800

{apify-4.0.1b4 → apify-4.0.2}/src/apify/_actor.py

@@ -1249,12 +1249,10 @@ class _ActorType:
             # the reboot. Typically, crawlers are listening for the MIGRATING event to stop processing new requests.
             # We can't just emit the events and wait for all listeners to finish,
             # because this method might be called from an event listener itself, and we would deadlock.
-            persist_state_listeners = flatten(
-                (self.event_manager._listeners_to_wrappers[Event.PERSIST_STATE] or {}).values()  # noqa: SLF001
-            )
-            migrating_listeners = flatten(
-                (self.event_manager._listeners_to_wrappers[Event.MIGRATING] or {}).values()  # noqa: SLF001
-            )
+            # Read the mapping with `get` - subscripting it would insert entries for events nobody listens to.
+            listeners_to_wrappers = self.event_manager._listeners_to_wrappers  # noqa: SLF001
+            persist_state_listeners = flatten(listeners_to_wrappers.get(Event.PERSIST_STATE, {}).values())
+            migrating_listeners = flatten(listeners_to_wrappers.get(Event.MIGRATING, {}).values())
 
             async def safe_dispatch(listener: Any, data: Any) -> None:
                 try:

{apify-4.0.1b4 → apify-4.0.2}/src/apify/events/_apify_event_manager.py

@@ -86,29 +86,47 @@ class ApifyEventManager(EventManager):
         self._platform_events_websocket: websockets.asyncio.client.ClientConnection | None = None
         """WebSocket connection to the platform events."""
 
-        self._process_platform_messages_task: asyncio.Task | None = None
+        self._process_platform_messages_task: asyncio.Task[None] | None = None
         """Task for processing messages from the platform websocket."""
 
-        self._connected_to_platform_websocket: asyncio.Future[bool] | None = None
-        """Future that resolves when the connection to the platform websocket is established."""
+        self._connected_to_platform_websocket: asyncio.Future[None] | None = None
+        """Resolves once the platform websocket is connected, or fails with the error that prevented the first
+        connection, so that `__aenter__` can report it.
+        """
 
     @override
     async def __aenter__(self) -> Self:
+        """Initialize the event manager upon entering the async context.
+
+        On the outermost entry, it connects to the platform events websocket and starts consuming its messages.
+        """
         await super().__aenter__()
-        self._connected_to_platform_websocket = asyncio.Future()
 
-        # Run tasks but don't await them
-        if self._configuration.actor_events_ws_url:
-            self._process_platform_messages_task = asyncio.create_task(
-                self._process_platform_messages(self._configuration.actor_events_ws_url)
-            )
-            is_connected = await self._connected_to_platform_websocket
-            if not is_connected:
-                # Exit the already-entered parent so the recurring persist state task does not leak.
-                await self.__aexit__(None, None, None)
-                raise RuntimeError('Error connecting to platform events websocket!')
-        else:
+        # Only the outermost context owns the websocket, so a nested one must not open a second connection.
+        if self._active_ref_count > 1:
+            return self
+
+        if not self._configuration.actor_events_ws_url:
             logger.debug('APIFY_ACTOR_EVENTS_WS_URL env var not set, no events from Apify platform will be emitted.')
+            return self
+
+        # The future has to exist before the task that resolves it starts running.
+        self._connected_to_platform_websocket = asyncio.Future()
+        self._process_platform_messages_task = asyncio.create_task(
+            self._process_platform_messages(self._configuration.actor_events_ws_url)
+        )
+
+        try:
+            await self._connected_to_platform_websocket
+        except Exception as exc:
+            # Exit the already-entered parent so the recurring persist state task does not leak.
+            await self.__aexit__(None, None, None)
+            raise RuntimeError('Error connecting to platform events websocket!') from exc
+        except BaseException:
+            # Cancellation has to clean up as well. A stale task left behind would make the next entry look nested,
+            # returning a manager that silently receives no platform events at all.
+            await self.__aexit__(None, None, None)
+            raise
 
         return self
 
@@ -119,17 +137,35 @@ class ApifyEventManager(EventManager):
         exc_value: BaseException | None,
         exc_traceback: TracebackType | None,
     ) -> None:
-        # Cancel the task before closing the websocket so that the closed connection is not treated as a drop
-        # and followed by a reconnect attempt.
-        if self._process_platform_messages_task and not self._process_platform_messages_task.done():
-            self._process_platform_messages_task.cancel()
-            with contextlib.suppress(asyncio.CancelledError):
-                await self._process_platform_messages_task
+        """Close the event manager upon exiting the async context.
 
-        if self._platform_events_websocket:
-            await self._platform_events_websocket.close()
-
-        await super().__aexit__(exc_type, exc_value, exc_traceback)
+        On the outermost exit, it stops consuming the platform messages and closes the websocket connection.
+        """
+        try:
+            if self._active_ref_count == 1:
+                await self._teardown_platform_websocket()
+        finally:
+            # The parent context has to be left even if the shutdown above fails. Staying active would mean never
+            # emitting `PersistState` again, as re-entering the context would be a no-op.
+            await super().__aexit__(exc_type, exc_value, exc_traceback)
+
+    async def _teardown_platform_websocket(self) -> None:
+        """Stop consuming the platform messages and close the websocket connection to the platform events."""
+        try:
+            # Cancel the task before closing the websocket so that the closed connection is not treated as a drop
+            # and followed by a reconnect attempt.
+            if self._process_platform_messages_task and not self._process_platform_messages_task.done():
+                self._process_platform_messages_task.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await self._process_platform_messages_task
+
+            if self._platform_events_websocket:
+                await self._platform_events_websocket.close()
+        finally:
+            # Leave no closed connection or resolved future behind, so that the context can be entered again.
+            self._process_platform_messages_task = None
+            self._platform_events_websocket = None
+            self._connected_to_platform_websocket = None
 
     def _process_connection_exception(self, exc: Exception) -> Exception | None:
         """Decide whether a failed connection attempt to the platform websocket should be retried.
@@ -159,7 +195,7 @@ class ApifyEventManager(EventManager):
                 async for websocket in connections:
                     self._platform_events_websocket = websocket
                     if self._connected_to_platform_websocket and not self._connected_to_platform_websocket.done():
-                        self._connected_to_platform_websocket.set_result(True)
+                        self._connected_to_platform_websocket.set_result(None)
                     else:
                         logger.info('Reconnected to the platform events websocket.')
 
@@ -178,10 +214,12 @@ class ApifyEventManager(EventManager):
                         backoff_delays = websockets.client.backoff()
                     else:
                         await asyncio.sleep(next(backoff_delays))
-        except Exception:
+        except Exception as exc:
             logger.exception('Error in websocket connection')
+
             if self._connected_to_platform_websocket is not None and not self._connected_to_platform_websocket.done():
-                self._connected_to_platform_websocket.set_result(False)
+                # `__aenter__` is still waiting for the first connection, so let it fail with this as the cause.
+                self._connected_to_platform_websocket.set_exception(exc)
 
     async def _consume_messages(self, websocket: websockets.asyncio.client.ClientConnection) -> bool:
         """Handle platform messages until the connection closes; return whether it was lost vs. closed cleanly."""

{apify-4.0.1b4 → apify-4.0.2}/src/apify/scrapy/_async_thread.py

@@ -12,6 +12,9 @@ if TYPE_CHECKING:
 
 logger = getLogger(__name__)
 
+SUBMITTED_PRUNE_THRESHOLD = 128
+"""How many `submit_coro` futures may pile up before the finished ones are dropped from the tracking list."""
+
 
 class AsyncThread:
     """Run an asyncio event loop in a dedicated background thread.
@@ -26,6 +29,9 @@ class AsyncThread:
         self._default_timeout = default_timeout
         self._eventloop = asyncio.new_event_loop()
 
+        self._submitted: list[futures.Future] = []
+        """Futures of the coroutines submitted via `submit_coro` that may still be running."""
+
         # Start the event loop in a dedicated daemon thread.
         self._thread = threading.Thread(
             target=self._start_event_loop,
@@ -74,6 +80,55 @@ class AsyncThread:
             future.cancel()
             raise
 
+    def submit_coro(self, coro: Coroutine) -> futures.Future:
+        """Schedule a coroutine on the event loop without waiting for its result.
+
+        Use this for work nothing depends on, so the calling thread is not blocked by the round trip. Failures
+        are logged, as there is no caller to propagate them to, and `close` cancels whatever is still pending -
+        call `wait_for_submitted` first if that matters.
+
+        Args:
+            coro: The coroutine to run.
+
+        Returns:
+            The future of the scheduled coroutine, for callers that want to inspect its outcome later.
+
+        Raises:
+            RuntimeError: If the event loop has been closed.
+        """
+        if self._eventloop.is_closed():
+            raise RuntimeError(f'The coroutine {coro} cannot be executed because the event loop is closed.')
+
+        # Callers may go a long time between `wait_for_submitted` calls, so without pruning the list would hold
+        # every coroutine ever submitted, with its result.
+        if len(self._submitted) >= SUBMITTED_PRUNE_THRESHOLD:
+            self._submitted = [submitted for submitted in self._submitted if not submitted.done()]
+
+        future = asyncio.run_coroutine_threadsafe(coro, self._eventloop)
+        future.add_done_callback(self._log_failure)
+        self._submitted.append(future)
+
+        return future
+
+    def wait_for_submitted(self, timeout: timedelta | None = None) -> None:
+        """Block until the coroutines submitted via `submit_coro` have finished.
+
+        Use this before anything that would observe their effects, or before `close`, which cancels whatever is
+        still running. Coroutines that do not finish within the timeout stay tracked for the next call.
+
+        Args:
+            timeout: The maximum time to wait for the submitted coroutines. Pass `None` to use the
+                `default_timeout` passed to the constructor.
+        """
+        if timeout is None:
+            timeout = self._default_timeout
+
+        self._submitted = list(futures.wait(self._submitted, timeout=timeout.total_seconds()).not_done)
+
+        # Callers rely on the effects having landed, so a timeout has to be visible.
+        if self._submitted:
+            logger.warning(f'{len(self._submitted)} submitted coroutines did not finish within the timeout.')
+
     def close(self, timeout: timedelta | None = None) -> None:
         """Close the event loop and its thread gracefully.
 
@@ -110,6 +165,15 @@ class AsyncThread:
                 logger.warning('Event loop thread did not exit cleanly! Forcing shutdown...')
                 self._force_exit_event_loop()
 
+    @staticmethod
+    def _log_failure(future: futures.Future) -> None:
+        """Log the failure of a coroutine submitted via `submit_coro`."""
+        if future.cancelled():
+            return
+
+        if (exc := future.exception()) is not None:
+            logger.error('A coroutine submitted to the event loop failed.', exc_info=exc)
+
     def _start_event_loop(self) -> None:
         """Set up and run the asyncio event loop in the dedicated thread."""
         asyncio.set_event_loop(self._eventloop)

apify-4.0.2/src/apify/scrapy/_warnings.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import logging
+
+from crawlee._utils.log import LoggerOnce
+
+logger_once = LoggerOnce(logging.getLogger(__name__))
+"""Process-wide deduplication: the scheduler and the graceful-stop extension share the one message below."""
+
+
+def warn_about_uninitialized_actor() -> None:
+    """Warn, once per process, that the integration cannot react to the Actor run being migrated or aborted.
+
+    The scheduler and the graceful-stop extension both need an initialized Actor to listen for its events, and both
+    run without one in a plain `scrapy crawl`. One message covers them both.
+    """
+    logger_once.log(
+        'The Actor is not initialized, so the Scrapy integration cannot react to a migration or an abort of the '
+        'Actor run; the requests Scrapy is working on when the run is interrupted stay pending in the request queue.',
+        key='uninitialized-actor',
+        level=logging.WARNING,
+    )

apify-4.0.2/src/apify/scrapy/extensions/__init__.py

@@ -0,0 +1,4 @@
+from apify.scrapy.extensions._graceful_stop import ApifyGracefulStopExtension
+from apify.scrapy.extensions._httpcache import ApifyCacheStorage
+
+__all__ = ['ApifyCacheStorage', 'ApifyGracefulStopExtension']

apify-4.0.2/src/apify/scrapy/extensions/_graceful_stop.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from contextlib import suppress
+from logging import getLogger
+from typing import TYPE_CHECKING
+
+from scrapy import signals
+
+from apify import Actor, Event
+from apify.scrapy._warnings import warn_about_uninitialized_actor
+
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
+
+logger = getLogger(__name__)
+
+
+class ApifyGracefulStopExtension:
+    """A Scrapy extension that stops the crawl gracefully when the Actor run is being aborted.
+
+    A graceful abort gives the run a moment before it is killed. The extension uses it to stop the engine: no new
+    requests are started, the ones in flight finish along with their callbacks and item pipelines, and the
+    scheduler marks them as handled in the request queue, so resurrecting the run does not download them again.
+    A migration is handled by `ApifyScheduler` instead: the platform restarts a migrating run only if its process
+    does not exit on its own, so the scheduler keeps the crawl running with nothing to do rather than stopping it.
+    """
+
+    def __init__(self, crawler: Crawler) -> None:
+        self._crawler = crawler
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> ApifyGracefulStopExtension:
+        """Create the extension and hook it up to the spider's lifecycle."""
+        extension = cls(crawler)
+        crawler.signals.connect(extension.spider_opened, signal=signals.spider_opened)
+        crawler.signals.connect(extension.spider_closed, signal=signals.spider_closed)
+        return extension
+
+    def spider_opened(self) -> None:
+        """Start listening for the abort of the Actor run."""
+        try:
+            Actor.on(Event.ABORTING, self._on_aborting)
+        except RuntimeError:
+            warn_about_uninitialized_actor()
+
+    def spider_closed(self) -> None:
+        """Stop listening for the abort of the Actor run."""
+        # Without an initialized Actor (never initialized, or exited already) there is nothing to unregister from.
+        with suppress(RuntimeError):
+            Actor.off(Event.ABORTING, self._on_aborting)
+
+    async def _on_aborting(self) -> None:
+        """Stop the crawler; it waits for the requests in flight and closes the scheduler, which marks them."""
+        logger.info('The Actor run is being aborted: stopping the crawl gracefully.')
+        await self._crawler.stop_async()

{apify-4.0.1b4 → apify-4.0.2}/src/apify/scrapy/requests.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from copy import deepcopy
 from logging import getLogger
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, get_args
 
 from scrapy import Request as ScrapyRequest
 from scrapy import Spider
@@ -11,11 +11,15 @@ from scrapy.utils.misc import load_object
 from scrapy.utils.request import request_from_dict
 
 from crawlee._request import UserData
-from crawlee._types import HttpHeaders
+from crawlee._types import HttpHeaders, HttpMethod
+from crawlee._utils.requests import compute_unique_key
 
 from ._serialization import decode_from_json, encode_to_json
 from apify import Request as ApifyRequest
 
+if TYPE_CHECKING:
+    from typing_extensions import TypeIs
+
 logger = getLogger(__name__)
 
 
@@ -47,6 +51,38 @@ def _ensure_known_request_class(request_dict: dict[str, Any]) -> None:
         )
 
 
+def _is_http_method(method: str) -> TypeIs[HttpMethod]:
+    """Tell whether `method` is one of the HTTP methods the RQ accepts; Scrapy itself takes any string."""
+    return method in get_args(HttpMethod)
+
+
+def _compute_fingerprint(scrapy_request: ScrapyRequest) -> str:
+    """Identify the request an RQ unique key was minted for.
+
+    `to_scrapy_request` stamps this beside the unique key so `to_apify_request` can tell a request that came out
+    of the RQ from one Scrapy derived from it. It covers the URL, the method and the body, so a redirect, a
+    method switch and a changed body all read as a different request rather than the parent.
+
+    Headers are left out even though the unique key covers them: `DefaultHeadersMiddleware` and
+    `UserAgentMiddleware` call `headers.setdefault()` on the request in place before the scheduler sees it
+    again, so an otherwise untouched request would no longer match its own stamp.
+    """
+    method = scrapy_request.method
+    if not _is_http_method(method):
+        raise ValueError(
+            f'Unsupported HTTP method {method!r} in {scrapy_request}; '
+            f'the request queue accepts {", ".join(get_args(HttpMethod))}.'
+        )
+
+    return compute_unique_key(
+        url=scrapy_request.url,
+        method=method,
+        payload=scrapy_request.body,
+        keep_url_fragment=False,
+        use_extended_unique_key=True,
+    )
+
+
 def to_apify_request(scrapy_request: ScrapyRequest, spider: Spider) -> ApifyRequest | None:
     """Convert a Scrapy request to an Apify request.
 
@@ -76,8 +112,14 @@ def to_apify_request(scrapy_request: ScrapyRequest, spider: Spider) -> ApifyRequ
     try:
         if scrapy_request.dont_filter:
             request_kwargs['always_enqueue'] = True
-        elif scrapy_request.meta.get('apify_request_unique_key'):
-            request_kwargs['unique_key'] = scrapy_request.meta['apify_request_unique_key']
+        elif unique_key := scrapy_request.meta.get('apify_request_unique_key'):
+            # Reuse the RQ unique key only while this is still the request it was minted for. Redirects
+            # (`Request.replace()`) and spiders forwarding `meta` to another URL inherit the stamp, and reusing
+            # it there deduplicates the derived request against its parent. A stamp without a fingerprint was
+            # set by hand, so it is taken at face value.
+            fingerprint = _compute_fingerprint(scrapy_request)
+            if scrapy_request.meta.get('apify_request_fingerprint', fingerprint) == fingerprint:
+                request_kwargs['unique_key'] = unique_key
 
         # Serialize the Scrapy request now, before `Request.from_url()` runs below. `from_url()` mutates the
         # `user_data` dict it receives in place (it injects a live `CrawleeRequestData` under `__crawlee`), and that
@@ -113,8 +155,9 @@ def to_apify_request(scrapy_request: ScrapyRequest, spider: Spider) -> ApifyRequ
         # the old behavior, which dropped such requests entirely.
         if isinstance(scrapy_request.headers, Headers):
             try:
-                headers = cast('dict[str, str]', dict(scrapy_request.headers.to_unicode_dict()))
-                request_kwargs['headers'] = HttpHeaders(headers)
+                # `to_unicode_dict()` yields str keys and values, but Scrapy types it as `UserDict[str | bytes, Any]`,
+                # so the mapping goes through the model's validator instead of a cast.
+                request_kwargs['headers'] = HttpHeaders.model_validate(scrapy_request.headers.to_unicode_dict())
             except UnicodeDecodeError:
                 logger.warning(
                     'Could not represent Scrapy request headers as Apify request headers (non-UTF-8 values); '
@@ -165,7 +208,7 @@ def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequ
     Returns:
         The converted Scrapy request.
     """
-    if not isinstance(cast('Any', apify_request), ApifyRequest):
+    if not isinstance(apify_request, ApifyRequest):
         raise TypeError('apify_request must be an apify.Request instance')
 
     # If the apify_request comes from the Scrapy
@@ -187,21 +230,14 @@ def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequ
         if not isinstance(scrapy_request, ScrapyRequest):
             raise TypeError('scrapy_request must be an instance of the ScrapyRequest class')
 
-        # Update the meta field with the meta field from the apify_request
-        meta = scrapy_request.meta or {}
-        meta.update({'apify_request_unique_key': apify_request.unique_key})
-        # scrapy_request.meta is a property, so we have to set it like this
-        scrapy_request._meta = meta  # noqa: SLF001
-
     # If the apify_request comes directly from the Scrapy, typically start URLs.
     else:
-        scrapy_request = ScrapyRequest(
-            url=apify_request.url,
-            method=apify_request.method,
-            meta={
-                'apify_request_unique_key': apify_request.unique_key,
-            },
-        )
+        scrapy_request = ScrapyRequest(url=apify_request.url, method=apify_request.method)
+
+    # Stamp the unique key with a fingerprint of the request it belongs to, so `to_apify_request` can tell
+    # this request apart from the ones Scrapy derives from it.
+    scrapy_request.meta['apify_request_unique_key'] = apify_request.unique_key
+    scrapy_request.meta['apify_request_fingerprint'] = _compute_fingerprint(scrapy_request)
 
     # Add optional 'headers' field
     if apify_request.headers: