apify 3.4.2b5__tar.gz → 3.4.2b6__tar.gz

{apify-3.4.2b5 → apify-3.4.2b6}/CHANGELOG.md

@@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file.
 
 - **scrapy:** Correct proxy middleware exception log and import ([#953](https://github.com/apify/apify-sdk-python/pull/953)) ([5bd6eb9](https://github.com/apify/apify-sdk-python/commit/5bd6eb9843d90844cec083372e932413bceedec9)) by [@vdusek](https://github.com/vdusek)
 - **scrapy:** Skip a request that fails to convert instead of crashing the run ([#952](https://github.com/apify/apify-sdk-python/pull/952)) ([db9444f](https://github.com/apify/apify-sdk-python/commit/db9444faeb0158c29aa394121cf733ff2e843f28)) by [@vdusek](https://github.com/vdusek)
+- **scrapy:** [**breaking**] Serialize requests and HTTP cache as JSON instead of pickle ([#951](https://github.com/apify/apify-sdk-python/pull/951)) ([a87e8d1](https://github.com/apify/apify-sdk-python/commit/a87e8d1597478b4f12fd5bb9b379f65f637d8e96)) by [@vdusek](https://github.com/vdusek)
 
 ### 🚜 Refactor
 

{apify-3.4.2b5 → apify-3.4.2b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 3.4.2b5
+Version: 3.4.2b6
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog

{apify-3.4.2b5 → apify-3.4.2b6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apify"
-version = "3.4.2b5"
+version = "3.4.2b6"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

apify-3.4.2b6/src/apify/scrapy/_serialization.py

@@ -0,0 +1,138 @@
+"""JSON serialization of Scrapy requests and cached responses for storage on the Apify platform.
+
+Scrapy requests and cached responses are stored in the Apify request queue and key-value store which hold JSON,
+so they are serialized as JSON here rather than pickled.
+
+Only `body` (`bytes`) and `headers` (`{bytes: [bytes]}`) are not natively JSON-serializable; both sit at fixed keys
+and are base64-encoded in place. A `str` `body` is encoded as its UTF-8 bytes and comes back as `bytes`, matching
+Scrapy, which always stores `body` as `bytes`. Pydantic models such as Crawlee's `UserData` are dumped via
+`model_dump()`. Everything else, notably `meta` and `cb_kwargs`, must already be JSON-serializable, otherwise
+serialization fails with a clear error naming the offending value. No in-band sentinel is used, so no user value
+can collide with the encoding.
+
+Known limitations of the pickle -> JSON switch (a documented breaking change): JSON has fewer types than pickle,
+so values in `meta`/`cb_kwargs` are subject to JSON's coercions. A `tuple` round-trips as a `list` and non-string
+`dict` keys round-trip as strings (e.g. `{1: 'a'}` becomes `{'1': 'a'}`). Values JSON cannot represent at all
+(`datetime`, `set`, `Decimal`, arbitrary objects, ...) are not coerced silently: serialization raises and the request
+is skipped loudly rather than stored in a corrupted form.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+from typing import Any
+
+from pydantic import BaseModel
+
+# Cap the offending value's repr in a serialization error message so a huge value cannot bloat the log.
+_MAX_ERROR_VALUE_REPR_LEN = 200
+
+
+def encode_to_json(data: dict[str, Any]) -> str:
+    """Serialize a Scrapy request/response dict to a JSON string.
+
+    The `body` and `headers` fields are base64-encoded in place (a `str` `body` via its UTF-8 bytes); pydantic
+    models are dumped to plain dicts. A `TypeError` is raised if any other value cannot be JSON-encoded.
+
+    Args:
+        data: The dict to serialize, e.g. the output of `scrapy.Request.to_dict()`.
+
+    Returns:
+        The JSON-encoded string.
+    """
+    if not isinstance(data, dict):
+        raise TypeError(f'Expected a dict to serialize, got {type(data)}')
+
+    safe = dict(data)
+
+    # `body` is base64-encoded so binary payloads survive; a `str` body is taken as its UTF-8 bytes, which keeps
+    # encode/decode symmetric (decode always base64-decodes `body` back to `bytes`).
+    body = safe.get('body')
+    if isinstance(body, (bytes, str)):
+        raw_body = body.encode('utf-8') if isinstance(body, str) else body
+        safe['body'] = base64.b64encode(raw_body).decode('ascii')
+
+    if isinstance(safe.get('headers'), dict):
+        safe['headers'] = _encode_headers(safe['headers'])
+
+    try:
+        # `ensure_ascii=False` keeps non-ASCII URLs/meta as their UTF-8 form instead of `\uXXXX` escapes, which
+        # would otherwise roughly double the size of non-Latin text in storage.
+        return json.dumps(safe, default=_json_default, ensure_ascii=False)
+    except TypeError as exc:
+        raise TypeError(
+            'Failed to JSON-serialize a Scrapy request/response for storage on the Apify platform. '
+            'All values in `meta` and `cb_kwargs` must be JSON-serializable (str, int, float, bool, None, '
+            'list, dict, or a pydantic model).'
+        ) from exc
+
+
+def decode_from_json(text: str) -> Any:
+    """Reconstruct a Scrapy request/response dict from a string produced by `encode_to_json`.
+
+    The base64-encoded `body` and `headers` fields are decoded back to their `bytes` representation.
+
+    Args:
+        text: The JSON-encoded string.
+
+    Returns:
+        The decoded object (a dict for valid request/response payloads).
+    """
+    data = json.loads(text)
+    if not isinstance(data, dict):
+        return data
+
+    # `validate=True` makes a non-base64 body raise loudly instead of silently decoding to garbage.
+    if isinstance(data.get('body'), str):
+        data['body'] = base64.b64decode(data['body'], validate=True)
+
+    if isinstance(data.get('headers'), dict):
+        data['headers'] = _decode_headers(data['headers'])
+
+    return data
+
+
+def _json_default(obj: Any) -> Any:
+    """Fallback for values `json.dumps` cannot serialize: pydantic models are dumped, anything else raises.
+
+    The error names the offending value (type and a truncated repr) so a failed serialization points straight
+    at the bad `meta`/`cb_kwargs` entry instead of just reporting that something failed.
+    """
+    if isinstance(obj, BaseModel):
+        return obj.model_dump(by_alias=True)
+    value_repr = repr(obj)
+    if len(value_repr) > _MAX_ERROR_VALUE_REPR_LEN:
+        value_repr = value_repr[:_MAX_ERROR_VALUE_REPR_LEN] + '...'
+    raise TypeError(f'Object of type {type(obj).__name__} is not JSON-serializable: {value_repr}')
+
+
+def _encode_headers(headers: dict[Any, Any]) -> dict[str, list[str]]:
+    """Encode a Scrapy `{bytes: [bytes]}` headers mapping to a JSON-safe `{str: [base64-str]}`."""
+    encoded: dict[str, list[str]] = {}
+    for key, value in headers.items():
+        str_key = key.decode('latin-1') if isinstance(key, bytes) else key
+        values = value if isinstance(value, (list, tuple)) else [value]
+        encoded[str_key] = [_b64encode_value(item) for item in values]
+    return encoded
+
+
+def _decode_headers(headers: dict[str, Any]) -> dict[bytes, list[bytes]]:
+    """Reverse `_encode_headers`, restoring the `{bytes: [bytes]}` mapping Scrapy expects."""
+    decoded: dict[bytes, list[bytes]] = {}
+    for key, value in headers.items():
+        bytes_key = key.encode('latin-1') if isinstance(key, str) else key
+        values = value if isinstance(value, list) else [value]
+        decoded[bytes_key] = [base64.b64decode(item, validate=True) for item in values]
+    return decoded
+
+
+def _b64encode_value(value: Any) -> str:
+    """Base64-encode a single header value.
+
+    Scrapy stores header values as `bytes`; a `str` is encoded as its UTF-8 bytes. Any other type is coerced with
+    `str()` as a lenient last resort. That coercion is lossy (e.g. `5` becomes `b'5'`), but Scrapy does not produce
+    non-`bytes`/`str` header values, so it is not hit on the real path.
+    """
+    raw = value if isinstance(value, bytes) else str(value).encode('utf-8')
+    return base64.b64encode(raw).decode('ascii')

{apify-3.4.2b5 → apify-3.4.2b6}/src/apify/scrapy/extensions/_httpcache.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import gzip
 import io
-import pickle
 import re
 import struct
 from logging import getLogger
@@ -14,6 +13,7 @@ from scrapy.responsetypes import responsetypes
 
 from apify import Configuration
 from apify.scrapy._async_thread import AsyncThread
+from apify.scrapy._serialization import decode_from_json, encode_to_json
 from apify.storage_clients import ApifyStorageClient
 from apify.storages import KeyValueStore
 
@@ -29,14 +29,14 @@ logger = getLogger(__name__)
 class ApifyCacheStorage:
     """A Scrapy cache storage that uses the Apify `KeyValueStore` to store responses.
 
-    It can be set as a storage for Scrapy's built-in `HttpCacheMiddleware`, which caches
-    responses to requests. See HTTPCache middleware settings (prefixed with `HTTPCACHE_`)
-    in the Scrapy documentation for more information. Requires the asyncio Twisted reactor
-    to be installed.
+    It can be set as a storage for Scrapy's built-in `HttpCacheMiddleware`, which caches responses to requests.
+    See HTTPCache middleware settings (prefixed with `HTTPCACHE_`) in the Scrapy documentation for more information.
+    Requires the asyncio Twisted reactor to be installed.
     """
 
     def __init__(self, settings: BaseSettings) -> None:
-        self._expiration_max_items = 100
+        # Upper bound on how many keys the per-spider-close cleanup sweeps (best-effort; `close_spider`).
+        self._expiration_max_items: int = settings.getint('APIFY_HTTPCACHE_EXPIRATION_MAX_ITEMS', 100)
         self._expiration_secs: int = settings.getint('HTTPCACHE_EXPIRATION_SECS')
         self._spider: Spider | None = None
         self._kvs: KeyValueStore | None = None
@@ -79,23 +79,26 @@ class ApifyCacheStorage:
             async def expire_kvs() -> None:
                 if self._kvs is None:
                     raise ValueError('Key value store not initialized')
-                i = 0
+                # Best-effort cleanup: at most `_expiration_max_items` keys per close, in no guaranteed order,
+                # so stale entries may linger. This only reclaims storage; `retrieve_response` already treats
+                # an expired entry as a cache miss.
+                processed = 0
                 async for item in self._kvs.iterate_keys():
+                    if processed >= self._expiration_max_items:
+                        break
+                    processed += 1
                     value = await self._kvs.get_value(item.key)
                     try:
                         gzip_time = read_gzip_time(value)
                     except Exception as e:
                         logger.warning(f'Malformed cache item {item.key}: {e}')
-                        await self._kvs.set_value(item.key, None)
+                        await self._kvs.delete_value(item.key)
                     else:
                         if self._expiration_secs < current_time - gzip_time:
                             logger.debug(f'Expired cache item {item.key}')
-                            await self._kvs.set_value(item.key, None)
+                            await self._kvs.delete_value(item.key)
                         else:
                             logger.debug(f'Valid cache item {item.key}')
-                    if i == self._expiration_max_items:
-                        break
-                    i += 1
 
             self._async_thread.run_coro(expire_kvs())
 
@@ -127,17 +130,25 @@ class ApifyCacheStorage:
 
         if current_time is None:
             current_time = int(time())
-        if 0 < self._expiration_secs < current_time - read_gzip_time(value):
-            logger.debug('Cache expired', extra={'request': request})
+
+        # A malformed or legacy cache entry must not crash retrieval; treat it as a cache miss so Scrapy re-fetches
+        # and re-stores it in the current format. The field reads stay inside the `try` as well: a value that decodes
+        # to a dict missing any expected key (a forward/older format, or a truncated-but-valid JSON payload) must
+        # also degrade to a miss rather than raising an uncaught `KeyError`.
+        try:
+            if 0 < self._expiration_secs < current_time - read_gzip_time(value):
+                logger.debug('Cache expired', extra={'request': request})
+                return None
+            data = from_gzip(value)
+            url = data['url']
+            status = data['status']
+            headers = Headers(data['headers'])
+            body = data['body']
+        except Exception as exc:
+            logger.warning(f'Ignoring malformed cache entry {key!r}: {exc}', extra={'request': request})
             return None
 
-        data = from_gzip(value)
-        url = data['url']
-        status = data['status']
-        headers = Headers(data['headers'])
-        body = data['body']
         respcls = responsetypes.from_args(headers=headers, url=url, body=body)
-
         logger.debug('Cache hit', extra={'request': request})
         return respcls(url=url, headers=headers, status=status, body=body)
 
@@ -162,18 +173,25 @@ class ApifyCacheStorage:
 
 
 def to_gzip(data: dict, mtime: int | None = None) -> bytes:
-    """Dump a dictionary to a gzip-compressed byte stream."""
+    """Dump a dictionary to a gzip-compressed JSON byte stream.
+
+    Cache entries live in the Apify key-value store, which holds JSON, so they are serialized as JSON rather
+    than pickled. See `apify.scrapy._serialization` for the encoding.
+    """
+    payload = encode_to_json(data).encode('utf-8')
     with io.BytesIO() as byte_stream:
         with gzip.GzipFile(fileobj=byte_stream, mode='wb', mtime=mtime) as gzip_file:
-            pickle.dump(data, gzip_file, protocol=4)
+            gzip_file.write(payload)
         return byte_stream.getvalue()
 
 
 def from_gzip(gzip_bytes: bytes) -> dict:
-    """Load a dictionary from a gzip-compressed byte stream."""
+    """Load a dictionary from a gzip-compressed JSON byte stream."""
     with io.BytesIO(gzip_bytes) as byte_stream, gzip.GzipFile(fileobj=byte_stream, mode='rb') as gzip_file:
-        data: dict = pickle.load(gzip_file)
-        return data
+        data = decode_from_json(gzip_file.read().decode('utf-8'))
+    if not isinstance(data, dict):
+        raise TypeError(f'Expected a dict from the cached payload, got {type(data)}')
+    return data
 
 
 def read_gzip_time(gzip_bytes: bytes) -> int:
@@ -187,17 +205,15 @@ def read_gzip_time(gzip_bytes: bytes) -> int:
 def get_kvs_name(spider_name: str, max_length: int = 60) -> str:
     """Get the key value store name for a spider.
 
-    The key value store name is derived from the spider name by replacing all special characters
-    with hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with
-    'httpcache-' and truncated to the maximum length.
+    The key value store name is derived from the spider name by replacing all special characters with hyphens
+    and trimming leading and trailing hyphens. The resulting name is prefixed with 'httpcache-' and truncated
+    to the maximum length.
 
-    The documentation
-    [about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)
+    The documentation [about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)
     mentions that names can be up to 63 characters long, so the default max length is set to 60.
 
-    Such naming isn't unique per spider, but should be sufficiently unique for most use cases.
-    The name of the key value store should indicate to which spider it belongs, e.g. in
-    the listing in the Apify's console.
+    Such naming isn't unique per spider, but should be sufficiently unique for most use cases. The name
+    of the key-value store should indicate to which spider it belongs, e.g. in the listing in the Apify's console.
 
     Args:
         spider_name: Value of the Spider instance's name attribute.

apify-3.4.2b6/src/apify/scrapy/requests.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+from logging import getLogger
+from typing import Any, cast
+
+from scrapy import Request as ScrapyRequest
+from scrapy import Spider
+from scrapy.http.headers import Headers
+from scrapy.utils.misc import load_object
+from scrapy.utils.request import request_from_dict
+
+from crawlee._request import UserData
+from crawlee._types import HttpHeaders
+
+from ._serialization import decode_from_json, encode_to_json
+from apify import Request as ApifyRequest
+
+logger = getLogger(__name__)
+
+
+def _ensure_known_request_class(request_dict: dict[str, Any]) -> None:
+    """Validate the optional `_class` entry before `request_from_dict` instantiates it.
+
+    `request_from_dict` resolves `_class` with `load_object` and calls it with the request kwargs. The dotted path
+    is resolved here first and rejected unless it is a `scrapy.Request` subclass, so a payload can never coerce
+    reconstruction into instantiating an arbitrary callable. Resolving may import the class's module (the same
+    import `request_from_dict` would do, and far safer than the arbitrary code execution the previous pickle
+    format allowed). That import is what lets a custom `Request` subclass be rebuilt in a fresh process after
+    an Actor migration, before the spider has lazily imported it.
+    """
+    class_path = request_dict.get('_class')
+    if class_path is None:
+        return
+
+    if not isinstance(class_path, str):
+        raise TypeError(f'Invalid scrapy_request `_class`, expected a string, got {type(class_path)}')
+
+    try:
+        request_cls = load_object(class_path)
+    except (ImportError, AttributeError, ValueError, NameError) as exc:
+        raise TypeError(f'Refusing to reconstruct a Scrapy request: cannot resolve `_class` {class_path!r}.') from exc
+
+    if not (isinstance(request_cls, type) and issubclass(request_cls, ScrapyRequest)):
+        raise TypeError(
+            f'Refusing to reconstruct a Scrapy request of type {class_path!r}: it is not a scrapy.Request subclass.'
+        )
+
+
+def to_apify_request(scrapy_request: ScrapyRequest, spider: Spider) -> ApifyRequest | None:
+    """Convert a Scrapy request to an Apify request.
+
+    Args:
+        scrapy_request: The Scrapy request to be converted.
+        spider: The Scrapy spider that the request is associated with.
+
+    Returns:
+        The converted Apify request if the conversion was successful, otherwise None.
+    """
+    if not isinstance(scrapy_request, ScrapyRequest):
+        logger.warning('Failed to convert to Apify request: Scrapy request must be a ScrapyRequest instance.')
+        return None
+
+    logger.debug(f'to_apify_request was called (scrapy_request={scrapy_request})...')
+
+    # Configuration to behave as similarly as possible to Scrapy's default RFPDupeFilter.
+    #
+    # The body is stored twice on purpose: as `payload` (used for the extended unique key) and inside the serialized
+    # Scrapy request below (used to reconstruct it). Both come from `scrapy_request.body`.
+    request_kwargs: dict[str, Any] = {
+        'url': scrapy_request.url,
+        'method': scrapy_request.method,
+        'payload': scrapy_request.body,
+        'use_extended_unique_key': True,
+        'keep_url_fragment': False,
+    }
+
+    try:
+        if scrapy_request.dont_filter:
+            request_kwargs['always_enqueue'] = True
+        elif scrapy_request.meta.get('apify_request_unique_key'):
+            request_kwargs['unique_key'] = scrapy_request.meta['apify_request_unique_key']
+
+        user_data = scrapy_request.meta.get('userData', {})
+
+        # Convert UserData Pydantic model to a plain dict to prevent CrawleeRequestData objects from leaking
+        # into Request.from_url() during Scrapy-Apify roundtrips.
+        if isinstance(user_data, UserData):
+            user_data = user_data.model_dump(by_alias=True)
+
+        # Remove internal Crawlee data since it's managed by Request.from_url() and values from previous roundtrips
+        # cause incorrect state.
+        if isinstance(user_data, dict):
+            user_data.pop('__crawlee', None)
+
+        request_kwargs['user_data'] = user_data if isinstance(user_data, dict) else {}
+
+        # Store an Apify-platform view of the headers. The authoritative copy with exact bytes travels in
+        # the serialized scrapy_request below, so non-UTF-8 headers (which make `to_unicode_dict()` raise) are
+        # tolerated rather than dropping the whole request.
+        #
+        # Trade-off: with `use_extended_unique_key=True` the unique key includes the headers, so when non-UTF-8
+        # headers are omitted here two requests differing only in those headers share a unique key and one is
+        # deduplicated away. This is rare (header values are normally ASCII/UTF-8) and still strictly better than
+        # the old behavior, which dropped such requests entirely.
+        if isinstance(scrapy_request.headers, Headers):
+            try:
+                headers = cast('dict[str, str]', dict(scrapy_request.headers.to_unicode_dict()))
+                request_kwargs['headers'] = HttpHeaders(headers)
+            except UnicodeDecodeError:
+                logger.warning(
+                    'Could not represent Scrapy request headers as Apify request headers (non-UTF-8 values); '
+                    'they are preserved in the serialized request instead.'
+                )
+        else:
+            logger.warning(
+                f'Invalid scrapy_request.headers type, not scrapy.http.headers.Headers: {scrapy_request.headers}'
+            )
+
+        apify_request = ApifyRequest.from_url(**request_kwargs)
+        scrapy_request_dict = scrapy_request.to_dict(spider=spider)
+
+    except Exception as exc:
+        logger.warning(f'Conversion of Scrapy request {scrapy_request} to Apify request failed; {exc}')
+        return None
+
+    # Serialize the Scrapy request as JSON under 'scrapy_request'. Kept outside the broad except above so
+    # a non-JSON-serializable `meta`/`cb_kwargs` is logged with a traceback and the request skipped (returning
+    # None per this function's contract), rather than crashing the crawl.
+    try:
+        scrapy_request_json = encode_to_json(scrapy_request_dict)
+    except TypeError:
+        logger.exception(
+            f'Failed to serialize Scrapy request {scrapy_request} for storage on the Apify platform; skipping it. '
+            'Ensure all values in `meta` and `cb_kwargs` are JSON-serializable.'
+        )
+        return None
+
+    # `scrapy_request_json` is already JSON-safe text (binary fields are base64-encoded inside it), so it is stored
+    # as-is. The request queue serializes `user_data` to JSON, which escapes the string correctly; wrapping it in
+    # a second base64 layer would only add ~33% overhead on the enqueue path.
+    apify_request.user_data['scrapy_request'] = scrapy_request_json
+
+    logger.debug(f'scrapy_request was converted to the apify_request={apify_request}')
+    return apify_request
+
+
+def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequest:
+    """Convert an Apify request to a Scrapy request.
+
+    Args:
+        apify_request: The Apify request to be converted.
+        spider: The Scrapy spider that the request is associated with.
+
+    Raises:
+        TypeError: If `apify_request` is not an `ApifyRequest`, if the stored Scrapy request payload is malformed,
+        or if its `_class` cannot be resolved to a `scrapy.Request` subclass.
+
+    Returns:
+        The converted Scrapy request.
+    """
+    if not isinstance(cast('Any', apify_request), ApifyRequest):
+        raise TypeError('apify_request must be an apify.Request instance')
+
+    logger.debug(f'to_scrapy_request was called (apify_request={apify_request})...')
+
+    # If the apify_request comes from the Scrapy
+    if 'scrapy_request' in apify_request.user_data:
+        # Deserialize the Scrapy ScrapyRequest from the apify_request by parsing the stored JSON and reconstructing
+        # the Scrapy ScrapyRequest object from its dictionary representation.
+        logger.debug('Restoring the Scrapy ScrapyRequest from the apify_request...')
+
+        scrapy_request_json = apify_request.user_data['scrapy_request']
+        if not isinstance(scrapy_request_json, str):
+            raise TypeError('the stored scrapy_request must be a string')
+
+        scrapy_request_dict = decode_from_json(scrapy_request_json)
+        if not isinstance(scrapy_request_dict, dict):
+            raise TypeError('scrapy_request_dict must be a dictionary')
+
+        # Validate any `_class` entry before request_from_dict resolves and imports it.
+        _ensure_known_request_class(scrapy_request_dict)
+
+        scrapy_request = request_from_dict(scrapy_request_dict, spider=spider)
+        if not isinstance(scrapy_request, ScrapyRequest):
+            raise TypeError('scrapy_request must be an instance of the ScrapyRequest class')
+
+        logger.debug(f'Scrapy ScrapyRequest successfully reconstructed (scrapy_request={scrapy_request})...')
+
+        # Update the meta field with the meta field from the apify_request
+        meta = scrapy_request.meta or {}
+        meta.update({'apify_request_unique_key': apify_request.unique_key})
+        # scrapy_request.meta is a property, so we have to set it like this
+        scrapy_request._meta = meta  # noqa: SLF001
+
+    # If the apify_request comes directly from the Scrapy, typically start URLs.
+    else:
+        logger.debug('Gonna create a new Scrapy ScrapyRequest (cannot be restored)')
+
+        scrapy_request = ScrapyRequest(
+            url=apify_request.url,
+            method=apify_request.method,
+            meta={
+                'apify_request_unique_key': apify_request.unique_key,
+            },
+        )
+
+    # Add optional 'headers' field
+    if apify_request.headers:
+        scrapy_request.headers |= Headers(apify_request.headers)
+
+    # Add optional 'userData' field
+    if apify_request.user_data:
+        scrapy_request.meta['userData'] = apify_request.user_data
+
+    logger.debug(f'an apify_request was converted to the scrapy_request={scrapy_request}')
+    return scrapy_request

{apify-3.4.2b5 → apify-3.4.2b6}/src/apify/scrapy/scheduler.py

@@ -162,20 +162,24 @@ class ApifyScheduler(BaseScheduler):
         if not isinstance(self.spider, Spider):
             raise TypeError('self.spider must be an instance of the Spider class')
 
-        # Let the request queue know that the request is being handled. Every request should
-        # be marked as handled, retrying is handled by the Scrapy's RetryMiddleware.
+        # Reconstruct the Scrapy request before consuming the queue entry. A malformed entry must not crash
+        # the whole run, so on failure it is logged and skipped (None) rather than propagating.
+        try:
+            scrapy_request = to_scrapy_request(apify_request, spider=self.spider)
+        except Exception:
+            logger.exception(f'Failed to convert Apify request {apify_request} to a Scrapy request; skipping it.')
+            scrapy_request = None
+
+        # Mark the request as handled. This runs even when reconstruction failed above: an unrecoverable entry
+        # (a corrupt or legacy payload) must still be consumed, otherwise the queue would keep handing it back
+        # forever. Retrying genuine failures is the RetryMiddleware's job.
         try:
             self._async_thread.run_coro(self._rq.mark_request_as_handled(apify_request))
         except Exception:
             traceback.print_exc()
             raise
 
-        # Reconstruct the Scrapy request. A malformed queue entry must not crash the whole run: it
-        # has already been marked handled above, so log it and skip it instead of propagating.
-        try:
-            scrapy_request = to_scrapy_request(apify_request, spider=self.spider)
-        except Exception:
-            logger.exception(f'Failed to convert Apify request {apify_request} to a Scrapy request; skipping it.')
+        if scrapy_request is None:
             return None
 
         logger.debug(f'Converted to scrapy_request: {scrapy_request}')

apify-3.4.2b5/src/apify/scrapy/requests.py

@@ -1,164 +0,0 @@
-from __future__ import annotations
-
-import codecs
-import pickle
-from logging import getLogger
-from typing import Any, cast
-
-from scrapy import Request as ScrapyRequest
-from scrapy import Spider
-from scrapy.http.headers import Headers
-from scrapy.utils.request import request_from_dict
-
-from crawlee._request import UserData
-from crawlee._types import HttpHeaders
-
-from apify import Request as ApifyRequest
-
-logger = getLogger(__name__)
-
-
-def to_apify_request(scrapy_request: ScrapyRequest, spider: Spider) -> ApifyRequest | None:
-    """Convert a Scrapy request to an Apify request.
-
-    Args:
-        scrapy_request: The Scrapy request to be converted.
-        spider: The Scrapy spider that the request is associated with.
-
-    Returns:
-        The converted Apify request if the conversion was successful, otherwise None.
-    """
-    if not isinstance(scrapy_request, ScrapyRequest):
-        logger.warning('Failed to convert to Apify request: Scrapy request must be a ScrapyRequest instance.')
-        return None
-
-    logger.debug(f'to_apify_request was called (scrapy_request={scrapy_request})...')
-
-    # Configuration to behave as similarly as possible to Scrapy's default RFPDupeFilter.
-    request_kwargs: dict[str, Any] = {
-        'url': scrapy_request.url,
-        'method': scrapy_request.method,
-        'payload': scrapy_request.body,
-        'use_extended_unique_key': True,
-        'keep_url_fragment': False,
-    }
-
-    try:
-        if scrapy_request.dont_filter:
-            request_kwargs['always_enqueue'] = True
-        else:
-            if scrapy_request.meta.get('apify_request_unique_key'):
-                request_kwargs['unique_key'] = scrapy_request.meta['apify_request_unique_key']
-
-            if scrapy_request.meta.get('apify_request_id'):
-                request_kwargs['id'] = scrapy_request.meta['apify_request_id']
-
-        user_data = scrapy_request.meta.get('userData', {})
-
-        # Convert UserData Pydantic model to a plain dict to prevent CrawleeRequestData objects
-        # from leaking into Request.from_url() during Scrapy-Apify roundtrips.
-        if isinstance(user_data, UserData):
-            user_data = user_data.model_dump(by_alias=True)
-
-        # Remove internal Crawlee data since it's managed by Request.from_url() and values
-        # from previous roundtrips cause incorrect state.
-        if isinstance(user_data, dict):
-            user_data.pop('__crawlee', None)
-
-        request_kwargs['user_data'] = user_data if isinstance(user_data, dict) else {}
-
-        # Convert Scrapy's headers to a HttpHeaders and store them in the apify_request
-        if isinstance(scrapy_request.headers, Headers):
-            headers = cast('dict[str, str]', dict(scrapy_request.headers.to_unicode_dict()))
-            request_kwargs['headers'] = HttpHeaders(headers)
-        else:
-            logger.warning(
-                f'Invalid scrapy_request.headers type, not scrapy.http.headers.Headers: {scrapy_request.headers}'
-            )
-
-        apify_request = ApifyRequest.from_url(**request_kwargs)
-
-        # Serialize the Scrapy ScrapyRequest and store it in the apify_request.
-        #   - This process involves converting the Scrapy ScrapyRequest object into a dictionary, encoding it to base64,
-        #     and storing it as 'scrapy_request' within the 'userData' dictionary of the apify_request.
-        #   - The serialization process can be referenced at: https://stackoverflow.com/questions/30469575/.
-        scrapy_request_dict = scrapy_request.to_dict(spider=spider)
-        scrapy_request_dict_encoded = codecs.encode(pickle.dumps(scrapy_request_dict), 'base64').decode()
-        apify_request.user_data['scrapy_request'] = scrapy_request_dict_encoded
-
-    except Exception as exc:
-        logger.warning(f'Conversion of Scrapy request {scrapy_request} to Apify request failed; {exc}')
-        return None
-
-    logger.debug(f'scrapy_request was converted to the apify_request={apify_request}')
-    return apify_request
-
-
-def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequest:
-    """Convert an Apify request to a Scrapy request.
-
-    Args:
-        apify_request: The Apify request to be converted.
-        spider: The Scrapy spider that the request is associated with.
-
-    Raises:
-        TypeError: If the Apify request is not an instance of the `ApifyRequest` class.
-        ValueError: If the Apify request does not contain the required keys.
-
-    Returns:
-        The converted Scrapy request.
-    """
-    if not isinstance(cast('Any', apify_request), ApifyRequest):
-        raise TypeError('apify_request must be a crawlee.ScrapyRequest instance')
-
-    logger.debug(f'to_scrapy_request was called (apify_request={apify_request})...')
-
-    # If the apify_request comes from the Scrapy
-    if 'scrapy_request' in apify_request.user_data:
-        # Deserialize the Scrapy ScrapyRequest from the apify_request.
-        #   - This process involves decoding the base64-encoded request data and reconstructing
-        #     the Scrapy ScrapyRequest object from its dictionary representation.
-        logger.debug('Restoring the Scrapy ScrapyRequest from the apify_request...')
-
-        scrapy_request_dict_encoded = apify_request.user_data['scrapy_request']
-        if not isinstance(scrapy_request_dict_encoded, str):
-            raise TypeError('scrapy_request_dict_encoded must be a string')
-
-        scrapy_request_dict = pickle.loads(codecs.decode(scrapy_request_dict_encoded.encode(), 'base64'))
-        if not isinstance(scrapy_request_dict, dict):
-            raise TypeError('scrapy_request_dict must be a dictionary')
-
-        scrapy_request = request_from_dict(scrapy_request_dict, spider=spider)
-        if not isinstance(scrapy_request, ScrapyRequest):
-            raise TypeError('scrapy_request must be an instance of the ScrapyRequest class')
-
-        logger.debug(f'Scrapy ScrapyRequest successfully reconstructed (scrapy_request={scrapy_request})...')
-
-        # Update the meta field with the meta field from the apify_request
-        meta = scrapy_request.meta or {}
-        meta.update({'apify_request_unique_key': apify_request.unique_key})
-        # scrapy_request.meta is a property, so we have to set it like this
-        scrapy_request._meta = meta  # noqa: SLF001
-
-    # If the apify_request comes directly from the Scrapy, typically start URLs.
-    else:
-        logger.debug('Gonna create a new Scrapy ScrapyRequest (cannot be restored)')
-
-        scrapy_request = ScrapyRequest(
-            url=apify_request.url,
-            method=apify_request.method,
-            meta={
-                'apify_request_unique_key': apify_request.unique_key,
-            },
-        )
-
-    # Add optional 'headers' field
-    if apify_request.headers:
-        scrapy_request.headers |= Headers(apify_request.headers)
-
-    # Add optional 'userData' field
-    if apify_request.user_data:
-        scrapy_request.meta['userData'] = apify_request.user_data
-
-    logger.debug(f'an apify_request was converted to the scrapy_request={scrapy_request}')
-    return scrapy_request