apify 3.4.2b4__tar.gz → 3.4.2b6__tar.gz

{apify-3.4.2b4 → apify-3.4.2b6}/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 <!-- git-cliff-unreleased-start -->
 ## 3.4.2 - **not yet released**
 
+### 🐛 Bug Fixes
+
+- **scrapy:** Correct proxy middleware exception log and import ([#953](https://github.com/apify/apify-sdk-python/pull/953)) ([5bd6eb9](https://github.com/apify/apify-sdk-python/commit/5bd6eb9843d90844cec083372e932413bceedec9)) by [@vdusek](https://github.com/vdusek)
+- **scrapy:** Skip a request that fails to convert instead of crashing the run ([#952](https://github.com/apify/apify-sdk-python/pull/952)) ([db9444f](https://github.com/apify/apify-sdk-python/commit/db9444faeb0158c29aa394121cf733ff2e843f28)) by [@vdusek](https://github.com/vdusek)
+- **scrapy:** [**breaking**] Serialize requests and HTTP cache as JSON instead of pickle ([#951](https://github.com/apify/apify-sdk-python/pull/951)) ([a87e8d1](https://github.com/apify/apify-sdk-python/commit/a87e8d1597478b4f12fd5bb9b379f65f637d8e96)) by [@vdusek](https://github.com/vdusek)
+
 ### 🚜 Refactor
 
 - [**breaking**] Remove deprecated APIs ([#918](https://github.com/apify/apify-sdk-python/pull/918)) ([3e5728d](https://github.com/apify/apify-sdk-python/commit/3e5728d94cb8fd879d5a76e33a03d55792d835d5)) by [@vdusek](https://github.com/vdusek), closes [#635](https://github.com/apify/apify-sdk-python/issues/635)

{apify-3.4.2b4 → apify-3.4.2b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 3.4.2b4
+Version: 3.4.2b6
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog

{apify-3.4.2b4 → apify-3.4.2b6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apify"
-version = "3.4.2b4"
+version = "3.4.2b6"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{apify-3.4.2b4 → apify-3.4.2b6}/src/apify/_charging.py

@@ -7,7 +7,7 @@ from datetime import UTC, datetime
 from decimal import Decimal
 from typing import TYPE_CHECKING, Annotated, Literal, Protocol, TypedDict
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import Field
 
 import apify_client._models as _client_models
 from apify_client._models import ActorChargeEvent as ClientActorChargeEvent
@@ -28,14 +28,17 @@ if TYPE_CHECKING:
 
     from apify._configuration import Configuration
 
-PricingModel = Literal['PAY_PER_EVENT', 'PRICE_PER_DATASET_ITEM', 'FLAT_PRICE_PER_MONTH', 'FREE']
-"""Pricing model for an Actor."""
+charging_manager_ctx: ContextVar[ChargingManager | None] = ContextVar('charging_manager_ctx', default=None)
+"""Holds the current `ChargingManager` instance, if any.
+
+Allows PPE-aware dataset clients to access the charging manager without needing to pass it explicitly.
+"""
 
 DEFAULT_DATASET_ITEM_EVENT = 'apify-default-dataset-item'
+"""Name of the synthetic event charged for each item pushed to the default dataset."""
 
-# Context variable to hold the current `ChargingManager` instance, if any. This allows PPE-aware dataset clients to
-# access the charging manager without needing to pass it explicitly.
-charging_manager_ctx: ContextVar[ChargingManager | None] = ContextVar('charging_manager_ctx', default=None)
+PricingModel = Literal['PAY_PER_EVENT', 'PRICE_PER_DATASET_ITEM', 'FLAT_PRICE_PER_MONTH', 'FREE']
+"""Pricing model for an Actor."""
 
 _ensure_context = ensure_context('active')
 
@@ -49,48 +52,91 @@ _ensure_context = ensure_context('active')
 # `apify-client` instance) flows through the same code paths without conversion.
 
 
-class _RelaxedPricingMetadata(BaseModel):
-    """Mixin relaxing the `CommonActorPricingInfo` metadata fields the platform env var omits."""
-
-    model_config = ConfigDict(populate_by_name=True, extra='allow')
-
-    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
-    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
-    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
-
-
 @docs_group('Charging')
 class ActorChargeEvent(ClientActorChargeEvent):
-    # `event_description` is required in apify-client but omitted from the env var.
+    """Definition of a single chargeable event in the pay-per-event pricing model."""
+
     event_description: Annotated[str | None, Field(alias='eventDescription')] = None
+    """Human-readable description of the event.
+
+    Required in apify-client but omitted from the env var, so it is relaxed to optional.
+    """
 
 
 @docs_group('Charging')
 class PricingPerEvent(ClientPricingPerEvent):
+    """Pay-per-event pricing details - the chargeable events and their prices."""
+
     actor_charge_events: Annotated[dict[str, ActorChargeEvent] | None, Field(alias='actorChargeEvents')] = None
+    """Mapping of event name to its charge definition."""
 
 
 @docs_group('Charging')
-class FreeActorPricingInfo(_RelaxedPricingMetadata, ClientFree):
-    pass
+class FreeActorPricingInfo(ClientFree):
+    """Pricing info for an Actor offered free of charge."""
+
+    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    """Apify's margin on the price, as a percentage."""
+
+    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    """Timestamp when this pricing info was created."""
+
+    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    """Timestamp when this pricing became effective."""
 
 
 @docs_group('Charging')
-class FlatPricePerMonthActorPricingInfo(_RelaxedPricingMetadata, ClientFlatPricePerMonth):
+class FlatPricePerMonthActorPricingInfo(ClientFlatPricePerMonth):
+    """Pricing info for an Actor billed at a flat monthly price."""
+
+    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    """Apify's margin on the price, as a percentage."""
+
+    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    """Timestamp when this pricing info was created."""
+
+    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    """Timestamp when this pricing became effective."""
+
     trial_minutes: Annotated[int | None, Field(alias='trialMinutes')] = None
+    """Length of the free trial period, in minutes."""
+
     price_per_unit_usd: Annotated[float | None, Field(alias='pricePerUnitUsd')] = None
+    """Price per unit, in USD."""
 
 
 @docs_group('Charging')
-class PricePerDatasetItemActorPricingInfo(_RelaxedPricingMetadata, ClientPricePerDatasetItem):
+class PricePerDatasetItemActorPricingInfo(ClientPricePerDatasetItem):
+    """Pricing info for an Actor billed per dataset item produced."""
+
+    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    """Apify's margin on the price, as a percentage."""
+
+    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    """Timestamp when this pricing info was created."""
+
+    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    """Timestamp when this pricing became effective."""
+
     unit_name: Annotated[str | None, Field(alias='unitName')] = None
-    # `price_per_unit_usd` is already optional in apify-client - inherited.
+    """Name of the billed unit."""
 
 
 @docs_group('Charging')
-class PayPerEventActorPricingInfo(_RelaxedPricingMetadata, ClientPayPerEvent):
-    # Re-typed to the relaxed element so an omitted `eventDescription` validates; the field stays required.
+class PayPerEventActorPricingInfo(ClientPayPerEvent):
+    """Pricing info for an Actor billed per charged event."""
+
+    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    """Apify's margin on the price, as a percentage."""
+
+    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    """Timestamp when this pricing info was created."""
+
+    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    """Timestamp when this pricing became effective."""
+
     pricing_per_event: Annotated[PricingPerEvent, Field(alias='pricingPerEvent')]
+    """The pay-per-event pricing details."""
 
 
 ActorPricingInfoModel = ClientFree | ClientFlatPricePerMonth | ClientPricePerDatasetItem | ClientPayPerEvent

{apify-3.4.2b4 → apify-3.4.2b6}/src/apify/events/_types.py

@@ -27,14 +27,31 @@ This is the Apify-specific subset of [`Event`][crawlee.events.Event] — for the
 
 @docs_group('Event data')
 class SystemInfoEventData(BaseModel):
+    """Resource usage metrics carried by a `systemInfo` event."""
+
     mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
+    """Average memory usage over the measured interval, in bytes."""
+
     mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
+    """Current memory usage, in bytes."""
+
     mem_max_bytes: Annotated[float, Field(alias='memMaxBytes')]
+    """Peak memory usage observed so far, in bytes."""
+
     cpu_avg_usage: Annotated[float, Field(alias='cpuAvgUsage')]
+    """Average CPU usage over the measured interval, in percent."""
+
     cpu_max_usage: Annotated[float, Field(alias='cpuMaxUsage')]
+    """Peak CPU usage observed so far, in percent."""
+
     cpu_current_usage: Annotated[float, Field(alias='cpuCurrentUsage')]
+    """Current CPU usage, in percent."""
+
     is_cpu_overloaded: Annotated[bool, Field(alias='isCpuOverloaded')]
+    """Whether the CPU is currently overloaded."""
+
     created_at: Annotated[datetime, Field(alias='createdAt')]
+    """Timestamp when the metrics were collected."""
 
     def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
         return EventSystemInfoData.model_validate(
@@ -54,36 +71,63 @@ class SystemInfoEventData(BaseModel):
 
 @docs_group('Events')
 class PersistStateEvent(BaseModel):
+    """A `persistState` event instructing the Actor to persist its state."""
+
     name: Literal[Event.PERSIST_STATE]
+    """The event name."""
+
     data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
+    """The event payload."""
 
 
 @docs_group('Events')
 class SystemInfoEvent(BaseModel):
+    """A `systemInfo` event carrying the Actor's resource usage metrics."""
+
     name: Literal[Event.SYSTEM_INFO]
+    """The event name."""
+
     data: SystemInfoEventData
+    """The event payload."""
 
 
 @docs_group('Events')
 class MigratingEvent(BaseModel):
+    """A `migrating` event signalling the Actor is about to be migrated to another host."""
+
     name: Literal[Event.MIGRATING]
+    """The event name."""
+
     data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
+    """The event payload."""
 
 
 @docs_group('Events')
 class AbortingEvent(BaseModel):
+    """An `aborting` event signalling the Actor run is being aborted."""
+
     name: Literal[Event.ABORTING]
+    """The event name."""
+
     data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
+    """The event payload."""
 
 
 @docs_group('Events')
 class ExitEvent(BaseModel):
+    """An `exit` event signalling the Actor process is about to exit."""
+
     name: Literal[Event.EXIT]
+    """The event name."""
+
     data: Annotated[EventExitData, Field(default_factory=EventExitData)]
+    """The event payload."""
 
 
 @docs_group('Events')
 class EventWithoutData(BaseModel):
+    """A framework-level event that carries no payload (e.g. browser and page lifecycle events)."""
+
     name: Literal[
         Event.SESSION_RETIRED,
         Event.BROWSER_LAUNCHED,
@@ -92,19 +136,32 @@ class EventWithoutData(BaseModel):
         Event.PAGE_CREATED,
         Event.PAGE_CLOSED,
     ]
+    """The event name."""
+
     data: Any = None
+    """The event payload, always empty for this event."""
 
 
 @docs_group('Events')
 class DeprecatedEvent(BaseModel):
+    """A deprecated event kept for backward compatibility (e.g. `cpuInfo`)."""
+
     name: Literal['cpuInfo']
+    """The event name."""
+
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
+    """The event payload."""
 
 
 @docs_group('Events')
 class UnknownEvent(BaseModel):
+    """A fallback for any event whose name is not recognized by the SDK."""
+
     name: str
+    """The event name."""
+
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
+    """The event payload."""
 
 
 EventMessage = PersistStateEvent | SystemInfoEvent | MigratingEvent | AbortingEvent | ExitEvent | EventWithoutData

{apify-3.4.2b4 → apify-3.4.2b6}/src/apify/scrapy/_async_thread.py

@@ -5,7 +5,7 @@ import threading
 from concurrent import futures
 from datetime import timedelta
 from logging import getLogger
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
 
 if TYPE_CHECKING:
     from collections.abc import Coroutine
@@ -14,13 +14,16 @@ logger = getLogger(__name__)
 
 
 class AsyncThread:
-    """Class for running an asyncio event loop in a separate thread.
+    """Run an asyncio event loop in a dedicated background thread.
 
-    This allows running asynchronous coroutines from synchronous code by executingthem on an event loop
-    that runs in its own dedicated thread.
+    This lets synchronous Scrapy callbacks drive asynchronous Apify and Crawlee coroutines. The
+    scheduler and the HTTP cache storage each own their own `AsyncThread`, so the request queue and
+    the key-value store never share an event loop; they only share the read-only global
+    `Configuration`. A single shared loop would also work but would couple their lifecycles.
     """
 
-    def __init__(self) -> None:
+    def __init__(self, default_timeout: timedelta = timedelta(seconds=60)) -> None:
+        self._default_timeout = default_timeout
         self._eventloop = asyncio.new_event_loop()
 
         # Start the event loop in a dedicated daemon thread.
@@ -33,7 +36,7 @@ class AsyncThread:
     def run_coro(
         self,
         coro: Coroutine,
-        timeout: timedelta = timedelta(seconds=60),
+        timeout: timedelta | Literal['default'] = 'default',
     ) -> Any:
         """Run a coroutine on an event loop running in a separate thread.
 
@@ -42,7 +45,8 @@ class AsyncThread:
 
         Args:
             coro: The coroutine to run.
-            timeout: The maximum number of seconds to wait for the coroutine to finish.
+            timeout: The maximum time to wait for the coroutine to finish. Pass `'default'` to use the
+                `default_timeout` passed to the constructor.
 
         Returns:
             The result returned by the coroutine.
@@ -52,6 +56,9 @@ class AsyncThread:
             TimeoutError: If the coroutine does not complete within the timeout.
             Exception: Any exception raised during coroutine execution.
         """
+        if timeout == 'default':
+            timeout = self._default_timeout
+
         if not self._eventloop.is_running():
             raise RuntimeError(f'The coroutine {coro} cannot be executed because the event loop is not running.')
 

apify-3.4.2b6/src/apify/scrapy/_serialization.py

@@ -0,0 +1,138 @@
+"""JSON serialization of Scrapy requests and cached responses for storage on the Apify platform.
+
+Scrapy requests and cached responses are stored in the Apify request queue and key-value store which hold JSON,
+so they are serialized as JSON here rather than pickled.
+
+Only `body` (`bytes`) and `headers` (`{bytes: [bytes]}`) are not natively JSON-serializable; both sit at fixed keys
+and are base64-encoded in place. A `str` `body` is encoded as its UTF-8 bytes and comes back as `bytes`, matching
+Scrapy, which always stores `body` as `bytes`. Pydantic models such as Crawlee's `UserData` are dumped via
+`model_dump()`. Everything else, notably `meta` and `cb_kwargs`, must already be JSON-serializable, otherwise
+serialization fails with a clear error naming the offending value. No in-band sentinel is used, so no user value
+can collide with the encoding.
+
+Known limitations of the pickle -> JSON switch (a documented breaking change): JSON has fewer types than pickle,
+so values in `meta`/`cb_kwargs` are subject to JSON's coercions. A `tuple` round-trips as a `list` and non-string
+`dict` keys round-trip as strings (e.g. `{1: 'a'}` becomes `{'1': 'a'}`). Values JSON cannot represent at all
+(`datetime`, `set`, `Decimal`, arbitrary objects, ...) are not coerced silently: serialization raises and the request
+is skipped loudly rather than stored in a corrupted form.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+from typing import Any
+
+from pydantic import BaseModel
+
+# Cap the offending value's repr in a serialization error message so a huge value cannot bloat the log.
+_MAX_ERROR_VALUE_REPR_LEN = 200
+
+
+def encode_to_json(data: dict[str, Any]) -> str:
+    """Serialize a Scrapy request/response dict to a JSON string.
+
+    The `body` and `headers` fields are base64-encoded in place (a `str` `body` via its UTF-8 bytes); pydantic
+    models are dumped to plain dicts. A `TypeError` is raised if any other value cannot be JSON-encoded.
+
+    Args:
+        data: The dict to serialize, e.g. the output of `scrapy.Request.to_dict()`.
+
+    Returns:
+        The JSON-encoded string.
+    """
+    if not isinstance(data, dict):
+        raise TypeError(f'Expected a dict to serialize, got {type(data)}')
+
+    safe = dict(data)
+
+    # `body` is base64-encoded so binary payloads survive; a `str` body is taken as its UTF-8 bytes, which keeps
+    # encode/decode symmetric (decode always base64-decodes `body` back to `bytes`).
+    body = safe.get('body')
+    if isinstance(body, (bytes, str)):
+        raw_body = body.encode('utf-8') if isinstance(body, str) else body
+        safe['body'] = base64.b64encode(raw_body).decode('ascii')
+
+    if isinstance(safe.get('headers'), dict):
+        safe['headers'] = _encode_headers(safe['headers'])
+
+    try:
+        # `ensure_ascii=False` keeps non-ASCII URLs/meta as their UTF-8 form instead of `\uXXXX` escapes, which
+        # would otherwise roughly double the size of non-Latin text in storage.
+        return json.dumps(safe, default=_json_default, ensure_ascii=False)
+    except TypeError as exc:
+        raise TypeError(
+            'Failed to JSON-serialize a Scrapy request/response for storage on the Apify platform. '
+            'All values in `meta` and `cb_kwargs` must be JSON-serializable (str, int, float, bool, None, '
+            'list, dict, or a pydantic model).'
+        ) from exc
+
+
+def decode_from_json(text: str) -> Any:
+    """Reconstruct a Scrapy request/response dict from a string produced by `encode_to_json`.
+
+    The base64-encoded `body` and `headers` fields are decoded back to their `bytes` representation.
+
+    Args:
+        text: The JSON-encoded string.
+
+    Returns:
+        The decoded object (a dict for valid request/response payloads).
+    """
+    data = json.loads(text)
+    if not isinstance(data, dict):
+        return data
+
+    # `validate=True` makes a non-base64 body raise loudly instead of silently decoding to garbage.
+    if isinstance(data.get('body'), str):
+        data['body'] = base64.b64decode(data['body'], validate=True)
+
+    if isinstance(data.get('headers'), dict):
+        data['headers'] = _decode_headers(data['headers'])
+
+    return data
+
+
+def _json_default(obj: Any) -> Any:
+    """Fallback for values `json.dumps` cannot serialize: pydantic models are dumped, anything else raises.
+
+    The error names the offending value (type and a truncated repr) so a failed serialization points straight
+    at the bad `meta`/`cb_kwargs` entry instead of just reporting that something failed.
+    """
+    if isinstance(obj, BaseModel):
+        return obj.model_dump(by_alias=True)
+    value_repr = repr(obj)
+    if len(value_repr) > _MAX_ERROR_VALUE_REPR_LEN:
+        value_repr = value_repr[:_MAX_ERROR_VALUE_REPR_LEN] + '...'
+    raise TypeError(f'Object of type {type(obj).__name__} is not JSON-serializable: {value_repr}')
+
+
+def _encode_headers(headers: dict[Any, Any]) -> dict[str, list[str]]:
+    """Encode a Scrapy `{bytes: [bytes]}` headers mapping to a JSON-safe `{str: [base64-str]}`."""
+    encoded: dict[str, list[str]] = {}
+    for key, value in headers.items():
+        str_key = key.decode('latin-1') if isinstance(key, bytes) else key
+        values = value if isinstance(value, (list, tuple)) else [value]
+        encoded[str_key] = [_b64encode_value(item) for item in values]
+    return encoded
+
+
+def _decode_headers(headers: dict[str, Any]) -> dict[bytes, list[bytes]]:
+    """Reverse `_encode_headers`, restoring the `{bytes: [bytes]}` mapping Scrapy expects."""
+    decoded: dict[bytes, list[bytes]] = {}
+    for key, value in headers.items():
+        bytes_key = key.encode('latin-1') if isinstance(key, str) else key
+        values = value if isinstance(value, list) else [value]
+        decoded[bytes_key] = [base64.b64decode(item, validate=True) for item in values]
+    return decoded
+
+
+def _b64encode_value(value: Any) -> str:
+    """Base64-encode a single header value.
+
+    Scrapy stores header values as `bytes`; a `str` is encoded as its UTF-8 bytes. Any other type is coerced with
+    `str()` as a lenient last resort. That coercion is lossy (e.g. `5` becomes `b'5'`), but Scrapy does not produce
+    non-`bytes`/`str` header values, so it is not hit on the real path.
+    """
+    raw = value if isinstance(value, bytes) else str(value).encode('utf-8')
+    return base64.b64encode(raw).decode('ascii')

{apify-3.4.2b4 → apify-3.4.2b6}/src/apify/scrapy/extensions/_httpcache.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import gzip
 import io
-import pickle
 import re
 import struct
 from logging import getLogger
@@ -14,6 +13,7 @@ from scrapy.responsetypes import responsetypes
 
 from apify import Configuration
 from apify.scrapy._async_thread import AsyncThread
+from apify.scrapy._serialization import decode_from_json, encode_to_json
 from apify.storage_clients import ApifyStorageClient
 from apify.storages import KeyValueStore
 
@@ -29,14 +29,14 @@ logger = getLogger(__name__)
 class ApifyCacheStorage:
     """A Scrapy cache storage that uses the Apify `KeyValueStore` to store responses.
 
-    It can be set as a storage for Scrapy's built-in `HttpCacheMiddleware`, which caches
-    responses to requests. See HTTPCache middleware settings (prefixed with `HTTPCACHE_`)
-    in the Scrapy documentation for more information. Requires the asyncio Twisted reactor
-    to be installed.
+    It can be set as a storage for Scrapy's built-in `HttpCacheMiddleware`, which caches responses to requests.
+    See HTTPCache middleware settings (prefixed with `HTTPCACHE_`) in the Scrapy documentation for more information.
+    Requires the asyncio Twisted reactor to be installed.
     """
 
     def __init__(self, settings: BaseSettings) -> None:
-        self._expiration_max_items = 100
+        # Upper bound on how many keys the per-spider-close cleanup sweeps (best-effort; `close_spider`).
+        self._expiration_max_items: int = settings.getint('APIFY_HTTPCACHE_EXPIRATION_MAX_ITEMS', 100)
         self._expiration_secs: int = settings.getint('HTTPCACHE_EXPIRATION_SECS')
         self._spider: Spider | None = None
         self._kvs: KeyValueStore | None = None
@@ -79,23 +79,26 @@ class ApifyCacheStorage:
             async def expire_kvs() -> None:
                 if self._kvs is None:
                     raise ValueError('Key value store not initialized')
-                i = 0
+                # Best-effort cleanup: at most `_expiration_max_items` keys per close, in no guaranteed order,
+                # so stale entries may linger. This only reclaims storage; `retrieve_response` already treats
+                # an expired entry as a cache miss.
+                processed = 0
                 async for item in self._kvs.iterate_keys():
+                    if processed >= self._expiration_max_items:
+                        break
+                    processed += 1
                     value = await self._kvs.get_value(item.key)
                     try:
                         gzip_time = read_gzip_time(value)
                     except Exception as e:
                         logger.warning(f'Malformed cache item {item.key}: {e}')
-                        await self._kvs.set_value(item.key, None)
+                        await self._kvs.delete_value(item.key)
                     else:
                         if self._expiration_secs < current_time - gzip_time:
                             logger.debug(f'Expired cache item {item.key}')
-                            await self._kvs.set_value(item.key, None)
+                            await self._kvs.delete_value(item.key)
                         else:
                             logger.debug(f'Valid cache item {item.key}')
-                    if i == self._expiration_max_items:
-                        break
-                    i += 1
 
             self._async_thread.run_coro(expire_kvs())
 
@@ -127,17 +130,25 @@ class ApifyCacheStorage:
 
         if current_time is None:
             current_time = int(time())
-        if 0 < self._expiration_secs < current_time - read_gzip_time(value):
-            logger.debug('Cache expired', extra={'request': request})
+
+        # A malformed or legacy cache entry must not crash retrieval; treat it as a cache miss so Scrapy re-fetches
+        # and re-stores it in the current format. The field reads stay inside the `try` as well: a value that decodes
+        # to a dict missing any expected key (a forward/older format, or a truncated-but-valid JSON payload) must
+        # also degrade to a miss rather than raising an uncaught `KeyError`.
+        try:
+            if 0 < self._expiration_secs < current_time - read_gzip_time(value):
+                logger.debug('Cache expired', extra={'request': request})
+                return None
+            data = from_gzip(value)
+            url = data['url']
+            status = data['status']
+            headers = Headers(data['headers'])
+            body = data['body']
+        except Exception as exc:
+            logger.warning(f'Ignoring malformed cache entry {key!r}: {exc}', extra={'request': request})
             return None
 
-        data = from_gzip(value)
-        url = data['url']
-        status = data['status']
-        headers = Headers(data['headers'])
-        body = data['body']
         respcls = responsetypes.from_args(headers=headers, url=url, body=body)
-
         logger.debug('Cache hit', extra={'request': request})
         return respcls(url=url, headers=headers, status=status, body=body)
 
@@ -162,18 +173,25 @@ class ApifyCacheStorage:
 
 
 def to_gzip(data: dict, mtime: int | None = None) -> bytes:
-    """Dump a dictionary to a gzip-compressed byte stream."""
+    """Dump a dictionary to a gzip-compressed JSON byte stream.
+
+    Cache entries live in the Apify key-value store, which holds JSON, so they are serialized as JSON rather
+    than pickled. See `apify.scrapy._serialization` for the encoding.
+    """
+    payload = encode_to_json(data).encode('utf-8')
     with io.BytesIO() as byte_stream:
         with gzip.GzipFile(fileobj=byte_stream, mode='wb', mtime=mtime) as gzip_file:
-            pickle.dump(data, gzip_file, protocol=4)
+            gzip_file.write(payload)
         return byte_stream.getvalue()
 
 
 def from_gzip(gzip_bytes: bytes) -> dict:
-    """Load a dictionary from a gzip-compressed byte stream."""
+    """Load a dictionary from a gzip-compressed JSON byte stream."""
     with io.BytesIO(gzip_bytes) as byte_stream, gzip.GzipFile(fileobj=byte_stream, mode='rb') as gzip_file:
-        data: dict = pickle.load(gzip_file)
-        return data
+        data = decode_from_json(gzip_file.read().decode('utf-8'))
+    if not isinstance(data, dict):
+        raise TypeError(f'Expected a dict from the cached payload, got {type(data)}')
+    return data
 
 
 def read_gzip_time(gzip_bytes: bytes) -> int:
@@ -187,17 +205,15 @@ def read_gzip_time(gzip_bytes: bytes) -> int:
 def get_kvs_name(spider_name: str, max_length: int = 60) -> str:
     """Get the key value store name for a spider.
 
-    The key value store name is derived from the spider name by replacing all special characters
-    with hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with
-    'httpcache-' and truncated to the maximum length.
+    The key value store name is derived from the spider name by replacing all special characters with hyphens
+    and trimming leading and trailing hyphens. The resulting name is prefixed with 'httpcache-' and truncated
+    to the maximum length.
 
-    The documentation
-    [about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)
+    The documentation [about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)
     mentions that names can be up to 63 characters long, so the default max length is set to 60.
 
-    Such naming isn't unique per spider, but should be sufficiently unique for most use cases.
-    The name of the key value store should indicate to which spider it belongs, e.g. in
-    the listing in the Apify's console.
+    Such naming isn't unique per spider, but should be sufficiently unique for most use cases. The name
+    of the key-value store should indicate to which spider it belongs, e.g. in the listing in the Apify's console.
 
     Args:
         spider_name: Value of the Spider instance's name attribute.