apify 3.4.2b5__tar.gz → 3.4.2b7__tar.gz

{apify-3.4.2b5 → apify-3.4.2b7}/CHANGELOG.md

@@ -9,6 +9,8 @@ All notable changes to this project will be documented in this file.
 
 - **scrapy:** Correct proxy middleware exception log and import ([#953](https://github.com/apify/apify-sdk-python/pull/953)) ([5bd6eb9](https://github.com/apify/apify-sdk-python/commit/5bd6eb9843d90844cec083372e932413bceedec9)) by [@vdusek](https://github.com/vdusek)
 - **scrapy:** Skip a request that fails to convert instead of crashing the run ([#952](https://github.com/apify/apify-sdk-python/pull/952)) ([db9444f](https://github.com/apify/apify-sdk-python/commit/db9444faeb0158c29aa394121cf733ff2e843f28)) by [@vdusek](https://github.com/vdusek)
+- **scrapy:** [**breaking**] Serialize requests and HTTP cache as JSON instead of pickle ([#951](https://github.com/apify/apify-sdk-python/pull/951)) ([a87e8d1](https://github.com/apify/apify-sdk-python/commit/a87e8d1597478b4f12fd5bb9b379f65f637d8e96)) by [@vdusek](https://github.com/vdusek)
+- **scrapy:** Make logging configuration idempotent ([#954](https://github.com/apify/apify-sdk-python/pull/954)) ([2cc5602](https://github.com/apify/apify-sdk-python/commit/2cc5602b741b93c81f264d4e09e0d9bcfc7200f2)) by [@vdusek](https://github.com/vdusek)
 
 ### 🚜 Refactor
 

{apify-3.4.2b5 → apify-3.4.2b7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 3.4.2b5
+Version: 3.4.2b7
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog

{apify-3.4.2b5 → apify-3.4.2b7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apify"
-version = "3.4.2b5"
+version = "3.4.2b7"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{apify-3.4.2b5 → apify-3.4.2b7}/src/apify/_actor.py

@@ -699,7 +699,15 @@ class _ActorType:
 
     @_ensure_context
     async def get_input(self) -> Any:
-        """Get the Actor input value from the default key-value store associated with the current Actor run."""
+        """Get the Actor input value from the default key-value store associated with the current Actor run.
+
+        The input is the deserialized contents of the input record (the `INPUT` key by default), so it is typically
+        a `dict` keyed by the fields declared in the Actor's input schema. Any secret input fields are decrypted to
+        plaintext before being returned.
+
+        Returns:
+            The Actor input, usually a `dict` of input fields, or `None` if the Actor has no input.
+        """
         input_value = await self.get_value(self.configuration.input_key)
         input_secrets_private_key = self.configuration.input_secrets_private_key_file
         input_secrets_key_passphrase = self.configuration.input_secrets_private_key_passphrase

{apify-3.4.2b5 → apify-3.4.2b7}/src/apify/_charging.py

@@ -5,9 +5,10 @@ from contextvars import ContextVar
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from decimal import Decimal
-from typing import TYPE_CHECKING, Annotated, Literal, Protocol, TypedDict
+from typing import TYPE_CHECKING, Literal, Protocol, TypedDict
 
-from pydantic import Field
+from pydantic import ConfigDict
+from pydantic.alias_generators import to_camel
 
 import apify_client._models as _client_models
 from apify_client._models import ActorChargeEvent as ClientActorChargeEvent
@@ -56,7 +57,9 @@ _ensure_context = ensure_context('active')
 class ActorChargeEvent(ClientActorChargeEvent):
     """Definition of a single chargeable event in the pay-per-event pricing model."""
 
-    event_description: Annotated[str | None, Field(alias='eventDescription')] = None
+    model_config = ConfigDict(alias_generator=to_camel)
+
+    event_description: str | None = None
     """Human-readable description of the event.
 
     Required in apify-client but omitted from the env var, so it is relaxed to optional.
@@ -67,7 +70,9 @@ class ActorChargeEvent(ClientActorChargeEvent):
 class PricingPerEvent(ClientPricingPerEvent):
     """Pay-per-event pricing details - the chargeable events and their prices."""
 
-    actor_charge_events: Annotated[dict[str, ActorChargeEvent] | None, Field(alias='actorChargeEvents')] = None
+    model_config = ConfigDict(alias_generator=to_camel)
+
+    actor_charge_events: dict[str, ActorChargeEvent] | None = None
     """Mapping of event name to its charge definition."""
 
 
@@ -75,13 +80,15 @@ class PricingPerEvent(ClientPricingPerEvent):
 class FreeActorPricingInfo(ClientFree):
     """Pricing info for an Actor offered free of charge."""
 
-    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    model_config = ConfigDict(alias_generator=to_camel)
+
+    apify_margin_percentage: float | None = None
     """Apify's margin on the price, as a percentage."""
 
-    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    created_at: datetime | None = None
     """Timestamp when this pricing info was created."""
 
-    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    started_at: datetime | None = None
     """Timestamp when this pricing became effective."""
 
 
@@ -89,19 +96,21 @@ class FreeActorPricingInfo(ClientFree):
 class FlatPricePerMonthActorPricingInfo(ClientFlatPricePerMonth):
     """Pricing info for an Actor billed at a flat monthly price."""
 
-    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    model_config = ConfigDict(alias_generator=to_camel)
+
+    apify_margin_percentage: float | None = None
     """Apify's margin on the price, as a percentage."""
 
-    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    created_at: datetime | None = None
     """Timestamp when this pricing info was created."""
 
-    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    started_at: datetime | None = None
     """Timestamp when this pricing became effective."""
 
-    trial_minutes: Annotated[int | None, Field(alias='trialMinutes')] = None
+    trial_minutes: int | None = None
     """Length of the free trial period, in minutes."""
 
-    price_per_unit_usd: Annotated[float | None, Field(alias='pricePerUnitUsd')] = None
+    price_per_unit_usd: float | None = None
     """Price per unit, in USD."""
 
 
@@ -109,16 +118,18 @@ class FlatPricePerMonthActorPricingInfo(ClientFlatPricePerMonth):
 class PricePerDatasetItemActorPricingInfo(ClientPricePerDatasetItem):
     """Pricing info for an Actor billed per dataset item produced."""
 
-    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    model_config = ConfigDict(alias_generator=to_camel)
+
+    apify_margin_percentage: float | None = None
     """Apify's margin on the price, as a percentage."""
 
-    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    created_at: datetime | None = None
     """Timestamp when this pricing info was created."""
 
-    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    started_at: datetime | None = None
     """Timestamp when this pricing became effective."""
 
-    unit_name: Annotated[str | None, Field(alias='unitName')] = None
+    unit_name: str | None = None
     """Name of the billed unit."""
 
 
@@ -126,16 +137,18 @@ class PricePerDatasetItemActorPricingInfo(ClientPricePerDatasetItem):
 class PayPerEventActorPricingInfo(ClientPayPerEvent):
     """Pricing info for an Actor billed per charged event."""
 
-    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    model_config = ConfigDict(alias_generator=to_camel)
+
+    apify_margin_percentage: float | None = None
     """Apify's margin on the price, as a percentage."""
 
-    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    created_at: datetime | None = None
     """Timestamp when this pricing info was created."""
 
-    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    started_at: datetime | None = None
     """Timestamp when this pricing became effective."""
 
-    pricing_per_event: Annotated[PricingPerEvent, Field(alias='pricingPerEvent')]
+    pricing_per_event: PricingPerEvent
     """The pay-per-event pricing details."""
 
 

{apify-3.4.2b5 → apify-3.4.2b7}/src/apify/events/_types.py

@@ -3,7 +3,8 @@ from __future__ import annotations
 from datetime import datetime
 from typing import Annotated, Any, Literal
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.alias_generators import to_camel
 
 from crawlee.events._types import (
     Event,
@@ -29,28 +30,30 @@ This is the Apify-specific subset of [`Event`][crawlee.events.Event] — for the
 class SystemInfoEventData(BaseModel):
     """Resource usage metrics carried by a `systemInfo` event."""
 
-    mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
+    mem_avg_bytes: float
     """Average memory usage over the measured interval, in bytes."""
 
-    mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
+    mem_current_bytes: float
     """Current memory usage, in bytes."""
 
-    mem_max_bytes: Annotated[float, Field(alias='memMaxBytes')]
+    mem_max_bytes: float
     """Peak memory usage observed so far, in bytes."""
 
-    cpu_avg_usage: Annotated[float, Field(alias='cpuAvgUsage')]
+    cpu_avg_usage: float
     """Average CPU usage over the measured interval, in percent."""
 
-    cpu_max_usage: Annotated[float, Field(alias='cpuMaxUsage')]
+    cpu_max_usage: float
     """Peak CPU usage observed so far, in percent."""
 
-    cpu_current_usage: Annotated[float, Field(alias='cpuCurrentUsage')]
+    cpu_current_usage: float
     """Current CPU usage, in percent."""
 
-    is_cpu_overloaded: Annotated[bool, Field(alias='isCpuOverloaded')]
+    is_cpu_overloaded: bool
     """Whether the CPU is currently overloaded."""
 
-    created_at: Annotated[datetime, Field(alias='createdAt')]
+    created_at: datetime
     """Timestamp when the metrics were collected."""
 
     def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
@@ -73,6 +76,8 @@ class SystemInfoEventData(BaseModel):
 class PersistStateEvent(BaseModel):
     """A `persistState` event instructing the Actor to persist its state."""
 
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
     name: Literal[Event.PERSIST_STATE]
     """The event name."""
 
@@ -84,6 +89,8 @@ class PersistStateEvent(BaseModel):
 class SystemInfoEvent(BaseModel):
     """A `systemInfo` event carrying the Actor's resource usage metrics."""
 
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
     name: Literal[Event.SYSTEM_INFO]
     """The event name."""
 
@@ -95,6 +102,8 @@ class SystemInfoEvent(BaseModel):
 class MigratingEvent(BaseModel):
     """A `migrating` event signalling the Actor is about to be migrated to another host."""
 
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
     name: Literal[Event.MIGRATING]
     """The event name."""
 
@@ -106,6 +115,8 @@ class MigratingEvent(BaseModel):
 class AbortingEvent(BaseModel):
     """An `aborting` event signalling the Actor run is being aborted."""
 
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
     name: Literal[Event.ABORTING]
     """The event name."""
 
@@ -117,6 +128,8 @@ class AbortingEvent(BaseModel):
 class ExitEvent(BaseModel):
     """An `exit` event signalling the Actor process is about to exit."""
 
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
     name: Literal[Event.EXIT]
     """The event name."""
 
@@ -128,6 +141,8 @@ class ExitEvent(BaseModel):
 class EventWithoutData(BaseModel):
     """A framework-level event that carries no payload (e.g. browser and page lifecycle events)."""
 
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
     name: Literal[
         Event.SESSION_RETIRED,
         Event.BROWSER_LAUNCHED,
@@ -146,6 +161,8 @@ class EventWithoutData(BaseModel):
 class DeprecatedEvent(BaseModel):
     """A deprecated event kept for backward compatibility (e.g. `cpuInfo`)."""
 
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
     name: Literal['cpuInfo']
     """The event name."""
 
@@ -157,6 +174,8 @@ class DeprecatedEvent(BaseModel):
 class UnknownEvent(BaseModel):
     """A fallback for any event whose name is not recognized by the SDK."""
 
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
     name: str
     """The event name."""
 

{apify-3.4.2b5 → apify-3.4.2b7}/src/apify/request_loaders/_apify_request_list.py

@@ -5,7 +5,8 @@ import re
 from itertools import chain
 from typing import Annotated, Any
 
-from pydantic import BaseModel, Field, TypeAdapter
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
+from pydantic.alias_generators import to_camel
 
 from crawlee._types import HttpMethod
 from crawlee.http_clients import HttpClient, ImpitHttpClient
@@ -20,14 +21,16 @@ URL_NO_COMMAS_REGEX = re.compile(
 
 
 class _RequestDetails(BaseModel):
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
     method: HttpMethod = 'GET'
     payload: str = ''
     headers: Annotated[dict[str, str], Field(default_factory=dict)]
-    user_data: Annotated[dict[str, str], Field(default_factory=dict, alias='userData')]
+    user_data: Annotated[dict[str, str], Field(default_factory=dict)]
 
 
 class _RequestsFromUrlInput(_RequestDetails):
-    requests_from_url: str = Field(alias='requestsFromUrl')
+    requests_from_url: str
 
 
 class _SimpleUrlInput(_RequestDetails):

{apify-3.4.2b5 → apify-3.4.2b7}/src/apify/scrapy/_logging_config.py

@@ -13,6 +13,11 @@ _PRIMARY_LOGGERS = ['apify', 'apify_client', 'scrapy']
 _SUPPLEMENTAL_LOGGERS = ['filelock', 'hpack', 'httpcore', 'protego', 'twisted']
 _ALL_LOGGERS = _PRIMARY_LOGGERS + _SUPPLEMENTAL_LOGGERS
 
+# Mutable state shared with the Scrapy monkey-patch below. `initialize_logging` refreshes
+# `level`/`handler` on each call; the patch (installed once) reads them so it always applies the
+# latest configuration rather than values captured the first time it ran.
+_state: dict[str, Any] = {'level': 'INFO', 'handler': None, 'patched': False}
+
 
 def _configure_logger(name: str | None, logging_level: str, handler: logging.Handler) -> None:
     """Clear and reconfigure the logger."""
@@ -23,26 +28,39 @@ def _configure_logger(name: str | None, logging_level: str, handler: logging.Han
     logger.propagate = False
 
 
+def _configure_all_loggers() -> None:
+    """Apply the Apify handler and level to the root logger and all defined loggers."""
+    handler = _state['handler']
+    if handler is None:
+        return
+    for logger_name in [None, *_ALL_LOGGERS]:
+        _configure_logger(logger_name, _state['level'], handler)
+
+
 def initialize_logging() -> None:
     """Configure logging for Apify Actors and adjust Scrapy's logging settings."""
     # Retrieve Scrapy project settings and determine the logging level.
     settings = get_project_settings()
-    logging_level = settings.get('LOG_LEVEL', 'INFO')  # Default to INFO.
+    _state['level'] = settings.get('LOG_LEVEL', 'INFO')  # Default to INFO.
 
     # Create a custom handler with the Apify log formatter.
     handler = logging.StreamHandler()
     handler.setFormatter(ActorLogFormatter(include_logger_name=True))
+    _state['handler'] = handler
 
     # Configure the root logger and all other defined loggers.
-    for logger_name in [None, *_ALL_LOGGERS]:
-        _configure_logger(logger_name, logging_level, handler)
+    _configure_all_loggers()
+
+    # Monkey-patch Scrapy's logging to re-apply our settings whenever it reconfigures logging.
+    # Install the wrapper at most once, otherwise repeated calls would nest wrappers.
+    if _state['patched']:
+        return
 
-    # Monkey-patch Scrapy's logging configuration to re-apply our settings.
     original_configure_logging = scrapy_logging.configure_logging
 
     def new_configure_logging(*args: Any, **kwargs: Any) -> None:
         original_configure_logging(*args, **kwargs)
-        for logger_name in [None, *_ALL_LOGGERS]:
-            _configure_logger(logger_name, logging_level, handler)
+        _configure_all_loggers()
 
     scrapy_logging.configure_logging = new_configure_logging  # ty: ignore[invalid-assignment]
+    _state['patched'] = True

apify-3.4.2b7/src/apify/scrapy/_serialization.py

@@ -0,0 +1,138 @@
+"""JSON serialization of Scrapy requests and cached responses for storage on the Apify platform.
+
+Scrapy requests and cached responses are stored in the Apify request queue and key-value store which hold JSON,
+so they are serialized as JSON here rather than pickled.
+
+Only `body` (`bytes`) and `headers` (`{bytes: [bytes]}`) are not natively JSON-serializable; both sit at fixed keys
+and are base64-encoded in place. A `str` `body` is encoded as its UTF-8 bytes and comes back as `bytes`, matching
+Scrapy, which always stores `body` as `bytes`. Pydantic models such as Crawlee's `UserData` are dumped via
+`model_dump()`. Everything else, notably `meta` and `cb_kwargs`, must already be JSON-serializable, otherwise
+serialization fails with a clear error naming the offending value. No in-band sentinel is used, so no user value
+can collide with the encoding.
+
+Known limitations of the pickle -> JSON switch (a documented breaking change): JSON has fewer types than pickle,
+so values in `meta`/`cb_kwargs` are subject to JSON's coercions. A `tuple` round-trips as a `list` and non-string
+`dict` keys round-trip as strings (e.g. `{1: 'a'}` becomes `{'1': 'a'}`). Values JSON cannot represent at all
+(`datetime`, `set`, `Decimal`, arbitrary objects, ...) are not coerced silently: serialization raises and the request
+is skipped loudly rather than stored in a corrupted form.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+from typing import Any
+
+from pydantic import BaseModel
+
+# Cap the offending value's repr in a serialization error message so a huge value cannot bloat the log.
+_MAX_ERROR_VALUE_REPR_LEN = 200
+
+
+def encode_to_json(data: dict[str, Any]) -> str:
+    """Serialize a Scrapy request/response dict to a JSON string.
+
+    The `body` and `headers` fields are base64-encoded in place (a `str` `body` via its UTF-8 bytes); pydantic
+    models are dumped to plain dicts. A `TypeError` is raised if any other value cannot be JSON-encoded.
+
+    Args:
+        data: The dict to serialize, e.g. the output of `scrapy.Request.to_dict()`.
+
+    Returns:
+        The JSON-encoded string.
+    """
+    if not isinstance(data, dict):
+        raise TypeError(f'Expected a dict to serialize, got {type(data)}')
+
+    safe = dict(data)
+
+    # `body` is base64-encoded so binary payloads survive; a `str` body is taken as its UTF-8 bytes, which keeps
+    # encode/decode symmetric (decode always base64-decodes `body` back to `bytes`).
+    body = safe.get('body')
+    if isinstance(body, (bytes, str)):
+        raw_body = body.encode('utf-8') if isinstance(body, str) else body
+        safe['body'] = base64.b64encode(raw_body).decode('ascii')
+
+    if isinstance(safe.get('headers'), dict):
+        safe['headers'] = _encode_headers(safe['headers'])
+
+    try:
+        # `ensure_ascii=False` keeps non-ASCII URLs/meta as their UTF-8 form instead of `\uXXXX` escapes, which
+        # would otherwise roughly double the size of non-Latin text in storage.
+        return json.dumps(safe, default=_json_default, ensure_ascii=False)
+    except TypeError as exc:
+        raise TypeError(
+            'Failed to JSON-serialize a Scrapy request/response for storage on the Apify platform. '
+            'All values in `meta` and `cb_kwargs` must be JSON-serializable (str, int, float, bool, None, '
+            'list, dict, or a pydantic model).'
+        ) from exc
+
+
+def decode_from_json(text: str) -> Any:
+    """Reconstruct a Scrapy request/response dict from a string produced by `encode_to_json`.
+
+    The base64-encoded `body` and `headers` fields are decoded back to their `bytes` representation.
+
+    Args:
+        text: The JSON-encoded string.
+
+    Returns:
+        The decoded object (a dict for valid request/response payloads).
+    """
+    data = json.loads(text)
+    if not isinstance(data, dict):
+        return data
+
+    # `validate=True` makes a non-base64 body raise loudly instead of silently decoding to garbage.
+    if isinstance(data.get('body'), str):
+        data['body'] = base64.b64decode(data['body'], validate=True)
+
+    if isinstance(data.get('headers'), dict):
+        data['headers'] = _decode_headers(data['headers'])
+
+    return data
+
+
+def _json_default(obj: Any) -> Any:
+    """Fallback for values `json.dumps` cannot serialize: pydantic models are dumped, anything else raises.
+
+    The error names the offending value (type and a truncated repr) so a failed serialization points straight
+    at the bad `meta`/`cb_kwargs` entry instead of just reporting that something failed.
+    """
+    if isinstance(obj, BaseModel):
+        return obj.model_dump(by_alias=True)
+    value_repr = repr(obj)
+    if len(value_repr) > _MAX_ERROR_VALUE_REPR_LEN:
+        value_repr = value_repr[:_MAX_ERROR_VALUE_REPR_LEN] + '...'
+    raise TypeError(f'Object of type {type(obj).__name__} is not JSON-serializable: {value_repr}')
+
+
+def _encode_headers(headers: dict[Any, Any]) -> dict[str, list[str]]:
+    """Encode a Scrapy `{bytes: [bytes]}` headers mapping to a JSON-safe `{str: [base64-str]}`."""
+    encoded: dict[str, list[str]] = {}
+    for key, value in headers.items():
+        str_key = key.decode('latin-1') if isinstance(key, bytes) else key
+        values = value if isinstance(value, (list, tuple)) else [value]
+        encoded[str_key] = [_b64encode_value(item) for item in values]
+    return encoded
+
+
+def _decode_headers(headers: dict[str, Any]) -> dict[bytes, list[bytes]]:
+    """Reverse `_encode_headers`, restoring the `{bytes: [bytes]}` mapping Scrapy expects."""
+    decoded: dict[bytes, list[bytes]] = {}
+    for key, value in headers.items():
+        bytes_key = key.encode('latin-1') if isinstance(key, str) else key
+        values = value if isinstance(value, list) else [value]
+        decoded[bytes_key] = [base64.b64decode(item, validate=True) for item in values]
+    return decoded
+
+
+def _b64encode_value(value: Any) -> str:
+    """Base64-encode a single header value.
+
+    Scrapy stores header values as `bytes`; a `str` is encoded as its UTF-8 bytes. Any other type is coerced with
+    `str()` as a lenient last resort. That coercion is lossy (e.g. `5` becomes `b'5'`), but Scrapy does not produce
+    non-`bytes`/`str` header values, so it is not hit on the real path.
+    """
+    raw = value if isinstance(value, bytes) else str(value).encode('utf-8')
+    return base64.b64encode(raw).decode('ascii')