apify 3.4.2b4__tar.gz → 3.4.2b5__tar.gz

{apify-3.4.2b4 → apify-3.4.2b5}/CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 <!-- git-cliff-unreleased-start -->
 ## 3.4.2 - **not yet released**
 
+### 🐛 Bug Fixes
+
+- **scrapy:** Correct proxy middleware exception log and import ([#953](https://github.com/apify/apify-sdk-python/pull/953)) ([5bd6eb9](https://github.com/apify/apify-sdk-python/commit/5bd6eb9843d90844cec083372e932413bceedec9)) by [@vdusek](https://github.com/vdusek)
+- **scrapy:** Skip a request that fails to convert instead of crashing the run ([#952](https://github.com/apify/apify-sdk-python/pull/952)) ([db9444f](https://github.com/apify/apify-sdk-python/commit/db9444faeb0158c29aa394121cf733ff2e843f28)) by [@vdusek](https://github.com/vdusek)
+
 ### 🚜 Refactor
 
 - [**breaking**] Remove deprecated APIs ([#918](https://github.com/apify/apify-sdk-python/pull/918)) ([3e5728d](https://github.com/apify/apify-sdk-python/commit/3e5728d94cb8fd879d5a76e33a03d55792d835d5)) by [@vdusek](https://github.com/vdusek), closes [#635](https://github.com/apify/apify-sdk-python/issues/635)

{apify-3.4.2b4 → apify-3.4.2b5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 3.4.2b4
+Version: 3.4.2b5
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog

{apify-3.4.2b4 → apify-3.4.2b5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apify"
-version = "3.4.2b4"
+version = "3.4.2b5"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{apify-3.4.2b4 → apify-3.4.2b5}/src/apify/_charging.py

@@ -7,7 +7,7 @@ from datetime import UTC, datetime
 from decimal import Decimal
 from typing import TYPE_CHECKING, Annotated, Literal, Protocol, TypedDict
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import Field
 
 import apify_client._models as _client_models
 from apify_client._models import ActorChargeEvent as ClientActorChargeEvent
@@ -28,14 +28,17 @@ if TYPE_CHECKING:
 
     from apify._configuration import Configuration
 
-PricingModel = Literal['PAY_PER_EVENT', 'PRICE_PER_DATASET_ITEM', 'FLAT_PRICE_PER_MONTH', 'FREE']
-"""Pricing model for an Actor."""
+charging_manager_ctx: ContextVar[ChargingManager | None] = ContextVar('charging_manager_ctx', default=None)
+"""Holds the current `ChargingManager` instance, if any.
+
+Allows PPE-aware dataset clients to access the charging manager without needing to pass it explicitly.
+"""
 
 DEFAULT_DATASET_ITEM_EVENT = 'apify-default-dataset-item'
+"""Name of the synthetic event charged for each item pushed to the default dataset."""
 
-# Context variable to hold the current `ChargingManager` instance, if any. This allows PPE-aware dataset clients to
-# access the charging manager without needing to pass it explicitly.
-charging_manager_ctx: ContextVar[ChargingManager | None] = ContextVar('charging_manager_ctx', default=None)
+PricingModel = Literal['PAY_PER_EVENT', 'PRICE_PER_DATASET_ITEM', 'FLAT_PRICE_PER_MONTH', 'FREE']
+"""Pricing model for an Actor."""
 
 _ensure_context = ensure_context('active')
 
@@ -49,48 +52,91 @@ _ensure_context = ensure_context('active')
 # `apify-client` instance) flows through the same code paths without conversion.
 
 
-class _RelaxedPricingMetadata(BaseModel):
-    """Mixin relaxing the `CommonActorPricingInfo` metadata fields the platform env var omits."""
-
-    model_config = ConfigDict(populate_by_name=True, extra='allow')
-
-    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
-    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
-    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
-
-
 @docs_group('Charging')
 class ActorChargeEvent(ClientActorChargeEvent):
-    # `event_description` is required in apify-client but omitted from the env var.
+    """Definition of a single chargeable event in the pay-per-event pricing model."""
+
     event_description: Annotated[str | None, Field(alias='eventDescription')] = None
+    """Human-readable description of the event.
+
+    Required in apify-client but omitted from the env var, so it is relaxed to optional.
+    """
 
 
 @docs_group('Charging')
 class PricingPerEvent(ClientPricingPerEvent):
+    """Pay-per-event pricing details - the chargeable events and their prices."""
+
     actor_charge_events: Annotated[dict[str, ActorChargeEvent] | None, Field(alias='actorChargeEvents')] = None
+    """Mapping of event name to its charge definition."""
 
 
 @docs_group('Charging')
-class FreeActorPricingInfo(_RelaxedPricingMetadata, ClientFree):
-    pass
+class FreeActorPricingInfo(ClientFree):
+    """Pricing info for an Actor offered free of charge."""
+
+    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    """Apify's margin on the price, as a percentage."""
+
+    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    """Timestamp when this pricing info was created."""
+
+    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    """Timestamp when this pricing became effective."""
 
 
 @docs_group('Charging')
-class FlatPricePerMonthActorPricingInfo(_RelaxedPricingMetadata, ClientFlatPricePerMonth):
+class FlatPricePerMonthActorPricingInfo(ClientFlatPricePerMonth):
+    """Pricing info for an Actor billed at a flat monthly price."""
+
+    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    """Apify's margin on the price, as a percentage."""
+
+    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    """Timestamp when this pricing info was created."""
+
+    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    """Timestamp when this pricing became effective."""
+
     trial_minutes: Annotated[int | None, Field(alias='trialMinutes')] = None
+    """Length of the free trial period, in minutes."""
+
     price_per_unit_usd: Annotated[float | None, Field(alias='pricePerUnitUsd')] = None
+    """Price per unit, in USD."""
 
 
 @docs_group('Charging')
-class PricePerDatasetItemActorPricingInfo(_RelaxedPricingMetadata, ClientPricePerDatasetItem):
+class PricePerDatasetItemActorPricingInfo(ClientPricePerDatasetItem):
+    """Pricing info for an Actor billed per dataset item produced."""
+
+    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    """Apify's margin on the price, as a percentage."""
+
+    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    """Timestamp when this pricing info was created."""
+
+    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    """Timestamp when this pricing became effective."""
+
     unit_name: Annotated[str | None, Field(alias='unitName')] = None
-    # `price_per_unit_usd` is already optional in apify-client - inherited.
+    """Name of the billed unit."""
 
 
 @docs_group('Charging')
-class PayPerEventActorPricingInfo(_RelaxedPricingMetadata, ClientPayPerEvent):
-    # Re-typed to the relaxed element so an omitted `eventDescription` validates; the field stays required.
+class PayPerEventActorPricingInfo(ClientPayPerEvent):
+    """Pricing info for an Actor billed per charged event."""
+
+    apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
+    """Apify's margin on the price, as a percentage."""
+
+    created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
+    """Timestamp when this pricing info was created."""
+
+    started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
+    """Timestamp when this pricing became effective."""
+
     pricing_per_event: Annotated[PricingPerEvent, Field(alias='pricingPerEvent')]
+    """The pay-per-event pricing details."""
 
 
 ActorPricingInfoModel = ClientFree | ClientFlatPricePerMonth | ClientPricePerDatasetItem | ClientPayPerEvent

{apify-3.4.2b4 → apify-3.4.2b5}/src/apify/events/_types.py

@@ -27,14 +27,31 @@ This is the Apify-specific subset of [`Event`][crawlee.events.Event] — for the
 
 @docs_group('Event data')
 class SystemInfoEventData(BaseModel):
+    """Resource usage metrics carried by a `systemInfo` event."""
+
     mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
+    """Average memory usage over the measured interval, in bytes."""
+
     mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
+    """Current memory usage, in bytes."""
+
     mem_max_bytes: Annotated[float, Field(alias='memMaxBytes')]
+    """Peak memory usage observed so far, in bytes."""
+
     cpu_avg_usage: Annotated[float, Field(alias='cpuAvgUsage')]
+    """Average CPU usage over the measured interval, in percent."""
+
     cpu_max_usage: Annotated[float, Field(alias='cpuMaxUsage')]
+    """Peak CPU usage observed so far, in percent."""
+
     cpu_current_usage: Annotated[float, Field(alias='cpuCurrentUsage')]
+    """Current CPU usage, in percent."""
+
     is_cpu_overloaded: Annotated[bool, Field(alias='isCpuOverloaded')]
+    """Whether the CPU is currently overloaded."""
+
     created_at: Annotated[datetime, Field(alias='createdAt')]
+    """Timestamp when the metrics were collected."""
 
     def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
         return EventSystemInfoData.model_validate(
@@ -54,36 +71,63 @@ class SystemInfoEventData(BaseModel):
 
 @docs_group('Events')
 class PersistStateEvent(BaseModel):
+    """A `persistState` event instructing the Actor to persist its state."""
+
     name: Literal[Event.PERSIST_STATE]
+    """The event name."""
+
     data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
+    """The event payload."""
 
 
 @docs_group('Events')
 class SystemInfoEvent(BaseModel):
+    """A `systemInfo` event carrying the Actor's resource usage metrics."""
+
     name: Literal[Event.SYSTEM_INFO]
+    """The event name."""
+
     data: SystemInfoEventData
+    """The event payload."""
 
 
 @docs_group('Events')
 class MigratingEvent(BaseModel):
+    """A `migrating` event signalling the Actor is about to be migrated to another host."""
+
     name: Literal[Event.MIGRATING]
+    """The event name."""
+
     data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
+    """The event payload."""
 
 
 @docs_group('Events')
 class AbortingEvent(BaseModel):
+    """An `aborting` event signalling the Actor run is being aborted."""
+
     name: Literal[Event.ABORTING]
+    """The event name."""
+
     data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
+    """The event payload."""
 
 
 @docs_group('Events')
 class ExitEvent(BaseModel):
+    """An `exit` event signalling the Actor process is about to exit."""
+
     name: Literal[Event.EXIT]
+    """The event name."""
+
     data: Annotated[EventExitData, Field(default_factory=EventExitData)]
+    """The event payload."""
 
 
 @docs_group('Events')
 class EventWithoutData(BaseModel):
+    """A framework-level event that carries no payload (e.g. browser and page lifecycle events)."""
+
     name: Literal[
         Event.SESSION_RETIRED,
         Event.BROWSER_LAUNCHED,
@@ -92,19 +136,32 @@ class EventWithoutData(BaseModel):
         Event.PAGE_CREATED,
         Event.PAGE_CLOSED,
     ]
+    """The event name."""
+
     data: Any = None
+    """The event payload, always empty for this event."""
 
 
 @docs_group('Events')
 class DeprecatedEvent(BaseModel):
+    """A deprecated event kept for backward compatibility (e.g. `cpuInfo`)."""
+
     name: Literal['cpuInfo']
+    """The event name."""
+
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
+    """The event payload."""
 
 
 @docs_group('Events')
 class UnknownEvent(BaseModel):
+    """A fallback for any event whose name is not recognized by the SDK."""
+
     name: str
+    """The event name."""
+
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
+    """The event payload."""
 
 
 EventMessage = PersistStateEvent | SystemInfoEvent | MigratingEvent | AbortingEvent | ExitEvent | EventWithoutData

{apify-3.4.2b4 → apify-3.4.2b5}/src/apify/scrapy/_async_thread.py

@@ -5,7 +5,7 @@ import threading
 from concurrent import futures
 from datetime import timedelta
 from logging import getLogger
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
 
 if TYPE_CHECKING:
     from collections.abc import Coroutine
@@ -14,13 +14,16 @@ logger = getLogger(__name__)
 
 
 class AsyncThread:
-    """Class for running an asyncio event loop in a separate thread.
+    """Run an asyncio event loop in a dedicated background thread.
 
-    This allows running asynchronous coroutines from synchronous code by executingthem on an event loop
-    that runs in its own dedicated thread.
+    This lets synchronous Scrapy callbacks drive asynchronous Apify and Crawlee coroutines. The
+    scheduler and the HTTP cache storage each own their own `AsyncThread`, so the request queue and
+    the key-value store never share an event loop; they only share the read-only global
+    `Configuration`. A single shared loop would also work but would couple their lifecycles.
     """
 
-    def __init__(self) -> None:
+    def __init__(self, default_timeout: timedelta = timedelta(seconds=60)) -> None:
+        self._default_timeout = default_timeout
         self._eventloop = asyncio.new_event_loop()
 
         # Start the event loop in a dedicated daemon thread.
@@ -33,7 +36,7 @@ class AsyncThread:
     def run_coro(
         self,
         coro: Coroutine,
-        timeout: timedelta = timedelta(seconds=60),
+        timeout: timedelta | Literal['default'] = 'default',
     ) -> Any:
         """Run a coroutine on an event loop running in a separate thread.
 
@@ -42,7 +45,8 @@ class AsyncThread:
 
         Args:
             coro: The coroutine to run.
-            timeout: The maximum number of seconds to wait for the coroutine to finish.
+            timeout: The maximum time to wait for the coroutine to finish. Pass `'default'` to use the
+                `default_timeout` passed to the constructor.
 
         Returns:
             The result returned by the coroutine.
@@ -52,6 +56,9 @@ class AsyncThread:
             TimeoutError: If the coroutine does not complete within the timeout.
             Exception: Any exception raised during coroutine execution.
         """
+        if timeout == 'default':
+            timeout = self._default_timeout
+
         if not self._eventloop.is_running():
             raise RuntimeError(f'The coroutine {coro} cannot be executed because the event loop is not running.')
 

{apify-3.4.2b4 → apify-3.4.2b5}/src/apify/scrapy/middlewares/apify_proxy.py

@@ -7,7 +7,7 @@ from scrapy.core.downloader.handlers.http11 import TunnelError
 from scrapy.exceptions import NotConfigured
 
 from apify import Actor, ProxyConfiguration
-from apify.scrapy import get_basic_auth_header
+from apify.scrapy.utils import get_basic_auth_header
 
 if TYPE_CHECKING:
     from scrapy import Request, Spider
@@ -30,7 +30,6 @@ class ApifyHttpProxyMiddleware:
 
         Args:
             proxy_settings: Dictionary containing proxy settings, provided by the Actor input.
-            auth_encoding: Encoding for basic authentication (default is 'latin-1').
         """
         self._proxy_settings = proxy_settings
         self._proxy_cfg_internal: ProxyConfiguration | None = None
@@ -111,7 +110,7 @@ class ApifyHttpProxyMiddleware:
         if isinstance(exception, TunnelError):
             Actor.log.warning(
                 f'ApifyHttpProxyMiddleware: TunnelError occurred for request="{request}", '
-                'reason="{exception}", skipping...'
+                f'reason="{exception}", skipping...'
             )
 
     async def _get_new_proxy_url(self) -> ParseResult:

{apify-3.4.2b4 → apify-3.4.2b5}/src/apify/scrapy/scheduler.py

@@ -170,6 +170,13 @@ class ApifyScheduler(BaseScheduler):
             traceback.print_exc()
             raise
 
-        scrapy_request = to_scrapy_request(apify_request, spider=self.spider)
+        # Reconstruct the Scrapy request. A malformed queue entry must not crash the whole run: it
+        # has already been marked handled above, so log it and skip it instead of propagating.
+        try:
+            scrapy_request = to_scrapy_request(apify_request, spider=self.spider)
+        except Exception:
+            logger.exception(f'Failed to convert Apify request {apify_request} to a Scrapy request; skipping it.')
+            return None
+
         logger.debug(f'Converted to scrapy_request: {scrapy_request}')
         return scrapy_request