apify 2.7.2__py3-none-any.whl → 3.0.0__py3-none-any.whl

apify/_charging.py

@@ -4,11 +4,10 @@ import math
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from decimal import Decimal
-from typing import TYPE_CHECKING, Protocol, Union
+from typing import TYPE_CHECKING, Protocol
 
 from pydantic import TypeAdapter
 
-from apify_shared.utils import ignore_docs
 from crawlee._utils.context import ensure_context
 
 from apify._models import ActorRun, PricingModel
@@ -23,13 +22,21 @@ if TYPE_CHECKING:
 
     from apify._configuration import Configuration
 
+run_validator = TypeAdapter[ActorRun | None](ActorRun | None)
 
-run_validator: TypeAdapter[ActorRun | None] = TypeAdapter(Union[ActorRun, None])
 
-
-@docs_group('Interfaces')
+@docs_group('Charging')
 class ChargingManager(Protocol):
-    """Provides fine-grained access to pay-per-event functionality."""
+    """Provides fine-grained access to pay-per-event functionality.
+
+    The ChargingManager allows you to charge for specific events in your Actor when using
+    the pay-per-event pricing model. This enables precise cost control and transparent
+    billing for different operations within your Actor.
+
+    ### References
+
+    - Apify platform documentation: https://docs.apify.com/platform/actors/publishing/monetize
+    """
 
     async def charge(self, event_name: str, count: int = 1) -> ChargeResult:
         """Charge for a specified number of events - sub-operations of the Actor.
@@ -57,8 +64,18 @@ class ChargingManager(Protocol):
         This can be used for instance when your code needs to support multiple pricing models in transition periods.
         """
 
+    def get_charged_event_count(self, event_name: str) -> int:
+        """Get the number of events with the given name that were charged so far.
+
+        Args:
+            event_name: Name of the inspected event.
+        """
+
+    def get_max_total_charge_usd(self) -> Decimal:
+        """Get the configured maximum total charge for this Actor run."""
 
-@docs_group('Data structures')
+
+@docs_group('Charging')
 @dataclass(frozen=True)
 class ChargeResult:
     """Result of the `ChargingManager.charge` method."""
@@ -73,7 +90,7 @@ class ChargeResult:
     """How many events of each known type can still be charged within the limit."""
 
 
-@docs_group('Data structures')
+@docs_group('Charging')
 @dataclass
 class ActorPricingInfo:
     """Result of the `ChargingManager.get_pricing_info` method."""
@@ -91,7 +108,6 @@ class ActorPricingInfo:
     """Price of every known event type."""
 
 
-@ignore_docs
 class ChargingManagerImplementation(ChargingManager):
     """Implementation of the `ChargingManager` Protocol - this is only meant to be instantiated internally."""
 
@@ -303,6 +319,15 @@ class ChargingManagerImplementation(ChargingManager):
             },
         )
 
+    @ensure_context
+    def get_charged_event_count(self, event_name: str) -> int:
+        item = self._charging_state.get(event_name)
+        return item.charge_count if item is not None else 0
+
+    @ensure_context
+    def get_max_total_charge_usd(self) -> Decimal:
+        return self._max_total_charge_usd
+
 
 @dataclass
 class ChargingStateItem:

apify/_configuration.py

@@ -8,6 +8,7 @@ from typing import Annotated, Any
 from pydantic import AliasChoices, BeforeValidator, Field, model_validator
 from typing_extensions import Self, deprecated
 
+from crawlee import service_locator
 from crawlee._utils.models import timedelta_ms
 from crawlee._utils.urls import validate_http_url
 from crawlee.configuration import Configuration as CrawleeConfiguration
@@ -25,7 +26,7 @@ def _transform_to_list(value: Any) -> list[str] | None:
     return value if isinstance(value, list) else str(value).split(',')
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 class Configuration(CrawleeConfiguration):
     """A class for specifying the configuration of an Actor.
 
@@ -140,6 +141,39 @@ class Configuration(CrawleeConfiguration):
         ),
     ] = None
 
+    default_dataset_id: Annotated[
+        str | None,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_dataset_id',
+                'apify_default_dataset_id',
+            ),
+            description='Default dataset ID used by the Apify storage client when no ID or name is provided.',
+        ),
+    ] = None
+
+    default_key_value_store_id: Annotated[
+        str | None,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_key_value_store_id',
+                'apify_default_key_value_store_id',
+            ),
+            description='Default key-value store ID for the Apify storage client when no ID or name is provided.',
+        ),
+    ] = None
+
+    default_request_queue_id: Annotated[
+        str | None,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_request_queue_id',
+                'apify_default_request_queue_id',
+            ),
+            description='Default request queue ID for the Apify storage client when no ID or name is provided.',
+        ),
+    ] = None
+
     disable_outdated_warning: Annotated[
         bool,
         Field(
@@ -334,6 +368,15 @@ class Configuration(CrawleeConfiguration):
         ),
     ] = None
 
+    user_is_paying: Annotated[
+        bool,
+        Field(
+            alias='apify_user_is_paying',
+            description='True if the user calling the Actor is paying user',
+        ),
+        BeforeValidator(lambda val: False if val == '' else val),
+    ] = False
+
     web_server_port: Annotated[
         int,
         Field(
@@ -382,11 +425,41 @@ class Configuration(CrawleeConfiguration):
     def get_global_configuration(cls) -> Configuration:
         """Retrieve the global instance of the configuration.
 
-        Mostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`
-        instead.
+        This method ensures that ApifyConfigration is returned, even if CrawleeConfiguration was set in the
+        service locator.
+        """
+        global_configuration = service_locator.get_configuration()
+
+        if isinstance(global_configuration, Configuration):
+            # If Apify configuration was already stored in service locator, return it.
+            return global_configuration
+
+        logger.warning(
+            'Non Apify Configration is set in the `service_locator` in the SDK context. '
+            'It is recommended to set `apify.Configuration` explicitly as early as possible by using '
+            'service_locator.set_configuration'
+        )
+
+        return cls.from_configuration(global_configuration)
+
+    @classmethod
+    def from_configuration(cls, configuration: CrawleeConfiguration) -> Configuration:
+        """Create Apify Configuration from existing Crawlee Configuration.
+
+        Args:
+            configuration: The existing Crawlee Configuration.
+
+        Returns:
+            The created Apify Configuration.
         """
-        return cls()
+        apify_configuration = cls()
 
+        # Ensure the returned configuration is of type Apify Configuration.
+        # Most likely crawlee configuration was already set. Create Apify configuration from it.
+        # Due to known Pydantic issue https://github.com/pydantic/pydantic/issues/9516, creating new instance of
+        # Configuration from existing one in situation where environment can have some fields set by alias is very
+        # unpredictable. Use the stable workaround.
+        for name in configuration.model_fields:
+            setattr(apify_configuration, name, getattr(configuration, name))
 
-# Monkey-patch the base class so that it works with the extended configuration
-CrawleeConfiguration.get_global_configuration = Configuration.get_global_configuration  # type: ignore[method-assign]
+        return apify_configuration

apify/_crypto.py

@@ -12,7 +12,6 @@ from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import padding, rsa
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 
-from apify_shared.utils import ignore_docs
 from crawlee._utils.crypto import crypto_random_object_id
 
 from apify._consts import ENCRYPTED_INPUT_VALUE_REGEXP, ENCRYPTED_JSON_VALUE_PREFIX, ENCRYPTED_STRING_VALUE_PREFIX
@@ -22,7 +21,6 @@ ENCRYPTION_IV_LENGTH = 16
 ENCRYPTION_AUTH_TAG_LENGTH = 16
 
 
-@ignore_docs
 def public_encrypt(value: str, *, public_key: rsa.RSAPublicKey) -> dict:
     """Encrypts the given value using AES cipher and the password for encryption using the public key.
 
@@ -66,7 +64,6 @@ def public_encrypt(value: str, *, public_key: rsa.RSAPublicKey) -> dict:
     }
 
 
-@ignore_docs
 def private_decrypt(
     encrypted_password: str,
     encrypted_value: str,
@@ -118,7 +115,6 @@ def private_decrypt(
     return decipher_bytes.decode('utf-8')
 
 
-@ignore_docs
 def load_private_key(private_key_file_base64: str, private_key_password: str) -> rsa.RSAPrivateKey:
     private_key = serialization.load_pem_private_key(
         base64.b64decode(private_key_file_base64.encode('utf-8')),
@@ -138,7 +134,6 @@ def _load_public_key(public_key_file_base64: str) -> rsa.RSAPublicKey:
     return public_key
 
 
-@ignore_docs
 def decrypt_input_secrets(private_key: rsa.RSAPrivateKey, input_data: Any) -> Any:
     """Decrypt input secrets."""
     if not isinstance(input_data, dict):
@@ -180,7 +175,6 @@ def encode_base62(num: int) -> str:
     return res
 
 
-@ignore_docs
 def create_hmac_signature(secret_key: str, message: str) -> str:
     """Generate an HMAC signature and encodes it using Base62. Base62 encoding reduces the signature length.
 

apify/_models.py

@@ -13,10 +13,10 @@ from crawlee._utils.urls import validate_http_url
 from apify._utils import docs_group
 
 if TYPE_CHECKING:
-    from typing_extensions import TypeAlias
+    from typing import TypeAlias
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class Webhook(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -35,14 +35,14 @@ class Webhook(BaseModel):
     ] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunMeta(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
     origin: Annotated[MetaOrigin, Field()]
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunStats(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -63,7 +63,7 @@ class ActorRunStats(BaseModel):
     compute_units: Annotated[float, Field(alias='computeUnits')]
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunOptions(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -74,7 +74,7 @@ class ActorRunOptions(BaseModel):
     max_total_charge_usd: Annotated[Decimal | None, Field(alias='maxTotalChargeUsd')] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunUsage(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -92,7 +92,7 @@ class ActorRunUsage(BaseModel):
     proxy_serps: Annotated[float | None, Field(alias='PROXY_SERPS')] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRun(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 

apify/_proxy_configuration.py

@@ -1,16 +1,17 @@
 from __future__ import annotations
 
 import ipaddress
+import json
 import re
 from dataclasses import dataclass, field
 from re import Pattern
 from typing import TYPE_CHECKING, Any
 from urllib.parse import urljoin, urlparse
 
-import httpx
+import impit
+from yarl import URL
 
 from apify_shared.consts import ApifyEnvVars
-from apify_shared.utils import ignore_docs
 from crawlee.proxy_configuration import ProxyConfiguration as CrawleeProxyConfiguration
 from crawlee.proxy_configuration import ProxyInfo as CrawleeProxyInfo
 from crawlee.proxy_configuration import _NewUrlFunction
@@ -21,14 +22,14 @@ from apify.log import logger
 
 if TYPE_CHECKING:
     from apify_client import ApifyClientAsync
-    from crawlee import Request
+
+    from apify import Request
 
 APIFY_PROXY_VALUE_REGEX = re.compile(r'^[\w._~]+$')
 COUNTRY_CODE_REGEX = re.compile(r'^[A-Z]{2}$')
 SESSION_ID_MAX_LENGTH = 50
 
 
-@ignore_docs
 def is_url(url: str) -> bool:
     """Check if the given string is a valid URL."""
     try:
@@ -69,7 +70,7 @@ def _check(
         raise ValueError(f'{error_str} does not match pattern {pattern.pattern!r}')
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 @dataclass
 class ProxyInfo(CrawleeProxyInfo):
     """Provides information about a proxy connection that is used for requests."""
@@ -89,7 +90,7 @@ class ProxyInfo(CrawleeProxyInfo):
     """
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 class ProxyConfiguration(CrawleeProxyConfiguration):
     """Configures a connection to a proxy server with the provided options.
 
@@ -104,7 +105,6 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
 
     _configuration: Configuration
 
-    @ignore_docs
     def __init__(
         self,
         *,
@@ -233,7 +233,7 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
             return None
 
         if self._uses_apify_proxy:
-            parsed_url = httpx.URL(proxy_info.url)
+            parsed_url = URL(proxy_info.url)
             username = self._get_username(session_id)
 
             return ProxyInfo(
@@ -277,11 +277,11 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
             return
 
         status = None
-        async with httpx.AsyncClient(proxy=proxy_info.url, timeout=10) as client:
+        async with impit.AsyncClient(proxy=proxy_info.url, timeout=10) as client:
             for _ in range(2):
                 try:
                     response = await client.get(proxy_status_url)
-                    status = response.json()
+                    status = json.loads(response.text)
                     break
                 except Exception:  # noqa: S110
                     # retry on connection errors

apify/_utils.py

@@ -2,8 +2,12 @@ from __future__ import annotations
 
 import builtins
 import sys
+from enum import Enum
 from importlib import metadata
-from typing import Callable, Literal
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
 
 
 def get_system_info() -> dict:
@@ -27,7 +31,19 @@ def is_running_in_ipython() -> bool:
     return getattr(builtins, '__IPYTHON__', False)
 
 
-GroupName = Literal['Classes', 'Abstract classes', 'Interfaces', 'Data structures', 'Errors', 'Functions']
+# The order of the rendered API groups is defined in the website/docusaurus.config.js file.
+GroupName = Literal[
+    'Actor',
+    'Charging',
+    'Configuration',
+    'Event data',
+    'Event managers',
+    'Events',
+    'Request loaders',
+    'Storage clients',
+    'Storage data',
+    'Storages',
+]
 
 
 def docs_group(group_name: GroupName) -> Callable:  # noqa: ARG001
@@ -66,3 +82,10 @@ def docs_name(symbol_name: str) -> Callable:  # noqa: ARG001
         return func
 
     return wrapper
+
+
+def maybe_extract_enum_member_value(maybe_enum_member: Any) -> Any:
+    """Extract the value of an enumeration member if it is an Enum, otherwise return the original value."""
+    if isinstance(maybe_enum_member, Enum):
+        return maybe_enum_member.value
+    return maybe_enum_member

apify/events/__init__.py

@@ -0,0 +1,5 @@
+from crawlee.events import Event, EventManager, LocalEventManager
+
+from ._apify_event_manager import ApifyEventManager
+
+__all__ = ['ApifyEventManager', 'Event', 'EventManager', 'LocalEventManager']

apify/events/_apify_event_manager.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from typing import TYPE_CHECKING, Annotated
+
+import websockets.asyncio.client
+from pydantic import Discriminator, TypeAdapter
+from typing_extensions import Self, Unpack, override
+
+from crawlee.events import EventManager
+from crawlee.events._types import Event, EventPersistStateData
+
+from apify._utils import docs_group
+from apify.events._types import DeprecatedEvent, EventMessage, SystemInfoEventData, UnknownEvent
+from apify.log import logger
+
+if TYPE_CHECKING:
+    from types import TracebackType
+
+    from crawlee.events._event_manager import EventManagerOptions
+
+    from apify._configuration import Configuration
+
+
+event_data_adapter = TypeAdapter[EventMessage | DeprecatedEvent | UnknownEvent](
+    Annotated[EventMessage, Discriminator('name')] | DeprecatedEvent | UnknownEvent
+)
+
+
+@docs_group('Event managers')
+class ApifyEventManager(EventManager):
+    """Event manager for the Apify platform.
+
+    This class extends Crawlee's `EventManager` to provide Apify-specific functionality, including websocket
+    connectivity to the Apify platform for receiving platform events.
+
+    The event manager handles:
+    - Registration and emission of events and their listeners.
+    - Websocket connection to Apify platform events.
+    - Processing and validation of platform messages.
+    - Automatic event forwarding from the platform to local event listeners.
+
+    This class should not be used directly. Use the `Actor.on` and `Actor.off` methods to interact
+    with the event system.
+    """
+
+    def __init__(self, configuration: Configuration, **kwargs: Unpack[EventManagerOptions]) -> None:
+        """Initialize a new instance.
+
+        Args:
+            configuration: The Actor configuration for the event manager.
+            **kwargs: Additional event manager options passed to the parent class.
+        """
+        super().__init__(**kwargs)
+
+        self._configuration = configuration
+        """The Actor configuration for the event manager."""
+
+        self._platform_events_websocket: websockets.asyncio.client.ClientConnection | None = None
+        """WebSocket connection to the platform events."""
+
+        self._process_platform_messages_task: asyncio.Task | None = None
+        """Task for processing messages from the platform websocket."""
+
+        self._connected_to_platform_websocket: asyncio.Future[bool] | None = None
+        """Future that resolves when the connection to the platform websocket is established."""
+
+    @override
+    async def __aenter__(self) -> Self:
+        await super().__aenter__()
+        self._connected_to_platform_websocket = asyncio.Future()
+
+        # Run tasks but don't await them
+        if self._configuration.actor_events_ws_url:
+            self._process_platform_messages_task = asyncio.create_task(
+                self._process_platform_messages(self._configuration.actor_events_ws_url)
+            )
+            is_connected = await self._connected_to_platform_websocket
+            if not is_connected:
+                raise RuntimeError('Error connecting to platform events websocket!')
+        else:
+            logger.debug('APIFY_ACTOR_EVENTS_WS_URL env var not set, no events from Apify platform will be emitted.')
+
+        return self
+
+    @override
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_traceback: TracebackType | None,
+    ) -> None:
+        if self._platform_events_websocket:
+            await self._platform_events_websocket.close()
+
+        if self._process_platform_messages_task and not self._process_platform_messages_task.done():
+            self._process_platform_messages_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._process_platform_messages_task
+
+        await super().__aexit__(exc_type, exc_value, exc_traceback)
+
+    async def _process_platform_messages(self, ws_url: str) -> None:
+        try:
+            async with websockets.asyncio.client.connect(ws_url) as websocket:
+                self._platform_events_websocket = websocket
+                if self._connected_to_platform_websocket is not None:
+                    self._connected_to_platform_websocket.set_result(True)
+
+                async for message in websocket:
+                    try:
+                        parsed_message = event_data_adapter.validate_json(message)
+
+                        if isinstance(parsed_message, DeprecatedEvent):
+                            continue
+
+                        if isinstance(parsed_message, UnknownEvent):
+                            logger.info(
+                                f'Unknown message received: event_name={parsed_message.name}, '
+                                f'event_data={parsed_message.data}'
+                            )
+                            continue
+
+                        self.emit(
+                            event=parsed_message.name,
+                            event_data=parsed_message.data
+                            if not isinstance(parsed_message.data, SystemInfoEventData)
+                            else parsed_message.data.to_crawlee_format(self._configuration.dedicated_cpus or 1),
+                        )
+
+                        if parsed_message.name == Event.MIGRATING:
+                            await self._emit_persist_state_event_rec_task.stop()
+                            self.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=True))
+                    except Exception:
+                        logger.exception('Cannot parse Actor event', extra={'message': message})
+        except Exception:
+            logger.exception('Error in websocket connection')
+            if self._connected_to_platform_websocket is not None:
+                self._connected_to_platform_websocket.set_result(False)

apify/events/_types.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, Field
+
+from crawlee.events._types import (
+    Event,
+    EventAbortingData,
+    EventExitData,
+    EventMigratingData,
+    EventPersistStateData,
+    EventSystemInfoData,
+)
+
+from apify._utils import docs_group
+
+
+@docs_group('Event data')
+class SystemInfoEventData(BaseModel):
+    mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
+    mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
+    mem_max_bytes: Annotated[float, Field(alias='memMaxBytes')]
+    cpu_avg_usage: Annotated[float, Field(alias='cpuAvgUsage')]
+    cpu_max_usage: Annotated[float, Field(alias='cpuMaxUsage')]
+    cpu_current_usage: Annotated[float, Field(alias='cpuCurrentUsage')]
+    is_cpu_overloaded: Annotated[bool, Field(alias='isCpuOverloaded')]
+    created_at: Annotated[datetime, Field(alias='createdAt')]
+
+    def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
+        return EventSystemInfoData.model_validate(
+            {
+                'cpu_info': {
+                    'used_ratio': (self.cpu_current_usage / 100) / dedicated_cpus,
+                    'created_at': self.created_at,
+                },
+                'memory_info': {
+                    'total_size': self.mem_max_bytes,
+                    'current_size': self.mem_current_bytes,
+                    'created_at': self.created_at,
+                },
+            }
+        )
+
+
+@docs_group('Events')
+class PersistStateEvent(BaseModel):
+    name: Literal[Event.PERSIST_STATE]
+    data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
+
+
+@docs_group('Events')
+class SystemInfoEvent(BaseModel):
+    name: Literal[Event.SYSTEM_INFO]
+    data: SystemInfoEventData
+
+
+@docs_group('Events')
+class MigratingEvent(BaseModel):
+    name: Literal[Event.MIGRATING]
+    data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
+
+
+@docs_group('Events')
+class AbortingEvent(BaseModel):
+    name: Literal[Event.ABORTING]
+    data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
+
+
+@docs_group('Events')
+class ExitEvent(BaseModel):
+    name: Literal[Event.EXIT]
+    data: Annotated[EventExitData, Field(default_factory=EventExitData)]
+
+
+@docs_group('Events')
+class EventWithoutData(BaseModel):
+    name: Literal[
+        Event.SESSION_RETIRED,
+        Event.BROWSER_LAUNCHED,
+        Event.BROWSER_RETIRED,
+        Event.BROWSER_CLOSED,
+        Event.PAGE_CREATED,
+        Event.PAGE_CLOSED,
+    ]
+    data: Any = None
+
+
+@docs_group('Events')
+class DeprecatedEvent(BaseModel):
+    name: Literal['cpuInfo']
+    data: Annotated[dict[str, Any], Field(default_factory=dict)]
+
+
+@docs_group('Events')
+class UnknownEvent(BaseModel):
+    name: str
+    data: Annotated[dict[str, Any], Field(default_factory=dict)]
+
+
+EventMessage = PersistStateEvent | SystemInfoEvent | MigratingEvent | AbortingEvent | ExitEvent | EventWithoutData