apify 2.7.3__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

apify/_actor.py

@@ -5,7 +5,7 @@ import os
 import sys
 from contextlib import suppress
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, cast, overload
+from typing import TYPE_CHECKING, Any, Literal, TypeVar, cast, overload
 
 from lazy_object_proxy import Proxy
 from more_itertools import flatten
@@ -13,7 +13,6 @@ from pydantic import AliasChoices
 
 from apify_client import ApifyClientAsync
 from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars
-from apify_shared.utils import ignore_docs, maybe_extract_enum_member_value
 from crawlee import service_locator
 from crawlee.events import (
     Event,
@@ -30,15 +29,16 @@ from apify._configuration import Configuration
 from apify._consts import EVENT_LISTENERS_TIMEOUT
 from apify._crypto import decrypt_input_secrets, load_private_key
 from apify._models import ActorRun
-from apify._platform_event_manager import EventManager, LocalEventManager, PlatformEventManager
 from apify._proxy_configuration import ProxyConfiguration
-from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython
-from apify.apify_storage_client import ApifyStorageClient
+from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython, maybe_extract_enum_member_value
+from apify.events import ApifyEventManager, EventManager, LocalEventManager
 from apify.log import _configure_logging, logger
+from apify.storage_clients import ApifyStorageClient
 from apify.storages import Dataset, KeyValueStore, RequestQueue
 
 if TYPE_CHECKING:
     import logging
+    from collections.abc import Callable
     from types import TracebackType
 
     from typing_extensions import Self
@@ -53,9 +53,46 @@ MainReturnType = TypeVar('MainReturnType')
 
 
 @docs_name('Actor')
-@docs_group('Classes')
+@docs_group('Actor')
 class _ActorType:
-    """The class of `Actor`. Only make a new instance if you're absolutely sure you need to."""
+    """The core class for building Actors on the Apify platform.
+
+    Actors are serverless programs running in the cloud that can perform anything from simple actions
+    (such as filling out a web form or sending an email) to complex operations (such as crawling an
+    entire website or removing duplicates from a large dataset). They are packaged as Docker containers
+    which accept well-defined JSON input, perform an action, and optionally produce well-defined output.
+
+    ### References
+
+    - Apify platform documentation: https://docs.apify.com/platform/actors
+    - Actor whitepaper: https://whitepaper.actor/
+
+    ### Usage
+
+    ```python
+    import asyncio
+
+    import httpx
+    from apify import Actor
+    from bs4 import BeautifulSoup
+
+
+    async def main() -> None:
+        async with Actor:
+            actor_input = await Actor.get_input()
+            async with httpx.AsyncClient() as client:
+                response = await client.get(actor_input['url'])
+            soup = BeautifulSoup(response.content, 'html.parser')
+            data = {
+                'url': actor_input['url'],
+                'title': soup.title.string if soup.title else None,
+            }
+            await Actor.push_data(data)
+
+    if __name__ == '__main__':
+        asyncio.run(main())
+    ```
+    """
 
     _is_rebooting = False
     _is_any_instance_initialized = False
@@ -88,12 +125,12 @@ class _ActorType:
 
         # Create an instance of the cloud storage client, the local storage client is obtained
         # from the service locator.
-        self._cloud_storage_client = ApifyStorageClient.from_config(config=self._configuration)
+        self._cloud_storage_client = ApifyStorageClient()
 
         # Set the event manager based on whether the Actor is running on the platform or locally.
         self._event_manager = (
-            PlatformEventManager(
-                config=self._configuration,
+            ApifyEventManager(
+                configuration=self._configuration,
                 persist_state_interval=self._configuration.persist_state_interval,
             )
             if self.is_at_home()
@@ -107,7 +144,6 @@ class _ActorType:
 
         self._is_initialized = False
 
-    @ignore_docs
     async def __aenter__(self) -> Self:
         """Initialize the Actor.
 
@@ -119,7 +155,6 @@ class _ActorType:
         await self.init()
         return self
 
-    @ignore_docs
     async def __aexit__(
         self,
         _exc_type: type[BaseException] | None,

apify/_charging.py

@@ -4,11 +4,10 @@ import math
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from decimal import Decimal
-from typing import TYPE_CHECKING, Protocol, Union
+from typing import TYPE_CHECKING, Protocol
 
 from pydantic import TypeAdapter
 
-from apify_shared.utils import ignore_docs
 from crawlee._utils.context import ensure_context
 
 from apify._models import ActorRun, PricingModel
@@ -23,13 +22,21 @@ if TYPE_CHECKING:
 
     from apify._configuration import Configuration
 
+run_validator = TypeAdapter[ActorRun | None](ActorRun | None)
 
-run_validator: TypeAdapter[ActorRun | None] = TypeAdapter(Union[ActorRun, None])
 
-
-@docs_group('Interfaces')
+@docs_group('Charging')
 class ChargingManager(Protocol):
-    """Provides fine-grained access to pay-per-event functionality."""
+    """Provides fine-grained access to pay-per-event functionality.
+
+    The ChargingManager allows you to charge for specific events in your Actor when using
+    the pay-per-event pricing model. This enables precise cost control and transparent
+    billing for different operations within your Actor.
+
+    ### References
+
+    - Apify platform documentation: https://docs.apify.com/platform/actors/publishing/monetize
+    """
 
     async def charge(self, event_name: str, count: int = 1) -> ChargeResult:
         """Charge for a specified number of events - sub-operations of the Actor.
@@ -58,7 +65,7 @@ class ChargingManager(Protocol):
         """
 
 
-@docs_group('Data structures')
+@docs_group('Charging')
 @dataclass(frozen=True)
 class ChargeResult:
     """Result of the `ChargingManager.charge` method."""
@@ -73,7 +80,7 @@ class ChargeResult:
     """How many events of each known type can still be charged within the limit."""
 
 
-@docs_group('Data structures')
+@docs_group('Charging')
 @dataclass
 class ActorPricingInfo:
     """Result of the `ChargingManager.get_pricing_info` method."""
@@ -91,7 +98,6 @@ class ActorPricingInfo:
     """Price of every known event type."""
 
 
-@ignore_docs
 class ChargingManagerImplementation(ChargingManager):
     """Implementation of the `ChargingManager` Protocol - this is only meant to be instantiated internally."""
 

apify/_configuration.py

@@ -25,7 +25,7 @@ def _transform_to_list(value: Any) -> list[str] | None:
     return value if isinstance(value, list) else str(value).split(',')
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 class Configuration(CrawleeConfiguration):
     """A class for specifying the configuration of an Actor.
 
@@ -140,6 +140,39 @@ class Configuration(CrawleeConfiguration):
         ),
     ] = None
 
+    default_dataset_id: Annotated[
+        str,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_dataset_id',
+                'apify_default_dataset_id',
+            ),
+            description='Default dataset ID used by the Apify storage client when no ID or name is provided.',
+        ),
+    ] = 'default'
+
+    default_key_value_store_id: Annotated[
+        str,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_key_value_store_id',
+                'apify_default_key_value_store_id',
+            ),
+            description='Default key-value store ID for the Apify storage client when no ID or name is provided.',
+        ),
+    ] = 'default'
+
+    default_request_queue_id: Annotated[
+        str,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_request_queue_id',
+                'apify_default_request_queue_id',
+            ),
+            description='Default request queue ID for the Apify storage client when no ID or name is provided.',
+        ),
+    ] = 'default'
+
     disable_outdated_warning: Annotated[
         bool,
         Field(

apify/_crypto.py

@@ -12,7 +12,6 @@ from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import padding, rsa
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 
-from apify_shared.utils import ignore_docs
 from crawlee._utils.crypto import crypto_random_object_id
 
 from apify._consts import ENCRYPTED_INPUT_VALUE_REGEXP, ENCRYPTED_JSON_VALUE_PREFIX, ENCRYPTED_STRING_VALUE_PREFIX
@@ -22,7 +21,6 @@ ENCRYPTION_IV_LENGTH = 16
 ENCRYPTION_AUTH_TAG_LENGTH = 16
 
 
-@ignore_docs
 def public_encrypt(value: str, *, public_key: rsa.RSAPublicKey) -> dict:
     """Encrypts the given value using AES cipher and the password for encryption using the public key.
 
@@ -66,7 +64,6 @@ def public_encrypt(value: str, *, public_key: rsa.RSAPublicKey) -> dict:
     }
 
 
-@ignore_docs
 def private_decrypt(
     encrypted_password: str,
     encrypted_value: str,
@@ -118,7 +115,6 @@ def private_decrypt(
     return decipher_bytes.decode('utf-8')
 
 
-@ignore_docs
 def load_private_key(private_key_file_base64: str, private_key_password: str) -> rsa.RSAPrivateKey:
     private_key = serialization.load_pem_private_key(
         base64.b64decode(private_key_file_base64.encode('utf-8')),
@@ -138,7 +134,6 @@ def _load_public_key(public_key_file_base64: str) -> rsa.RSAPublicKey:
     return public_key
 
 
-@ignore_docs
 def decrypt_input_secrets(private_key: rsa.RSAPrivateKey, input_data: Any) -> Any:
     """Decrypt input secrets."""
     if not isinstance(input_data, dict):
@@ -180,7 +175,6 @@ def encode_base62(num: int) -> str:
     return res
 
 
-@ignore_docs
 def create_hmac_signature(secret_key: str, message: str) -> str:
     """Generate an HMAC signature and encodes it using Base62. Base62 encoding reduces the signature length.
 

apify/_models.py

@@ -13,10 +13,10 @@ from crawlee._utils.urls import validate_http_url
 from apify._utils import docs_group
 
 if TYPE_CHECKING:
-    from typing_extensions import TypeAlias
+    from typing import TypeAlias
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class Webhook(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -35,14 +35,14 @@ class Webhook(BaseModel):
     ] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunMeta(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
     origin: Annotated[MetaOrigin, Field()]
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunStats(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -63,7 +63,7 @@ class ActorRunStats(BaseModel):
     compute_units: Annotated[float, Field(alias='computeUnits')]
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunOptions(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -74,7 +74,7 @@ class ActorRunOptions(BaseModel):
     max_total_charge_usd: Annotated[Decimal | None, Field(alias='maxTotalChargeUsd')] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunUsage(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -92,7 +92,7 @@ class ActorRunUsage(BaseModel):
     proxy_serps: Annotated[float | None, Field(alias='PROXY_SERPS')] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRun(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 

apify/_proxy_configuration.py

@@ -1,16 +1,17 @@
 from __future__ import annotations
 
 import ipaddress
+import json
 import re
 from dataclasses import dataclass, field
 from re import Pattern
 from typing import TYPE_CHECKING, Any
 from urllib.parse import urljoin, urlparse
 
-import httpx
+import impit
+from yarl import URL
 
 from apify_shared.consts import ApifyEnvVars
-from apify_shared.utils import ignore_docs
 from crawlee.proxy_configuration import ProxyConfiguration as CrawleeProxyConfiguration
 from crawlee.proxy_configuration import ProxyInfo as CrawleeProxyInfo
 from crawlee.proxy_configuration import _NewUrlFunction
@@ -21,14 +22,14 @@ from apify.log import logger
 
 if TYPE_CHECKING:
     from apify_client import ApifyClientAsync
-    from crawlee import Request
+
+    from apify import Request
 
 APIFY_PROXY_VALUE_REGEX = re.compile(r'^[\w._~]+$')
 COUNTRY_CODE_REGEX = re.compile(r'^[A-Z]{2}$')
 SESSION_ID_MAX_LENGTH = 50
 
 
-@ignore_docs
 def is_url(url: str) -> bool:
     """Check if the given string is a valid URL."""
     try:
@@ -69,7 +70,7 @@ def _check(
         raise ValueError(f'{error_str} does not match pattern {pattern.pattern!r}')
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 @dataclass
 class ProxyInfo(CrawleeProxyInfo):
     """Provides information about a proxy connection that is used for requests."""
@@ -89,7 +90,7 @@ class ProxyInfo(CrawleeProxyInfo):
     """
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 class ProxyConfiguration(CrawleeProxyConfiguration):
     """Configures a connection to a proxy server with the provided options.
 
@@ -104,7 +105,6 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
 
     _configuration: Configuration
 
-    @ignore_docs
     def __init__(
         self,
         *,
@@ -233,7 +233,7 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
             return None
 
         if self._uses_apify_proxy:
-            parsed_url = httpx.URL(proxy_info.url)
+            parsed_url = URL(proxy_info.url)
             username = self._get_username(session_id)
 
             return ProxyInfo(
@@ -277,11 +277,11 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
             return
 
         status = None
-        async with httpx.AsyncClient(proxy=proxy_info.url, timeout=10) as client:
+        async with impit.AsyncClient(proxy=proxy_info.url, timeout=10) as client:
             for _ in range(2):
                 try:
                     response = await client.get(proxy_status_url)
-                    status = response.json()
+                    status = json.loads(response.text)
                     break
                 except Exception:  # noqa: S110
                     # retry on connection errors

apify/_utils.py

@@ -2,8 +2,12 @@ from __future__ import annotations
 
 import builtins
 import sys
+from enum import Enum
 from importlib import metadata
-from typing import Callable, Literal
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
 
 
 def get_system_info() -> dict:
@@ -27,7 +31,19 @@ def is_running_in_ipython() -> bool:
     return getattr(builtins, '__IPYTHON__', False)
 
 
-GroupName = Literal['Classes', 'Abstract classes', 'Interfaces', 'Data structures', 'Errors', 'Functions']
+# The order of the rendered API groups is defined in the website/docusaurus.config.js file.
+GroupName = Literal[
+    'Actor',
+    'Charging',
+    'Configuration',
+    'Event data',
+    'Event managers',
+    'Events',
+    'Request loaders',
+    'Storage clients',
+    'Storage data',
+    'Storages',
+]
 
 
 def docs_group(group_name: GroupName) -> Callable:  # noqa: ARG001
@@ -66,3 +82,10 @@ def docs_name(symbol_name: str) -> Callable:  # noqa: ARG001
         return func
 
     return wrapper
+
+
+def maybe_extract_enum_member_value(maybe_enum_member: Any) -> Any:
+    """Extract the value of an enumeration member if it is an Enum, otherwise return the original value."""
+    if isinstance(maybe_enum_member, Enum):
+        return maybe_enum_member.value
+    return maybe_enum_member

apify/events/__init__.py

@@ -0,0 +1,5 @@
+from crawlee.events import EventManager, LocalEventManager
+
+from ._apify_event_manager import ApifyEventManager
+
+__all__ = ['ApifyEventManager', 'EventManager', 'LocalEventManager']

apify/events/_apify_event_manager.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from typing import TYPE_CHECKING, Annotated
+
+import websockets.asyncio.client
+from pydantic import Discriminator, TypeAdapter
+from typing_extensions import Self, Unpack, override
+
+from crawlee.events import EventManager
+from crawlee.events._types import Event, EventPersistStateData
+
+from apify._utils import docs_group
+from apify.events._types import DeprecatedEvent, EventMessage, SystemInfoEventData, UnknownEvent
+from apify.log import logger
+
+if TYPE_CHECKING:
+    from types import TracebackType
+
+    from crawlee.events._event_manager import EventManagerOptions
+
+    from apify._configuration import Configuration
+
+
+event_data_adapter = TypeAdapter[EventMessage | DeprecatedEvent | UnknownEvent](
+    Annotated[EventMessage, Discriminator('name')] | DeprecatedEvent | UnknownEvent
+)
+
+
+@docs_group('Event managers')
+class ApifyEventManager(EventManager):
+    """Event manager for the Apify platform.
+
+    This class extends Crawlee's `EventManager` to provide Apify-specific functionality, including websocket
+    connectivity to the Apify platform for receiving platform events.
+
+    The event manager handles:
+    - Registration and emission of events and their listeners.
+    - Websocket connection to Apify platform events.
+    - Processing and validation of platform messages.
+    - Automatic event forwarding from the platform to local event listeners.
+
+    This class should not be used directly. Use the `Actor.on` and `Actor.off` methods to interact
+    with the event system.
+    """
+
+    def __init__(self, configuration: Configuration, **kwargs: Unpack[EventManagerOptions]) -> None:
+        """Initialize a new instance.
+
+        Args:
+            configuration: The Actor configuration for the event manager.
+            **kwargs: Additional event manager options passed to the parent class.
+        """
+        super().__init__(**kwargs)
+
+        self._configuration = configuration
+        """The Actor configuration for the event manager."""
+
+        self._platform_events_websocket: websockets.asyncio.client.ClientConnection | None = None
+        """WebSocket connection to the platform events."""
+
+        self._process_platform_messages_task: asyncio.Task | None = None
+        """Task for processing messages from the platform websocket."""
+
+        self._connected_to_platform_websocket: asyncio.Future[bool] | None = None
+        """Future that resolves when the connection to the platform websocket is established."""
+
+    @override
+    async def __aenter__(self) -> Self:
+        await super().__aenter__()
+        self._connected_to_platform_websocket = asyncio.Future()
+
+        # Run tasks but don't await them
+        if self._configuration.actor_events_ws_url:
+            self._process_platform_messages_task = asyncio.create_task(
+                self._process_platform_messages(self._configuration.actor_events_ws_url)
+            )
+            is_connected = await self._connected_to_platform_websocket
+            if not is_connected:
+                raise RuntimeError('Error connecting to platform events websocket!')
+        else:
+            logger.debug('APIFY_ACTOR_EVENTS_WS_URL env var not set, no events from Apify platform will be emitted.')
+
+        return self
+
+    @override
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_traceback: TracebackType | None,
+    ) -> None:
+        if self._platform_events_websocket:
+            await self._platform_events_websocket.close()
+
+        if self._process_platform_messages_task and not self._process_platform_messages_task.done():
+            self._process_platform_messages_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._process_platform_messages_task
+
+        await super().__aexit__(exc_type, exc_value, exc_traceback)
+
+    async def _process_platform_messages(self, ws_url: str) -> None:
+        try:
+            async with websockets.asyncio.client.connect(ws_url) as websocket:
+                self._platform_events_websocket = websocket
+                if self._connected_to_platform_websocket is not None:
+                    self._connected_to_platform_websocket.set_result(True)
+
+                async for message in websocket:
+                    try:
+                        parsed_message = event_data_adapter.validate_json(message)
+
+                        if isinstance(parsed_message, DeprecatedEvent):
+                            continue
+
+                        if isinstance(parsed_message, UnknownEvent):
+                            logger.info(
+                                f'Unknown message received: event_name={parsed_message.name}, '
+                                f'event_data={parsed_message.data}'
+                            )
+                            continue
+
+                        self.emit(
+                            event=parsed_message.name,
+                            event_data=parsed_message.data
+                            if not isinstance(parsed_message.data, SystemInfoEventData)
+                            else parsed_message.data.to_crawlee_format(self._configuration.dedicated_cpus or 1),
+                        )
+
+                        if parsed_message.name == Event.MIGRATING:
+                            await self._emit_persist_state_event_rec_task.stop()
+                            self.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=True))
+                    except Exception:
+                        logger.exception('Cannot parse Actor event', extra={'message': message})
+        except Exception:
+            logger.exception('Error in websocket connection')
+            if self._connected_to_platform_websocket is not None:
+                self._connected_to_platform_websocket.set_result(False)