apify 2.7.2__py3-none-any.whl → 3.0.0__py3-none-any.whl

apify/_platform_event_manager.py

@@ -1,231 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from datetime import datetime
-from typing import TYPE_CHECKING, Annotated, Any, Literal, Union
-
-import websockets.asyncio.client
-from pydantic import BaseModel, Discriminator, Field, TypeAdapter
-from typing_extensions import Self, Unpack, override
-
-from crawlee.events._event_manager import EventManager, EventManagerOptions
-from crawlee.events._local_event_manager import LocalEventManager
-from crawlee.events._types import (
-    Event,
-    EventAbortingData,
-    EventExitData,
-    EventMigratingData,
-    EventPersistStateData,
-    EventSystemInfoData,
-)
-
-from apify._utils import docs_group
-from apify.log import logger
-
-if TYPE_CHECKING:
-    from types import TracebackType
-
-    from apify._configuration import Configuration
-
-
-__all__ = ['EventManager', 'LocalEventManager', 'PlatformEventManager']
-
-
-@docs_group('Data structures')
-class PersistStateEvent(BaseModel):
-    name: Literal[Event.PERSIST_STATE]
-    data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
-
-
-@docs_group('Data structures')
-class SystemInfoEventData(BaseModel):
-    mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
-    mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
-    mem_max_bytes: Annotated[float, Field(alias='memMaxBytes')]
-    cpu_avg_usage: Annotated[float, Field(alias='cpuAvgUsage')]
-    cpu_max_usage: Annotated[float, Field(alias='cpuMaxUsage')]
-    cpu_current_usage: Annotated[float, Field(alias='cpuCurrentUsage')]
-    is_cpu_overloaded: Annotated[bool, Field(alias='isCpuOverloaded')]
-    created_at: Annotated[datetime, Field(alias='createdAt')]
-
-    def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
-        return EventSystemInfoData.model_validate(
-            {
-                'cpu_info': {
-                    'used_ratio': (self.cpu_current_usage / 100) / dedicated_cpus,
-                    'created_at': self.created_at,
-                },
-                'memory_info': {
-                    'total_size': self.mem_max_bytes,
-                    'current_size': self.mem_current_bytes,
-                    'created_at': self.created_at,
-                },
-            }
-        )
-
-
-@docs_group('Data structures')
-class SystemInfoEvent(BaseModel):
-    name: Literal[Event.SYSTEM_INFO]
-    data: SystemInfoEventData
-
-
-@docs_group('Data structures')
-class MigratingEvent(BaseModel):
-    name: Literal[Event.MIGRATING]
-    data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
-
-
-@docs_group('Data structures')
-class AbortingEvent(BaseModel):
-    name: Literal[Event.ABORTING]
-    data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
-
-
-@docs_group('Data structures')
-class ExitEvent(BaseModel):
-    name: Literal[Event.EXIT]
-    data: Annotated[EventExitData, Field(default_factory=EventExitData)]
-
-
-@docs_group('Data structures')
-class EventWithoutData(BaseModel):
-    name: Literal[
-        Event.SESSION_RETIRED,
-        Event.BROWSER_LAUNCHED,
-        Event.BROWSER_RETIRED,
-        Event.BROWSER_CLOSED,
-        Event.PAGE_CREATED,
-        Event.PAGE_CLOSED,
-    ]
-    data: Any = None
-
-
-@docs_group('Data structures')
-class DeprecatedEvent(BaseModel):
-    name: Literal['cpuInfo']
-    data: Annotated[dict[str, Any], Field(default_factory=dict)]
-
-
-@docs_group('Data structures')
-class UnknownEvent(BaseModel):
-    name: str
-    data: Annotated[dict[str, Any], Field(default_factory=dict)]
-
-
-EventMessage = Union[
-    PersistStateEvent,
-    SystemInfoEvent,
-    MigratingEvent,
-    AbortingEvent,
-    ExitEvent,
-    EventWithoutData,
-]
-
-
-event_data_adapter: TypeAdapter[EventMessage | DeprecatedEvent | UnknownEvent] = TypeAdapter(
-    Union[
-        Annotated[
-            EventMessage,
-            Discriminator('name'),
-        ],
-        DeprecatedEvent,
-        UnknownEvent,
-    ]
-)
-
-
-@docs_group('Classes')
-class PlatformEventManager(EventManager):
-    """A class for managing Actor events.
-
-    You shouldn't use this class directly,
-    but instead use it via the `Actor.on()` and `Actor.off()` methods.
-    """
-
-    _platform_events_websocket: websockets.asyncio.client.ClientConnection | None = None
-    _process_platform_messages_task: asyncio.Task | None = None
-    _send_system_info_interval_task: asyncio.Task | None = None
-    _connected_to_platform_websocket: asyncio.Future = asyncio.Future()
-
-    def __init__(self, config: Configuration, **kwargs: Unpack[EventManagerOptions]) -> None:
-        """Create an instance of the EventManager.
-
-        Args:
-            config: The Actor configuration to be used in this event manager.
-            kwargs: Event manager options - forwarded to the base class
-        """
-        super().__init__(**kwargs)
-
-        self._config = config
-        self._listener_tasks = set()
-        self._connected_to_platform_websocket = asyncio.Future[bool]()
-
-    @override
-    async def __aenter__(self) -> Self:
-        await super().__aenter__()
-        self._connected_to_platform_websocket = asyncio.Future()
-
-        # Run tasks but don't await them
-        if self._config.actor_events_ws_url:
-            self._process_platform_messages_task = asyncio.create_task(
-                self._process_platform_messages(self._config.actor_events_ws_url)
-            )
-            is_connected = await self._connected_to_platform_websocket
-            if not is_connected:
-                raise RuntimeError('Error connecting to platform events websocket!')
-        else:
-            logger.debug('APIFY_ACTOR_EVENTS_WS_URL env var not set, no events from Apify platform will be emitted.')
-
-        return self
-
-    @override
-    async def __aexit__(
-        self,
-        exc_type: type[BaseException] | None,
-        exc_value: BaseException | None,
-        exc_traceback: TracebackType | None,
-    ) -> None:
-        if self._platform_events_websocket:
-            await self._platform_events_websocket.close()
-
-        if self._process_platform_messages_task:
-            await self._process_platform_messages_task
-
-        await super().__aexit__(exc_type, exc_value, exc_traceback)
-
-    async def _process_platform_messages(self, ws_url: str) -> None:
-        try:
-            async with websockets.asyncio.client.connect(ws_url) as websocket:
-                self._platform_events_websocket = websocket
-                self._connected_to_platform_websocket.set_result(True)
-
-                async for message in websocket:
-                    try:
-                        parsed_message = event_data_adapter.validate_json(message)
-
-                        if isinstance(parsed_message, DeprecatedEvent):
-                            continue
-
-                        if isinstance(parsed_message, UnknownEvent):
-                            logger.info(
-                                f'Unknown message received: event_name={parsed_message.name}, '
-                                f'event_data={parsed_message.data}'
-                            )
-                            continue
-
-                        self.emit(
-                            event=parsed_message.name,
-                            event_data=parsed_message.data
-                            if not isinstance(parsed_message.data, SystemInfoEventData)
-                            else parsed_message.data.to_crawlee_format(self._config.dedicated_cpus or 1),
-                        )
-
-                        if parsed_message.name == Event.MIGRATING:
-                            await self._emit_persist_state_event_rec_task.stop()
-                            self.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=True))
-                    except Exception:
-                        logger.exception('Cannot parse Actor event', extra={'message': message})
-        except Exception:
-            logger.exception('Error in websocket connection')
-            self._connected_to_platform_websocket.set_result(False)

apify/apify_storage_client/__init__.py

@@ -1,3 +0,0 @@
-from apify.apify_storage_client._apify_storage_client import ApifyStorageClient
-
-__all__ = ['ApifyStorageClient']

apify/apify_storage_client/_apify_storage_client.py

@@ -1,72 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from typing_extensions import override
-
-from apify_client import ApifyClientAsync
-from crawlee._utils.crypto import crypto_random_object_id
-from crawlee.storage_clients import StorageClient
-
-from apify._utils import docs_group
-from apify.apify_storage_client._dataset_client import DatasetClient
-from apify.apify_storage_client._dataset_collection_client import DatasetCollectionClient
-from apify.apify_storage_client._key_value_store_client import KeyValueStoreClient
-from apify.apify_storage_client._key_value_store_collection_client import KeyValueStoreCollectionClient
-from apify.apify_storage_client._request_queue_client import RequestQueueClient
-from apify.apify_storage_client._request_queue_collection_client import RequestQueueCollectionClient
-
-if TYPE_CHECKING:
-    from apify._configuration import Configuration
-
-
-@docs_group('Classes')
-class ApifyStorageClient(StorageClient):
-    """A storage client implementation based on the Apify platform storage."""
-
-    def __init__(self, *, configuration: Configuration) -> None:
-        self._client_key = crypto_random_object_id()
-        self._apify_client = ApifyClientAsync(
-            token=configuration.token,
-            api_url=configuration.api_base_url,
-            max_retries=8,
-            min_delay_between_retries_millis=500,
-            timeout_secs=360,
-        )
-        self._configuration = configuration
-
-    @classmethod
-    def from_config(cls, config: Configuration) -> ApifyStorageClient:
-        return cls(configuration=config)
-
-    @override
-    def dataset(self, id: str) -> DatasetClient:
-        return DatasetClient(self._apify_client.dataset(id))
-
-    @override
-    def datasets(self) -> DatasetCollectionClient:
-        return DatasetCollectionClient(self._apify_client.datasets())
-
-    @override
-    def key_value_store(self, id: str) -> KeyValueStoreClient:
-        return KeyValueStoreClient(self._apify_client.key_value_store(id), self._configuration.api_public_base_url)
-
-    @override
-    def key_value_stores(self) -> KeyValueStoreCollectionClient:
-        return KeyValueStoreCollectionClient(self._apify_client.key_value_stores())
-
-    @override
-    def request_queue(self, id: str) -> RequestQueueClient:
-        return RequestQueueClient(self._apify_client.request_queue(id, client_key=self._client_key))
-
-    @override
-    def request_queues(self) -> RequestQueueCollectionClient:
-        return RequestQueueCollectionClient(self._apify_client.request_queues())
-
-    @override
-    async def purge_on_start(self) -> None:
-        pass
-
-    @override
-    def get_rate_limit_errors(self) -> dict[int, int]:
-        return self._apify_client.stats.rate_limit_errors

apify/apify_storage_client/_dataset_client.py

@@ -1,190 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from typing_extensions import override
-
-from crawlee.storage_clients._base import DatasetClient as BaseDatasetClient
-from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
-
-if TYPE_CHECKING:
-    from collections.abc import AsyncIterator
-    from contextlib import AbstractAsyncContextManager
-
-    from httpx import Response
-
-    from apify_client.clients import DatasetClientAsync
-    from crawlee._types import JsonSerializable
-
-
-class DatasetClient(BaseDatasetClient):
-    """Dataset resource client implementation based on the Apify platform storage."""
-
-    def __init__(self, apify_dataset_client: DatasetClientAsync) -> None:
-        self._client = apify_dataset_client
-
-    @override
-    async def get(self) -> DatasetMetadata | None:
-        result = await self._client.get()
-        return DatasetMetadata.model_validate(result) if result else None
-
-    @override
-    async def update(
-        self,
-        *,
-        name: str | None = None,
-    ) -> DatasetMetadata:
-        return DatasetMetadata.model_validate(
-            await self._client.update(
-                name=name,
-            )
-        )
-
-    @override
-    async def delete(self) -> None:
-        await self._client.delete()
-
-    @override
-    async def list_items(
-        self,
-        *,
-        offset: int | None = 0,
-        limit: int | None = BaseDatasetClient._LIST_ITEMS_LIMIT,  # noqa: SLF001
-        clean: bool = False,
-        desc: bool = False,
-        fields: list[str] | None = None,
-        omit: list[str] | None = None,
-        unwind: str | None = None,
-        skip_empty: bool = False,
-        skip_hidden: bool = False,
-        flatten: list[str] | None = None,
-        view: str | None = None,
-    ) -> DatasetItemsListPage:
-        return DatasetItemsListPage.model_validate(
-            vars(
-                await self._client.list_items(
-                    offset=offset,
-                    limit=limit,
-                    clean=clean,
-                    desc=desc,
-                    fields=fields,
-                    omit=omit,
-                    unwind=unwind,
-                    skip_empty=skip_empty,
-                    skip_hidden=skip_hidden,
-                    flatten=flatten,
-                    view=view,
-                )
-            )
-        )
-
-    @override
-    async def iterate_items(
-        self,
-        *,
-        offset: int = 0,
-        limit: int | None = None,
-        clean: bool = False,
-        desc: bool = False,
-        fields: list[str] | None = None,
-        omit: list[str] | None = None,
-        unwind: str | None = None,
-        skip_empty: bool = False,
-        skip_hidden: bool = False,
-    ) -> AsyncIterator[dict]:
-        async for item in self._client.iterate_items(
-            offset=offset,
-            limit=limit,
-            clean=clean,
-            desc=desc,
-            fields=fields,
-            omit=omit,
-            unwind=unwind,
-            skip_empty=skip_empty,
-            skip_hidden=skip_hidden,
-        ):
-            yield item
-
-    @override
-    async def get_items_as_bytes(
-        self,
-        *,
-        item_format: str = 'json',
-        offset: int | None = None,
-        limit: int | None = None,
-        desc: bool = False,
-        clean: bool = False,
-        bom: bool = False,
-        delimiter: str | None = None,
-        fields: list[str] | None = None,
-        omit: list[str] | None = None,
-        unwind: str | None = None,
-        skip_empty: bool = False,
-        skip_header_row: bool = False,
-        skip_hidden: bool = False,
-        xml_root: str | None = None,
-        xml_row: str | None = None,
-        flatten: list[str] | None = None,
-    ) -> bytes:
-        return await self._client.get_items_as_bytes(
-            item_format=item_format,
-            offset=offset,
-            limit=limit,
-            desc=desc,
-            clean=clean,
-            bom=bom,
-            delimiter=delimiter,
-            fields=fields,
-            omit=omit,
-            unwind=unwind,
-            skip_empty=skip_empty,
-            skip_header_row=skip_header_row,
-            skip_hidden=skip_hidden,
-            xml_root=xml_root,
-            xml_row=xml_row,
-            flatten=flatten,
-        )
-
-    @override
-    async def stream_items(
-        self,
-        *,
-        item_format: str = 'json',
-        offset: int | None = None,
-        limit: int | None = None,
-        desc: bool = False,
-        clean: bool = False,
-        bom: bool = False,
-        delimiter: str | None = None,
-        fields: list[str] | None = None,
-        omit: list[str] | None = None,
-        unwind: str | None = None,
-        skip_empty: bool = False,
-        skip_header_row: bool = False,
-        skip_hidden: bool = False,
-        xml_root: str | None = None,
-        xml_row: str | None = None,
-    ) -> AbstractAsyncContextManager[Response | None]:
-        return self._client.stream_items(
-            item_format=item_format,
-            offset=offset,
-            limit=limit,
-            desc=desc,
-            clean=clean,
-            bom=bom,
-            delimiter=delimiter,
-            fields=fields,
-            omit=omit,
-            unwind=unwind,
-            skip_empty=skip_empty,
-            skip_header_row=skip_header_row,
-            skip_hidden=skip_hidden,
-            xml_root=xml_root,
-            xml_row=xml_row,
-        )
-
-    @override
-    async def push_items(self, items: JsonSerializable) -> None:
-        await self._client.push_items(
-            items=items,
-        )

apify/apify_storage_client/_dataset_collection_client.py

@@ -1,51 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from typing_extensions import override
-
-from crawlee.storage_clients._base import DatasetCollectionClient as BaseDatasetCollectionClient
-from crawlee.storage_clients.models import DatasetListPage, DatasetMetadata
-
-if TYPE_CHECKING:
-    from apify_client.clients import DatasetCollectionClientAsync
-
-
-class DatasetCollectionClient(BaseDatasetCollectionClient):
-    """Dataset collection resource client implementation based on the Apify platform storage."""
-
-    def __init__(self, apify_dataset_collection_client: DatasetCollectionClientAsync) -> None:
-        self._client = apify_dataset_collection_client
-
-    @override
-    async def get_or_create(
-        self,
-        *,
-        id: str | None = None,
-        name: str | None = None,
-        schema: dict | None = None,
-    ) -> DatasetMetadata:
-        return DatasetMetadata.model_validate(
-            await self._client.get_or_create(
-                name=id if id is not None else name,
-                schema=schema,
-            )
-        )
-
-    @override
-    async def list(
-        self,
-        *,
-        unnamed: bool = False,
-        limit: int | None = None,
-        offset: int | None = None,
-        desc: bool = False,
-    ) -> DatasetListPage:
-        return DatasetListPage.model_validate(
-            await self._client.list(
-                unnamed=unnamed,
-                limit=limit,
-                offset=offset,
-                desc=desc,
-            )
-        )

apify/apify_storage_client/_key_value_store_client.py

@@ -1,109 +0,0 @@
-from __future__ import annotations
-
-from contextlib import asynccontextmanager
-from typing import TYPE_CHECKING, Any
-
-from typing_extensions import override
-from yarl import URL
-
-from crawlee.storage_clients._base import KeyValueStoreClient as BaseKeyValueStoreClient
-from crawlee.storage_clients.models import KeyValueStoreListKeysPage, KeyValueStoreMetadata, KeyValueStoreRecord
-
-from apify._crypto import create_hmac_signature
-
-if TYPE_CHECKING:
-    from collections.abc import AsyncIterator
-    from contextlib import AbstractAsyncContextManager
-
-    from httpx import Response
-
-    from apify_client.clients import KeyValueStoreClientAsync
-
-
-class KeyValueStoreClient(BaseKeyValueStoreClient):
-    """Key-value store resource client implementation based on the Apify platform storage."""
-
-    def __init__(self, apify_key_value_store_client: KeyValueStoreClientAsync, api_public_base_url: str) -> None:
-        self._client = apify_key_value_store_client
-        self._api_public_base_url = api_public_base_url
-
-    @override
-    async def get(self) -> KeyValueStoreMetadata | None:
-        result = await self._client.get()
-        return KeyValueStoreMetadata.model_validate(result) if result else None
-
-    @override
-    async def update(
-        self,
-        *,
-        name: str | None = None,
-    ) -> KeyValueStoreMetadata:
-        return KeyValueStoreMetadata.model_validate(await self._client.update())
-
-    @override
-    async def delete(self) -> None:
-        await self._client.delete()
-
-    @override
-    async def list_keys(
-        self,
-        *,
-        limit: int = 1000,
-        exclusive_start_key: str | None = None,
-    ) -> KeyValueStoreListKeysPage:
-        return KeyValueStoreListKeysPage.model_validate(await self._client.list_keys())
-
-    @override
-    async def get_record(self, key: str) -> KeyValueStoreRecord | None:
-        result = await self._client.get_record(key)
-        return KeyValueStoreRecord.model_validate(result) if result else None
-
-    @override
-    async def get_record_as_bytes(self, key: str) -> KeyValueStoreRecord | None:
-        result = await self._client.get_record_as_bytes(key)
-        return KeyValueStoreRecord.model_validate(result) if result else None
-
-    @override
-    async def stream_record(self, key: str) -> AbstractAsyncContextManager[KeyValueStoreRecord[Response] | None]:
-        return self._stream_record_internal(key)
-
-    @asynccontextmanager
-    async def _stream_record_internal(self, key: str) -> AsyncIterator[KeyValueStoreRecord[Response] | None]:
-        async with self._client.stream_record(key) as response:
-            yield KeyValueStoreRecord.model_validate(response)
-
-    @override
-    async def set_record(self, key: str, value: Any, content_type: str | None = None) -> None:
-        await self._client.set_record(
-            key=key,
-            value=value,
-            content_type=content_type,
-        )
-
-    @override
-    async def delete_record(self, key: str) -> None:
-        await self._client.delete_record(
-            key=key,
-        )
-
-    async def get_public_url(self, key: str) -> str:
-        """Get a URL for the given key that may be used to publicly access the value in the remote key-value store.
-
-        Args:
-            key: The key for which the URL should be generated.
-        """
-        if self._client.resource_id is None:
-            raise ValueError('resource_id cannot be None when generating a public URL')
-
-        public_url = (
-            URL(self._api_public_base_url) / 'v2' / 'key-value-stores' / self._client.resource_id / 'records' / key
-        )
-
-        key_value_store = await self.get()
-
-        if key_value_store is not None and isinstance(key_value_store.model_extra, dict):
-            url_signing_secret_key = key_value_store.model_extra.get('urlSigningSecretKey')
-            if url_signing_secret_key:
-                public_url = public_url.with_query(signature=create_hmac_signature(url_signing_secret_key, key))
-
-        return str(public_url)