apify 1.7.3b4__py3-none-any.whl → 2.0.0__py3-none-any.whl

apify/apify_storage_client/_dataset_client.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from typing_extensions import override
+
+from crawlee.base_storage_client import BaseDatasetClient, DatasetItemsListPage, DatasetMetadata
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+    from contextlib import AbstractAsyncContextManager
+
+    from httpx import Response
+
+    from apify_client.clients import DatasetClientAsync
+    from crawlee._types import JsonSerializable
+
+
+class DatasetClient(BaseDatasetClient):
+    """Dataset resource client implementation based on the Apify platform storage."""
+
+    def __init__(self, apify_dataset_client: DatasetClientAsync) -> None:
+        self._client = apify_dataset_client
+
+    @override
+    async def get(self) -> DatasetMetadata | None:
+        result = await self._client.get()
+        return DatasetMetadata.model_validate(result) if result else None
+
+    @override
+    async def update(
+        self,
+        *,
+        name: str | None = None,
+    ) -> DatasetMetadata:
+        return DatasetMetadata.model_validate(
+            await self._client.update(
+                name=name,
+            )
+        )
+
+    @override
+    async def delete(self) -> None:
+        await self._client.delete()
+
+    @override
+    async def list_items(
+        self,
+        *,
+        offset: int | None = 0,
+        limit: int | None = BaseDatasetClient._LIST_ITEMS_LIMIT,  # noqa: SLF001
+        clean: bool = False,
+        desc: bool = False,
+        fields: list[str] | None = None,
+        omit: list[str] | None = None,
+        unwind: str | None = None,
+        skip_empty: bool = False,
+        skip_hidden: bool = False,
+        flatten: list[str] | None = None,
+        view: str | None = None,
+    ) -> DatasetItemsListPage:
+        return DatasetItemsListPage.model_validate(
+            vars(
+                await self._client.list_items(
+                    offset=offset,
+                    limit=limit,
+                    clean=clean,
+                    desc=desc,
+                    fields=fields,
+                    omit=omit,
+                    unwind=unwind,
+                    skip_empty=skip_empty,
+                    skip_hidden=skip_hidden,
+                    flatten=flatten,
+                    view=view,
+                )
+            )
+        )
+
+    @override
+    async def iterate_items(
+        self,
+        *,
+        offset: int = 0,
+        limit: int | None = None,
+        clean: bool = False,
+        desc: bool = False,
+        fields: list[str] | None = None,
+        omit: list[str] | None = None,
+        unwind: str | None = None,
+        skip_empty: bool = False,
+        skip_hidden: bool = False,
+    ) -> AsyncIterator[dict]:
+        return self._client.iterate_items(
+            offset=offset,
+            limit=limit,
+            clean=clean,
+            desc=desc,
+            fields=fields,
+            omit=omit,
+            unwind=unwind,
+            skip_empty=skip_empty,
+            skip_hidden=skip_hidden,
+        )
+
+    @override
+    async def get_items_as_bytes(
+        self,
+        *,
+        item_format: str = 'json',
+        offset: int | None = None,
+        limit: int | None = None,
+        desc: bool = False,
+        clean: bool = False,
+        bom: bool = False,
+        delimiter: str | None = None,
+        fields: list[str] | None = None,
+        omit: list[str] | None = None,
+        unwind: str | None = None,
+        skip_empty: bool = False,
+        skip_header_row: bool = False,
+        skip_hidden: bool = False,
+        xml_root: str | None = None,
+        xml_row: str | None = None,
+        flatten: list[str] | None = None,
+    ) -> bytes:
+        return await self._client.get_items_as_bytes(
+            item_format=item_format,
+            offset=offset,
+            limit=limit,
+            desc=desc,
+            clean=clean,
+            bom=bom,
+            delimiter=delimiter,
+            fields=fields,
+            omit=omit,
+            unwind=unwind,
+            skip_empty=skip_empty,
+            skip_header_row=skip_header_row,
+            skip_hidden=skip_hidden,
+            xml_root=xml_root,
+            xml_row=xml_row,
+            flatten=flatten,
+        )
+
+    @override
+    async def stream_items(
+        self,
+        *,
+        item_format: str = 'json',
+        offset: int | None = None,
+        limit: int | None = None,
+        desc: bool = False,
+        clean: bool = False,
+        bom: bool = False,
+        delimiter: str | None = None,
+        fields: list[str] | None = None,
+        omit: list[str] | None = None,
+        unwind: str | None = None,
+        skip_empty: bool = False,
+        skip_header_row: bool = False,
+        skip_hidden: bool = False,
+        xml_root: str | None = None,
+        xml_row: str | None = None,
+    ) -> AbstractAsyncContextManager[Response | None]:
+        return self._client.stream_items(
+            item_format=item_format,
+            offset=offset,
+            limit=limit,
+            desc=desc,
+            clean=clean,
+            bom=bom,
+            delimiter=delimiter,
+            fields=fields,
+            omit=omit,
+            unwind=unwind,
+            skip_empty=skip_empty,
+            skip_header_row=skip_header_row,
+            skip_hidden=skip_hidden,
+            xml_root=xml_root,
+            xml_row=xml_row,
+        )
+
+    @override
+    async def push_items(self, items: JsonSerializable) -> None:
+        await self._client.push_items(
+            items=items,
+        )

apify/apify_storage_client/_dataset_collection_client.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from typing_extensions import override
+
+from crawlee.base_storage_client import BaseDatasetCollectionClient, DatasetListPage, DatasetMetadata
+
+if TYPE_CHECKING:
+    from apify_client.clients import DatasetCollectionClientAsync
+
+
+class DatasetCollectionClient(BaseDatasetCollectionClient):
+    """Dataset collection resource client implementation based on the Apify platform storage."""
+
+    def __init__(self, apify_dataset_collection_client: DatasetCollectionClientAsync) -> None:
+        self._client = apify_dataset_collection_client
+
+    @override
+    async def get_or_create(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        schema: dict | None = None,
+    ) -> DatasetMetadata:
+        return DatasetMetadata.model_validate(
+            await self._client.get_or_create(
+                name=id if id is not None else name,
+                schema=schema,
+            )
+        )
+
+    @override
+    async def list(
+        self,
+        *,
+        unnamed: bool = False,
+        limit: int | None = None,
+        offset: int | None = None,
+        desc: bool = False,
+    ) -> DatasetListPage:
+        return DatasetListPage.model_validate(
+            await self._client.list(
+                unnamed=unnamed,
+                limit=limit,
+                offset=offset,
+                desc=desc,
+            )
+        )

apify/apify_storage_client/_key_value_store_client.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from typing import TYPE_CHECKING, Any
+
+from typing_extensions import override
+
+from crawlee.base_storage_client import (
+    BaseKeyValueStoreClient,
+    KeyValueStoreListKeysPage,
+    KeyValueStoreMetadata,
+    KeyValueStoreRecord,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+    from contextlib import AbstractAsyncContextManager
+
+    from httpx import Response
+
+    from apify_client.clients import KeyValueStoreClientAsync
+
+
+class KeyValueStoreClient(BaseKeyValueStoreClient):
+    """Key-value store resource client implementation based on the Apify platform storage."""
+
+    def __init__(self, apify_key_value_store_client: KeyValueStoreClientAsync, api_public_base_url: str) -> None:
+        self._client = apify_key_value_store_client
+        self._api_public_base_url = api_public_base_url
+
+    @override
+    async def get(self) -> KeyValueStoreMetadata | None:
+        result = await self._client.get()
+        return KeyValueStoreMetadata.model_validate(result) if result else None
+
+    @override
+    async def update(
+        self,
+        *,
+        name: str | None = None,
+    ) -> KeyValueStoreMetadata:
+        return KeyValueStoreMetadata.model_validate(await self._client.update())
+
+    @override
+    async def delete(self) -> None:
+        await self._client.delete()
+
+    @override
+    async def list_keys(
+        self,
+        *,
+        limit: int = 1000,
+        exclusive_start_key: str | None = None,
+    ) -> KeyValueStoreListKeysPage:
+        return KeyValueStoreListKeysPage.model_validate(await self._client.list_keys())
+
+    @override
+    async def get_record(self, key: str) -> KeyValueStoreRecord | None:
+        result = await self._client.get_record(key)
+        return KeyValueStoreRecord.model_validate(result) if result else None
+
+    @override
+    async def get_record_as_bytes(self, key: str) -> KeyValueStoreRecord | None:
+        result = await self._client.get_record_as_bytes(key)
+        return KeyValueStoreRecord.model_validate(result) if result else None
+
+    @override
+    async def stream_record(self, key: str) -> AbstractAsyncContextManager[KeyValueStoreRecord[Response] | None]:
+        return self._stream_record_internal(key)
+
+    @asynccontextmanager
+    async def _stream_record_internal(self, key: str) -> AsyncIterator[KeyValueStoreRecord[Response] | None]:
+        async with self._client.stream_record(key) as response:
+            yield KeyValueStoreRecord.model_validate(response)
+
+    @override
+    async def set_record(self, key: str, value: Any, content_type: str | None = None) -> None:
+        await self._client.set_record(
+            key=key,
+            value=value,
+            content_type=content_type,
+        )
+
+    @override
+    async def delete_record(self, key: str) -> None:
+        await self._client.delete_record(
+            key=key,
+        )
+
+    async def get_public_url(self, key: str) -> str:
+        """Get a URL for the given key that may be used to publicly access the value in the remote key-value store.
+
+        Args:
+            key: The key for which the URL should be generated.
+        """
+        public_api_url = self._api_public_base_url
+
+        return f'{public_api_url}/v2/key-value-stores/{self._client.resource_id}/records/{key}'

apify/apify_storage_client/_key_value_store_collection_client.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from typing_extensions import override
+
+from crawlee.base_storage_client import BaseKeyValueStoreCollectionClient, KeyValueStoreListPage, KeyValueStoreMetadata
+
+if TYPE_CHECKING:
+    from apify_client.clients import KeyValueStoreCollectionClientAsync
+
+
+class KeyValueStoreCollectionClient(BaseKeyValueStoreCollectionClient):
+    """Key-value store collection resource client implementation based on the Apify platform storage."""
+
+    def __init__(self, apify_dataset_collection_client: KeyValueStoreCollectionClientAsync) -> None:
+        self._client = apify_dataset_collection_client
+
+    @override
+    async def get_or_create(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        schema: dict | None = None,
+    ) -> KeyValueStoreMetadata:
+        return KeyValueStoreMetadata.model_validate(
+            await self._client.get_or_create(
+                name=id if id is not None else name,
+                schema=schema,
+            )
+        )
+
+    @override
+    async def list(
+        self,
+        *,
+        unnamed: bool = False,
+        limit: int | None = None,
+        offset: int | None = None,
+        desc: bool = False,
+    ) -> KeyValueStoreListPage:
+        return KeyValueStoreListPage.model_validate(
+            await self._client.list(
+                unnamed=unnamed,
+                limit=limit,
+                offset=offset,
+                desc=desc,
+            )
+        )

apify/apify_storage_client/_request_queue_client.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from more_itertools import chunked
+from typing_extensions import override
+
+from crawlee import Request
+from crawlee.base_storage_client import (
+    BaseRequestQueueClient,
+    BatchRequestsOperationResponse,
+    ProcessedRequest,
+    ProlongRequestLockResponse,
+    RequestQueueHead,
+    RequestQueueHeadWithLocks,
+    RequestQueueMetadata,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from apify_client.clients import RequestQueueClientAsync
+
+
+class RequestQueueClient(BaseRequestQueueClient):
+    """Request queue resource client implementation based on the Apify platform storage."""
+
+    def __init__(self, apify_request_queue_client: RequestQueueClientAsync) -> None:
+        self._client = apify_request_queue_client
+
+    @override
+    async def get(self) -> RequestQueueMetadata | None:
+        result = await self._client.get()
+        return RequestQueueMetadata.model_validate({'resourceDirectory': ''} | result) if result else None
+
+    @override
+    async def update(
+        self,
+        *,
+        name: str | None = None,
+    ) -> RequestQueueMetadata:
+        return RequestQueueMetadata.model_validate(
+            {'resourceDirectory': ''}
+            | await self._client.update(
+                name=name,
+            )
+        )
+
+    @override
+    async def delete(self) -> None:
+        await self._client.delete()
+
+    @override
+    async def list_head(self, *, limit: int | None = None) -> RequestQueueHead:
+        return RequestQueueHead.model_validate(
+            await self._client.list_head(
+                limit=limit,
+            ),
+        )
+
+    @override
+    async def list_and_lock_head(self, *, lock_secs: int, limit: int | None = None) -> RequestQueueHeadWithLocks:
+        return RequestQueueHeadWithLocks.model_validate(
+            await self._client.list_and_lock_head(
+                lock_secs=lock_secs,
+                limit=limit,
+            )
+        )
+
+    @override
+    async def add_request(
+        self,
+        request: Request,
+        *,
+        forefront: bool = False,
+    ) -> ProcessedRequest:
+        return ProcessedRequest.model_validate(
+            {'id': request.id, 'uniqueKey': request.unique_key}
+            | await self._client.add_request(
+                request=request.model_dump(
+                    by_alias=True,
+                    exclude={
+                        'id',
+                        'json_',
+                        'order_no',
+                        'query_params',
+                        'data',
+                    },
+                ),
+                forefront=forefront,
+            )
+        )
+
+    @override
+    async def get_request(self, request_id: str) -> Request | None:
+        result = await self._client.get_request(request_id)
+        return Request.model_validate(result) if result else None
+
+    @override
+    async def update_request(
+        self,
+        request: Request,
+        *,
+        forefront: bool = False,
+    ) -> ProcessedRequest:
+        return ProcessedRequest.model_validate(
+            {'id': request.id, 'uniqueKey': request.unique_key}
+            | await self._client.update_request(
+                request=request.model_dump(
+                    by_alias=True,
+                    exclude={
+                        'json_',
+                        'order_no',
+                        'query_params',
+                        'data',
+                    },
+                ),
+                forefront=forefront,
+            )
+        )
+
+    @override
+    async def delete_request(self, request_id: str) -> None:
+        await self._client.delete_request(request_id)
+
+    @override
+    async def prolong_request_lock(
+        self,
+        request_id: str,
+        *,
+        forefront: bool = False,
+        lock_secs: int,
+    ) -> ProlongRequestLockResponse:
+        return ProlongRequestLockResponse.model_validate(
+            await self._client.prolong_request_lock(
+                request_id=request_id,
+                forefront=forefront,
+                lock_secs=lock_secs,
+            )
+        )
+
+    @override
+    async def delete_request_lock(
+        self,
+        request_id: str,
+        *,
+        forefront: bool = False,
+    ) -> None:
+        await self._client.delete_request_lock(
+            request_id=request_id,
+            forefront=forefront,
+        )
+
+    @override
+    async def batch_add_requests(
+        self,
+        requests: Sequence[Request],
+        *,
+        forefront: bool = False,
+    ) -> BatchRequestsOperationResponse:
+        processed = []
+        unprocessed = []
+
+        for chunk in chunked(requests, 25):  # The API endpoint won't accept more than 25 requests at once
+            response = await self._client.batch_add_requests(
+                requests=[
+                    r.model_dump(
+                        by_alias=True,
+                        exclude={
+                            'id',
+                            'json_',
+                            'order_no',
+                            'query_params',
+                            'data',
+                        },
+                    )
+                    for r in chunk
+                ],
+                forefront=forefront,
+            )
+            processed.extend(response['processedRequests'])
+            unprocessed.extend(response['unprocessedRequests'])
+
+        return BatchRequestsOperationResponse.model_validate(
+            {
+                'processedRequests': processed,
+                'unprocessedRequests': unprocessed,
+            }
+        )
+
+    @override
+    async def batch_delete_requests(self, requests: list[Request]) -> BatchRequestsOperationResponse:
+        return BatchRequestsOperationResponse.model_validate(
+            await self._client.batch_delete_requests(
+                requests=[
+                    r.model_dump(
+                        by_alias=True,
+                        exclude={
+                            'json_',
+                            'order_no',
+                            'query_params',
+                            'data',
+                        },
+                    )
+                    for r in requests
+                ],
+            )
+        )

apify/apify_storage_client/_request_queue_collection_client.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from typing_extensions import override
+
+from crawlee.base_storage_client import BaseRequestQueueCollectionClient, RequestQueueListPage, RequestQueueMetadata
+
+if TYPE_CHECKING:
+    from apify_client.clients import RequestQueueCollectionClientAsync
+
+
+class RequestQueueCollectionClient(BaseRequestQueueCollectionClient):
+    """Request queue collection resource client implementation based on the Apify platform storage."""
+
+    def __init__(self, apify_request_queue_collection_client: RequestQueueCollectionClientAsync) -> None:
+        self._client = apify_request_queue_collection_client
+
+    @override
+    async def get_or_create(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        schema: dict | None = None,
+    ) -> RequestQueueMetadata:
+        return RequestQueueMetadata.model_validate(
+            {'resourceDirectory': ''}
+            | await self._client.get_or_create(
+                name=id if id is not None else name,
+            )
+        )
+
+    @override
+    async def list(
+        self,
+        *,
+        unnamed: bool = False,
+        limit: int | None = None,
+        offset: int | None = None,
+        desc: bool = False,
+    ) -> RequestQueueListPage:
+        return RequestQueueListPage.model_validate(
+            await self._client.list(
+                unnamed=unnamed,
+                limit=limit,
+                offset=offset,
+                desc=desc,
+            )
+        )