apify 3.0.3b1__tar.gz → 3.0.4b1__tar.gz

{apify-3.0.3b1 → apify-3.0.4b1}/CHANGELOG.md

@@ -3,14 +3,21 @@
 All notable changes to this project will be documented in this file.
 
 <!-- git-cliff-unreleased-start -->
-## 3.0.3 - **not yet released**
+## 3.0.4 - **not yet released**
 
 ### 🐛 Bug Fixes
 
-- Cache requests in RQ implementations by `id` ([#633](https://github.com/apify/apify-sdk-python/pull/633)) ([76886ce](https://github.com/apify/apify-sdk-python/commit/76886ce496165346a01f67e018547287c211ea54)) by [@Pijukatel](https://github.com/Pijukatel), closes [#630](https://github.com/apify/apify-sdk-python/issues/630)
+- Fix type of `cloud_storage_client` in `SmartApifyStorageClient` ([#642](https://github.com/apify/apify-sdk-python/pull/642)) ([3bf285d](https://github.com/apify/apify-sdk-python/commit/3bf285d60f507730954986a80c19ed2e27a38f9c)) by [@vdusek](https://github.com/vdusek)
 
 
 <!-- git-cliff-unreleased-end -->
+## [3.0.3](https://github.com/apify/apify-sdk-python/releases/tag/v3.0.3) (2025-10-21)
+
+### 🐛 Bug Fixes
+
+- Cache requests in RQ implementations by `id` ([#633](https://github.com/apify/apify-sdk-python/pull/633)) ([76886ce](https://github.com/apify/apify-sdk-python/commit/76886ce496165346a01f67e018547287c211ea54)) by [@Pijukatel](https://github.com/Pijukatel), closes [#630](https://github.com/apify/apify-sdk-python/issues/630)
+
+
 ## [3.0.2](https://github.com/apify/apify-sdk-python/releases/tag/v3.0.2) (2025-10-17)
 
 ### 🐛 Bug Fixes

{apify-3.0.3b1 → apify-3.0.4b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 3.0.3b1
+Version: 3.0.4b1
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog

{apify-3.0.3b1 → apify-3.0.4b1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apify"
-version = "3.0.3b1"
+version = "3.0.4b1"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{apify-3.0.3b1 → apify-3.0.4b1}/src/apify/_actor.py

@@ -48,7 +48,6 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
     from crawlee.proxy_configuration import _NewUrlFunction
-    from crawlee.storage_clients import StorageClient
 
     from apify._models import Webhook
 
@@ -140,7 +139,6 @@ class _ActorType:
         # `__init__` method should not be considered final.
 
         self._apify_client: ApifyClientAsync | None = None
-        self._local_storage_client: StorageClient | None = None
 
         self._is_exiting = False
         self._is_initialized = False

{apify-3.0.3b1 → apify-3.0.4b1}/src/apify/storage_clients/_apify/_models.py

@@ -120,7 +120,7 @@ class RequestQueueStats(BaseModel):
     """The number of request queue reads."""
 
     storage_bytes: Annotated[int, Field(alias='storageBytes', default=0)]
-    """Storage size in Bytes."""
+    """Storage size in bytes."""
 
     write_count: Annotated[int, Field(alias='writeCount', default=0)]
     """The number of request queue writes."""

apify-3.0.4b1/src/apify/storage_clients/_apify/_storage_client.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Literal
+
+from typing_extensions import override
+
+from crawlee.storage_clients._base import StorageClient
+
+from ._dataset_client import ApifyDatasetClient
+from ._key_value_store_client import ApifyKeyValueStoreClient
+from ._request_queue_client import ApifyRequestQueueClient
+from ._utils import hash_api_base_url_and_token
+from apify._configuration import Configuration as ApifyConfiguration
+from apify._utils import docs_group
+
+if TYPE_CHECKING:
+    from collections.abc import Hashable
+
+    from crawlee.configuration import Configuration as CrawleeConfiguration
+
+
+@docs_group('Storage clients')
+class ApifyStorageClient(StorageClient):
+    """Apify platform implementation of the storage client.
+
+    This storage client provides access to datasets, key-value stores, and request queues that persist data
+    to the Apify platform. Each storage type is implemented with its own specific Apify client that stores data
+    in the cloud, making it accessible from anywhere.
+
+    The communication with the Apify platform is handled via the Apify API client for Python, which is an HTTP API
+    wrapper. For maximum efficiency and performance of the storage clients, various caching mechanisms are used to
+    minimize the number of API calls made to the Apify platform. Data can be inspected and manipulated through
+    the Apify console web interface or via the Apify API.
+
+    The request queue client supports two access modes controlled by the `request_queue_access` parameter:
+
+    ### Single mode
+
+    The `single` mode is optimized for scenarios with only one consumer. It minimizes API calls, making it faster
+    and more cost-efficient compared to the `shared` mode. This option is ideal when a single Actor is responsible
+    for consuming the entire request queue. Using multiple consumers simultaneously may lead to inconsistencies
+    or unexpected behavior.
+
+    In this mode, multiple producers can safely add new requests, but forefront requests may not be processed
+    immediately, as the client relies on local head estimation instead of frequent forefront fetching. Requests can
+    also be added or marked as handled by other clients, but they must not be deleted or modified, since such changes
+    would not be reflected in the local cache. If a request is already fully cached locally, marking it as handled
+    by another client will be ignored by this client. This does not cause errors but can occasionally result in
+    reprocessing a request that was already handled elsewhere. If the request was not yet cached locally, marking
+    it as handled poses no issue.
+
+    ### Shared mode
+
+    The `shared` mode is designed for scenarios with multiple concurrent consumers. It ensures proper synchronization
+    and consistency across clients, at the cost of higher API usage and slightly worse performance. This mode is safe
+    for concurrent access from multiple processes, including Actors running in parallel on the Apify platform. It
+    should be used when multiple consumers need to process requests from the same queue simultaneously.
+    """
+
+    _LSP_ERROR_MSG = 'Expected "configuration" to be an instance of "apify.Configuration", but got {} instead.'
+    """This class (intentionally) violates the Liskov Substitution Principle.
+
+    It requires a specialized `Configuration` instance compared to its parent class.
+    """
+
+    def __init__(self, *, request_queue_access: Literal['single', 'shared'] = 'single') -> None:
+        """Initialize a new instance.
+
+        Args:
+            request_queue_access: Defines how the request queue client behaves. Use `single` mode for a single
+                consumer. It has fewer API calls, meaning better performance and lower costs. If you need multiple
+                concurrent consumers use `shared` mode, but expect worse performance and higher costs due to
+                the additional overhead.
+        """
+        self._request_queue_access = request_queue_access
+
+    @override
+    async def create_dataset_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: CrawleeConfiguration | None = None,
+    ) -> ApifyDatasetClient:
+        configuration = configuration or ApifyConfiguration.get_global_configuration()
+        if isinstance(configuration, ApifyConfiguration):
+            return await ApifyDatasetClient.open(id=id, name=name, alias=alias, configuration=configuration)
+
+        raise TypeError(self._LSP_ERROR_MSG.format(type(configuration).__name__))
+
+    @override
+    async def create_kvs_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: CrawleeConfiguration | None = None,
+    ) -> ApifyKeyValueStoreClient:
+        configuration = configuration or ApifyConfiguration.get_global_configuration()
+        if isinstance(configuration, ApifyConfiguration):
+            return await ApifyKeyValueStoreClient.open(id=id, name=name, alias=alias, configuration=configuration)
+
+        raise TypeError(self._LSP_ERROR_MSG.format(type(configuration).__name__))
+
+    @override
+    async def create_rq_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: CrawleeConfiguration | None = None,
+    ) -> ApifyRequestQueueClient:
+        configuration = configuration or ApifyConfiguration.get_global_configuration()
+        if isinstance(configuration, ApifyConfiguration):
+            return await ApifyRequestQueueClient.open(
+                id=id, name=name, alias=alias, configuration=configuration, access=self._request_queue_access
+            )
+
+        raise TypeError(self._LSP_ERROR_MSG.format(type(configuration).__name__))
+
+    @override
+    def get_storage_client_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
+        if isinstance(configuration, ApifyConfiguration):
+            # It is not supported to open exactly same queue with 'single' and 'shared' client at the same time.
+            # Whichever client variation gets used first, wins.
+            return super().get_storage_client_cache_key(configuration), hash_api_base_url_and_token(configuration)
+
+        config_class = type(configuration)
+        raise TypeError(self._LSP_ERROR_MSG.format(f'{config_class.__module__}.{config_class.__name__}'))

{apify-3.0.3b1 → apify-3.0.4b1}/src/apify/storage_clients/_smart_apify/_storage_client.py

@@ -8,8 +8,7 @@ from crawlee.storage_clients._base import DatasetClient, KeyValueStoreClient, Re
 
 from apify._configuration import Configuration as ApifyConfiguration
 from apify._utils import docs_group
-from apify.storage_clients import ApifyStorageClient
-from apify.storage_clients._file_system import ApifyFileSystemStorageClient
+from apify.storage_clients import ApifyStorageClient, FileSystemStorageClient
 
 if TYPE_CHECKING:
     from collections.abc import Hashable
@@ -19,28 +18,36 @@ if TYPE_CHECKING:
 
 @docs_group('Storage clients')
 class SmartApifyStorageClient(StorageClient):
-    """SmartApifyStorageClient that delegates to cloud_storage_client or local_storage_client.
+    """Storage client that automatically selects cloud or local storage client based on the environment.
 
-    When running on Apify platform use cloud_storage_client, else use local_storage_client. This storage client is
-    designed to work specifically in Actor context.
+    This storage client provides access to datasets, key-value stores, and request queues by intelligently
+    delegating to either the cloud or local storage client based on the execution environment and configuration.
+
+    When running on the Apify platform (which is detected via environment variables), this client automatically
+    uses the `cloud_storage_client` to store storage data there. When running locally, it uses the
+    `local_storage_client` to store storage data there. You can also force cloud storage usage from your
+    local machine by using the `force_cloud` argument.
+
+    This storage client is designed to work specifically in `Actor` context and provides a seamless development
+    experience where the same code works both locally and on the Apify platform without any changes.
     """
 
     def __init__(
         self,
         *,
-        cloud_storage_client: ApifyStorageClient | None = None,
+        cloud_storage_client: StorageClient | None = None,
         local_storage_client: StorageClient | None = None,
     ) -> None:
-        """Initialize the Apify storage client.
+        """Initialize a new instance.
 
         Args:
-            cloud_storage_client: Client used to communicate with the Apify platform storage. Either through
-                `force_cloud` argument when opening storages or automatically when running on the Apify platform.
-            local_storage_client: Client used to communicate with the storage when not running on the Apify
-                platform and not using `force_cloud` argument when opening storages.
+            cloud_storage_client: Storage client used when an Actor is running on the Apify platform, or when
+                explicitly enabled via the `force_cloud` argument. Defaults to `ApifyStorageClient`.
+            local_storage_client: Storage client used when an Actor is not running on the Apify platform and when
+                `force_cloud` flag is not set. Defaults to `FileSystemStorageClient`.
         """
-        self._cloud_storage_client = cloud_storage_client or ApifyStorageClient(request_queue_access='single')
-        self._local_storage_client = local_storage_client or ApifyFileSystemStorageClient()
+        self._cloud_storage_client = cloud_storage_client or ApifyStorageClient()
+        self._local_storage_client = local_storage_client or FileSystemStorageClient()
 
     def __str__(self) -> str:
         return (
@@ -48,26 +55,6 @@ class SmartApifyStorageClient(StorageClient):
             f' local_storage_client={self._local_storage_client.__class__.__name__})'
         )
 
-    def get_suitable_storage_client(self, *, force_cloud: bool = False) -> StorageClient:
-        """Get a suitable storage client based on the global configuration and the value of the force_cloud flag.
-
-        Args:
-            force_cloud: If True, return `cloud_storage_client`.
-        """
-        if ApifyConfiguration.get_global_configuration().is_at_home:
-            return self._cloud_storage_client
-
-        configuration = ApifyConfiguration.get_global_configuration()
-        if force_cloud:
-            if configuration.token is None:
-                raise RuntimeError(
-                    'In order to use the Apify cloud storage from your computer, '
-                    'you need to provide an Apify token using the APIFY_TOKEN environment variable.'
-                )
-            return self._cloud_storage_client
-
-        return self._local_storage_client
-
     @override
     def get_storage_client_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
         if ApifyConfiguration.get_global_configuration().is_at_home:
@@ -115,3 +102,23 @@ class SmartApifyStorageClient(StorageClient):
         return await self.get_suitable_storage_client().create_rq_client(
             id=id, name=id, alias=alias, configuration=configuration
         )
+
+    def get_suitable_storage_client(self, *, force_cloud: bool = False) -> StorageClient:
+        """Get a suitable storage client based on the global configuration and the value of the force_cloud flag.
+
+        Args:
+            force_cloud: If True, return `cloud_storage_client`.
+        """
+        if ApifyConfiguration.get_global_configuration().is_at_home:
+            return self._cloud_storage_client
+
+        configuration = ApifyConfiguration.get_global_configuration()
+        if force_cloud:
+            if configuration.token is None:
+                raise RuntimeError(
+                    'In order to use the Apify cloud storage from your computer, '
+                    'you need to provide an Apify token using the APIFY_TOKEN environment variable.'
+                )
+            return self._cloud_storage_client
+
+        return self._local_storage_client

{apify-3.0.3b1 → apify-3.0.4b1}/uv.lock

@@ -28,7 +28,7 @@ wheels = [
 
 [[package]]
 name = "apify"
-version = "3.0.3b1"
+version = "3.0.4b1"
 source = { editable = "." }
 dependencies = [
     { name = "apify-client" },

{apify-3.0.3b1 → apify-3.0.4b1}/website/docusaurus.config.js

@@ -239,6 +239,10 @@ module.exports = {
                         url: 'https://crawlee.dev/python/api/class/FileSystemStorageClient',
                         group: 'Storage clients',
                     },
+                    {
+                        url: 'https://crawlee.dev/python/api/class/SqlStorageClient',
+                        group: 'Storage clients',
+                    },
                     // Request loaders
                     {
                         url: 'https://crawlee.dev/python/api/class/RequestLoader',

{apify-3.0.3b1 → apify-3.0.4b1}/website/package-lock.json

@@ -1671,9 +1671,9 @@
             }
         },
         "node_modules/@apify/docs-theme": {
-            "version": "1.0.218",
-            "resolved": "https://registry.npmjs.org/@apify/docs-theme/-/docs-theme-1.0.218.tgz",
-            "integrity": "sha512-2kAF+2CZmwGgK8Pq3SRk5cB2knZ5GLuNTW0sH1UohK4oFCtFFjh0HYyqeedRcf2Pk2mQSVgac4GNMB9E7+yBTQ==",
+            "version": "1.0.221",
+            "resolved": "https://registry.npmjs.org/@apify/docs-theme/-/docs-theme-1.0.221.tgz",
+            "integrity": "sha512-bZBjMCnqPg04gf+ZfQaiRNXZGgnLKvsx5UvvDYWJdLu1O5otPtAZMrMDTprwaQ7ofhb+tk9CUgZCpuZZF0e5dg==",
             "license": "ISC",
             "dependencies": {
                 "@apify/docs-search-modal": "^1.3.3",

apify-3.0.3b1/src/apify/storage_clients/_apify/_storage_client.py

@@ -1,106 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Literal
-
-from typing_extensions import override
-
-from crawlee.storage_clients._base import StorageClient
-
-from ._dataset_client import ApifyDatasetClient
-from ._key_value_store_client import ApifyKeyValueStoreClient
-from ._request_queue_client import ApifyRequestQueueClient
-from ._utils import hash_api_base_url_and_token
-from apify._configuration import Configuration as ApifyConfiguration
-from apify._utils import docs_group
-
-if TYPE_CHECKING:
-    from collections.abc import Hashable
-
-    from crawlee.configuration import Configuration as CrawleeConfiguration
-
-
-@docs_group('Storage clients')
-class ApifyStorageClient(StorageClient):
-    """Apify storage client."""
-
-    def __init__(self, *, request_queue_access: Literal['single', 'shared'] = 'single') -> None:
-        """Initialize the Apify storage client.
-
-        Args:
-            request_queue_access: Controls the implementation of the request queue client based on expected scenario:
-                - 'single' is suitable for single consumer scenarios. It makes less API calls, is cheaper and faster.
-                - 'shared' is suitable for multiple consumers scenarios at the cost of higher API usage.
-                Detailed constraints for the 'single' access type:
-                - Only one client is consuming the request queue at the time.
-                - Multiple producers can put requests to the queue, but their forefront requests are not guaranteed to
-                  be handled so quickly as this client does not aggressively fetch the forefront and relies on local
-                  head estimation.
-                - Requests are only added to the queue, never deleted by other clients. (Marking as handled is ok.)
-                - Other producers can add new requests, but not modify existing ones.
-                  (Modifications would not be included in local cache)
-        """
-        self._request_queue_access = request_queue_access
-
-    # This class breaches Liskov Substitution Principle. It requires specialized Configuration compared to its parent.
-    _lsp_violation_error_message_template = (
-        'Expected "configuration" to be an instance of "apify.Configuration", but got {} instead.'
-    )
-
-    @override
-    def get_storage_client_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
-        if isinstance(configuration, ApifyConfiguration):
-            # It is not supported to open exactly same queue with 'single' and 'shared' client at the same time.
-            # Whichever client variation gets used first, wins.
-            return super().get_storage_client_cache_key(configuration), hash_api_base_url_and_token(configuration)
-
-        config_class = type(configuration)
-        raise TypeError(
-            self._lsp_violation_error_message_template.format(f'{config_class.__module__}.{config_class.__name__}')
-        )
-
-    @override
-    async def create_dataset_client(
-        self,
-        *,
-        id: str | None = None,
-        name: str | None = None,
-        alias: str | None = None,
-        configuration: CrawleeConfiguration | None = None,
-    ) -> ApifyDatasetClient:
-        configuration = configuration or ApifyConfiguration.get_global_configuration()
-        if isinstance(configuration, ApifyConfiguration):
-            return await ApifyDatasetClient.open(id=id, name=name, alias=alias, configuration=configuration)
-
-        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))
-
-    @override
-    async def create_kvs_client(
-        self,
-        *,
-        id: str | None = None,
-        name: str | None = None,
-        alias: str | None = None,
-        configuration: CrawleeConfiguration | None = None,
-    ) -> ApifyKeyValueStoreClient:
-        configuration = configuration or ApifyConfiguration.get_global_configuration()
-        if isinstance(configuration, ApifyConfiguration):
-            return await ApifyKeyValueStoreClient.open(id=id, name=name, alias=alias, configuration=configuration)
-
-        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))
-
-    @override
-    async def create_rq_client(
-        self,
-        *,
-        id: str | None = None,
-        name: str | None = None,
-        alias: str | None = None,
-        configuration: CrawleeConfiguration | None = None,
-    ) -> ApifyRequestQueueClient:
-        configuration = configuration or ApifyConfiguration.get_global_configuration()
-        if isinstance(configuration, ApifyConfiguration):
-            return await ApifyRequestQueueClient.open(
-                id=id, name=name, alias=alias, configuration=configuration, access=self._request_queue_access
-            )
-
-        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))