apify 3.0.3__py3-none-any.whl → 3.0.4b1__py3-none-any.whl

apify/_actor.py

@@ -48,7 +48,6 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
     from crawlee.proxy_configuration import _NewUrlFunction
-    from crawlee.storage_clients import StorageClient
 
     from apify._models import Webhook
 
@@ -140,7 +139,6 @@ class _ActorType:
         # `__init__` method should not be considered final.
 
         self._apify_client: ApifyClientAsync | None = None
-        self._local_storage_client: StorageClient | None = None
 
         self._is_exiting = False
         self._is_initialized = False

apify/storage_clients/_apify/_models.py

@@ -120,7 +120,7 @@ class RequestQueueStats(BaseModel):
     """The number of request queue reads."""
 
     storage_bytes: Annotated[int, Field(alias='storageBytes', default=0)]
-    """Storage size in Bytes."""
+    """Storage size in bytes."""
 
     write_count: Annotated[int, Field(alias='writeCount', default=0)]
     """The number of request queue writes."""

apify/storage_clients/_apify/_storage_client.py

@@ -21,43 +21,59 @@ if TYPE_CHECKING:
 
 @docs_group('Storage clients')
 class ApifyStorageClient(StorageClient):
-    """Apify storage client."""
+    """Apify platform implementation of the storage client.
+
+    This storage client provides access to datasets, key-value stores, and request queues that persist data
+    to the Apify platform. Each storage type is implemented with its own specific Apify client that stores data
+    in the cloud, making it accessible from anywhere.
+
+    The communication with the Apify platform is handled via the Apify API client for Python, which is an HTTP API
+    wrapper. For maximum efficiency and performance of the storage clients, various caching mechanisms are used to
+    minimize the number of API calls made to the Apify platform. Data can be inspected and manipulated through
+    the Apify console web interface or via the Apify API.
+
+    The request queue client supports two access modes controlled by the `request_queue_access` parameter:
+
+    ### Single mode
+
+    The `single` mode is optimized for scenarios with only one consumer. It minimizes API calls, making it faster
+    and more cost-efficient compared to the `shared` mode. This option is ideal when a single Actor is responsible
+    for consuming the entire request queue. Using multiple consumers simultaneously may lead to inconsistencies
+    or unexpected behavior.
+
+    In this mode, multiple producers can safely add new requests, but forefront requests may not be processed
+    immediately, as the client relies on local head estimation instead of frequent forefront fetching. Requests can
+    also be added or marked as handled by other clients, but they must not be deleted or modified, since such changes
+    would not be reflected in the local cache. If a request is already fully cached locally, marking it as handled
+    by another client will be ignored by this client. This does not cause errors but can occasionally result in
+    reprocessing a request that was already handled elsewhere. If the request was not yet cached locally, marking
+    it as handled poses no issue.
+
+    ### Shared mode
+
+    The `shared` mode is designed for scenarios with multiple concurrent consumers. It ensures proper synchronization
+    and consistency across clients, at the cost of higher API usage and slightly worse performance. This mode is safe
+    for concurrent access from multiple processes, including Actors running in parallel on the Apify platform. It
+    should be used when multiple consumers need to process requests from the same queue simultaneously.
+    """
+
+    _LSP_ERROR_MSG = 'Expected "configuration" to be an instance of "apify.Configuration", but got {} instead.'
+    """This class (intentionally) violates the Liskov Substitution Principle.
+
+    It requires a specialized `Configuration` instance compared to its parent class.
+    """
 
     def __init__(self, *, request_queue_access: Literal['single', 'shared'] = 'single') -> None:
-        """Initialize the Apify storage client.
+        """Initialize a new instance.
 
         Args:
-            request_queue_access: Controls the implementation of the request queue client based on expected scenario:
-                - 'single' is suitable for single consumer scenarios. It makes less API calls, is cheaper and faster.
-                - 'shared' is suitable for multiple consumers scenarios at the cost of higher API usage.
-                Detailed constraints for the 'single' access type:
-                - Only one client is consuming the request queue at the time.
-                - Multiple producers can put requests to the queue, but their forefront requests are not guaranteed to
-                  be handled so quickly as this client does not aggressively fetch the forefront and relies on local
-                  head estimation.
-                - Requests are only added to the queue, never deleted by other clients. (Marking as handled is ok.)
-                - Other producers can add new requests, but not modify existing ones.
-                  (Modifications would not be included in local cache)
+            request_queue_access: Defines how the request queue client behaves. Use `single` mode for a single
+                consumer. It has fewer API calls, meaning better performance and lower costs. If you need multiple
+                concurrent consumers use `shared` mode, but expect worse performance and higher costs due to
+                the additional overhead.
         """
         self._request_queue_access = request_queue_access
 
-    # This class breaches Liskov Substitution Principle. It requires specialized Configuration compared to its parent.
-    _lsp_violation_error_message_template = (
-        'Expected "configuration" to be an instance of "apify.Configuration", but got {} instead.'
-    )
-
-    @override
-    def get_storage_client_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
-        if isinstance(configuration, ApifyConfiguration):
-            # It is not supported to open exactly same queue with 'single' and 'shared' client at the same time.
-            # Whichever client variation gets used first, wins.
-            return super().get_storage_client_cache_key(configuration), hash_api_base_url_and_token(configuration)
-
-        config_class = type(configuration)
-        raise TypeError(
-            self._lsp_violation_error_message_template.format(f'{config_class.__module__}.{config_class.__name__}')
-        )
-
     @override
     async def create_dataset_client(
         self,
@@ -71,7 +87,7 @@ class ApifyStorageClient(StorageClient):
         if isinstance(configuration, ApifyConfiguration):
             return await ApifyDatasetClient.open(id=id, name=name, alias=alias, configuration=configuration)
 
-        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))
+        raise TypeError(self._LSP_ERROR_MSG.format(type(configuration).__name__))
 
     @override
     async def create_kvs_client(
@@ -86,7 +102,7 @@ class ApifyStorageClient(StorageClient):
         if isinstance(configuration, ApifyConfiguration):
             return await ApifyKeyValueStoreClient.open(id=id, name=name, alias=alias, configuration=configuration)
 
-        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))
+        raise TypeError(self._LSP_ERROR_MSG.format(type(configuration).__name__))
 
     @override
     async def create_rq_client(
@@ -103,4 +119,14 @@ class ApifyStorageClient(StorageClient):
                 id=id, name=name, alias=alias, configuration=configuration, access=self._request_queue_access
             )
 
-        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))
+        raise TypeError(self._LSP_ERROR_MSG.format(type(configuration).__name__))
+
+    @override
+    def get_storage_client_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
+        if isinstance(configuration, ApifyConfiguration):
+            # It is not supported to open exactly same queue with 'single' and 'shared' client at the same time.
+            # Whichever client variation gets used first, wins.
+            return super().get_storage_client_cache_key(configuration), hash_api_base_url_and_token(configuration)
+
+        config_class = type(configuration)
+        raise TypeError(self._LSP_ERROR_MSG.format(f'{config_class.__module__}.{config_class.__name__}'))

apify/storage_clients/_smart_apify/_storage_client.py

@@ -8,8 +8,7 @@ from crawlee.storage_clients._base import DatasetClient, KeyValueStoreClient, Re
 
 from apify._configuration import Configuration as ApifyConfiguration
 from apify._utils import docs_group
-from apify.storage_clients import ApifyStorageClient
-from apify.storage_clients._file_system import ApifyFileSystemStorageClient
+from apify.storage_clients import ApifyStorageClient, FileSystemStorageClient
 
 if TYPE_CHECKING:
     from collections.abc import Hashable
@@ -19,28 +18,36 @@ if TYPE_CHECKING:
 
 @docs_group('Storage clients')
 class SmartApifyStorageClient(StorageClient):
-    """SmartApifyStorageClient that delegates to cloud_storage_client or local_storage_client.
+    """Storage client that automatically selects cloud or local storage client based on the environment.
 
-    When running on Apify platform use cloud_storage_client, else use local_storage_client. This storage client is
-    designed to work specifically in Actor context.
+    This storage client provides access to datasets, key-value stores, and request queues by intelligently
+    delegating to either the cloud or local storage client based on the execution environment and configuration.
+
+    When running on the Apify platform (which is detected via environment variables), this client automatically
+    uses the `cloud_storage_client` to store storage data there. When running locally, it uses the
+    `local_storage_client` to store storage data there. You can also force cloud storage usage from your
+    local machine by using the `force_cloud` argument.
+
+    This storage client is designed to work specifically in `Actor` context and provides a seamless development
+    experience where the same code works both locally and on the Apify platform without any changes.
     """
 
     def __init__(
         self,
         *,
-        cloud_storage_client: ApifyStorageClient | None = None,
+        cloud_storage_client: StorageClient | None = None,
         local_storage_client: StorageClient | None = None,
     ) -> None:
-        """Initialize the Apify storage client.
+        """Initialize a new instance.
 
         Args:
-            cloud_storage_client: Client used to communicate with the Apify platform storage. Either through
-                `force_cloud` argument when opening storages or automatically when running on the Apify platform.
-            local_storage_client: Client used to communicate with the storage when not running on the Apify
-                platform and not using `force_cloud` argument when opening storages.
+            cloud_storage_client: Storage client used when an Actor is running on the Apify platform, or when
+                explicitly enabled via the `force_cloud` argument. Defaults to `ApifyStorageClient`.
+            local_storage_client: Storage client used when an Actor is not running on the Apify platform and when
+                `force_cloud` flag is not set. Defaults to `FileSystemStorageClient`.
         """
-        self._cloud_storage_client = cloud_storage_client or ApifyStorageClient(request_queue_access='single')
-        self._local_storage_client = local_storage_client or ApifyFileSystemStorageClient()
+        self._cloud_storage_client = cloud_storage_client or ApifyStorageClient()
+        self._local_storage_client = local_storage_client or FileSystemStorageClient()
 
     def __str__(self) -> str:
         return (
@@ -48,26 +55,6 @@ class SmartApifyStorageClient(StorageClient):
             f' local_storage_client={self._local_storage_client.__class__.__name__})'
         )
 
-    def get_suitable_storage_client(self, *, force_cloud: bool = False) -> StorageClient:
-        """Get a suitable storage client based on the global configuration and the value of the force_cloud flag.
-
-        Args:
-            force_cloud: If True, return `cloud_storage_client`.
-        """
-        if ApifyConfiguration.get_global_configuration().is_at_home:
-            return self._cloud_storage_client
-
-        configuration = ApifyConfiguration.get_global_configuration()
-        if force_cloud:
-            if configuration.token is None:
-                raise RuntimeError(
-                    'In order to use the Apify cloud storage from your computer, '
-                    'you need to provide an Apify token using the APIFY_TOKEN environment variable.'
-                )
-            return self._cloud_storage_client
-
-        return self._local_storage_client
-
     @override
     def get_storage_client_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
         if ApifyConfiguration.get_global_configuration().is_at_home:
@@ -115,3 +102,23 @@ class SmartApifyStorageClient(StorageClient):
         return await self.get_suitable_storage_client().create_rq_client(
             id=id, name=id, alias=alias, configuration=configuration
         )
+
+    def get_suitable_storage_client(self, *, force_cloud: bool = False) -> StorageClient:
+        """Get a suitable storage client based on the global configuration and the value of the force_cloud flag.
+
+        Args:
+            force_cloud: If True, return `cloud_storage_client`.
+        """
+        if ApifyConfiguration.get_global_configuration().is_at_home:
+            return self._cloud_storage_client
+
+        configuration = ApifyConfiguration.get_global_configuration()
+        if force_cloud:
+            if configuration.token is None:
+                raise RuntimeError(
+                    'In order to use the Apify cloud storage from your computer, '
+                    'you need to provide an Apify token using the APIFY_TOKEN environment variable.'
+                )
+            return self._cloud_storage_client
+
+        return self._local_storage_client

{apify-3.0.3.dist-info → apify-3.0.4b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 3.0.3
+Version: 3.0.4b1
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog

{apify-3.0.3.dist-info → apify-3.0.4b1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 apify/__init__.py,sha256=HpgKg2FZWJuSPfDygzJ62psylhw4NN4tKFnoYUIhcd4,838
-apify/_actor.py,sha256=kfrwD8gaeN4NcdNMD_Pj66agNh78jJjwMuNOuwLdo-E,57370
+apify/_actor.py,sha256=eJzlRicWkVuO-151ikjJnRMRl-UgJ2QKg6wahtoc2Rc,57252
 apify/_charging.py,sha256=KjZ2DnEMS0Tt8ibizmmt0RwBq8FOAsD1z-hKFgdazcY,13143
 apify/_configuration.py,sha256=7ZHhgRp98kr35zx4k4EB2aImq7Dq1FJjPg7r5bucv_M,14984
 apify/_consts.py,sha256=CjhyEJ4Mi0lcIrzfqz8dN7nPJWGjCeBrrXQy1PZ6zRI,440
@@ -37,21 +37,21 @@ apify/storage_clients/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 apify/storage_clients/_apify/__init__.py,sha256=mtbVDsxqWL3kx30elnh0kAn2kZ4s3BBsWa15Y5e7RMU,347
 apify/storage_clients/_apify/_dataset_client.py,sha256=qmCJyL1MN83tYRXmc31P6yMIXVZMyRrGjr7R6-86FSE,11869
 apify/storage_clients/_apify/_key_value_store_client.py,sha256=994a5bM_BGHIeirnny6QlXjy5CzMU2I9SmMksCbHCUY,9357
-apify/storage_clients/_apify/_models.py,sha256=XxBru5XFdj0jqX6V-uVahT-pMQU3pZ501aTNzXCuoMU,4556
+apify/storage_clients/_apify/_models.py,sha256=szYdJOvWQ6hrmwUp7y7QsoQrRh5TlIYNxmO7nOe-M14,4556
 apify/storage_clients/_apify/_request_queue_client.py,sha256=tAyap34gpxvPiQ0McDjX5ojq1ZIZc4EI3PrW8VQqS4k,13292
 apify/storage_clients/_apify/_request_queue_shared_client.py,sha256=pWmd6aPxM-eZ6PC1MfsfTcjD2mGGpCDS3ZZ3cG_2MEA,20971
 apify/storage_clients/_apify/_request_queue_single_client.py,sha256=d2txMwxW1nlYnvjdOH8xpxhcOYNeyc1ousGHRE7jsPg,17468
-apify/storage_clients/_apify/_storage_client.py,sha256=hFl_PuX1UgOydBD6pieZ0u2NWbDmZV-i0qygKdsuHt4,4873
+apify/storage_clients/_apify/_storage_client.py,sha256=TcmMzbEMgyndxaT6lIMl2fTG4oQR_lpLSHTmwwH0tXs,6515
 apify/storage_clients/_apify/_utils.py,sha256=375gk_TJyMWIIgRbE9SS0hQup0h6sA3mzpTG53XIjkM,8769
 apify/storage_clients/_apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/storage_clients/_file_system/__init__.py,sha256=rDbXatXV9wHKPhKTrXDzWnexhTm7sIJQWucMi-P-SD4,130
 apify/storage_clients/_file_system/_key_value_store_client.py,sha256=gxM3ap67PnY80Rd7P3onPAf2pksYpU0LoAlJdayEMdU,4179
 apify/storage_clients/_file_system/_storage_client.py,sha256=rcwpKYlrWzvlSA2xoxftg-EZAi_iGZ3vOCbu0C5lKDE,1396
 apify/storage_clients/_smart_apify/__init__.py,sha256=614B2AaWY-dx6RQ6mod7VVR8gFh75-_jnq5BeDD7hSc,53
-apify/storage_clients/_smart_apify/_storage_client.py,sha256=GCPmVe_xWAFcO2Cuej4su4i97_d33Q9Ih_Sc5xW2Wa4,4674
+apify/storage_clients/_smart_apify/_storage_client.py,sha256=ZNNY4Qm9Cx_UFqBaforT28gC4hhOnCcKWpUYCIvzj48,5218
 apify/storages/__init__.py,sha256=-9tEYJVabVs_eRVhUehxN58GH0UG8OfuGjGwuDieP2M,122
 apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify-3.0.3.dist-info/METADATA,sha256=BghvJ18oGZT-KqyoV3wlC6WdJ-rVupfzSUaWXLix2-k,22580
-apify-3.0.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-apify-3.0.3.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
-apify-3.0.3.dist-info/RECORD,,
+apify-3.0.4b1.dist-info/METADATA,sha256=Y3LBVJMWeWazp-foQTEHw-5kD1AGlTkmOPRwJZ2KAcU,22582
+apify-3.0.4b1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+apify-3.0.4b1.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
+apify-3.0.4b1.dist-info/RECORD,,