apify 2.2.0b11__py3-none-any.whl → 2.2.0b12__py3-none-any.whl

apify/_actor.py

@@ -13,7 +13,7 @@ from pydantic import AliasChoices
 from apify_client import ApifyClientAsync
 from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars
 from apify_shared.utils import ignore_docs, maybe_extract_enum_member_value
-from crawlee import service_container
+from crawlee import service_locator
 from crawlee.events._types import Event, EventMigratingData, EventPersistStateData
 
 from apify._configuration import Configuration
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
     from crawlee.proxy_configuration import _NewUrlFunction
+    from crawlee.storage_clients import BaseStorageClient
 
     from apify._models import Webhook
 
@@ -71,17 +72,22 @@ class _ActorType:
         self._configure_logging = configure_logging
         self._apify_client = self.new_client()
 
-        self._event_manager: EventManager
-        if self._configuration.is_at_home:
-            self._event_manager = PlatformEventManager(
+        # Create an instance of the cloud storage client, the local storage client is obtained
+        # from the service locator.
+        self._cloud_storage_client = ApifyStorageClient.from_config(config=self._configuration)
+
+        # Set the event manager based on whether the Actor is running on the platform or locally.
+        self._event_manager = (
+            PlatformEventManager(
                 config=self._configuration,
                 persist_state_interval=self._configuration.persist_state_interval,
             )
-        else:
-            self._event_manager = LocalEventManager(
+            if self.is_at_home()
+            else LocalEventManager(
                 system_info_interval=self._configuration.system_info_interval,
                 persist_state_interval=self._configuration.persist_state_interval,
             )
+        )
 
         self._is_initialized = False
 
@@ -94,9 +100,6 @@ class _ActorType:
         When you exit the `async with` block, the `Actor.exit()` method is called, and if any exception happens while
         executing the block code, the `Actor.fail` method is called.
         """
-        if self._configure_logging:
-            _configure_logging(self._configuration)
-
         await self.init()
         return self
 
@@ -156,6 +159,11 @@ class _ActorType:
         """The logging.Logger instance the Actor uses."""
         return logger
 
+    @property
+    def _local_storage_client(self) -> BaseStorageClient:
+        """The local storage client the Actor instance uses."""
+        return service_locator.get_storage_client()
+
     def _raise_if_not_initialized(self) -> None:
         if not self._is_initialized:
             raise RuntimeError('The Actor was not initialized!')
@@ -184,18 +192,19 @@ class _ActorType:
         if self._is_initialized:
             raise RuntimeError('The Actor was already initialized!')
 
-        if self._configuration.token:
-            service_container.set_cloud_storage_client(ApifyStorageClient(configuration=self._configuration))
+        self._is_exiting = False
+        self._was_final_persist_state_emitted = False
 
-        if self._configuration.is_at_home:
-            service_container.set_default_storage_client_type('cloud')
-        else:
-            service_container.set_default_storage_client_type('local')
+        # If the Actor is running on the Apify platform, we set the cloud storage client.
+        if self.is_at_home():
+            service_locator.set_storage_client(self._cloud_storage_client)
 
-        service_container.set_event_manager(self._event_manager)
+        service_locator.set_event_manager(self.event_manager)
+        service_locator.set_configuration(self.configuration)
 
-        self._is_exiting = False
-        self._was_final_persist_state_emitted = False
+        # The logging configuration has to be called after all service_locator set methods.
+        if self._configure_logging:
+            _configure_logging()
 
         self.log.info('Initializing Actor...')
         self.log.info('System info', extra=get_system_info())
@@ -245,7 +254,6 @@ class _ActorType:
                 await self._event_manager.wait_for_all_listeners_to_complete(timeout=event_listeners_timeout)
 
             await self._event_manager.__aexit__(None, None, None)
-            cast(dict, service_container._services).clear()  # noqa: SLF001
 
         await asyncio.wait_for(finalize(), cleanup_timeout.total_seconds())
         self._is_initialized = False
@@ -349,11 +357,13 @@ class _ActorType:
         self._raise_if_not_initialized()
         self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
 
+        storage_client = self._cloud_storage_client if force_cloud else self._local_storage_client
+
         return await Dataset.open(
             id=id,
             name=name,
             configuration=self._configuration,
-            storage_client=service_container.get_storage_client(client_type='cloud' if force_cloud else None),
+            storage_client=storage_client,
         )
 
     async def open_key_value_store(
@@ -381,12 +391,13 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
         self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
+        storage_client = self._cloud_storage_client if force_cloud else self._local_storage_client
 
         return await KeyValueStore.open(
             id=id,
             name=name,
             configuration=self._configuration,
-            storage_client=service_container.get_storage_client(client_type='cloud' if force_cloud else None),
+            storage_client=storage_client,
         )
 
     async def open_request_queue(
@@ -417,11 +428,13 @@ class _ActorType:
         self._raise_if_not_initialized()
         self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
 
+        storage_client = self._cloud_storage_client if force_cloud else self._local_storage_client
+
         return await RequestQueue.open(
             id=id,
             name=name,
             configuration=self._configuration,
-            storage_client=service_container.get_storage_client(client_type='cloud' if force_cloud else None),
+            storage_client=storage_client,
         )
 
     async def push_data(self, data: dict | list[dict]) -> None:
@@ -963,7 +976,7 @@ class _ActorType:
         password: str | None = None,
         groups: list[str] | None = None,
         country_code: str | None = None,
-        proxy_urls: list[str] | None = None,
+        proxy_urls: list[str | None] | None = None,
         new_url_function: _NewUrlFunction | None = None,
     ) -> ProxyConfiguration | None:
         """Create a ProxyConfiguration object with the passed proxy configuration.

apify/_configuration.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from datetime import datetime, timedelta
+from logging import getLogger
 from typing import Annotated, Any
 
 from pydantic import AliasChoices, BeforeValidator, Field
@@ -12,6 +13,8 @@ from crawlee.configuration import Configuration as CrawleeConfiguration
 
 from apify._utils import docs_group
 
+logger = getLogger(__name__)
+
 
 def _transform_to_list(value: Any) -> list[str] | None:
     if value is None:
@@ -353,6 +356,15 @@ class Configuration(CrawleeConfiguration):
         ),
     ] = None
 
+    @classmethod
+    def get_global_configuration(cls) -> Configuration:
+        """Retrieve the global instance of the configuration.
+
+        Mostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`
+        instead.
+        """
+        return cls()
+
 
 # Monkey-patch the base class so that it works with the extended configuration
 CrawleeConfiguration.get_global_configuration = Configuration.get_global_configuration  # type: ignore[method-assign]

apify/_proxy_configuration.py

@@ -111,9 +111,9 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
         password: str | None = None,
         groups: list[str] | None = None,
         country_code: str | None = None,
-        proxy_urls: list[str] | None = None,
+        proxy_urls: list[str | None] | None = None,
         new_url_function: _NewUrlFunction | None = None,
-        tiered_proxy_urls: list[list[str]] | None = None,
+        tiered_proxy_urls: list[list[str | None]] | None = None,
         _actor_config: Configuration | None = None,
         _apify_client: ApifyClientAsync | None = None,
     ) -> None:
@@ -148,7 +148,7 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
                 ' "groups" or "country_code".'
             )
 
-        if proxy_urls and any('apify.com' in url for url in proxy_urls):
+        if proxy_urls and any('apify.com' in (url or '') for url in proxy_urls):
             logger.warning(
                 'Some Apify proxy features may work incorrectly. Please consider setting up Apify properties '
                 'instead of `proxy_urls`.\n'

apify/apify_storage_client/_apify_storage_client.py

@@ -1,10 +1,13 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from typing_extensions import override
 
 from apify_client import ApifyClientAsync
 from crawlee._utils.crypto import crypto_random_object_id
-from crawlee.base_storage_client import BaseStorageClient
+from crawlee.storage_clients import BaseStorageClient
 
-from apify._configuration import Configuration
 from apify._utils import docs_group
 from apify.apify_storage_client._dataset_client import DatasetClient
 from apify.apify_storage_client._dataset_collection_client import DatasetCollectionClient
@@ -13,6 +16,9 @@ from apify.apify_storage_client._key_value_store_collection_client import KeyVal
 from apify.apify_storage_client._request_queue_client import RequestQueueClient
 from apify.apify_storage_client._request_queue_collection_client import RequestQueueCollectionClient
 
+if TYPE_CHECKING:
+    from apify._configuration import Configuration
+
 
 @docs_group('Classes')
 class ApifyStorageClient(BaseStorageClient):
@@ -29,6 +35,10 @@ class ApifyStorageClient(BaseStorageClient):
         )
         self._configuration = configuration
 
+    @classmethod
+    def from_config(cls, config: Configuration) -> ApifyStorageClient:
+        return cls(configuration=config)
+
     @override
     def dataset(self, id: str) -> DatasetClient:
         return DatasetClient(self._apify_client.dataset(id))

apify/apify_storage_client/_dataset_client.py

@@ -4,7 +4,8 @@ from typing import TYPE_CHECKING
 
 from typing_extensions import override
 
-from crawlee.base_storage_client import BaseDatasetClient, DatasetItemsListPage, DatasetMetadata
+from crawlee.storage_clients._base import BaseDatasetClient
+from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator

apify/apify_storage_client/_dataset_collection_client.py

@@ -4,7 +4,8 @@ from typing import TYPE_CHECKING
 
 from typing_extensions import override
 
-from crawlee.base_storage_client import BaseDatasetCollectionClient, DatasetListPage, DatasetMetadata
+from crawlee.storage_clients._base import BaseDatasetCollectionClient
+from crawlee.storage_clients.models import DatasetListPage, DatasetMetadata
 
 if TYPE_CHECKING:
     from apify_client.clients import DatasetCollectionClientAsync

apify/apify_storage_client/_key_value_store_client.py

@@ -5,12 +5,8 @@ from typing import TYPE_CHECKING, Any
 
 from typing_extensions import override
 
-from crawlee.base_storage_client import (
-    BaseKeyValueStoreClient,
-    KeyValueStoreListKeysPage,
-    KeyValueStoreMetadata,
-    KeyValueStoreRecord,
-)
+from crawlee.storage_clients._base import BaseKeyValueStoreClient
+from crawlee.storage_clients.models import KeyValueStoreListKeysPage, KeyValueStoreMetadata, KeyValueStoreRecord
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator

apify/apify_storage_client/_key_value_store_collection_client.py

@@ -4,7 +4,8 @@ from typing import TYPE_CHECKING
 
 from typing_extensions import override
 
-from crawlee.base_storage_client import BaseKeyValueStoreCollectionClient, KeyValueStoreListPage, KeyValueStoreMetadata
+from crawlee.storage_clients._base import BaseKeyValueStoreCollectionClient
+from crawlee.storage_clients.models import KeyValueStoreListPage, KeyValueStoreMetadata
 
 if TYPE_CHECKING:
     from apify_client.clients import KeyValueStoreCollectionClientAsync

apify/apify_storage_client/_request_queue_client.py

@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING
 from typing_extensions import override
 
 from crawlee import Request
-from crawlee.base_storage_client import (
-    BaseRequestQueueClient,
+from crawlee.storage_clients._base import BaseRequestQueueClient
+from crawlee.storage_clients.models import (
     BatchRequestsOperationResponse,
     ProcessedRequest,
     ProlongRequestLockResponse,
@@ -80,10 +80,6 @@ class RequestQueueClient(BaseRequestQueueClient):
                     by_alias=True,
                     exclude={
                         'id',
-                        'json_',
-                        'order_no',
-                        'query_params',
-                        'data',
                     },
                 ),
                 forefront=forefront,
@@ -107,12 +103,6 @@ class RequestQueueClient(BaseRequestQueueClient):
             | await self._client.update_request(
                 request=request.model_dump(
                     by_alias=True,
-                    exclude={
-                        'json_',
-                        'order_no',
-                        'query_params',
-                        'data',
-                    },
                 ),
                 forefront=forefront,
             )
@@ -164,10 +154,6 @@ class RequestQueueClient(BaseRequestQueueClient):
                         by_alias=True,
                         exclude={
                             'id',
-                            'json_',
-                            'order_no',
-                            'query_params',
-                            'data',
                         },
                     )
                     for r in requests
@@ -183,12 +169,6 @@ class RequestQueueClient(BaseRequestQueueClient):
                 requests=[
                     r.model_dump(
                         by_alias=True,
-                        exclude={
-                            'json_',
-                            'order_no',
-                            'query_params',
-                            'data',
-                        },
                     )
                     for r in requests
                 ],

apify/apify_storage_client/_request_queue_collection_client.py

@@ -4,7 +4,8 @@ from typing import TYPE_CHECKING
 
 from typing_extensions import override
 
-from crawlee.base_storage_client import BaseRequestQueueCollectionClient, RequestQueueListPage, RequestQueueMetadata
+from crawlee.storage_clients._base import BaseRequestQueueCollectionClient
+from crawlee.storage_clients.models import RequestQueueListPage, RequestQueueMetadata
 
 if TYPE_CHECKING:
     from apify_client.clients import RequestQueueCollectionClientAsync

apify/log.py

@@ -1,14 +1,10 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING
 
 from apify_shared.utils import ignore_docs
 from crawlee._log_config import CrawleeLogFormatter, configure_logger, get_configured_log_level
 
-if TYPE_CHECKING:
-    from apify import Configuration
-
 # Name of the logger used throughout the library (resolves to 'apify')
 logger_name = __name__.split('.')[0]
 
@@ -21,11 +17,11 @@ class ActorLogFormatter(CrawleeLogFormatter):  # noqa: D101 (Inherited from pare
     pass
 
 
-def _configure_logging(configuration: Configuration) -> None:
+def _configure_logging() -> None:
     apify_client_logger = logging.getLogger('apify_client')
-    configure_logger(apify_client_logger, configuration, remove_old_handlers=True)
+    configure_logger(apify_client_logger, remove_old_handlers=True)
 
-    level = get_configured_log_level(configuration)
+    level = get_configured_log_level()
 
     # Keep apify_client logger quiet unless debug logging is requested
     if level > logging.DEBUG:
@@ -42,4 +38,4 @@ def _configure_logging(configuration: Configuration) -> None:
 
     # Use configured log level for apify logger
     apify_logger = logging.getLogger('apify')
-    configure_logger(apify_logger, configuration, remove_old_handlers=True)
+    configure_logger(apify_logger, remove_old_handlers=True)

apify/storages/_request_list.py

@@ -11,7 +11,7 @@ from pydantic import BaseModel, Field, TypeAdapter
 from crawlee import Request
 from crawlee._types import HttpMethod
 from crawlee.http_clients import BaseHttpClient, HttpxHttpClient
-from crawlee.storages import RequestList as CrawleeRequestList
+from crawlee.request_loaders import RequestList as CrawleeRequestList
 
 from apify._utils import docs_group
 

{apify-2.2.0b11.dist-info → apify-2.2.0b12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: apify
-Version: 2.2.0b11
+Version: 2.2.0b12
 Summary: Apify SDK for Python
 License: Apache-2.0
 Keywords: apify,sdk,automation,chrome,crawlee,crawler,headless,scraper,scraping
@@ -21,7 +21,7 @@ Classifier: Topic :: Software Development :: Libraries
 Provides-Extra: scrapy
 Requires-Dist: apify-client (>=1.8.1)
 Requires-Dist: apify-shared (>=1.2.1)
-Requires-Dist: crawlee (>=0.4.0,<0.5.0)
+Requires-Dist: crawlee (>=0.5.1,<0.6.0)
 Requires-Dist: cryptography (>=42.0.0)
 Requires-Dist: httpx (>=0.27.0)
 Requires-Dist: lazy-object-proxy (>=1.10.0)

{apify-2.2.0b11.dist-info → apify-2.2.0b12.dist-info}/RECORD

@@ -1,22 +1,22 @@
 apify/__init__.py,sha256=99ynaDWBLEcCjdLq7R0Exy_iACsXiXoQ8VUZKmbzTeM,550
-apify/_actor.py,sha256=bDCiS-IswmR2bQi4r71BK0wjvqA4qFUZpIAbN878-W8,43800
-apify/_configuration.py,sha256=_pPkesm1NEE6IxT1Mgxu3fGM89b1rtnLyeC3Hfpbh-Q,10516
+apify/_actor.py,sha256=X8UCTYVJmsq0Nus7aru1ayEgrbo8QE__0jKaN8aWMZ0,44313
+apify/_configuration.py,sha256=T3Z_o_W98iSyTbrutfb578yW51aexZ_V0FcLwTxFLjI,10878
 apify/_consts.py,sha256=_Xq4hOfOA1iZ3n1P967YWdyncKivpbX6RTlp_qanUoE,330
 apify/_crypto.py,sha256=e0_aM3l9_5Osk-jszYOOjrAKK60OggSHbiw5c30QnsU,5638
 apify/_models.py,sha256=Btlz-23obKY5tJ75JnUwkVNC2lmU1IEBbdU3HvWaVhg,5748
 apify/_platform_event_manager.py,sha256=44xyV0Lpzf4h4VZ0rkyYg_nhbQkEONNor8_Z9gIKO40,7899
-apify/_proxy_configuration.py,sha256=vdDiE5dfyNQYEXKXnj0jcgL3rG3-qanwNSybrVl1xT8,13167
+apify/_proxy_configuration.py,sha256=c-O6_PZ9pUD-i4J0RFEKTtfyJPP2rTRJJA1TH8NVsV8,13189
 apify/_utils.py,sha256=CCLkpAsZKp00ykm88Z_Fbck5PNT0j6mJYOuD0RxzZUs,1620
 apify/apify_storage_client/__init__.py,sha256=-UbR68bFsDR6ln8OFs4t50eqcnY36hujO-SeOt-KmcA,114
-apify/apify_storage_client/_apify_storage_client.py,sha256=NsZBleJNHLBXVyG__bVjdCGEI30cnmVZngCbQaVekfk,2397
-apify/apify_storage_client/_dataset_client.py,sha256=FfXew6tBiZRkpovyPaQ__xhtZZ-rZvjijwBIIyRahH8,5536
-apify/apify_storage_client/_dataset_collection_client.py,sha256=fkYvYGQCigHD2CDzpWk0swNAkfvAinAhMGpYqllle3E,1445
-apify/apify_storage_client/_key_value_store_client.py,sha256=uyeQgb75sGFsqIS4sq4hEZ3QP81COLfS3tmTqHc0tso,3340
-apify/apify_storage_client/_key_value_store_collection_client.py,sha256=vCtMTI-jx89Qp5WHILDNkCthwLuv0MAwm1J_5E4aypU,1519
-apify/apify_storage_client/_request_queue_client.py,sha256=jAiFkaJ38_myHFGTw-Rk21wmpbN0UCR2w2SFoimFGFc,5826
-apify/apify_storage_client/_request_queue_collection_client.py,sha256=NnO73UJ9ZrjV8xoudo30wfaM-SojRkG0guhxDyB-K1g,1527
+apify/apify_storage_client/_apify_storage_client.py,sha256=0rS75JoRHt7stRYS9-oqm3DmaSIZQN5C11N5MZQUvlA,2616
+apify/apify_storage_client/_dataset_client.py,sha256=UUodnR_MQBg5RkURrfegkGJWR5OmdPPgPfGepvkdQoU,5580
+apify/apify_storage_client/_dataset_collection_client.py,sha256=qCcKZlA0bkO-sL7xED0Yose85NlrRa9AKr4oCSrYX6k,1489
+apify/apify_storage_client/_key_value_store_client.py,sha256=MSuoIeqEHLu92WfUU7kyB3Cc_gKUlm8TghnU3_xkPtE,3363
+apify/apify_storage_client/_key_value_store_collection_client.py,sha256=NxD-3XDJP6JGMDyIa6ib0gl8op7rQjSQ0vlToCiV190,1563
+apify/apify_storage_client/_request_queue_client.py,sha256=n-CR-hA5LM6_8IwiMwQ9tT2juavq7X2zC3ZNlrtv-2s,5156
+apify/apify_storage_client/_request_queue_collection_client.py,sha256=MdzgbQb2D8rHWpUlPCrQSHRlAi0fI0PSZ9bYagr-MhY,1571
 apify/apify_storage_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify/log.py,sha256=zElFyEp2RJN0kiHEwJhcjSCAuHrba5zYiq4pK2xsL_o,1450
+apify/log.py,sha256=j-E4t-WeA93bc1NCQRG8sTntehQCiiN8ia-MdQe3_Ts,1291
 apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/scrapy/__init__.py,sha256=HE5wCN7-DZKPydLCOvjNyLuL3CvN2fUFweXfrDfe1Ss,348
 apify/scrapy/middlewares/__init__.py,sha256=tfW-d3WFWLeNEjL8fTmon6NwgD-OXx1Bw2fBdU-wPy4,114
@@ -30,9 +30,9 @@ apify/scrapy/requests.py,sha256=yZ9hIsz2YyqOoOwzN9F1h76wG4qwawrI6h_6xq0I7Iw,7599
 apify/scrapy/scheduler.py,sha256=03kZxejWWb-TofJ-vpSZuQ28rT-qNjhhpC-QeO2OzoU,5977
 apify/scrapy/utils.py,sha256=758DcHCSAgCTProY0QX74uJ1XrzVsQwvCmFanj2f_3Q,2928
 apify/storages/__init__.py,sha256=FW-z6ubuPnHGM-Wp15T8mR5q6lnpDGrCW-IkgZd5L30,177
-apify/storages/_request_list.py,sha256=4nrvSdMUF-kiwGVIPEfIOygLKgjUpO37Jl8Om-jRbIU,5858
+apify/storages/_request_list.py,sha256=-lZJcE5nq69aJhGFJ7Sh2ctqgAWUDyOwYm5_0y1hdAE,5865
 apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify-2.2.0b11.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
-apify-2.2.0b11.dist-info/METADATA,sha256=X9DPgGULfgdcJ2oPVcokk0AppJ9iMD1GHhuuAkpZb8s,8714
-apify-2.2.0b11.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-apify-2.2.0b11.dist-info/RECORD,,
+apify-2.2.0b12.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
+apify-2.2.0b12.dist-info/METADATA,sha256=mievbjU3RcmtIBtb1Tidw1riATyuC0qrFCZ-FocCz3A,8714
+apify-2.2.0b12.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+apify-2.2.0b12.dist-info/RECORD,,