apify 1.5.2b5__py3-none-any.whl → 1.5.3b2__py3-none-any.whl

apify/scrapy/utils.py

@@ -8,6 +8,8 @@ from urllib.parse import unquote
 
 try:
     from scrapy import Request, Spider
+    from scrapy.settings import Settings  # noqa: TCH002
+    from scrapy.utils.project import get_project_settings
     from scrapy.utils.python import to_bytes
     from scrapy.utils.request import request_from_dict
 except ImportError as exc:
@@ -153,6 +155,45 @@ def to_scrapy_request(apify_request: dict, spider: Spider) -> Request:
     return scrapy_request
 
 
+def apply_apify_settings(*, settings: Settings | None = None, proxy_config: dict | None = None) -> Settings:
+    """Integrates Apify configuration into a Scrapy project settings.
+
+    Note: The function directly modifies the passed `settings` object and also returns it.
+
+    Args:
+        settings: Scrapy project settings to be modified.
+        proxy_config: Proxy configuration to be stored in the settings.
+
+    Returns:
+        Scrapy project settings with custom configurations.
+    """
+    if settings is None:
+        settings = get_project_settings()
+
+    # Use ApifyScheduler as the scheduler
+    settings['SCHEDULER'] = 'apify.scrapy.scheduler.ApifyScheduler'
+
+    # Add the ActorDatasetPushPipeline into the item pipelines, assigning it the highest integer (1000),
+    # ensuring it is executed as the final step in the pipeline sequence
+    settings['ITEM_PIPELINES']['apify.scrapy.pipelines.ActorDatasetPushPipeline'] = 1000
+
+    # Disable the default RobotsTxtMiddleware, Apify's custom scheduler already handles robots.txt
+    settings['DOWNLOADER_MIDDLEWARES']['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware'] = None
+
+    # Disable the default HttpProxyMiddleware and add ApifyHttpProxyMiddleware
+    settings['DOWNLOADER_MIDDLEWARES']['scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware'] = None
+    settings['DOWNLOADER_MIDDLEWARES']['apify.scrapy.middlewares.ApifyHttpProxyMiddleware'] = 950
+
+    # Disable the default RetryMiddleware and add ApifyRetryMiddleware with the highest integer (1000)
+    settings['DOWNLOADER_MIDDLEWARES']['scrapy.downloadermiddlewares.retry.RetryMiddleware'] = None
+    settings['DOWNLOADER_MIDDLEWARES']['apify.scrapy.middlewares.ApifyRetryMiddleware'] = 1000
+
+    # Store the proxy configuration
+    settings['APIFY_PROXY_SETTINGS'] = proxy_config
+
+    return settings
+
+
 async def open_queue_with_custom_client() -> RequestQueue:
     """Open a Request Queue with custom Apify Client.
 

{apify-1.5.2b5.dist-info → apify-1.5.3b2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify
-Version: 1.5.2b5
+Version: 1.5.3b2
 Summary: Apify SDK for Python
 Author-email: "Apify Technologies s.r.o." <support@apify.com>
 License: Apache Software License

{apify-1.5.2b5.dist-info → apify-1.5.3b2.dist-info}/RECORD

@@ -22,7 +22,7 @@ apify/_memory_storage/resource_clients/request_queue.py,sha256=6skV2fGpDxrDtn43o
 apify/_memory_storage/resource_clients/request_queue_collection.py,sha256=qW7kPLKRl5mukIGCT8iNCsXT82BUYSwCV_KH7MLj528,1713
 apify/scrapy/__init__.py,sha256=tCnqsdzcCx0Rpx13r1THeEJ6SzEgjmyrmHVKdCgfEfo,174
 apify/scrapy/scheduler.py,sha256=-6onJc3BPJbOfCs9yypzrp9N0hXdPVre7i7thHlZJ4k,4950
-apify/scrapy/utils.py,sha256=cv-lgnzuP8zDrGNlMiGYXVdPXbmEnrNpEA8h3My_2ec,7427
+apify/scrapy/utils.py,sha256=ixFoLbihzLZjjUuMtT8pqGItqVj-eOHrv7y1UEjELZ0,9387
 apify/scrapy/middlewares/__init__.py,sha256=zzosV8BD8SZQIrVKsSaGFGV9rHinNLKm5GPL3ZNxSZQ,96
 apify/scrapy/middlewares/apify_proxy.py,sha256=pDNx31y7llSv3ZM1SSujEYZSWQTfNfAqKvsNph5zbbU,5919
 apify/scrapy/middlewares/apify_retry.py,sha256=6VeWoYW8c_WgxFNE5aoZ4OwlFZwMIdOXF-q-UOSO3QE,4607
@@ -34,8 +34,8 @@ apify/storages/dataset.py,sha256=gfMlJ6dSXDdjCykkStZCyp4u8xKCIZqflGS9-jLzK74,233
 apify/storages/key_value_store.py,sha256=BUGYPI4MuRlOJ_aPzZr8bq7ay_K04qAZ7yKW7C8ItV4,10760
 apify/storages/request_queue.py,sha256=PP7KVZAQTR8-BPT54f-nk8Rbgkc5-bkDGFf4u_XQJLs,26957
 apify/storages/storage_client_manager.py,sha256=fvXg3PRojATNamEN29BBZyZZ5GWN_s0r29A59aiL-wQ,2465
-apify-1.5.2b5.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
-apify-1.5.2b5.dist-info/METADATA,sha256=mzFxqVGympnEbRPS4bMlnBmYNgwV6n4sHk-5I9AZmaE,6236
-apify-1.5.2b5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-apify-1.5.2b5.dist-info/top_level.txt,sha256=2oFNsHggn5m_rCaaP7xijQg_-Va2ByOSYuvKgACsS5w,6
-apify-1.5.2b5.dist-info/RECORD,,
+apify-1.5.3b2.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
+apify-1.5.3b2.dist-info/METADATA,sha256=EQzJ1o7R60UE5ThurCkoelSeRUPb-VfZrvCcQeEslmc,6236
+apify-1.5.3b2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+apify-1.5.3b2.dist-info/top_level.txt,sha256=2oFNsHggn5m_rCaaP7xijQg_-Va2ByOSYuvKgACsS5w,6
+apify-1.5.3b2.dist-info/RECORD,,