apify 1.4.2b3__py3-none-any.whl → 1.5.0__py3-none-any.whl

apify/scrapy/__init__.py

@@ -1,4 +1,3 @@
-from .middlewares import ApifyRetryMiddleware
 from .pipelines import ActorDatasetPushPipeline
 from .scheduler import ApifyScheduler
-from .utils import get_running_event_loop_id, open_queue_with_custom_client, to_apify_request, to_scrapy_request
+from .utils import get_basic_auth_header, get_running_event_loop_id, open_queue_with_custom_client, to_apify_request, to_scrapy_request

apify/scrapy/middlewares/__init__.py

@@ -0,0 +1,2 @@
+from .apify_proxy import ApifyHttpProxyMiddleware
+from .apify_retry import ApifyRetryMiddleware

apify/scrapy/middlewares/apify_proxy.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from urllib.parse import ParseResult, urlparse
+
+from scrapy.core.downloader.handlers.http11 import TunnelError
+from scrapy.exceptions import NotConfigured
+
+from ...actor import Actor
+from ...proxy_configuration import ProxyConfiguration
+from ..utils import get_basic_auth_header
+
+if TYPE_CHECKING:
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
+
+
+class ApifyHttpProxyMiddleware:
+    """Apify HTTP proxy middleware for Scrapy.
+
+    This middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication
+    header. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL
+    is sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be
+    provided by the Actor input. An example of the proxy settings:
+
+    proxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}
+    """
+
+    def __init__(self: ApifyHttpProxyMiddleware, proxy_settings: dict) -> None:
+        """Create a new instance.
+
+        Args:
+            proxy_settings: Dictionary containing proxy settings, provided by the Actor input.
+            auth_encoding: Encoding for basic authentication (default is 'latin-1').
+        """
+        self._proxy_settings = proxy_settings
+        self._proxy_cfg_internal: ProxyConfiguration | None = None
+
+    @classmethod
+    def from_crawler(cls: type[ApifyHttpProxyMiddleware], crawler: Crawler) -> ApifyHttpProxyMiddleware:
+        """Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.
+
+        Args:
+            cls: Class type.
+            crawler: Scrapy Crawler object.
+
+        Returns:
+            ApifyHttpProxyMiddleware: Instance of the class.
+        """
+        proxy_settings: dict | None = crawler.settings.get('APIFY_PROXY_SETTINGS')
+
+        if proxy_settings is None:
+            Actor.log.warning('ApifyHttpProxyMiddleware is not going to be used. Object "proxyConfiguration" is probably missing in the Actor input.')
+            raise NotConfigured
+
+        use_apify_proxy = proxy_settings.get('useApifyProxy', False)
+
+        if use_apify_proxy is not True:
+            Actor.log.warning(
+                'ApifyHttpProxyMiddleware is not going to be used. Actor input field "proxyConfiguration.useApifyProxy" is probably set to False.'
+            )
+            raise NotConfigured
+
+        return cls(proxy_settings)
+
+    async def process_request(self: ApifyHttpProxyMiddleware, request: Request, spider: Spider) -> None:
+        """Process a Scrapy request by assigning a new proxy.
+
+        Args:
+            request: Scrapy Request object.
+            spider: Scrapy Spider object.
+
+        Raises:
+            ValueError: If username and password are not provided in the proxy URL.
+
+        Returns:
+            None: The request is processed and middleware pipeline can continue.
+        """
+        Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: request={request}, spider={spider}')
+        url = await self._get_new_proxy_url()
+
+        if not (url.username and url.password):
+            raise ValueError('Username and password must be provided in the proxy URL.')
+
+        request.meta['proxy'] = url.geturl()
+        basic_auth_header = get_basic_auth_header(url.username, url.password)
+        request.headers[b'Proxy-Authorization'] = basic_auth_header
+
+        Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: updated request.meta={request.meta}')
+
+    def process_exception(
+        self: ApifyHttpProxyMiddleware,
+        request: Request,
+        exception: Exception,
+        spider: Spider,
+    ) -> None | Request:
+        """Process an exception that occurs during request processing.
+
+        Args:
+            request: Scrapy Request object.
+            exception: Exception object.
+            spider: Scrapy Spider object.
+
+        Returns:
+            If a TunnelError occurs, return the request object to halt its processing in the middleware pipeline.
+            Return None otherwise to allow the continuation of request processing.
+        """
+        Actor.log.debug(
+            f'ApifyHttpProxyMiddleware.process_exception: request={request}, exception={exception}, spider={spider}',
+        )
+
+        if isinstance(exception, TunnelError):
+            Actor.log.warning(f'ApifyHttpProxyMiddleware: TunnelError occurred for request="{request}", reason="{exception}", skipping...')
+            return request
+
+        return None
+
+    async def _get_new_proxy_url(self: ApifyHttpProxyMiddleware) -> ParseResult:
+        """Get a new proxy URL.
+
+        Raises:
+            NotConfigured: If creation of the proxy configuration fails.
+
+        Returns:
+            ParseResult: New proxy URL.
+        """
+        # Get proxy configuration, creating it if necessary
+        proxy_cfg = (
+            self._proxy_cfg_internal
+            if isinstance(self._proxy_cfg_internal, ProxyConfiguration)
+            else await Actor.create_proxy_configuration(actor_proxy_input=self._proxy_settings)
+        )
+
+        # If the proxy configuration is still not available, raise an error. However, this should not happen due
+        # to the checks in the `from_crawler` method.
+        if proxy_cfg is None:
+            Actor.log.error('Creation of proxy configuration failed. Check the field "proxyConfiguration" in the Actor input.')
+            raise NotConfigured
+
+        # Store the proxy configuration for future use
+        self._proxy_cfg_internal = proxy_cfg
+
+        # Get a new proxy URL and return it
+        new_url = await proxy_cfg.new_url()
+        return urlparse(new_url)

apify/scrapy/{middlewares.py → middlewares/apify_retry.py}

@@ -11,14 +11,14 @@ except ImportError as exc:
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
     ) from exc
 
-from ..actor import Actor
-from .utils import nested_event_loop, open_queue_with_custom_client, to_apify_request
+from ...actor import Actor
+from ..utils import nested_event_loop, open_queue_with_custom_client, to_apify_request
 
 if TYPE_CHECKING:
     from scrapy import Spider
     from scrapy.http import Request, Response
 
-    from ..storages import RequestQueue
+    from ...storages import RequestQueue
 
 
 class ApifyRetryMiddleware(RetryMiddleware):

apify/scrapy/utils.py

@@ -3,6 +3,10 @@ from __future__ import annotations
 import asyncio
 import codecs
 import pickle
+from base64 import b64encode
+from urllib.parse import unquote
+
+from scrapy.utils.python import to_bytes
 
 try:
     from scrapy import Request, Spider
@@ -19,6 +23,13 @@ from ..storages import RequestQueue, StorageClientManager
 nested_event_loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
 
 
+def get_basic_auth_header(username: str, password: str, auth_encoding: str = 'latin-1') -> bytes:
+    """Generate a basic authentication header for the given username and password."""
+    string = f'{unquote(username)}:{unquote(password)}'
+    user_pass = to_bytes(string, encoding=auth_encoding)
+    return b'Basic ' + b64encode(user_pass)
+
+
 def get_running_event_loop_id() -> int:
     """Get the ID of the currently running event loop.
 

{apify-1.4.2b3.dist-info → apify-1.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify
-Version: 1.4.2b3
+Version: 1.5.0
 Summary: Apify SDK for Python
 Author-email: "Apify Technologies s.r.o." <support@apify.com>
 License: Apache Software License

{apify-1.4.2b3.dist-info → apify-1.5.0.dist-info}/RECORD

@@ -20,19 +20,21 @@ apify/_memory_storage/resource_clients/key_value_store.py,sha256=Fbw2dDsO8WveQBM
 apify/_memory_storage/resource_clients/key_value_store_collection.py,sha256=H_Uc2inqk_q4wn5rFxSD9G2swCWuPt8jJFsR7bCFe7s,1678
 apify/_memory_storage/resource_clients/request_queue.py,sha256=7LS_jrBBJvylFZedZHrgwMPyCsLz8X9-mAvvhOaYzXI,19614
 apify/_memory_storage/resource_clients/request_queue_collection.py,sha256=2WKOoE7FrAJmgyXbxCmowDRdkauYhdA3yGTJw5LeoAg,1651
-apify/scrapy/__init__.py,sha256=BgVp2s6SamGU99y1Dg9zGAYoXlUq_cuShciDbXA9ly4,245
-apify/scrapy/middlewares.py,sha256=iQltZDis-2cKJCDE87v4pPiMKlW3r9eurPbKqer1GW8,4556
+apify/scrapy/__init__.py,sha256=z8uIRyjdp03tm2mmgRXff3LasUwuInnvkFt_L2Q-cGw,222
 apify/scrapy/pipelines.py,sha256=beBC1JwPU-51vEHL4k3GC1iljY74e5XAor8YBE8lE-I,955
 apify/scrapy/scheduler.py,sha256=A6lKB7Bp5F2uoFy6l_gDb-CNHrVfznNORRnLWhFWOBY,4899
-apify/scrapy/utils.py,sha256=tYsCABtInRoDMytrvwQ5bx4Y-gkO2DcBBq_KYcbwzsI,6330
+apify/scrapy/utils.py,sha256=DcspnpCY7LVPGpFPrZzzn9nRB4p5gWgK-UOb1ChCwMA,6777
+apify/scrapy/middlewares/__init__.py,sha256=zzosV8BD8SZQIrVKsSaGFGV9rHinNLKm5GPL3ZNxSZQ,96
+apify/scrapy/middlewares/apify_proxy.py,sha256=SAbEtzYB-vGHpS1FaLth0afWeHxLsd4-qqdNLJI5fA4,5740
+apify/scrapy/middlewares/apify_retry.py,sha256=RrUMrXgk9FTydBG99VbD7m1nDtWccMsO_Kf-rNivunI,4559
 apify/storages/__init__.py,sha256=rBdwhyZxUMG6m_7uAb4sl5eg_dxiLvYVas5aRcZ6PIE,268
 apify/storages/base_storage.py,sha256=gV2izLpzM20kW__RgXo6Z_qf7dRPqiI_LLUup1ul3GU,7422
 apify/storages/dataset.py,sha256=fbWyr7IZwM19UruUFrQLl8VEdgmF-FeVHmAHI01r4sQ,23305
 apify/storages/key_value_store.py,sha256=vnCo-BgmVB4x1i_8A3cIypnCSItxHI_mZji_Xv8_fyk,10744
 apify/storages/request_queue.py,sha256=wUpPLXyDAwoVwhEMeeCyVE_QzbQZSRNYZ9-YoIL0GMs,26929
 apify/storages/storage_client_manager.py,sha256=QAGbu47pwFkHa-AFfolNW3W5hvR7zNz2yxK9Sv0wQbA,2457
-apify-1.4.2b3.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
-apify-1.4.2b3.dist-info/METADATA,sha256=SXCMib2QcYmL8XXA5cVs16xEMTyQkkdb1t5-GfnU6Xo,6235
-apify-1.4.2b3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-apify-1.4.2b3.dist-info/top_level.txt,sha256=2oFNsHggn5m_rCaaP7xijQg_-Va2ByOSYuvKgACsS5w,6
-apify-1.4.2b3.dist-info/RECORD,,
+apify-1.5.0.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
+apify-1.5.0.dist-info/METADATA,sha256=HP-3DvhaO_IRYFjnqyGzlIkseaXqGY_yIxU86Du57O0,6233
+apify-1.5.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+apify-1.5.0.dist-info/top_level.txt,sha256=2oFNsHggn5m_rCaaP7xijQg_-Va2ByOSYuvKgACsS5w,6
+apify-1.5.0.dist-info/RECORD,,