apify 1.4.2b3__tar.gz → 1.5.0b1__tar.gz

{apify-1.4.2b3 → apify-1.5.0b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify
-Version: 1.4.2b3
+Version: 1.5.0b1
 Summary: Apify SDK for Python
 Author-email: "Apify Technologies s.r.o." <support@apify.com>
 License: Apache Software License

{apify-1.4.2b3 → apify-1.5.0b1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "apify"
-version = "1.4.2b3"
+version = "1.5.0b1"
 description = "Apify SDK for Python"
 readme = "README.md"
 license = { text = "Apache Software License" }

apify-1.5.0b1/src/apify/scrapy/__init__.py

@@ -0,0 +1,3 @@
+from .pipelines import ActorDatasetPushPipeline
+from .scheduler import ApifyScheduler
+from .utils import get_basic_auth_header, get_running_event_loop_id, open_queue_with_custom_client, to_apify_request, to_scrapy_request

apify-1.5.0b1/src/apify/scrapy/middlewares/__init__.py

@@ -0,0 +1,2 @@
+from .apify_proxy import ApifyHttpProxyMiddleware
+from .apify_retry import ApifyRetryMiddleware

apify-1.5.0b1/src/apify/scrapy/middlewares/apify_proxy.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from urllib.parse import ParseResult, urlparse
+
+from scrapy.core.downloader.handlers.http11 import TunnelError
+from scrapy.exceptions import NotConfigured
+
+from ...actor import Actor
+from ...proxy_configuration import ProxyConfiguration
+from ..utils import get_basic_auth_header
+
+if TYPE_CHECKING:
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
+
+
+class ApifyHttpProxyMiddleware:
+    """Apify HTTP proxy middleware for Scrapy.
+
+    This middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication
+    header. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL
+    is sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be
+    provided by the Actor input. An example of the proxy settings:
+
+    proxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}
+    """
+
+    def __init__(self: ApifyHttpProxyMiddleware, proxy_settings: dict) -> None:
+        """Create a new instance.
+
+        Args:
+            proxy_settings: Dictionary containing proxy settings, provided by the Actor input.
+            auth_encoding: Encoding for basic authentication (default is 'latin-1').
+        """
+        self._proxy_settings = proxy_settings
+        self._proxy_cfg_internal: ProxyConfiguration | None = None
+
+    @classmethod
+    def from_crawler(cls: type[ApifyHttpProxyMiddleware], crawler: Crawler) -> ApifyHttpProxyMiddleware:
+        """Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.
+
+        Args:
+            cls: Class type.
+            crawler: Scrapy Crawler object.
+
+        Returns:
+            ApifyHttpProxyMiddleware: Instance of the class.
+        """
+        proxy_settings: dict | None = crawler.settings.get('APIFY_PROXY_SETTINGS')
+
+        if proxy_settings is None:
+            Actor.log.warning('ApifyHttpProxyMiddleware is not going to be used. Object "proxyConfiguration" is probably missing in the Actor input.')
+            raise NotConfigured
+
+        use_apify_proxy = proxy_settings.get('useApifyProxy', False)
+
+        if use_apify_proxy is not True:
+            Actor.log.warning(
+                'ApifyHttpProxyMiddleware is not going to be used. Actor input field "proxyConfiguration.useApifyProxy" is probably set to False.'
+            )
+            raise NotConfigured
+
+        return cls(proxy_settings)
+
+    async def process_request(self: ApifyHttpProxyMiddleware, request: Request, spider: Spider) -> None:
+        """Process a Scrapy request by assigning a new proxy.
+
+        Args:
+            request: Scrapy Request object.
+            spider: Scrapy Spider object.
+
+        Raises:
+            ValueError: If username and password are not provided in the proxy URL.
+
+        Returns:
+            None: The request is processed and middleware pipeline can continue.
+        """
+        Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: request={request}, spider={spider}')
+        url = await self._get_new_proxy_url()
+
+        if not (url.username and url.password):
+            raise ValueError('Username and password must be provided in the proxy URL.')
+
+        request.meta['proxy'] = url.geturl()
+        basic_auth_header = get_basic_auth_header(url.username, url.password)
+        request.headers[b'Proxy-Authorization'] = basic_auth_header
+
+        Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: updated request.meta={request.meta}')
+
+    def process_exception(
+        self: ApifyHttpProxyMiddleware,
+        request: Request,
+        exception: Exception,
+        spider: Spider,
+    ) -> None | Request:
+        """Process an exception that occurs during request processing.
+
+        Args:
+            request: Scrapy Request object.
+            exception: Exception object.
+            spider: Scrapy Spider object.
+
+        Returns:
+            If a TunnelError occurs, return the request object to halt its processing in the middleware pipeline.
+            Return None otherwise to allow the continuation of request processing.
+        """
+        Actor.log.debug(
+            f'ApifyHttpProxyMiddleware.process_exception: request={request}, exception={exception}, spider={spider}',
+        )
+
+        if isinstance(exception, TunnelError):
+            Actor.log.warning(f'ApifyHttpProxyMiddleware: TunnelError occurred for request="{request}", reason="{exception}", skipping...')
+            return request
+
+        return None
+
+    async def _get_new_proxy_url(self: ApifyHttpProxyMiddleware) -> ParseResult:
+        """Get a new proxy URL.
+
+        Raises:
+            NotConfigured: If creation of the proxy configuration fails.
+
+        Returns:
+            ParseResult: New proxy URL.
+        """
+        # Get proxy configuration, creating it if necessary
+        proxy_cfg = (
+            self._proxy_cfg_internal
+            if isinstance(self._proxy_cfg_internal, ProxyConfiguration)
+            else await Actor.create_proxy_configuration(actor_proxy_input=self._proxy_settings)
+        )
+
+        # If the proxy configuration is still not available, raise an error. However, this should not happen due
+        # to the checks in the `from_crawler` method.
+        if proxy_cfg is None:
+            Actor.log.error('Creation of proxy configuration failed. Check the field "proxyConfiguration" in the Actor input.')
+            raise NotConfigured
+
+        # Store the proxy configuration for future use
+        self._proxy_cfg_internal = proxy_cfg
+
+        # Get a new proxy URL and return it
+        new_url = await proxy_cfg.new_url()
+        return urlparse(new_url)

apify-1.4.2b3/src/apify/scrapy/middlewares.py → apify-1.5.0b1/src/apify/scrapy/middlewares/apify_retry.py

@@ -11,14 +11,14 @@ except ImportError as exc:
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
     ) from exc
 
-from ..actor import Actor
-from .utils import nested_event_loop, open_queue_with_custom_client, to_apify_request
+from ...actor import Actor
+from ..utils import nested_event_loop, open_queue_with_custom_client, to_apify_request
 
 if TYPE_CHECKING:
     from scrapy import Spider
     from scrapy.http import Request, Response
 
-    from ..storages import RequestQueue
+    from ...storages import RequestQueue
 
 
 class ApifyRetryMiddleware(RetryMiddleware):

{apify-1.4.2b3 → apify-1.5.0b1}/src/apify/scrapy/utils.py

@@ -3,6 +3,10 @@ from __future__ import annotations
 import asyncio
 import codecs
 import pickle
+from base64 import b64encode
+from urllib.parse import unquote
+
+from scrapy.utils.python import to_bytes
 
 try:
     from scrapy import Request, Spider
@@ -19,6 +23,13 @@ from ..storages import RequestQueue, StorageClientManager
 nested_event_loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
 
 
+def get_basic_auth_header(username: str, password: str, auth_encoding: str = 'latin-1') -> bytes:
+    """Generate a basic authentication header for the given username and password."""
+    string = f'{unquote(username)}:{unquote(password)}'
+    user_pass = to_bytes(string, encoding=auth_encoding)
+    return b'Basic ' + b64encode(user_pass)
+
+
 def get_running_event_loop_id() -> int:
     """Get the ID of the currently running event loop.
 

{apify-1.4.2b3 → apify-1.5.0b1}/src/apify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify
-Version: 1.4.2b3
+Version: 1.5.0b1
 Summary: Apify SDK for Python
 Author-email: "Apify Technologies s.r.o." <support@apify.com>
 License: Apache Software License

{apify-1.4.2b3 → apify-1.5.0b1}/src/apify.egg-info/SOURCES.txt

@@ -29,10 +29,12 @@ src/apify/_memory_storage/resource_clients/key_value_store_collection.py
 src/apify/_memory_storage/resource_clients/request_queue.py
 src/apify/_memory_storage/resource_clients/request_queue_collection.py
 src/apify/scrapy/__init__.py
-src/apify/scrapy/middlewares.py
 src/apify/scrapy/pipelines.py
 src/apify/scrapy/scheduler.py
 src/apify/scrapy/utils.py
+src/apify/scrapy/middlewares/__init__.py
+src/apify/scrapy/middlewares/apify_proxy.py
+src/apify/scrapy/middlewares/apify_retry.py
 src/apify/storages/__init__.py
 src/apify/storages/base_storage.py
 src/apify/storages/dataset.py

apify-1.4.2b3/src/apify/scrapy/__init__.py

@@ -1,4 +0,0 @@
-from .middlewares import ApifyRetryMiddleware
-from .pipelines import ActorDatasetPushPipeline
-from .scheduler import ApifyScheduler
-from .utils import get_running_event_loop_id, open_queue_with_custom_client, to_apify_request, to_scrapy_request