aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/core/downloader/handlers/curl_cffi.py

@@ -1,3 +1,13 @@
+"""
+Download handler implementation using curl_cffi.
+使用curl_cffi的下载处理程序实现。
+
+This module provides a download handler that uses curl_cffi to perform HTTP/HTTPS requests.
+It supports features like browser impersonation, proxies, and cookies.
+此模块提供了一个使用curl_cffi执行HTTP/HTTPS请求的下载处理程序。
+它支持浏览器模拟、代理和Cookie等功能。
+"""
+
 from curl_cffi.curl import CurlError
 from curl_cffi.requests import AsyncSession
 
@@ -10,50 +20,151 @@ from aioscrapy.utils.log import logger
 
 
 class CurlCffiDownloadHandler(BaseDownloadHandler):
+    """
+    Download handler that uses curl_cffi to perform HTTP/HTTPS requests.
+    使用curl_cffi执行HTTP/HTTPS请求的下载处理程序。
+
+    This handler implements the BaseDownloadHandler interface using the curl_cffi
+    library, which provides high-performance HTTP requests with browser fingerprinting
+    capabilities.
+    此处理程序使用curl_cffi库实现BaseDownloadHandler接口，该库提供具有浏览器指纹
+    功能的高性能HTTP请求。
+    """
 
     def __init__(self, settings):
+        """
+        Initialize the CurlCffiDownloadHandler.
+        初始化CurlCffiDownloadHandler。
+
+        Args:
+            settings: The settings object containing configuration for the handler.
+                     包含处理程序配置的设置对象。
+        """
         self.settings: Settings = settings
-        self.httpx_client_session_args: dict = self.settings.get('CURL_CFFI_CLIENT_SESSION_ARGS', {})
+
+        # Arguments to pass to curl_cffi AsyncSession constructor
+        # 传递给curl_cffi AsyncSession构造函数的参数
+        self.curl_cffi_args: dict = self.settings.get('CURL_CFFI_ARGS', {})
+
+        # SSL verification setting
+        # SSL验证设置
         self.verify_ssl: bool = self.settings.get("VERIFY_SSL", True)
 
     @classmethod
     def from_settings(cls, settings: Settings):
+        """
+        Create a download handler from settings.
+        从设置创建下载处理程序。
+
+        This is a factory method that creates a new CurlCffiDownloadHandler
+        instance with the given settings.
+        这是一个工厂方法，使用给定的设置创建一个新的CurlCffiDownloadHandler实例。
+
+        Args:
+            settings: The settings to use for the handler.
+                     用于处理程序的设置。
+
+        Returns:
+            CurlCffiDownloadHandler: A new download handler instance.
+                                    一个新的下载处理程序实例。
+        """
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        """
+        Download a request using curl_cffi.
+        使用curl_cffi下载请求。
+
+        This method implements the BaseDownloadHandler.download_request interface.
+        It wraps the actual download logic in _download_request and handles
+        curl_cffi-specific exceptions.
+        此方法实现了BaseDownloadHandler.download_request接口。
+        它将实际的下载逻辑包装在_download_request中，并处理curl_cffi特定的异常。
+
+        Args:
+            request: The request to download.
+                    要下载的请求。
+            _: The spider (not used in this implementation).
+               爬虫（在此实现中未使用）。
+
+        Returns:
+            HtmlResponse: The response from the server.
+                         来自服务器的响应。
+
+        Raises:
+            DownloadError: If a CurlError occurs during the download.
+                          如果在下载过程中发生CurlError。
+        """
         try:
             return await self._download_request(request)
         except CurlError as e:
+            # Wrap curl_cffi-specific exceptions in a generic DownloadError
+            # 将curl_cffi特定的异常包装在通用的DownloadError中
             raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
+        """
+        Internal method to perform the actual download using curl_cffi.
+        使用curl_cffi执行实际下载的内部方法。
+
+        This method configures and uses a curl_cffi.AsyncSession to perform the request,
+        handling SSL settings, proxies, cookies, browser impersonation, and other request parameters.
+        此方法配置并使用curl_cffi.AsyncSession执行请求，处理SSL设置、代理、Cookie、
+        浏览器模拟和其他请求参数。
+
+        Args:
+            request: The request to download.
+                    要下载的请求。
+
+        Returns:
+            HtmlResponse: The response from the server.
+                         来自服务器的响应。
+        """
+        # Configure request parameters
+        # 配置请求参数
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
             'verify': request.meta.get('verify_ssl', self.verify_ssl),
             'allow_redirects': self.settings.getbool('REDIRECT_ENABLED', True) if request.meta.get(
                 'dont_redirect') is None else request.meta.get('dont_redirect'),
-            'impersonate': request.meta.get('impersonate'),
+            'impersonate': request.meta.get('impersonate'),  # Browser fingerprinting feature
+                                                            # 浏览器指纹功能
         }
+
+        # Handle request body data
+        # 处理请求体数据
         post_data = request.body or None
         if isinstance(post_data, dict):
-            kwargs['json'] = post_data
+            kwargs['json'] = post_data  # Send as JSON
+                                       # 作为JSON发送
         else:
-            kwargs['data'] = post_data
+            kwargs['data'] = post_data  # Send as form data or raw bytes
+                                       # 作为表单数据或原始字节发送
 
+        # Set request headers
+        # 设置请求头
         headers = request.headers or self.settings.get('DEFAULT_REQUEST_HEADERS')
         kwargs['headers'] = headers
 
+        # Configure proxy if specified
+        # 如果指定，配置代理
         proxy = request.meta.get("proxy")
         if proxy:
             kwargs["proxies"] = {'http': proxy, 'https': proxy}
             logger.debug(f"use proxy {proxy}: {request.url}")
 
-        session_args = self.httpx_client_session_args.copy()
+        # Configure curl_cffi session
+        # 配置curl_cffi会话
+        session_args = self.curl_cffi_args.copy()
 
+        # Perform the request
+        # 执行请求
         async with AsyncSession(**session_args) as session:
             response = await session.request(request.method, request.url, **kwargs)
 
+        # Convert curl_cffi response to HtmlResponse
+        # 将curl_cffi响应转换为HtmlResponse
         return HtmlResponse(
             str(response.url),
             status=response.status_code,
@@ -64,4 +175,14 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
         )
 
     async def close(self):
+        """
+        Close the download handler and release resources.
+        关闭下载处理程序并释放资源。
+
+        This method is called when the spider is closing. In this implementation,
+        there are no persistent resources to clean up since curl_cffi.AsyncSession
+        is created and closed for each request.
+        当爬虫关闭时调用此方法。在此实现中，没有需要清理的持久资源，
+        因为curl_cffi.AsyncSession是为每个请求创建和关闭的。
+        """
         pass

aioscrapy/core/downloader/handlers/httpx.py

@@ -1,3 +1,13 @@
+"""
+Download handler implementation using httpx.
+使用httpx的下载处理程序实现。
+
+This module provides a download handler that uses httpx to perform HTTP/HTTPS requests.
+It supports HTTP/2, SSL customization, proxies, and cookies.
+此模块提供了一个使用httpx执行HTTP/HTTPS请求的下载处理程序。
+它支持HTTP/2、SSL自定义、代理和Cookie。
+"""
+
 import ssl
 
 import httpx
@@ -12,65 +22,175 @@ from aioscrapy.utils.log import logger
 
 
 class HttpxDownloadHandler(BaseDownloadHandler):
+    """
+    Download handler that uses httpx to perform HTTP/HTTPS requests.
+    使用httpx执行HTTP/HTTPS请求的下载处理程序。
+
+    This handler implements the BaseDownloadHandler interface using the httpx
+    library, which provides modern HTTP client features including HTTP/2 support,
+    connection pooling, and async capabilities.
+    此处理程序使用httpx库实现BaseDownloadHandler接口，该库提供现代HTTP客户端功能，
+    包括HTTP/2支持、连接池和异步功能。
+    """
 
     def __init__(self, settings):
+        """
+        Initialize the HttpxDownloadHandler.
+        初始化HttpxDownloadHandler。
+
+        Args:
+            settings: The settings object containing configuration for the handler.
+                     包含处理程序配置的设置对象。
+        """
         self.settings: Settings = settings
-        self.httpx_client_session_args: dict = self.settings.get('HTTPX_CLIENT_SESSION_ARGS', {})
+
+        # Arguments to pass to httpx AsyncClient constructor
+        # 传递给httpx AsyncClient构造函数的参数
+        self.httpx_args: dict = self.settings.get('HTTPX_ARGS', {})
+
+        # SSL verification setting
+        # SSL验证设置
         self.verify_ssl: bool = self.settings.get("VERIFY_SSL", True)
-        self.ssl_protocol = self.settings.get("SSL_PROTOCOL")  # ssl.PROTOCOL_TLSv1_2
+
+        # SSL protocol version to use (e.g., ssl.PROTOCOL_TLSv1_2)
+        # 要使用的SSL协议版本（例如，ssl.PROTOCOL_TLSv1_2）
+        self.ssl_protocol = self.settings.get("SSL_PROTOCOL")
+
+        # Fix for non-standard HTTP headers in responses
+        # 修复响应中的非标准HTTP头
         if self.settings.getbool("FIX_HTTPX_HEADER", True):
-            # Fixed non-standard response's header 修复不标准的响应头
             import h11
             import re
             h11._readers.header_field_re = re.compile(b"(?P<field_name>.*?):[ \t](?P<field_value>.*?)")
 
     @classmethod
     def from_settings(cls, settings: Settings):
+        """
+        Create a download handler from settings.
+        从设置创建下载处理程序。
+
+        This is a factory method that creates a new HttpxDownloadHandler
+        instance with the given settings.
+        这是一个工厂方法，使用给定的设置创建一个新的HttpxDownloadHandler实例。
+
+        Args:
+            settings: The settings to use for the handler.
+                     用于处理程序的设置。
+
+        Returns:
+            HttpxDownloadHandler: A new download handler instance.
+                                 一个新的下载处理程序实例。
+        """
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        """
+        Download a request using httpx.
+        使用httpx下载请求。
+
+        This method implements the BaseDownloadHandler.download_request interface.
+        It wraps the actual download logic in _download_request and handles
+        httpx-specific exceptions.
+        此方法实现了BaseDownloadHandler.download_request接口。
+        它将实际的下载逻辑包装在_download_request中，并处理httpx特定的异常。
+
+        Args:
+            request: The request to download.
+                    要下载的请求。
+            _: The spider (not used in this implementation).
+               爬虫（在此实现中未使用）。
+
+        Returns:
+            HtmlResponse: The response from the server.
+                         来自服务器的响应。
+
+        Raises:
+            DownloadError: If an HttpxError occurs during the download.
+                          如果在下载过程中发生HttpxError。
+        """
         try:
             return await self._download_request(request)
         except HttpxError as e:
+            # Wrap httpx-specific exceptions in a generic DownloadError
+            # 将httpx特定的异常包装在通用的DownloadError中
             raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
+        """
+        Internal method to perform the actual download using httpx.
+        使用httpx执行实际下载的内部方法。
+
+        This method configures and uses an httpx.AsyncClient to perform the request,
+        handling SSL settings, proxies, cookies, and other request parameters.
+        此方法配置并使用httpx.AsyncClient执行请求，处理SSL设置、代理、Cookie和其他请求参数。
+
+        Args:
+            request: The request to download.
+                    要下载的请求。
+
+        Returns:
+            HtmlResponse: The response from the server.
+                         来自服务器的响应。
+        """
+        # Configure request parameters
+        # 配置请求参数
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
             'data': request.body or None
         }
+
+        # Set request headers
+        # 设置请求头
         headers = request.headers or self.settings.get('DEFAULT_REQUEST_HEADERS')
         kwargs['headers'] = headers
 
-        session_args = self.httpx_client_session_args.copy()
-        session_args.setdefault('http2', True)
+        # Configure httpx client session
+        # 配置httpx客户端会话
+        session_args = self.httpx_args.copy()
+        session_args.setdefault('http2', True)  # Enable HTTP/2 by default
+                                               # 默认启用HTTP/2
         session_args.update({
             'verify': request.meta.get('verify_ssl', self.verify_ssl),
             'follow_redirects': self.settings.getbool('REDIRECT_ENABLED', True) if request.meta.get(
                 'dont_redirect') is None else request.meta.get('dont_redirect'),
             'max_redirects': self.settings.getint('REDIRECT_MAX_TIMES', 20),
         })
+
+        # Configure SSL settings if specified
+        # 如果指定，配置SSL设置
         ssl_ciphers = request.meta.get('TLS_CIPHERS')
         ssl_protocol = request.meta.get('ssl_protocol', self.ssl_protocol)
         if ssl_ciphers or ssl_protocol:
             if ssl_protocol:
+                # Create SSL context with specific protocol
+                # 使用特定协议创建SSL上下文
                 context = ssl.SSLContext(protocol=ssl_protocol)
             else:
+                # Use default SSL context
+                # 使用默认SSL上下文
                 context = ssl.create_default_context()
 
+            # Set SSL ciphers if specified
+            # 如果指定，设置SSL密码
             ssl_ciphers and context.set_ciphers(ssl_ciphers)
             session_args['verify'] = context
 
+        # Configure proxy if specified
+        # 如果指定，配置代理
         proxy = request.meta.get("proxy")
         if proxy:
             session_args["proxies"] = proxy
             logger.debug(f"使用代理{proxy}抓取: {request.url}")
 
+        # Perform the request
+        # 执行请求
         async with httpx.AsyncClient(**session_args) as session:
             response = await session.request(request.method, request.url, **kwargs)
             content = response.read()
 
+        # Convert httpx response to HtmlResponse
+        # 将httpx响应转换为HtmlResponse
         return HtmlResponse(
             str(response.url),
             status=response.status_code,
@@ -81,4 +201,14 @@ class HttpxDownloadHandler(BaseDownloadHandler):
         )
 
     async def close(self):
+        """
+        Close the download handler and release resources.
+        关闭下载处理程序并释放资源。
+
+        This method is called when the spider is closing. In this implementation,
+        there are no persistent resources to clean up since httpx.AsyncClient
+        is created and closed for each request.
+        当爬虫关闭时调用此方法。在此实现中，没有需要清理的持久资源，
+        因为httpx.AsyncClient是为每个请求创建和关闭的。
+        """
         pass

aioscrapy/core/downloader/handlers/pyhttpx.py

@@ -1,3 +1,13 @@
+"""
+Download handler implementation using pyhttpx.
+使用pyhttpx的下载处理程序实现。
+
+This module provides a download handler that uses pyhttpx to perform HTTP/HTTPS requests.
+It supports HTTP/2, proxies, and cookies, and runs synchronous pyhttpx in a thread pool.
+此模块提供了一个使用pyhttpx执行HTTP/HTTPS请求的下载处理程序。
+它支持HTTP/2、代理和Cookie，并在线程池中运行同步的pyhttpx。
+"""
+
 import asyncio
 
 import pyhttpx
@@ -12,24 +22,113 @@ from aioscrapy.utils.log import logger
 
 
 class PyhttpxDownloadHandler(BaseDownloadHandler):
+    """
+    Download handler that uses pyhttpx to perform HTTP/HTTPS requests.
+    使用pyhttpx执行HTTP/HTTPS请求的下载处理程序。
+
+    This handler implements the BaseDownloadHandler interface using the pyhttpx
+    library, which provides HTTP client features including HTTP/2 support.
+    Since pyhttpx is synchronous, this handler runs it in a thread pool.
+    此处理程序使用pyhttpx库实现BaseDownloadHandler接口，该库提供包括HTTP/2支持的HTTP客户端功能。
+    由于pyhttpx是同步的，此处理程序在线程池中运行它。
+    """
 
     def __init__(self, settings):
+        """
+        Initialize the PyhttpxDownloadHandler.
+        初始化PyhttpxDownloadHandler。
+
+        Args:
+            settings: The settings object containing configuration for the handler.
+                     包含处理程序配置的设置对象。
+        """
         self.settings: Settings = settings
-        self.pyhttpx_client_args: dict = self.settings.get('PYHTTPX_CLIENT_ARGS', {})
+
+        # Arguments to pass to pyhttpx HttpSession constructor
+        # 传递给pyhttpx HttpSession构造函数的参数
+        self.pyhttpx_args: dict = self.settings.get('PYHTTPX_ARGS', {})
+
+        # SSL verification setting
+        # SSL验证设置
         self.verify_ssl = self.settings.get("VERIFY_SSL", True)
+
+        # Get the current event loop for running pyhttpx in a thread pool
+        # 获取当前事件循环，用于在线程池中运行pyhttpx
         self.loop = asyncio.get_running_loop()
 
     @classmethod
     def from_settings(cls, settings: Settings):
+        """
+        Create a download handler from settings.
+        从设置创建下载处理程序。
+
+        This is a factory method that creates a new PyhttpxDownloadHandler
+        instance with the given settings.
+        这是一个工厂方法，使用给定的设置创建一个新的PyhttpxDownloadHandler实例。
+
+        Args:
+            settings: The settings to use for the handler.
+                     用于处理程序的设置。
+
+        Returns:
+            PyhttpxDownloadHandler: A new download handler instance.
+                                   一个新的下载处理程序实例。
+        """
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        """
+        Download a request using pyhttpx.
+        使用pyhttpx下载请求。
+
+        This method implements the BaseDownloadHandler.download_request interface.
+        It wraps the actual download logic in _download_request and handles
+        pyhttpx-specific exceptions.
+        此方法实现了BaseDownloadHandler.download_request接口。
+        它将实际的下载逻辑包装在_download_request中，并处理pyhttpx特定的异常。
+
+        Args:
+            request: The request to download.
+                    要下载的请求。
+            _: The spider (not used in this implementation).
+               爬虫（在此实现中未使用）。
+
+        Returns:
+            HtmlResponse: The response from the server.
+                         来自服务器的响应。
+
+        Raises:
+            DownloadError: If a PyHttpxError occurs during the download.
+                          如果在下载过程中发生PyHttpxError。
+        """
         try:
             return await self._download_request(request)
         except PyHttpxError as e:
+            # Wrap pyhttpx-specific exceptions in a generic DownloadError
+            # 将pyhttpx特定的异常包装在通用的DownloadError中
             raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
+        """
+        Internal method to perform the actual download using pyhttpx.
+        使用pyhttpx执行实际下载的内部方法。
+
+        This method configures and uses a pyhttpx.HttpSession to perform the request,
+        handling SSL settings, proxies, cookies, and other request parameters.
+        Since pyhttpx is synchronous, it runs in a thread pool using asyncio.to_thread.
+        此方法配置并使用pyhttpx.HttpSession执行请求，处理SSL设置、代理、Cookie和其他请求参数。
+        由于pyhttpx是同步的，它使用asyncio.to_thread在线程池中运行。
+
+        Args:
+            request: The request to download.
+                    要下载的请求。
+
+        Returns:
+            HtmlResponse: The response from the server.
+                         来自服务器的响应。
+        """
+        # Configure request parameters
+        # 配置请求参数
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
@@ -37,24 +136,47 @@ class PyhttpxDownloadHandler(BaseDownloadHandler):
             'allow_redirects': self.settings.getbool('REDIRECT_ENABLED', True) if request.meta.get(
                 'dont_redirect') is None else request.meta.get('dont_redirect')
         }
+
+        # Handle request body data
+        # 处理请求体数据
         post_data = request.body or None
         if isinstance(post_data, dict):
-            kwargs['json'] = post_data
+            kwargs['json'] = post_data  # Send as JSON
+                                       # 作为JSON发送
         else:
-            kwargs['data'] = post_data
+            kwargs['data'] = post_data  # Send as form data or raw bytes
+                                       # 作为表单数据或原始字节发送
 
+        # Set request headers
+        # 设置请求头
         headers = request.headers or self.settings.get('DEFAULT_REQUEST_HEADERS')
         kwargs['headers'] = headers
 
+        # Configure proxy if specified
+        # 如果指定，配置代理
         proxy = request.meta.get("proxy")
         if proxy:
             kwargs["proxies"] = {'https': proxy}
             logger.debug(f"use proxy {proxy}: {request.url}")
 
-        session_args = self.pyhttpx_client_args.copy()
-        session_args.setdefault('http2', True)
+        # Configure pyhttpx session
+        # 配置pyhttpx会话
+        session_args = self.pyhttpx_args.copy()
+        session_args.setdefault('http2', True)  # Enable HTTP/2 by default
+                                               # 默认启用HTTP/2
+
+        if ja3 := request.meta.get("ja3"):
+            session_args['ja3'] = ja3
+
+        # Execute the request in a thread pool since pyhttpx is synchronous
+        # 由于pyhttpx是同步的，在线程池中执行请求
         with pyhttpx.HttpSession(**session_args) as session:
+            # Run the synchronous pyhttpx request in a thread pool
+            # 在线程池中运行同步的pyhttpx请求
             response = await asyncio.to_thread(session.request, request.method, request.url, **kwargs)
+
+            # Convert pyhttpx response to HtmlResponse
+            # 将pyhttpx响应转换为HtmlResponse
             return HtmlResponse(
                 request.url,
                 status=response.status_code,
@@ -65,4 +187,14 @@ class PyhttpxDownloadHandler(BaseDownloadHandler):
             )
 
     async def close(self):
+        """
+        Close the download handler and release resources.
+        关闭下载处理程序并释放资源。
+
+        This method is called when the spider is closing. In this implementation,
+        there are no persistent resources to clean up since pyhttpx.HttpSession
+        is created and closed for each request.
+        当爬虫关闭时调用此方法。在此实现中，没有需要清理的持久资源，
+        因为pyhttpx.HttpSession是为每个请求创建和关闭的。
+        """
         pass