aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/core/downloader/handlers/__init__.py

@@ -1,4 +1,11 @@
-"""Download handlers for different schemes"""
+"""
+Download handlers for different URL schemes.
+不同URL方案的下载处理程序。
+
+This module provides the base classes and manager for download handlers,
+which are responsible for handling different URL schemes (http, https, ftp, etc.).
+此模块提供了下载处理程序的基类和管理器，负责处理不同的URL方案（http、https、ftp等）。
+"""
 
 from abc import abstractmethod
 from typing import Optional
@@ -13,73 +20,248 @@ from aioscrapy.utils.python import without_none_values
 
 
 class BaseDownloadHandler:
+    """
+    Base class for download handlers.
+    下载处理程序的基类。
+
+    Download handlers are responsible for handling requests with specific URL schemes
+    (http, https, ftp, etc.). Each scheme has its own handler implementation.
+    下载处理程序负责处理具有特定URL方案的请求（http、https、ftp等）。每个方案都有自己的处理程序实现。
+    """
+
     @abstractmethod
-    async def download_request(self, requests: Request, spider: Spider):
+    async def download_request(self, request: Request, spider: Spider):
+        """
+        Download the given request and return a response.
+        下载给定的请求并返回响应。
+
+        Args:
+            request: The request to download.
+                    要下载的请求。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            A response object.
+            响应对象。
+        """
         raise NotImplementedError()
 
     @abstractmethod
     async def close(self):
+        """
+        Close the handler and release its resources.
+        关闭处理程序并释放其资源。
+
+        This method is called when the spider is closed.
+        当爬虫关闭时调用此方法。
+        """
         pass
 
 
 class DownloadHandlerManager:
+    """
+    Manager for download handlers.
+    下载处理程序的管理器。
+
+    This class manages download handlers for different URL schemes.
+    It lazily loads handlers when they are first needed and keeps track
+    of which schemes are supported.
+    此类管理不同URL方案的下载处理程序。它在首次需要时懒加载处理程序，并跟踪支持哪些方案。
+    """
 
     def __init__(self, crawler):
+        """
+        Initialize the download handler manager.
+        初始化下载处理程序管理器。
+
+        Args:
+            crawler: The crawler instance that this manager belongs to.
+                    此管理器所属的爬虫实例。
+        """
         self._crawler = crawler
 
-        # stores acceptable schemes on instancing
+        # Load scheme handlers configuration from settings
+        # 从设置加载方案处理程序配置
+        # First try DOWNLOAD_HANDLERS_MAP[DOWNLOAD_HANDLERS_TYPE], then fall back to DOWNLOAD_HANDLERS
+        # 首先尝试DOWNLOAD_HANDLERS_MAP[DOWNLOAD_HANDLERS_TYPE]，然后回退到DOWNLOAD_HANDLERS
         self._schemes: dict = without_none_values(
             crawler.settings.get('DOWNLOAD_HANDLERS_MAP', {}).get(crawler.settings.get('DOWNLOAD_HANDLERS_TYPE')) or
             crawler.settings.getwithbase('DOWNLOAD_HANDLERS')
         )
+
+        # Dictionary of scheme -> handler instance
+        # 方案 -> 处理程序实例的字典
         self._handlers: dict = {}  # stores instanced handlers for schemes
+
+        # Dictionary of scheme -> error message for failed handlers
+        # 方案 -> 失败处理程序的错误消息的字典
         self._notconfigured: dict = {}  # remembers failed handlers
+
+        # Connect to engine_stopped signal to close handlers
+        # 连接到engine_stopped信号以关闭处理程序
         crawler.signals.connect(self._close, signals.engine_stopped)
 
     @classmethod
     def from_crawler(cls, crawler) -> "DownloadHandlerManager":
+        """
+        Create a download handler manager from a crawler.
+        从爬虫创建下载处理程序管理器。
+
+        This is a factory method that creates a new download handler manager
+        instance with the given crawler.
+        这是一个工厂方法，使用给定的爬虫创建一个新的下载处理程序管理器实例。
+
+        Args:
+            crawler: The crawler instance that will use this manager.
+                    将使用此管理器的爬虫实例。
+
+        Returns:
+            DownloadHandlerManager: A new download handler manager instance.
+                                   一个新的下载处理程序管理器实例。
+        """
         return cls(crawler)
 
     async def _get_handler(self, scheme: str) -> Optional[BaseDownloadHandler]:
-        """Lazy-load the downloadhandler for a scheme
-        only on the first request for that scheme.
         """
+        Lazy-load the download handler for a scheme.
+        懒加载方案的下载处理程序。
+
+        This method only loads the handler on the first request for that scheme.
+        此方法仅在首次请求该方案时加载处理程序。
+
+        Args:
+            scheme: The URL scheme to get a handler for (e.g., 'http', 'https', 'ftp').
+                   要获取处理程序的URL方案（例如，'http'、'https'、'ftp'）。
+
+        Returns:
+            BaseDownloadHandler: The handler for the scheme, or None if no handler
+                                is available or could be loaded.
+                                方案的处理程序，如果没有可用或无法加载的处理程序，则为None。
+        """
+        # Return cached handler if available
+        # 如果可用，返回缓存的处理程序
         if scheme in self._handlers:
             return self._handlers[scheme]
+
+        # Return None if we already know this scheme is not configured
+        # 如果我们已经知道此方案未配置，则返回None
         if scheme in self._notconfigured:
             return None
+
+        # Return None if no handler is defined for this scheme
+        # 如果没有为此方案定义处理程序，则返回None
         if scheme not in self._schemes:
             self._notconfigured[scheme] = 'no handler available for that scheme'
             return None
 
+        # Load the handler for this scheme
+        # 加载此方案的处理程序
         return await self._load_handler(scheme)
 
     async def _load_handler(self, scheme: str) -> Optional[BaseDownloadHandler]:
+        """
+        Load a download handler for a scheme.
+        加载方案的下载处理程序。
+
+        This method attempts to load the handler class specified in the settings
+        for the given scheme.
+        此方法尝试加载设置中为给定方案指定的处理程序类。
+
+        Args:
+            scheme: The URL scheme to load a handler for.
+                   要加载处理程序的URL方案。
+
+        Returns:
+            BaseDownloadHandler: The loaded handler, or None if the handler
+                                could not be loaded.
+                                加载的处理程序，如果无法加载处理程序，则为None。
+        """
+        # Get the handler class path from settings
+        # 从设置获取处理程序类路径
         path: str = self._schemes[scheme]
+
         try:
+            # Load the handler class
+            # 加载处理程序类
             dh: BaseDownloadHandler = await load_instance(
                 path,
                 settings=self._crawler.settings,
             )
         except NotConfigured as ex:
+            # Handler explicitly raised NotConfigured
+            # 处理程序明确引发NotConfigured
             self._notconfigured[scheme] = str(ex)
             return None
         except Exception as ex:
+            # Any other exception during loading
+            # 加载期间的任何其他异常
             logger.exception(f'Loading "{path}" for scheme "{scheme}"')
             self._notconfigured[scheme] = str(ex)
             return None
         else:
+            # Successfully loaded the handler
+            # 成功加载处理程序
             self._handlers[scheme] = dh
             return dh
 
     async def download_request(self, request: Request, spider: Spider) -> HtmlResponse:
+        """
+        Download a request using the appropriate handler for its URL scheme.
+        使用适合其URL方案的处理程序下载请求。
+
+        This method determines the URL scheme of the request, gets the appropriate
+        handler, and delegates the download to that handler.
+        此方法确定请求的URL方案，获取适当的处理程序，并将下载委托给该处理程序。
+
+        Args:
+            request: The request to download.
+                    要下载的请求。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            HtmlResponse: The response from the handler.
+                         来自处理程序的响应。
+
+        Raises:
+            NotSupported: If no handler is available for the request's URL scheme.
+                         如果请求的URL方案没有可用的处理程序。
+        """
+        # Extract the scheme from the URL (http, https, ftp, etc.)
+        # 从URL提取方案（http、https、ftp等）
         scheme = urlparse_cached(request).scheme
+
+        # Get the handler for this scheme
+        # 获取此方案的处理程序
         handler: BaseDownloadHandler = await self._get_handler(scheme)
+
+        # Raise an exception if no handler is available
+        # 如果没有可用的处理程序，则引发异常
         if not handler:
             raise NotSupported("Unsupported URL scheme '%s': %s" %
                                (scheme, self._notconfigured[scheme]))
+
+        # Delegate the download to the handler
+        # 将下载委托给处理程序
         return await handler.download_request(request, spider)
 
     async def _close(self, *_a, **_kw) -> None:
+        """
+        Close all download handlers.
+        关闭所有下载处理程序。
+
+        This method is called when the engine is stopped. It closes all
+        download handlers that have been loaded.
+        当引擎停止时调用此方法。它关闭所有已加载的下载处理程序。
+
+        Args:
+            *_a: Variable positional arguments from the signal (not used).
+                 来自信号的可变位置参数（未使用）。
+            **_kw: Variable keyword arguments from the signal (not used).
+                  来自信号的可变关键字参数（未使用）。
+        """
+        # Close each handler
+        # 关闭每个处理程序
         for dh in self._handlers.values():
             await dh.close()

aioscrapy/core/downloader/handlers/aiohttp.py

@@ -1,3 +1,13 @@
+"""
+
+Download handler implementation using aiohttp.
+使用aiohttp的下载处理程序实现。
+
+This module provides a download handler that uses aiohttp to perform HTTP/HTTPS requests.
+It supports features like browser impersonation, proxies, and cookies.
+此模块提供了一个使用aiohttp执行HTTP/HTTPS请求的下载处理程序。
+它支持浏览器模拟、代理和Cookie等功能。
+"""
 import asyncio
 import re
 import ssl
@@ -15,31 +25,168 @@ from aioscrapy.utils.log import logger
 
 
 class AioHttpDownloadHandler(BaseDownloadHandler):
-    session: Optional[aiohttp.ClientSession] = None
+    """
+    Download handler that uses aiohttp to download HTTP/HTTPS requests.
+    使用aiohttp下载HTTP/HTTPS请求的下载处理程序。
+
+    This handler implements the BaseDownloadHandler interface using the aiohttp
+    library to perform HTTP/HTTPS requests.
+    此处理程序使用aiohttp库执行HTTP/HTTPS请求，实现了BaseDownloadHandler接口。
+    """
+
+    session: Optional[aiohttp.ClientSession] = None  # Shared session when USE_SESSION is True
+                                                    # 当USE_SESSION为True时的共享会话
 
     def __init__(self, settings: Settings):
+        """
+        Initialize the AioHttpDownloadHandler.
+        初始化AioHttpDownloadHandler。
+
+        Args:
+            settings: The settings object containing configuration for the handler.
+                     包含处理程序配置的设置对象。
+        """
         self.settings = settings
-        self.aiohttp_client_session_args: dict = settings.getdict('AIOHTTP_CLIENT_SESSION_ARGS')
+
+        # Arguments to pass to aiohttp.ClientSession constructor
+        # 传递给aiohttp.ClientSession构造函数的参数
+        self.aiohttp_args: dict = settings.getdict('AIOHTTP_ARGS')
+
+        # SSL verification setting
+        # SSL验证设置
         self.verify_ssl: Optional[bool] = settings.get("VERIFY_SSL")
+
+        # SSL protocol version (e.g., ssl.PROTOCOL_TLSv1_2)
+        # SSL协议版本（例如，ssl.PROTOCOL_TLSv1_2）
         self.ssl_protocol = settings.get("SSL_PROTOCOL")  # ssl.PROTOCOL_TLSv1_2
+
+        # Whether to use a persistent session for all requests
+        # 是否对所有请求使用持久会话
         self.use_session: bool = settings.getbool("USE_SESSION", False)
 
     @classmethod
     def from_settings(cls, settings: Settings):
+        """
+        Create a download handler from settings.
+        从设置创建下载处理程序。
+
+        This is a factory method that creates a new AioHttpDownloadHandler
+        instance with the given settings.
+        这是一个工厂方法，使用给定的设置创建一个新的AioHttpDownloadHandler实例。
+
+        Args:
+            settings: The settings to use for the handler.
+                     用于处理程序的设置。
+
+        Returns:
+            AioHttpDownloadHandler: A new download handler instance.
+                                   一个新的下载处理程序实例。
+        """
         return cls(settings)
 
     def get_session(self, *args, **kwargs) -> aiohttp.ClientSession:
+        """
+        Get or create a shared aiohttp ClientSession.
+        获取或创建共享的aiohttp ClientSession。
+
+        This method returns the existing session if one exists, or creates
+        a new one if none exists yet. This is used when USE_SESSION is True
+        to reuse the same session for multiple requests.
+        如果会话已存在，此方法返回现有会话；如果尚不存在，则创建一个新会话。
+        当USE_SESSION为True时使用此方法，为多个请求重用相同的会话。
+
+        Args:
+            *args: Positional arguments to pass to aiohttp.ClientSession constructor.
+                  传递给aiohttp.ClientSession构造函数的位置参数。
+            **kwargs: Keyword arguments to pass to aiohttp.ClientSession constructor.
+                     传递给aiohttp.ClientSession构造函数的关键字参数。
+
+        Returns:
+            aiohttp.ClientSession: The shared client session.
+                                  共享的客户端会话。
+        """
         if self.session is None:
             self.session = aiohttp.ClientSession(*args, **kwargs)
         return self.session
 
-    async def download_request(self, request: Request, _) -> HtmlResponse:
+    async def download_request(self, request: Request, spider) -> HtmlResponse:
+        """
+        Download a request using aiohttp.
+        使用aiohttp下载请求。
+
+        This method implements the BaseDownloadHandler.download_request interface.
+        It wraps the actual download logic in _download_request and handles
+        aiohttp-specific exceptions.
+        此方法实现了BaseDownloadHandler.download_request接口。
+        它将实际的下载逻辑包装在_download_request中，并处理aiohttp特定的异常。
+
+        Args:
+            request: The request to download.
+                    要下载的请求。
+            spider: The spider making the request. This parameter is required by the
+                   BaseDownloadHandler interface but is not used in this implementation.
+                   发出请求的爬虫。此参数是BaseDownloadHandler接口所需的，但在此实现中未使用。
+                   It is included to maintain compatibility with the interface and to allow
+                   subclasses to use it if needed.
+                   包含它是为了保持与接口的兼容性，并允许子类在需要时使用它。
+
+        Returns:
+            HtmlResponse: The response from the server.
+                         来自服务器的响应。
+
+        Raises:
+            DownloadError: If an aiohttp ClientError occurs during the download.
+                          如果在下载过程中发生aiohttp ClientError。
+        """
         try:
+            # The spider parameter is intentionally unused in this implementation
+            # 在此实现中有意不使用spider参数
             return await self._download_request(request)
         except ClientError as e:
+            # Wrap aiohttp-specific exceptions in a generic DownloadError
+            # 将aiohttp特定的异常包装在通用的DownloadError中
             raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
+        """
+        Perform the actual download of a request using aiohttp.
+        使用aiohttp执行请求的实际下载。
+
+        This method handles the details of configuring and performing the HTTP request,
+        including SSL settings, proxies, cookies, and session management. It supports
+        various request options through request.meta:
+        此方法处理配置和执行HTTP请求的详细信息，包括SSL设置、代理、Cookie和会话管理。
+        它通过request.meta支持各种请求选项：
+
+        - verify_ssl: Whether to verify SSL certificates
+                     是否验证SSL证书
+        - download_timeout: Timeout for the request in seconds
+                          请求超时时间（秒）
+        - dont_redirect: Whether to disable following redirects
+                        是否禁用跟随重定向
+        - TLS_CIPHERS: Custom SSL cipher suite to use
+                      要使用的自定义SSL密码套件
+        - ssl_protocol: SSL protocol version to use
+                       要使用的SSL协议版本
+        - proxy: Proxy URL to use for the request
+                用于请求的代理URL
+
+        Args:
+            request: The request to download.
+                    要下载的请求。
+                    This includes the URL, method, headers, body, cookies, and
+                    meta information for configuring the request.
+                    这包括URL、方法、标头、正文、Cookie和用于配置请求的元信息。
+
+        Returns:
+            HtmlResponse: The response from the server.
+                         来自服务器的响应。
+                         This includes the status code, headers, body, cookies,
+                         and encoding of the response.
+                         这包括响应的状态码、标头、正文、Cookie和编码。
+        """
+        # Prepare request parameters
+        # 准备请求参数
         kwargs = {
             'verify_ssl': request.meta.get('verify_ssl', self.verify_ssl),
             'timeout': request.meta.get('download_timeout', 180),
@@ -50,9 +197,13 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
             'max_redirects': self.settings.getint('REDIRECT_MAX_TIMES', 20),
         }
 
+        # Set headers from request or default settings
+        # 从请求或默认设置设置标头
         headers = request.headers or self.settings.get('DEFAULT_REQUEST_HEADERS')
         kwargs['headers'] = headers
 
+        # Configure SSL context if needed
+        # 如果需要，配置SSL上下文
         ssl_ciphers: str = request.meta.get('TLS_CIPHERS')
         ssl_protocol = request.meta.get('ssl_protocol', self.ssl_protocol)
         if ssl_ciphers or ssl_protocol:
@@ -65,28 +216,38 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
             kwargs['ssl'] = context
             kwargs['verify_ssl'] = True
 
+        # Configure proxy if specified
+        # 如果指定，配置代理
         proxy: str = request.meta.get("proxy")
         if proxy:
             kwargs["proxy"] = proxy
             logger.debug(f"使用代理{proxy}抓取: {request.url}")
 
+        # Perform the request using either a persistent session or a new session
+        # 使用持久会话或新会话执行请求
         if self.use_session:
             # Not recommended to use session, The abnormal phenomena will occurs when using tunnel proxy
-            session = self.get_session(**self.aiohttp_client_session_args)
+            # 不建议使用会话，使用隧道代理时会出现异常现象
+            session = self.get_session(**self.aiohttp_args)
             async with session.request(request.method, request.url, **kwargs) as response:
                 content: bytes = await response.read()
-
         else:
-            async with aiohttp.ClientSession(**self.aiohttp_client_session_args) as session:
+            # Create a new session for each request (recommended)
+            # 为每个请求创建一个新会话（推荐）
+            async with aiohttp.ClientSession(**self.aiohttp_args) as session:
                 async with session.request(request.method, request.url, **kwargs) as response:
                     content: bytes = await response.read()
 
+        # Process cookies from response
+        # 处理响应中的Cookie
         r_cookies = response.cookies.output() or None
         if r_cookies:
             r_cookies = {
                 cookie[0]: cookie[1] for cookie in re.findall(r'Set-Cookie: (.*?)=(.*?); Domain', r_cookies, re.S)
             }
 
+        # Create and return the response object
+        # 创建并返回响应对象
         return HtmlResponse(
             str(response.url),
             status=response.status,
@@ -97,9 +258,32 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
         )
 
     async def close(self):
+        """
+        Close the download handler and release its resources.
+        关闭下载处理程序并释放其资源。
+
+        This method closes the shared session if one exists and waits for
+        the underlying SSL connections to close properly. It follows the
+        recommended graceful shutdown procedure for aiohttp sessions.
+        此方法关闭共享会话（如果存在），并等待底层SSL连接正确关闭。
+        它遵循aiohttp会话的推荐优雅关闭程序。
+
+        The 250ms sleep after closing the session is recommended by the aiohttp
+        documentation to allow the underlying SSL connections to be properly closed.
+        Without this delay, SSL connections might be terminated abruptly, which
+        can cause issues with some servers.
+        关闭会话后的250毫秒睡眠是aiohttp文档推荐的，以允许底层SSL连接正确关闭。
+        没有这个延迟，SSL连接可能会突然终止，这可能会导致某些服务器出现问题。
+
+        See: https://docs.aiohttp.org/en/latest/client_advanced.html#graceful-shutdown
+        参见：https://docs.aiohttp.org/en/latest/client_advanced.html#graceful-shutdown
+        """
         if self.session is not None:
+            # Close the shared session
+            # 关闭共享会话
             await self.session.close()
 
             # Wait 250 ms for the underlying SSL connections to close
+            # 等待250毫秒让底层SSL连接关闭
             # https://docs.aiohttp.org/en/latest/client_advanced.html#graceful-shutdown
             await asyncio.sleep(0.250)