aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/http/response/web_driver.py

@@ -0,0 +1,144 @@
+"""
+Playwright response implementation for aioscrapy.
+aioscrapy的WebDriverResponse响应实现。
+
+This module provides the PlaywrightResponse class, which is a specialized TextResponse
+for handling responses from Playwright browser automation. It adds support for
+browser driver management and response caching.
+此模块提供了WebDriverResponse类，这是一个专门用于处理来自Playwright/DrissionPage等浏览器自动化的响应的TextResponse。
+它添加了对浏览器驱动程序管理和响应缓存的支持。
+"""
+
+from typing import Optional, Any
+
+from aioscrapy.http.response.text import TextResponse
+
+
+class WebDriverResponse(TextResponse):
+    """
+    A Response subclass for handling Playwright browser automation responses.
+    用于处理Playwright浏览器自动化响应的Response子类。
+
+    This class extends TextResponse to handle responses from Playwright browser automation.
+    It adds support for:
+    此类扩展了TextResponse以处理来自Playwright浏览器自动化的响应。
+    它添加了对以下内容的支持：
+
+    - Browser driver management
+      浏览器驱动程序管理
+    - Response caching
+      响应缓存
+    - Text content override
+      文本内容覆盖
+    - Intercepted request data
+      拦截的请求数据
+    """
+
+    def __init__(
+            self,
+            *args,
+            text: str = '',
+            cache_response: Optional[dict] = None,
+            driver: Optional["WebDriverBase"] = None,
+            driver_pool: Optional["WebDriverPool"] = None,
+            intercept_request: Optional[dict] = None,
+            **kwargs
+    ):
+        """
+        Initialize a PlaywrightResponse.
+        初始化PlaywrightResponse。
+
+        Args:
+            *args: Positional arguments passed to the TextResponse constructor.
+                  传递给TextResponse构造函数的位置参数。
+            text: The text content of the response, which can override the body's decoded text.
+                 响应的文本内容，可以覆盖正文的解码文本。
+            cache_response: A dictionary of cached response data.
+                          缓存的响应数据字典。
+            driver: The Playwright driver instance used for this response.
+                   用于此响应的Playwright驱动程序实例。
+            driver_pool: The WebDriverPool that manages the driver.
+                        管理驱动程序的WebDriverPool。
+            intercept_request: A dictionary of intercepted request data.
+                             拦截的请求数据字典。
+            **kwargs: Keyword arguments passed to the TextResponse constructor.
+                     传递给TextResponse构造函数的关键字参数。
+        """
+        # Store Playwright-specific attributes
+        # 存储Playwright特定的属性
+        self.driver = driver
+        self.driver_pool = driver_pool
+        self._text = text
+        self.cache_response = cache_response or {}
+        self.intercept_request = intercept_request
+
+        # Initialize the base TextResponse
+        # 初始化基本TextResponse
+        super().__init__(*args, **kwargs)
+
+    async def release(self):
+        """
+        Release the Playwright driver back to the pool.
+        将Playwright驱动程序释放回池中。
+
+        This method releases the driver instance back to the WebDriverPool
+        if both the driver and pool are available.
+        如果驱动程序和池都可用，此方法将驱动程序实例释放回WebDriverPool。
+
+        Returns:
+            None
+        """
+        self.driver_pool and self.driver and await self.driver_pool.release(self.driver)
+
+    @property
+    def text(self):
+        """
+        Get the response text content.
+        获取响应文本内容。
+
+        This property overrides the base TextResponse.text property to return
+        the explicitly set text content if available, otherwise falls back to
+        the decoded body text from the parent class.
+        此属性重写了基本TextResponse.text属性，如果可用，则返回明确设置的文本内容，
+        否则回退到父类的解码正文文本。
+
+        Returns:
+            str: The response text content.
+                响应文本内容。
+        """
+        return self._text or super().text
+
+    @text.setter
+    def text(self, text):
+        """
+        Set the response text content.
+        设置响应文本内容。
+
+        This setter allows explicitly setting the text content of the response,
+        which will override the decoded body text.
+        此设置器允许明确设置响应的文本内容，这将覆盖解码的正文文本。
+
+        Args:
+            text: The text content to set.
+                 要设置的文本内容。
+        """
+        self._text = text
+
+    def get_response(self, key) -> Any:
+        """
+        Get a value from the cached response data.
+        从缓存的响应数据中获取值。
+
+        This method retrieves a value from the cache_response dictionary
+        using the provided key.
+        此方法使用提供的键从cache_response字典中检索值。
+
+        Args:
+            key: The key to look up in the cached response data.
+                 在缓存的响应数据中查找的键。
+
+        Returns:
+            Any: The value associated with the key, or None if the key is not found.
+                与键关联的值，如果未找到键，则为None。
+        """
+        return self.cache_response.get(key)

aioscrapy/http/response/xml.py

@@ -1,12 +1,54 @@
 """
-This module implements the XmlResponse class which adds encoding
-discovering through XML encoding declarations to the TextResponse class.
+XML response implementation for aioscrapy.
+aioscrapy的XML响应实现。
 
-See documentation in docs/topics/request-response.rst
+This module provides the XmlResponse class, which is a specialized TextResponse
+for handling XML content. It inherits all functionality from TextResponse
+but is specifically intended for XML responses, with support for XML encoding
+declarations.
+此模块提供了XmlResponse类，这是一个专门用于处理XML内容的TextResponse。
+它继承了TextResponse的所有功能，但专门用于XML响应，支持XML编码声明。
 """
 
 from aioscrapy.http.response.text import TextResponse
 
 
 class XmlResponse(TextResponse):
+    """
+    A Response subclass specifically for XML responses.
+    专门用于XML响应的Response子类。
+
+    This class extends TextResponse to handle XML content. It inherits all the
+    functionality of TextResponse, including:
+    此类扩展了TextResponse以处理XML内容。它继承了TextResponse的所有功能，包括：
+
+    - Automatic encoding detection (including from XML declarations)
+      自动编码检测（包括从XML声明中）
+    - Unicode conversion
+      Unicode转换
+    - CSS and XPath selectors (particularly useful for XML)
+      CSS和XPath选择器（对XML特别有用）
+    - Enhanced link following
+      增强的链接跟踪
+
+    The main purpose of this class is to provide a specific type for XML responses,
+    which can be useful for type checking and middleware processing.
+    此类的主要目的是为XML响应提供特定类型，这对类型检查和中间件处理很有用。
+
+    Example:
+        ```python
+        def parse(self, response):
+            if isinstance(response, XmlResponse):
+                # Process XML response
+                items = response.xpath('//item')
+                for item in items:
+                    yield {
+                        'name': item.xpath('./name/text()').get(),
+                        'value': item.xpath('./value/text()').get()
+                    }
+            else:
+                # Handle other response types
+                pass
+        ```
+    """
     pass

aioscrapy/libs/downloader/defaultheaders.py

@@ -1,22 +1,86 @@
 """
-DefaultHeaders downloader middleware
+DefaultHeaders Downloader Middleware
+默认头部下载器中间件
 
-See documentation in docs/topics/downloader-middleware.rst
+This middleware sets default headers for all requests, as specified in the
+DEFAULT_REQUEST_HEADERS setting. These headers are only set if they are not
+already present in the request.
+此中间件为所有请求设置默认头部，如DEFAULT_REQUEST_HEADERS设置中指定的那样。
+这些头部仅在请求中尚未存在时才会设置。
 """
 
 from aioscrapy.utils.python import without_none_values
 
 
 class DefaultHeadersMiddleware:
+    """
+    Middleware for setting default headers on requests.
+    用于在请求上设置默认头部的中间件。
+
+    This middleware adds default headers to all outgoing requests, as specified in the
+    DEFAULT_REQUEST_HEADERS setting. Headers are only added if they are not already
+    present in the request, allowing request-specific headers to take precedence.
+    此中间件向所有传出请求添加默认头部，如DEFAULT_REQUEST_HEADERS设置中指定的那样。
+    仅当请求中尚未存在头部时才会添加头部，允许特定于请求的头部优先。
+    """
 
     def __init__(self, headers):
+        """
+        Initialize the DefaultHeadersMiddleware.
+        初始化DefaultHeadersMiddleware。
+
+        Args:
+            headers: An iterable of (name, value) pairs representing the default headers.
+                    表示默认头部的(名称, 值)对的可迭代对象。
+        """
+        # Store the default headers
+        # 存储默认头部
         self._headers = headers
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a DefaultHeadersMiddleware instance from a crawler.
+        从爬虫创建DefaultHeadersMiddleware实例。
+
+        This is the factory method used by AioScrapy to create the middleware.
+        这是AioScrapy用于创建中间件的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this middleware.
+                    将使用此中间件的爬虫。
+
+        Returns:
+            DefaultHeadersMiddleware: A new DefaultHeadersMiddleware instance.
+                                     一个新的DefaultHeadersMiddleware实例。
+        """
+        # Get the default headers from settings, filtering out None values
+        # 从设置获取默认头部，过滤掉None值
         headers = without_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS'])
+
+        # Create and return a new instance with the headers as (name, value) pairs
+        # 使用作为(名称, 值)对的头部创建并返回一个新实例
         return cls(headers.items())
 
     def process_request(self, request, spider):
+        """
+        Process a request before it is sent to the downloader.
+        在请求发送到下载器之前处理它。
+
+        This method adds the default headers to the request if they are not already present.
+        如果请求中尚未存在默认头部，此方法会将其添加到请求中。
+
+        Args:
+            request: The request being processed.
+                    正在处理的请求。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            None: This method does not return a response or a deferred.
+                 此方法不返回响应或延迟对象。
+        """
+        # Add each default header to the request if it's not already set
+        # 如果尚未设置，则将每个默认头部添加到请求中
         for k, v in self._headers:
             request.headers.setdefault(k, v)

aioscrapy/libs/downloader/downloadtimeout.py

@@ -1,26 +1,115 @@
 """
-Download timeout middleware
+Download Timeout Middleware
+下载超时中间件
 
-See documentation in docs/topics/downloader-middleware.rst
+This middleware sets a default timeout for all requests, as specified in the
+DOWNLOAD_TIMEOUT setting or the spider's download_timeout attribute. The timeout
+can be overridden on a per-request basis by setting the 'download_timeout' key
+in the request's meta dictionary.
+此中间件为所有请求设置默认超时，如DOWNLOAD_TIMEOUT设置或爬虫的download_timeout
+属性中指定的那样。可以通过在请求的meta字典中设置'download_timeout'键来覆盖每个
+请求的超时。
 """
 
 from aioscrapy import signals
 
 
 class DownloadTimeoutMiddleware:
+    """
+    Middleware for setting default download timeouts on requests.
+    用于在请求上设置默认下载超时的中间件。
+
+    This middleware sets a default timeout for all outgoing requests, as specified in the
+    DOWNLOAD_TIMEOUT setting or the spider's download_timeout attribute. The timeout
+    can be overridden on a per-request basis by setting the 'download_timeout' key
+    in the request's meta dictionary.
+    此中间件为所有传出请求设置默认超时，如DOWNLOAD_TIMEOUT设置或爬虫的download_timeout
+    属性中指定的那样。可以通过在请求的meta字典中设置'download_timeout'键来覆盖每个
+    请求的超时。
+    """
 
     def __init__(self, timeout=180):
+        """
+        Initialize the DownloadTimeoutMiddleware.
+        初始化DownloadTimeoutMiddleware。
+
+        Args:
+            timeout: The default download timeout in seconds.
+                    默认下载超时（以秒为单位）。
+                    Defaults to 180 seconds.
+                    默认为180秒。
+        """
+        # Store the default timeout
+        # 存储默认超时
         self._timeout = timeout
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a DownloadTimeoutMiddleware instance from a crawler.
+        从爬虫创建DownloadTimeoutMiddleware实例。
+
+        This is the factory method used by AioScrapy to create the middleware.
+        这是AioScrapy用于创建中间件的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this middleware.
+                    将使用此中间件的爬虫。
+
+        Returns:
+            DownloadTimeoutMiddleware: A new DownloadTimeoutMiddleware instance.
+                                      一个新的DownloadTimeoutMiddleware实例。
+        """
+        # Create a new instance with the timeout from settings
+        # 使用来自设置的超时创建一个新实例
         o = cls(crawler.settings.getfloat('DOWNLOAD_TIMEOUT'))
+
+        # Connect to the spider_opened signal
+        # 连接到spider_opened信号
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
+
+        # Return the new instance
+        # 返回新实例
         return o
 
     def spider_opened(self, spider):
+        """
+        Handle the spider_opened signal.
+        处理spider_opened信号。
+
+        This method is called when a spider is opened. It updates the default timeout
+        with the spider's download_timeout attribute if it exists.
+        当爬虫打开时调用此方法。如果存在，它会使用爬虫的download_timeout属性更新默认超时。
+
+        Args:
+            spider: The spider that was opened.
+                   被打开的爬虫。
+        """
+        # Update the timeout with the spider's download_timeout attribute if it exists
+        # 如果存在，则使用爬虫的download_timeout属性更新超时
         self._timeout = getattr(spider, 'download_timeout', self._timeout)
 
     def process_request(self, request, spider):
+        """
+        Process a request before it is sent to the downloader.
+        在请求发送到下载器之前处理它。
+
+        This method sets the default download timeout in the request's meta dictionary
+        if it's not already set and if a default timeout is configured.
+        如果尚未设置默认下载超时且已配置默认超时，此方法会在请求的meta字典中设置它。
+
+        Args:
+            request: The request being processed.
+                    正在处理的请求。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            None: This method does not return a response or a deferred.
+                 此方法不返回响应或延迟对象。
+        """
+        # Set the default download timeout in the request's meta if it's not already set
+        # and if a default timeout is configured
+        # 如果尚未设置默认下载超时且已配置默认超时，则在请求的meta中设置它
         if self._timeout:
             request.meta.setdefault('download_timeout', self._timeout)

aioscrapy/libs/downloader/ja3fingerprint.py

@@ -1,36 +1,129 @@
 """
-SSL/TLS指纹随机中间件
+JA3 Fingerprint Randomization Middleware
+JA3指纹随机化中间件
+
+This module provides a middleware for randomizing SSL/TLS cipher suites to help
+avoid fingerprinting and detection when making HTTPS requests. JA3 is a method
+for creating SSL/TLS client fingerprints that can be used to identify specific
+clients regardless of the presented hostname or client IP address.
+此模块提供了一个中间件，用于随机化SSL/TLS密码套件，以帮助在发出HTTPS请求时
+避免指纹识别和检测。JA3是一种创建SSL/TLS客户端指纹的方法，可用于识别特定
+客户端，而不考虑所呈现的主机名或客户端IP地址。
+
+By randomizing the order of cipher suites, this middleware helps to generate
+different JA3 fingerprints for each request, making it harder for servers to
+track or block the crawler based on its TLS fingerprint.
+通过随机化密码套件的顺序，此中间件有助于为每个请求生成不同的JA3指纹，
+使服务器更难基于其TLS指纹跟踪或阻止爬虫。
 """
 import random
 
 
+# Default cipher suite string used when no custom ciphers are specified
+# 未指定自定义密码套件时使用的默认密码套件字符串
 ORIGIN_CIPHERS = ('ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
                   'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:!eNULL:!MD5')
 
 
 class TLSCiphersMiddleware:
-    """SSL/TLS指纹中间件"""
+    """
+    SSL/TLS Fingerprint Randomization Middleware.
+    SSL/TLS指纹随机化中间件。
+
+    This middleware modifies the SSL/TLS cipher suites used in HTTPS requests
+    to help avoid fingerprinting and detection. It can use custom cipher suites
+    or randomize the order of the default cipher suites.
+    此中间件修改HTTPS请求中使用的SSL/TLS密码套件，以帮助避免指纹识别和检测。
+    它可以使用自定义密码套件或随机化默认密码套件的顺序。
+    """
 
     def __init__(self, ciphers, is_random):
+        """
+        Initialize the TLS Ciphers Middleware.
+        初始化TLS密码套件中间件。
+
+        Args:
+            ciphers: The cipher suites to use, or 'DEFAULT' to use the default ciphers.
+                    要使用的密码套件，或'DEFAULT'以使用默认密码套件。
+            is_random: Whether to randomize the order of cipher suites.
+                      是否随机化密码套件的顺序。
+        """
+        # If ciphers is 'DEFAULT', set self.ciphers to None to use ORIGIN_CIPHERS later
+        # 如果ciphers是'DEFAULT'，将self.ciphers设置为None以便稍后使用ORIGIN_CIPHERS
         if ciphers == 'DEFAULT':
             self.ciphers = None
+        else:
+            self.ciphers = ciphers
 
         self.is_random = is_random
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a TLSCiphersMiddleware instance from a crawler.
+        从爬虫创建TLSCiphersMiddleware实例。
+
+        This is the factory method used by AioScrapy to create middleware instances.
+        这是AioScrapy用于创建中间件实例的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this middleware.
+                    将使用此中间件的爬虫。
+
+        Returns:
+            TLSCiphersMiddleware: A new TLSCiphersMiddleware instance.
+                                 一个新的TLSCiphersMiddleware实例。
+        """
         return cls(
+            # Get custom cipher suites from settings, or use 'DEFAULT'
+            # 从设置获取自定义密码套件，或使用'DEFAULT'
             ciphers=crawler.settings.get('DOWNLOADER_CLIENT_TLS_CIPHERS', 'DEFAULT'),
+            # Get whether to randomize cipher suites from settings
+            # 从设置获取是否随机化密码套件
             is_random=crawler.settings.get('RANDOM_TLS_CIPHERS', False)
         )
 
     def process_request(self, request, spider):
+        """
+        Process a request before it is sent to the downloader.
+        在请求发送到下载器之前处理它。
+
+        This method sets the TLS cipher suites for the request, optionally
+        randomizing their order to generate different JA3 fingerprints.
+        此方法为请求设置TLS密码套件，可选择随机化它们的顺序以生成不同的JA3指纹。
+
+        Args:
+            request: The request being processed.
+                    正在处理的请求。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            None: This method returns None to continue processing the request.
+                 此方法返回None以继续处理请求。
+        """
+        # Skip if neither custom ciphers nor randomization is enabled
+        # 如果既没有启用自定义密码套件也没有启用随机化，则跳过
         if not (self.ciphers or self.is_random):
             return
 
+        # Use custom ciphers if specified, otherwise use default
+        # 如果指定了自定义密码套件则使用它，否则使用默认值
         ciphers = self.ciphers or ORIGIN_CIPHERS
+
+        # Randomize cipher suite order if enabled
+        # 如果启用了随机化，则随机化密码套件顺序
         if self.is_random:
+            # Split the cipher string into individual ciphers
+            # 将密码字符串拆分为单个密码
             ciphers = ciphers.split(":")
+            # Shuffle the ciphers randomly
+            # 随机打乱密码
             random.shuffle(ciphers)
+            # Join the ciphers back into a string
+            # 将密码重新连接成字符串
             ciphers = ":".join(ciphers)
+
+        # Set the cipher suites in the request metadata
+        # 在请求元数据中设置密码套件
         request.meta['TLS_CIPHERS'] = ciphers