aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/utils/reqser.py

@@ -1,5 +1,12 @@
 """
-Helper functions for serializing (and deserializing) requests.
+Request serialization utilities for aioscrapy.
+aioscrapy的请求序列化实用工具。
+
+This module provides helper functions for serializing and deserializing Request objects.
+These functions are particularly useful for storing requests in queues, databases,
+or transmitting them between different processes or systems.
+此模块提供了用于序列化和反序列化Request对象的辅助函数。
+这些函数对于在队列、数据库中存储请求或在不同进程或系统之间传输请求特别有用。
 """
 from typing import Optional
 
@@ -8,8 +15,75 @@ from aioscrapy.utils.request import request_from_dict as _from_dict
 
 
 def request_to_dict(request: "aioscrapy.Request", spider: Optional["aioscrapy.Spider"] = None) -> dict:
+    """
+    Convert a Request object to a dictionary representation.
+    将Request对象转换为字典表示。
+
+    This function serializes a Request object into a dictionary that can be easily
+    stored or transmitted. The dictionary contains all the necessary information
+    to reconstruct the Request object later using request_from_dict().
+    此函数将Request对象序列化为可以轻松存储或传输的字典。
+    该字典包含稍后使用request_from_dict()重建Request对象所需的所有信息。
+
+    Args:
+        request: The Request object to serialize.
+                要序列化的Request对象。
+        spider: Optional Spider instance that may be used to customize the
+               serialization process. Some Request subclasses may use the spider
+               to properly serialize their attributes.
+               可选的Spider实例，可用于自定义序列化过程。
+               某些Request子类可能使用spider来正确序列化其属性。
+
+    Returns:
+        dict: A dictionary representation of the Request object.
+              Request对象的字典表示。
+
+    Example:
+        >>> request = Request('http://example.com', callback='parse_item')
+        >>> request_dict = request_to_dict(request, spider)
+        >>> # The dictionary can be stored or transmitted
+        >>> new_request = await request_from_dict(request_dict, spider)
+    """
+    # Delegate to the Request object's to_dict method
+    # 委托给Request对象的to_dict方法
     return request.to_dict(spider=spider)
 
 
 async def request_from_dict(d: dict, spider: Optional["aioscrapy.Spider"] = None) -> "aioscrapy.Request":
+    """
+    Convert a dictionary representation back to a Request object.
+    将字典表示转换回Request对象。
+
+    This function deserializes a dictionary (previously created by request_to_dict)
+    back into a Request object. It reconstructs all the attributes and properties
+    of the original Request, including callback and errback methods if a spider
+    is provided.
+    此函数将（先前由request_to_dict创建的）字典反序列化回Request对象。
+    它重建原始Request的所有属性和属性，如果提供了spider，
+    还包括回调和错误回调方法。
+
+    Args:
+        d: The dictionary representation of a Request object.
+           Request对象的字典表示。
+        spider: Optional Spider instance that may be used to resolve callback
+               and errback method names to actual methods on the spider.
+               可选的Spider实例，可用于将回调和错误回调方法名称
+               解析为spider上的实际方法。
+
+    Returns:
+        aioscrapy.Request: A reconstructed Request object.
+                          重建的Request对象。
+
+    Example:
+        >>> request_dict = {
+        ...     'url': 'http://example.com',
+        ...     'callback': 'parse_item',
+        ...     'method': 'GET'
+        ... }
+        >>> request = await request_from_dict(request_dict, spider)
+        >>> request.url
+        'http://example.com'
+    """
+    # Delegate to the imported _from_dict function from aioscrapy.utils.request
+    # 委托给从aioscrapy.utils.request导入的_from_dict函数
     return await _from_dict(d, spider=spider)

aioscrapy/utils/request.py

@@ -1,6 +1,12 @@
 """
-This module provides some useful functions for working with
-aioscrapy.http.Request objects
+Request utility functions for aioscrapy.
+aioscrapy的请求实用函数。
+
+This module provides utility functions for working with aioscrapy.http.Request objects.
+It includes functions for converting requests to raw HTTP representations, extracting
+referrer information, and creating Request objects from dictionaries.
+此模块提供了用于处理aioscrapy.http.Request对象的实用函数。
+它包括将请求转换为原始HTTP表示、提取引用者信息以及从字典创建Request对象的函数。
 """
 
 from typing import Optional
@@ -15,53 +21,208 @@ from aioscrapy.utils.python import to_bytes, to_unicode
 
 
 def request_httprepr(request: Request) -> bytes:
-    """Return the raw HTTP representation (as bytes) of the given request.
-    This is provided only for reference since it's not the actual stream of
-    bytes that will be send when performing the request (that's controlled
-    by Twisted).
     """
+    Return the raw HTTP representation of a request as bytes.
+    以字节形式返回请求的原始HTTP表示。
+
+    This function converts a Request object to its raw HTTP representation,
+    including the request line, headers, and body. This is useful for debugging
+    and logging purposes.
+    此函数将Request对象转换为其原始HTTP表示，包括请求行、头部和正文。
+    这对于调试和日志记录目的很有用。
+
+    Note:
+        This is provided only for reference since it's not the actual stream of
+        bytes that will be sent when performing the request (that's controlled
+        by the HTTP client implementation).
+        这仅供参考，因为它不是执行请求时将发送的实际字节流
+        （那由HTTP客户端实现控制）。
+
+    Args:
+        request: The Request object to convert.
+                要转换的Request对象。
+
+    Returns:
+        bytes: The raw HTTP representation of the request.
+               请求的原始HTTP表示。
+
+    Example:
+        >>> request = Request('http://example.com', method='POST',
+        ...                   headers={'Content-Type': 'application/json'},
+        ...                   body='{"key": "value"}')
+        >>> print(request_httprepr(request).decode())
+        POST / HTTP/1.1
+        Host: example.com
+        Content-Type: application/json
+
+        {"key": "value"}
+    """
+    # Parse the URL
+    # 解析URL
     parsed = urlparse_cached(request)
+
+    # Construct the path including params and query
+    # 构造包含参数和查询的路径
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
+
+    # Start with the request line
+    # 从请求行开始
     s = to_bytes(request.method) + b" " + to_bytes(path) + b" HTTP/1.1\r\n"
+
+    # Add the Host header
+    # 添加Host头部
     s += b"Host: " + to_bytes(parsed.hostname or b'') + b"\r\n"
+
+    # Add other headers if present
+    # 如果存在，添加其他头部
     if request.headers:
         s += headers_dict_to_raw({to_bytes(k): to_bytes(v) for k, v in request.headers.items()}) + b"\r\n"
+
+    # Add the empty line that separates headers from body
+    # 添加分隔头部和正文的空行
     s += b"\r\n"
+
+    # Add the body
+    # 添加正文
     s += to_bytes(request.body)
+
     return s
 
 
 def referer_str(request: Request) -> Optional[str]:
-    """ Return Referer HTTP header suitable for logging. """
+    """
+    Return the Referer HTTP header in a format suitable for logging.
+    以适合日志记录的格式返回Referer HTTP头。
+
+    This function extracts the 'Referer' header from a request and converts it
+    to a unicode string, replacing any invalid characters. This is useful for
+    logging purposes to avoid encoding errors.
+    此函数从请求中提取'Referer'头并将其转换为unicode字符串，
+    替换任何无效字符。这对于日志记录很有用，可以避免编码错误。
+
+    Args:
+        request: The Request object to extract the Referer from.
+                要提取Referer的Request对象。
+
+    Returns:
+        Optional[str]: The Referer header as a unicode string, or None if the
+                      header is not present.
+                      作为unicode字符串的Referer头，如果头不存在则为None。
+    """
+    # Get the Referer header from the request
+    # 从请求中获取Referer头
     referrer = request.headers.get('Referer')
+
+    # If there's no Referer header, return None
+    # 如果没有Referer头，返回None
     if referrer is None:
         return referrer
+
+    # Convert the Referer to unicode, replacing any invalid characters
+    # 将Referer转换为unicode，替换任何无效字符
     return to_unicode(referrer, errors='replace')
 
 
 async def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
-    """Create a :class:`~scrapy.Request` object from a dict.
+    """
+    Create a Request object from a dictionary.
+    从字典创建Request对象。
+
+    This function converts a dictionary representation of a request into an actual
+    Request object. It's useful for deserializing requests, for example when
+    loading them from a queue or a file.
+    此函数将请求的字典表示转换为实际的Request对象。
+    它对于反序列化请求很有用，例如从队列或文件加载请求时。
+
+    If a spider is provided, the function will:
+    1. First call the spider's request_from_dict method to allow custom processing
+    2. Try to resolve callback and errback strings to actual methods on the spider
 
-    If a spider is given, it will try to resolve the callbacks looking at the
-    spider for methods with the same name.
+    如果提供了爬虫，该函数将：
+    1. 首先调用爬虫的request_from_dict方法以允许自定义处理
+    2. 尝试将callback和errback字符串解析为爬虫上的实际方法
+
+    Args:
+        d: Dictionary containing the request attributes.
+           包含请求属性的字典。
+        spider: Optional spider instance to resolve callbacks and errbacks.
+               可选的爬虫实例，用于解析回调和错误回调。
+
+    Returns:
+        Request: A Request object (or subclass) with the attributes from the dictionary.
+                具有字典中属性的Request对象（或子类）。
+
+    Raises:
+        ValueError: If a callback or errback name cannot be resolved to a method.
+                   如果回调或错误回调名称无法解析为方法。
     """
-    d = await spider.request_from_dict(d) or d
+    # Allow the spider to customize the dictionary
+    # 允许爬虫自定义字典
+    if spider:
+        d = await spider.request_from_dict(d) or d
+
+    # If the spider already returned a Request object, return it directly
+    # 如果爬虫已经返回了一个Request对象，直接返回它
     if isinstance(d, Request):
         return d
 
+    # Determine the request class to use (default is Request)
+    # 确定要使用的请求类（默认为Request）
     request_cls = load_object(d["_class"]) if "_class" in d else Request
+
+    # Filter the dictionary to only include valid attributes for the request class
+    # 过滤字典，只包含请求类的有效属性
     kwargs = {key: value for key, value in d.items() if key in request_cls.attributes}
+
+    # Resolve callback string to actual method if spider is provided
+    # 如果提供了爬虫，将回调字符串解析为实际方法
     if d.get("callback") and spider:
         kwargs["callback"] = _get_method(spider, d["callback"])
+
+    # Resolve errback string to actual method if spider is provided
+    # 如果提供了爬虫，将错误回调字符串解析为实际方法
     if d.get("errback") and spider:
         kwargs["errback"] = _get_method(spider, d["errback"])
+
+    # Create and return the request object
+    # 创建并返回请求对象
     return request_cls(**kwargs)
 
 
 def _get_method(obj, name):
-    """Helper function for request_from_dict"""
+    """
+    Get a method from an object by name.
+    通过名称从对象获取方法。
+
+    This is a helper function for request_from_dict that resolves method names
+    to actual method objects. It's used to convert callback and errback strings
+    to callable methods on a spider.
+    这是request_from_dict的辅助函数，用于将方法名称解析为实际的方法对象。
+    它用于将回调和错误回调字符串转换为爬虫上的可调用方法。
+
+    Args:
+        obj: The object to get the method from (typically a spider).
+             要从中获取方法的对象（通常是爬虫）。
+        name: The name of the method to get.
+              要获取的方法的名称。
+
+    Returns:
+        callable: The method object.
+                 方法对象。
+
+    Raises:
+        ValueError: If the method is not found on the object.
+                   如果在对象上找不到该方法。
+    """
+    # Ensure the name is a string
+    # 确保名称是字符串
     name = str(name)
+
+    # Try to get the method from the object
+    # 尝试从对象获取方法
     try:
         return getattr(obj, name)
     except AttributeError:
+        # Raise a more informative error if the method is not found
+        # 如果找不到该方法，引发更多信息的错误
         raise ValueError(f"Method {name!r} not found in: {obj}")

aioscrapy/utils/response.py

@@ -1,6 +1,11 @@
 """
-This module provides some useful functions for working with
-scrapy.http.Response objects
+Response utility functions for aioscrapy.
+aioscrapy的响应实用函数。
+
+This module provides utility functions for working with aioscrapy.http.Response objects.
+It includes functions for extracting base URLs and meta refresh directives from HTML responses.
+此模块提供了用于处理aioscrapy.http.Response对象的实用函数。
+它包括从HTML响应中提取基本URL和元刷新指令的函数。
 """
 from typing import Iterable, Optional, Tuple, Union
 from weakref import WeakKeyDictionary
@@ -10,17 +15,59 @@ from w3lib import html
 import aioscrapy
 from aioscrapy.http.response import Response
 
+# Cache for storing base URLs to avoid repeated parsing of the same response
+# 缓存存储基本URL，以避免重复解析相同的响应
 _baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
 
 
 def get_base_url(response: "aioscrapy.http.response.TextResponse") -> str:
-    """Return the base url of the given response, joined with the response url"""
+    """
+    Extract the base URL from an HTML response.
+    从HTML响应中提取基本URL。
+
+    This function extracts the base URL from an HTML response by looking for
+    the <base> tag in the HTML. If found, it returns the href attribute of the
+    base tag, resolved against the response URL. If not found, it returns the
+    response URL.
+    此函数通过查找HTML中的<base>标签来从HTML响应中提取基本URL。
+    如果找到，它返回base标签的href属性，相对于响应URL解析。
+    如果未找到，它返回响应URL。
+
+    The function uses a cache to avoid repeated parsing of the same response.
+    Only the first 4KB of the response text are examined for performance reasons.
+    该函数使用缓存来避免重复解析相同的响应。
+    出于性能原因，只检查响应文本的前4KB。
+
+    Args:
+        response: The HTML response to extract the base URL from.
+                 要从中提取基本URL的HTML响应。
+
+    Returns:
+        str: The base URL of the response, which could be either:
+             响应的基本URL，可能是：
+             - The href attribute of the <base> tag, resolved against the response URL
+               <base>标签的href属性，相对于响应URL解析
+             - The response URL if no <base> tag is found
+               如果未找到<base>标签，则为响应URL
+    """
+    # Check if the base URL is already cached for this response
+    # 检查此响应的基本URL是否已缓存
     if response not in _baseurl_cache:
+        # Only examine the first 4KB of the response for performance
+        # 出于性能考虑，只检查响应的前4KB
         text = response.text[0:4096]
+        # Extract the base URL using w3lib.html
+        # 使用w3lib.html提取基本URL
         _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding)
+    # Return the cached base URL
+    # 返回缓存的基本URL
     return _baseurl_cache[response]
 
 
+# Cache for storing meta refresh directives to avoid repeated parsing of the same response
+# 缓存存储元刷新指令，以避免重复解析相同的响应
+# The cache stores either (None, None) if no meta refresh is found, or (seconds, url) if found
+# 如果未找到元刷新，缓存存储(None, None)，如果找到，则存储(秒数, url)
 _metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary()
 
 
@@ -28,11 +75,49 @@ def get_meta_refresh(
     response: "aioscrapy.http.response.TextResponse",
     ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'),
 ) -> Union[Tuple[None, None], Tuple[float, str]]:
-    """Parse the http-equiv refrsh parameter from the given response"""
+    """
+    Extract the meta refresh directive from an HTML response.
+    从HTML响应中提取元刷新指令。
+
+    This function looks for the HTML meta refresh tag in the response and extracts
+    the delay (in seconds) and the URL to redirect to. The meta refresh tag is
+    typically used for automatic page redirection or refreshing.
+    此函数在响应中查找HTML元刷新标签，并提取延迟（以秒为单位）和要重定向到的URL。
+    元刷新标签通常用于自动页面重定向或刷新。
+
+    Example of a meta refresh tag:
+    元刷新标签的示例：
+    <meta http-equiv="refresh" content="5; url=https://example.com">
+
+    The function uses a cache to avoid repeated parsing of the same response.
+    Only the first 4KB of the response text are examined for performance reasons.
+    该函数使用缓存来避免重复解析相同的响应。
+    出于性能原因，只检查响应文本的前4KB。
+
+    Args:
+        response: The HTML response to extract the meta refresh from.
+                 要从中提取元刷新的HTML响应。
+        ignore_tags: HTML tags to ignore when parsing. Default is ('script', 'noscript').
+                    解析时要忽略的HTML标签。默认为('script', 'noscript')。
+
+    Returns:
+        A tuple containing:
+        包含以下内容的元组：
+        - If meta refresh is found: (delay_seconds, url)
+          如果找到元刷新：(延迟秒数, url)
+        - If no meta refresh is found: (None, None)
+          如果未找到元刷新：(None, None)
+    """
+    # Check if the meta refresh is already cached for this response
+    # 检查此响应的元刷新是否已缓存
     if response not in _metaref_cache:
+        # Only examine the first 4KB of the response for performance
+        # 出于性能考虑，只检查响应的前4KB
         text = response.text[0:4096]
+        # Extract the meta refresh using w3lib.html
+        # 使用w3lib.html提取元刷新
         _metaref_cache[response] = html.get_meta_refresh(
             text, response.url, response.encoding, ignore_tags=ignore_tags)
+    # Return the cached meta refresh
+    # 返回缓存的元刷新
     return _metaref_cache[response]
-
-