aio-scrapy 2.1.3__py3-none-any.whl → 2.1.6__py3-none-any.whl

aioscrapy/http/request/json_request.py

@@ -1,8 +1,13 @@
-"""
-This module implements the JsonRequest class which is a more convenient class
-(than Request) to generate JSON Requests.
 
-See documentation in docs/topics/request-response.rst
+"""
+JSON request implementation for aioscrapy.
+aioscrapy的JSON请求实现。
+
+This module provides the JsonRequest class, which is a specialized Request
+that handles JSON data, automatically setting appropriate headers and
+serializing Python objects to JSON.
+此模块提供了JsonRequest类，这是一个专门处理JSON数据的Request，
+自动设置适当的头部并将Python对象序列化为JSON。
 """
 
 import copy
@@ -11,53 +16,160 @@ import warnings
 from typing import Optional, Tuple
 
 from aioscrapy.http.request import Request
-from aioscrapy.utils.deprecate import create_deprecated_class
 
 
 class JsonRequest(Request):
+    """
+    A Request that handles JSON data.
+    处理JSON数据的Request。
+
+    This class extends the base Request to handle JSON data, automatically
+    setting appropriate headers for JSON content and serializing Python
+    objects to JSON format.
+    此类扩展了基本Request以处理JSON数据，自动设置JSON内容的
+    适当头部，并将Python对象序列化为JSON格式。
+    """
+
+    # Add dumps_kwargs to the list of attributes to be included in serialization
+    # 将dumps_kwargs添加到要包含在序列化中的属性列表中
     attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
 
     def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None:
+        """
+        Initialize a JsonRequest.
+        初始化JsonRequest。
+
+        This constructor extends the base Request constructor to handle JSON data.
+        It accepts either a 'body' parameter with pre-serialized JSON or a 'data'
+        parameter with a Python object to be serialized to JSON.
+        此构造函数扩展了基本Request构造函数以处理JSON数据。
+        它接受带有预序列化JSON的'body'参数或带有要序列化为JSON的Python对象的'data'参数。
+
+        Args:
+            *args: Positional arguments passed to the Request constructor.
+                  传递给Request构造函数的位置参数。
+            dumps_kwargs: Optional keyword arguments to pass to json.dumps().
+                         可选的关键字参数，传递给json.dumps()。
+            **kwargs: Keyword arguments passed to the Request constructor.
+                     May include 'data' (a Python object to serialize to JSON)
+                     or 'body' (pre-serialized JSON string).
+                     传递给Request构造函数的关键字参数。
+                     可能包括'data'（要序列化为JSON的Python对象）
+                     或'body'（预序列化的JSON字符串）。
+        """
+        # Make a deep copy of dumps_kwargs to avoid modifying the original
+        # 深拷贝dumps_kwargs以避免修改原始对象
         dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {}
         self._dumps_kwargs = dumps_kwargs
 
+        # Check if body or data parameters were provided
+        # 检查是否提供了body或data参数
         body_passed = kwargs.get('body', None) is not None
         data = kwargs.pop('data', None)
         data_passed = data is not None
 
+        # Handle the case where both body and data are provided
+        # 处理同时提供body和data的情况
         if body_passed and data_passed:
             warnings.warn('Both body and data passed. data will be ignored')
 
+        # Handle the case where only data is provided
+        # 处理只提供data的情况
         elif not body_passed and data_passed:
+            # Serialize the data to JSON and set it as the body
+            # 将数据序列化为JSON并将其设置为body
             kwargs['body'] = self._dumps(data)
 
+            # Default to POST method if not specified
+            # 如果未指定，则默认为POST方法
             if 'method' not in kwargs:
                 kwargs['method'] = 'POST'
 
+        # Initialize the base Request
+        # 初始化基本Request
         super().__init__(*args, **kwargs)
+
+        # Set default headers for JSON content
+        # 设置JSON内容的默认头部
         self.headers.setdefault('Content-Type', 'application/json')
         self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
 
     @property
     def dumps_kwargs(self) -> dict:
+        """
+        Get the keyword arguments used for JSON serialization.
+        获取用于JSON序列化的关键字参数。
+
+        These arguments are passed to json.dumps() when serializing data.
+        这些参数在序列化数据时传递给json.dumps()。
+
+        Returns:
+            dict: The keyword arguments for json.dumps().
+                 json.dumps()的关键字参数。
+        """
         return self._dumps_kwargs
 
     def replace(self, *args, **kwargs) -> Request:
+        """
+        Create a new JsonRequest with the same attributes except for those given new values.
+        创建一个新的JsonRequest，除了给定的新值外，其他属性与当前JsonRequest相同。
+
+        This method extends the base Request.replace() method to handle the 'data'
+        parameter, serializing it to JSON if provided.
+        此方法扩展了基本Request.replace()方法以处理'data'参数，
+        如果提供了该参数，则将其序列化为JSON。
+
+        Args:
+            *args: Positional arguments passed to the base replace() method.
+                  传递给基本replace()方法的位置参数。
+            **kwargs: Keyword arguments passed to the base replace() method.
+                     May include 'data' (a Python object to serialize to JSON)
+                     or 'body' (pre-serialized JSON string).
+                     传递给基本replace()方法的关键字参数。
+                     可能包括'data'（要序列化为JSON的Python对象）
+                     或'body'（预序列化的JSON字符串）。
+
+        Returns:
+            Request: A new JsonRequest object.
+                    一个新的JsonRequest对象。
+        """
+        # Check if body or data parameters were provided
+        # 检查是否提供了body或data参数
         body_passed = kwargs.get('body', None) is not None
         data = kwargs.pop('data', None)
         data_passed = data is not None
 
+        # Handle the case where both body and data are provided
+        # 处理同时提供body和data的情况
         if body_passed and data_passed:
             warnings.warn('Both body and data passed. data will be ignored')
 
+        # Handle the case where only data is provided
+        # 处理只提供data的情况
         elif not body_passed and data_passed:
+            # Serialize the data to JSON and set it as the body
+            # 将数据序列化为JSON并将其设置为body
             kwargs['body'] = self._dumps(data)
 
+        # Call the base replace() method
+        # 调用基本replace()方法
         return super().replace(*args, **kwargs)
 
     def _dumps(self, data: dict) -> str:
-        """Convert to JSON """
+        """
+        Convert Python data to a JSON string.
+        将Python数据转换为JSON字符串。
+
+        This internal method serializes the given data to JSON using the
+        json.dumps() function with the configured keyword arguments.
+        此内部方法使用json.dumps()函数和配置的关键字参数将给定数据序列化为JSON。
+
+        Args:
+            data: The Python object to serialize to JSON.
+                 要序列化为JSON的Python对象。
+
+        Returns:
+            str: The JSON string representation of the data.
+                数据的JSON字符串表示。
+        """
         return json.dumps(data, **self._dumps_kwargs)
-
-
-JSONRequest = create_deprecated_class("JSONRequest", JsonRequest)

aioscrapy/http/response/__init__.py

@@ -1,9 +1,15 @@
+
 """
-This module implements the Response class which is used to represent HTTP
-responses in aioscrapy.
+HTTP Response implementation for aioscrapy.
+aioscrapy的HTTP响应实现。
 
-See documentation in docs/topics/request-response.rst
+This module provides the Response class, which represents an HTTP response
+received by the crawler. It handles response data, headers, cookies, and
+provides methods for URL joining and following links.
+此模块提供了Response类，表示爬虫接收到的HTTP响应。它处理响应数据、
+头部、Cookie，并提供URL连接和跟踪链接的方法。
 """
+
 from typing import Generator, Optional
 from urllib.parse import urljoin
 
@@ -24,6 +30,19 @@ class Response(object):
             flags: Optional[list] = None,
             request: Optional[Request] = None,
     ):
+        """
+        Initialize a Response object.
+        初始化Response对象。
+
+        Args:
+            url: URL for this response. 此响应的URL。
+            status: HTTP status code. HTTP状态码。
+            headers: HTTP headers. HTTP头信息。
+            cookies: Cookies from the response. 响应中的Cookie。
+            body: Response body. 响应体。
+            flags: Response flags. 响应标志。
+            request: The Request object that generated this response. 生成此响应的Request对象。
+        """
         self.headers = headers or {}
         self.status = int(status)
         self._set_body(body)
@@ -34,6 +53,24 @@ class Response(object):
 
     @property
     def cb_kwargs(self):
+        """
+        Get the callback keyword arguments from the request that generated this response.
+        从生成此响应的请求中获取回调关键字参数。
+
+        This property provides access to the cb_kwargs dictionary of the request
+        that generated this response, allowing callback functions to access
+        data passed from the request.
+        此属性提供对生成此响应的请求的cb_kwargs字典的访问，
+        允许回调函数访问从请求传递的数据。
+
+        Returns:
+            dict: The callback keyword arguments dictionary.
+                 回调关键字参数字典。
+
+        Raises:
+            AttributeError: If this response is not tied to any request.
+                           如果此响应未与任何请求关联。
+        """
         try:
             return self.request.cb_kwargs
         except AttributeError:
@@ -44,6 +81,24 @@ class Response(object):
 
     @property
     def meta(self):
+        """
+        Get the metadata from the request that generated this response.
+        从生成此响应的请求中获取元数据。
+
+        This property provides access to the meta dictionary of the request
+        that generated this response, allowing callback functions to access
+        metadata passed from the request.
+        此属性提供对生成此响应的请求的meta字典的访问，
+        允许回调函数访问从请求传递的元数据。
+
+        Returns:
+            dict: The request metadata dictionary.
+                 请求元数据字典。
+
+        Raises:
+            AttributeError: If this response is not tied to any request.
+                           如果此响应未与任何请求关联。
+        """
         try:
             return self.request.meta
         except AttributeError:
@@ -53,21 +108,75 @@ class Response(object):
             )
 
     def _get_url(self):
+        """
+        Get the response URL.
+        获取响应URL。
+
+        This is an internal method used by the url property.
+        这是由url属性使用的内部方法。
+
+        Returns:
+            str: The response URL.
+                响应URL。
+        """
         return self._url
 
     def _set_url(self, url):
+        """
+        Set the response URL.
+        设置响应URL。
+
+        This method validates that the URL is a string.
+        此方法验证URL是一个字符串。
+
+        Args:
+            url: The URL to set.
+                要设置的URL。
+
+        Raises:
+            TypeError: If the URL is not a string.
+                      如果URL不是字符串。
+        """
         if isinstance(url, str):
             self._url = url
         else:
             raise TypeError(f'{type(self).__name__} url must be str, '
                             f'got {type(url).__name__}')
 
+    # Property that uses the getter and setter methods
+    # 使用getter和setter方法的属性
     url = property(_get_url, _set_url)
 
     def _get_body(self):
+        """
+        Get the response body.
+        获取响应体。
+
+        This is an internal method used by the body property.
+        这是由body属性使用的内部方法。
+
+        Returns:
+            bytes: The response body.
+                 响应体。
+        """
         return self._body
 
     def _set_body(self, body):
+        """
+        Set the response body.
+        设置响应体。
+
+        This method validates that the body is bytes and converts None to an empty bytes object.
+        此方法验证body是字节对象，并将None转换为空字节对象。
+
+        Args:
+            body: The body to set.
+                 要设置的响应体。
+
+        Raises:
+            TypeError: If the body is not bytes.
+                      如果body不是字节对象。
+        """
         if body is None:
             self._body = b''
         elif not isinstance(body, bytes):
@@ -78,20 +187,49 @@ class Response(object):
         else:
             self._body = body
 
+    # Property that uses the getter and setter methods
+    # 使用getter和setter方法的属性
     body = property(_get_body, _set_body)
 
     def __str__(self):
+        """
+        Return a string representation of the response.
+        返回响应的字符串表示。
+
+        The string representation includes the HTTP status code and URL.
+        字符串表示包括HTTP状态码和URL。
+
+        Returns:
+            str: A string representation of the response.
+                响应的字符串表示。
+        """
         return f"<{self.status} {self.url}>"
 
+    # Use the same implementation for __repr__
+    # 对__repr__使用相同的实现
     __repr__ = __str__
 
     def copy(self):
-        """Return a copy of this Response"""
+        """
+        Return a copy of this Response.
+        返回此Response的副本。
+
+        Returns:
+            A copy of this Response. 此Response的副本。
+        """
         return self.replace()
 
     def replace(self, *args, **kwargs):
-        """Create a new Response with the same attributes except for those
-        given new values.
+        """
+        Create a new Response with the same attributes except for those given new values.
+        创建一个新的Response，除了给定的新值外，其他属性与当前Response相同。
+
+        Args:
+            *args: Positional arguments for the new Response. 新Response的位置参数。
+            **kwargs: Keyword arguments for the new Response. 新Response的关键字参数。
+
+        Returns:
+            A new Response object. 一个新的Response对象。
         """
         for x in [
             "url", "status", "headers", "body", "request", "flags"
@@ -101,32 +239,102 @@ class Response(object):
         return cls(*args, **kwargs)
 
     def urljoin(self, url):
-        """Join this Response's url with a possible relative url to form an
-        absolute interpretation of the latter."""
+        """
+        Join this Response's url with a possible relative url to form an absolute interpretation of the latter.
+        将此Response的url与可能的相对url连接，形成后者的绝对解释。
+
+        Args:
+            url: The URL to join. 要连接的URL。
+
+        Returns:
+            The absolute URL. 绝对URL。
+        """
         return urljoin(self.url, url)
 
     @property
     def text(self):
-        """For subclasses of TextResponse, this will return the body
-        as str
+        """
+        Get the response body as text.
+        将响应体作为文本获取。
+
+        This property is only implemented by subclasses of TextResponse.
+        In the base Response class, it raises an AttributeError.
+        此属性仅由TextResponse的子类实现。
+        在基本Response类中，它会引发AttributeError。
+
+        Returns:
+            str: The response body as text (in subclasses).
+                响应体作为文本（在子类中）。
+
+        Raises:
+            AttributeError: In the base Response class.
+                           在基本Response类中。
         """
         raise AttributeError("Response content isn't text")
 
     def css(self, *a, **kw):
-        """Shortcut method implemented only by responses whose content
-        is text (subclasses of TextResponse).
+        """
+        Apply the given CSS selector to this response's content.
+        将给定的CSS选择器应用于此响应的内容。
+
+        This method is only implemented by subclasses of TextResponse.
+        In the base Response class, it raises a NotSupported exception.
+        此方法仅由TextResponse的子类实现。
+        在基本Response类中，它会引发NotSupported异常。
+
+        Args:
+            *a: Positional arguments for the CSS selector.
+                CSS选择器的位置参数。
+            **kw: Keyword arguments for the CSS selector.
+                 CSS选择器的关键字参数。
+
+        Raises:
+            NotSupported: In the base Response class.
+                         在基本Response类中。
         """
         raise NotSupported("Response content isn't text")
 
     def xpath(self, *a, **kw):
-        """Shortcut method implemented only by responses whose content
-        is text (subclasses of TextResponse).
+        """
+        Apply the given XPath selector to this response's content.
+        将给定的XPath选择器应用于此响应的内容。
+
+        This method is only implemented by subclasses of TextResponse.
+        In the base Response class, it raises a NotSupported exception.
+        此方法仅由TextResponse的子类实现。
+        在基本Response类中，它会引发NotSupported异常。
+
+        Args:
+            *a: Positional arguments for the XPath selector.
+                XPath选择器的位置参数。
+            **kw: Keyword arguments for the XPath selector.
+                 XPath选择器的关键字参数。
+
+        Raises:
+            NotSupported: In the base Response class.
+                         在基本Response类中。
         """
         raise NotSupported("Response content isn't text")
 
     def json(self, *a, **kw):
-        """Shortcut method implemented only by responses whose content
-        is text (subclasses of TextResponse).
+        """
+        Parse this response's body as JSON.
+        将此响应的正文解析为JSON。
+
+        This method is only implemented by subclasses of TextResponse.
+        In the base Response class, it raises a NotSupported exception.
+        此方法仅由TextResponse的子类实现。
+        在基本Response类中，它会引发NotSupported异常。
+
+        Args:
+            *a: Positional arguments for the JSON parser.
+                JSON解析器的位置参数。
+            **kw: Keyword arguments for the JSON parser.
+                 JSON解析器的关键字参数。
+
+        Raises:
+            NotSupported: In the base Response class.
+                         在基本Response类中。
         """
         raise NotSupported("Response content isn't text")
 
@@ -135,17 +343,48 @@ class Response(object):
                fingerprint=None, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Request
         """
-        Return a :class:`~.Request` instance to follow a link ``url``.
-        It accepts the same arguments as ``Request.__init__`` method,
-        but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
-        not only an absolute URL.
+        Return a Request instance to follow a link.
+        返回一个Request实例以跟踪链接。
+
+        This method creates a new Request to follow the given URL. The URL can be
+        a relative URL, a Link object, or an absolute URL. If it's a relative URL,
+        it will be joined with the current response's URL.
+        此方法创建一个新的Request以跟踪给定的URL。URL可以是相对URL、Link对象或绝对URL。
+        如果是相对URL，它将与当前响应的URL连接。
 
-        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
-        method which supports selectors in addition to absolute/relative URLs
-        and Link objects.
+        Args:
+            url: The URL to follow. Can be a string or a Link object.
+                要跟踪的URL。可以是字符串或Link对象。
+            callback: A function to be called with the response from the request.
+                     使用请求的响应调用的函数。
+            method: The HTTP method to use.
+                   要使用的HTTP方法。
+            headers: The headers to use for the request.
+                    请求使用的头部。
+            body: The body of the request.
+                 请求的正文。
+            cookies: The cookies to send with the request.
+                    与请求一起发送的Cookie。
+            meta: Extra data to pass to the request.
+                 传递给请求的额外数据。
+            encoding: The encoding to use for the request.
+                     请求使用的编码。
+            priority: The priority of the request.
+                     请求的优先级。
+            dont_filter: Whether to filter duplicate requests.
+                        是否过滤重复请求。
+            fingerprint: The fingerprint for the request.
+                        请求的指纹。
+            errback: A function to be called if the request fails.
+                    如果请求失败时调用的函数。
+            cb_kwargs: Additional keyword arguments to pass to the callback.
+                      传递给回调的额外关键字参数。
+            flags: Flags for the request.
+                  请求的标志。
 
-        .. versionadded:: 2.0
-           The *flags* parameter.
+        Returns:
+            Request: A new Request instance.
+                    一个新的Request实例。
         """
         if isinstance(url, Link):
             url = url.url
@@ -175,16 +414,50 @@ class Response(object):
                    dont_filter=False, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Generator[Request, None, None]
         """
-        .. versionadded:: 2.0
+        Return an iterable of Request instances to follow all links in urls.
+        返回一个Request实例的可迭代对象，以跟踪urls中的所有链接。
+
+        This method creates multiple Requests to follow the given URLs. Each URL can be
+        a relative URL, a Link object, or an absolute URL. If it's a relative URL,
+        it will be joined with the current response's URL.
+        此方法创建多个Request以跟踪给定的URL。每个URL可以是相对URL、Link对象或绝对URL。
+        如果是相对URL，它将与当前响应的URL连接。
+
+        Args:
+            urls: An iterable of URLs to follow. Each can be a string or a Link object.
+                 要跟踪的URL的可迭代对象。每个可以是字符串或Link对象。
+            callback: A function to be called with the response from each request.
+                     使用每个请求的响应调用的函数。
+            method: The HTTP method to use.
+                   要使用的HTTP方法。
+            headers: The headers to use for the requests.
+                    请求使用的头部。
+            body: The body of the requests.
+                 请求的正文。
+            cookies: The cookies to send with the requests.
+                    与请求一起发送的Cookie。
+            meta: Extra data to pass to the requests.
+                 传递给请求的额外数据。
+            encoding: The encoding to use for the requests.
+                     请求使用的编码。
+            priority: The priority of the requests.
+                     请求的优先级。
+            dont_filter: Whether to filter duplicate requests.
+                        是否过滤重复请求。
+            errback: A function to be called if the requests fail.
+                    如果请求失败时调用的函数。
+            cb_kwargs: Additional keyword arguments to pass to the callback.
+                      传递给回调的额外关键字参数。
+            flags: Flags for the requests.
+                  请求的标志。
 
-        Return an iterable of :class:`~.Request` instances to follow all links
-        in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
-        but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects,
-        not only absolute URLs.
+        Returns:
+            Generator[Request, None, None]: A generator of Request instances.
+                                           Request实例的生成器。
 
-        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`
-        method which supports selectors in addition to absolute/relative URLs
-        and Link objects.
+        Raises:
+            TypeError: If urls is not an iterable.
+                      如果urls不是可迭代的。
         """
         if not hasattr(urls, '__iter__'):
             raise TypeError("'urls' argument must be an iterable")

aioscrapy/http/response/html.py

@@ -1,12 +1,51 @@
+
 """
-This module implements the HtmlResponse class which adds encoding
-discovering through HTML encoding declarations to the TextResponse class.
+HTML response implementation for aioscrapy.
+aioscrapy的HTML响应实现。
 
-See documentation in docs/topics/request-response.rst
+This module provides the HtmlResponse class, which is a specialized TextResponse
+for handling HTML content. It inherits all functionality from TextResponse
+but is specifically intended for HTML responses.
+此模块提供了HtmlResponse类，这是一个专门用于处理HTML内容的TextResponse。
+它继承了TextResponse的所有功能，但专门用于HTML响应。
 """
 
 from aioscrapy.http.response.text import TextResponse
 
 
 class HtmlResponse(TextResponse):
+    """
+    A Response subclass specifically for HTML responses.
+    专门用于HTML响应的Response子类。
+
+    This class extends TextResponse to handle HTML content. It inherits all the
+    functionality of TextResponse, including:
+    此类扩展了TextResponse以处理HTML内容。它继承了TextResponse的所有功能，包括：
+
+    - Automatic encoding detection
+      自动编码检测
+    - Unicode conversion
+      Unicode转换
+    - CSS and XPath selectors
+      CSS和XPath选择器
+    - JSON parsing
+      JSON解析
+    - Enhanced link following
+      增强的链接跟踪
+
+    The main purpose of this class is to provide a specific type for HTML responses,
+    which can be useful for type checking and middleware processing.
+    此类的主要目的是为HTML响应提供特定类型，这对类型检查和中间件处理很有用。
+
+    Example:
+        ```python
+        def parse(self, response):
+            if isinstance(response, HtmlResponse):
+                # Process HTML response
+                title = response.css('title::text').get()
+            else:
+                # Handle other response types
+                pass
+        ```
+    """
     pass