aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/queue/__init__.py

@@ -7,7 +7,15 @@ from aioscrapy.utils.reqser import request_from_dict
 
 
 class AbsQueue(metaclass=ABCMeta):
-    """Per-spider base queue class"""
+    """
+    Per-spider base queue class.
+    每个爬虫的基础队列类。
+
+    This abstract class defines the interface for request queues used by spiders.
+    It provides methods for pushing, popping, and managing requests in a queue.
+    此抽象类定义了爬虫使用的请求队列的接口。
+    它提供了推送、弹出和管理队列中请求的方法。
+    """
 
     def __init__(
             self,
@@ -16,52 +24,196 @@ class AbsQueue(metaclass=ABCMeta):
             key: Optional[str] = None,
             serializer: Optional[AbsSerializer] = None
     ) -> None:
-        """Initialize per-spider queue"""
-        self.container = container
-        self.spider = spider
-        self.key = key
-        self.serializer = serializer
+        """
+        Initialize per-spider queue.
+        初始化每个爬虫的队列。
+
+        Args:
+            container: The underlying data structure to store the queue.
+                      存储队列的底层数据结构。
+            spider: The spider instance that will use this queue.
+                   将使用此队列的爬虫实例。
+            key: Optional key to identify this queue.
+                可选的键，用于标识此队列。
+            serializer: Optional serializer for encoding/decoding requests.
+                       可选的序列化器，用于编码/解码请求。
+        """
+        self.container = container  # The underlying data structure
+                                   # 底层数据结构
+        self.spider = spider  # Associated spider
+                             # 关联的爬虫
+        self.key = key  # Queue identifier
+                        # 队列标识符
+        self.serializer = serializer  # For serializing requests
+                                     # 用于序列化请求
 
     @property
     @abstractmethod
     def inc_key(self) -> str:
-        """stats inc_value"""
+        """
+        Get the key used for incrementing stats.
+        获取用于增加统计信息的键。
+
+        This property should return a string key that will be used with
+        the stats collector's inc_value method to track queue operations.
+        此属性应返回一个字符串键，该键将与统计收集器的inc_value方法一起使用，
+        以跟踪队列操作。
+
+        Returns:
+            str: The stats key for this queue.
+                此队列的统计键。
+        """
 
     @classmethod
     @abstractmethod
     async def from_spider(cls, spider: aioscrapy.Spider) -> "AbsQueue":
-        """get queue instance from spider"""
+        """
+        Create a queue instance for a spider.
+        为爬虫创建队列实例。
+
+        This factory method creates a new queue instance configured
+        for the given spider.
+        此工厂方法创建一个为给定爬虫配置的新队列实例。
+
+        Args:
+            spider: The spider that will use the queue.
+                   将使用队列的爬虫。
+
+        Returns:
+            AbsQueue: A new queue instance.
+                     一个新的队列实例。
+        """
 
     def _encode_request(self, request: aioscrapy.Request) -> Any:
-        """Encode a request object"""
+        """
+        Encode a request object for storage.
+        编码请求对象以进行存储。
+
+        This method converts a Request object to a serialized form that can
+        be stored in the queue's container.
+        此方法将Request对象转换为可以存储在队列容器中的序列化形式。
+
+        Args:
+            request: The Request object to encode.
+                    要编码的Request对象。
+
+        Returns:
+            Any: The serialized form of the request.
+                请求的序列化形式。
+        """
         obj = request.to_dict(spider=self.spider)
         return self.serializer.dumps(obj)
 
     async def _decode_request(self, encoded_request: Any) -> aioscrapy.Request:
-        """Decode an request previously encoded"""
+        """
+        Decode a previously encoded request.
+        解码先前编码的请求。
+
+        This method converts a serialized request back into a Request object.
+        此方法将序列化的请求转换回Request对象。
+
+        Args:
+            encoded_request: The serialized request to decode.
+                           要解码的序列化请求。
+
+        Returns:
+            Request: The reconstructed Request object.
+                    重建的Request对象。
+        """
         obj = self.serializer.loads(encoded_request)
         return await request_from_dict(obj, spider=self.spider)
 
     def __len__(self) -> None:
-        """Return the length of the queue"""
+        """
+        Return the length of the queue (synchronous version).
+        返回队列的长度（同步版本）。
+
+        This method is overridden to prevent synchronous access to the queue length.
+        Use the async len() method instead.
+        此方法被重写以防止同步访问队列长度。
+        请改用异步len()方法。
+
+        Raises:
+            Exception: Always raises an exception to remind users to use the async len() method.
+                      始终引发异常，以提醒用户使用异步len()方法。
+        """
         raise Exception('please use len()')
 
     @abstractmethod
     async def len(self) -> int:
-        """Return the length of the queue"""
+        """
+        Return the length of the queue (asynchronous version).
+        返回队列的长度（异步版本）。
+
+        This method should return the number of requests currently in the queue.
+        此方法应返回当前队列中的请求数量。
+
+        Returns:
+            int: The number of requests in the queue.
+                队列中的请求数量。
+        """
 
     @abstractmethod
     async def push(self, request: aioscrapy.Request) -> None:
-        """Push a request"""
+        """
+        Push a request to the queue.
+        将请求推送到队列。
+
+        This method adds a single request to the queue.
+        此方法将单个请求添加到队列中。
+
+        Args:
+            request: The request to add to the queue.
+                    要添加到队列的请求。
+        """
 
     @abstractmethod
     async def push_batch(self, requests: List[aioscrapy.Request]) -> None:
-        """Push a batch requests"""
+        """
+        Push multiple requests to the queue.
+        将多个请求推送到队列。
+
+        This method adds multiple requests to the queue at once,
+        which may be more efficient than calling push() multiple times.
+        此方法一次将多个请求添加到队列中，
+        这可能比多次调用push()更有效率。
+
+        Args:
+            requests: A list of requests to add to the queue.
+                     要添加到队列的请求列表。
+        """
 
     @abstractmethod
     async def pop(self, timeout: int = 0) -> Optional[aioscrapy.Request]:
-        """Pop a request"""
+        """
+        Pop a request from the queue.
+        从队列中弹出请求。
+
+        This method removes and returns a request from the queue.
+        If the queue is empty, it may wait up to timeout seconds
+        before returning None.
+        此方法从队列中移除并返回一个请求。
+        如果队列为空，它可能会等待最多timeout秒，
+        然后返回None。
+
+        Args:
+            timeout: Maximum time to wait for a request, in seconds.
+                    等待请求的最长时间，以秒为单位。
+
+        Returns:
+            Optional[Request]: The next request from the queue, or None if
+                              the queue is empty or the timeout expires.
+                              队列中的下一个请求，如果队列为空或超时，则为None。
+        """
 
     @abstractmethod
     async def clear(self) -> None:
-        """Clear queue/stack"""
+        """
+        Clear all requests from the queue.
+        清除队列中的所有请求。
+
+        This method removes all pending requests from the queue,
+        effectively resetting it to an empty state.
+        此方法从队列中删除所有待处理的请求，
+        有效地将其重置为空状态。
+        """

aioscrapy/scrapyd/runner.py

@@ -1,4 +1,28 @@
 
+"""
+Scrapyd Runner Module
+Scrapyd运行器模块
+
+This module provides utilities for running AioScrapy spiders from egg files deployed
+with Scrapyd. It handles the activation of egg files, setting up the project environment,
+and launching the spider.
+此模块提供了从使用Scrapyd部署的egg文件运行AioScrapy爬虫的实用程序。它处理egg文件的激活、
+设置项目环境和启动爬虫。
+
+The main components are:
+主要组件包括：
+
+1. activate_egg: Activates a Scrapy egg file and sets up the environment
+                激活Scrapy egg文件并设置环境
+2. project_environment: Context manager that sets up the project environment
+                       设置项目环境的上下文管理器
+3. main: Entry point for running spiders from Scrapyd
+        从Scrapyd运行爬虫的入口点
+
+This module is designed to be used by Scrapyd to run AioScrapy spiders, but it can
+also be used directly to run spiders from egg files.
+此模块设计用于Scrapyd运行AioScrapy爬虫，但也可以直接用于从egg文件运行爬虫。
+"""
 import os
 import shutil
 import sys
@@ -15,9 +39,28 @@ except ImportError:
 
 
 def activate_egg(eggpath):
-    """Activate a Scrapy egg file. This is meant to be used from egg runners
-    to activate a Scrapy egg file. Don't use it from other code as it may
-    leave unwanted side effects.
+    """
+    Activate a Scrapy egg file.
+    激活aioscrapy egg文件。
+
+    This function activates a aioscrapy egg file by adding it to the Python path
+    and setting the AIOSCRAPY_SETTINGS_MODULE environment variable to the
+    settings module specified in the egg's entry points.
+    此函数通过将aioscrapy egg文件添加到Python路径并将AIOSCRAPY_SETTINGS_MODULE
+    环境变量设置为egg入口点中指定的设置模块来激活它。
+
+    This is meant to be used from egg runners to activate a Scrapy egg file.
+    Don't use it from other code as it may leave unwanted side effects.
+    这旨在从egg运行器使用，以激活Scrapy egg文件。不要从其他代码中使用它，
+    因为它可能会留下不必要的副作用。
+
+    Args:
+        eggpath: Path to the egg file to activate.
+                要激活的egg文件的路径。
+
+    Raises:
+        ValueError: If the egg file is unknown or corrupt.
+                   如果egg文件未知或损坏。
     """
     try:
         d = next(pkg_resources.find_distributions(eggpath))
@@ -30,31 +73,109 @@ def activate_egg(eggpath):
 
 @contextmanager
 def project_environment(project):
+    """
+    Set up the environment for a aioscrapy project.
+    为aioscrapy项目设置环境。
+
+    This context manager sets up the environment for a aioscrapy project by:
+    此上下文管理器通过以下方式为aioscrapy项目设置环境：
+
+    1. Retrieving the egg file for the project from aioscrapyd's egg storage
+       从aioscrapyd的egg存储中检索项目的egg文件
+    2. Creating a temporary copy of the egg file
+       创建egg文件的临时副本
+    3. Activating the egg file
+       激活egg文件
+    4. Cleaning up the temporary egg file when done
+       完成后清理临时egg文件
+
+    Args:
+        project: The name of the project to set up the environment for.
+                要为其设置环境的项目名称。
+
+    Yields:
+        None: This context manager doesn't yield a value, but sets up the
+             environment for the code inside the with block.
+             此上下文管理器不产生值，但为with块内的代码设置环境。
+
+    Raises:
+        AssertionError: If aioscrapy settings are already loaded.
+                       如果aioscrapy设置已加载。
+    """
+    # Get the Scrapyd application and egg storage
+    # 获取Scrapyd应用程序和egg存储
     app = get_application()
     eggstorage = app.getComponent(IEggStorage)
+
+    # Get the egg version from environment or use the latest
+    # 从环境获取egg版本或使用最新版本
     eggversion = os.environ.get('AIOSCRAPY_EGG_VERSION', None)
+
+    # Get the egg file from storage
+    # 从存储中获取egg文件
     version, eggfile = eggstorage.get(project, eggversion)
+
     if eggfile:
+        # Create a temporary copy of the egg file
+        # 创建egg文件的临时副本
         prefix = '%s-%s-' % (project, version)
         fd, eggpath = tempfile.mkstemp(prefix=prefix, suffix='.egg')
         lf = os.fdopen(fd, 'wb')
         shutil.copyfileobj(eggfile, lf)
         lf.close()
+
+        # Activate the egg file
+        # 激活egg文件
         activate_egg(eggpath)
     else:
         eggpath = None
+
     try:
+        # Ensure settings aren't already loaded
+        # 确保设置尚未加载
         assert 'aioscrapy.conf' not in sys.modules, "aioscrapy settings already loaded"
         yield
     finally:
+        # Clean up the temporary egg file
+        # 清理临时egg文件
         if eggpath:
             os.remove(eggpath)
 
 
 def main():
+    """
+    Main entry point for running spiders from Scrapyd.
+    从Scrapyd运行爬虫的主入口点。
+
+    This function:
+    此函数：
+
+    1. Updates environment variables by converting SCRAPY_* variables to AIO* variables
+       通过将SCRAPY_*变量转换为AIO*变量来更新环境变量
+    2. Gets the project name from the AIOSCRAPY_PROJECT environment variable
+       从AIOSCRAPY_PROJECT环境变量获取项目名称
+    3. Sets up the project environment using the project_environment context manager
+       使用project_environment上下文管理器设置项目环境
+    4. Imports and executes the aioscrapy.cmdline.execute function to run the spider
+       导入并执行aioscrapy.cmdline.execute函数来运行爬虫
+
+    This function is designed to be called by Scrapyd to run AioScrapy spiders.
+    此函数设计用于Scrapyd调用以运行AioScrapy爬虫。
+
+    Raises:
+        KeyError: If the AIOSCRAPY_PROJECT environment variable is not set.
+                 如果未设置AIOSCRAPY_PROJECT环境变量。
+    """
+    # Update environment variables by converting SCRAPY_* to AIO*
+    # 通过将SCRAPY_*转换为AIO*来更新环境变量
     os.environ.update({f'AIO{k}': v for k, v in os.environ.items() if k.startswith('SCRAPY_')})
 
+    # Get the project name from environment
+    # 从环境获取项目名称
     project = os.environ['AIOSCRAPY_PROJECT']
+
+    # Set up the project environment and run the spider
+    # 设置项目环境并运行爬虫
     with project_environment(project):
         from aioscrapy.cmdline import execute
         execute()

aioscrapy/serializer.py

@@ -1,3 +1,14 @@
+"""
+Serialization utilities for AioScrapy.
+AioScrapy的序列化实用工具。
+
+This module provides serializer classes for converting Python objects to and from
+serialized formats like JSON and Pickle. These serializers are used throughout
+AioScrapy for data persistence, message passing, and caching.
+此模块提供了用于将Python对象转换为序列化格式（如JSON和Pickle）以及从这些格式转换回来的
+序列化器类。这些序列化器在AioScrapy中用于数据持久化、消息传递和缓存。
+"""
+
 import ujson
 import pickle
 from abc import ABCMeta, abstractmethod
@@ -6,34 +17,203 @@ __all__ = ['PickleSerializer', 'JsonSerializer', 'AbsSerializer']
 
 
 class AbsSerializer(object, metaclass=ABCMeta):
+    """
+    Abstract base class for serializers.
+    序列化器的抽象基类。
+
+    This class defines the interface that all serializers must implement.
+    It provides methods for serializing Python objects to a string format
+    and deserializing strings back to Python objects.
+    此类定义了所有序列化器必须实现的接口。
+    它提供了将Python对象序列化为字符串格式以及将字符串反序列化回Python对象的方法。
+    """
 
     @staticmethod
     @abstractmethod
     def loads(s):
-        """Serializer object"""
+        """
+        Deserialize a string to a Python object.
+        将字符串反序列化为Python对象。
+
+        This method takes a serialized string and converts it back to
+        a Python object.
+        此方法接受一个序列化的字符串并将其转换回Python对象。
+
+        Args:
+            s: The serialized string to deserialize.
+               要反序列化的序列化字符串。
+
+        Returns:
+            The deserialized Python object.
+            反序列化的Python对象。
+
+        Raises:
+            Depends on the implementation.
+            取决于实现。
+        """
+        pass
 
     @staticmethod
     @abstractmethod
     def dumps(obj):
-        """Serializer object"""
+        """
+        Serialize a Python object to a string.
+        将Python对象序列化为字符串。
+
+        This method takes a Python object and converts it to a serialized
+        string format.
+        此方法接受一个Python对象并将其转换为序列化的字符串格式。
+
+        Args:
+            obj: The Python object to serialize.
+                 要序列化的Python对象。
+
+        Returns:
+            The serialized string representation of the object.
+            对象的序列化字符串表示。
+
+        Raises:
+            Depends on the implementation.
+            取决于实现。
+        """
+        pass
 
 
 class PickleSerializer(AbsSerializer):
+    """
+    Serializer that uses Python's pickle module.
+    使用Python的pickle模块的序列化器。
+
+    This serializer uses Python's built-in pickle module to serialize and
+    deserialize Python objects. Pickle can handle a wide range of Python
+    objects, including custom classes, but the resulting serialized data
+    is not human-readable and may not be compatible across different
+    Python versions.
+    此序列化器使用Python内置的pickle模块来序列化和反序列化Python对象。
+    Pickle可以处理各种Python对象，包括自定义类，但生成的序列化数据
+    不是人类可读的，并且可能在不同的Python版本之间不兼容。
+
+    Warning:
+        Pickle is not secure against maliciously constructed data. Never unpickle
+        data received from untrusted or unauthenticated sources.
+        Pickle对恶意构造的数据不安全。切勿对来自不受信任或未经身份验证的
+        来源的数据进行反序列化。
+    """
+
     @staticmethod
     def loads(s):
+        """
+        Deserialize a pickle-encoded string to a Python object.
+        将pickle编码的字符串反序列化为Python对象。
+
+        Args:
+            s: The pickle-encoded string to deserialize.
+               要反序列化的pickle编码字符串。
+
+        Returns:
+            The deserialized Python object.
+            反序列化的Python对象。
+
+        Raises:
+            pickle.UnpicklingError: If the data cannot be unpickled.
+                                   如果数据无法被反序列化。
+            ValueError: If the pickle data is truncated.
+                       如果pickle数据被截断。
+            TypeError: If the serialized data is not a bytes-like object.
+                      如果序列化数据不是类字节对象。
+        """
         return pickle.loads(s)
 
     @staticmethod
     def dumps(obj):
+        """
+        Serialize a Python object to a pickle-encoded string.
+        将Python对象序列化为pickle编码的字符串。
+
+        Args:
+            obj: The Python object to serialize.
+                要序列化的Python对象。
+
+        Returns:
+            bytes: The pickle-encoded representation of the object.
+                  对象的pickle编码表示。
+
+        Raises:
+            pickle.PicklingError: If the object cannot be pickled.
+                                 如果对象无法被序列化。
+
+        Note:
+            Uses the highest available pickle protocol for maximum efficiency.
+            使用最高可用的pickle协议以获得最大效率。
+        """
+        # protocol=-1 means use the highest available protocol
+        # protocol=-1表示使用最高可用的协议
         return pickle.dumps(obj, protocol=-1)
 
 
 class JsonSerializer(AbsSerializer):
+    """
+    Serializer that uses the ujson module.
+    使用ujson模块的序列化器。
+
+    This serializer uses the ujson module (UltraJSON) to serialize and
+    deserialize Python objects to and from JSON format. UltraJSON is a fast
+    JSON encoder and decoder written in C with Python bindings.
+    此序列化器使用ujson模块（UltraJSON）将Python对象序列化为JSON格式
+    以及从JSON格式反序列化。UltraJSON是一个用C编写的快速JSON编码器和
+    解码器，带有Python绑定。
+
+    JSON serialization is more limited than pickle in terms of the types it can
+    handle (primarily: dict, list, str, int, float, bool, None), but it produces
+    human-readable output and is safe to use with untrusted data.
+    JSON序列化在可以处理的类型方面比pickle更有限（主要是：dict、list、str、
+    int、float、bool、None），但它产生人类可读的输出，并且可以安全地
+    用于不受信任的数据。
+    """
+
     @staticmethod
     def loads(s):
+        """
+        Deserialize a JSON string to a Python object.
+        将JSON字符串反序列化为Python对象。
+
+        Args:
+            s: The JSON string to deserialize.
+               要反序列化的JSON字符串。
+
+        Returns:
+            The deserialized Python object (typically a dict, list, or primitive type).
+            反序列化的Python对象（通常是dict、list或原始类型）。
+
+        Raises:
+            ValueError: If the string is not valid JSON.
+                       如果字符串不是有效的JSON。
+        """
         return ujson.loads(s)
 
     @staticmethod
     def dumps(obj):
+        """
+        Serialize a Python object to a JSON string.
+        将Python对象序列化为JSON字符串。
+
+        Args:
+            obj: The Python object to serialize.
+                要序列化的Python对象。
+                Must be a type that can be represented in JSON (dict, list, str,
+                int, float, bool, None, or a combination of these).
+                必须是可以在JSON中表示的类型（dict、list、str、int、float、
+                bool、None或这些的组合）。
+
+        Returns:
+            str: The JSON string representation of the object.
+                 对象的JSON字符串表示。
+
+        Raises:
+            TypeError: If the object contains types that cannot be serialized to JSON.
+                      如果对象包含无法序列化为JSON的类型。
+            OverflowError: If an integer is too large to be represented in JSON.
+                          如果整数太大而无法在JSON中表示。
+        """
         return ujson.dumps(obj)