aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/utils/httpobj.py

@@ -1,4 +1,13 @@
-"""Helper functions for scrapy.http objects (Request, Response)"""
+"""
+HTTP object utility functions for aioscrapy.
+aioscrapy的HTTP对象实用函数。
+
+This module provides utility functions for working with HTTP objects (Request, Response)
+in aioscrapy. It includes functions for parsing and caching URL information to improve
+performance when the same URLs are processed multiple times.
+此模块提供了用于处理aioscrapy中HTTP对象（Request, Response）的实用函数。
+它包括用于解析和缓存URL信息的函数，以提高多次处理相同URL时的性能。
+"""
 
 from typing import Union
 from urllib.parse import urlparse, ParseResult
@@ -7,13 +16,56 @@ from weakref import WeakKeyDictionary
 from aioscrapy.http import Request, Response
 
 
+# Cache for storing parsed URLs to avoid repeated parsing of the same URL
+# Uses WeakKeyDictionary so entries are automatically removed when the Request/Response is garbage collected
+# 用于存储已解析URL的缓存，以避免重复解析相同的URL
+# 使用WeakKeyDictionary，因此当Request/Response被垃圾回收时，条目会自动删除
 _urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary()
 
 
 def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult:
-    """Return urlparse.urlparse caching the result, where the argument can be a
-    Request or Response object
     """
+    Parse the URL of a Request or Response object with caching.
+    解析Request或Response对象的URL，并进行缓存。
+
+    This function parses the URL of the given Request or Response object using
+    urllib.parse.urlparse and caches the result. If the same object is passed
+    again, the cached result is returned instead of re-parsing the URL.
+    此函数使用urllib.parse.urlparse解析给定Request或Response对象的URL，
+    并缓存结果。如果再次传递相同的对象，则返回缓存的结果，而不是重新解析URL。
+
+    The caching mechanism uses a WeakKeyDictionary, so the cache entries are
+    automatically removed when the Request or Response objects are garbage collected.
+    This prevents memory leaks while still providing performance benefits.
+    缓存机制使用WeakKeyDictionary，因此当Request或Response对象被垃圾回收时，
+    缓存条目会自动删除。这可以防止内存泄漏，同时仍然提供性能优势。
+
+    Args:
+        request_or_response: A Request or Response object whose URL will be parsed.
+                            将解析其URL的Request或Response对象。
+
+    Returns:
+        ParseResult: The parsed URL components (scheme, netloc, path, params,
+                    query, fragment).
+                    解析的URL组件（scheme, netloc, path, params, query, fragment）。
+
+    Example:
+        >>> request = Request('https://example.com/path?query=value')
+        >>> parsed = urlparse_cached(request)
+        >>> parsed.netloc
+        'example.com'
+        >>> parsed.path
+        '/path'
+        >>> parsed.query
+        'query=value'
+    """
+    # Check if this object's URL has already been parsed and cached
+    # 检查此对象的URL是否已被解析和缓存
     if request_or_response not in _urlparse_cache:
+        # If not in cache, parse the URL and store the result
+        # 如果不在缓存中，解析URL并存储结果
         _urlparse_cache[request_or_response] = urlparse(request_or_response.url)
+
+    # Return the cached parse result
+    # 返回缓存的解析结果
     return _urlparse_cache[request_or_response]

aioscrapy/utils/log.py

@@ -1,3 +1,13 @@
+"""
+Logging utilities for aioscrapy.
+aioscrapy的日志工具。
+
+This module provides logging functionality for aioscrapy using the loguru library.
+It configures logging based on settings and provides a spider-aware logger.
+此模块使用loguru库为aioscrapy提供日志功能。
+它根据设置配置日志记录，并提供一个感知爬虫的日志记录器。
+"""
+
 import asyncio
 import sys
 import warnings
@@ -7,21 +17,46 @@ from loguru import logger as _logger
 
 from aioscrapy.settings import Settings
 
+# Remove the default stderr handler to avoid duplicate logs
+# 移除默认的stderr处理程序以避免重复日志
 for _handler in _logger._core.handlers.values():
     if _handler._name == '<stderr>':
         _logger.remove(_handler._id)
 
 
 def configure_logging(spider: Type["Spider"], settings: Settings):
+    """
+    Configure logging for a spider based on settings.
+    根据设置为爬虫配置日志记录。
+
+    This function sets up logging handlers for a specific spider based on the provided settings.
+    It can configure logging to stderr and/or to a file, with various options like log level,
+    rotation, retention, etc.
+    此函数根据提供的设置为特定爬虫设置日志处理程序。
+    它可以配置日志记录到stderr和/或文件，具有各种选项，如日志级别、轮换、保留等。
+
+    Args:
+        spider: The spider instance for which to configure logging.
+               要为其配置日志记录的爬虫实例。
+        settings: The settings object containing logging configuration.
+                 包含日志配置的设置对象。
+    """
+    # Get logging configuration from settings
+    # 从设置中获取日志配置
     formatter = settings.get('LOG_FORMAT')
     level = settings.get('LOG_LEVEL', 'INFO')
     enqueue = settings.get('ENQUEUE', True)
+
+    # Configure stderr logging if enabled
+    # 如果启用，配置stderr日志记录
     if settings.get('LOG_STDOUT', True):
         _logger.add(
             sys.stderr, format=formatter, level=level, enqueue=enqueue,
             filter=lambda record: record["extra"].get("spidername") == spider.name,
         )
 
+    # Configure file logging if a filename is provided
+    # 如果提供了文件名，配置文件日志记录
     if filename := settings.get('LOG_FILE'):
         rotation = settings.get('LOG_ROTATION', '20MB')
         retention = settings.get('LOG_RETENTION', 10)
@@ -34,18 +69,62 @@ def configure_logging(spider: Type["Spider"], settings: Settings):
 
 
 class AioScrapyLogger:
+    """
+    Spider-aware logger for aioscrapy.
+    aioscrapy的爬虫感知日志记录器。
+
+    This class provides a wrapper around the loguru logger that automatically
+    binds the current spider name to log records. This allows for filtering
+    logs by spider name and provides context about which spider generated each log.
+    此类提供了loguru日志记录器的包装器，它自动将当前爬虫名称绑定到日志记录。
+    这允许按爬虫名称过滤日志，并提供关于哪个爬虫生成了每条日志的上下文。
+
+    The logger methods (debug, info, warning, etc.) are dynamically accessed
+    through __getattr__, so they're not explicitly defined.
+    日志记录器方法（debug、info、warning等）是通过__getattr__动态访问的，
+    因此它们没有明确定义。
+    """
     __slots__ = (
         'catch', 'complete', 'critical', 'debug', 'error', 'exception',
         'info', 'log', 'patch', 'success', 'trace', 'warning'
     )
 
     def __getattr__(self, method):
+        """
+        Dynamically access logger methods with spider name binding.
+        动态访问带有爬虫名称绑定的日志记录器方法。
+
+        This method intercepts attribute access to provide logger methods that
+        automatically include the current spider name in the log context.
+        此方法拦截属性访问，以提供自动在日志上下文中包含当前爬虫名称的日志记录器方法。
+
+        Args:
+            method: The name of the logger method to access (e.g., 'info', 'debug').
+                   要访问的日志记录器方法的名称（例如，'info'、'debug'）。
+
+        Returns:
+            The requested logger method, bound with the current spider name.
+            请求的日志记录器方法，绑定了当前爬虫名称。
+
+        Note:
+            If the current task name cannot be determined, it falls back to the
+            original logger method without binding a spider name.
+            如果无法确定当前任务名称，它会回退到原始日志记录器方法，而不绑定爬虫名称。
+        """
         try:
+            # Get the current task name as the spider name
+            # 获取当前任务名称作为爬虫名称
             spider_name = asyncio.current_task().get_name()
+            # Return the logger method bound with the spider name
+            # 返回绑定了爬虫名称的日志记录器方法
             return getattr(_logger.bind(spidername=spider_name), method)
         except Exception as e:
+            # Fall back to the original logger method if binding fails
+            # 如果绑定失败，回退到原始日志记录器方法
             warnings.warn(f'Error on get logger: {e}')
             return getattr(_logger, method)
 
 
+# Create a singleton instance of the spider-aware logger
+# 创建爬虫感知日志记录器的单例实例
 logger = AioScrapyLogger()

aioscrapy/utils/misc.py

@@ -1,4 +1,12 @@
-"""Helper functions which don't fit anywhere else"""
+"""
+Miscellaneous utility functions for aioscrapy.
+aioscrapy的杂项实用函数。
+
+This module contains helper functions that don't fit into other utility categories.
+It provides functionality for module walking, object loading, and instance creation.
+此模块包含不适合其他实用程序类别的辅助函数。
+它提供了模块遍历、对象加载和实例创建的功能。
+"""
 
 from importlib import import_module
 from pkgutil import iter_modules
@@ -7,41 +15,112 @@ from aioscrapy.utils.tools import call_helper
 
 
 def walk_modules(path):
-    """Loads a module and all its submodules from the given module path and
-    returns them. If *any* module throws an exception while importing, that
-    exception is thrown back.
-
-    For example: walk_modules('aioscrapy.utils')
     """
+    Load a module and all its submodules recursively.
+    递归加载模块及其所有子模块。
+
+    This function imports a module and all its submodules from the given module path
+    and returns them as a list. It performs a recursive traversal of the module tree.
+    If any module raises an exception during import, that exception is propagated.
+    此函数从给定的模块路径导入模块及其所有子模块，并将它们作为列表返回。
+    它执行模块树的递归遍历。如果任何模块在导入过程中引发异常，则该异常会被传播。
+
+    Args:
+        path: The module path to load (e.g., 'aioscrapy.utils').
+              要加载的模块路径（例如，'aioscrapy.utils'）。
 
+    Returns:
+        list: A list of imported modules, including the root module and all submodules.
+              导入的模块列表，包括根模块和所有子模块。
+
+    Raises:
+        ImportError: If any module cannot be imported.
+                    如果任何模块无法导入。
+
+    Example:
+        >>> mods = walk_modules('aioscrapy.utils')
+        >>> 'aioscrapy.utils.url' in [mod.__name__ for mod in mods]
+        True
+    """
+    # Initialize the list of modules
+    # 初始化模块列表
     mods = []
+
+    # Import the root module
+    # 导入根模块
     mod = import_module(path)
     mods.append(mod)
+
+    # If the module is a package (has a __path__), process its submodules
+    # 如果模块是一个包（有__path__），处理其子模块
     if hasattr(mod, '__path__'):
         for _, subpath, ispkg in iter_modules(mod.__path__):
+            # Construct the full path for the submodule
+            # 构造子模块的完整路径
             fullpath = path + '.' + subpath
+
+            # If the submodule is a package, recursively walk it
+            # 如果子模块是一个包，递归遍历它
             if ispkg:
                 mods += walk_modules(fullpath)
+            # Otherwise, import the submodule and add it to the list
+            # 否则，导入子模块并将其添加到列表中
             else:
                 submod = import_module(fullpath)
                 mods.append(submod)
+
     return mods
 
 
 def load_object(path: str):
-    """Load an object given its absolute object path, and return it.
+    """
+    Load an object by its fully qualified name.
+    通过完全限定名称加载对象。
+
+    This function imports a module and retrieves an object from it based on the
+    provided import path. The object can be a class, function, variable, or an
+    instance defined in the specified module.
+    此函数根据提供的导入路径导入模块并从中检索对象。
+    对象可以是在指定模块中定义的类、函数、变量或实例。
+
+    Args:
+        path: The absolute object path (e.g., 'aioscrapy.libs.downloader.redirect.RedirectMiddleware').
+              对象的绝对路径（例如，'aioscrapy.libs.downloader.redirect.RedirectMiddleware'）。
 
-    The object can be the import path of a class, function, variable or an
-    instance, e.g. 'aioscrapy.libs.downloader.redirect.RedirectMiddleware'..
+    Returns:
+        The loaded object.
+        加载的对象。
+
+    Raises:
+        ValueError: If the path is not a full path (doesn't contain a dot).
+                   如果路径不是完整路径（不包含点）。
+        NameError: If the module doesn't define the specified object.
+                  如果模块未定义指定的对象。
+        ImportError: If the module cannot be imported.
+                    如果无法导入模块。
+
+    Example:
+        >>> middleware = load_object('aioscrapy.libs.downloader.redirect.RedirectMiddleware')
+        >>> middleware.__name__
+        'RedirectMiddleware'
     """
+    # Find the last dot in the path to separate module path from object name
+    # 在路径中查找最后一个点，以将模块路径与对象名称分开
     try:
         dot = path.rindex('.')
     except ValueError:
         raise ValueError(f"Error loading object '{path}': not a full path")
 
+    # Split the path into module path and object name
+    # 将路径分割为模块路径和对象名称
     module, name = path[:dot], path[dot + 1:]
+
+    # Import the module
+    # 导入模块
     mod = import_module(module)
 
+    # Get the object from the module
+    # 从模块中获取对象
     try:
         obj = getattr(mod, name)
     except AttributeError:
@@ -51,49 +130,138 @@ def load_object(path: str):
 
 
 async def create_instance(objcls, settings, crawler, *args, spider=None, **kwargs):
-    """Construct a class instance using its ``from_crawler`` or
-    ``from_settings`` constructors, if available.
+    """
+    Create an instance of a class using its factory methods.
+    使用类的工厂方法创建类的实例。
 
-    At least one of ``settings`` and ``crawler`` needs to be different from
-    ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
-    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
-    tried.
+    This function tries to create an instance of the given class using one of its
+    factory methods in the following order of preference:
+    1. from_crawler(crawler, *args, **kwargs) - if crawler is provided
+    2. from_spider(spider, *args, **kwargs) - if spider is provided
+    3. from_settings(settings, *args, **kwargs) - if settings is provided
+    4. Regular constructor: objcls(*args, **kwargs) - as a fallback
 
-    ``*args`` and ``**kwargs`` are forwarded to the constructors.
+    此函数尝试使用给定类的一个工厂方法创建实例，优先顺序如下：
+    1. from_crawler(crawler, *args, **kwargs) - 如果提供了crawler
+    2. from_spider(spider, *args, **kwargs) - 如果提供了spider
+    3. from_settings(settings, *args, **kwargs) - 如果提供了settings
+    4. 常规构造函数：objcls(*args, **kwargs) - 作为后备选项
 
-    Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
+    Args:
+        objcls: The class to instantiate.
+               要实例化的类。
+        settings: The settings object to use. Can be None if crawler is provided.
+                 要使用的设置对象。如果提供了crawler，可以为None。
+        crawler: The crawler object to use. Can be None.
+                要使用的爬虫对象。可以为None。
+        *args: Positional arguments to pass to the constructor.
+               传递给构造函数的位置参数。
+        spider: The spider object to use. Can be None.
+               要使用的蜘蛛对象。可以为None。
+        **kwargs: Keyword arguments to pass to the constructor.
+                 传递给构造函数的关键字参数。
 
-    .. versionchanged:: 2.2
-       Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
-       extension has not been implemented correctly).
+    Returns:
+        An instance of the specified class.
+        指定类的实例。
+
+    Raises:
+        ValueError: If settings, crawler, and spider are all None.
+                   如果settings、crawler和spider都为None。
+        TypeError: If the factory method returns None.
+                  如果工厂方法返回None。
     """
+    # Ensure we have settings from either crawler, spider, or directly provided
+    # 确保我们从crawler、spider或直接提供的参数中获取设置
     if settings is None:
         if crawler is None and spider is None:
             raise ValueError("Specify at least one of settings, crawler and spider.")
 
+        # Get settings from crawler or spider
+        # 从crawler或spider获取设置
         settings = crawler and crawler.settings or spider and spider.settings
+        # Get spider from crawler if not directly provided
+        # 如果没有直接提供，从crawler获取spider
         spider = spider or crawler and crawler.spider
 
+    # Try to create instance using the appropriate factory method
+    # 尝试使用适当的工厂方法创建实例
     if crawler and hasattr(objcls, 'from_crawler'):
+        # Use from_crawler if available and crawler is provided
+        # 如果可用且提供了crawler，则使用from_crawler
         instance = await call_helper(objcls.from_crawler, crawler, *args, **kwargs)
         method_name = 'from_crawler'
     elif spider and hasattr(objcls, 'from_spider'):
+        # Use from_spider if available and spider is provided
+        # 如果可用且提供了spider，则使用from_spider
         instance = await call_helper(objcls.from_spider, spider, *args, **kwargs)
         method_name = 'from_spider'
     elif hasattr(objcls, 'from_settings'):
+        # Use from_settings if available
+        # 如果可用，则使用from_settings
         instance = await call_helper(objcls.from_settings, settings, *args, **kwargs)
         method_name = 'from_settings'
     else:
+        # Fall back to regular constructor
+        # 回退到常规构造函数
         instance = objcls(*args, **kwargs)
         method_name = '__new__'
+
+    # Ensure the factory method returned a valid instance
+    # 确保工厂方法返回了有效的实例
     if instance is None:
         raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
+
     return instance
 
 
 async def load_instance(clspath: str, *args, settings=None, spider=None, crawler=None, **kwargs):
+    """
+    Load a class by its path and create an instance of it.
+    通过路径加载类并创建其实例。
+
+    This function combines load_object() and create_instance() to load a class
+    by its fully qualified name and then create an instance of it using the
+    appropriate factory method.
+    此函数结合了load_object()和create_instance()，通过完全限定名称加载类，
+    然后使用适当的工厂方法创建其实例。
+
+    Args:
+        clspath: The fully qualified class path (e.g., 'aioscrapy.libs.downloader.redirect.RedirectMiddleware').
+                完全限定的类路径（例如，'aioscrapy.libs.downloader.redirect.RedirectMiddleware'）。
+        *args: Positional arguments to pass to the constructor.
+               传递给构造函数的位置参数。
+        settings: The settings object to use. Can be None if crawler is provided.
+                 要使用的设置对象。如果提供了crawler，可以为None。
+        spider: The spider object to use. Can be None.
+               要使用的蜘蛛对象。可以为None。
+        crawler: The crawler object to use. Can be None.
+                要使用的爬虫对象。可以为None。
+        **kwargs: Keyword arguments to pass to the constructor.
+                 传递给构造函数的关键字参数。
+
+    Returns:
+        An instance of the specified class.
+        指定类的实例。
+
+    Raises:
+        ValueError: If settings, crawler, and spider are all None.
+                   如果settings、crawler和spider都为None。
+        TypeError: If the factory method returns None.
+                  如果工厂方法返回None。
+        ImportError: If the class cannot be imported.
+                    如果无法导入类。
+        NameError: If the module doesn't define the specified class.
+                  如果模块未定义指定的类。
+    """
+    # First load the class by its path
+    # 首先通过路径加载类
+    cls = load_object(clspath)
+
+    # Then create an instance of the class
+    # 然后创建类的实例
     return await create_instance(
-        load_object(clspath),
+        cls,
         settings,
         crawler,
         *args,

aioscrapy/utils/ossignal.py

@@ -1,23 +1,85 @@
+"""
+Operating system signal utilities for aioscrapy.
+aioscrapy的操作系统信号实用工具。
+
+This module provides utilities for working with operating system signals in aioscrapy.
+It includes functions for installing signal handlers and mapping between signal
+numbers and their names.
+此模块提供了用于处理aioscrapy中操作系统信号的实用工具。
+它包括用于安装信号处理程序以及在信号编号和其名称之间映射的函数。
+"""
+
 import signal
 
 
+# Dictionary mapping signal numbers to their names (e.g., {2: 'SIGINT', 15: 'SIGTERM'})
+# 将信号编号映射到其名称的字典（例如，{2: 'SIGINT', 15: 'SIGTERM'}）
 signal_names = {}
+
+# Populate the signal_names dictionary by iterating through all attributes in the signal module
+# 通过迭代信号模块中的所有属性来填充signal_names字典
 for signame in dir(signal):
+    # Only process attributes that start with 'SIG' but not 'SIG_'
+    # (SIG_ prefixed constants are signal handlers, not signal types)
+    # 只处理以'SIG'开头但不以'SIG_'开头的属性
+    # （SIG_前缀的常量是信号处理程序，而不是信号类型）
     if signame.startswith('SIG') and not signame.startswith('SIG_'):
+        # Get the signal number for this signal name
+        # 获取此信号名称的信号编号
         signum = getattr(signal, signame)
+        # Only add to the dictionary if it's an integer (a valid signal number)
+        # 只有当它是整数（有效的信号编号）时才添加到字典中
         if isinstance(signum, int):
             signal_names[signum] = signame
 
 
 def install_shutdown_handlers(function, override_sigint=True):
-    """Install the given function as a signal handler for all common shutdown
-    signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the
-    SIGINT handler won't be install if there is already a handler in place
-    (e.g.  Pdb)
     """
+    Install a function as a signal handler for common shutdown signals.
+    为常见的关闭信号安装函数作为信号处理程序。
+
+    This function installs the provided function as a handler for common shutdown
+    signals such as SIGTERM (terminate), SIGINT (keyboard interrupt), and SIGBREAK
+    (Ctrl-Break on Windows). This is useful for graceful shutdown of applications.
+    此函数将提供的函数安装为常见关闭信号的处理程序，如SIGTERM（终止）、
+    SIGINT（键盘中断）和SIGBREAK（Windows上的Ctrl-Break）。
+    这对于应用程序的优雅关闭很有用。
+
+    Args:
+        function: The function to be called when a shutdown signal is received.
+                 当收到关闭信号时要调用的函数。
+                 This function should accept two parameters: signal number and frame.
+                 此函数应接受两个参数：信号编号和帧。
+        override_sigint: Whether to override an existing SIGINT handler.
+                        是否覆盖现有的SIGINT处理程序。
+                        If False, the SIGINT handler won't be installed if there's
+                        already a custom handler in place (e.g., a debugger like Pdb).
+                        如果为False，则在已有自定义处理程序（例如Pdb调试器）的情况下
+                        不会安装SIGINT处理程序。
+                        Defaults to True.
+                        默认为True。
+
+    Example:
+        >>> def handle_shutdown(signum, frame):
+        ...     print(f"Received signal {signal_names.get(signum, signum)}")
+        ...     # Perform cleanup operations
+        ...     sys.exit(0)
+        >>> install_shutdown_handlers(handle_shutdown)
+    """
+    # Always install handler for SIGTERM (terminate signal)
+    # 始终为SIGTERM（终止信号）安装处理程序
     signal.signal(signal.SIGTERM, function)
+
+    # Install handler for SIGINT (keyboard interrupt) if:
+    # - The current handler is the default handler, or
+    # - override_sigint is True (forcing override of any existing handler)
+    # 在以下情况下为SIGINT（键盘中断）安装处理程序：
+    # - 当前处理程序是默认处理程序，或
+    # - override_sigint为True（强制覆盖任何现有处理程序）
     if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
         signal.signal(signal.SIGINT, function)
-    # Catch Ctrl-Break in windows
+
+    # Install handler for SIGBREAK (Ctrl-Break) on Windows if available
+    # 如果可用，在Windows上为SIGBREAK（Ctrl-Break）安装处理程序
     if hasattr(signal, 'SIGBREAK'):
         signal.signal(signal.SIGBREAK, function)