aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/utils/signal.py

@@ -1,4 +1,13 @@
-"""Helper functions for working with signals"""
+"""
+Signal utility functions for aioscrapy.
+aioscrapy的信号实用函数。
+
+This module provides utility functions for working with signals in aioscrapy.
+It includes functions for sending signals, catching and logging exceptions,
+and managing signal connections.
+此模块提供了用于处理aioscrapy中信号的实用函数。
+它包括用于发送信号、捕获和记录异常以及管理信号连接的函数。
+"""
 import asyncio
 
 from pydispatch.dispatcher import Anonymous, Any, disconnect, getAllReceivers, liveReceivers
@@ -10,52 +19,225 @@ from aioscrapy.utils.tools import create_task
 
 
 class _IgnoredException(Exception):
+    """
+    Internal exception class used to mark exceptions that should be ignored in logs.
+    内部异常类，用于标记应在日志中忽略的异常。
+
+    This exception is used as a marker for exceptions that should not be logged
+    when caught in signal handlers. It's used in conjunction with the 'dont_log'
+    parameter in signal sending functions.
+    此异常用作在信号处理程序中捕获时不应记录的异常的标记。
+    它与信号发送函数中的'dont_log'参数一起使用。
+    """
     pass
 
 
 async def robustApplyWrap(f, recv, *args, **kw):
+    """
+    Wrap a function call with exception handling and async support.
+    使用异常处理和异步支持包装函数调用。
+
+    This function wraps the application of a function to a receiver with robust
+    exception handling. It also supports awaiting coroutines returned by the function.
+    If an exception occurs, it logs the error (unless the exception type is in dont_log)
+    and returns the exception object instead of raising it.
+    此函数使用健壮的异常处理包装函数对接收器的应用。它还支持等待函数返回的协程。
+    如果发生异常，它会记录错误（除非异常类型在dont_log中），并返回异常对象而不是引发它。
+
+    Args:
+        f: The function to apply (typically robustApply).
+           要应用的函数（通常是robustApply）。
+        recv: The receiver object (signal handler).
+              接收器对象（信号处理程序）。
+        *args: Positional arguments to pass to the function.
+               传递给函数的位置参数。
+        **kw: Keyword arguments to pass to the function.
+              传递给函数的关键字参数。
+              Special keys:
+              特殊键：
+              - dont_log: Exception types to not log if caught.
+                         如果捕获，不记录的异常类型。
+              - spider: The spider instance (for context in logs).
+                       爬虫实例（用于日志中的上下文）。
+
+    Returns:
+        The result of the function call, or the exception if one was caught.
+        函数调用的结果，如果捕获到异常则返回异常。
+    """
+    # Extract special parameters
+    # 提取特殊参数
     dont_log = kw.pop('dont_log', None)
-    spider = kw.get('spider', None)
+    # Spider is kept in kw for context but extracted here for potential future use
+    # 爬虫保留在kw中作为上下文，但在此提取以供将来使用
+    spider = kw.get('spider', None)  # noqa: F841
+
     try:
+        # Apply the function to the receiver
+        # 将函数应用于接收器
         result = f(recv, *args, **kw)
+        # If the result is a coroutine, await it
+        # 如果结果是协程，等待它
         if asyncio.iscoroutine(result):
             return await result
+        return result
     except (Exception, BaseException) as exc:  # noqa: E722
+        # Log the exception unless it's a type we should ignore
+        # 记录异常，除非它是我们应该忽略的类型
         if dont_log is None or not isinstance(exc, dont_log):
             logger.exception(f"Error caught on signal handler: {recv}")
+        # Return the exception instead of raising it
+        # 返回异常而不是引发它
         return exc
 
 
 async def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
-    """Like pydispatcher.robust.sendRobust but it also logs errors and returns
-    Failures instead of exceptions.
     """
+    Send a signal and catch any exceptions raised by handlers.
+    发送信号并捕获处理程序引发的任何异常。
+
+    This function is similar to pydispatcher.robust.sendRobust but with additional
+    features for aioscrapy:
+    1. It logs errors that occur in signal handlers
+    2. It returns the exceptions instead of raising them
+    3. It supports async signal handlers
+    4. It has special handling for StopDownload exceptions
+
+    此函数类似于pydispatcher.robust.sendRobust，但为aioscrapy提供了额外功能：
+    1. 它记录信号处理程序中发生的错误
+    2. 它返回异常而不是引发它们
+    3. 它支持异步信号处理程序
+    4. 它对StopDownload异常有特殊处理
+
+    Args:
+        signal: The signal to send. Default is Any (all signals).
+               要发送的信号。默认为Any（所有信号）。
+        sender: The sender of the signal. Default is Anonymous.
+               信号的发送者。默认为Anonymous。
+        *arguments: Positional arguments to pass to the signal handlers.
+                   传递给信号处理程序的位置参数。
+        **named: Keyword arguments to pass to the signal handlers.
+                传递给信号处理程序的关键字参数。
+                Special keys:
+                特殊键：
+                - dont_log: Exception types to not log if caught.
+                           如果捕获，不记录的异常类型。
+
+    Returns:
+        list: A list of (receiver, result) tuples, where result is either the
+              return value of the handler or the exception that was caught.
+              (接收器, 结果)元组的列表，其中结果是处理程序的返回值或捕获的异常。
+    """
+    # Configure which exceptions should not be logged
+    # 配置不应记录的异常
     named['dont_log'] = (named.pop('dont_log', _IgnoredException), StopDownload)
+
+    # Collect responses from all receivers
+    # 收集所有接收器的响应
     responses = []
+
+    # Get all receivers for this signal and sender
+    # 获取此信号和发送者的所有接收器
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
-        result = await robustApplyWrap(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)
+        # Apply the handler function robustly, catching any exceptions
+        # 健壮地应用处理程序函数，捕获任何异常
+        result = await robustApplyWrap(
+            robustApply,
+            receiver,
+            signal=signal,
+            sender=sender,
+            *arguments,
+            **named
+        )
+        # Store the receiver and its result (or exception)
+        # 存储接收器及其结果（或异常）
         responses.append((receiver, result))
+
     return responses
 
 
 async def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
-    """Like send_catch_log but supports returning deferreds on signal handlers.
-    Returns a deferred that gets fired once all signal handlers deferreds were
-    fired.
     """
-    dfds = []
+    Send a signal and gather results from all handlers concurrently.
+    发送信号并同时收集所有处理程序的结果。
+
+    This function is similar to send_catch_log but runs all signal handlers
+    concurrently using asyncio tasks. It waits for all handlers to complete
+    before returning the results.
+    此函数类似于send_catch_log，但使用asyncio任务同时运行所有信号处理程序。
+    它在返回结果之前等待所有处理程序完成。
+
+    This is useful when signal handlers are independent of each other and
+    can run in parallel, potentially improving performance.
+    当信号处理程序彼此独立并且可以并行运行时，这很有用，可能会提高性能。
+
+    Args:
+        signal: The signal to send. Default is Any (all signals).
+               要发送的信号。默认为Any（所有信号）。
+        sender: The sender of the signal. Default is Anonymous.
+               信号的发送者。默认为Anonymous。
+        *arguments: Positional arguments to pass to the signal handlers.
+                   传递给信号处理程序的位置参数。
+        **named: Keyword arguments to pass to the signal handlers.
+                传递给信号处理程序的关键字参数。
+
+    Returns:
+        list: A list of results from all signal handlers, in the order they were
+              registered. Each result is either the return value of the handler
+              or the exception that was caught.
+              所有信号处理程序的结果列表，按它们注册的顺序排列。
+              每个结果要么是处理程序的返回值，要么是捕获的异常。
+    """
+    # List to store tasks for each receiver
+    # 用于存储每个接收器的任务的列表
+    tasks = []
+
+    # Get all receivers for this signal and sender
+    # 获取此信号和发送者的所有接收器
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
-        dfds.append(
+        # Create a task for each receiver to run concurrently
+        # 为每个接收器创建一个任务以同时运行
+        tasks.append(
             create_task(
-                robustApplyWrap(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)
+                robustApplyWrap(
+                    robustApply,
+                    receiver,
+                    signal=signal,
+                    sender=sender,
+                    *arguments,
+                    **named
+                )
             )
         )
-    return await asyncio.gather(*dfds)
+
+    # Wait for all tasks to complete and return their results
+    # 等待所有任务完成并返回它们的结果
+    return await asyncio.gather(*tasks)
 
 
 def disconnect_all(signal=Any, sender=Any):
-    """Disconnect all signal handlers. Useful for cleaning up after running
-    tests
     """
+    Disconnect all signal handlers for a given signal and sender.
+    断开给定信号和发送者的所有信号处理程序。
+
+    This function disconnects all signal handlers that match the specified
+    signal and sender. It's particularly useful for cleaning up after running
+    tests to ensure that signal handlers from one test don't affect other tests.
+    此函数断开与指定信号和发送者匹配的所有信号处理程序。
+    它对于在运行测试后进行清理特别有用，以确保一个测试的信号处理程序不会影响其他测试。
+
+    Args:
+        signal: The signal to disconnect handlers from. Default is Any (all signals).
+               要断开处理程序的信号。默认为Any（所有信号）。
+        sender: The sender to disconnect handlers for. Default is Any (all senders).
+               要断开处理程序的发送者。默认为Any（所有发送者）。
+
+    Note:
+        This function modifies the global signal registry maintained by PyDispatcher.
+        此函数修改由PyDispatcher维护的全局信号注册表。
+    """
+    # Get all receivers for this signal and sender
+    # 获取此信号和发送者的所有接收器
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
+        # Disconnect each receiver
+        # 断开每个接收器
         disconnect(receiver, signal=signal, sender=sender)

aioscrapy/utils/spider.py

@@ -1,20 +1,67 @@
+"""
+Spider utility functions for aioscrapy.
+aioscrapy的爬虫实用函数。
+
+This module provides utility functions for working with spider classes in aioscrapy.
+It includes functions for discovering and iterating over spider classes.
+此模块提供了用于处理aioscrapy中爬虫类的实用函数。
+它包括用于发现和迭代爬虫类的函数。
+"""
+
 import inspect
 
 from aioscrapy.spiders import Spider
 
 
 def iter_spider_classes(module):
-    """Return an iterator over all spider classes defined in the given module
-    that can be instantiated (i.e. which have name)
     """
-    # this needs to be imported here until get rid of the spider manager
-    # singleton in aioscrapy.spider.spiders
+    Iterate over all valid spider classes defined in a module.
+    迭代模块中定义的所有有效爬虫类。
+
+    This function finds all classes in the given module that:
+    1. Are subclasses of the Spider class
+    2. Are defined in the module itself (not imported)
+    3. Have a non-empty 'name' attribute (required for instantiation)
+
+    此函数查找给定模块中满足以下条件的所有类：
+    1. 是Spider类的子类
+    2. 在模块本身中定义（非导入）
+    3. 具有非空的'name'属性（实例化所必需的）
 
+    The function is used by the spider loader to discover spiders in a module.
+    该函数被爬虫加载器用来在模块中发现爬虫。
+
+    Args:
+        module: The module object to inspect for spider classes.
+               要检查爬虫类的模块对象。
+
+    Yields:
+        class: Spider classes that can be instantiated.
+              可以实例化的爬虫类。
+
+    Note:
+        This implementation avoids importing the spider manager singleton
+        from aioscrapy.spider.spiders, which would create circular imports.
+        此实现避免从aioscrapy.spider.spiders导入爬虫管理器单例，
+        这会创建循环导入。
+    """
+    # Iterate through all objects in the module
+    # 迭代模块中的所有对象
     for obj in vars(module).values():
+        # Check if the object meets all criteria for a valid spider class
+        # 检查对象是否满足有效爬虫类的所有条件
         if (
+                # Must be a class
+                # 必须是一个类
                 inspect.isclass(obj)
+                # Must be a subclass of Spider
+                # 必须是Spider的子类
                 and issubclass(obj, Spider)
+                # Must be defined in this module (not imported)
+                # 必须在此模块中定义（非导入）
                 and obj.__module__ == module.__name__
+                # Must have a name attribute (required for instantiation)
+                # 必须有name属性（实例化所必需的）
                 and getattr(obj, 'name', None)
         ):
             yield obj

aioscrapy/utils/template.py

@@ -1,4 +1,13 @@
-"""Helper functions for working with templates"""
+"""
+Template utility functions for aioscrapy.
+aioscrapy的模板实用函数。
+
+This module provides utility functions for working with templates in aioscrapy.
+It includes functions for rendering template files and string transformations
+commonly used in code generation.
+此模块提供了用于处理aioscrapy中模板的实用函数。
+它包括用于渲染模板文件和在代码生成中常用的字符串转换的函数。
+"""
 
 import os
 import re
@@ -6,31 +15,109 @@ import string
 
 
 def render_templatefile(path, **kwargs):
+    """
+    Render a template file with the given parameters.
+    使用给定参数渲染模板文件。
+
+    This function reads a template file, substitutes variables using Python's
+    string.Template, and writes the result back to the file system. If the file
+    has a '.tmpl' extension, it will be renamed to remove this extension after
+    rendering.
+    此函数读取模板文件，使用Python的string.Template替换变量，
+    并将结果写回文件系统。如果文件有'.tmpl'扩展名，
+    渲染后将重命名以删除此扩展名。
+
+    The template uses the syntax defined by string.Template, where variables are
+    marked with a $ prefix (e.g., $variable or ${variable}).
+    模板使用string.Template定义的语法，其中变量用$前缀标记
+    （例如，$variable或${variable}）。
+
+    Args:
+        path: Path to the template file to render.
+              要渲染的模板文件的路径。
+        **kwargs: Variables to substitute in the template.
+                 要在模板中替换的变量。
+
+    Example:
+        >>> render_templatefile('spider.py.tmpl',
+        ...                     classname='MySpider',
+        ...                     domain='example.com')
+
+    Note:
+        This function modifies the file system by:
+        此函数通过以下方式修改文件系统：
+        1. Potentially renaming the template file (if it ends with .tmpl)
+           可能重命名模板文件（如果以.tmpl结尾）
+        2. Writing the rendered content to the target file
+           将渲染的内容写入目标文件
+    """
+    # Read the template file as UTF-8
+    # 以UTF-8格式读取模板文件
     with open(path, 'rb') as fp:
         raw = fp.read().decode('utf8')
 
+    # Substitute variables in the template
+    # 替换模板中的变量
     content = string.Template(raw).substitute(**kwargs)
 
+    # Determine the output path (remove .tmpl extension if present)
+    # 确定输出路径（如果存在，则删除.tmpl扩展名）
     render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path
 
+    # Rename the file if it has a .tmpl extension
+    # 如果文件有.tmpl扩展名，则重命名文件
     if path.endswith('.tmpl'):
         os.rename(path, render_path)
 
+    # Write the rendered content back to the file
+    # 将渲染的内容写回文件
     with open(render_path, 'wb') as fp:
         fp.write(content.encode('utf8'))
 
 
+# Regular expression pattern to match characters that are not letters or digits
+# Used by string_camelcase to remove invalid characters when converting to CamelCase
+# 匹配非字母或数字的字符的正则表达式模式
+# 由string_camelcase用于在转换为驼峰命名法时删除无效字符
 CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')
 
 
 def string_camelcase(string):
-    """ Convert a word  to its CamelCase version and remove invalid chars
+    """
+    Convert a string to CamelCase and remove invalid characters.
+    将字符串转换为驼峰命名法并删除无效字符。
+
+    This function converts a string to CamelCase by:
+    1. Capitalizing the first letter of each word (using str.title())
+    2. Removing all non-alphanumeric characters (using CAMELCASE_INVALID_CHARS regex)
+
+    此函数通过以下方式将字符串转换为驼峰命名法：
+    1. 将每个单词的首字母大写（使用str.title()）
+    2. 删除所有非字母数字字符（使用CAMELCASE_INVALID_CHARS正则表达式）
+
+    This is commonly used in code generation to convert variable names or
+    identifiers from different formats (snake_case, kebab-case, etc.) to CamelCase.
+    这在代码生成中常用于将变量名或标识符从不同格式
+    （snake_case、kebab-case等）转换为驼峰命名法。
+
+    Args:
+        string: The input string to convert to CamelCase.
+               要转换为驼峰命名法的输入字符串。
+
+    Returns:
+        str: The CamelCase version of the input string with invalid characters removed.
+             输入字符串的驼峰命名法版本，已删除无效字符。
 
-    >>> string_camelcase('lost-pound')
-    'LostPound'
+    Examples:
+        >>> string_camelcase('lost-pound')
+        'LostPound'
 
-    >>> string_camelcase('missing_images')
-    'MissingImages'
+        >>> string_camelcase('missing_images')
+        'MissingImages'
 
+        >>> string_camelcase('hello world')
+        'HelloWorld'
     """
+    # Convert to title case (capitalize first letter of each word) and remove invalid chars
+    # 转换为标题大小写（每个单词的首字母大写）并删除无效字符
     return CAMELCASE_INVALID_CHARS.sub('', string.title())