aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/utils/tools.py

@@ -1,30 +1,100 @@
 # _*_ coding: utf-8 _*_
+"""
+Utility tools for aioscrapy.
+aioscrapy的实用工具。
+
+This module provides various utility functions for working with asynchronous code,
+singletons, JavaScript execution, and task creation in aioscrapy.
+此模块提供了各种实用函数，用于在aioscrapy中处理异步代码、单例、JavaScript执行和任务创建。
+"""
+
 import asyncio
 from types import CoroutineType, GeneratorType, AsyncGeneratorType
 
 
-async def call_helper(f, *args, **kwargs):
-    if asyncio.iscoroutinefunction(f):
-        return await f(*args, **kwargs)
-    return f(*args, **kwargs)
+async def call_helper(fn, *args, **kwargs):
+    """
+    Call a function or coroutine function with the given arguments.
+    使用给定参数调用函数或协程函数。
+
+    This helper function automatically detects whether the provided function is
+    a coroutine function and awaits it if necessary. This allows for uniform
+    handling of both synchronous and asynchronous functions.
+    此辅助函数自动检测提供的函数是否为协程函数，并在必要时等待它。
+    这允许统一处理同步和异步函数。
+
+    Args:
+        fn: The function or coroutine function to call.
+           要调用的函数或协程函数。
+        *args: Positional arguments to pass to the function.
+               传递给函数的位置参数。
+        **kwargs: Keyword arguments to pass to the function.
+                  传递给函数的关键字参数。
+
+    Returns:
+        The result of calling the function or awaiting the coroutine function.
+        调用函数或等待协程函数的结果。
+    """
+    # Check if the function is a coroutine function
+    # 检查函数是否为协程函数
+    if asyncio.iscoroutinefunction(fn):
+        # If it is, await it
+        # 如果是，则等待它
+        return await fn(*args, **kwargs)
+    # Otherwise, call it directly
+    # 否则，直接调用它
+    return fn(*args, **kwargs)
 
 
 async def async_generator_wrapper(wrapped):
-    """将传入的对象变成AsyncGeneratorType类型"""
+    """
+    Convert any object into an asynchronous generator.
+    将任何对象转换为异步生成器。
+
+    This function takes any object and converts it into an AsyncGeneratorType.
+    It handles different types of input differently:
+    - AsyncGeneratorType: returned as is
+    - CoroutineType: wrapped in an async generator that yields the awaited result
+    - GeneratorType: wrapped in an async generator that yields each item
+    - Any other type: wrapped in an async generator that yields the object itself
+
+    此函数接受任何对象并将其转换为AsyncGeneratorType。
+    它对不同类型的输入有不同的处理方式：
+    - AsyncGeneratorType：按原样返回
+    - CoroutineType：包装在一个异步生成器中，该生成器产生等待的结果
+    - GeneratorType：包装在一个异步生成器中，该生成器产生每个项目
+    - 任何其他类型：包装在一个异步生成器中，该生成器产生对象本身
+
+    Args:
+        wrapped: The object to convert to an async generator.
+                要转换为异步生成器的对象。
+
+    Returns:
+        AsyncGeneratorType: An asynchronous generator that yields the appropriate values.
+                           产生适当值的异步生成器。
+    """
+    # If it's already an async generator, return it as is
+    # 如果它已经是一个异步生成器，按原样返回它
     if isinstance(wrapped, AsyncGeneratorType):
         return wrapped
 
+    # If it's a coroutine, wrap it in an async generator that yields the awaited result
+    # 如果它是一个协程，将其包装在一个异步生成器中，该生成器产生等待的结果
     elif isinstance(wrapped, CoroutineType):
         async def anonymous(c):
             yield await c
         return anonymous(wrapped)
 
+    # If it's a generator, wrap it in an async generator that yields each item
+    # 如果它是一个生成器，将其包装在一个异步生成器中，该生成器产生每个项目
     elif isinstance(wrapped, GeneratorType):
         async def anonymous(c):
             for r in c:
                 yield r
         return anonymous(wrapped)
 
+    # For any other type, wrap it in an async generator that yields the object itself
+    # 对于任何其他类型，将其包装在一个异步生成器中，该生成器产生对象本身
     else:
         async def anonymous(c):
             yield c
@@ -32,11 +102,45 @@ async def async_generator_wrapper(wrapped):
 
 
 def singleton(cls):
+    """
+    Decorator to implement the singleton pattern for a class.
+    为类实现单例模式的装饰器。
+
+    This decorator ensures that only one instance of the decorated class is created.
+    Subsequent calls to the class constructor will return the same instance.
+    此装饰器确保只创建一个被装饰类的实例。
+    对类构造函数的后续调用将返回相同的实例。
+
+    Args:
+        cls: The class to make a singleton.
+             要变成单例的类。
+
+    Returns:
+        function: A wrapper function that implements the singleton pattern.
+                 实现单例模式的包装函数。
+
+    Example:
+        @singleton
+        class MyClass:
+            pass
+
+        # These will be the same instance
+        # 这些将是相同的实例
+        instance1 = MyClass()
+        instance2 = MyClass()
+        assert instance1 is instance2
+    """
+    # Dictionary to store class instances
+    # 用于存储类实例的字典
     _instance = {}
 
     def _singleton(*args, **kwargs):
+        # If the class doesn't have an instance yet, create one
+        # 如果类还没有实例，则创建一个
         if cls not in _instance:
             _instance[cls] = cls(*args, **kwargs)
+        # Return the existing instance
+        # 返回现有实例
         return _instance[cls]
 
     return _singleton
@@ -44,35 +148,105 @@ def singleton(cls):
 
 def exec_js_func(js_file_path, func_name, func_params=None, cwd_path=None, cmd_path='node'):
     """
-    调用node执行js文件
-    :param js_file_path: js文件路径
-    :param func_name: 要调用的js文件内的函数名称
-    :param func_params: func的参数
-    :param cwd_path: node_modules文件所在路径，如果不指定将使用全局的node_modules
-    :param cmd_path: node的位置， 例如：r'D:\\path\to\node.exe'
-    :return: func_name函数的执行结果
+    Execute a JavaScript function using Node.js.
+    使用Node.js执行JavaScript函数。
+
+    This function reads a JavaScript file, compiles it using Node.js, and calls
+    a specified function with the provided parameters. It's useful for executing
+    JavaScript code that can't be easily translated to Python, such as browser
+    fingerprinting or encryption algorithms.
+    此函数读取JavaScript文件，使用Node.js编译它，并使用提供的参数调用指定的函数。
+    它对于执行不容易转换为Python的JavaScript代码很有用，例如浏览器指纹或加密算法。
+
+    Note:
+        This function requires the PyExecJS package to be installed.
+        此函数需要安装PyExecJS包。
+
+    Args:
+        js_file_path (str): Path to the JavaScript file.
+                           JavaScript文件的路径。
+        func_name (str): Name of the function to call in the JavaScript file.
+                        要在JavaScript文件中调用的函数名称。
+        func_params (list, optional): Parameters to pass to the JavaScript function.
+                                     要传递给JavaScript函数的参数。
+        cwd_path (str, optional): Path to the directory containing node_modules.
+                                 包含node_modules的目录路径。
+                                 If not specified, global node_modules will be used.
+                                 如果未指定，将使用全局的node_modules。
+        cmd_path (str, optional): Path to the Node.js executable.
+                                 Node.js可执行文件的路径。
+                                 Default is 'node', which assumes Node.js is in PATH.
+                                 默认为'node'，这假设Node.js在PATH中。
+
+    Returns:
+        The result of the JavaScript function call.
+        JavaScript函数调用的结果。
+
+    Raises:
+        ImportError: If PyExecJS is not installed.
+                    如果未安装PyExecJS。
+        FileNotFoundError: If the JavaScript file or Node.js executable is not found.
+                          如果找不到JavaScript文件或Node.js可执行文件。
     """
+    # Import execjs here to avoid making it a required dependency
+    # 在这里导入execjs以避免使其成为必需的依赖项
     import execjs
 
+    # Initialize function parameters if None
+    # 如果为None，则初始化函数参数
     if func_params is None:
         func_params = []
-    name = "MyNode"
-    execjs.register(name, execjs._external_runtime.ExternalRuntime(
+
+    # Register a custom Node.js runtime
+    # 注册自定义Node.js运行时
+    node_runtime_name = "MyNode"
+    execjs.register(node_runtime_name, execjs._external_runtime.ExternalRuntime(
         name="Node.js (V8)",
         command=[cmd_path],
         encoding='UTF-8',
         runner_source=execjs._runner_sources.Node
     ))
+
+    # Read the JavaScript file
+    # 读取JavaScript文件
     with open(js_file_path, 'r', encoding='utf-8') as f:
         lines = f.readlines()
     js = ''.join(lines)
-    js_context = execjs.get(name).compile(js, cwd=cwd_path)
+
+    # Compile the JavaScript code and call the function
+    # 编译JavaScript代码并调用函数
+    js_context = execjs.get(node_runtime_name).compile(js, cwd=cwd_path)
     return js_context.call(func_name, *func_params)
 
 
 def create_task(coros, name=None):
-    """ 感染所有的协程函数 """
+    """
+    Create an asyncio task with the current task's name.
+    创建具有当前任务名称的asyncio任务。
+
+    This function creates an asyncio task from a coroutine and sets its name
+    to the name of the current task. This helps with task tracking and debugging
+    by maintaining the task hierarchy in the task names.
+    此函数从协程创建asyncio任务，并将其名称设置为当前任务的名称。
+    这通过在任务名称中维护任务层次结构来帮助任务跟踪和调试。
+
+    Args:
+        coros: The coroutine to schedule for execution.
+               要安排执行的协程。
+        name: Optional name for the task. If not provided, the current task's name is used.
+              任务的可选名称。如果未提供，则使用当前任务的名称。
+
+    Returns:
+        asyncio.Task: The created task.
+                     创建的任务。
+
+    Raises:
+        RuntimeError: If there is no current task.
+                     如果没有当前任务。
+    """
+    # Create a new task with the coroutine and inherit the current task's name
+    # 使用协程创建新任务并继承当前任务的名称
     return asyncio.create_task(
-        coros, 
+        coros,
         name=asyncio.current_task().get_name()
     )

aioscrapy/utils/trackref.py

@@ -1,12 +1,26 @@
-"""This module provides some functions and classes to record and report
-references to live object instances.
+"""
+Object reference tracking utilities for aioscrapy.
+aioscrapy的对象引用跟踪实用工具。
+
+This module provides functions and classes to record and report references to
+live object instances. It's useful for debugging memory leaks and tracking
+object lifetimes in aioscrapy applications.
+此模块提供了用于记录和报告活动对象实例引用的函数和类。
+它对于调试内存泄漏和跟踪aioscrapy应用程序中的对象生命周期很有用。
 
+Usage:
+使用方法：
 If you want live objects for a particular class to be tracked, you only have to
 subclass from object_ref (instead of object).
-
-About performance: This library has a minimal performance impact when enabled,
-and no performance penalty at all when disabled (as object_ref becomes just an
-alias to object in that case).
+如果您希望跟踪特定类的活动对象，只需从object_ref（而不是object）继承即可。
+
+About performance:
+关于性能：
+This library has a minimal performance impact when enabled, and no performance
+penalty at all when disabled (as object_ref becomes just an alias to object
+in that case).
+启用时，此库对性能的影响最小，禁用时完全没有性能损失
+（因为在这种情况下，object_ref只是object的别名）。
 """
 
 from collections import defaultdict
@@ -16,52 +30,224 @@ from typing import DefaultDict
 from weakref import WeakKeyDictionary
 
 
+# Type alias for the None type, used for default ignore parameter in format_live_refs
+# None类型的类型别名，用于format_live_refs中的默认ignore参数
 NoneType = type(None)
+
+# Global dictionary that maps classes to WeakKeyDictionaries of their instances
+# 全局字典，将类映射到其实例的WeakKeyDictionary
+# The WeakKeyDictionary for each class maps object instances to their creation time
+# 每个类的WeakKeyDictionary将对象实例映射到它们的创建时间
 live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)
 
 
 class object_ref:
-    """Inherit from this class to a keep a record of live instances"""
-
+    """
+    Base class for tracking live object instances.
+    用于跟踪活动对象实例的基类。
+
+    Inherit from this class instead of directly from 'object' to enable tracking
+    of instances of your class. Each instance will be recorded in the global
+    'live_refs' dictionary with its creation time.
+    从此类继承而不是直接从'object'继承，以启用对您的类实例的跟踪。
+    每个实例都将与其创建时间一起记录在全局'live_refs'字典中。
+
+    This tracking uses weak references, so it doesn't prevent garbage collection
+    of objects that are no longer referenced elsewhere.
+    此跟踪使用弱引用，因此不会阻止垃圾收集不再在其他地方引用的对象。
+    """
+
+    # Using __slots__ to reduce memory footprint
+    # 使用__slots__减少内存占用
     __slots__ = ()
 
     def __new__(cls, *args, **kwargs):
+        """
+        Create a new instance and register it in the tracking system.
+        创建一个新实例并将其注册到跟踪系统中。
+
+        This method creates a new instance of the class and adds it to the
+        'live_refs' dictionary with the current timestamp. This allows tracking
+        when the object was created and how many instances exist.
+        此方法创建类的新实例，并将其与当前时间戳一起添加到'live_refs'字典中。
+        这允许跟踪对象的创建时间以及存在多少个实例。
+
+        Args:
+            *args: Variable length argument list passed to the constructor.
+                  传递给构造函数的可变长度参数列表。
+            **kwargs: Arbitrary keyword arguments passed to the constructor.
+                     传递给构造函数的任意关键字参数。
+
+        Returns:
+            The newly created object instance.
+            新创建的对象实例。
+        """
+        # Create the object using the standard object.__new__
+        # 使用标准的object.__new__创建对象
         obj = object.__new__(cls)
+
+        # Register the object in live_refs with current timestamp
+        # 使用当前时间戳在live_refs中注册对象
         live_refs[cls][obj] = time()
+
         return obj
 
 
 def format_live_refs(ignore=NoneType):
-    """Return a tabular representation of tracked objects"""
+    """
+    Generate a formatted table of tracked live objects.
+    生成跟踪的活动对象的格式化表格。
+
+    This function creates a human-readable table showing statistics about
+    tracked objects, including:
+    - The class name
+    - The number of live instances
+    - The age of the oldest instance
+
+    此函数创建一个人类可读的表格，显示有关跟踪对象的统计信息，包括：
+    - 类名
+    - 活动实例的数量
+    - 最旧实例的年龄
+
+    Args:
+        ignore: A class or type to ignore in the output. Instances of this class
+               and its subclasses will not be included in the report.
+               要在输出中忽略的类或类型。此类及其子类的实例将不会包含在报告中。
+               Default is NoneType.
+               默认为NoneType。
+
+    Returns:
+        str: A formatted string containing the tabular representation of tracked objects.
+             包含跟踪对象的表格表示的格式化字符串。
+    """
+    # Start with a header
+    # 以标题开始
     s = "Live References\n\n"
+
+    # Get current time for age calculation
+    # 获取当前时间用于年龄计算
     now = time()
+
+    # Sort classes by name for consistent output
+    # 按名称对类进行排序以获得一致的输出
     for cls, wdict in sorted(live_refs.items(),
                              key=lambda x: x[0].__name__):
+        # Skip empty dictionaries (no instances)
+        # 跳过空字典（没有实例）
         if not wdict:
             continue
+
+        # Skip ignored classes and their subclasses
+        # 跳过被忽略的类及其子类
         if issubclass(cls, ignore):
             continue
+
+        # Find the oldest instance
+        # 查找最旧的实例
         oldest = min(wdict.values())
+
+        # Format the line: class name, instance count, and age of oldest instance
+        # 格式化行：类名、实例计数和最旧实例的年龄
         s += f"{cls.__name__:<30} {len(wdict):6}   oldest: {int(now - oldest)}s ago\n"
+
     return s
 
 
 def print_live_refs(*a, **kw):
-    """Print tracked objects"""
+    """
+    Print a formatted table of tracked live objects to stdout.
+    将跟踪的活动对象的格式化表格打印到标准输出。
+
+    This is a convenience function that calls format_live_refs() and prints
+    the result to the standard output. It's useful for interactive debugging
+    or for logging object tracking information.
+    这是一个便捷函数，它调用format_live_refs()并将结果打印到标准输出。
+    它对于交互式调试或记录对象跟踪信息很有用。
+
+    Args:
+        *a: Positional arguments passed to format_live_refs().
+            传递给format_live_refs()的位置参数。
+        **kw: Keyword arguments passed to format_live_refs().
+              传递给format_live_refs()的关键字参数。
+
+    Returns:
+        None
+    """
+    # Format the live references and print the result
+    # 格式化活动引用并打印结果
     print(format_live_refs(*a, **kw))
 
 
 def get_oldest(class_name):
-    """Get the oldest object for a specific class name"""
+    """
+    Get the oldest tracked instance of a class by its name.
+    通过名称获取类的最旧跟踪实例。
+
+    This function finds the oldest (longest-living) instance of a class
+    with the given name. The age is determined by the timestamp recorded
+    when the object was created.
+    此函数查找具有给定名称的类的最旧（存活时间最长）实例。
+    年龄由创建对象时记录的时间戳确定。
+
+    Args:
+        class_name: The name of the class to search for.
+                   要搜索的类的名称。
+
+    Returns:
+        object: The oldest instance of the specified class, or None if no
+               instances are found.
+               指定类的最旧实例，如果未找到实例则为None。
+    """
+    # Iterate through all tracked classes
+    # 遍历所有跟踪的类
     for cls, wdict in live_refs.items():
+        # Find the class with the matching name
+        # 查找具有匹配名称的类
         if cls.__name__ == class_name:
+            # If there are no instances, return None
+            # 如果没有实例，返回None
             if not wdict:
                 break
+
+            # Find the instance with the minimum timestamp (oldest)
+            # 查找具有最小时间戳（最旧）的实例
             return min(wdict.items(), key=itemgetter(1))[0]
 
+    # Return None if no matching class or instances are found
+    # 如果未找到匹配的类或实例，则返回None
+    return None
+
 
 def iter_all(class_name):
-    """Iterate over all objects of the same class by its class name"""
+    """
+    Iterate over all tracked instances of a class by its name.
+    通过名称迭代类的所有跟踪实例。
+
+    This function returns an iterator over all live instances of a class
+    with the given name. It's useful for inspecting or manipulating all
+    instances of a particular class during debugging.
+    此函数返回具有给定名称的类的所有活动实例的迭代器。
+    它对于在调试期间检查或操作特定类的所有实例很有用。
+
+    Args:
+        class_name: The name of the class to search for.
+                   要搜索的类的名称。
+
+    Returns:
+        iterator: An iterator over all instances of the specified class,
+                 or None if no matching class is found.
+                 指定类的所有实例的迭代器，如果未找到匹配的类则为None。
+    """
+    # Iterate through all tracked classes
+    # 遍历所有跟踪的类
     for cls, wdict in live_refs.items():
+        # Find the class with the matching name
+        # 查找具有匹配名称的类
         if cls.__name__ == class_name:
+            # Return an iterator over all instances of this class
+            # 返回此类的所有实例的迭代器
             return wdict.keys()
+
+    # Return None if no matching class is found
+    # 如果未找到匹配的类，则返回None
+    return None