aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/utils/project.py

@@ -1,3 +1,14 @@
+"""
+Project utility functions for aioscrapy.
+aioscrapy的项目实用函数。
+
+This module provides utility functions for working with aioscrapy projects.
+It includes functions for determining if code is running inside a project,
+accessing project data directories, and loading project settings.
+此模块提供了用于处理aioscrapy项目的实用函数。
+它包括用于确定代码是否在项目内运行、访问项目数据目录和加载项目设置的函数。
+"""
+
 import os
 import warnings
 
@@ -9,73 +20,221 @@ from aioscrapy.settings import Settings
 from aioscrapy.exceptions import NotConfigured, AioScrapyDeprecationWarning
 
 
+# Environment variable name that defines the settings module path
+# 定义设置模块路径的环境变量名称
 ENVVAR = 'AIOSCRAPY_SETTINGS_MODULE'
+
+# Configuration section name for data directory settings in scrapy.cfg
+# scrapy.cfg中数据目录设置的配置部分名称
 DATADIR_CFG_SECTION = 'datadir'
 
 
 def inside_project():
+    """
+    Check if the code is running inside an aioscrapy project.
+    检查代码是否在aioscrapy项目内运行。
+
+    This function determines if the current code is running inside an aioscrapy project
+    by checking either:
+    1. If the AIOSCRAPY_SETTINGS_MODULE environment variable is set and the module can be imported
+    2. If a scrapy.cfg file can be found in the current directory or any parent directory
+
+    此函数通过以下方式确定当前代码是否在aioscrapy项目内运行：
+    1. 检查AIOSCRAPY_SETTINGS_MODULE环境变量是否设置且模块可以导入
+    2. 检查当前目录或任何父目录中是否可以找到scrapy.cfg文件
+
+    Returns:
+        bool: True if running inside a project, False otherwise.
+              如果在项目内运行则为True，否则为False。
+    """
+    # Check if the settings module environment variable is set
+    # 检查是否设置了设置模块环境变量
     scrapy_module = os.environ.get('AIOSCRAPY_SETTINGS_MODULE')
     if scrapy_module is not None:
         try:
+            # Try to import the settings module
+            # 尝试导入设置模块
             import_module(scrapy_module)
         except ImportError as exc:
+            # If import fails, warn but continue checking
+            # 如果导入失败，发出警告但继续检查
             warnings.warn(f"Cannot import scrapy settings module {scrapy_module}: {exc}")
         else:
+            # If import succeeds, we're inside a project
+            # 如果导入成功，我们在项目内
             return True
+
+    # If no settings module or import failed, check for scrapy.cfg file
+    # 如果没有设置模块或导入失败，检查scrapy.cfg文件
     return bool(closest_aioscrapy_cfg())
 
 
 def project_data_dir(project='default'):
-    """Return the current project data dir, creating it if it doesn't exist"""
+    """
+    Get the project data directory, creating it if it doesn't exist.
+    获取项目数据目录，如果不存在则创建它。
+
+    This function returns the path to the data directory for the specified project.
+    The directory is determined in the following order:
+    1. From the [datadir] section in scrapy.cfg if it exists
+    2. Otherwise, defaults to a '.scrapy' directory in the same directory as scrapy.cfg
+
+    此函数返回指定项目的数据目录的路径。
+    目录按以下顺序确定：
+    1. 如果存在，从scrapy.cfg的[datadir]部分获取
+    2. 否则，默认为与scrapy.cfg相同目录中的'.scrapy'目录
+
+    The function will create the directory if it doesn't exist.
+    如果目录不存在，该函数将创建它。
+
+    Args:
+        project: The project name to get the data directory for.
+                要获取数据目录的项目名称。
+                Defaults to 'default'.
+                默认为'default'。
+
+    Returns:
+        str: The absolute path to the project data directory.
+             项目数据目录的绝对路径。
+
+    Raises:
+        NotConfigured: If not running inside a project or if scrapy.cfg cannot be found.
+                      如果不在项目内运行或找不到scrapy.cfg。
+    """
+    # Check if we're inside a project
+    # 检查我们是否在项目内
     if not inside_project():
         raise NotConfigured("Not inside a project")
+
+    # Get the configuration
+    # 获取配置
     cfg = get_config()
+
+    # Try to get the data directory from the config
+    # 尝试从配置中获取数据目录
     if cfg.has_option(DATADIR_CFG_SECTION, project):
         d = cfg.get(DATADIR_CFG_SECTION, project)
     else:
+        # Fall back to default location
+        # 回退到默认位置
         scrapy_cfg = closest_aioscrapy_cfg()
         if not scrapy_cfg:
             raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir")
         d = abspath(join(dirname(scrapy_cfg), '.scrapy'))
+
+    # Create the directory if it doesn't exist
+    # 如果目录不存在，则创建它
     if not exists(d):
         os.makedirs(d)
+
     return d
 
 
 def data_path(path, createdir=False):
     """
-    Return the given path joined with the .scrapy data directory.
-    If given an absolute path, return it unmodified.
+    Get the absolute path for a file within the project data directory.
+    获取项目数据目录中文件的绝对路径。
+
+    This function resolves a path relative to the project data directory.
+    If the given path is already absolute, it returns it unmodified.
+    If not inside a project, it uses a '.scrapy' directory in the current directory.
+
+    此函数解析相对于项目数据目录的路径。
+    如果给定的路径已经是绝对路径，则原样返回。
+    如果不在项目内，则使用当前目录中的'.scrapy'目录。
+
+    Args:
+        path: The path to resolve. Can be absolute or relative.
+              要解析的路径。可以是绝对路径或相对路径。
+        createdir: Whether to create the directory if it doesn't exist.
+                  如果目录不存在，是否创建它。
+                  Defaults to False.
+                  默认为False。
+
+    Returns:
+        str: The absolute path to the file or directory.
+             文件或目录的绝对路径。
     """
+    # If the path is not absolute, make it relative to the data directory
+    # 如果路径不是绝对的，使其相对于数据目录
     if not isabs(path):
         if inside_project():
+            # If inside a project, use the project data directory
+            # 如果在项目内，使用项目数据目录
             path = join(project_data_dir(), path)
         else:
+            # Otherwise, use a .scrapy directory in the current directory
+            # 否则，使用当前目录中的.scrapy目录
             path = join('.scrapy', path)
+
+    # Create the directory if requested and it doesn't exist
+    # 如果请求且目录不存在，则创建目录
     if createdir and not exists(path):
         os.makedirs(path)
+
     return path
 
 
 def get_project_settings():
+    """
+    Get a Settings instance with the project settings.
+    获取包含项目设置的Settings实例。
+
+    This function loads the project settings from the module specified in the
+    AIOSCRAPY_SETTINGS_MODULE environment variable. If the variable is not set,
+    it tries to initialize the environment using the project name from the
+    AIOSCRAPY_PROJECT environment variable (defaulting to 'default').
+
+    此函数从AIOSCRAPY_SETTINGS_MODULE环境变量指定的模块加载项目设置。
+    如果未设置该变量，它会尝试使用AIOSCRAPY_PROJECT环境变量中的项目名称
+    （默认为'default'）初始化环境。
+
+    The function also handles environment variables prefixed with AIOSCRAPY_
+    as a way to override settings, though this method is deprecated.
+
+    该函数还处理以AIOSCRAPY_为前缀的环境变量作为覆盖设置的方式，
+    尽管此方法已弃用。
+
+    Returns:
+        Settings: A Settings instance with the project settings loaded.
+                 加载了项目设置的Settings实例。
+    """
+    # Initialize the environment if the settings module is not set
+    # 如果未设置设置模块，则初始化环境
     if ENVVAR not in os.environ:
         project = os.environ.get('AIOSCRAPY_PROJECT', 'default')
         init_env(project)
 
+    # Create a new Settings instance
+    # 创建一个新的Settings实例
     settings = Settings()
+
+    # Load settings from the module specified in the environment variable
+    # 从环境变量指定的模块加载设置
     settings_module_path = os.environ.get(ENVVAR)
     if settings_module_path:
         settings.setmodule(settings_module_path, priority='project')
 
+    # Get all environment variables prefixed with AIOSCRAPY_
+    # 获取所有以AIOSCRAPY_为前缀的环境变量
     aioscrapy_envvars = {k[10:]: v for k, v in os.environ.items() if
                          k.startswith('AIOSCRAPY_')}
+
+    # Define which environment variables are valid and not settings
+    # 定义哪些环境变量是有效的且不是设置
     valid_envvars = {
         'CHECK',
         'PROJECT',
         'PYTHON_SHELL',
         'SETTINGS_MODULE',
     }
+
+    # Find environment variables that are being used to override settings
+    # 查找用于覆盖设置的环境变量
     setting_envvars = {k for k in aioscrapy_envvars if k not in valid_envvars}
+
+    # Warn about deprecated usage of environment variables to override settings
+    # 警告关于使用环境变量覆盖设置的已弃用用法
     if setting_envvars:
         setting_envvar_list = ', '.join(sorted(setting_envvars))
         warnings.warn(
@@ -84,6 +243,9 @@ def get_project_settings():
             f'currently defined: {setting_envvar_list}',
             AioScrapyDeprecationWarning
         )
+
+    # Apply the environment variable overrides
+    # 应用环境变量覆盖
     settings.setdict(aioscrapy_envvars, priority='project')
 
     return settings

aioscrapy/utils/python.py

@@ -1,5 +1,12 @@
 """
-This module contains essential stuff that should've come with Python itself ;)
+Python utility functions for aioscrapy.
+aioscrapy的Python实用函数。
+
+This module contains essential utility functions for Python that enhance the standard library.
+It provides functions for type conversion, string handling, regular expression searching,
+memoization, and other common operations.
+此模块包含增强标准库的Python基本实用函数。
+它提供了用于类型转换、字符串处理、正则表达式搜索、记忆化和其他常见操作的函数。
 """
 import gc
 import re
@@ -12,31 +19,72 @@ from aioscrapy.utils.decorators import deprecated
 
 def is_listlike(x):
     """
-    >>> is_listlike("foo")
-    False
-    >>> is_listlike(5)
-    False
-    >>> is_listlike(b"foo")
-    False
-    >>> is_listlike([b"foo"])
-    True
-    >>> is_listlike((b"foo",))
-    True
-    >>> is_listlike({})
-    True
-    >>> is_listlike(set())
-    True
-    >>> is_listlike((x for x in range(3)))
-    True
-    >>> is_listlike(range(5))
-    True
+    Check if the given object is list-like (iterable but not a string or bytes).
+    检查给定对象是否类似列表（可迭代但不是字符串或字节）。
+
+    This function determines if an object is iterable (has the __iter__ method)
+    but is not a string or bytes object. This is useful for functions that
+    should treat strings differently from other iterables.
+    此函数确定对象是否可迭代（具有__iter__方法）但不是字符串或字节对象。
+    这对于应该将字符串与其他可迭代对象区别对待的函数很有用。
+
+    Args:
+        x: The object to check.
+           要检查的对象。
+
+    Returns:
+        bool: True if the object is list-like, False otherwise.
+              如果对象类似列表则为True，否则为False。
+
+    Examples:
+        >>> is_listlike("foo")
+        False
+        >>> is_listlike(5)
+        False
+        >>> is_listlike(b"foo")
+        False
+        >>> is_listlike([b"foo"])
+        True
+        >>> is_listlike((b"foo",))
+        True
+        >>> is_listlike({})
+        True
+        >>> is_listlike(set())
+        True
+        >>> is_listlike((x for x in range(3)))
+        True
+        >>> is_listlike(range(5))
+        True
     """
     return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
 
 
 def to_unicode(text, encoding=None, errors='strict'):
-    """Return the unicode representation of a bytes object ``text``. If
-    ``text`` is already an unicode object, return it as-is."""
+    """
+    Convert a bytes object to a unicode (str) object.
+    将字节对象转换为unicode（str）对象。
+
+    This function converts a bytes object to a unicode string using the specified
+    encoding. If the input is already a unicode string, it is returned unchanged.
+    此函数使用指定的编码将字节对象转换为unicode字符串。
+    如果输入已经是unicode字符串，则原样返回。
+
+    Args:
+        text: The text to convert. Must be a bytes or str object.
+              要转换的文本。必须是bytes或str对象。
+        encoding: The encoding to use for decoding bytes. Defaults to 'utf-8'.
+                 用于解码字节的编码。默认为'utf-8'。
+        errors: The error handling scheme for decoding. Defaults to 'strict'.
+               解码的错误处理方案。默认为'strict'。
+
+    Returns:
+        str: The unicode representation of the input text.
+             输入文本的unicode表示。
+
+    Raises:
+        TypeError: If the input is not a bytes or str object.
+                  如果输入不是bytes或str对象。
+    """
     if isinstance(text, str):
         return text
     if not isinstance(text, (bytes, str)):
@@ -48,8 +96,31 @@ def to_unicode(text, encoding=None, errors='strict'):
 
 
 def to_bytes(text, encoding=None, errors='strict'):
-    """Return the binary representation of ``text``. If ``text``
-    is already a bytes object, return it as-is."""
+    """
+    Convert a unicode (str) object to a bytes object.
+    将unicode（str）对象转换为字节对象。
+
+    This function converts a unicode string to a bytes object using the specified
+    encoding. If the input is already a bytes object, it is returned unchanged.
+    此函数使用指定的编码将unicode字符串转换为字节对象。
+    如果输入已经是字节对象，则原样返回。
+
+    Args:
+        text: The text to convert. Must be a str or bytes object.
+              要转换的文本。必须是str或bytes对象。
+        encoding: The encoding to use for encoding the string. Defaults to 'utf-8'.
+                 用于编码字符串的编码。默认为'utf-8'。
+        errors: The error handling scheme for encoding. Defaults to 'strict'.
+               编码的错误处理方案。默认为'strict'。
+
+    Returns:
+        bytes: The binary representation of the input text.
+               输入文本的二进制表示。
+
+    Raises:
+        TypeError: If the input is not a str or bytes object.
+                  如果输入不是str或bytes对象。
+    """
     if isinstance(text, bytes):
         return text
     if not isinstance(text, str):
@@ -62,24 +133,71 @@ def to_bytes(text, encoding=None, errors='strict'):
 
 @deprecated('to_unicode')
 def to_native_str(text, encoding=None, errors='strict'):
-    """ Return str representation of ``text``. """
+    """
+    Convert text to native string type (str).
+    将文本转换为本地字符串类型（str）。
+
+    This function is deprecated. Use to_unicode() instead.
+    此函数已弃用。请改用to_unicode()。
+
+    Args:
+        text: The text to convert.
+              要转换的文本。
+        encoding: The encoding to use for decoding bytes. Defaults to 'utf-8'.
+                 用于解码字节的编码。默认为'utf-8'。
+        errors: The error handling scheme. Defaults to 'strict'.
+               错误处理方案。默认为'strict'。
+
+    Returns:
+        str: The string representation of the input text.
+             输入文本的字符串表示。
+    """
     return to_unicode(text, encoding, errors)
 
 
 def re_rsearch(pattern, text, chunk_size=1024):
     """
-    This function does a reverse search in a text using a regular expression
-    given in the attribute 'pattern'.
-    Since the re module does not provide this functionality, we have to find for
-    the expression into chunks of text extracted from the end (for the sake of efficiency).
-    At first, a chunk of 'chunk_size' kilobytes is extracted from the end, and searched for
-    the pattern. If the pattern is not found, another chunk is extracted, and another
-    search is performed.
-    This process continues until a match is found, or until the whole file is read.
-    In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
-    the start position of the match, and the ending (regarding the entire text).
-    """
+    Perform a reverse search in text using a regular expression pattern.
+    使用正则表达式模式在文本中执行反向搜索。
+
+    This function searches for the last occurrence of a pattern in a text,
+    starting from the end. Since the re module does not provide reverse search
+    functionality, this function implements it by searching in chunks from the
+    end of the text for efficiency.
+    此函数从文本末尾开始搜索模式的最后一次出现。
+    由于re模块不提供反向搜索功能，此函数通过从文本末尾的块中搜索来实现它，以提高效率。
+
+    The algorithm works as follows:
+    1. Extract a chunk of 'chunk_size' kilobytes from the end of the text
+    2. Search for the pattern in this chunk
+    3. If not found, extract another chunk further from the end and search again
+    4. Continue until a match is found or the entire text has been searched
+
+    算法工作原理如下：
+    1. 从文本末尾提取'chunk_size'千字节的块
+    2. 在此块中搜索模式
+    3. 如果未找到，从末尾进一步提取另一个块并再次搜索
+    4. 继续直到找到匹配项或已搜索整个文本
+
+    Args:
+        pattern: The regular expression pattern to search for.
+                要搜索的正则表达式模式。
+                Can be a string or a compiled regex pattern.
+                可以是字符串或已编译的正则表达式模式。
+        text: The text to search in.
+              要搜索的文本。
+        chunk_size: The size of each chunk in kilobytes. Defaults to 1024 (1MB).
+                   每个块的大小（千字节）。默认为1024（1MB）。
 
+    Returns:
+        tuple or None: If a match is found, returns a tuple (start, end) with the
+                      positions of the match in the entire text. If no match is found,
+                      returns None.
+                      如果找到匹配项，返回一个元组(start, end)，其中包含整个文本中
+                      匹配项的位置。如果未找到匹配项，返回None。
+    """
+    # Inner function to generate chunks from the end of the text
+    # 从文本末尾生成块的内部函数
     def _chunk_iter():
         offset = len(text)
         while True:
@@ -89,60 +207,152 @@ def re_rsearch(pattern, text, chunk_size=1024):
             yield (text[offset:], offset)
         yield (text, 0)
 
+    # Compile the pattern if it's a string
+    # 如果模式是字符串，则编译它
     if isinstance(pattern, str):
         pattern = re.compile(pattern)
 
+    # Search for the pattern in each chunk
+    # 在每个块中搜索模式
     for chunk, offset in _chunk_iter():
         matches = [match for match in pattern.finditer(chunk)]
         if matches:
+            # Return the position of the last match in the chunk
+            # 返回块中最后一个匹配项的位置
             start, end = matches[-1].span()
             return offset + start, offset + end
     return None
 
 
 def memoizemethod_noargs(method):
-    """Decorator to cache the result of a method (without arguments) using a
-    weak reference to its object
     """
+    Decorator to cache the result of a method with no arguments.
+    装饰器，用于缓存无参数方法的结果。
+
+    This decorator caches the result of a method call using a weak reference
+    to the object. This means the cache entry will be automatically removed
+    when the object is garbage collected.
+    此装饰器使用对对象的弱引用缓存方法调用的结果。
+    这意味着当对象被垃圾回收时，缓存条目将自动被移除。
+
+    Note that while the decorated method can accept arguments, the caching
+    is based only on the object instance, not on the arguments. This means
+    that only the first call's result will be cached, regardless of arguments.
+    请注意，虽然装饰的方法可以接受参数，但缓存仅基于对象实例，而不是参数。
+    这意味着无论参数如何，只有第一次调用的结果将被缓存。
+
+    Args:
+        method: The method to be decorated.
+               要装饰的方法。
+
+    Returns:
+        function: A new method that caches its result.
+                 缓存其结果的新方法。
+    """
+    # Create a weak key dictionary to store cached results
+    # 创建一个弱键字典来存储缓存的结果
     cache = weakref.WeakKeyDictionary()
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
+        # If the result is not cached for this object, call the method
+        # 如果此对象的结果未缓存，则调用该方法
         if self not in cache:
             cache[self] = method(self, *args, **kwargs)
+        # Return the cached result
+        # 返回缓存的结果
         return cache[self]
 
     return new_method
 
 
 def without_none_values(iterable):
-    """Return a copy of ``iterable`` with all ``None`` entries removed.
+    """
+    Return a copy of an iterable with all None entries removed.
+    返回一个去除所有None条目的可迭代对象的副本。
+
+    This function creates a new iterable of the same type as the input,
+    but with all None values removed. It handles both mappings (like dictionaries)
+    and sequences (like lists, tuples).
+    此函数创建一个与输入相同类型的新可迭代对象，但移除了所有None值。
+    它处理映射（如字典）和序列（如列表、元组）。
+
+    Args:
+        iterable: The iterable to process. Can be a mapping or a sequence.
+                 要处理的可迭代对象。可以是映射或序列。
+
+    Returns:
+        A new iterable of the same type as the input, but with all None values removed.
+        一个与输入相同类型的新可迭代对象，但移除了所有None值。
 
-    If ``iterable`` is a mapping, return a dictionary where all pairs that have
-    value ``None`` have been removed.
+    Examples:
+        >>> without_none_values({'a': 1, 'b': None, 'c': 3})
+        {'a': 1, 'c': 3}
+        >>> without_none_values([1, None, 3, None, 5])
+        [1, 3, 5]
     """
     try:
+        # Handle mappings (objects with .items() method)
+        # 处理映射（具有.items()方法的对象）
         return {k: v for k, v in iterable.items() if v is not None}
     except AttributeError:
+        # Handle sequences and other iterables
+        # 处理序列和其他可迭代对象
         return type(iterable)((v for v in iterable if v is not None))
 
 
 def global_object_name(obj):
     """
-    Return full name of a global object.
+    Return the full qualified name of a global object.
+    返回全局对象的完全限定名称。
 
-    >>> from aioscrapy import Request
-    >>> global_object_name(Request)
-    'aioscrapy.http.request.Request'
+    This function returns the full name of an object, including its module path.
+    It's useful for debugging, logging, and serialization purposes.
+    此函数返回对象的完整名称，包括其模块路径。
+    它对于调试、日志记录和序列化目的很有用。
+
+    Args:
+        obj: The object to get the name for. Must have __module__ and __name__ attributes.
+             要获取名称的对象。必须具有__module__和__name__属性。
+
+    Returns:
+        str: The full qualified name of the object in the format "module.name".
+             对象的完全限定名称，格式为"module.name"。
+
+    Examples:
+        >>> from aioscrapy import Request
+        >>> global_object_name(Request)
+        'aioscrapy.http.request.Request'
     """
     return f"{obj.__module__}.{obj.__name__}"
 
 
 if hasattr(sys, "pypy_version_info"):
     def garbage_collect():
+        """
+        Force garbage collection, with special handling for PyPy.
+        强制垃圾回收，对PyPy进行特殊处理。
+
+        On PyPy, collecting weak references can take two collection cycles,
+        so this function calls gc.collect() twice.
+        在PyPy上，收集弱引用可能需要两个收集周期，
+        因此此函数调用gc.collect()两次。
+        """
         # Collecting weakreferences can take two collections on PyPy.
+        # 在PyPy上收集弱引用可能需要两次收集。
         gc.collect()
         gc.collect()
 else:
     def garbage_collect():
+        """
+        Force garbage collection.
+        强制垃圾回收。
+
+        This function calls Python's garbage collector to force a collection cycle.
+        It's useful when you need to ensure that objects with no references are
+        properly cleaned up, especially those with __del__ methods or weak references.
+        此函数调用Python的垃圾收集器来强制进行收集周期。
+        当您需要确保没有引用的对象被正确清理时，这很有用，
+        特别是那些具有__del__方法或弱引用的对象。
+        """
         gc.collect()