crawlo 1.1.9__py3-none-any.whl → 1.2.1__py3-none-any.whl

crawlo/utils/env_config.py

@@ -1,106 +1,106 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-环境变量配置工具
-提供统一的环境变量读取和配置管理机制
-"""
-import os
-from typing import Optional, Union, Any
-
-
-class EnvConfigManager:
-    """环境变量配置管理器"""
-    
-    @staticmethod
-    def get_env_var(var_name: str, default: Any = None, var_type: type = str) -> Any:
-        """
-        获取环境变量值
-        
-        Args:
-            var_name: 环境变量名称
-            default: 默认值
-            var_type: 变量类型 (str, int, float, bool)
-            
-        Returns:
-            环境变量值或默认值
-        """
-        value = os.getenv(var_name)
-        if value is None:
-            return default
-        
-        try:
-            if var_type == bool:
-                return value.lower() in ('1', 'true', 'yes', 'on')
-            elif var_type == int:
-                return int(value)
-            elif var_type == float:
-                return float(value)
-            else:
-                return value
-        except (ValueError, TypeError):
-            return default
-    
-    @staticmethod
-    def get_redis_config() -> dict:
-        """
-        获取 Redis 配置
-        
-        Returns:
-            Redis 配置字典
-        """
-        return {
-            'REDIS_HOST': EnvConfigManager.get_env_var('REDIS_HOST', '127.0.0.1', str),
-            'REDIS_PORT': EnvConfigManager.get_env_var('REDIS_PORT', 6379, int),
-            'REDIS_PASSWORD': EnvConfigManager.get_env_var('REDIS_PASSWORD', '', str),
-            'REDIS_DB': EnvConfigManager.get_env_var('REDIS_DB', 0, int),
-        }
-    
-    @staticmethod
-    def get_runtime_config() -> dict:
-        """
-        获取运行时配置
-        
-        Returns:
-            运行时配置字典
-        """
-        return {
-            'CRAWLO_MODE': EnvConfigManager.get_env_var('CRAWLO_MODE', 'standalone', str),
-            'PROJECT_NAME': EnvConfigManager.get_env_var('PROJECT_NAME', 'crawlo', str),
-            'CONCURRENCY': EnvConfigManager.get_env_var('CONCURRENCY', 8, int),
-        }
-
-
-# 便捷函数
-def get_env_var(var_name: str, default: Any = None, var_type: type = str) -> Any:
-    """
-    便捷函数：获取环境变量值
-    
-    Args:
-        var_name: 环境变量名称
-        default: 默认值
-        var_type: 变量类型 (str, int, float, bool)
-        
-    Returns:
-        环境变量值或默认值
-    """
-    return EnvConfigManager.get_env_var(var_name, default, var_type)
-
-
-def get_redis_config() -> dict:
-    """
-    便捷函数：获取 Redis 配置
-    
-    Returns:
-        Redis 配置字典
-    """
-    return EnvConfigManager.get_redis_config()
-
-
-def get_runtime_config() -> dict:
-    """
-    便捷函数：获取运行时配置
-    
-    Returns:
-        运行时配置字典
-    """
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+环境变量配置工具
+提供统一的环境变量读取和配置管理机制
+"""
+import os
+from typing import Optional, Union, Any
+
+
+class EnvConfigManager:
+    """环境变量配置管理器"""
+    
+    @staticmethod
+    def get_env_var(var_name: str, default: Any = None, var_type: type = str) -> Any:
+        """
+        获取环境变量值
+        
+        Args:
+            var_name: 环境变量名称
+            default: 默认值
+            var_type: 变量类型 (str, int, float, bool)
+            
+        Returns:
+            环境变量值或默认值
+        """
+        value = os.getenv(var_name)
+        if value is None:
+            return default
+        
+        try:
+            if var_type == bool:
+                return value.lower() in ('1', 'true', 'yes', 'on')
+            elif var_type == int:
+                return int(value)
+            elif var_type == float:
+                return float(value)
+            else:
+                return value
+        except (ValueError, TypeError):
+            return default
+    
+    @staticmethod
+    def get_redis_config() -> dict:
+        """
+        获取 Redis 配置
+        
+        Returns:
+            Redis 配置字典
+        """
+        return {
+            'REDIS_HOST': EnvConfigManager.get_env_var('REDIS_HOST', '127.0.0.1', str),
+            'REDIS_PORT': EnvConfigManager.get_env_var('REDIS_PORT', 6379, int),
+            'REDIS_PASSWORD': EnvConfigManager.get_env_var('REDIS_PASSWORD', '', str),
+            'REDIS_DB': EnvConfigManager.get_env_var('REDIS_DB', 0, int),
+        }
+    
+    @staticmethod
+    def get_runtime_config() -> dict:
+        """
+        获取运行时配置
+        
+        Returns:
+            运行时配置字典
+        """
+        return {
+            'CRAWLO_MODE': EnvConfigManager.get_env_var('CRAWLO_MODE', 'standalone', str),
+            'PROJECT_NAME': EnvConfigManager.get_env_var('PROJECT_NAME', 'crawlo', str),
+            'CONCURRENCY': EnvConfigManager.get_env_var('CONCURRENCY', 8, int),
+        }
+
+
+# 便捷函数
+def get_env_var(var_name: str, default: Any = None, var_type: type = str) -> Any:
+    """
+    便捷函数：获取环境变量值
+    
+    Args:
+        var_name: 环境变量名称
+        default: 默认值
+        var_type: 变量类型 (str, int, float, bool)
+        
+    Returns:
+        环境变量值或默认值
+    """
+    return EnvConfigManager.get_env_var(var_name, default, var_type)
+
+
+def get_redis_config() -> dict:
+    """
+    便捷函数：获取 Redis 配置
+    
+    Returns:
+        Redis 配置字典
+    """
+    return EnvConfigManager.get_redis_config()
+
+
+def get_runtime_config() -> dict:
+    """
+    便捷函数：获取运行时配置
+    
+    Returns:
+        运行时配置字典
+    """
     return EnvConfigManager.get_runtime_config()

crawlo/utils/error_handler.py

@@ -1,126 +1,126 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-统一错误处理工具
-提供一致的错误处理和日志记录机制
-"""
-import traceback
-import logging
-from typing import Optional, Callable, Any
-from functools import wraps
-
-from crawlo.utils.log import get_logger
-from crawlo.utils.enhanced_error_handler import EnhancedErrorHandler, ErrorContext
-
-
-class ErrorHandler:
-    """统一错误处理器（简化版，使用增强版作为后端实现）"""
-    
-    def __init__(self, logger_name: str = __name__, log_level: str = 'ERROR'):
-        self.logger = get_logger(logger_name, log_level)
-        # 使用增强版错误处理器作为后端
-        self._enhanced_handler = EnhancedErrorHandler(logger_name, log_level)
-    
-    def handle_error(self, exception: Exception, context: str = "", 
-                     raise_error: bool = True, log_error: bool = True) -> None:
-        """
-        统一处理错误
-        
-        Args:
-            exception: 异常对象
-            context: 错误上下文描述
-            raise_error: 是否重新抛出异常
-            log_error: 是否记录错误日志
-        """
-        # 转换为增强版错误上下文
-        error_context = ErrorContext(context=context) if context else None
-        self._enhanced_handler.handle_error(
-            exception, context=error_context, 
-            raise_error=raise_error, log_error=log_error
-        )
-    
-    def safe_call(self, func: Callable, *args, default_return=None, 
-                  context: str = "", **kwargs) -> Any:
-        """
-        安全调用函数，捕获并处理异常
-        
-        Args:
-            func: 要调用的函数
-            *args: 函数参数
-            default_return: 默认返回值
-            context: 错误上下文描述
-            **kwargs: 函数关键字参数
-            
-        Returns:
-            函数返回值或默认值
-        """
-        error_context = ErrorContext(context=context) if context else None
-        return self._enhanced_handler.safe_call(
-            func, *args, default_return=default_return, 
-            context=error_context, **kwargs
-        )
-    
-    def retry_on_failure(self, max_retries: int = 3, delay: float = 1.0, 
-                         exceptions: tuple = (Exception,)):
-        """
-        装饰器：失败时重试
-        
-        Args:
-            max_retries: 最大重试次数
-            delay: 重试间隔（秒）
-            exceptions: 需要重试的异常类型
-        """
-        def decorator(func):
-            # 直接使用增强版处理器的重试装饰器
-            return self._enhanced_handler.retry_on_failure(
-                max_retries=max_retries, delay=delay, exceptions=exceptions
-            )(func)
-        return decorator
-
-
-# 全局错误处理器实例
-default_error_handler = ErrorHandler()
-
-
-def handle_exception(context: str = "", raise_error: bool = True, log_error: bool = True):
-    """
-    装饰器：处理函数异常
-    
-    Args:
-        context: 错误上下文描述
-        raise_error: 是否重新抛出异常
-        log_error: 是否记录错误日志
-    """
-    def decorator(func):
-        @wraps(func)
-        async def async_wrapper(*args, **kwargs):
-            try:
-                return await func(*args, **kwargs)
-            except Exception as e:
-                default_error_handler.handle_error(
-                    e, context=f"{context} - {func.__name__}", 
-                    raise_error=raise_error, log_error=log_error
-                )
-                if not raise_error:
-                    return None
-        
-        @wraps(func)
-        def sync_wrapper(*args, **kwargs):
-            try:
-                return func(*args, **kwargs)
-            except Exception as e:
-                default_error_handler.handle_error(
-                    e, context=f"{context} - {func.__name__}", 
-                    raise_error=raise_error, log_error=log_error
-                )
-                if not raise_error:
-                    return None
-        
-        # 根据函数是否为异步函数返回相应的包装器
-        import inspect
-        if inspect.iscoroutinefunction(func):
-            return async_wrapper
-        else:
-            return sync_wrapper
-    
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+统一错误处理工具
+提供一致的错误处理和日志记录机制
+"""
+import traceback
+import logging
+from typing import Optional, Callable, Any
+from functools import wraps
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.enhanced_error_handler import EnhancedErrorHandler, ErrorContext
+
+
+class ErrorHandler:
+    """统一错误处理器（简化版，使用增强版作为后端实现）"""
+    
+    def __init__(self, logger_name: str = __name__, log_level: str = 'ERROR'):
+        self.logger = get_logger(logger_name, log_level)
+        # 使用增强版错误处理器作为后端
+        self._enhanced_handler = EnhancedErrorHandler(logger_name, log_level)
+    
+    def handle_error(self, exception: Exception, context: str = "", 
+                     raise_error: bool = True, log_error: bool = True) -> None:
+        """
+        统一处理错误
+        
+        Args:
+            exception: 异常对象
+            context: 错误上下文描述
+            raise_error: 是否重新抛出异常
+            log_error: 是否记录错误日志
+        """
+        # 转换为增强版错误上下文
+        error_context = ErrorContext(context=context) if context else None
+        self._enhanced_handler.handle_error(
+            exception, context=error_context, 
+            raise_error=raise_error, log_error=log_error
+        )
+    
+    def safe_call(self, func: Callable, *args, default_return=None, 
+                  context: str = "", **kwargs) -> Any:
+        """
+        安全调用函数，捕获并处理异常
+        
+        Args:
+            func: 要调用的函数
+            *args: 函数参数
+            default_return: 默认返回值
+            context: 错误上下文描述
+            **kwargs: 函数关键字参数
+            
+        Returns:
+            函数返回值或默认值
+        """
+        error_context = ErrorContext(context=context) if context else None
+        return self._enhanced_handler.safe_call(
+            func, *args, default_return=default_return, 
+            context=error_context, **kwargs
+        )
+    
+    def retry_on_failure(self, max_retries: int = 3, delay: float = 1.0, 
+                         exceptions: tuple = (Exception,)):
+        """
+        装饰器：失败时重试
+        
+        Args:
+            max_retries: 最大重试次数
+            delay: 重试间隔（秒）
+            exceptions: 需要重试的异常类型
+        """
+        def decorator(func):
+            # 直接使用增强版处理器的重试装饰器
+            return self._enhanced_handler.retry_on_failure(
+                max_retries=max_retries, delay=delay, exceptions=exceptions
+            )(func)
+        return decorator
+
+
+# 全局错误处理器实例
+default_error_handler = ErrorHandler()
+
+
+def handle_exception(context: str = "", raise_error: bool = True, log_error: bool = True):
+    """
+    装饰器：处理函数异常
+    
+    Args:
+        context: 错误上下文描述
+        raise_error: 是否重新抛出异常
+        log_error: 是否记录错误日志
+    """
+    def decorator(func):
+        @wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            try:
+                return await func(*args, **kwargs)
+            except Exception as e:
+                default_error_handler.handle_error(
+                    e, context=f"{context} - {func.__name__}", 
+                    raise_error=raise_error, log_error=log_error
+                )
+                if not raise_error:
+                    return None
+        
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                default_error_handler.handle_error(
+                    e, context=f"{context} - {func.__name__}", 
+                    raise_error=raise_error, log_error=log_error
+                )
+                if not raise_error:
+                    return None
+        
+        # 根据函数是否为异步函数返回相应的包装器
+        import inspect
+        if inspect.iscoroutinefunction(func):
+            return async_wrapper
+        else:
+            return sync_wrapper
+    
     return decorator

crawlo/utils/func_tools.py

@@ -1,82 +1,82 @@
-# -*- coding: UTF-8 -*-
-from typing import Union, AsyncGenerator, Generator
-from inspect import isgenerator, isasyncgen
-from crawlo import Response, Request, Item
-from crawlo.exceptions import TransformTypeError
-
-T = Union[Request, Item]
-
-
-async def transform(
-        func: Union[Generator[T, None, None], AsyncGenerator[T, None]],
-        response: Response
-) -> AsyncGenerator[Union[T, Exception], None]:
-    """
-    转换回调函数的输出为统一异步生成器
-
-    Args:
-        func: 同步或异步生成器函数
-        response: 当前响应对象
-
-    Yields:
-        Union[T, Exception]: 生成请求/Item或异常对象
-
-    Raises:
-        TransformTypeError: 当输入类型不符合要求时
-    """
-
-    def _set_meta(obj: T) -> T:
-        """统一设置请求的depth元数据"""
-        if isinstance(obj, Request):
-            obj.meta.setdefault('depth', response.meta.get('depth', 0))
-        return obj
-
-    # 类型检查前置
-    if not (isgenerator(func) or isasyncgen(func)):
-        raise TransformTypeError(
-            f'Callback must return generator or async generator, got {type(func).__name__}'
-        )
-
-    try:
-        if isgenerator(func):
-            # 同步生成器处理
-            for item in func:
-                yield _set_meta(item)
-        else:
-            # 异步生成器处理
-            async for item in func:
-                yield _set_meta(item)
-
-    except Exception as e:
-        yield e
-
-# #!/usr/bin/python
-# # -*- coding:UTF-8 -*-
-# from typing import Callable, Union
-# from inspect import isgenerator, isasyncgen
-# from crawlo import Response, Request, Item
-# from crawlo.exceptions import TransformTypeError
-#
-#
-# T = Union[Request, Item]
-#
-#
-# async def transform(func: Callable, response: Response):
-#     def set_request(t: T) -> T:
-#         if isinstance(t, Request):
-#             t.meta['depth'] = response.meta['depth']
-#         return t
-#     try:
-#         if isgenerator(func):
-#             for f in func:
-#                 yield set_request(f)
-#         elif isasyncgen(func):
-#             async for f in func:
-#                 yield set_request(f)
-#         else:
-#             raise TransformTypeError(
-#                 f'callback return type error: {type(func)} must be `generator` or `async generator`'
-#             )
-#     except Exception as exp:
-#         yield exp
-
+# -*- coding: UTF-8 -*-
+from typing import Union, AsyncGenerator, Generator
+from inspect import isgenerator, isasyncgen
+from crawlo import Response, Request, Item
+from crawlo.exceptions import TransformTypeError
+
+T = Union[Request, Item]
+
+
+async def transform(
+        func: Union[Generator[T, None, None], AsyncGenerator[T, None]],
+        response: Response
+) -> AsyncGenerator[Union[T, Exception], None]:
+    """
+    转换回调函数的输出为统一异步生成器
+
+    Args:
+        func: 同步或异步生成器函数
+        response: 当前响应对象
+
+    Yields:
+        Union[T, Exception]: 生成请求/Item或异常对象
+
+    Raises:
+        TransformTypeError: 当输入类型不符合要求时
+    """
+
+    def _set_meta(obj: T) -> T:
+        """统一设置请求的depth元数据"""
+        if isinstance(obj, Request):
+            obj.meta.setdefault('depth', response.meta.get('depth', 0))
+        return obj
+
+    # 类型检查前置
+    if not (isgenerator(func) or isasyncgen(func)):
+        raise TransformTypeError(
+            f'Callback must return generator or async generator, got {type(func).__name__}'
+        )
+
+    try:
+        if isgenerator(func):
+            # 同步生成器处理
+            for item in func:
+                yield _set_meta(item)
+        else:
+            # 异步生成器处理
+            async for item in func:
+                yield _set_meta(item)
+
+    except Exception as e:
+        yield e
+
+# #!/usr/bin/python
+# # -*- coding:UTF-8 -*-
+# from typing import Callable, Union
+# from inspect import isgenerator, isasyncgen
+# from crawlo import Response, Request, Item
+# from crawlo.exceptions import TransformTypeError
+#
+#
+# T = Union[Request, Item]
+#
+#
+# async def transform(func: Callable, response: Response):
+#     def set_request(t: T) -> T:
+#         if isinstance(t, Request):
+#             t.meta['depth'] = response.meta['depth']
+#         return t
+#     try:
+#         if isgenerator(func):
+#             for f in func:
+#                 yield set_request(f)
+#         elif isasyncgen(func):
+#             async for f in func:
+#                 yield set_request(f)
+#         else:
+#             raise TransformTypeError(
+#                 f'callback return type error: {type(func)} must be `generator` or `async generator`'
+#             )
+#     except Exception as exp:
+#         yield exp
+