crawlo 1.4.4__py3-none-any.whl → 1.4.6__py3-none-any.whl

crawlo/utils/enhanced_error_handler.py

@@ -1,357 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-误处理工具
-提供更详细、更一致的错误处理和日志记录机制
-"""
-import traceback
-from datetime import datetime
-from functools import wraps
-from typing import Optional, Callable, Any, Dict, List
-
-from crawlo.utils.log import get_logger
-
-
-class ErrorContext:
-    """错误上下文信息"""
-    
-    def __init__(self, context: str = "", module: str = "", function: str = ""):
-        self.context = context
-        self.module = module
-        self.function = function
-        self.timestamp = datetime.now()
-        
-    def __str__(self):
-        parts = []
-        if self.module:
-            parts.append(f"Module: {self.module}")
-        if self.function:
-            parts.append(f"Function: {self.function}")
-        if self.context:
-            parts.append(f"Context: {self.context}")
-        parts.append(f"Time: {self.timestamp.strftime('%Y-%m-%d %H:%M:%S')}")
-        return " | ".join(parts)
-
-
-class DetailedException(Exception):
-    """带有详细信息的异常基类"""
-    
-    def __init__(self, message: str, context: Optional[ErrorContext] = None, 
-                 error_code: Optional[str] = None, **kwargs):
-        super().__init__(message)
-        self.context = context
-        self.error_code = error_code
-        self.details = kwargs
-        self.timestamp = datetime.now()
-        
-    def __str__(self):
-        base_msg = super().__str__()
-        if self.context:
-            return f"{base_msg} ({self.context})"
-        return base_msg
-    
-    def get_full_details(self) -> Dict:
-        """获取完整的错误详情"""
-        return {
-            "message": str(self),
-            "error_code": self.error_code,
-            "context": str(self.context) if self.context else None,
-            "details": self.details,
-            "timestamp": self.timestamp.isoformat(),
-            "exception_type": self.__class__.__name__
-        }
-
-
-class EnhancedErrorHandler:
-    """增强版错误处理器"""
-    
-    def __init__(self, logger_name: str = __name__, log_level: str = 'ERROR'):
-        self.logger = get_logger(logger_name, log_level)
-        self.error_history: List[Dict] = []  # 错误历史记录
-        self.max_history_size = 100  # 最大历史记录数
-    
-    def handle_error(self, exception: Exception, context: Optional[ErrorContext] = None, 
-                     raise_error: bool = True, log_error: bool = True,
-                     extra_info: Optional[Dict] = None) -> Dict:
-        """
-        增强版错误处理
-        
-        Args:
-            exception: 异常对象
-            context: 错误上下文信息
-            raise_error: 是否重新抛出异常
-            log_error: 是否记录错误日志
-            extra_info: 额外的错误信息
-            
-        Returns:
-            包含错误详情的字典
-        """
-        # 构建错误详情
-        error_details = {
-            "exception": exception,
-            "exception_type": type(exception).__name__,
-            "message": str(exception),
-            "context": str(context) if context else None,
-            "timestamp": datetime.now().isoformat(),
-            "traceback": traceback.format_exc() if log_error else None,
-            "extra_info": extra_info or {}
-        }
-        
-        # 记录到历史
-        self._record_error(error_details)
-        
-        # 记录日志
-        if log_error:
-            self._log_error(error_details)
-        
-        # 重新抛出异常
-        if raise_error:
-            raise exception
-        
-        return error_details
-    
-    def _log_error(self, error_details: Dict):
-        """记录错误日志"""
-        # 基本错误信息
-        context_info = error_details.get("context", "")
-        message = error_details["message"]
-        error_msg = f"{message} [{context_info}]" if context_info else message
-        
-        # 记录错误
-        self.logger.error(error_msg)
-        
-        # 记录详细信息
-        if error_details.get("traceback"):
-            self.logger.debug(f"详细错误信息:\n{error_details['traceback']}")
-        
-        # 记录额外信息
-        if error_details.get("extra_info"):
-            self.logger.debug(f"额外信息: {error_details['extra_info']}")
-    
-    def _record_error(self, error_details: Dict):
-        """记录错误到历史"""
-        self.error_history.append(error_details)
-        # 限制历史记录大小
-        if len(self.error_history) > self.max_history_size:
-            self.error_history.pop(0)
-    
-    def safe_call(self, func: Callable, *args, default_return=None, 
-                  context: Optional[ErrorContext] = None, **kwargs) -> Any:
-        """
-        安全调用函数，捕获并处理异常
-        
-        Args:
-            func: 要调用的函数
-            *args: 函数参数
-            default_return: 默认返回值
-            context: 错误上下文
-            **kwargs: 函数关键字参数
-            
-        Returns:
-            函数返回值或默认值
-        """
-        try:
-            return func(*args, **kwargs)
-        except Exception as e:
-            self.handle_error(e, context=context, raise_error=False)
-            return default_return
-    
-    def retry_on_failure(self, max_retries: int = 3, delay: float = 1.0, 
-                         exceptions: tuple = (Exception,), backoff_factor: float = 1.0,
-                         context: Optional[ErrorContext] = None):
-        """
-        装饰器：失败时重试（增强版）
-        
-        Args:
-            max_retries: 最大重试次数
-            delay: 初始重试间隔（秒）
-            exceptions: 需要重试的异常类型
-            backoff_factor: 退避因子（每次重试间隔乘以此因子）
-            context: 错误上下文
-        """
-        def decorator(func):
-            @wraps(func)
-            async def async_wrapper(*args, **kwargs):
-                last_exception = None
-                current_delay = delay
-                
-                for attempt in range(max_retries + 1):
-                    try:
-                        return await func(*args, **kwargs)
-                    except exceptions as e:
-                        last_exception = e
-                        if attempt < max_retries:
-                            # 记录重试信息
-                            retry_context = ErrorContext(
-                                context=f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1})",
-                                module=context.module if context else "",
-                                function=func.__name__
-                            ) if context else None
-                            
-                            self.logger.warning(
-                                f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1}): {e}"
-                            )
-                            
-                            import asyncio
-                            await asyncio.sleep(current_delay)
-                            current_delay *= backoff_factor  # 指数退避
-                        else:
-                            # 最后一次尝试失败
-                            final_context = ErrorContext(
-                                context=f"函数 {func.__name__} 执行失败，已达到最大重试次数",
-                                module=context.module if context else "",
-                                function=func.__name__
-                            ) if context else None
-                            
-                            self.logger.error(
-                                f"函数 {func.__name__} 执行失败，已达到最大重试次数: {e}"
-                            )
-                raise last_exception
-            
-            @wraps(func)
-            def sync_wrapper(*args, **kwargs):
-                last_exception = None
-                current_delay = delay
-                
-                for attempt in range(max_retries + 1):
-                    try:
-                        return func(*args, **kwargs)
-                    except exceptions as e:
-                        last_exception = e
-                        if attempt < max_retries:
-                            # 记录重试信息
-                            retry_context = ErrorContext(
-                                context=f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1})",
-                                module=context.module if context else "",
-                                function=func.__name__
-                            ) if context else None
-            
-                            self.logger.warning(
-                                f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1}): {e}"
-                            )
-                            
-                            import time
-                            time.sleep(current_delay)
-                            current_delay *= backoff_factor  # 指数退避
-                        else:
-                            # 最后一次尝试失败
-                            final_context = ErrorContext(
-                                context=f"函数 {func.__name__} 执行失败，已达到最大重试次数",
-                                module=context.module if context else "",
-                                function=func.__name__
-                            ) if context else None
-                            
-                            self.logger.error(
-                                f"函数 {func.__name__} 执行失败，已达到最大重试次数: {e}"
-                            )
-                raise last_exception
-            
-            # 根据函数是否为异步函数返回相应的包装器
-            import inspect
-            if inspect.iscoroutinefunction(func):
-                return async_wrapper
-            else:
-                return sync_wrapper
-        
-        return decorator
-    
-    def get_error_history(self) -> List[Dict]:
-        """获取错误历史记录"""
-        return self.error_history.copy()
-    
-    def clear_error_history(self):
-        """清空错误历史记录"""
-        self.error_history.clear()
-
-
-# 全局增强错误处理器实例
-enhanced_error_handler = EnhancedErrorHandler()
-
-
-def handle_exception(context: str = "", module: str = "", function: str = "",
-                     raise_error: bool = True, log_error: bool = True,
-                     error_code: Optional[str] = None):
-    """
-    装饰器：处理函数异常（增强版）
-    
-    Args:
-        context: 错误上下文描述
-        module: 模块名称
-        function: 函数名称
-        raise_error: 是否重新抛出异常
-        log_error: 是否记录错误日志
-        error_code: 错误代码
-    """
-    def decorator(func):
-        @wraps(func)
-        async def async_wrapper(*args, **kwargs):
-            try:
-                return await func(*args, **kwargs)
-            except Exception as e:
-                error_context = ErrorContext(
-                    context=f"{context} - {func.__name__}",
-                    module=module,
-                    function=func.__name__
-                )
-                
-                # 如果是详细异常，保留原有信息
-                if isinstance(e, DetailedException):
-                    # 确保上下文信息完整
-                    if not e.context:
-                        e.context = error_context
-                    enhanced_error_handler.handle_error(
-                        e, context=e.context,
-                        raise_error=raise_error, log_error=log_error
-                    )
-                else:
-                    # 包装为详细异常
-                    detailed_e = DetailedException(
-                        str(e), context=error_context, error_code=error_code
-                    )
-                    enhanced_error_handler.handle_error(
-                        detailed_e, context=error_context,
-                        raise_error=raise_error, log_error=log_error
-                    )
-                if not raise_error:
-                    return None
-        
-        @wraps(func)
-        def sync_wrapper(*args, **kwargs):
-            try:
-                return func(*args, **kwargs)
-            except Exception as e:
-                error_context = ErrorContext(
-                    context=f"{context} - {func.__name__}",
-                    module=module,
-                    function=func.__name__
-                )
-                
-                # 如果是详细异常，保留原有信息
-                if isinstance(e, DetailedException):
-                    # 确保上下文信息完整
-                    if not e.context:
-                        e.context = error_context
-                    enhanced_error_handler.handle_error(
-                        e, context=e.context,
-                        raise_error=raise_error, log_error=log_error
-                    )
-                else:
-                    # 包装为详细异常
-                    detailed_e = DetailedException(
-                        str(e), context=error_context, error_code=error_code
-                    )
-                    enhanced_error_handler.handle_error(
-                        detailed_e, context=error_context,
-                        raise_error=raise_error, log_error=log_error
-                    )
-                if not raise_error:
-                    return None
-        
-        # 根据函数是否为异步函数返回相应的包装器
-        import inspect
-        if inspect.iscoroutinefunction(func):
-            return async_wrapper
-        else:
-            return sync_wrapper
-    
-    return decorator

crawlo-1.4.4.dist-info/METADATA

@@ -1,190 +0,0 @@
-Metadata-Version: 2.4
-Name: crawlo
-Version: 1.4.4
-Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
-Home-page: https://github.com/crawl-coder/Crawlo.git
-Author: crawl-coder
-Author-email: crawlo@qq.com
-License: MIT
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.6
-Description-Content-Type: text/markdown
-Requires-Dist: aiohttp>=3.12.14
-Requires-Dist: aiomysql>=0.2.0
-Requires-Dist: aioredis>=2.0.1
-Requires-Dist: asyncmy>=0.2.10
-Requires-Dist: cssselect>=1.2.0
-Requires-Dist: dateparser>=1.2.2
-Requires-Dist: httpx[http2]>=0.27.0
-Requires-Dist: curl-cffi>=0.13.0
-Requires-Dist: lxml>=5.2.1
-Requires-Dist: motor>=3.7.0
-Requires-Dist: parsel>=1.9.1
-Requires-Dist: pydantic>=2.11.7
-Requires-Dist: pymongo>=4.11
-Requires-Dist: PyMySQL>=1.1.1
-Requires-Dist: python-dateutil>=2.9.0.post0
-Requires-Dist: redis>=6.2.0
-Requires-Dist: requests>=2.32.4
-Requires-Dist: six>=1.17.0
-Requires-Dist: ujson>=5.9.0
-Requires-Dist: urllib3>=2.5.0
-Requires-Dist: w3lib>=2.1.2
-Requires-Dist: rich>=14.1.0
-Requires-Dist: astor>=0.8.1
-Requires-Dist: watchdog>=6.0.0
-Provides-Extra: render
-Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
-Requires-Dist: playwright; extra == "render"
-Requires-Dist: selenium>=3.141.0; extra == "render"
-Provides-Extra: all
-Requires-Dist: bitarray>=1.5.3; extra == "all"
-Requires-Dist: PyExecJS>=1.5.1; extra == "all"
-Requires-Dist: pymongo>=3.10.1; extra == "all"
-Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
-Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
-Requires-Dist: playwright; extra == "all"
-Requires-Dist: selenium>=3.141.0; extra == "all"
-
-# Crawlo 爬虫框架
-
-Crawlo 是一个高性能、可扩展的 Python 爬虫框架，支持单机和分布式部署。
-
-## 特性
-
-- 高性能异步爬取
-- 支持多种下载器 (aiohttp, httpx, curl-cffi)
-- 内置数据清洗和验证
-- 分布式爬取支持
-- 灵活的中间件系统
-- 强大的配置管理系统
-- 详细的日志记录和监控
-- Windows 和 Linux 兼容
-
-## 安装
-
-```bash
-pip install crawlo
-```
-
-或者从源码安装：
-
-```bash
-git clone https://github.com/your-username/crawlo.git
-cd crawlo
-pip install -r requirements.txt
-pip install .
-```
-
-## 快速开始
-
-```python
-from crawlo import Spider
-
-class MySpider(Spider):
-    name = 'example'
-    
-    def parse(self, response):
-        # 解析逻辑
-        pass
-
-# 运行爬虫
-# crawlo run example
-```
-
-## 日志系统
-
-Crawlo 拥有一个功能强大的日志系统，支持多种配置选项：
-
-### 基本配置
-
-```python
-from crawlo.logging import configure_logging, get_logger
-
-# 配置日志系统
-configure_logging(
-    LOG_LEVEL='INFO',
-    LOG_FILE='logs/app.log',
-    LOG_MAX_BYTES=10*1024*1024,  # 10MB
-    LOG_BACKUP_COUNT=5
-)
-
-# 获取logger
-logger = get_logger('my_module')
-logger.info('这是一条日志消息')
-```
-
-### 高级配置
-
-```python
-# 分别配置控制台和文件日志级别
-configure_logging(
-    LOG_LEVEL='INFO',
-    LOG_CONSOLE_LEVEL='WARNING',  # 控制台只显示WARNING及以上级别
-    LOG_FILE_LEVEL='DEBUG',       # 文件记录DEBUG及以上级别
-    LOG_FILE='logs/app.log',
-    LOG_INCLUDE_THREAD_ID=True,   # 包含线程ID
-    LOG_INCLUDE_PROCESS_ID=True   # 包含进程ID
-)
-
-# 模块特定日志级别
-configure_logging(
-    LOG_LEVEL='WARNING',
-    LOG_LEVELS={
-        'my_module.debug': 'DEBUG',
-        'my_module.info': 'INFO'
-    }
-)
-```
-
-### 性能监控
-
-```python
-from crawlo.logging import get_monitor
-
-# 启用日志性能监控
-monitor = get_monitor()
-monitor.enable_monitoring()
-
-# 获取性能报告
-report = monitor.get_performance_report()
-print(report)
-```
-
-### 日志采样
-
-```python
-from crawlo.logging import get_sampler
-
-# 设置采样率（只记录30%的日志）
-sampler = get_sampler()
-sampler.set_sample_rate('my_module', 0.3)
-
-# 设置速率限制（每秒最多100条日志）
-sampler.set_rate_limit('my_module', 100)
-```
-
-## Windows 兼容性说明
-
-在 Windows 系统上使用日志轮转功能时，可能会遇到文件锁定问题。为了解决这个问题，建议安装 `concurrent-log-handler` 库：
-
-```bash
-pip install concurrent-log-handler
-```
-
-Crawlo 框架会自动检测并使用这个库来提供更好的 Windows 兼容性。
-
-如果未安装 `concurrent-log-handler`，在 Windows 上运行时可能会出现以下错误：
-```
-PermissionError: [WinError 32] 另一个程序正在使用此文件，进程无法访问。
-```
-
-## 文档
-
-请查看 [文档](https://your-docs-url.com) 获取更多信息。
-
-## 许可证
-
-MIT

tests/simple_log_test.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-"""
-简单的日志系统测试
-"""
-import sys
-import os
-sys.path.insert(0, '/')
-
-# 确保日志目录存在
-os.makedirs('/examples/ofweek_standalone/logs', exist_ok=True)
-
-# 测试日志系统
-from crawlo.utils.log import LoggerManager, get_logger
-
-print("=== 简单日志系统测试 ===")
-
-# 1. 直接配置日志系统
-print("1. 配置日志系统...")
-LoggerManager.configure(
-    LOG_LEVEL='INFO',
-    LOG_FILE='/Users/oscar/projects/Crawlo/examples/ofweek_standalone/logs/simple_test.log'
-)
-
-# 2. 创建logger
-print("2. 创建logger...")
-logger = get_logger('test.logger')
-print(f"   Logger: {logger}")
-print(f"   Handlers: {len(logger.handlers)}")
-
-for i, handler in enumerate(logger.handlers):
-    handler_type = type(handler).__name__
-    print(f"     Handler {i}: {handler_type}")
-    if hasattr(handler, 'baseFilename'):
-        print(f"       File: {handler.baseFilename}")
-
-# 3. 测试日志输出
-print("3. 测试日志输出...")
-logger.info("这是一条测试INFO日志")
-logger.debug("这是一条测试DEBUG日志")
-logger.warning("这是一条测试WARNING日志")
-
-print("4. 检查日志文件...")
-log_file = '/Users/oscar/projects/Crawlo/examples/ofweek_standalone/logs/simple_test.log'
-if os.path.exists(log_file):
-    print(f"   日志文件存在: {log_file}")
-    with open(log_file, 'r', encoding='utf-8') as f:
-        content = f.read()
-        print(f"   文件内容长度: {len(content)} 字符")
-        if content:
-            print("   文件内容:")
-            print(content)
-        else:
-            print("   文件为空")
-else:
-    print(f"   日志文件不存在: {log_file}")
-
-print("=== 测试完成 ===")

tests/simple_test.py

@@ -1,48 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-简化的框架测试
-"""
-import os
-import sys
-sys.path.insert(0, '/')
-
-# 设置基本配置
-test_log_file = '/Users/oscar/projects/Crawlo/simple_test.log'
-if os.path.exists(test_log_file):
-    os.remove(test_log_file)
-
-# 最简单的测试
-try:
-    from crawlo.utils.log import LoggerManager
-    
-    print("配置日志系统...")
-    LoggerManager.configure(
-        LOG_LEVEL='INFO',
-        LOG_FILE=test_log_file
-    )
-    
-    from crawlo.utils.log import get_logger
-    logger = get_logger('test.simple')
-    
-    print("测试日志输出...")
-    logger.info("这是一条测试信息")
-    logger.info("Crawlo框架初始化完成")
-    logger.info("Crawlo Framework Started 1.3.3")
-    
-    print("检查日志文件...")
-    if os.path.exists(test_log_file):
-        with open(test_log_file, 'r', encoding='utf-8') as f:
-            content = f.read()
-            print(f"日志文件内容: {len(content)} 字符")
-            print("内容:")
-            print(content)
-    else:
-        print("日志文件未创建")
-        
-except Exception as e:
-    print(f"错误: {e}")
-    import traceback
-    traceback.print_exc()
-
-print("测试完成")