crawlo 1.4.4__py3-none-any.whl → 1.4.5__py3-none-any.whl

crawlo/tools/anti_crawler.py

@@ -1,269 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 反爬虫应对工具
-"""
-
-import asyncio
-import random
-import time
-from typing import Dict, Any, Optional, List, Callable
-
-
-class ProxyPoolManager:
-    """代理池管理器类"""
-
-    def __init__(self, proxies: Optional[List[Dict[str, str]]] = None):
-        """
-        初始化代理池管理器
-        
-        Args:
-            proxies (Optional[List[Dict[str, str]]]): 代理列表
-        """
-        self.proxies = proxies or [
-            {"http": "http://proxy1.example.com:8080", "https": "https://proxy1.example.com:8080"},
-            {"http": "http://proxy2.example.com:8080", "https": "https://proxy2.example.com:8080"},
-            {"http": "http://proxy3.example.com:8080", "https": "https://proxy3.example.com:8080"}
-        ]
-        self.proxy_status = {id(proxy): {"last_used": 0, "success_count": 0, "fail_count": 0} 
-                           for proxy in self.proxies}
-
-    def get_random_proxy(self) -> Dict[str, str]:
-        """
-        获取随机代理
-        
-        Returns:
-            Dict[str, str]: 代理配置
-        """
-        return random.choice(self.proxies)
-
-    def get_best_proxy(self) -> Dict[str, str]:
-        """
-        根据成功率获取最佳代理
-        
-        Returns:
-            Dict[str, str]: 代理配置
-        """
-        if not self.proxy_status:
-            return self.get_random_proxy()
-            
-        # 计算每个代理的成功率
-        proxy_scores = []
-        for proxy in self.proxies:
-            proxy_id = id(proxy)
-            status = self.proxy_status.get(proxy_id, {"success_count": 0, "fail_count": 0})
-            total = status["success_count"] + status["fail_count"]
-            
-            if total == 0:
-                score = 0.5  # 默认成功率
-            else:
-                score = status["success_count"] / total
-                
-            proxy_scores.append((proxy, score))
-            
-        # 按成功率排序，返回成功率最高的代理
-        proxy_scores.sort(key=lambda x: x[1], reverse=True)
-        return proxy_scores[0][0]
-
-    def report_proxy_result(self, proxy: Dict[str, str], success: bool) -> None:
-        """
-        报告代理使用结果
-        
-        Args:
-            proxy (Dict[str, str]): 代理配置
-            success (bool): 是否成功
-        """
-        proxy_id = id(proxy)
-        if proxy_id not in self.proxy_status:
-            self.proxy_status[proxy_id] = {"last_used": 0, "success_count": 0, "fail_count": 0}
-            
-        status = self.proxy_status[proxy_id]
-        status["last_used"] = time.time()
-        
-        if success:
-            status["success_count"] += 1
-        else:
-            status["fail_count"] += 1
-
-    def remove_invalid_proxy(self, proxy: Dict[str, str]) -> None:
-        """
-        移除无效代理
-        
-        Args:
-            proxy (Dict[str, str]): 代理配置
-        """
-        if proxy in self.proxies:
-            self.proxies.remove(proxy)
-            proxy_id = id(proxy)
-            if proxy_id in self.proxy_status:
-                del self.proxy_status[proxy_id]
-
-
-class CaptchaHandler:
-    """验证码处理器类"""
-
-    def __init__(self, captcha_service: Optional[Callable] = None):
-        """
-        初始化验证码处理器
-        
-        Args:
-            captcha_service (Optional[Callable]): 验证码识别服务
-        """
-        self.captcha_service = captcha_service
-
-    async def recognize_captcha(self, image_data: bytes, 
-                               captcha_type: str = "image") -> Optional[str]:
-        """
-        识别验证码
-        
-        Args:
-            image_data (bytes): 验证码图片数据
-            captcha_type (str): 验证码类型
-            
-        Returns:
-            Optional[str]: 识别结果
-        """
-        if self.captcha_service:
-            try:
-                return await self.captcha_service(image_data, captcha_type)
-            except Exception:
-                return None
-        else:
-            # 如果没有配置验证码服务，返回None
-            return None
-
-    async def handle_manual_captcha(self, prompt: str = "请输入验证码: ") -> str:
-        """
-        处理手动验证码输入
-        
-        Args:
-            prompt (str): 提示信息
-            
-        Returns:
-            str: 用户输入的验证码
-        """
-        # 在实际应用中，这里可能需要与用户界面交互
-        # 为了演示目的，我们模拟用户输入
-        print(prompt)
-        return input() if not asyncio.get_event_loop().is_running() else ""
-
-
-class AntiCrawler:
-    """反爬虫应对工具类"""
-
-    def __init__(self, proxies: Optional[List[Dict[str, str]]] = None,
-                 captcha_service: Optional[Callable] = None):
-        """
-        初始化反爬虫应对工具
-        
-        Args:
-            proxies (Optional[List[Dict[str, str]]]): 代理列表
-            captcha_service (Optional[Callable]): 验证码识别服务
-        """
-        self.proxy_manager = ProxyPoolManager(proxies)
-        self.captcha_handler = CaptchaHandler(captcha_service)
-
-    def get_random_user_agent(self) -> str:
-        """
-        获取随机User-Agent
-        
-        Returns:
-            str: 随机User-Agent
-        """
-        user_agents = [
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0",
-            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59",
-            "Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Mobile Safari/537.36"
-        ]
-        return random.choice(user_agents)
-
-    def rotate_proxy(self) -> Dict[str, str]:
-        """
-        轮换代理
-        
-        Returns:
-            Dict[str, str]: 代理配置
-        """
-        return self.proxy_manager.get_best_proxy()
-
-    def handle_captcha(self, response_text: str) -> bool:
-        """
-        检测是否遇到验证码
-        
-        Args:
-            response_text (str): 响应文本
-            
-        Returns:
-            bool: 是否遇到验证码
-        """
-        captcha_keywords = ["captcha", "verify", "验证", "验证码", "human verification"]
-        return any(keyword in response_text.lower() for keyword in captcha_keywords)
-
-    def detect_rate_limiting(self, status_code: int, response_headers: Dict[str, Any]) -> bool:
-        """
-        检测是否遇到频率限制
-        
-        Args:
-            status_code (int): HTTP状态码
-            response_headers (Dict[str, Any]): 响应头
-            
-        Returns:
-            bool: 是否遇到频率限制
-        """
-        # 检查状态码
-        if status_code in [429, 503]:
-            return True
-            
-        # 检查响应头
-        rate_limit_headers = ["x-ratelimit-remaining", "retry-after", "x-ratelimit-reset"]
-        return any(header.lower() in [k.lower() for k in response_headers.keys()] 
-                  for header in rate_limit_headers)
-
-    def random_delay(self, min_delay: float = 1.0, max_delay: float = 3.0) -> None:
-        """
-        随机延迟，避免请求过于频繁
-        
-        Args:
-            min_delay (float): 最小延迟时间（秒）
-            max_delay (float): 最大延迟时间（秒）
-        """
-        delay = random.uniform(min_delay, max_delay)
-        time.sleep(delay)
-
-    async def async_random_delay(self, min_delay: float = 1.0, max_delay: float = 3.0) -> None:
-        """
-        异步随机延迟，避免请求过于频繁
-        
-        Args:
-            min_delay (float): 最小延迟时间（秒）
-            max_delay (float): 最大延迟时间（秒）
-        """
-        delay = random.uniform(min_delay, max_delay)
-        await asyncio.sleep(delay)
-
-
-# 便捷函数
-def get_random_user_agent() -> str:
-    """获取随机User-Agent"""
-    return AntiCrawler().get_random_user_agent()
-
-
-def rotate_proxy(proxies: Optional[List[Dict[str, str]]] = None) -> Dict[str, str]:
-    """轮换代理"""
-    return AntiCrawler(proxies).rotate_proxy()
-
-
-def handle_captcha(response_text: str) -> bool:
-    """检测是否遇到验证码"""
-    return AntiCrawler().handle_captcha(response_text)
-
-
-def detect_rate_limiting(status_code: int, response_headers: Dict[str, Any]) -> bool:
-    """检测是否遇到频率限制"""
-    return AntiCrawler().detect_rate_limiting(status_code, response_headers)

crawlo/utils/class_loader.py

@@ -1,26 +0,0 @@
-# -*- coding: UTF-8 -*-
-"""
-类加载器工具模块
-==============
-提供动态类加载功能，避免循环依赖问题。
-"""
-import importlib
-from typing import Any
-
-
-def load_class(path: str) -> Any:
-    """
-    动态加载类
-    
-    Args:
-        path: 类的完整路径，如 'package.module.ClassName'
-        
-    Returns:
-        加载的类对象
-    """
-    try:
-        module_path, class_name = path.rsplit('.', 1)
-        module = importlib.import_module(module_path)
-        return getattr(module, class_name)
-    except (ValueError, ImportError, AttributeError) as e:
-        raise ImportError(f"无法加载类 '{path}': {e}")

crawlo/utils/enhanced_error_handler.py

@@ -1,357 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-误处理工具
-提供更详细、更一致的错误处理和日志记录机制
-"""
-import traceback
-from datetime import datetime
-from functools import wraps
-from typing import Optional, Callable, Any, Dict, List
-
-from crawlo.utils.log import get_logger
-
-
-class ErrorContext:
-    """错误上下文信息"""
-    
-    def __init__(self, context: str = "", module: str = "", function: str = ""):
-        self.context = context
-        self.module = module
-        self.function = function
-        self.timestamp = datetime.now()
-        
-    def __str__(self):
-        parts = []
-        if self.module:
-            parts.append(f"Module: {self.module}")
-        if self.function:
-            parts.append(f"Function: {self.function}")
-        if self.context:
-            parts.append(f"Context: {self.context}")
-        parts.append(f"Time: {self.timestamp.strftime('%Y-%m-%d %H:%M:%S')}")
-        return " | ".join(parts)
-
-
-class DetailedException(Exception):
-    """带有详细信息的异常基类"""
-    
-    def __init__(self, message: str, context: Optional[ErrorContext] = None, 
-                 error_code: Optional[str] = None, **kwargs):
-        super().__init__(message)
-        self.context = context
-        self.error_code = error_code
-        self.details = kwargs
-        self.timestamp = datetime.now()
-        
-    def __str__(self):
-        base_msg = super().__str__()
-        if self.context:
-            return f"{base_msg} ({self.context})"
-        return base_msg
-    
-    def get_full_details(self) -> Dict:
-        """获取完整的错误详情"""
-        return {
-            "message": str(self),
-            "error_code": self.error_code,
-            "context": str(self.context) if self.context else None,
-            "details": self.details,
-            "timestamp": self.timestamp.isoformat(),
-            "exception_type": self.__class__.__name__
-        }
-
-
-class EnhancedErrorHandler:
-    """增强版错误处理器"""
-    
-    def __init__(self, logger_name: str = __name__, log_level: str = 'ERROR'):
-        self.logger = get_logger(logger_name, log_level)
-        self.error_history: List[Dict] = []  # 错误历史记录
-        self.max_history_size = 100  # 最大历史记录数
-    
-    def handle_error(self, exception: Exception, context: Optional[ErrorContext] = None, 
-                     raise_error: bool = True, log_error: bool = True,
-                     extra_info: Optional[Dict] = None) -> Dict:
-        """
-        增强版错误处理
-        
-        Args:
-            exception: 异常对象
-            context: 错误上下文信息
-            raise_error: 是否重新抛出异常
-            log_error: 是否记录错误日志
-            extra_info: 额外的错误信息
-            
-        Returns:
-            包含错误详情的字典
-        """
-        # 构建错误详情
-        error_details = {
-            "exception": exception,
-            "exception_type": type(exception).__name__,
-            "message": str(exception),
-            "context": str(context) if context else None,
-            "timestamp": datetime.now().isoformat(),
-            "traceback": traceback.format_exc() if log_error else None,
-            "extra_info": extra_info or {}
-        }
-        
-        # 记录到历史
-        self._record_error(error_details)
-        
-        # 记录日志
-        if log_error:
-            self._log_error(error_details)
-        
-        # 重新抛出异常
-        if raise_error:
-            raise exception
-        
-        return error_details
-    
-    def _log_error(self, error_details: Dict):
-        """记录错误日志"""
-        # 基本错误信息
-        context_info = error_details.get("context", "")
-        message = error_details["message"]
-        error_msg = f"{message} [{context_info}]" if context_info else message
-        
-        # 记录错误
-        self.logger.error(error_msg)
-        
-        # 记录详细信息
-        if error_details.get("traceback"):
-            self.logger.debug(f"详细错误信息:\n{error_details['traceback']}")
-        
-        # 记录额外信息
-        if error_details.get("extra_info"):
-            self.logger.debug(f"额外信息: {error_details['extra_info']}")
-    
-    def _record_error(self, error_details: Dict):
-        """记录错误到历史"""
-        self.error_history.append(error_details)
-        # 限制历史记录大小
-        if len(self.error_history) > self.max_history_size:
-            self.error_history.pop(0)
-    
-    def safe_call(self, func: Callable, *args, default_return=None, 
-                  context: Optional[ErrorContext] = None, **kwargs) -> Any:
-        """
-        安全调用函数，捕获并处理异常
-        
-        Args:
-            func: 要调用的函数
-            *args: 函数参数
-            default_return: 默认返回值
-            context: 错误上下文
-            **kwargs: 函数关键字参数
-            
-        Returns:
-            函数返回值或默认值
-        """
-        try:
-            return func(*args, **kwargs)
-        except Exception as e:
-            self.handle_error(e, context=context, raise_error=False)
-            return default_return
-    
-    def retry_on_failure(self, max_retries: int = 3, delay: float = 1.0, 
-                         exceptions: tuple = (Exception,), backoff_factor: float = 1.0,
-                         context: Optional[ErrorContext] = None):
-        """
-        装饰器：失败时重试（增强版）
-        
-        Args:
-            max_retries: 最大重试次数
-            delay: 初始重试间隔（秒）
-            exceptions: 需要重试的异常类型
-            backoff_factor: 退避因子（每次重试间隔乘以此因子）
-            context: 错误上下文
-        """
-        def decorator(func):
-            @wraps(func)
-            async def async_wrapper(*args, **kwargs):
-                last_exception = None
-                current_delay = delay
-                
-                for attempt in range(max_retries + 1):
-                    try:
-                        return await func(*args, **kwargs)
-                    except exceptions as e:
-                        last_exception = e
-                        if attempt < max_retries:
-                            # 记录重试信息
-                            retry_context = ErrorContext(
-                                context=f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1})",
-                                module=context.module if context else "",
-                                function=func.__name__
-                            ) if context else None
-                            
-                            self.logger.warning(
-                                f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1}): {e}"
-                            )
-                            
-                            import asyncio
-                            await asyncio.sleep(current_delay)
-                            current_delay *= backoff_factor  # 指数退避
-                        else:
-                            # 最后一次尝试失败
-                            final_context = ErrorContext(
-                                context=f"函数 {func.__name__} 执行失败，已达到最大重试次数",
-                                module=context.module if context else "",
-                                function=func.__name__
-                            ) if context else None
-                            
-                            self.logger.error(
-                                f"函数 {func.__name__} 执行失败，已达到最大重试次数: {e}"
-                            )
-                raise last_exception
-            
-            @wraps(func)
-            def sync_wrapper(*args, **kwargs):
-                last_exception = None
-                current_delay = delay
-                
-                for attempt in range(max_retries + 1):
-                    try:
-                        return func(*args, **kwargs)
-                    except exceptions as e:
-                        last_exception = e
-                        if attempt < max_retries:
-                            # 记录重试信息
-                            retry_context = ErrorContext(
-                                context=f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1})",
-                                module=context.module if context else "",
-                                function=func.__name__
-                            ) if context else None
-            
-                            self.logger.warning(
-                                f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1}): {e}"
-                            )
-                            
-                            import time
-                            time.sleep(current_delay)
-                            current_delay *= backoff_factor  # 指数退避
-                        else:
-                            # 最后一次尝试失败
-                            final_context = ErrorContext(
-                                context=f"函数 {func.__name__} 执行失败，已达到最大重试次数",
-                                module=context.module if context else "",
-                                function=func.__name__
-                            ) if context else None
-                            
-                            self.logger.error(
-                                f"函数 {func.__name__} 执行失败，已达到最大重试次数: {e}"
-                            )
-                raise last_exception
-            
-            # 根据函数是否为异步函数返回相应的包装器
-            import inspect
-            if inspect.iscoroutinefunction(func):
-                return async_wrapper
-            else:
-                return sync_wrapper
-        
-        return decorator
-    
-    def get_error_history(self) -> List[Dict]:
-        """获取错误历史记录"""
-        return self.error_history.copy()
-    
-    def clear_error_history(self):
-        """清空错误历史记录"""
-        self.error_history.clear()
-
-
-# 全局增强错误处理器实例
-enhanced_error_handler = EnhancedErrorHandler()
-
-
-def handle_exception(context: str = "", module: str = "", function: str = "",
-                     raise_error: bool = True, log_error: bool = True,
-                     error_code: Optional[str] = None):
-    """
-    装饰器：处理函数异常（增强版）
-    
-    Args:
-        context: 错误上下文描述
-        module: 模块名称
-        function: 函数名称
-        raise_error: 是否重新抛出异常
-        log_error: 是否记录错误日志
-        error_code: 错误代码
-    """
-    def decorator(func):
-        @wraps(func)
-        async def async_wrapper(*args, **kwargs):
-            try:
-                return await func(*args, **kwargs)
-            except Exception as e:
-                error_context = ErrorContext(
-                    context=f"{context} - {func.__name__}",
-                    module=module,
-                    function=func.__name__
-                )
-                
-                # 如果是详细异常，保留原有信息
-                if isinstance(e, DetailedException):
-                    # 确保上下文信息完整
-                    if not e.context:
-                        e.context = error_context
-                    enhanced_error_handler.handle_error(
-                        e, context=e.context,
-                        raise_error=raise_error, log_error=log_error
-                    )
-                else:
-                    # 包装为详细异常
-                    detailed_e = DetailedException(
-                        str(e), context=error_context, error_code=error_code
-                    )
-                    enhanced_error_handler.handle_error(
-                        detailed_e, context=error_context,
-                        raise_error=raise_error, log_error=log_error
-                    )
-                if not raise_error:
-                    return None
-        
-        @wraps(func)
-        def sync_wrapper(*args, **kwargs):
-            try:
-                return func(*args, **kwargs)
-            except Exception as e:
-                error_context = ErrorContext(
-                    context=f"{context} - {func.__name__}",
-                    module=module,
-                    function=func.__name__
-                )
-                
-                # 如果是详细异常，保留原有信息
-                if isinstance(e, DetailedException):
-                    # 确保上下文信息完整
-                    if not e.context:
-                        e.context = error_context
-                    enhanced_error_handler.handle_error(
-                        e, context=e.context,
-                        raise_error=raise_error, log_error=log_error
-                    )
-                else:
-                    # 包装为详细异常
-                    detailed_e = DetailedException(
-                        str(e), context=error_context, error_code=error_code
-                    )
-                    enhanced_error_handler.handle_error(
-                        detailed_e, context=error_context,
-                        raise_error=raise_error, log_error=log_error
-                    )
-                if not raise_error:
-                    return None
-        
-        # 根据函数是否为异步函数返回相应的包装器
-        import inspect
-        if inspect.iscoroutinefunction(func):
-            return async_wrapper
-        else:
-            return sync_wrapper
-    
-    return decorator