crawlo 1.4.5__py3-none-any.whl → 1.4.7__py3-none-any.whl

crawlo/tools/request_tools.py

@@ -1,83 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-20 10:00
-# @Author  : crawl-coder
-# @Desc    : 请求处理工具
-"""
-from typing import Dict, Any, Optional
-from urllib.parse import urlparse, parse_qs, urlencode, urlunparse
-from copy import deepcopy
-
-
-def build_url(base_url: str, path: str = "", query_params: Optional[Dict[str, Any]] = None) -> str:
-    """
-    构建完整的URL
-    
-    :param base_url: 基础URL
-    :param path: 路径
-    :param query_params: 查询参数字典
-    :return: 完整的URL
-    """
-    # 解析基础URL
-    parsed = urlparse(base_url)
-    
-    # 合并路径
-    if path:
-        # 确保路径以/开头
-        if not path.startswith('/'):
-            path = '/' + path
-        # 合并路径
-        new_path = parsed.path.rstrip('/') + path
-    else:
-        new_path = parsed.path
-    
-    # 处理查询参数
-    if query_params:
-        # 解析现有查询参数
-        existing_params = parse_qs(parsed.query)
-        # 合并新参数
-        for key, value in query_params.items():
-            existing_params[key] = [str(value)]
-        # 编码查询参数
-        new_query = urlencode(existing_params, doseq=True)
-    else:
-        new_query = parsed.query
-    
-    # 构建新的URL
-    new_parsed = parsed._replace(path=new_path, query=new_query)
-    return urlunparse(new_parsed)
-
-
-def add_query_params(url: str, params: Dict[str, Any]) -> str:
-    """
-    向URL添加查询参数
-    
-    :param url: 原始URL
-    :param params: 要添加的参数字典
-    :return: 添加参数后的URL
-    """
-    parsed = urlparse(url)
-    # 解析现有查询参数
-    existing_params = parse_qs(parsed.query)
-    # 添加新参数
-    for key, value in params.items():
-        existing_params[key] = [str(value)]
-    # 编码查询参数
-    new_query = urlencode(existing_params, doseq=True)
-    # 构建新的URL
-    new_parsed = parsed._replace(query=new_query)
-    return urlunparse(new_parsed)
-
-
-def merge_headers(base_headers: Dict[str, str], additional_headers: Dict[str, str]) -> Dict[str, str]:
-    """
-    合并请求头
-    
-    :param base_headers: 基础请求头
-    :param additional_headers: 要添加的请求头
-    :return: 合并后的请求头
-    """
-    merged = deepcopy(base_headers)
-    merged.update(additional_headers)
-    return merged

crawlo/tools/retry_mechanism.py

@@ -1,224 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 重试机制工具
-"""
-
-import time
-import random
-import asyncio
-from typing import Callable, Any, Optional, Tuple, Set
-from functools import wraps
-
-
-class RetryMechanism:
-    """重试机制工具类"""
-
-    # 默认应该重试的HTTP状态码
-    DEFAULT_RETRY_STATUS_CODES = {429, 500, 502, 503, 504}
-
-    # 默认应该重试的异常类型
-    DEFAULT_RETRY_EXCEPTIONS = (
-        ConnectionError,
-        TimeoutError,
-        asyncio.TimeoutError,
-    )
-
-    def __init__(self, max_retries: int = 3,
-                 retry_status_codes: Optional[Set[int]] = None,
-                 retry_exceptions: Optional[Tuple[type, ...]] = None):
-        """
-        初始化重试机制
-        
-        Args:
-            max_retries (int): 最大重试次数
-            retry_status_codes (Optional[Set[int]]): 应该重试的HTTP状态码
-            retry_exceptions (Optional[Tuple[type, ...]]): 应该重试的异常类型
-        """
-        self.max_retries = max_retries
-        self.retry_status_codes = retry_status_codes or self.DEFAULT_RETRY_STATUS_CODES
-        self.retry_exceptions = retry_exceptions or self.DEFAULT_RETRY_EXCEPTIONS
-
-    def should_retry(self, status_code: Optional[int] = None,
-                     exception: Optional[Exception] = None) -> bool:
-        """
-        判断是否应该重试
-        
-        Args:
-            status_code (Optional[int]): HTTP状态码
-            exception (Optional[Exception]): 异常对象
-            
-        Returns:
-            bool: 是否应该重试
-        """
-        # 如果有状态码，检查是否在重试列表中
-        if status_code is not None and status_code in self.retry_status_codes:
-            return True
-
-        # 如果有异常，检查是否在重试列表中
-        if exception is not None and isinstance(exception, self.retry_exceptions):
-            return True
-
-        return False
-
-    def exponential_backoff(self, attempt: int, base_delay: float = 1.0,
-                            max_delay: float = 60.0) -> float:
-        """
-        计算指数退避延迟时间
-        
-        Args:
-            attempt (int): 当前重试次数
-            base_delay (float): 基础延迟时间（秒）
-            max_delay (float): 最大延迟时间（秒）
-            
-        Returns:
-            float: 延迟时间（秒）
-        """
-        # 计算基本延迟：base_delay * (2 ^ attempt)
-        delay = base_delay * (2 ** attempt)
-
-        # 添加随机抖动，避免惊群效应
-        jitter = random.uniform(0, 0.1) * delay
-
-        # 返回最终延迟时间，不超过最大延迟
-        return min(delay + jitter, max_delay)
-
-    async def async_retry(self, func: Callable, *args, **kwargs) -> Any:
-        """
-        异步重试执行函数
-        
-        Args:
-            func (Callable): 要执行的函数
-            *args: 函数参数
-            **kwargs: 函数关键字参数
-            
-        Returns:
-            Any: 函数执行结果
-            
-        Raises:
-            Exception: 如果超过最大重试次数仍未成功，则抛出最后一个异常
-        """
-        last_exception = None
-
-        for attempt in range(self.max_retries + 1):
-            try:
-                result = await func(*args, **kwargs)
-
-                # 如果函数返回状态码，检查是否需要重试
-                if hasattr(result, 'status') and self.should_retry(status_code=result.status):
-                    if attempt < self.max_retries:
-                        delay = self.exponential_backoff(attempt)
-                        await asyncio.sleep(delay)
-                        continue
-                    else:
-                        raise Exception(f"HTTP {result.status} after {self.max_retries} retries")
-
-                return result
-
-            except Exception as e:
-                last_exception = e
-
-                # 检查是否应该重试
-                if self.should_retry(exception=e) and attempt < self.max_retries:
-                    delay = self.exponential_backoff(attempt)
-                    await asyncio.sleep(delay)
-                    continue
-                else:
-                    raise e
-
-        # 如果到达这里，说明所有重试都失败了
-        raise last_exception
-
-    def sync_retry(self, func: Callable, *args, **kwargs) -> Any:
-        """
-        同步重试执行函数
-        
-        Args:
-            func (Callable): 要执行的函数
-            *args: 函数参数
-            **kwargs: 函数关键字参数
-            
-        Returns:
-            Any: 函数执行结果
-            
-        Raises:
-            Exception: 如果超过最大重试次数仍未成功，则抛出最后一个异常
-        """
-        last_exception = None
-
-        for attempt in range(self.max_retries + 1):
-            try:
-                result = func(*args, **kwargs)
-
-                # 如果函数返回状态码，检查是否需要重试
-                if hasattr(result, 'status') and self.should_retry(status_code=result.status):
-                    if attempt < self.max_retries:
-                        delay = self.exponential_backoff(attempt)
-                        time.sleep(delay)
-                        continue
-                    else:
-                        raise Exception(f"HTTP {result.status} after {self.max_retries} retries")
-
-                return result
-
-            except Exception as e:
-                last_exception = e
-
-                # 检查是否应该重试
-                if self.should_retry(exception=e) and attempt < self.max_retries:
-                    delay = self.exponential_backoff(attempt)
-                    time.sleep(delay)
-                    continue
-                else:
-                    raise e
-
-        # 如果到达这里，说明所有重试都失败了
-        raise last_exception
-
-
-def retry(max_retries: int = 3,
-          retry_status_codes: Optional[Set[int]] = None,
-          retry_exceptions: Optional[Tuple[type, ...]] = None):
-    """
-    重试装饰器
-    
-    Args:
-        max_retries (int): 最大重试次数
-        retry_status_codes (Optional[Set[int]]): 应该重试的HTTP状态码
-        retry_exceptions (Optional[Tuple[type, ...]]): 应该重试的异常类型
-    """
-
-    def decorator(func: Callable) -> Callable:
-        retry_mechanism = RetryMechanism(max_retries, retry_status_codes, retry_exceptions)
-
-        if asyncio.iscoroutinefunction(func):
-            @wraps(func)
-            async def async_wrapper(*args, **kwargs):
-                return await retry_mechanism.async_retry(func, *args, **kwargs)
-
-            return async_wrapper
-        else:
-            @wraps(func)
-            def sync_wrapper(*args, **kwargs):
-                return retry_mechanism.sync_retry(func, *args, **kwargs)
-
-            return sync_wrapper
-
-    return decorator
-
-
-# 便捷函数
-def should_retry(status_code: Optional[int] = None,
-                 exception: Optional[Exception] = None) -> bool:
-    """判断是否应该重试"""
-    retry_mechanism = RetryMechanism()
-    return retry_mechanism.should_retry(status_code, exception)
-
-
-def exponential_backoff(attempt: int, base_delay: float = 1.0,
-                        max_delay: float = 60.0) -> float:
-    """计算指数退避延迟时间"""
-    retry_mechanism = RetryMechanism()
-    return retry_mechanism.exponential_backoff(attempt, base_delay, max_delay)

crawlo/utils/env_config.py

@@ -1,143 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-环境变量配置工具
-提供统一的环境变量读取和配置管理机制
-"""
-import os
-import re
-from typing import Any
-
-
-class EnvConfigManager:
-    """环境变量配置管理器"""
-    
-    @staticmethod
-    def get_env_var(var_name: str, default: Any = None, var_type: type = str) -> Any:
-        """
-        获取环境变量值
-        
-        Args:
-            var_name: 环境变量名称
-            default: 默认值
-            var_type: 变量类型 (str, int, float, bool)
-            
-        Returns:
-            环境变量值或默认值
-        """
-        value = os.getenv(var_name)
-        if value is None:
-            return default
-        
-        try:
-            if var_type == bool:
-                return value.lower() in ('1', 'true', 'yes', 'on')
-            elif var_type == int:
-                return int(value)
-            elif var_type == float:
-                return float(value)
-            else:
-                return value
-        except (ValueError, TypeError):
-            return default
-    
-    @staticmethod
-    def get_redis_config() -> dict:
-        """
-        获取 Redis 配置
-        
-        Returns:
-            Redis 配置字典
-        """
-        return {
-            'REDIS_HOST': EnvConfigManager.get_env_var('REDIS_HOST', '127.0.0.1', str),
-            'REDIS_PORT': EnvConfigManager.get_env_var('REDIS_PORT', 6379, int),
-            'REDIS_PASSWORD': EnvConfigManager.get_env_var('REDIS_PASSWORD', '', str),
-            'REDIS_DB': EnvConfigManager.get_env_var('REDIS_DB', 0, int),
-        }
-    
-    @staticmethod
-    def get_runtime_config() -> dict:
-        """
-        获取运行时配置
-        
-        Returns:
-            运行时配置字典
-        """
-        return {
-            'CRAWLO_MODE': EnvConfigManager.get_env_var('CRAWLO_MODE', 'standalone', str),
-            'PROJECT_NAME': EnvConfigManager.get_env_var('PROJECT_NAME', 'crawlo', str),
-            'CONCURRENCY': EnvConfigManager.get_env_var('CONCURRENCY', 8, int),
-        }
-
-    @staticmethod
-    def get_version() -> str:
-        """
-        获取框架版本号
-        
-        Returns:
-            框架版本号字符串
-        """
-        # 获取版本文件路径
-        version_file = os.path.join(os.path.dirname(__file__), '..', '__version__.py')
-        default_version = '1.0.0'
-        
-        if os.path.exists(version_file):
-            try:
-                with open(version_file, 'r', encoding='utf-8') as f:
-                    content = f.read()
-                    # 使用正则表达式提取版本号
-                    version_match = re.search(r"__version__\s*=\s*['\"]([^'\"]*)['\"]", content)
-                    if version_match:
-                        return version_match.group(1)
-            except Exception:
-                # 如果读取失败，使用默认版本号
-                pass
-        
-        return default_version
-
-
-# 便捷函数
-def get_env_var(var_name: str, default: Any = None, var_type: type = str) -> Any:
-    """
-    便捷函数：获取环境变量值
-    
-    Args:
-        var_name: 环境变量名称
-        default: 默认值
-        var_type: 变量类型 (str, int, float, bool)
-        
-    Returns:
-        环境变量值或默认值
-    """
-    return EnvConfigManager.get_env_var(var_name, default, var_type)
-
-
-def get_redis_config() -> dict:
-    """
-    便捷函数：获取 Redis 配置
-    
-    Returns:
-        Redis 配置字典
-    """
-    return EnvConfigManager.get_redis_config()
-
-
-def get_runtime_config() -> dict:
-    """
-    便捷函数：获取运行时配置
-    
-    Returns:
-        运行时配置字典
-    """
-    return EnvConfigManager.get_runtime_config()
-
-
-def get_version() -> str:
-    """
-    便捷函数：获取框架版本号
-    
-    Returns:
-        框架版本号字符串
-    """
-    return EnvConfigManager.get_version()