crawlo 1.4.6__py3-none-any.whl → 1.4.7__py3-none-any.whl

crawlo/logging/monitor.py

@@ -1,153 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-日志性能监控器
-"""
-
-import time
-import threading
-from typing import Dict, List
-from collections import defaultdict, deque
-from .manager import get_config
-
-
-class LogPerformanceMonitor:
-    """
-    日志性能监控器
-    用于监控日志系统的性能指标
-    """
-    
-    def __init__(self):
-        self._lock = threading.Lock()
-        self._log_stats: Dict[str, deque] = defaultdict(lambda: deque(maxlen=1000))
-        self._enabled = False
-        
-    def enable_monitoring(self):
-        """启用性能监控"""
-        with self._lock:
-            self._enabled = True
-            
-    def disable_monitoring(self):
-        """禁用性能监控"""
-        with self._lock:
-            self._enabled = False
-            
-    def record_log_event(self, logger_name: str, level: str, message: str):
-        """
-        记录日志事件
-        
-        Args:
-            logger_name: Logger名称
-            level: 日志级别
-            message: 日志消息
-        """
-        if not self._enabled:
-            return
-            
-        with self._lock:
-            event = {
-                'timestamp': time.time(),
-                'level': level,
-                'message_length': len(message),
-                'thread_id': threading.get_ident()
-            }
-            self._log_stats[logger_name].append(event)
-            
-    def get_statistics(self, logger_name: str = None) -> Dict:
-        """
-        获取日志统计信息
-        
-        Args:
-            logger_name: Logger名称，如果为None则返回所有统计信息
-            
-        Returns:
-            统计信息字典
-        """
-        with self._lock:
-            if logger_name:
-                return self._calculate_stats(logger_name, self._log_stats[logger_name])
-            else:
-                result = {}
-                for name, events in self._log_stats.items():
-                    result[name] = self._calculate_stats(name, events)
-                return result
-                
-    def _calculate_stats(self, logger_name: str, events: deque) -> Dict:
-        """计算统计信息"""
-        if not events:
-            return {
-                'logger_name': logger_name,
-                'total_logs': 0,
-                'log_rates': {},
-                'avg_message_length': 0
-            }
-            
-        # 计算日志级别分布
-        level_counts = defaultdict(int)
-        total_length = 0
-        
-        for event in events:
-            level_counts[event['level']] += 1
-            total_length += event['message_length']
-            
-        # 计算日志速率（每分钟）
-        if len(events) > 1:
-            time_span = events[-1]['timestamp'] - events[0]['timestamp']
-            if time_span > 0:
-                logs_per_minute = len(events) / (time_span / 60)
-            else:
-                logs_per_minute = len(events) * 60
-        else:
-            logs_per_minute = 0
-            
-        return {
-            'logger_name': logger_name,
-            'total_logs': len(events),
-            'log_rates': {
-                'per_minute': logs_per_minute
-            },
-            'level_distribution': dict(level_counts),
-            'avg_message_length': total_length / len(events) if events else 0
-        }
-        
-    def get_performance_report(self) -> str:
-        """
-        获取性能报告
-        
-        Returns:
-            格式化的性能报告字符串
-        """
-        stats = self.get_statistics()
-        
-        report = ["=" * 50]
-        report.append("日志系统性能报告")
-        report.append("=" * 50)
-        
-        config = get_config()
-        if config:
-            report.append(f"日志文件: {config.file_path or 'N/A'}")
-            report.append(f"文件启用: {config.file_enabled}")
-            report.append(f"控制台启用: {config.console_enabled}")
-            report.append("-" * 50)
-        
-        for logger_name, stat in stats.items():
-            report.append(f"Logger: {logger_name}")
-            report.append(f"  总日志数: {stat['total_logs']}")
-            report.append(f"  日志速率: {stat['log_rates']['per_minute']:.2f} 条/分钟")
-            report.append(f"  平均消息长度: {stat['avg_message_length']:.2f} 字符")
-            
-            if 'level_distribution' in stat:
-                levels = ", ".join([f"{k}: {v}" for k, v in stat['level_distribution'].items()])
-                report.append(f"  级别分布: {levels}")
-            report.append("")
-            
-        return "\n".join(report)
-
-
-# 全局实例
-_log_monitor = LogPerformanceMonitor()
-
-
-def get_monitor() -> LogPerformanceMonitor:
-    """获取日志监控器实例"""
-    return _log_monitor

crawlo/logging/sampler.py

@@ -1,167 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-日志采样器
-用于在高负载时减少日志输出
-"""
-
-import random
-import time
-import threading
-from typing import Dict, Set
-from collections import defaultdict
-
-
-class LogSampler:
-    """
-    日志采样器
-    支持多种采样策略以减少日志输出量
-    """
-    
-    def __init__(self):
-        self._lock = threading.RLock()
-        self._sample_rates: Dict[str, float] = {}  # logger_name -> sample_rate
-        self._rate_limiters: Dict[str, TokenBucket] = {}  # logger_name -> rate_limiter
-        self._message_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
-        self._time_windows: Dict[str, Dict[str, float]] = defaultdict(lambda: defaultdict(float))
-        
-    def set_sample_rate(self, logger_name: str, rate: float):
-        """
-        设置采样率
-        
-        Args:
-            logger_name: Logger名称
-            rate: 采样率 (0.0-1.0)，1.0表示全部记录，0.0表示不记录
-        """
-        with self._lock:
-            self._sample_rates[logger_name] = max(0.0, min(1.0, rate))
-            
-    def set_rate_limit(self, logger_name: str, messages_per_second: float):
-        """
-        设置速率限制
-        
-        Args:
-            logger_name: Logger名称
-            messages_per_second: 每秒最大消息数
-        """
-        with self._lock:
-            self._rate_limiters[logger_name] = TokenBucket(messages_per_second, messages_per_second)
-            
-    def should_log(self, logger_name: str, message: str = None, level: str = None) -> bool:
-        """
-        判断是否应该记录日志
-        
-        Args:
-            logger_name: Logger名称
-            message: 日志消息（用于去重）
-            level: 日志级别
-            
-        Returns:
-            是否应该记录日志
-        """
-        with self._lock:
-            # 检查采样率
-            if logger_name in self._sample_rates:
-                sample_rate = self._sample_rates[logger_name]
-                if random.random() > sample_rate:
-                    return False
-                    
-            # 检查速率限制
-            if logger_name in self._rate_limiters:
-                if not self._rate_limiters[logger_name].consume(1):
-                    return False
-                    
-            # 检查消息去重（相同消息在短时间内只记录一次）
-            if message:
-                key = f"{level}:{message}" if level else message
-                current_time = time.time()
-                
-                # 如果距离上次记录超过60秒，重置计数
-                if current_time - self._time_windows[logger_name][key] > 60:
-                    self._message_counts[logger_name][key] = 0
-                    self._time_windows[logger_name][key] = current_time
-                    
-                # 限制相同消息的记录次数
-                if self._message_counts[logger_name][key] >= 5:  # 最多记录5次相同消息
-                    return False
-                    
-                self._message_counts[logger_name][key] += 1
-                
-            return True
-            
-    def reset(self):
-        """重置采样器状态"""
-        with self._lock:
-            self._sample_rates.clear()
-            self._rate_limiters.clear()
-            self._message_counts.clear()
-            self._time_windows.clear()
-
-
-class TokenBucket:
-    """
-    令牌桶算法实现
-    用于速率限制
-    """
-    
-    def __init__(self, rate: float, capacity: float):
-        """
-        初始化令牌桶
-        
-        Args:
-            rate: 每秒生成的令牌数
-            capacity: 桶的最大容量
-        """
-        self._rate = rate
-        self._capacity = capacity
-        self._tokens = capacity
-        self._last_time = time.time()
-        self._lock = threading.Lock()
-        
-    def consume(self, tokens: float) -> bool:
-        """
-        消费令牌
-        
-        Args:
-            tokens: 要消费的令牌数
-            
-        Returns:
-            是否消费成功
-        """
-        with self._lock:
-            current_time = time.time()
-            # 补充令牌
-            elapsed = current_time - self._last_time
-            self._tokens = min(self._capacity, self._tokens + elapsed * self._rate)
-            self._last_time = current_time
-            
-            # 尝试消费令牌
-            if self._tokens >= tokens:
-                self._tokens -= tokens
-                return True
-            else:
-                return False
-
-
-# 全局实例
-_log_sampler = LogSampler()
-
-
-def get_sampler() -> LogSampler:
-    """获取日志采样器实例"""
-    return _log_sampler
-
-
-def should_log(logger_name: str, message: str = None, level: str = None) -> bool:
-    """
-    判断是否应该记录日志的便捷函数
-    
-    Args:
-        logger_name: Logger名称
-        message: 日志消息
-        level: 日志级别
-        
-    Returns:
-        是否应该记录日志
-    """
-    return get_sampler().should_log(logger_name, message, level)

crawlo/tools/authenticated_proxy.py

@@ -1,241 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-带认证代理工具
-=============
-支持带用户名密码认证的代理和非认证代理的统一处理工具
-
-功能特性:
-- 支持HTTP/HTTPS/SOCKS代理
-- 支持带认证和不带认证的代理
-- 统一的代理格式处理
-- 代理有效性检测
-"""
-
-from typing import Dict, Any, Optional, Union
-from urllib.parse import urlparse, urlunparse
-
-
-class AuthenticatedProxy:
-    """带认证代理处理类"""
-
-    def __init__(self, proxy_url: str):
-        """
-        初始化代理对象
-        
-        Args:
-            proxy_url (str): 代理URL，支持带认证信息的格式
-                - 带认证: http://username:password@proxy.example.com:8080
-                - 无认证: http://proxy.example.com:8080
-        """
-        self.proxy_url = proxy_url
-        self.parsed = urlparse(proxy_url)
-        
-        # 提取认证信息
-        self.username = self.parsed.username
-        self.password = self.parsed.password
-        
-        # 构造不带认证信息的URL
-        self.clean_url = urlunparse((
-            self.parsed.scheme,
-            f"{self.parsed.hostname}:{self.parsed.port}" if self.parsed.port else self.parsed.hostname,
-            self.parsed.path,
-            self.parsed.params,
-            self.parsed.query,
-            self.parsed.fragment
-        ))
-        
-        # 构造下载器兼容的代理字典
-        self.proxy_dict = {
-            "http": self.clean_url,
-            "https": self.clean_url
-        }
-        
-        # 如果有认证信息，构造认证字符串
-        if self.username and self.password:
-            self.auth_string = f"{self.username}:{self.password}"
-        else:
-            self.auth_string = None
-
-    def get_proxy_for_downloader(self) -> Union[str, Dict[str, str]]:
-        """
-        获取适用于下载器的代理配置
-        
-        Returns:
-            Union[str, Dict[str, str]]: 代理配置
-                - 对于AioHttp/CurlCffi: 返回字典格式 {"http": "...", "https": "..."}
-                - 对于HttpX: 可以直接使用字符串或字典格式
-        """
-        return self.proxy_dict
-
-    def get_auth_credentials(self) -> Optional[Dict[str, str]]:
-        """
-        获取认证凭据
-        
-        Returns:
-            Optional[Dict[str, str]]: 认证凭据 {"username": "...", "password": "..."}
-        """
-        if self.username and self.password:
-            return {
-                "username": self.username,
-                "password": self.password
-            }
-        return None
-
-    def get_auth_header(self) -> Optional[str]:
-        """
-        获取Basic Auth认证头
-        
-        Returns:
-            Optional[str]: Basic Auth头信息
-        """
-        if self.username and self.password:
-            import base64
-            credentials = f"{self.username}:{self.password}"
-            encoded = base64.b64encode(credentials.encode()).decode()
-            return f"Basic {encoded}"
-        return None
-
-    def is_valid(self) -> bool:
-        """
-        检查代理URL是否有效
-        
-        Returns:
-            bool: 代理URL是否有效
-        """
-        # 检查协议
-        if self.parsed.scheme not in ["http", "https", "socks4", "socks5"]:
-            return False
-            
-        # 检查主机名
-        if not self.parsed.hostname:
-            return False
-            
-        # 检查端口（如果指定）
-        if self.parsed.port and (self.parsed.port < 1 or self.parsed.port > 65535):
-            return False
-            
-        return True
-
-    def __str__(self) -> str:
-        return self.proxy_url
-
-    def __repr__(self) -> str:
-        return f"AuthenticatedProxy(url='{self.proxy_url}', username={self.username is not None})"
-
-
-def create_proxy_config(proxy_url: str) -> Dict[str, Any]:
-    """
-    创建代理配置，兼容各种下载器
-    
-    Args:
-        proxy_url (str): 代理URL
-        
-    Returns:
-        Dict[str, Any]: 代理配置字典
-    """
-    proxy = AuthenticatedProxy(proxy_url)
-    
-    if not proxy.is_valid():
-        raise ValueError(f"Invalid proxy URL: {proxy_url}")
-    
-    config = {
-        "url": proxy.clean_url,
-        "proxy_dict": proxy.proxy_dict,
-        "has_auth": proxy.auth_string is not None
-    }
-    
-    if proxy.auth_string:
-        config["auth"] = proxy.get_auth_credentials()
-        config["auth_header"] = proxy.get_auth_header()
-        
-    return config
-
-
-def format_proxy_for_request(proxy_config: Dict[str, Any], downloader_type: str = "aiohttp") -> Dict[str, Any]:
-    """
-    格式化代理配置以适配特定下载器
-    
-    Args:
-        proxy_config (Dict[str, Any]): 代理配置
-        downloader_type (str): 下载器类型 (aiohttp, httpx, curl_cffi)
-        
-    Returns:
-        Dict[str, Any]: 适配下载器的代理配置
-    """
-    formatted = {}
-    
-    if downloader_type.lower() == "aiohttp":
-        # AioHttp使用proxy和proxy_auth参数
-        formatted["proxy"] = proxy_config["url"]
-        if proxy_config.get("has_auth") and proxy_config.get("auth"):
-            from aiohttp import BasicAuth
-            auth = proxy_config["auth"]
-            formatted["proxy_auth"] = BasicAuth(auth["username"], auth["password"])
-            
-    elif downloader_type.lower() == "httpx":
-        # HttpX可以直接使用代理URL字符串或字典
-        formatted["proxy"] = proxy_config["url"]
-        
-    elif downloader_type.lower() == "curl_cffi":
-        # CurlCffi使用proxies字典
-        formatted["proxies"] = proxy_config["proxy_dict"]
-        # 认证信息包含在URL中或通过headers传递
-        if proxy_config.get("auth_header"):
-            formatted["headers"] = {"Proxy-Authorization": proxy_config["auth_header"]}
-            
-    return formatted
-
-
-# 便捷函数
-def parse_proxy_url(proxy_url: str) -> Dict[str, Any]:
-    """
-    解析代理URL并返回详细信息
-    
-    Args:
-        proxy_url (str): 代理URL
-        
-    Returns:
-        Dict[str, Any]: 代理详细信息
-    """
-    return create_proxy_config(proxy_url)
-
-
-def validate_proxy_url(proxy_url: str) -> bool:
-    """
-    验证代理URL是否有效
-    
-    Args:
-        proxy_url (str): 代理URL
-        
-    Returns:
-        bool: 是否有效
-    """
-    try:
-        proxy = AuthenticatedProxy(proxy_url)
-        return proxy.is_valid()
-    except:
-        return False
-
-
-def get_proxy_info(proxy_url: str) -> Dict[str, Any]:
-    """
-    获取代理详细信息
-    
-    Args:
-        proxy_url (str): 代理URL
-        
-    Returns:
-        Dict[str, Any]: 代理详细信息
-    """
-    proxy = AuthenticatedProxy(proxy_url)
-    return {
-        "original_url": proxy.proxy_url,
-        "clean_url": proxy.clean_url,
-        "scheme": proxy.parsed.scheme,
-        "hostname": proxy.parsed.hostname,
-        "port": proxy.parsed.port,
-        "has_auth": proxy.auth_string is not None,
-        "username": proxy.username,
-        "is_valid": proxy.is_valid()
-    }