crawlo 1.4.2__py3-none-any.whl → 1.4.3__py3-none-any.whl

crawlo/logging/manager.py

@@ -1,112 +1,112 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-日志管理器 - 核心组件
-"""
-
-import threading
-from typing import Optional, Any
-from .config import LogConfig
-
-
-class LogManager:
-    """
-    日志管理器 - 单例模式
-    
-    职责：
-    1. 全局日志配置管理
-    2. 配置状态跟踪
-    3. 线程安全的配置更新
-    """
-    
-    _instance: Optional['LogManager'] = None
-    _lock = threading.Lock()
-    
-    def __new__(cls) -> 'LogManager':
-        if cls._instance is None:
-            with cls._lock:
-                if cls._instance is None:
-                    cls._instance = super(LogManager, cls).__new__(cls)
-                    cls._instance._initialized = False
-        return cls._instance
-    
-    def __init__(self):
-        if hasattr(self, '_initialized') and self._initialized:
-            return
-            
-        self._config: Optional[LogConfig] = None
-        self._configured = False
-        self._config_lock = threading.RLock()
-        self._initialized = True
-    
-    @property
-    def config(self) -> Optional[LogConfig]:
-        """获取当前配置"""
-        with self._config_lock:
-            return self._config
-    
-    @property
-    def is_configured(self) -> bool:
-        """检查是否已配置"""
-        return self._configured
-    
-    def configure(self, settings=None, **kwargs) -> LogConfig:
-        """
-        配置日志系统
-        
-        Args:
-            settings: 配置对象或None
-            **kwargs: 关键字参数配置
-            
-        Returns:
-            LogConfig: 生效的配置对象
-        """
-        with self._config_lock:
-            # 总是重新配置，即使已经配置过
-            # 从不同来源创建配置
-            if settings is not None:
-                # 检查settings是否已经是LogConfig对象
-                if isinstance(settings, LogConfig):
-                    config = settings
-                else:
-                    config = LogConfig.from_settings(settings)
-            elif kwargs:
-                config = LogConfig.from_dict(kwargs)
-            else:
-                config = LogConfig()  # 使用默认配置
-            
-            # 验证配置
-            if not config.validate():
-                raise ValueError("Invalid log configuration")
-            
-            self._config = config
-            self._configured = True
-            
-            return config
-    
-    def reset(self):
-        """重置配置（主要用于测试）"""
-        with self._config_lock:
-            self._config = None
-            self._configured = False
-
-
-# 全局实例
-_log_manager = LogManager()
-
-# 模块级便捷函数
-def configure(settings=None, **kwargs) -> LogConfig:
-    """配置日志系统"""
-    return _log_manager.configure(settings, **kwargs)
-
-def is_configured() -> bool:
-    """检查是否已配置"""
-    return _log_manager.is_configured
-
-def get_config() -> Optional[LogConfig]:
-    """获取当前配置"""
-    return _log_manager.config
-
-def reset():
-    """重置配置"""
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+日志管理器 - 核心组件
+"""
+
+import threading
+from typing import Optional, Any
+from .config import LogConfig
+
+
+class LogManager:
+    """
+    日志管理器 - 单例模式
+    
+    职责：
+    1. 全局日志配置管理
+    2. 配置状态跟踪
+    3. 线程安全的配置更新
+    """
+    
+    _instance: Optional['LogManager'] = None
+    _lock = threading.Lock()
+    
+    def __new__(cls) -> 'LogManager':
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super(LogManager, cls).__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+    
+    def __init__(self):
+        if hasattr(self, '_initialized') and self._initialized:
+            return
+            
+        self._config: Optional[LogConfig] = None
+        self._configured = False
+        self._config_lock = threading.RLock()
+        self._initialized = True
+    
+    @property
+    def config(self) -> Optional[LogConfig]:
+        """获取当前配置"""
+        with self._config_lock:
+            return self._config
+    
+    @property
+    def is_configured(self) -> bool:
+        """检查是否已配置"""
+        return self._configured
+    
+    def configure(self, settings=None, **kwargs) -> LogConfig:
+        """
+        配置日志系统
+        
+        Args:
+            settings: 配置对象或None
+            **kwargs: 关键字参数配置
+            
+        Returns:
+            LogConfig: 生效的配置对象
+        """
+        with self._config_lock:
+            # 总是重新配置，即使已经配置过
+            # 从不同来源创建配置
+            if settings is not None:
+                # 检查settings是否已经是LogConfig对象
+                if isinstance(settings, LogConfig):
+                    config = settings
+                else:
+                    config = LogConfig.from_settings(settings)
+            elif kwargs:
+                config = LogConfig.from_dict(kwargs)
+            else:
+                config = LogConfig()  # 使用默认配置
+            
+            # 验证配置
+            if not config.validate():
+                raise ValueError("Invalid log configuration")
+            
+            self._config = config
+            self._configured = True
+            
+            return config
+    
+    def reset(self):
+        """重置配置（主要用于测试）"""
+        with self._config_lock:
+            self._config = None
+            self._configured = False
+
+
+# 全局实例
+_log_manager = LogManager()
+
+# 模块级便捷函数
+def configure(settings=None, **kwargs) -> LogConfig:
+    """配置日志系统"""
+    return _log_manager.configure(settings, **kwargs)
+
+def is_configured() -> bool:
+    """检查是否已配置"""
+    return _log_manager.is_configured
+
+def get_config() -> Optional[LogConfig]:
+    """获取当前配置"""
+    return _log_manager.config
+
+def reset():
+    """重置配置"""
     _log_manager.reset()

crawlo/logging/monitor.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+日志性能监控器
+"""
+
+import time
+import threading
+from typing import Dict, List
+from collections import defaultdict, deque
+from .manager import get_config
+
+
+class LogPerformanceMonitor:
+    """
+    日志性能监控器
+    用于监控日志系统的性能指标
+    """
+    
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._log_stats: Dict[str, deque] = defaultdict(lambda: deque(maxlen=1000))
+        self._enabled = False
+        
+    def enable_monitoring(self):
+        """启用性能监控"""
+        with self._lock:
+            self._enabled = True
+            
+    def disable_monitoring(self):
+        """禁用性能监控"""
+        with self._lock:
+            self._enabled = False
+            
+    def record_log_event(self, logger_name: str, level: str, message: str):
+        """
+        记录日志事件
+        
+        Args:
+            logger_name: Logger名称
+            level: 日志级别
+            message: 日志消息
+        """
+        if not self._enabled:
+            return
+            
+        with self._lock:
+            event = {
+                'timestamp': time.time(),
+                'level': level,
+                'message_length': len(message),
+                'thread_id': threading.get_ident()
+            }
+            self._log_stats[logger_name].append(event)
+            
+    def get_statistics(self, logger_name: str = None) -> Dict:
+        """
+        获取日志统计信息
+        
+        Args:
+            logger_name: Logger名称，如果为None则返回所有统计信息
+            
+        Returns:
+            统计信息字典
+        """
+        with self._lock:
+            if logger_name:
+                return self._calculate_stats(logger_name, self._log_stats[logger_name])
+            else:
+                result = {}
+                for name, events in self._log_stats.items():
+                    result[name] = self._calculate_stats(name, events)
+                return result
+                
+    def _calculate_stats(self, logger_name: str, events: deque) -> Dict:
+        """计算统计信息"""
+        if not events:
+            return {
+                'logger_name': logger_name,
+                'total_logs': 0,
+                'log_rates': {},
+                'avg_message_length': 0
+            }
+            
+        # 计算日志级别分布
+        level_counts = defaultdict(int)
+        total_length = 0
+        
+        for event in events:
+            level_counts[event['level']] += 1
+            total_length += event['message_length']
+            
+        # 计算日志速率（每分钟）
+        if len(events) > 1:
+            time_span = events[-1]['timestamp'] - events[0]['timestamp']
+            if time_span > 0:
+                logs_per_minute = len(events) / (time_span / 60)
+            else:
+                logs_per_minute = len(events) * 60
+        else:
+            logs_per_minute = 0
+            
+        return {
+            'logger_name': logger_name,
+            'total_logs': len(events),
+            'log_rates': {
+                'per_minute': logs_per_minute
+            },
+            'level_distribution': dict(level_counts),
+            'avg_message_length': total_length / len(events) if events else 0
+        }
+        
+    def get_performance_report(self) -> str:
+        """
+        获取性能报告
+        
+        Returns:
+            格式化的性能报告字符串
+        """
+        stats = self.get_statistics()
+        
+        report = ["=" * 50]
+        report.append("日志系统性能报告")
+        report.append("=" * 50)
+        
+        config = get_config()
+        if config:
+            report.append(f"日志文件: {config.file_path or 'N/A'}")
+            report.append(f"文件启用: {config.file_enabled}")
+            report.append(f"控制台启用: {config.console_enabled}")
+            report.append("-" * 50)
+        
+        for logger_name, stat in stats.items():
+            report.append(f"Logger: {logger_name}")
+            report.append(f"  总日志数: {stat['total_logs']}")
+            report.append(f"  日志速率: {stat['log_rates']['per_minute']:.2f} 条/分钟")
+            report.append(f"  平均消息长度: {stat['avg_message_length']:.2f} 字符")
+            
+            if 'level_distribution' in stat:
+                levels = ", ".join([f"{k}: {v}" for k, v in stat['level_distribution'].items()])
+                report.append(f"  级别分布: {levels}")
+            report.append("")
+            
+        return "\n".join(report)
+
+
+# 全局实例
+_log_monitor = LogPerformanceMonitor()
+
+
+def get_monitor() -> LogPerformanceMonitor:
+    """获取日志监控器实例"""
+    return _log_monitor

crawlo/logging/sampler.py

@@ -0,0 +1,167 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+日志采样器
+用于在高负载时减少日志输出
+"""
+
+import random
+import time
+import threading
+from typing import Dict, Set
+from collections import defaultdict
+
+
+class LogSampler:
+    """
+    日志采样器
+    支持多种采样策略以减少日志输出量
+    """
+    
+    def __init__(self):
+        self._lock = threading.RLock()
+        self._sample_rates: Dict[str, float] = {}  # logger_name -> sample_rate
+        self._rate_limiters: Dict[str, TokenBucket] = {}  # logger_name -> rate_limiter
+        self._message_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
+        self._time_windows: Dict[str, Dict[str, float]] = defaultdict(lambda: defaultdict(float))
+        
+    def set_sample_rate(self, logger_name: str, rate: float):
+        """
+        设置采样率
+        
+        Args:
+            logger_name: Logger名称
+            rate: 采样率 (0.0-1.0)，1.0表示全部记录，0.0表示不记录
+        """
+        with self._lock:
+            self._sample_rates[logger_name] = max(0.0, min(1.0, rate))
+            
+    def set_rate_limit(self, logger_name: str, messages_per_second: float):
+        """
+        设置速率限制
+        
+        Args:
+            logger_name: Logger名称
+            messages_per_second: 每秒最大消息数
+        """
+        with self._lock:
+            self._rate_limiters[logger_name] = TokenBucket(messages_per_second, messages_per_second)
+            
+    def should_log(self, logger_name: str, message: str = None, level: str = None) -> bool:
+        """
+        判断是否应该记录日志
+        
+        Args:
+            logger_name: Logger名称
+            message: 日志消息（用于去重）
+            level: 日志级别
+            
+        Returns:
+            是否应该记录日志
+        """
+        with self._lock:
+            # 检查采样率
+            if logger_name in self._sample_rates:
+                sample_rate = self._sample_rates[logger_name]
+                if random.random() > sample_rate:
+                    return False
+                    
+            # 检查速率限制
+            if logger_name in self._rate_limiters:
+                if not self._rate_limiters[logger_name].consume(1):
+                    return False
+                    
+            # 检查消息去重（相同消息在短时间内只记录一次）
+            if message:
+                key = f"{level}:{message}" if level else message
+                current_time = time.time()
+                
+                # 如果距离上次记录超过60秒，重置计数
+                if current_time - self._time_windows[logger_name][key] > 60:
+                    self._message_counts[logger_name][key] = 0
+                    self._time_windows[logger_name][key] = current_time
+                    
+                # 限制相同消息的记录次数
+                if self._message_counts[logger_name][key] >= 5:  # 最多记录5次相同消息
+                    return False
+                    
+                self._message_counts[logger_name][key] += 1
+                
+            return True
+            
+    def reset(self):
+        """重置采样器状态"""
+        with self._lock:
+            self._sample_rates.clear()
+            self._rate_limiters.clear()
+            self._message_counts.clear()
+            self._time_windows.clear()
+
+
+class TokenBucket:
+    """
+    令牌桶算法实现
+    用于速率限制
+    """
+    
+    def __init__(self, rate: float, capacity: float):
+        """
+        初始化令牌桶
+        
+        Args:
+            rate: 每秒生成的令牌数
+            capacity: 桶的最大容量
+        """
+        self._rate = rate
+        self._capacity = capacity
+        self._tokens = capacity
+        self._last_time = time.time()
+        self._lock = threading.Lock()
+        
+    def consume(self, tokens: float) -> bool:
+        """
+        消费令牌
+        
+        Args:
+            tokens: 要消费的令牌数
+            
+        Returns:
+            是否消费成功
+        """
+        with self._lock:
+            current_time = time.time()
+            # 补充令牌
+            elapsed = current_time - self._last_time
+            self._tokens = min(self._capacity, self._tokens + elapsed * self._rate)
+            self._last_time = current_time
+            
+            # 尝试消费令牌
+            if self._tokens >= tokens:
+                self._tokens -= tokens
+                return True
+            else:
+                return False
+
+
+# 全局实例
+_log_sampler = LogSampler()
+
+
+def get_sampler() -> LogSampler:
+    """获取日志采样器实例"""
+    return _log_sampler
+
+
+def should_log(logger_name: str, message: str = None, level: str = None) -> bool:
+    """
+    判断是否应该记录日志的便捷函数
+    
+    Args:
+        logger_name: Logger名称
+        message: 日志消息
+        level: 日志级别
+        
+    Returns:
+        是否应该记录日志
+    """
+    return get_sampler().should_log(logger_name, message, level)

crawlo/middleware/__init__.py

@@ -1,21 +1,21 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo import Request, Response
-
-
-class BaseMiddleware(object):
-    def process_request(self, request, spider) -> None | Request | Response:
-        # 请求预处理
-        pass
-
-    def process_response(self, request, response, spider) -> Request | Response:
-        # 响应预处理
-        pass
-
-    def process_exception(self, request, exp, spider) -> None | Request | Response:
-        # 异常预处理
-        pass
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo import Request, Response
+
+
+class BaseMiddleware(object):
+    def process_request(self, request, spider) -> None | Request | Response:
+        # 请求预处理
+        pass
+
+    def process_response(self, request, response, spider) -> Request | Response:
+        # 响应预处理
+        pass
+
+    def process_exception(self, request, exp, spider) -> None | Request | Response:
+        # 异常预处理
+        pass
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls()