crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

crawlo/utils/performance_monitor.py

@@ -0,0 +1,285 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+性能监控工具
+提供系统性能监控和资源使用情况跟踪
+"""
+import time
+import psutil
+import asyncio
+from typing import Dict, Any, Optional, Callable
+from functools import wraps
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.error_handler import ErrorHandler
+
+
+class PerformanceMonitor:
+    """性能监控器"""
+    
+    def __init__(self, logger_name: str = __name__):
+        self.logger = get_logger(logger_name)
+        self.error_handler = ErrorHandler(logger_name)
+        self.process = psutil.Process()
+        self.start_time = time.time()
+        
+        # 性能指标
+        self.metrics = {
+            'cpu_usage': [],
+            'memory_usage': [],
+            'network_io': [],
+            'disk_io': []
+        }
+    
+    def get_system_metrics(self) -> Dict[str, Any]:
+        """
+        获取系统性能指标
+        
+        Returns:
+            包含各种性能指标的字典
+        """
+        try:
+            # CPU使用率
+            cpu_percent = psutil.cpu_percent(interval=1)
+            
+            # 内存使用情况
+            memory = psutil.virtual_memory()
+            
+            # 网络IO
+            net_io = psutil.net_io_counters()
+            
+            # 磁盘IO
+            disk_io = psutil.disk_io_counters()
+            
+            # 进程特定信息
+            process_memory = self.process.memory_info()
+            process_cpu = self.process.cpu_percent()
+            
+            return {
+                'timestamp': time.time(),
+                'uptime': time.time() - self.start_time,
+                'cpu': {
+                    'percent': cpu_percent,
+                    'count': psutil.cpu_count(),
+                    'freq': psutil.cpu_freq()._asdict() if psutil.cpu_freq() else {}
+                },
+                'memory': {
+                    'total': memory.total,
+                    'available': memory.available,
+                    'percent': memory.percent,
+                    'used': memory.used,
+                    'free': memory.free
+                },
+                'process': {
+                    'memory_rss': process_memory.rss,
+                    'memory_vms': process_memory.vms,
+                    'cpu_percent': process_cpu,
+                    'num_threads': self.process.num_threads(),
+                    'num_fds': self.process.num_fds() if hasattr(self.process, 'num_fds') else 0
+                },
+                'network': {
+                    'bytes_sent': net_io.bytes_sent,
+                    'bytes_recv': net_io.bytes_recv,
+                    'packets_sent': net_io.packets_sent,
+                    'packets_recv': net_io.packets_recv
+                },
+                'disk': {
+                    'read_bytes': disk_io.read_bytes,
+                    'write_bytes': disk_io.write_bytes,
+                    'read_count': disk_io.read_count,
+                    'write_count': disk_io.write_count
+                }
+            }
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="获取系统性能指标失败", 
+                raise_error=False
+            )
+            return {}
+    
+    def log_system_metrics(self, detailed: bool = False):
+        """
+        记录系统性能指标
+        
+        Args:
+            detailed: 是否记录详细信息
+        """
+        try:
+            metrics = self.get_system_metrics()
+            if not metrics:
+                return
+            
+            # 基本信息
+            basic_info = (
+                f"📊 系统性能指标 | "
+                f"CPU: {metrics['cpu']['percent']:.1f}% | "
+                f"内存: {metrics['memory']['percent']:.1f}% | "
+                f"进程CPU: {metrics['process']['cpu_percent']:.1f}% | "
+                f"进程内存: {metrics['process']['memory_rss'] / 1024 / 1024:.1f}MB"
+            )
+            self.logger.info(basic_info)
+            
+            # 详细信息
+            if detailed:
+                detailed_info = (
+                    f"   详细信息:\n"
+                    f"   - CPU: {metrics['cpu']['count']} 核心\n"
+                    f"   - 内存: 总计 {metrics['memory']['total'] / 1024 / 1024 / 1024:.1f}GB, "
+                    f"可用 {metrics['memory']['available'] / 1024 / 1024 / 1024:.1f}GB\n"
+                    f"   - 网络: 发送 {metrics['network']['bytes_sent'] / 1024 / 1024:.1f}MB, "
+                    f"接收 {metrics['network']['bytes_recv'] / 1024 / 1024:.1f}MB\n"
+                    f"   - 磁盘: 读取 {metrics['disk']['read_bytes'] / 1024 / 1024:.1f}MB, "
+                    f"写入 {metrics['disk']['write_bytes'] / 1024 / 1024:.1f}MB"
+                )
+                self.logger.debug(detailed_info)
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="记录系统性能指标失败", 
+                raise_error=False
+            )
+    
+    def start_monitoring(self, interval: int = 60, detailed: bool = False):
+        """
+        开始定期监控
+        
+        Args:
+            interval: 监控间隔（秒）
+            detailed: 是否记录详细信息
+        """
+        async def monitor_loop():
+            while True:
+                try:
+                    self.log_system_metrics(detailed)
+                    await asyncio.sleep(interval)
+                except asyncio.CancelledError:
+                    break
+                except Exception as e:
+                    self.logger.error(f"监控循环错误: {e}")
+        
+        # 启动监控任务
+        self.monitor_task = asyncio.create_task(monitor_loop())
+        self.logger.info(f"开始性能监控，间隔: {interval}秒")
+    
+    async def stop_monitoring(self):
+        """停止监控"""
+        if hasattr(self, 'monitor_task') and self.monitor_task:
+            self.monitor_task.cancel()
+            try:
+                await self.monitor_task
+            except asyncio.CancelledError:
+                pass
+            self.logger.info("性能监控已停止")
+
+
+class PerformanceTimer:
+    """性能计时器"""
+    
+    def __init__(self, name: str = "timer"):
+        self.name = name
+        self.start_time = None
+        self.end_time = None
+        self.logger = get_logger(f"{__name__}.{self.__class__.__name__}")
+        self.error_handler = ErrorHandler(f"{__name__}.{self.__class__.__name__}")
+    
+    def start(self):
+        """开始计时"""
+        self.start_time = time.time()
+        self.logger.debug(f"⏱️  开始计时: {self.name}")
+    
+    def stop(self) -> float:
+        """
+        停止计时并返回耗时
+        
+        Returns:
+            耗时（秒）
+        """
+        self.end_time = time.time()
+        if self.start_time is None:
+            raise RuntimeError("计时器未启动")
+        
+        elapsed = self.end_time - self.start_time
+        self.logger.debug(f"⏱️  停止计时: {self.name}, 耗时: {elapsed:.3f}秒")
+        return elapsed
+    
+    def __enter__(self):
+        self.start()
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        try:
+            elapsed = self.stop()
+            if exc_type is None:
+                self.logger.info(f"✅ {self.name} 执行成功，耗时: {elapsed:.3f}秒")
+            else:
+                self.logger.error(f"❌ {self.name} 执行失败，耗时: {elapsed:.3f}秒")
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context=f"计时器退出时发生错误: {self.name}", 
+                raise_error=False
+            )
+
+
+def performance_monitor_decorator(name: str = None, log_level: str = "INFO"):
+    """
+    装饰器：监控函数性能
+    
+    Args:
+        name: 函数名称（如果为None则使用函数名）
+        log_level: 日志级别
+    """
+    def decorator(func):
+        @wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            timer_name = name or f"{func.__module__}.{func.__name__}"
+            logger = get_logger(timer_name)
+            
+            with PerformanceTimer(timer_name) as timer:
+                if asyncio.iscoroutinefunction(func):
+                    return await func(*args, **kwargs)
+                else:
+                    return func(*args, **kwargs)
+        
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            timer_name = name or f"{func.__module__}.{func.__name__}"
+            logger = get_logger(timer_name)
+            
+            with PerformanceTimer(timer_name) as timer:
+                return func(*args, **kwargs)
+        
+        # 根据函数是否为异步函数返回相应的包装器
+        import inspect
+        if inspect.iscoroutinefunction(func):
+            return async_wrapper
+        else:
+            return sync_wrapper
+    
+    return decorator
+
+
+# 全局性能监控器实例
+default_performance_monitor = PerformanceMonitor()
+
+
+def monitor_performance(interval: int = 60, detailed: bool = False):
+    """
+    便捷函数：开始性能监控
+    
+    Args:
+        interval: 监控间隔（秒）
+        detailed: 是否记录详细信息
+    """
+    default_performance_monitor.start_monitoring(interval, detailed)
+
+
+def get_current_metrics() -> Dict[str, Any]:
+    """
+    便捷函数：获取当前性能指标
+    
+    Returns:
+        性能指标字典
+    """
+    return default_performance_monitor.get_system_metrics()

crawlo/utils/redis_connection_pool.py

@@ -0,0 +1,335 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Redis连接池优化工具
+提供优化的Redis连接池管理和配置
+"""
+import asyncio
+import redis.asyncio as aioredis
+from typing import Dict, Any, Optional, Union
+from contextlib import asynccontextmanager
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.error_handler import ErrorHandler
+
+
+class OptimizedRedisConnectionPool:
+    """优化的Redis连接池管理器"""
+    
+    # 默认连接池配置
+    DEFAULT_CONFIG = {
+        'max_connections': 50,
+        'socket_connect_timeout': 5,
+        'socket_timeout': 30,
+        'socket_keepalive': True,
+        'health_check_interval': 30,
+        'retry_on_timeout': True,
+        'encoding': 'utf-8',
+        'decode_responses': False,
+    }
+    
+    def __init__(self, redis_url: str, **kwargs):
+        self.redis_url = redis_url
+        self.config = {**self.DEFAULT_CONFIG, **kwargs}
+        self.logger = get_logger(self.__class__.__name__)
+        self.error_handler = ErrorHandler(self.__class__.__name__)
+        
+        # 连接池实例
+        self._connection_pool: Optional[aioredis.ConnectionPool] = None
+        self._redis_client: Optional[aioredis.Redis] = None
+        
+        # 连接池统计信息
+        self._stats = {
+            'created_connections': 0,
+            'active_connections': 0,
+            'idle_connections': 0,
+            'errors': 0
+        }
+        
+        # 初始化连接池
+        self._initialize_pool()
+    
+    def _initialize_pool(self):
+        """初始化连接池"""
+        try:
+            self._connection_pool = aioredis.ConnectionPool.from_url(
+                self.redis_url,
+                **self.config
+            )
+            
+            self._redis_client = aioredis.Redis(
+                connection_pool=self._connection_pool
+            )
+            
+            self.logger.info(f"✅ Redis连接池初始化成功: {self.redis_url}")
+            self.logger.debug(f"   连接池配置: {self.config}")
+            
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="Redis连接池初始化失败", 
+                raise_error=True
+            )
+    
+    async def get_connection(self) -> aioredis.Redis:
+        """
+        获取Redis连接实例
+        
+        Returns:
+            Redis连接实例
+        """
+        if not self._redis_client:
+            await self._initialize_pool()
+        
+        self._stats['active_connections'] += 1
+        return self._redis_client
+    
+    async def ping(self) -> bool:
+        """
+        检查Redis连接是否正常
+        
+        Returns:
+            连接是否正常
+        """
+        try:
+            if self._redis_client:
+                await self._redis_client.ping()
+                return True
+            return False
+        except Exception as e:
+            self.logger.warning(f"Redis连接检查失败: {e}")
+            return False
+    
+    async def close(self):
+        """关闭连接池"""
+        try:
+            if self._redis_client:
+                await self._redis_client.close()
+                self._redis_client = None
+            
+            if self._connection_pool:
+                await self._connection_pool.disconnect()
+                self._connection_pool = None
+                
+            self.logger.info("✅ Redis连接池已关闭")
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="关闭Redis连接池失败", 
+                raise_error=False
+            )
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """
+        获取连接池统计信息
+        
+        Returns:
+            统计信息字典
+        """
+        if self._connection_pool:
+            pool_stats = {
+                'max_connections': self._connection_pool.max_connections,
+                'created_connections': self._connection_pool.created_connections,
+                'available_connections': len(self._connection_pool._available_connections),
+                'in_use_connections': len(self._connection_pool._in_use_connections),
+            }
+            self._stats.update(pool_stats)
+        
+        return self._stats.copy()
+    
+    @asynccontextmanager
+    async def connection_context(self):
+        """
+        连接上下文管理器
+        
+        Yields:
+            Redis连接实例
+        """
+        connection = await self.get_connection()
+        try:
+            yield connection
+        finally:
+            self._stats['active_connections'] -= 1
+            self._stats['idle_connections'] += 1
+
+
+class RedisBatchOperationHelper:
+    """Redis批量操作助手"""
+    
+    def __init__(self, redis_client: aioredis.Redis, batch_size: int = 100):
+        self.redis_client = redis_client
+        self.batch_size = batch_size
+        self.logger = get_logger(self.__class__.__name__)
+        self.error_handler = ErrorHandler(self.__class__.__name__)
+    
+    async def batch_execute(self, operations: list, batch_size: Optional[int] = None) -> list:
+        """
+        批量执行Redis操作
+        
+        Args:
+            operations: 操作列表，每个操作是一个包含(command, *args)的元组
+            batch_size: 批次大小（如果为None则使用实例的batch_size）
+            
+        Returns:
+            执行结果列表
+        """
+        actual_batch_size = batch_size or self.batch_size
+        results = []
+        
+        try:
+            for i in range(0, len(operations), actual_batch_size):
+                batch = operations[i:i + actual_batch_size]
+                self.logger.debug(f"执行批次 {i//actual_batch_size + 1}/{(len(operations)-1)//actual_batch_size + 1}")
+                
+                try:
+                    pipe = self.redis_client.pipeline()
+                    for operation in batch:
+                        command, *args = operation
+                        getattr(pipe, command)(*args)
+                    
+                    batch_results = await pipe.execute()
+                    results.extend(batch_results)
+                    
+                except Exception as e:
+                    self.logger.error(f"执行批次失败: {e}")
+                    # 继续执行下一个批次而不是中断
+        
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="Redis批量操作执行失败", 
+                raise_error=False
+            )
+        
+        return results
+    
+    async def batch_set_hash(self, hash_key: str, items: Dict[str, Any]) -> int:
+        """
+        批量设置Hash字段
+        
+        Args:
+            hash_key: Hash键名
+            items: 要设置的字段字典
+            
+        Returns:
+            成功设置的字段数量
+        """
+        try:
+            if not items:
+                return 0
+            
+            pipe = self.redis_client.pipeline()
+            count = 0
+            
+            for key, value in items.items():
+                pipe.hset(hash_key, key, value)
+                count += 1
+                
+                # 每达到批次大小就执行一次
+                if count % self.batch_size == 0:
+                    await pipe.execute()
+                    pipe = self.redis_client.pipeline()
+            
+            # 执行剩余的操作
+            if count % self.batch_size != 0:
+                await pipe.execute()
+            
+            self.logger.debug(f"批量设置Hash {count} 个字段")
+            return count
+            
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="Redis批量设置Hash失败", 
+                raise_error=False
+            )
+            return 0
+    
+    async def batch_get_hash(self, hash_key: str, fields: list) -> Dict[str, Any]:
+        """
+        批量获取Hash字段值
+        
+        Args:
+            hash_key: Hash键名
+            fields: 要获取的字段列表
+            
+        Returns:
+            字段值字典
+        """
+        try:
+            if not fields:
+                return {}
+            
+            # 使用管道批量获取
+            pipe = self.redis_client.pipeline()
+            for field in fields:
+                pipe.hget(hash_key, field)
+            
+            results = await pipe.execute()
+            
+            # 构建结果字典
+            result_dict = {}
+            for i, field in enumerate(fields):
+                if results[i] is not None:
+                    result_dict[field] = results[i]
+            
+            self.logger.debug(f"批量获取Hash {len(result_dict)} 个字段")
+            return result_dict
+            
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="Redis批量获取Hash失败", 
+                raise_error=False
+            )
+            return {}
+
+
+# 全局连接池管理器
+_connection_pools: Dict[str, OptimizedRedisConnectionPool] = {}
+
+
+def get_redis_pool(redis_url: str, **kwargs) -> OptimizedRedisConnectionPool:
+    """
+    获取Redis连接池实例（单例模式）
+    
+    Args:
+        redis_url: Redis URL
+        **kwargs: 连接池配置参数
+        
+    Returns:
+        Redis连接池实例
+    """
+    if redis_url not in _connection_pools:
+        _connection_pools[redis_url] = OptimizedRedisConnectionPool(redis_url, **kwargs)
+    
+    return _connection_pools[redis_url]
+
+
+async def close_all_pools():
+    """关闭所有连接池"""
+    global _connection_pools
+    
+    for pool in _connection_pools.values():
+        await pool.close()
+    
+    _connection_pools.clear()
+
+
+# 便捷函数
+async def execute_redis_batch(redis_url: str, operations: list, batch_size: int = 100) -> list:
+    """
+    便捷函数：执行Redis批量操作
+    
+    Args:
+        redis_url: Redis URL
+        operations: 操作列表
+        batch_size: 批次大小
+        
+    Returns:
+        执行结果列表
+    """
+    pool = get_redis_pool(redis_url)
+    redis_client = await pool.get_connection()
+    helper = RedisBatchOperationHelper(redis_client, batch_size)
+    return await helper.batch_execute(operations)