crawlo 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl

crawlo/utils/log.py

@@ -1,129 +1,129 @@
-# -*- coding: UTF-8 -*-
-"""
-日志管理器：安全版本，使用字符串化 key 避免 unhashable 问题
-"""
-import os
-from logging import (
-    Formatter,
-    StreamHandler,
-    FileHandler,
-    Logger,
-    DEBUG,
-    INFO,
-    WARNING,
-    ERROR,
-    CRITICAL,
-)
-
-LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s： %(message)s'
-
-
-class LoggerManager:
-    logger_cache = {}
-    _default_filename = None
-    _default_level = INFO
-    _default_file_level = INFO
-    _default_console_level = INFO
-    _default_log_format = LOG_FORMAT
-    _default_encoding = 'utf-8'
-
-    _level_map = {
-        'DEBUG': DEBUG,
-        'INFO': INFO,
-        'WARNING': WARNING,
-        'ERROR': ERROR,
-        'CRITICAL': CRITICAL,
-    }
-
-    @classmethod
-    def _to_level(cls, level):
-        """安全转换为日志级别 int"""
-        if level is None:
-            return INFO
-        if isinstance(level, int):
-            return level
-        if isinstance(level, str):
-            return cls._level_map.get(level.upper(), INFO)
-        if hasattr(level, 'get'):  # 如 SettingManager 或 dict
-            lv = level.get('LOG_LEVEL')
-            if isinstance(lv, int):
-                return lv
-            if isinstance(lv, str):
-                return cls._level_map.get(lv.upper(), INFO)
-        return INFO
-
-    @classmethod
-    def configure(cls, settings=None, **kwargs):
-        """
-        使用 settings 对象或关键字参数配置日志
-        """
-        # 优先使用 settings，否则用 kwargs
-        get_val = settings.get if hasattr(settings, 'get') else (lambda k, d=None: kwargs.get(k, d))
-
-        filename = get_val('LOG_FILE')
-        level = get_val('LOG_LEVEL', 'INFO')
-        file_level = get_val('LOG_FILE_LEVEL', level)
-        console_level = get_val('LOG_CONSOLE_LEVEL', level)
-        log_format = get_val('LOG_FORMAT', LOG_FORMAT)
-        encoding = get_val('LOG_ENCODING', 'utf-8')
-
-        cls._default_filename = filename
-        cls._default_level = cls._to_level(level)
-        cls._default_file_level = cls._to_level(file_level)
-        cls._default_console_level = cls._to_level(console_level)
-        cls._default_log_format = log_format
-        cls._default_encoding = encoding
-
-    @classmethod
-    def get_logger(cls, name='default', level=None, filename=None):
-        """
-        简化接口，只暴露必要参数
-        """
-        # 确定最终参数
-        final_level = cls._to_level(level) if level is not None else cls._default_level
-        final_filename = filename if filename is not None else cls._default_filename
-
-        # ✅ 安全的字符串化 key，避免任何 unhashable 类型
-        key_parts = [
-            name,
-            str(final_level),
-            final_filename or 'no_file',
-        ]
-        key = '|'.join(key_parts)  # 如 "my_spider|20|logs/app.log"
-
-        if key in cls.logger_cache:
-            return cls.logger_cache[key]
-
-        # 创建 logger
-        _logger = Logger(name=name)
-        _logger.setLevel(final_level)
-
-        formatter = Formatter(cls._default_log_format)
-
-        # 控制台
-        if cls._default_console_level is not False:
-            ch = StreamHandler()
-            ch.setFormatter(formatter)
-            ch.setLevel(cls._default_console_level)
-            _logger.addHandler(ch)
-
-        # 文件
-        if final_filename:
-            try:
-                log_dir = os.path.dirname(final_filename)
-                if log_dir and not os.path.exists(log_dir):
-                    os.makedirs(log_dir, exist_ok=True)
-
-                fh = FileHandler(final_filename, encoding=cls._default_encoding)
-                fh.setFormatter(formatter)
-                fh.setLevel(cls._default_file_level)
-                _logger.addHandler(fh)
-            except Exception as e:
-                print(f"[Logger] 无法创建日志文件 {final_filename}: {e}")
-
-        cls.logger_cache[key] = _logger
-        return _logger
-
-
-# 全局快捷函数
+# -*- coding: UTF-8 -*-
+"""
+日志管理器：安全版本，使用字符串化 key 避免 unhashable 问题
+"""
+import os
+from logging import (
+    Formatter,
+    StreamHandler,
+    FileHandler,
+    Logger,
+    DEBUG,
+    INFO,
+    WARNING,
+    ERROR,
+    CRITICAL,
+)
+
+LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s： %(message)s'
+
+
+class LoggerManager:
+    logger_cache = {}
+    _default_filename = None
+    _default_level = INFO
+    _default_file_level = INFO
+    _default_console_level = INFO
+    _default_log_format = LOG_FORMAT
+    _default_encoding = 'utf-8'
+
+    _level_map = {
+        'DEBUG': DEBUG,
+        'INFO': INFO,
+        'WARNING': WARNING,
+        'ERROR': ERROR,
+        'CRITICAL': CRITICAL,
+    }
+
+    @classmethod
+    def _to_level(cls, level):
+        """安全转换为日志级别 int"""
+        if level is None:
+            return INFO
+        if isinstance(level, int):
+            return level
+        if isinstance(level, str):
+            return cls._level_map.get(level.upper(), INFO)
+        if hasattr(level, 'get'):  # 如 SettingManager 或 dict
+            lv = level.get('LOG_LEVEL')
+            if isinstance(lv, int):
+                return lv
+            if isinstance(lv, str):
+                return cls._level_map.get(lv.upper(), INFO)
+        return INFO
+
+    @classmethod
+    def configure(cls, settings=None, **kwargs):
+        """
+        使用 settings 对象或关键字参数配置日志
+        """
+        # 优先使用 settings，否则用 kwargs
+        get_val = settings.get if hasattr(settings, 'get') else (lambda k, d=None: kwargs.get(k, d))
+
+        filename = get_val('LOG_FILE')
+        level = get_val('LOG_LEVEL', 'INFO')
+        file_level = get_val('LOG_FILE_LEVEL', level)
+        console_level = get_val('LOG_CONSOLE_LEVEL', level)
+        log_format = get_val('LOG_FORMAT', LOG_FORMAT)
+        encoding = get_val('LOG_ENCODING', 'utf-8')
+
+        cls._default_filename = filename
+        cls._default_level = cls._to_level(level)
+        cls._default_file_level = cls._to_level(file_level)
+        cls._default_console_level = cls._to_level(console_level)
+        cls._default_log_format = log_format
+        cls._default_encoding = encoding
+
+    @classmethod
+    def get_logger(cls, name='default', level=None, filename=None):
+        """
+        简化接口，只暴露必要参数
+        """
+        # 确定最终参数
+        final_level = cls._to_level(level) if level is not None else cls._default_level
+        final_filename = filename if filename is not None else cls._default_filename
+
+        # ✅ 安全的字符串化 key，避免任何 unhashable 类型
+        key_parts = [
+            name,
+            str(final_level),
+            final_filename or 'no_file',
+        ]
+        key = '|'.join(key_parts)  # 如 "my_spider|20|logs/app.log"
+
+        if key in cls.logger_cache:
+            return cls.logger_cache[key]
+
+        # 创建 logger
+        _logger = Logger(name=name)
+        _logger.setLevel(final_level)
+
+        formatter = Formatter(cls._default_log_format)
+
+        # 控制台
+        if cls._default_console_level is not False:
+            ch = StreamHandler()
+            ch.setFormatter(formatter)
+            ch.setLevel(cls._default_console_level)
+            _logger.addHandler(ch)
+
+        # 文件
+        if final_filename:
+            try:
+                log_dir = os.path.dirname(final_filename)
+                if log_dir and not os.path.exists(log_dir):
+                    os.makedirs(log_dir, exist_ok=True)
+
+                fh = FileHandler(final_filename, encoding=cls._default_encoding)
+                fh.setFormatter(formatter)
+                fh.setLevel(cls._default_file_level)
+                _logger.addHandler(fh)
+            except Exception as e:
+                print(f"[Logger] 无法创建日志文件 {final_filename}: {e}")
+
+        cls.logger_cache[key] = _logger
+        return _logger
+
+
+# 全局快捷函数
 get_logger = LoggerManager.get_logger

crawlo/utils/performance_monitor.py

@@ -0,0 +1,285 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+性能监控工具
+提供系统性能监控和资源使用情况跟踪
+"""
+import time
+import psutil
+import asyncio
+from typing import Dict, Any, Optional, Callable
+from functools import wraps
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.error_handler import ErrorHandler
+
+
+class PerformanceMonitor:
+    """性能监控器"""
+    
+    def __init__(self, logger_name: str = __name__):
+        self.logger = get_logger(logger_name)
+        self.error_handler = ErrorHandler(logger_name)
+        self.process = psutil.Process()
+        self.start_time = time.time()
+        
+        # 性能指标
+        self.metrics = {
+            'cpu_usage': [],
+            'memory_usage': [],
+            'network_io': [],
+            'disk_io': []
+        }
+    
+    def get_system_metrics(self) -> Dict[str, Any]:
+        """
+        获取系统性能指标
+        
+        Returns:
+            包含各种性能指标的字典
+        """
+        try:
+            # CPU使用率
+            cpu_percent = psutil.cpu_percent(interval=1)
+            
+            # 内存使用情况
+            memory = psutil.virtual_memory()
+            
+            # 网络IO
+            net_io = psutil.net_io_counters()
+            
+            # 磁盘IO
+            disk_io = psutil.disk_io_counters()
+            
+            # 进程特定信息
+            process_memory = self.process.memory_info()
+            process_cpu = self.process.cpu_percent()
+            
+            return {
+                'timestamp': time.time(),
+                'uptime': time.time() - self.start_time,
+                'cpu': {
+                    'percent': cpu_percent,
+                    'count': psutil.cpu_count(),
+                    'freq': psutil.cpu_freq()._asdict() if psutil.cpu_freq() else {}
+                },
+                'memory': {
+                    'total': memory.total,
+                    'available': memory.available,
+                    'percent': memory.percent,
+                    'used': memory.used,
+                    'free': memory.free
+                },
+                'process': {
+                    'memory_rss': process_memory.rss,
+                    'memory_vms': process_memory.vms,
+                    'cpu_percent': process_cpu,
+                    'num_threads': self.process.num_threads(),
+                    'num_fds': self.process.num_fds() if hasattr(self.process, 'num_fds') else 0
+                },
+                'network': {
+                    'bytes_sent': net_io.bytes_sent,
+                    'bytes_recv': net_io.bytes_recv,
+                    'packets_sent': net_io.packets_sent,
+                    'packets_recv': net_io.packets_recv
+                },
+                'disk': {
+                    'read_bytes': disk_io.read_bytes,
+                    'write_bytes': disk_io.write_bytes,
+                    'read_count': disk_io.read_count,
+                    'write_count': disk_io.write_count
+                }
+            }
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="获取系统性能指标失败", 
+                raise_error=False
+            )
+            return {}
+    
+    def log_system_metrics(self, detailed: bool = False):
+        """
+        记录系统性能指标
+        
+        Args:
+            detailed: 是否记录详细信息
+        """
+        try:
+            metrics = self.get_system_metrics()
+            if not metrics:
+                return
+            
+            # 基本信息
+            basic_info = (
+                f"📊 系统性能指标 | "
+                f"CPU: {metrics['cpu']['percent']:.1f}% | "
+                f"内存: {metrics['memory']['percent']:.1f}% | "
+                f"进程CPU: {metrics['process']['cpu_percent']:.1f}% | "
+                f"进程内存: {metrics['process']['memory_rss'] / 1024 / 1024:.1f}MB"
+            )
+            self.logger.info(basic_info)
+            
+            # 详细信息
+            if detailed:
+                detailed_info = (
+                    f"   详细信息:\n"
+                    f"   - CPU: {metrics['cpu']['count']} 核心\n"
+                    f"   - 内存: 总计 {metrics['memory']['total'] / 1024 / 1024 / 1024:.1f}GB, "
+                    f"可用 {metrics['memory']['available'] / 1024 / 1024 / 1024:.1f}GB\n"
+                    f"   - 网络: 发送 {metrics['network']['bytes_sent'] / 1024 / 1024:.1f}MB, "
+                    f"接收 {metrics['network']['bytes_recv'] / 1024 / 1024:.1f}MB\n"
+                    f"   - 磁盘: 读取 {metrics['disk']['read_bytes'] / 1024 / 1024:.1f}MB, "
+                    f"写入 {metrics['disk']['write_bytes'] / 1024 / 1024:.1f}MB"
+                )
+                self.logger.debug(detailed_info)
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="记录系统性能指标失败", 
+                raise_error=False
+            )
+    
+    def start_monitoring(self, interval: int = 60, detailed: bool = False):
+        """
+        开始定期监控
+        
+        Args:
+            interval: 监控间隔（秒）
+            detailed: 是否记录详细信息
+        """
+        async def monitor_loop():
+            while True:
+                try:
+                    self.log_system_metrics(detailed)
+                    await asyncio.sleep(interval)
+                except asyncio.CancelledError:
+                    break
+                except Exception as e:
+                    self.logger.error(f"监控循环错误: {e}")
+        
+        # 启动监控任务
+        self.monitor_task = asyncio.create_task(monitor_loop())
+        self.logger.info(f"开始性能监控，间隔: {interval}秒")
+    
+    async def stop_monitoring(self):
+        """停止监控"""
+        if hasattr(self, 'monitor_task') and self.monitor_task:
+            self.monitor_task.cancel()
+            try:
+                await self.monitor_task
+            except asyncio.CancelledError:
+                pass
+            self.logger.info("性能监控已停止")
+
+
+class PerformanceTimer:
+    """性能计时器"""
+    
+    def __init__(self, name: str = "timer"):
+        self.name = name
+        self.start_time = None
+        self.end_time = None
+        self.logger = get_logger(f"{__name__}.{self.__class__.__name__}")
+        self.error_handler = ErrorHandler(f"{__name__}.{self.__class__.__name__}")
+    
+    def start(self):
+        """开始计时"""
+        self.start_time = time.time()
+        self.logger.debug(f"⏱️  开始计时: {self.name}")
+    
+    def stop(self) -> float:
+        """
+        停止计时并返回耗时
+        
+        Returns:
+            耗时（秒）
+        """
+        self.end_time = time.time()
+        if self.start_time is None:
+            raise RuntimeError("计时器未启动")
+        
+        elapsed = self.end_time - self.start_time
+        self.logger.debug(f"⏱️  停止计时: {self.name}, 耗时: {elapsed:.3f}秒")
+        return elapsed
+    
+    def __enter__(self):
+        self.start()
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        try:
+            elapsed = self.stop()
+            if exc_type is None:
+                self.logger.info(f"✅ {self.name} 执行成功，耗时: {elapsed:.3f}秒")
+            else:
+                self.logger.error(f"❌ {self.name} 执行失败，耗时: {elapsed:.3f}秒")
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context=f"计时器退出时发生错误: {self.name}", 
+                raise_error=False
+            )
+
+
+def performance_monitor_decorator(name: str = None, log_level: str = "INFO"):
+    """
+    装饰器：监控函数性能
+    
+    Args:
+        name: 函数名称（如果为None则使用函数名）
+        log_level: 日志级别
+    """
+    def decorator(func):
+        @wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            timer_name = name or f"{func.__module__}.{func.__name__}"
+            logger = get_logger(timer_name)
+            
+            with PerformanceTimer(timer_name) as timer:
+                if asyncio.iscoroutinefunction(func):
+                    return await func(*args, **kwargs)
+                else:
+                    return func(*args, **kwargs)
+        
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            timer_name = name or f"{func.__module__}.{func.__name__}"
+            logger = get_logger(timer_name)
+            
+            with PerformanceTimer(timer_name) as timer:
+                return func(*args, **kwargs)
+        
+        # 根据函数是否为异步函数返回相应的包装器
+        import inspect
+        if inspect.iscoroutinefunction(func):
+            return async_wrapper
+        else:
+            return sync_wrapper
+    
+    return decorator
+
+
+# 全局性能监控器实例
+default_performance_monitor = PerformanceMonitor()
+
+
+def monitor_performance(interval: int = 60, detailed: bool = False):
+    """
+    便捷函数：开始性能监控
+    
+    Args:
+        interval: 监控间隔（秒）
+        detailed: 是否记录详细信息
+    """
+    default_performance_monitor.start_monitoring(interval, detailed)
+
+
+def get_current_metrics() -> Dict[str, Any]:
+    """
+    便捷函数：获取当前性能指标
+    
+    Returns:
+        性能指标字典
+    """
+    return default_performance_monitor.get_system_metrics()