crawlo 1.4.6__py3-none-any.whl → 1.4.8__py3-none-any.whl

tests/test_cli_arguments.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试CLI参数解析
+验证crawlo run命令是否正确解析--log-level、--config和--concurrency参数
+"""
+
+import sys
+import os
+import unittest
+from unittest.mock import patch, MagicMock
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.commands.run import main as run_main
+
+
+class TestCLIArguments(unittest.TestCase):
+    
+    def setUp(self):
+        """测试前准备"""
+        # 保存原始的sys.path
+        self.original_path = sys.path[:]
+        
+    def tearDown(self):
+        """测试后清理"""
+        # 恢复原始的sys.path
+        sys.path = self.original_path[:]
+    
+    @patch('crawlo.commands.run._find_project_root')
+    @patch('crawlo.commands.run.initialize_framework')
+    @patch('crawlo.commands.run.CrawlerProcess')
+    def test_log_level_argument(self, mock_crawler_process, mock_initialize, mock_find_project):
+        """测试--log-level参数解析"""
+        # 模拟项目环境
+        mock_find_project.return_value = os.path.join(os.path.dirname(__file__), '..')
+        mock_initialize.return_value = {}
+        mock_process_instance = MagicMock()
+        mock_crawler_process.return_value = mock_process_instance
+        mock_process_instance.get_spider_names.return_value = ['test_spider']
+        mock_process_instance.is_spider_registered.return_value = True
+        mock_process_instance.get_spider_class.return_value = MagicMock(__name__='TestSpider')
+        
+        # 测试参数解析
+        args = ['test_spider', '--log-level', 'DEBUG']
+        result = run_main(args)
+        
+        # 验证initialize_framework被调用时传入了正确的日志级别
+        mock_initialize.assert_called_once()
+        call_args = mock_initialize.call_args
+        if call_args and call_args[0]:  # 检查位置参数
+            settings = call_args[0][0]
+            self.assertEqual(settings.get('LOG_LEVEL'), 'DEBUG')
+        elif call_args and call_args[1]:  # 检查关键字参数
+            settings = call_args[1].get('custom_settings', {})
+            self.assertEqual(settings.get('LOG_LEVEL'), 'DEBUG')
+    
+    @patch('crawlo.commands.run._find_project_root')
+    @patch('crawlo.commands.run.initialize_framework')
+    @patch('crawlo.commands.run.CrawlerProcess')
+    def test_concurrency_argument(self, mock_crawler_process, mock_initialize, mock_find_project):
+        """测试--concurrency参数解析"""
+        # 模拟项目环境
+        mock_find_project.return_value = os.path.join(os.path.dirname(__file__), '..')
+        mock_initialize.return_value = {}
+        mock_process_instance = MagicMock()
+        mock_crawler_process.return_value = mock_process_instance
+        mock_process_instance.get_spider_names.return_value = ['test_spider']
+        mock_process_instance.is_spider_registered.return_value = True
+        mock_process_instance.get_spider_class.return_value = MagicMock(__name__='TestSpider')
+        
+        # 测试参数解析
+        args = ['test_spider', '--concurrency', '32']
+        result = run_main(args)
+        
+        # 验证initialize_framework被调用时传入了正确的并发数
+        mock_initialize.assert_called_once()
+        call_args = mock_initialize.call_args
+        if call_args and call_args[0]:  # 检查位置参数
+            settings = call_args[0][0]
+            self.assertEqual(settings.get('CONCURRENCY'), 32)
+        elif call_args and call_args[1]:  # 检查关键字参数
+            settings = call_args[1].get('custom_settings', {})
+            self.assertEqual(settings.get('CONCURRENCY'), 32)
+    
+    @patch('crawlo.commands.run._find_project_root')
+    @patch('crawlo.commands.run.initialize_framework')
+    @patch('crawlo.commands.run.CrawlerProcess')
+    def test_combined_arguments(self, mock_crawler_process, mock_initialize, mock_find_project):
+        """测试组合参数解析"""
+        # 模拟项目环境
+        mock_find_project.return_value = os.path.join(os.path.dirname(__file__), '..')
+        mock_initialize.return_value = {}
+        mock_process_instance = MagicMock()
+        mock_crawler_process.return_value = mock_process_instance
+        mock_process_instance.get_spider_names.return_value = ['test_spider']
+        mock_process_instance.is_spider_registered.return_value = True
+        mock_process_instance.get_spider_class.return_value = MagicMock(__name__='TestSpider')
+        
+        # 测试参数解析
+        args = ['test_spider', '--log-level', 'DEBUG', '--concurrency', '16']
+        result = run_main(args)
+        
+        # 验证initialize_framework被调用时传入了正确的参数
+        mock_initialize.assert_called_once()
+        call_args = mock_initialize.call_args
+        if call_args and call_args[0]:  # 检查位置参数
+            settings = call_args[0][0]
+            self.assertEqual(settings.get('LOG_LEVEL'), 'DEBUG')
+            self.assertEqual(settings.get('CONCURRENCY'), 16)
+        elif call_args and call_args[1]:  # 检查关键字参数
+            settings = call_args[1].get('custom_settings', {})
+            self.assertEqual(settings.get('LOG_LEVEL'), 'DEBUG')
+            self.assertEqual(settings.get('CONCURRENCY'), 16)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_dedup_fix.py

@@ -19,7 +19,7 @@ from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
 from crawlo.pipelines.memory_dedup_pipeline import MemoryDedupPipeline
 from crawlo.pipelines.bloom_dedup_pipeline import BloomDedupPipeline
 from crawlo.pipelines.pipeline_manager import PipelineManager
-from crawlo.exceptions import ItemDiscard, DropItem
+from crawlo.exceptions import ItemDiscard
 
 
 class TestDedupFix(unittest.TestCase):
@@ -61,7 +61,7 @@ class TestDedupFix(unittest.TestCase):
                 log_level='INFO'
             )
             
-            # 验证抛出的是ItemDiscard异常而不是DropItem异常
+            # 验证抛出的是ItemDiscard异常
             with self.assertRaises(ItemDiscard) as context:
                 pipeline.process_item(self.test_item, Mock())
             
@@ -77,7 +77,7 @@ class TestDedupFix(unittest.TestCase):
         fingerprint = pipeline._generate_item_fingerprint(self.test_item)
         pipeline.seen_items.add(fingerprint)
         
-        # 验证抛出的是ItemDiscard异常而不是DropItem异常
+        # 验证抛出的是ItemDiscard异常
         with self.assertRaises(ItemDiscard) as context:
             pipeline.process_item(self.test_item, Mock())
         
@@ -93,7 +93,7 @@ class TestDedupFix(unittest.TestCase):
         fingerprint = pipeline._generate_item_fingerprint(self.test_item)
         pipeline.bloom_filter.add(fingerprint)
         
-        # 验证抛出的是ItemDiscard异常而不是DropItem异常
+        # 验证抛出的是ItemDiscard异常
         with self.assertRaises(ItemDiscard) as context:
             pipeline.process_item(self.test_item, Mock())
         
@@ -101,7 +101,7 @@ class TestDedupFix(unittest.TestCase):
         self.assertIn("可能重复的数据项:", str(context.exception))
 
     async def test_pipeline_manager_exception_handling(self):
-        """测试管道管理器能正确处理两种异常类型"""
+        """测试管道管理器能正确处理ItemDiscard异常"""
         # 创建管道管理器实例
         pipeline_manager = PipelineManager(self.mock_crawler)
         
@@ -148,7 +148,7 @@ class TestDedupFix(unittest.TestCase):
             mock_mysql_method.assert_not_called()
 
     async def test_pipeline_manager_dropitem_exception_handling(self):
-        """测试管道管理器能正确处理DropItem异常"""
+        """测试管道管理器能正确处理ItemDiscard异常（重复测试）"""
         # 创建管道管理器实例
         pipeline_manager = PipelineManager(self.mock_crawler)
         
@@ -162,9 +162,9 @@ class TestDedupFix(unittest.TestCase):
         # 模拟管道方法列表
         pipeline_manager.methods = []
         
-        # 创建模拟的去重管道方法（抛出DropItem异常）
+        # 创建模拟的去重管道方法（抛出ItemDiscard异常）
         mock_dedup_method = Mock()
-        mock_dedup_method.side_effect = DropItem("测试DropItem异常")
+        mock_dedup_method.side_effect = ItemDiscard("测试ItemDiscard异常")
         
         # 创建模拟的MySQL管道方法
         mock_mysql_method = Mock()
@@ -179,7 +179,7 @@ class TestDedupFix(unittest.TestCase):
             # 设置common_call的副作用来模拟异常
             async def mock_common_call_func(method, *args, **kwargs):
                 if method == mock_dedup_method:
-                    raise DropItem("测试DropItem异常")
+                    raise ItemDiscard("测试ItemDiscard异常")
                 return test_item
                 
             mock_common_call.side_effect = mock_common_call_func
@@ -187,7 +187,7 @@ class TestDedupFix(unittest.TestCase):
             # 调用处理方法
             await pipeline_manager.process_item(test_item)
             
-            # 验证DropItem异常被正确处理
+            # 验证ItemDiscard异常被正确处理
             # 验证create_task被调用了一次（item_discard事件）
             self.assertEqual(mock_create_task.call_count, 1)
             

crawlo/logging/async_handler.py

@@ -1,181 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-异步日志处理器
-用于提高日志写入性能
-"""
-
-import asyncio
-import logging
-import threading
-import queue
-from typing import Optional
-from concurrent_log_handler import ConcurrentRotatingFileHandler
-
-
-class AsyncLogHandler(logging.Handler):
-    """
-    异步日志处理器
-    将日志记录放入队列中，由后台线程异步处理
-    """
-    
-    def __init__(self, handler: logging.Handler, queue_size: int = 10000):
-        """
-        初始化异步日志处理器
-        
-        Args:
-            handler: 实际的日志处理器
-            queue_size: 队列大小
-        """
-        super().__init__()
-        self._handler = handler
-        self._queue = queue.Queue(maxsize=queue_size)
-        self._thread: Optional[threading.Thread] = None
-        self._stop_event = threading.Event()
-        self._started = False
-        
-    def start(self):
-        """启动异步处理线程"""
-        if self._started:
-            return
-            
-        self._started = True
-        self._stop_event.clear()
-        self._thread = threading.Thread(target=self._worker, daemon=True)
-        self._thread.start()
-        
-    def stop(self):
-        """停止异步处理线程"""
-        if not self._started:
-            return
-            
-        self._started = False
-        self._stop_event.set()
-        
-        # 发送一个哨兵消息来唤醒工作线程
-        try:
-            self._queue.put_nowait(None)
-        except queue.Full:
-            pass
-            
-        # 等待线程结束
-        if self._thread and self._thread.is_alive():
-            self._thread.join(timeout=5.0)
-            
-        # 关闭底层处理器
-        if self._handler:
-            self._handler.close()
-            
-    def _worker(self):
-        """工作线程函数"""
-        while not self._stop_event.is_set():
-            try:
-                # 从队列中获取日志记录
-                record = self._queue.get(timeout=1.0)
-                
-                # 哨兵消息，表示停止
-                if record is None:
-                    break
-                    
-                # 处理日志记录
-                try:
-                    self._handler.emit(record)
-                except Exception:
-                    pass  # 忽略处理错误
-                    
-                self._queue.task_done()
-                
-            except queue.Empty:
-                continue
-            except Exception:
-                if not self._stop_event.is_set():
-                    continue
-                else:
-                    break
-                    
-    def emit(self, record):
-        """
-        发出日志记录
-        
-        Args:
-            record: 日志记录
-        """
-        if not self._started:
-            self.start()
-            
-        # 将日志记录放入队列
-        try:
-            self._queue.put_nowait(record)
-        except queue.Full:
-            # 队列满时丢弃日志记录
-            pass
-            
-    def flush(self):
-        """刷新日志处理器"""
-        if self._handler:
-            self._handler.flush()
-            
-    def close(self):
-        """关闭日志处理器"""
-        self.stop()
-        super().close()
-
-
-class AsyncConcurrentRotatingFileHandler(AsyncLogHandler):
-    """
-    异步并发轮转文件处理器
-    结合了异步处理和并发轮转文件的功能
-    """
-    
-    def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, 
-                 encoding=None, delay=False, queue_size: int = 10000):
-        """
-        初始化异步并发轮转文件处理器
-        
-        Args:
-            filename: 日志文件名
-            mode: 文件打开模式
-            maxBytes: 最大文件大小
-            backupCount: 备份文件数量
-            encoding: 文件编码
-            delay: 是否延迟打开文件
-            queue_size: 队列大小
-        """
-        handler = ConcurrentRotatingFileHandler(
-            filename=filename,
-            mode=mode,
-            maxBytes=maxBytes,
-            backupCount=backupCount,
-            encoding=encoding,
-            delay=delay
-        )
-        super().__init__(handler, queue_size)
-        
-    @property
-    def baseFilename(self):
-        """获取基础文件名"""
-        return self._handler.baseFilename if self._handler else None
-        
-    @property
-    def maxBytes(self):
-        """获取最大字节数"""
-        return self._handler.maxBytes if self._handler else 0
-        
-    @property
-    def backupCount(self):
-        """获取备份计数"""
-        return self._handler.backupCount if self._handler else 0
-
-
-def wrap_handler_async(handler: logging.Handler, queue_size: int = 10000) -> AsyncLogHandler:
-    """
-    将现有的日志处理器包装为异步处理器
-    
-    Args:
-        handler: 要包装的日志处理器
-        queue_size: 队列大小
-        
-    Returns:
-        异步日志处理器
-    """
-    return AsyncLogHandler(handler, queue_size)

crawlo/logging/monitor.py

@@ -1,153 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-日志性能监控器
-"""
-
-import time
-import threading
-from typing import Dict, List
-from collections import defaultdict, deque
-from .manager import get_config
-
-
-class LogPerformanceMonitor:
-    """
-    日志性能监控器
-    用于监控日志系统的性能指标
-    """
-    
-    def __init__(self):
-        self._lock = threading.Lock()
-        self._log_stats: Dict[str, deque] = defaultdict(lambda: deque(maxlen=1000))
-        self._enabled = False
-        
-    def enable_monitoring(self):
-        """启用性能监控"""
-        with self._lock:
-            self._enabled = True
-            
-    def disable_monitoring(self):
-        """禁用性能监控"""
-        with self._lock:
-            self._enabled = False
-            
-    def record_log_event(self, logger_name: str, level: str, message: str):
-        """
-        记录日志事件
-        
-        Args:
-            logger_name: Logger名称
-            level: 日志级别
-            message: 日志消息
-        """
-        if not self._enabled:
-            return
-            
-        with self._lock:
-            event = {
-                'timestamp': time.time(),
-                'level': level,
-                'message_length': len(message),
-                'thread_id': threading.get_ident()
-            }
-            self._log_stats[logger_name].append(event)
-            
-    def get_statistics(self, logger_name: str = None) -> Dict:
-        """
-        获取日志统计信息
-        
-        Args:
-            logger_name: Logger名称，如果为None则返回所有统计信息
-            
-        Returns:
-            统计信息字典
-        """
-        with self._lock:
-            if logger_name:
-                return self._calculate_stats(logger_name, self._log_stats[logger_name])
-            else:
-                result = {}
-                for name, events in self._log_stats.items():
-                    result[name] = self._calculate_stats(name, events)
-                return result
-                
-    def _calculate_stats(self, logger_name: str, events: deque) -> Dict:
-        """计算统计信息"""
-        if not events:
-            return {
-                'logger_name': logger_name,
-                'total_logs': 0,
-                'log_rates': {},
-                'avg_message_length': 0
-            }
-            
-        # 计算日志级别分布
-        level_counts = defaultdict(int)
-        total_length = 0
-        
-        for event in events:
-            level_counts[event['level']] += 1
-            total_length += event['message_length']
-            
-        # 计算日志速率（每分钟）
-        if len(events) > 1:
-            time_span = events[-1]['timestamp'] - events[0]['timestamp']
-            if time_span > 0:
-                logs_per_minute = len(events) / (time_span / 60)
-            else:
-                logs_per_minute = len(events) * 60
-        else:
-            logs_per_minute = 0
-            
-        return {
-            'logger_name': logger_name,
-            'total_logs': len(events),
-            'log_rates': {
-                'per_minute': logs_per_minute
-            },
-            'level_distribution': dict(level_counts),
-            'avg_message_length': total_length / len(events) if events else 0
-        }
-        
-    def get_performance_report(self) -> str:
-        """
-        获取性能报告
-        
-        Returns:
-            格式化的性能报告字符串
-        """
-        stats = self.get_statistics()
-        
-        report = ["=" * 50]
-        report.append("日志系统性能报告")
-        report.append("=" * 50)
-        
-        config = get_config()
-        if config:
-            report.append(f"日志文件: {config.file_path or 'N/A'}")
-            report.append(f"文件启用: {config.file_enabled}")
-            report.append(f"控制台启用: {config.console_enabled}")
-            report.append("-" * 50)
-        
-        for logger_name, stat in stats.items():
-            report.append(f"Logger: {logger_name}")
-            report.append(f"  总日志数: {stat['total_logs']}")
-            report.append(f"  日志速率: {stat['log_rates']['per_minute']:.2f} 条/分钟")
-            report.append(f"  平均消息长度: {stat['avg_message_length']:.2f} 字符")
-            
-            if 'level_distribution' in stat:
-                levels = ", ".join([f"{k}: {v}" for k, v in stat['level_distribution'].items()])
-                report.append(f"  级别分布: {levels}")
-            report.append("")
-            
-        return "\n".join(report)
-
-
-# 全局实例
-_log_monitor = LogPerformanceMonitor()
-
-
-def get_monitor() -> LogPerformanceMonitor:
-    """获取日志监控器实例"""
-    return _log_monitor

crawlo/logging/sampler.py

@@ -1,167 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-日志采样器
-用于在高负载时减少日志输出
-"""
-
-import random
-import time
-import threading
-from typing import Dict, Set
-from collections import defaultdict
-
-
-class LogSampler:
-    """
-    日志采样器
-    支持多种采样策略以减少日志输出量
-    """
-    
-    def __init__(self):
-        self._lock = threading.RLock()
-        self._sample_rates: Dict[str, float] = {}  # logger_name -> sample_rate
-        self._rate_limiters: Dict[str, TokenBucket] = {}  # logger_name -> rate_limiter
-        self._message_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
-        self._time_windows: Dict[str, Dict[str, float]] = defaultdict(lambda: defaultdict(float))
-        
-    def set_sample_rate(self, logger_name: str, rate: float):
-        """
-        设置采样率
-        
-        Args:
-            logger_name: Logger名称
-            rate: 采样率 (0.0-1.0)，1.0表示全部记录，0.0表示不记录
-        """
-        with self._lock:
-            self._sample_rates[logger_name] = max(0.0, min(1.0, rate))
-            
-    def set_rate_limit(self, logger_name: str, messages_per_second: float):
-        """
-        设置速率限制
-        
-        Args:
-            logger_name: Logger名称
-            messages_per_second: 每秒最大消息数
-        """
-        with self._lock:
-            self._rate_limiters[logger_name] = TokenBucket(messages_per_second, messages_per_second)
-            
-    def should_log(self, logger_name: str, message: str = None, level: str = None) -> bool:
-        """
-        判断是否应该记录日志
-        
-        Args:
-            logger_name: Logger名称
-            message: 日志消息（用于去重）
-            level: 日志级别
-            
-        Returns:
-            是否应该记录日志
-        """
-        with self._lock:
-            # 检查采样率
-            if logger_name in self._sample_rates:
-                sample_rate = self._sample_rates[logger_name]
-                if random.random() > sample_rate:
-                    return False
-                    
-            # 检查速率限制
-            if logger_name in self._rate_limiters:
-                if not self._rate_limiters[logger_name].consume(1):
-                    return False
-                    
-            # 检查消息去重（相同消息在短时间内只记录一次）
-            if message:
-                key = f"{level}:{message}" if level else message
-                current_time = time.time()
-                
-                # 如果距离上次记录超过60秒，重置计数
-                if current_time - self._time_windows[logger_name][key] > 60:
-                    self._message_counts[logger_name][key] = 0
-                    self._time_windows[logger_name][key] = current_time
-                    
-                # 限制相同消息的记录次数
-                if self._message_counts[logger_name][key] >= 5:  # 最多记录5次相同消息
-                    return False
-                    
-                self._message_counts[logger_name][key] += 1
-                
-            return True
-            
-    def reset(self):
-        """重置采样器状态"""
-        with self._lock:
-            self._sample_rates.clear()
-            self._rate_limiters.clear()
-            self._message_counts.clear()
-            self._time_windows.clear()
-
-
-class TokenBucket:
-    """
-    令牌桶算法实现
-    用于速率限制
-    """
-    
-    def __init__(self, rate: float, capacity: float):
-        """
-        初始化令牌桶
-        
-        Args:
-            rate: 每秒生成的令牌数
-            capacity: 桶的最大容量
-        """
-        self._rate = rate
-        self._capacity = capacity
-        self._tokens = capacity
-        self._last_time = time.time()
-        self._lock = threading.Lock()
-        
-    def consume(self, tokens: float) -> bool:
-        """
-        消费令牌
-        
-        Args:
-            tokens: 要消费的令牌数
-            
-        Returns:
-            是否消费成功
-        """
-        with self._lock:
-            current_time = time.time()
-            # 补充令牌
-            elapsed = current_time - self._last_time
-            self._tokens = min(self._capacity, self._tokens + elapsed * self._rate)
-            self._last_time = current_time
-            
-            # 尝试消费令牌
-            if self._tokens >= tokens:
-                self._tokens -= tokens
-                return True
-            else:
-                return False
-
-
-# 全局实例
-_log_sampler = LogSampler()
-
-
-def get_sampler() -> LogSampler:
-    """获取日志采样器实例"""
-    return _log_sampler
-
-
-def should_log(logger_name: str, message: str = None, level: str = None) -> bool:
-    """
-    判断是否应该记录日志的便捷函数
-    
-    Args:
-        logger_name: Logger名称
-        message: 日志消息
-        level: 日志级别
-        
-    Returns:
-        是否应该记录日志
-    """
-    return get_sampler().should_log(logger_name, message, level)