crawlo 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

tests/simple_optimization_test.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+简单的Crawlo框架优化验证测试
+用于快速验证优化效果
+"""
+
+import asyncio
+import time
+import sys
+import os
+from pathlib import Path
+
+# 添加项目根目录到 Python 路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import Spider
+from crawlo import Request, Item
+
+
+class SimpleTestSpider(Spider):
+    """简单的测试爬虫"""
+    name = 'simple_test'
+    
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # 使用较少的测试页面以加快测试速度
+        self.start_urls = ['https://httpbin.org/delay/0' for _ in range(10)]
+    
+    def parse(self, response):
+        """简单解析响应"""
+        yield {'url': response.url, 'status': response.status_code}
+
+
+async def test_concurrent_performance():
+    """测试并发性能"""
+    print("开始并发性能测试...")
+    
+    # 配置设置
+    settings = {
+        'CONCURRENT_REQUESTS': 10,
+        'DOWNLOAD_DELAY': 0.1,
+        'RANDOMIZE_DOWNLOAD_DELAY': False,
+        'SCHEDULER_MAX_QUEUE_SIZE': 5000,
+        'BACKPRESSURE_RATIO': 0.9,
+    }
+    
+    # 创建爬虫进程
+    process = CrawlerProcess(settings=settings)
+    
+    # 记录开始时间
+    start_time = time.time()
+    
+    # 添加测试爬虫
+    crawler = await process.crawl(SimpleTestSpider)
+    
+    # 计算性能指标
+    metrics = crawler.metrics
+    duration = metrics.get_total_duration()
+    pages = 10
+    rps = pages / duration if duration > 0 else 0
+    
+    print(f"完成时间: {duration:.2f} 秒")
+    print(f"每秒请求数: {rps:.2f} RPS")
+    
+    return duration, rps
+
+
+async def test_sequential_performance():
+    """测试顺序执行性能"""
+    print("\n开始顺序执行性能测试...")
+    
+    # 配置设置
+    settings = {
+        'CONCURRENT_REQUESTS': 1,  # 顺序执行
+        'DOWNLOAD_DELAY': 0.1,
+        'RANDOMIZE_DOWNLOAD_DELAY': False,
+        'SCHEDULER_MAX_QUEUE_SIZE': 5000,
+        'BACKPRESSURE_RATIO': 0.9,
+    }
+    
+    # 创建爬虫进程
+    process = CrawlerProcess(settings=settings)
+    
+    # 记录开始时间
+    start_time = time.time()
+    
+    # 添加测试爬虫
+    crawler = await process.crawl(SimpleTestSpider)
+    
+    # 计算性能指标
+    metrics = crawler.metrics
+    duration = metrics.get_total_duration()
+    pages = 10
+    rps = pages / duration if duration > 0 else 0
+    
+    print(f"完成时间: {duration:.2f} 秒")
+    print(f"每秒请求数: {rps:.2f} RPS")
+    
+    return duration, rps
+
+
+async def main():
+    """主函数"""
+    print("Crawlo 框架优化效果验证测试")
+    print("=" * 40)
+    
+    # 测试并发性能
+    concurrent_duration, concurrent_rps = await test_concurrent_performance()
+    
+    # 测试顺序执行性能
+    sequential_duration, sequential_rps = await test_sequential_performance()
+    
+    # 输出比较结果
+    print("\n=== 性能对比 ===")
+    print(f"并发执行: {concurrent_duration:.2f}s, {concurrent_rps:.2f} RPS")
+    print(f"顺序执行: {sequential_duration:.2f}s, {sequential_rps:.2f} RPS")
+    
+    if concurrent_duration < sequential_duration:
+        improvement = (sequential_duration - concurrent_duration) / sequential_duration * 100
+        print(f"并发执行比顺序执行快 {improvement:.1f}%")
+    else:
+        print("并发执行性能未达到预期，请检查优化效果")
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

tests/simple_spider_test.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+简单的爬虫测试脚本
+"""
+
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class TestSpider(Spider):
+    """测试爬虫"""
+    name = 'test_spider'
+    
+    def start_requests(self):
+        """发起测试请求"""
+        yield Request('https://httpbin.org/get', callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功获取响应: {response.url}")
+        print(f"状态码: {response.status_code}")
+        return []
+
+
+def main():
+    """主函数"""
+    print("开始测试爬虫功能...")
+    
+    # 初始化框架
+    from crawlo.initialization import initialize_framework
+    settings = initialize_framework()
+    
+    # 创建爬虫进程
+    from crawlo.crawler import CrawlerProcess
+    process = CrawlerProcess(settings=settings)
+    
+    # 运行爬虫
+    import asyncio
+    asyncio.run(process.crawl(TestSpider))
+    
+    print("爬虫测试完成！")
+
+
+if __name__ == "__main__":
+    main()

tests/simple_test.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+简化的框架测试
+"""
+import os
+import sys
+sys.path.insert(0, '/')
+
+# 设置基本配置
+test_log_file = '/Users/oscar/projects/Crawlo/simple_test.log'
+if os.path.exists(test_log_file):
+    os.remove(test_log_file)
+
+# 最简单的测试
+try:
+    from crawlo.utils.log import LoggerManager
+    
+    print("配置日志系统...")
+    LoggerManager.configure(
+        LOG_LEVEL='INFO',
+        LOG_FILE=test_log_file
+    )
+    
+    from crawlo.utils.log import get_logger
+    logger = get_logger('test.simple')
+    
+    print("测试日志输出...")
+    logger.info("这是一条测试信息")
+    logger.info("Crawlo框架初始化完成")
+    logger.info("Crawlo Framework Started 1.3.3")
+    
+    print("检查日志文件...")
+    if os.path.exists(test_log_file):
+        with open(test_log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"日志文件内容: {len(content)} 字符")
+            print("内容:")
+            print(content)
+    else:
+        print("日志文件未创建")
+        
+except Exception as e:
+    print(f"错误: {e}")
+    import traceback
+    traceback.print_exc()
+
+print("测试完成")

tests/test_advanced_tools.py

@@ -1,149 +1,149 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-高级工具测试
-"""
-import unittest
-from crawlo.tools import (
-    # 数据处理工具
-    clean_text,
-    format_currency,
-    validate_email,
-    validate_url,
-    check_data_integrity,
-    
-    # 重试机制
-    RetryMechanism,
-    should_retry,
-    exponential_backoff,
-    
-    # 反爬虫应对工具
-    AntiCrawler,
-    rotate_proxy,
-    handle_captcha,
-    detect_rate_limiting,
-    
-    # 分布式协调工具
-    generate_pagination_tasks,
-    distribute_tasks,
-    DistributedCoordinator,
-    TaskDistributor,
-    DeduplicationTool
-)
-
-
-class TestAdvancedTools(unittest.TestCase):
-    """高级工具测试类"""
-
-    def test_data_processing_tools(self):
-        """测试数据处理工具"""
-        # 测试数据清洗
-        dirty_text = "<p>这是一个&nbsp;<b>测试</b>&amp;文本</p>"
-        clean_result = clean_text(dirty_text)
-        self.assertEqual(clean_result, "这是一个 测试&文本")
-        
-        # 测试数据格式化
-        price = 1234.567
-        formatted_price = format_currency(price, "¥", 2)
-        self.assertEqual(formatted_price, "¥1,234.57")
-        
-        # 测试字段验证
-        self.assertTrue(validate_email("test@example.com"))
-        self.assertFalse(validate_email("invalid-email"))
-        
-        self.assertTrue(validate_url("https://example.com"))
-        self.assertFalse(validate_url("invalid-url"))
-        
-        # 测试数据完整性检查
-        data = {
-            "name": "张三",
-            "email": "test@example.com",
-            "phone": "13812345678"
-        }
-        required_fields = ["name", "email", "phone"]
-        integrity_result = check_data_integrity(data, required_fields)
-        self.assertTrue(integrity_result["is_valid"])
-
-    def test_retry_mechanism(self):
-        """测试重试机制"""
-        # 测试指数退避
-        delay = exponential_backoff(0)
-        self.assertGreater(delay, 0)
-        
-        # 测试是否应该重试
-        self.assertTrue(should_retry(status_code=500))
-        self.assertTrue(should_retry(exception=ConnectionError()))
-        self.assertFalse(should_retry(status_code=200))
-
-    def test_anti_crawler_tools(self):
-        """测试反爬虫应对工具"""
-        # 测试反爬虫工具
-        anti_crawler = AntiCrawler()
-        
-        # 测试随机User-Agent
-        user_agent = anti_crawler.get_random_user_agent()
-        self.assertIsInstance(user_agent, str)
-        self.assertGreater(len(user_agent), 0)
-        
-        # 测试代理轮换
-        proxy = anti_crawler.rotate_proxy()
-        self.assertIsInstance(proxy, dict)
-        
-        # 测试验证码检测
-        self.assertTrue(anti_crawler.handle_captcha("请输入验证码进行验证"))
-        self.assertFalse(anti_crawler.handle_captcha("正常页面内容"))
-        
-        # 测试频率限制检测
-        self.assertTrue(anti_crawler.detect_rate_limiting(429, {}))
-        self.assertFalse(anti_crawler.detect_rate_limiting(200, {}))
-
-    def test_distributed_coordinator_tools(self):
-        """测试分布式协调工具"""
-        # 测试任务分发器
-        distributor = TaskDistributor()
-        
-        # 测试分页任务生成
-        base_url = "https://example.com/products"
-        pagination_tasks = distributor.generate_pagination_tasks(base_url, 1, 5)
-        self.assertEqual(len(pagination_tasks), 5)
-        
-        # 测试任务分发
-        tasks = list(range(1, 21))  # 20个任务
-        distributed = distributor.distribute_tasks(tasks, 4)  # 分发给4个工作节点
-        self.assertEqual(len(distributed), 4)
-        self.assertEqual(sum(len(worker_tasks) for worker_tasks in distributed), 20)
-        
-        # 测试去重工具
-        dedup_tool = DeduplicationTool()
-        
-        # 测试数据指纹生成
-        fingerprint1 = dedup_tool.generate_fingerprint({"name": "test", "value": 123})
-        fingerprint2 = dedup_tool.generate_fingerprint({"name": "test", "value": 123})
-        self.assertEqual(fingerprint1, fingerprint2)
-        
-        # 测试去重功能
-        self.assertFalse(dedup_tool.is_duplicate({"name": "test", "value": 123}))
-        self.assertTrue(dedup_tool.add_to_dedup({"name": "test", "value": 123}))
-        self.assertTrue(dedup_tool.is_duplicate({"name": "test", "value": 123}))
-        self.assertFalse(dedup_tool.add_to_dedup({"name": "test", "value": 123}))
-        
-        # 测试分布式协调器
-        coordinator = DistributedCoordinator()
-        
-        # 测试任务ID生成
-        task_id = coordinator.generate_task_id("https://example.com", "test_spider")
-        self.assertIsInstance(task_id, str)
-        self.assertEqual(len(task_id), 32)  # MD5 hash长度
-        
-        # 测试分页任务生成
-        pagination_tasks = coordinator.generate_pagination_tasks("https://example.com/products", 1, 5)
-        self.assertEqual(len(pagination_tasks), 5)
-        
-        # 测试任务分发
-        tasks = list(range(1, 21))  # 20个任务
-        distributed = coordinator.distribute_tasks(tasks, 4)  # 分发给4个工作节点
-        self.assertEqual(len(distributed), 4)
-
-
-if __name__ == '__main__':
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+高级工具测试
+"""
+import unittest
+from crawlo.tools import (
+    # 数据处理工具
+    clean_text,
+    format_currency,
+    validate_email,
+    validate_url,
+    check_data_integrity,
+    
+    # 重试机制
+    RetryMechanism,
+    should_retry,
+    exponential_backoff,
+    
+    # 反爬虫应对工具
+    AntiCrawler,
+    rotate_proxy,
+    handle_captcha,
+    detect_rate_limiting,
+    
+    # 分布式协调工具
+    generate_pagination_tasks,
+    distribute_tasks,
+    DistributedCoordinator,
+    TaskDistributor,
+    DeduplicationTool
+)
+
+
+class TestAdvancedTools(unittest.TestCase):
+    """高级工具测试类"""
+
+    def test_data_processing_tools(self):
+        """测试数据处理工具"""
+        # 测试数据清洗
+        dirty_text = "<p>这是一个&nbsp;<b>测试</b>&amp;文本</p>"
+        clean_result = clean_text(dirty_text)
+        self.assertEqual(clean_result, "这是一个 测试&文本")
+        
+        # 测试数据格式化
+        price = 1234.567
+        formatted_price = format_currency(price, "¥", 2)
+        self.assertEqual(formatted_price, "¥1,234.57")
+        
+        # 测试字段验证
+        self.assertTrue(validate_email("test@example.com"))
+        self.assertFalse(validate_email("invalid-email"))
+        
+        self.assertTrue(validate_url("https://example.com"))
+        self.assertFalse(validate_url("invalid-url"))
+        
+        # 测试数据完整性检查
+        data = {
+            "name": "张三",
+            "email": "test@example.com",
+            "phone": "13812345678"
+        }
+        required_fields = ["name", "email", "phone"]
+        integrity_result = check_data_integrity(data, required_fields)
+        self.assertTrue(integrity_result["is_valid"])
+
+    def test_retry_mechanism(self):
+        """测试重试机制"""
+        # 测试指数退避
+        delay = exponential_backoff(0)
+        self.assertGreater(delay, 0)
+        
+        # 测试是否应该重试
+        self.assertTrue(should_retry(status_code=500))
+        self.assertTrue(should_retry(exception=ConnectionError()))
+        self.assertFalse(should_retry(status_code=200))
+
+    def test_anti_crawler_tools(self):
+        """测试反爬虫应对工具"""
+        # 测试反爬虫工具
+        anti_crawler = AntiCrawler()
+        
+        # 测试随机User-Agent
+        user_agent = anti_crawler.get_random_user_agent()
+        self.assertIsInstance(user_agent, str)
+        self.assertGreater(len(user_agent), 0)
+        
+        # 测试代理轮换
+        proxy = anti_crawler.rotate_proxy()
+        self.assertIsInstance(proxy, dict)
+        
+        # 测试验证码检测
+        self.assertTrue(anti_crawler.handle_captcha("请输入验证码进行验证"))
+        self.assertFalse(anti_crawler.handle_captcha("正常页面内容"))
+        
+        # 测试频率限制检测
+        self.assertTrue(anti_crawler.detect_rate_limiting(429, {}))
+        self.assertFalse(anti_crawler.detect_rate_limiting(200, {}))
+
+    def test_distributed_coordinator_tools(self):
+        """测试分布式协调工具"""
+        # 测试任务分发器
+        distributor = TaskDistributor()
+        
+        # 测试分页任务生成
+        base_url = "https://example.com/products"
+        pagination_tasks = distributor.generate_pagination_tasks(base_url, 1, 5)
+        self.assertEqual(len(pagination_tasks), 5)
+        
+        # 测试任务分发
+        tasks = list(range(1, 21))  # 20个任务
+        distributed = distributor.distribute_tasks(tasks, 4)  # 分发给4个工作节点
+        self.assertEqual(len(distributed), 4)
+        self.assertEqual(sum(len(worker_tasks) for worker_tasks in distributed), 20)
+        
+        # 测试去重工具
+        dedup_tool = DeduplicationTool()
+        
+        # 测试数据指纹生成
+        fingerprint1 = dedup_tool.generate_fingerprint({"name": "test", "value": 123})
+        fingerprint2 = dedup_tool.generate_fingerprint({"name": "test", "value": 123})
+        self.assertEqual(fingerprint1, fingerprint2)
+        
+        # 测试去重功能
+        self.assertFalse(dedup_tool.is_duplicate({"name": "test", "value": 123}))
+        self.assertTrue(dedup_tool.add_to_dedup({"name": "test", "value": 123}))
+        self.assertTrue(dedup_tool.is_duplicate({"name": "test", "value": 123}))
+        self.assertFalse(dedup_tool.add_to_dedup({"name": "test", "value": 123}))
+        
+        # 测试分布式协调器
+        coordinator = DistributedCoordinator()
+        
+        # 测试任务ID生成
+        task_id = coordinator.generate_task_id("https://example.com", "test_spider")
+        self.assertIsInstance(task_id, str)
+        self.assertEqual(len(task_id), 32)  # MD5 hash长度
+        
+        # 测试分页任务生成
+        pagination_tasks = coordinator.generate_pagination_tasks("https://example.com/products", 1, 5)
+        self.assertEqual(len(pagination_tasks), 5)
+        
+        # 测试任务分发
+        tasks = list(range(1, 21))  # 20个任务
+        distributed = coordinator.distribute_tasks(tasks, 4)  # 分发给4个工作节点
+        self.assertEqual(len(distributed), 4)
+
+
+if __name__ == '__main__':
     unittest.main()