crawlo 1.4.7__py3-none-any.whl → 1.4.8__py3-none-any.whl

tests/optimized_performance_test.py

@@ -1,212 +1,212 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-优化后的Crawlo框架性能测试脚本
-用于评估优化后的框架性能表现
-"""
-
-import asyncio
-import time
-import sys
-import os
-from pathlib import Path
-
-# 添加项目根目录到 Python 路径
-project_root = Path(__file__).parent.parent
-sys.path.insert(0, str(project_root))
-
-from crawlo.crawler import CrawlerProcess
-from crawlo.spider import Spider
-from crawlo import Request, Item
-
-
-class TestItem(Item):
-    """测试用的Item类"""
-    def __init__(self):
-        super().__init__()
-        self.url = ''
-        self.status = 0
-        self.page_id = ''
-
-
-def create_test_spider_class(spider_name, page_count):
-    """动态创建测试爬虫类"""
-    class TestSpider(Spider):
-        # 显式设置name属性
-        name = spider_name
-        
-        def __init__(self, *args, **kwargs):
-            super().__init__(*args, **kwargs)
-            # 使用较少的测试页面以加快测试速度
-            self.start_urls = [f'https://httpbin.org/delay/0?page={i}' for i in range(page_count)]
-        
-        def parse(self, response):
-            """简单解析响应"""
-            # 正确返回Item对象
-            item = TestItem()
-            item['url'] = response.url
-            item['status'] = response.status_code
-            item['page_id'] = response.url.split('page=')[-1] if 'page=' in response.url else 'unknown'
-            yield item
-    
-    return TestSpider
-
-
-class OptimizedPerformanceTester:
-    """优化后的性能测试器"""
-    
-    def __init__(self):
-        self.results = {}
-    
-    def test_initialization_performance(self):
-        """测试初始化性能"""
-        print("测试初始化性能...")
-        
-        start_time = time.time()
-        settings = {
-            'CONCURRENT_REQUESTS': 10,
-        }
-        process = CrawlerProcess(settings=settings)
-        end_time = time.time()
-        
-        init_time = end_time - start_time
-        print(f"初始化时间: {init_time:.4f} 秒")
-        return init_time
-    
-    async def run_crawler_test(self, test_pages=20, concurrent_requests=10, test_name="performance_test"):
-        """运行爬虫性能测试"""
-        # 配置设置
-        settings = {
-            'CONCURRENT_REQUESTS': concurrent_requests,
-            'DOWNLOAD_DELAY': 0.1,  # 减少延迟以提高性能
-            'RANDOMIZE_DOWNLOAD_DELAY': False,
-            'SCHEDULER_MAX_QUEUE_SIZE': 5000,  # 使用优化后的队列大小
-            'BACKPRESSURE_RATIO': 0.9,  # 使用优化后的背压比率
-        }
-        
-        # 创建测试爬虫类
-        TestSpiderClass = create_test_spider_class(test_name, test_pages)
-        
-        # 注册爬虫类
-        from crawlo.spider import get_global_spider_registry
-        registry = get_global_spider_registry()
-        registry[TestSpiderClass.name] = TestSpiderClass
-        
-        # 创建爬虫进程
-        process = CrawlerProcess(settings=settings)
-        
-        # 添加测试爬虫
-        crawler = await process.crawl(TestSpiderClass.name)
-        
-        # 计算性能指标
-        metrics = crawler.metrics
-        duration = metrics.get_total_duration()
-        rps = test_pages / duration if duration > 0 else 0
-        
-        return {
-            'duration': duration,
-            'rps': rps,
-            'pages': test_pages,
-            'concurrent': concurrent_requests
-        }
-    
-    async def run_scale_tests(self):
-        """运行不同规模的测试"""
-        print("\n=== 运行规模测试 ===")
-        scales = [10, 20, 50]  # 减少测试规模以加快测试速度
-        results = []
-        
-        for i, scale in enumerate(scales):
-            test_name = f"scale_test_{i}_{scale}"
-            print(f"测试规模: {scale} 个页面")
-            try:
-                result = await self.run_crawler_test(test_pages=scale, test_name=test_name)
-                results.append(result)
-                print(f"  完成时间: {result['duration']:.2f} 秒")
-                print(f"  每秒请求数: {result['rps']:.2f} RPS")
-            except Exception as e:
-                print(f"  测试失败: {e}")
-                import traceback
-                traceback.print_exc()
-            print()
-        
-        return results
-    
-    async def run_concurrency_tests(self):
-        """运行不同并发数的测试"""
-        print("\n=== 运行并发测试 ===")
-        concurrencies = [1, 5, 10, 20]  # 增加更高并发数的测试
-        results = []
-        
-        for i, concurrency in enumerate(concurrencies):
-            test_name = f"concurrency_test_{i}_{concurrency}"
-            print(f"测试并发数: {concurrency}")
-            try:
-                result = await self.run_crawler_test(
-                    test_pages=50,  # 增加页面数以更好地测试并发效果
-                    concurrent_requests=concurrency,
-                    test_name=test_name
-                )
-                results.append(result)
-                print(f"  完成时间: {result['duration']:.2f} 秒")
-                print(f"  每秒请求数: {result['rps']:.2f} RPS")
-            except Exception as e:
-                print(f"  测试失败: {e}")
-                import traceback
-                traceback.print_exc()
-            print()
-        
-        return results
-    
-    async def run_performance_suite(self):
-        """运行完整的性能测试套件"""
-        print("开始 Crawlo 框架优化后性能测试")
-        print("=" * 50)
-        
-        # 测试初始化性能
-        init_time = self.test_initialization_performance()
-        
-        # 运行规模测试
-        scale_results = await self.run_scale_tests()
-        
-        # 运行并发测试
-        concurrency_results = await self.run_concurrency_tests()
-        
-        # 汇总结果
-        print("\n=== 性能测试汇总 ===")
-        print(f"初始化时间: {init_time:.4f} 秒")
-        
-        print("\n规模测试结果:")
-        for result in scale_results:
-            if 'duration' in result:
-                print(f"  {result['pages']} 页面: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
-        
-        print("\n并发测试结果:")
-        for result in concurrency_results:
-            if 'duration' in result:
-                print(f"  {result['concurrent']} 并发: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
-        
-        return {
-            'initialization': init_time,
-            'scale_tests': scale_results,
-            'concurrency_tests': concurrency_results
-        }
-
-
-async def main():
-    """主函数"""
-    tester = OptimizedPerformanceTester()
-    results = await tester.run_performance_suite()
-    
-    print("\n=== 测试完成 ===")
-    print("优化后性能测试已完成，结果如上所示。")
-    
-    # 保存结果到文件
-    import json
-    with open('optimized_performance_test_results.json', 'w', encoding='utf-8') as f:
-        json.dump(results, f, ensure_ascii=False, indent=2)
-    print("结果已保存到 optimized_performance_test_results.json")
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+优化后的Crawlo框架性能测试脚本
+用于评估优化后的框架性能表现
+"""
+
+import asyncio
+import time
+import sys
+import os
+from pathlib import Path
+
+# 添加项目根目录到 Python 路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import Spider
+from crawlo import Request, Item
+
+
+class TestItem(Item):
+    """测试用的Item类"""
+    def __init__(self):
+        super().__init__()
+        self.url = ''
+        self.status = 0
+        self.page_id = ''
+
+
+def create_test_spider_class(spider_name, page_count):
+    """动态创建测试爬虫类"""
+    class TestSpider(Spider):
+        # 显式设置name属性
+        name = spider_name
+        
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            # 使用较少的测试页面以加快测试速度
+            self.start_urls = [f'https://httpbin.org/delay/0?page={i}' for i in range(page_count)]
+        
+        def parse(self, response):
+            """简单解析响应"""
+            # 正确返回Item对象
+            item = TestItem()
+            item['url'] = response.url
+            item['status'] = response.status_code
+            item['page_id'] = response.url.split('page=')[-1] if 'page=' in response.url else 'unknown'
+            yield item
+    
+    return TestSpider
+
+
+class OptimizedPerformanceTester:
+    """优化后的性能测试器"""
+    
+    def __init__(self):
+        self.results = {}
+    
+    def test_initialization_performance(self):
+        """测试初始化性能"""
+        print("测试初始化性能...")
+        
+        start_time = time.time()
+        settings = {
+            'CONCURRENT_REQUESTS': 10,
+        }
+        process = CrawlerProcess(settings=settings)
+        end_time = time.time()
+        
+        init_time = end_time - start_time
+        print(f"初始化时间: {init_time:.4f} 秒")
+        return init_time
+    
+    async def run_crawler_test(self, test_pages=20, concurrent_requests=10, test_name="performance_test"):
+        """运行爬虫性能测试"""
+        # 配置设置
+        settings = {
+            'CONCURRENT_REQUESTS': concurrent_requests,
+            'DOWNLOAD_DELAY': 0.1,  # 减少延迟以提高性能
+            'RANDOMIZE_DOWNLOAD_DELAY': False,
+            'SCHEDULER_MAX_QUEUE_SIZE': 5000,  # 使用优化后的队列大小
+            'BACKPRESSURE_RATIO': 0.9,  # 使用优化后的背压比率
+        }
+        
+        # 创建测试爬虫类
+        TestSpiderClass = create_test_spider_class(test_name, test_pages)
+        
+        # 注册爬虫类
+        from crawlo.spider import get_global_spider_registry
+        registry = get_global_spider_registry()
+        registry[TestSpiderClass.name] = TestSpiderClass
+        
+        # 创建爬虫进程
+        process = CrawlerProcess(settings=settings)
+        
+        # 添加测试爬虫
+        crawler = await process.crawl(TestSpiderClass.name)
+        
+        # 计算性能指标
+        metrics = crawler.metrics
+        duration = metrics.get_total_duration()
+        rps = test_pages / duration if duration > 0 else 0
+        
+        return {
+            'duration': duration,
+            'rps': rps,
+            'pages': test_pages,
+            'concurrent': concurrent_requests
+        }
+    
+    async def run_scale_tests(self):
+        """运行不同规模的测试"""
+        print("\n=== 运行规模测试 ===")
+        scales = [10, 20, 50]  # 减少测试规模以加快测试速度
+        results = []
+        
+        for i, scale in enumerate(scales):
+            test_name = f"scale_test_{i}_{scale}"
+            print(f"测试规模: {scale} 个页面")
+            try:
+                result = await self.run_crawler_test(test_pages=scale, test_name=test_name)
+                results.append(result)
+                print(f"  完成时间: {result['duration']:.2f} 秒")
+                print(f"  每秒请求数: {result['rps']:.2f} RPS")
+            except Exception as e:
+                print(f"  测试失败: {e}")
+                import traceback
+                traceback.print_exc()
+            print()
+        
+        return results
+    
+    async def run_concurrency_tests(self):
+        """运行不同并发数的测试"""
+        print("\n=== 运行并发测试 ===")
+        concurrencies = [1, 5, 10, 20]  # 增加更高并发数的测试
+        results = []
+        
+        for i, concurrency in enumerate(concurrencies):
+            test_name = f"concurrency_test_{i}_{concurrency}"
+            print(f"测试并发数: {concurrency}")
+            try:
+                result = await self.run_crawler_test(
+                    test_pages=50,  # 增加页面数以更好地测试并发效果
+                    concurrent_requests=concurrency,
+                    test_name=test_name
+                )
+                results.append(result)
+                print(f"  完成时间: {result['duration']:.2f} 秒")
+                print(f"  每秒请求数: {result['rps']:.2f} RPS")
+            except Exception as e:
+                print(f"  测试失败: {e}")
+                import traceback
+                traceback.print_exc()
+            print()
+        
+        return results
+    
+    async def run_performance_suite(self):
+        """运行完整的性能测试套件"""
+        print("开始 Crawlo 框架优化后性能测试")
+        print("=" * 50)
+        
+        # 测试初始化性能
+        init_time = self.test_initialization_performance()
+        
+        # 运行规模测试
+        scale_results = await self.run_scale_tests()
+        
+        # 运行并发测试
+        concurrency_results = await self.run_concurrency_tests()
+        
+        # 汇总结果
+        print("\n=== 性能测试汇总 ===")
+        print(f"初始化时间: {init_time:.4f} 秒")
+        
+        print("\n规模测试结果:")
+        for result in scale_results:
+            if 'duration' in result:
+                print(f"  {result['pages']} 页面: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
+        
+        print("\n并发测试结果:")
+        for result in concurrency_results:
+            if 'duration' in result:
+                print(f"  {result['concurrent']} 并发: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
+        
+        return {
+            'initialization': init_time,
+            'scale_tests': scale_results,
+            'concurrency_tests': concurrency_results
+        }
+
+
+async def main():
+    """主函数"""
+    tester = OptimizedPerformanceTester()
+    results = await tester.run_performance_suite()
+    
+    print("\n=== 测试完成 ===")
+    print("优化后性能测试已完成，结果如上所示。")
+    
+    # 保存结果到文件
+    import json
+    with open('optimized_performance_test_results.json', 'w', encoding='utf-8') as f:
+        json.dump(results, f, ensure_ascii=False, indent=2)
+    print("结果已保存到 optimized_performance_test_results.json")
+
+
+if __name__ == '__main__':
     asyncio.run(main())