crawlo 1.3.3__py3-none-any.whl → 1.3.5__py3-none-any.whl

tests/framework_performance_test.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Crawlo 框架性能测试脚本
+用于评估框架在不同场景下的性能表现
+"""
+
+import asyncio
+import time
+import tracemalloc
+import threading
+import sys
+import os
+from pathlib import Path
+
+# 添加项目根目录到 Python 路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import Spider
+
+
+def create_test_spider_class(spider_name, page_count):
+    """动态创建测试爬虫类"""
+    class TestSpider(Spider):
+        # 显式设置name属性
+        name = spider_name
+        
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            # 使用较少的测试页面以加快测试速度
+            self.start_urls = [f'https://httpbin.org/delay/0?page={i}' for i in range(page_count)]
+        
+        def parse(self, response):
+            """简单解析响应"""
+            yield {
+                'url': response.url,
+                'status': response.status_code,  # 修复：使用status_code而不是status
+                'page_id': response.url.split('page=')[-1] if 'page=' in response.url else 'unknown'
+            }
+    
+    return TestSpider
+
+
+class PerformanceTester:
+    """性能测试器"""
+    
+    def __init__(self):
+        self.results = {}
+    
+    def test_initialization_performance(self):
+        """测试初始化性能"""
+        print("测试初始化性能...")
+        
+        start_time = time.time()
+        settings = {
+            'CONCURRENT_REQUESTS': 10,
+        }
+        process = CrawlerProcess(settings=settings)
+        end_time = time.time()
+        
+        init_time = end_time - start_time
+        print(f"初始化时间: {init_time:.4f} 秒")
+        return init_time
+    
+    async def run_crawler_test(self, test_pages=20, concurrent_requests=10, test_name="performance_test"):
+        """运行爬虫性能测试"""
+        # 配置设置
+        settings = {
+            'CONCURRENT_REQUESTS': concurrent_requests,
+            'DOWNLOAD_DELAY': 0,
+            'RANDOMIZE_DOWNLOAD_DELAY': False,
+            'SCHEDULER_MAX_QUEUE_SIZE': 1000,
+            'BACKPRESSURE_RATIO': 0.8,
+        }
+        
+        # 创建测试爬虫类
+        TestSpiderClass = create_test_spider_class(test_name, test_pages)
+        
+        # 注册爬虫类
+        from crawlo.spider import get_global_spider_registry
+        registry = get_global_spider_registry()
+        registry[TestSpiderClass.name] = TestSpiderClass
+        
+        # 创建爬虫进程
+        process = CrawlerProcess(settings=settings)
+        
+        # 添加测试爬虫
+        crawler = await process.crawl(TestSpiderClass.name)
+        
+        # 计算性能指标
+        metrics = crawler.metrics
+        duration = metrics.get_total_duration()
+        rps = test_pages / duration if duration > 0 else 0
+        
+        return {
+            'duration': duration,
+            'rps': rps,
+            'pages': test_pages,
+            'concurrent': concurrent_requests
+        }
+    
+    async def run_scale_tests(self):
+        """运行不同规模的测试"""
+        print("\n=== 运行规模测试 ===")
+        scales = [10, 20, 50]  # 减少测试规模以加快测试速度
+        results = []
+        
+        for i, scale in enumerate(scales):
+            test_name = f"scale_test_{i}_{scale}"
+            print(f"测试规模: {scale} 个页面")
+            try:
+                result = await self.run_crawler_test(test_pages=scale, test_name=test_name)
+                results.append(result)
+                print(f"  完成时间: {result['duration']:.2f} 秒")
+                print(f"  每秒请求数: {result['rps']:.2f} RPS")
+            except Exception as e:
+                print(f"  测试失败: {e}")
+                import traceback
+                traceback.print_exc()
+            print()
+        
+        return results
+    
+    async def run_concurrency_tests(self):
+        """运行不同并发数的测试"""
+        print("\n=== 运行并发测试 ===")
+        concurrencies = [1, 5, 10]  # 减少并发数以避免对测试服务器造成过大压力
+        results = []
+        
+        for i, concurrency in enumerate(concurrencies):
+            test_name = f"concurrency_test_{i}_{concurrency}"
+            print(f"测试并发数: {concurrency}")
+            try:
+                result = await self.run_crawler_test(
+                    test_pages=20, 
+                    concurrent_requests=concurrency,
+                    test_name=test_name
+                )
+                results.append(result)
+                print(f"  完成时间: {result['duration']:.2f} 秒")
+                print(f"  每秒请求数: {result['rps']:.2f} RPS")
+            except Exception as e:
+                print(f"  测试失败: {e}")
+                import traceback
+                traceback.print_exc()
+            print()
+        
+        return results
+    
+    async def run_performance_suite(self):
+        """运行完整的性能测试套件"""
+        print("开始 Crawlo 框架性能测试")
+        print("=" * 50)
+        
+        # 测试初始化性能
+        init_time = self.test_initialization_performance()
+        
+        # 运行规模测试
+        scale_results = await self.run_scale_tests()
+        
+        # 运行并发测试
+        concurrency_results = await self.run_concurrency_tests()
+        
+        # 汇总结果
+        print("\n=== 性能测试汇总 ===")
+        print(f"初始化时间: {init_time:.4f} 秒")
+        
+        print("\n规模测试结果:")
+        for result in scale_results:
+            if 'duration' in result:
+                print(f"  {result['pages']} 页面: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
+        
+        print("\n并发测试结果:")
+        for result in concurrency_results:
+            if 'duration' in result:
+                print(f"  {result['concurrent']} 并发: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
+        
+        return {
+            'initialization': init_time,
+            'scale_tests': scale_results,
+            'concurrency_tests': concurrency_results
+        }
+
+
+async def main():
+    """主函数"""
+    tester = PerformanceTester()
+    results = await tester.run_performance_suite()
+    
+    print("\n=== 测试完成 ===")
+    print("性能测试已完成，结果如上所示。")
+    
+    # 保存结果到文件
+    import json
+    with open('performance_test_results.json', 'w', encoding='utf-8') as f:
+        json.dump(results, f, ensure_ascii=False, indent=2)
+    print("结果已保存到 performance_test_results.json")
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

tests/log_buffering_test.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+测试日志缓冲问题
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# 添加项目根目录到Python路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.logging import configure_logging as configure, get_logger
+import logging
+
+
+def test_log_buffering():
+    """测试日志缓冲行为"""
+    print("=== 测试日志缓冲行为 ===")
+    
+    # 设置日志文件路径
+    log_file = "logs/buffering_test2.log"
+    
+    # 删除可能存在的旧日志文件
+    if os.path.exists(log_file):
+        os.remove(log_file)
+    
+    # 配置日志系统
+    configure(
+        level='INFO',
+        file_path=log_file,
+        console_enabled=True,
+        file_enabled=True
+    )
+    
+    # 获取logger
+    logger = get_logger('buffering.test2')
+    
+    print("1. 检查handler的自动刷新设置...")
+    for handler in logger.handlers:
+        handler_type = type(handler).__name__
+        print(f"   Handler类型: {handler_type}")
+        if hasattr(handler, 'stream'):
+            print(f"   流类型: {type(handler.stream).__name__}")
+            if hasattr(handler.stream, 'flush'):
+                print(f"   支持flush方法: True")
+        
+        # 检查是否有自动刷新设置
+        if hasattr(handler, 'flush'):
+            print(f"   Handler有flush方法")
+    
+    print("\n2. 写入日志并强制刷新...")
+    logger.info("测试日志1")
+    logger.info("测试日志2")
+    
+    # 强制刷新所有handler
+    for handler in logger.handlers:
+        if hasattr(handler, 'flush'):
+            handler.flush()
+            print(f"   已刷新 {type(handler).__name__}")
+    
+    # 检查文件内容
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"\n日志文件内容:")
+            print(f"  行数: {len(content.splitlines())}")
+            print(f"  内容: {repr(content)}")
+    
+    print("\n3. 测试不同日志级别的缓冲...")
+    logger.debug("DEBUG消息（不应显示）")
+    logger.info("INFO消息")
+    logger.warning("WARNING消息")
+    logger.error("ERROR消息")
+    
+    # 再次强制刷新
+    for handler in logger.handlers:
+        if hasattr(handler, 'flush'):
+            handler.flush()
+    
+    # 检查最终文件内容
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+            print(f"\n最终日志文件内容:")
+            print(f"  总行数: {len(lines)}")
+            for i, line in enumerate(lines):
+                print(f"  {i+1}: {line.strip()}")
+
+
+def main():
+    """主函数"""
+    print("开始测试日志缓冲问题...")
+    
+    try:
+        test_log_buffering()
+        
+        print("\n=== 所有测试完成 ===")
+        
+    except Exception as e:
+        print(f"\n测试过程中出现错误: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

tests/log_generation_timing_test.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+测试日志文件的生成时机
+"""
+
+import os
+import sys
+import time
+from pathlib import Path
+
+# 添加项目根目录到Python路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.logging import configure_logging as configure, get_logger
+
+
+def test_log_file_generation_timing():
+    """测试日志文件的生成时机"""
+    print("=== 测试日志文件的生成时机 ===")
+    
+    # 设置日志文件路径
+    log_file = "logs/timing_test.log"
+    
+    # 确保日志目录存在
+    os.makedirs("logs", exist_ok=True)
+    
+    # 删除可能存在的旧日志文件
+    if os.path.exists(log_file):
+        os.remove(log_file)
+        print(f"已删除旧日志文件: {log_file}")
+    
+    print(f"日志文件路径: {log_file}")
+    print(f"日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 配置日志系统
+    print("\n1. 配置日志系统...")
+    configure(
+        level='INFO',
+        file_path=log_file,
+        console_enabled=True,
+        file_enabled=True
+    )
+    
+    print(f"配置后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 获取logger
+    print("\n2. 获取logger...")
+    logger = get_logger('timing.test')
+    print(f"获取logger后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 立即写入日志
+    print("\n3. 立即写入日志...")
+    logger.info("立即写入的第一条日志")
+    print(f"写入第一条日志后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 检查文件内容
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"日志文件内容行数: {len(content.splitlines())}")
+            print(f"日志文件大小: {os.path.getsize(log_file)} 字节")
+    
+    # 等待一小段时间
+    print("\n4. 等待1秒...")
+    time.sleep(1)
+    print(f"等待后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 再写入一些日志
+    print("\n5. 再写入一些日志...")
+    for i in range(5):
+        logger.info(f"第{i+1}条测试日志")
+        time.sleep(0.1)
+    
+    print(f"写入多条日志后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 检查最终文件内容
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+            print(f"\n最终日志文件内容:")
+            print(f"  总行数: {len(lines)}")
+            print(f"  文件大小: {os.path.getsize(log_file)} 字节")
+            if lines:
+                print(f"  第一行: {lines[0].strip()}")
+                print(f"  最后一行: {lines[-1].strip()}")
+    
+    print("\n=== 测试完成 ===")
+
+
+def test_buffering_behavior():
+    """测试日志缓冲行为"""
+    print("\n=== 测试日志缓冲行为 ===")
+    
+    # 设置日志文件路径
+    log_file = "logs/buffering_test.log"
+    
+    # 删除可能存在的旧日志文件
+    if os.path.exists(log_file):
+        os.remove(log_file)
+    
+    # 配置日志系统
+    configure(
+        level='INFO',
+        file_path=log_file,
+        console_enabled=True,
+        file_enabled=True
+    )
+    
+    logger = get_logger('buffering.test')
+    
+    print("1. 写入日志后立即检查文件...")
+    logger.info("缓冲测试日志1")
+    print(f"   写入后文件是否存在: {os.path.exists(log_file)}")
+    
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"   文件内容: '{content.strip()}'")
+    
+    print("2. 强制刷新并检查...")
+    # 获取文件处理器并强制刷新
+    for handler in logger.handlers:
+        if hasattr(handler, 'flush'):
+            handler.flush()
+    
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"   刷新后文件内容: '{content.strip()}'")
+
+
+def main():
+    """主函数"""
+    print("开始测试日志文件的生成时机...")
+    
+    try:
+        test_log_file_generation_timing()
+        test_buffering_behavior()
+        
+        print("\n=== 所有测试完成 ===")
+        
+    except Exception as e:
+        print(f"\n测试过程中出现错误: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

tests/optimized_performance_test.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+优化后的Crawlo框架性能测试脚本
+用于评估优化后的框架性能表现
+"""
+
+import asyncio
+import time
+import sys
+import os
+from pathlib import Path
+
+# 添加项目根目录到 Python 路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import Spider
+from crawlo import Request, Item
+
+
+class TestItem(Item):
+    """测试用的Item类"""
+    def __init__(self):
+        super().__init__()
+        self.url = ''
+        self.status = 0
+        self.page_id = ''
+
+
+def create_test_spider_class(spider_name, page_count):
+    """动态创建测试爬虫类"""
+    class TestSpider(Spider):
+        # 显式设置name属性
+        name = spider_name
+        
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            # 使用较少的测试页面以加快测试速度
+            self.start_urls = [f'https://httpbin.org/delay/0?page={i}' for i in range(page_count)]
+        
+        def parse(self, response):
+            """简单解析响应"""
+            # 正确返回Item对象
+            item = TestItem()
+            item['url'] = response.url
+            item['status'] = response.status_code
+            item['page_id'] = response.url.split('page=')[-1] if 'page=' in response.url else 'unknown'
+            yield item
+    
+    return TestSpider
+
+
+class OptimizedPerformanceTester:
+    """优化后的性能测试器"""
+    
+    def __init__(self):
+        self.results = {}
+    
+    def test_initialization_performance(self):
+        """测试初始化性能"""
+        print("测试初始化性能...")
+        
+        start_time = time.time()
+        settings = {
+            'CONCURRENT_REQUESTS': 10,
+        }
+        process = CrawlerProcess(settings=settings)
+        end_time = time.time()
+        
+        init_time = end_time - start_time
+        print(f"初始化时间: {init_time:.4f} 秒")
+        return init_time
+    
+    async def run_crawler_test(self, test_pages=20, concurrent_requests=10, test_name="performance_test"):
+        """运行爬虫性能测试"""
+        # 配置设置
+        settings = {
+            'CONCURRENT_REQUESTS': concurrent_requests,
+            'DOWNLOAD_DELAY': 0.1,  # 减少延迟以提高性能
+            'RANDOMIZE_DOWNLOAD_DELAY': False,
+            'SCHEDULER_MAX_QUEUE_SIZE': 5000,  # 使用优化后的队列大小
+            'BACKPRESSURE_RATIO': 0.9,  # 使用优化后的背压比率
+        }
+        
+        # 创建测试爬虫类
+        TestSpiderClass = create_test_spider_class(test_name, test_pages)
+        
+        # 注册爬虫类
+        from crawlo.spider import get_global_spider_registry
+        registry = get_global_spider_registry()
+        registry[TestSpiderClass.name] = TestSpiderClass
+        
+        # 创建爬虫进程
+        process = CrawlerProcess(settings=settings)
+        
+        # 添加测试爬虫
+        crawler = await process.crawl(TestSpiderClass.name)
+        
+        # 计算性能指标
+        metrics = crawler.metrics
+        duration = metrics.get_total_duration()
+        rps = test_pages / duration if duration > 0 else 0
+        
+        return {
+            'duration': duration,
+            'rps': rps,
+            'pages': test_pages,
+            'concurrent': concurrent_requests
+        }
+    
+    async def run_scale_tests(self):
+        """运行不同规模的测试"""
+        print("\n=== 运行规模测试 ===")
+        scales = [10, 20, 50]  # 减少测试规模以加快测试速度
+        results = []
+        
+        for i, scale in enumerate(scales):
+            test_name = f"scale_test_{i}_{scale}"
+            print(f"测试规模: {scale} 个页面")
+            try:
+                result = await self.run_crawler_test(test_pages=scale, test_name=test_name)
+                results.append(result)
+                print(f"  完成时间: {result['duration']:.2f} 秒")
+                print(f"  每秒请求数: {result['rps']:.2f} RPS")
+            except Exception as e:
+                print(f"  测试失败: {e}")
+                import traceback
+                traceback.print_exc()
+            print()
+        
+        return results
+    
+    async def run_concurrency_tests(self):
+        """运行不同并发数的测试"""
+        print("\n=== 运行并发测试 ===")
+        concurrencies = [1, 5, 10, 20]  # 增加更高并发数的测试
+        results = []
+        
+        for i, concurrency in enumerate(concurrencies):
+            test_name = f"concurrency_test_{i}_{concurrency}"
+            print(f"测试并发数: {concurrency}")
+            try:
+                result = await self.run_crawler_test(
+                    test_pages=50,  # 增加页面数以更好地测试并发效果
+                    concurrent_requests=concurrency,
+                    test_name=test_name
+                )
+                results.append(result)
+                print(f"  完成时间: {result['duration']:.2f} 秒")
+                print(f"  每秒请求数: {result['rps']:.2f} RPS")
+            except Exception as e:
+                print(f"  测试失败: {e}")
+                import traceback
+                traceback.print_exc()
+            print()
+        
+        return results
+    
+    async def run_performance_suite(self):
+        """运行完整的性能测试套件"""
+        print("开始 Crawlo 框架优化后性能测试")
+        print("=" * 50)
+        
+        # 测试初始化性能
+        init_time = self.test_initialization_performance()
+        
+        # 运行规模测试
+        scale_results = await self.run_scale_tests()
+        
+        # 运行并发测试
+        concurrency_results = await self.run_concurrency_tests()
+        
+        # 汇总结果
+        print("\n=== 性能测试汇总 ===")
+        print(f"初始化时间: {init_time:.4f} 秒")
+        
+        print("\n规模测试结果:")
+        for result in scale_results:
+            if 'duration' in result:
+                print(f"  {result['pages']} 页面: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
+        
+        print("\n并发测试结果:")
+        for result in concurrency_results:
+            if 'duration' in result:
+                print(f"  {result['concurrent']} 并发: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
+        
+        return {
+            'initialization': init_time,
+            'scale_tests': scale_results,
+            'concurrency_tests': concurrency_results
+        }
+
+
+async def main():
+    """主函数"""
+    tester = OptimizedPerformanceTester()
+    results = await tester.run_performance_suite()
+    
+    print("\n=== 测试完成 ===")
+    print("优化后性能测试已完成，结果如上所示。")
+    
+    # 保存结果到文件
+    import json
+    with open('optimized_performance_test_results.json', 'w', encoding='utf-8') as f:
+        json.dump(results, f, ensure_ascii=False, indent=2)
+    print("结果已保存到 optimized_performance_test_results.json")
+
+
+if __name__ == '__main__':
+    asyncio.run(main())