crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

tests/final_validation_test.py

@@ -1,183 +1,183 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-最终验证测试脚本
-验证所有修改的功能是否正常工作
-"""
-
-import sys
-import os
-import asyncio
-import time
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.spider import Spider
-from crawlo import Request
-
-
-class ValidationTestSpider(Spider):
-    """验证测试爬虫"""
-    name = 'validation_test_spider'
-    
-    def start_requests(self):
-        """发起测试请求"""
-        # 生成一些测试请求
-        for i in range(3):
-            yield Request(f'https://httpbin.org/get?page={i}', callback=self.parse)
-    
-    def parse(self, response):
-        """解析响应"""
-        print(f"成功获取响应: {response.url}")
-        print(f"状态码: {response.status_code}")
-        return []
-
-
-async def test_framework_startup_logging():
-    """测试框架启动日志"""
-    print("测试框架启动日志...")
-    
-    from crawlo.initialization import initialize_framework
-    from crawlo.logging import get_logger
-    
-    # 初始化框架
-    settings = initialize_framework({
-        'PROJECT_NAME': 'validation_test'
-    })
-    
-    # 获取框架logger并检查启动日志
-    logger = get_logger('crawlo.framework')
-    print("框架启动日志已记录")
-
-
-async def test_queue_blocking_behavior():
-    """测试队列阻塞行为"""
-    print("测试队列阻塞行为...")
-    
-    from crawlo.queue.queue_manager import QueueConfig, QueueManager
-    from crawlo import Request
-    
-    # 创建小队列配置进行测试
-    queue_config = QueueConfig(
-        queue_type='memory',
-        max_queue_size=3  # 非常小的队列
-    )
-    
-    # 创建队列管理器
-    queue_manager = QueueManager(queue_config)
-    await queue_manager.initialize()
-    
-    # 测试添加超过队列大小的请求
-    print("添加5个请求到大小为3的队列...")
-    start_time = time.time()
-    
-    # 创建生产者任务
-    async def producer():
-        for i in range(5):
-            request = Request(f'https://example.com/test{i}')
-            await queue_manager.put(request)
-            print(f"添加请求 {i}")
-    
-    # 创建消费者任务
-    async def consumer():
-        retrieved = 0
-        while retrieved < 5:
-            request = await queue_manager.get(timeout=2.0)
-            if request:
-                print(f"获取请求: {request.url}")
-                retrieved += 1
-                await asyncio.sleep(0.1)  # 模拟处理时间
-    
-    # 并发运行生产者和消费者
-    await asyncio.gather(producer(), consumer())
-    
-    end_time = time.time()
-    print(f"队列测试完成，耗时 {end_time - start_time:.2f} 秒")
-    
-    # 关闭队列
-    await queue_manager.close()
-
-
-async def test_stats_output():
-    """测试统计信息输出"""
-    print("测试统计信息输出...")
-    
-    from crawlo.initialization import initialize_framework
-    from crawlo.crawler import CrawlerProcess
-    
-    # 初始化框架
-    settings = initialize_framework({
-        'PROJECT_NAME': 'stats_test'
-    })
-    
-    # 创建爬虫进程
-    process = CrawlerProcess(settings=settings)
-    
-    # 运行爬虫
-    await process.crawl(ValidationTestSpider)
-    
-    print("统计信息输出测试完成")
-
-
-async def test_spider_auto_import():
-    """测试爬虫自动导入"""
-    print("测试爬虫自动导入...")
-    
-    from crawlo.initialization import initialize_framework
-    from crawlo.crawler import CrawlerProcess
-    
-    # 初始化框架
-    settings = initialize_framework({
-        'PROJECT_NAME': 'auto_import_test'
-    })
-    
-    # 创建爬虫进程，指定spider_modules
-    spider_modules = ['crawlo.spider']  # 使用框架的spider模块进行测试
-    process = CrawlerProcess(settings=settings, spider_modules=spider_modules)
-    
-    # 检查爬虫是否已注册
-    spider_name = ValidationTestSpider.name
-    is_registered = process.is_spider_registered(spider_name)
-    print(f"爬虫 '{spider_name}' 是否已注册: {is_registered}")
-    
-    if is_registered:
-        print("爬虫自动导入功能正常")
-    else:
-        print("爬虫自动导入功能异常")
-
-
-async def main():
-    """主函数"""
-    print("开始最终验证测试...")
-    print("=" * 50)
-    
-    try:
-        # 1. 测试框架启动日志
-        await test_framework_startup_logging()
-        print()
-        
-        # 2. 测试队列阻塞行为
-        await test_queue_blocking_behavior()
-        print()
-        
-        # 3. 测试统计信息输出
-        await test_stats_output()
-        print()
-        
-        # 4. 测试爬虫自动导入
-        await test_spider_auto_import()
-        print()
-        
-        print("=" * 50)
-        print("所有验证测试通过！框架修改的功能工作正常。")
-        
-    except Exception as e:
-        print("=" * 50)
-        print(f"验证测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-
-
-if __name__ == "__main__":
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+最终验证测试脚本
+验证所有修改的功能是否正常工作
+"""
+
+import sys
+import os
+import asyncio
+import time
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class ValidationTestSpider(Spider):
+    """验证测试爬虫"""
+    name = 'validation_test_spider'
+    
+    def start_requests(self):
+        """发起测试请求"""
+        # 生成一些测试请求
+        for i in range(3):
+            yield Request(f'https://httpbin.org/get?page={i}', callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功获取响应: {response.url}")
+        print(f"状态码: {response.status_code}")
+        return []
+
+
+async def test_framework_startup_logging():
+    """测试框架启动日志"""
+    print("测试框架启动日志...")
+    
+    from crawlo.initialization import initialize_framework
+    from crawlo.logging import get_logger
+    
+    # 初始化框架
+    settings = initialize_framework({
+        'PROJECT_NAME': 'validation_test'
+    })
+    
+    # 获取框架logger并检查启动日志
+    logger = get_logger('crawlo.framework')
+    print("框架启动日志已记录")
+
+
+async def test_queue_blocking_behavior():
+    """测试队列阻塞行为"""
+    print("测试队列阻塞行为...")
+    
+    from crawlo.queue.queue_manager import QueueConfig, QueueManager
+    from crawlo import Request
+    
+    # 创建小队列配置进行测试
+    queue_config = QueueConfig(
+        queue_type='memory',
+        max_queue_size=3  # 非常小的队列
+    )
+    
+    # 创建队列管理器
+    queue_manager = QueueManager(queue_config)
+    await queue_manager.initialize()
+    
+    # 测试添加超过队列大小的请求
+    print("添加5个请求到大小为3的队列...")
+    start_time = time.time()
+    
+    # 创建生产者任务
+    async def producer():
+        for i in range(5):
+            request = Request(f'https://example.com/test{i}')
+            await queue_manager.put(request)
+            print(f"添加请求 {i}")
+    
+    # 创建消费者任务
+    async def consumer():
+        retrieved = 0
+        while retrieved < 5:
+            request = await queue_manager.get(timeout=2.0)
+            if request:
+                print(f"获取请求: {request.url}")
+                retrieved += 1
+                await asyncio.sleep(0.1)  # 模拟处理时间
+    
+    # 并发运行生产者和消费者
+    await asyncio.gather(producer(), consumer())
+    
+    end_time = time.time()
+    print(f"队列测试完成，耗时 {end_time - start_time:.2f} 秒")
+    
+    # 关闭队列
+    await queue_manager.close()
+
+
+async def test_stats_output():
+    """测试统计信息输出"""
+    print("测试统计信息输出...")
+    
+    from crawlo.initialization import initialize_framework
+    from crawlo.crawler import CrawlerProcess
+    
+    # 初始化框架
+    settings = initialize_framework({
+        'PROJECT_NAME': 'stats_test'
+    })
+    
+    # 创建爬虫进程
+    process = CrawlerProcess(settings=settings)
+    
+    # 运行爬虫
+    await process.crawl(ValidationTestSpider)
+    
+    print("统计信息输出测试完成")
+
+
+async def test_spider_auto_import():
+    """测试爬虫自动导入"""
+    print("测试爬虫自动导入...")
+    
+    from crawlo.initialization import initialize_framework
+    from crawlo.crawler import CrawlerProcess
+    
+    # 初始化框架
+    settings = initialize_framework({
+        'PROJECT_NAME': 'auto_import_test'
+    })
+    
+    # 创建爬虫进程，指定spider_modules
+    spider_modules = ['crawlo.spider']  # 使用框架的spider模块进行测试
+    process = CrawlerProcess(settings=settings, spider_modules=spider_modules)
+    
+    # 检查爬虫是否已注册
+    spider_name = ValidationTestSpider.name
+    is_registered = process.is_spider_registered(spider_name)
+    print(f"爬虫 '{spider_name}' 是否已注册: {is_registered}")
+    
+    if is_registered:
+        print("爬虫自动导入功能正常")
+    else:
+        print("爬虫自动导入功能异常")
+
+
+async def main():
+    """主函数"""
+    print("开始最终验证测试...")
+    print("=" * 50)
+    
+    try:
+        # 1. 测试框架启动日志
+        await test_framework_startup_logging()
+        print()
+        
+        # 2. 测试队列阻塞行为
+        await test_queue_blocking_behavior()
+        print()
+        
+        # 3. 测试统计信息输出
+        await test_stats_output()
+        print()
+        
+        # 4. 测试爬虫自动导入
+        await test_spider_auto_import()
+        print()
+        
+        print("=" * 50)
+        print("所有验证测试通过！框架修改的功能工作正常。")
+        
+    except Exception as e:
+        print("=" * 50)
+        print(f"验证测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
     asyncio.run(main())

tests/fix_log_test.py

@@ -1,143 +1,143 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-"""
-测试日志配置修复
-"""
-
-import os
-import sys
-from pathlib import Path
-
-# 添加项目根目录到Python路径
-project_root = Path(__file__).parent.parent
-sys.path.insert(0, str(project_root))
-
-from crawlo.logging import configure_logging as configure, get_logger, LogManager
-from crawlo.logging.config import LogConfig
-
-
-def test_correct_configuration():
-    """测试正确的配置方式"""
-    print("=== 测试正确的配置方式 ===")
-    
-    # 1. 使用配置字典（推荐方式）
-    print("1. 使用配置字典...")
-    LogManager().reset()
-    
-    config_dict = {
-        'level': 'DEBUG',
-        'file_path': 'fix_test.log',
-        'max_bytes': 2048,
-        'backup_count': 2,
-        'console_enabled': True,
-        'file_enabled': True,
-        'encoding': 'utf-8'
-    }
-    
-    config = configure(**config_dict)
-    print(f"   配置文件路径: {config.file_path}")
-    
-    logger = get_logger('test.fix')
-    file_handler_found = False
-    for handler in logger.handlers:
-        if 'FileHandler' in type(handler).__name__:
-            file_handler_found = True
-            print(f"   文件处理器文件名: {handler.baseFilename}")
-    
-    print(f"   文件处理器找到: {file_handler_found}")
-    logger.info("修复测试消息")
-    
-    # 2. 使用LogConfig对象
-    print("2. 使用LogConfig对象...")
-    LogManager().reset()
-    
-    log_config = LogConfig(
-        level='INFO',
-        file_path='object_test.log',
-        max_bytes=1024,
-        backup_count=1,
-        console_enabled=True,
-        file_enabled=True
-    )
-    
-    config = configure(log_config)
-    print(f"   配置文件路径: {config.file_path}")
-    
-    logger = get_logger('test.object')
-    file_handler_found = False
-    for handler in logger.handlers:
-        if 'FileHandler' in type(handler).__name__:
-            file_handler_found = True
-            print(f"   文件处理器文件名: {handler.baseFilename}")
-    
-    print(f"   文件处理器找到: {file_handler_found}")
-    logger.info("对象配置测试消息")
-
-
-def create_settings_class():
-    """创建一个模拟的settings类来测试from_settings方法"""
-    print("\n=== 测试from_settings方法 ===")
-    
-    class MockSettings:
-        def __init__(self):
-            self.LOG_LEVEL = 'DEBUG'
-            self.LOG_FILE = 'settings_test.log'
-            self.LOG_MAX_BYTES = 1024
-            self.LOG_BACKUP_COUNT = 2
-            self.LOG_ENCODING = 'utf-8'
-            self.LOG_CONSOLE_ENABLED = True
-            self.LOG_FILE_ENABLED = True
-            self.LOG_LEVELS = {}
-    
-    # 测试from_settings方法
-    settings = MockSettings()
-    config = LogConfig.from_settings(settings)
-    
-    print(f"   从settings创建的配置:")
-    print(f"     级别: {config.level}")
-    print(f"     文件路径: {config.file_path}")
-    print(f"     轮转大小: {config.max_bytes}")
-    print(f"     备份数量: {config.backup_count}")
-    print(f"     控制台启用: {config.console_enabled}")
-    print(f"     文件启用: {config.file_enabled}")
-    
-    # 应用配置
-    LogManager().reset()
-    configure(config)
-    
-    logger = get_logger('test.settings')
-    file_handler_found = False
-    for handler in logger.handlers:
-        if 'FileHandler' in type(handler).__name__:
-            file_handler_found = True
-            print(f"   文件处理器文件名: {handler.baseFilename}")
-    
-    print(f"   文件处理器找到: {file_handler_found}")
-    logger.info("Settings配置测试消息")
-
-
-def main():
-    """主函数"""
-    print("测试日志配置修复...")
-    
-    try:
-        test_correct_configuration()
-        create_settings_class()
-        
-        print("\n=== 修复测试完成 ===")
-        print("\n推荐的日志配置方式:")
-        print("1. 使用配置字典: configure(level='DEBUG', file_path='test.log', ...)")
-        print("2. 使用LogConfig对象: configure(LogConfig(...))")
-        print("3. 使用settings对象: configure(settings_object)")
-        
-    except Exception as e:
-        print(f"\n测试过程中出现错误: {e}")
-        import traceback
-        traceback.print_exc()
-        return 1
-    
-    return 0
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+测试日志配置修复
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# 添加项目根目录到Python路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.logging import configure_logging as configure, get_logger, LogManager
+from crawlo.logging.config import LogConfig
+
+
+def test_correct_configuration():
+    """测试正确的配置方式"""
+    print("=== 测试正确的配置方式 ===")
+    
+    # 1. 使用配置字典（推荐方式）
+    print("1. 使用配置字典...")
+    LogManager().reset()
+    
+    config_dict = {
+        'level': 'DEBUG',
+        'file_path': 'fix_test.log',
+        'max_bytes': 2048,
+        'backup_count': 2,
+        'console_enabled': True,
+        'file_enabled': True,
+        'encoding': 'utf-8'
+    }
+    
+    config = configure(**config_dict)
+    print(f"   配置文件路径: {config.file_path}")
+    
+    logger = get_logger('test.fix')
+    file_handler_found = False
+    for handler in logger.handlers:
+        if 'FileHandler' in type(handler).__name__:
+            file_handler_found = True
+            print(f"   文件处理器文件名: {handler.baseFilename}")
+    
+    print(f"   文件处理器找到: {file_handler_found}")
+    logger.info("修复测试消息")
+    
+    # 2. 使用LogConfig对象
+    print("2. 使用LogConfig对象...")
+    LogManager().reset()
+    
+    log_config = LogConfig(
+        level='INFO',
+        file_path='object_test.log',
+        max_bytes=1024,
+        backup_count=1,
+        console_enabled=True,
+        file_enabled=True
+    )
+    
+    config = configure(log_config)
+    print(f"   配置文件路径: {config.file_path}")
+    
+    logger = get_logger('test.object')
+    file_handler_found = False
+    for handler in logger.handlers:
+        if 'FileHandler' in type(handler).__name__:
+            file_handler_found = True
+            print(f"   文件处理器文件名: {handler.baseFilename}")
+    
+    print(f"   文件处理器找到: {file_handler_found}")
+    logger.info("对象配置测试消息")
+
+
+def create_settings_class():
+    """创建一个模拟的settings类来测试from_settings方法"""
+    print("\n=== 测试from_settings方法 ===")
+    
+    class MockSettings:
+        def __init__(self):
+            self.LOG_LEVEL = 'DEBUG'
+            self.LOG_FILE = 'settings_test.log'
+            self.LOG_MAX_BYTES = 1024
+            self.LOG_BACKUP_COUNT = 2
+            self.LOG_ENCODING = 'utf-8'
+            self.LOG_CONSOLE_ENABLED = True
+            self.LOG_FILE_ENABLED = True
+            self.LOG_LEVELS = {}
+    
+    # 测试from_settings方法
+    settings = MockSettings()
+    config = LogConfig.from_settings(settings)
+    
+    print(f"   从settings创建的配置:")
+    print(f"     级别: {config.level}")
+    print(f"     文件路径: {config.file_path}")
+    print(f"     轮转大小: {config.max_bytes}")
+    print(f"     备份数量: {config.backup_count}")
+    print(f"     控制台启用: {config.console_enabled}")
+    print(f"     文件启用: {config.file_enabled}")
+    
+    # 应用配置
+    LogManager().reset()
+    configure(config)
+    
+    logger = get_logger('test.settings')
+    file_handler_found = False
+    for handler in logger.handlers:
+        if 'FileHandler' in type(handler).__name__:
+            file_handler_found = True
+            print(f"   文件处理器文件名: {handler.baseFilename}")
+    
+    print(f"   文件处理器找到: {file_handler_found}")
+    logger.info("Settings配置测试消息")
+
+
+def main():
+    """主函数"""
+    print("测试日志配置修复...")
+    
+    try:
+        test_correct_configuration()
+        create_settings_class()
+        
+        print("\n=== 修复测试完成 ===")
+        print("\n推荐的日志配置方式:")
+        print("1. 使用配置字典: configure(level='DEBUG', file_path='test.log', ...)")
+        print("2. 使用LogConfig对象: configure(LogConfig(...))")
+        print("3. 使用settings对象: configure(settings_object)")
+        
+    except Exception as e:
+        print(f"\n测试过程中出现错误: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == '__main__':
     sys.exit(main())