crawlo 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

tests/debug_log_levels.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+调试日志级别配置脚本
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+from crawlo.core.framework_initializer import initialize_framework
+from crawlo.utils.log import LoggerManager, get_logger
+import logging
+
+def main():
+    print("=== 开始调试日志级别配置 ===")
+    
+    # 初始化框架
+    print("1. 初始化框架...")
+    settings = initialize_framework()
+    
+    # 打印配置信息
+    print(f"2. 配置信息:")
+    print(f"   LOG_LEVEL: {settings.get('LOG_LEVEL')}")
+    print(f"   LOG_FILE: {settings.get('LOG_FILE')}")
+    print(f"   LoggerManager._default_level: {LoggerManager._default_level}")
+    print(f"   LoggerManager._default_console_level: {LoggerManager._default_console_level}")
+    print(f"   LoggerManager._default_file_level: {LoggerManager._default_file_level}")
+    
+    # 测试不同组件的日志级别
+    components = [
+        'crawlo.framework',
+        'crawlo.crawler', 
+        'QueueManager',
+        'Scheduler',
+        'AioHttpDownloader',
+        'MiddlewareManager',
+        'PipelineManager',
+        'ExtensionManager',
+        'of_week_standalone'
+    ]
+    
+    print("3. 组件日志级别测试:")
+    for component_name in components:
+        logger = get_logger(component_name)
+        print(f"   {component_name}:")
+        print(f"     logger.level: {logger.level} ({logging.getLevelName(logger.level)})")
+        
+        for handler in logger.handlers:
+            handler_type = type(handler).__name__
+            print(f"     {handler_type}.level: {handler.level} ({logging.getLevelName(handler.level)})")
+    
+    # 实际测试日志输出
+    print("4. 测试日志输出:")
+    test_logger = get_logger('TestLogger')
+    
+    print("   控制台应该看到以下日志:")
+    test_logger.debug("这是DEBUG级别日志 - 控制台应该看不到")
+    test_logger.info("这是INFO级别日志 - 控制台应该能看到")
+    test_logger.warning("这是WARNING级别日志 - 控制台应该能看到")
+    
+    print("=== 调试完成 ===")
+
+if __name__ == '__main__':
+    main()

tests/distributed_test.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+分布式模式测试脚本
+"""
+
+import sys
+import os
+import asyncio
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class DistributedTestSpider(Spider):
+    """分布式测试爬虫"""
+    name = 'distributed_test_spider'
+    
+    def start_requests(self):
+        """发起测试请求"""
+        # 生成一些测试请求
+        for i in range(5):
+            yield Request(f'https://httpbin.org/get?page={i}', callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功获取响应: {response.url}")
+        print(f"状态码: {response.status_code}")
+        return []
+
+
+async def test_distributed_mode():
+    """测试分布式模式"""
+    print("开始测试分布式模式...")
+    
+    # 初始化框架，设置为分布式模式
+    from crawlo.initialization import initialize_framework
+    custom_settings = {
+        'RUN_MODE': 'distributed',
+        'QUEUE_TYPE': 'redis',
+        'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
+        'REDIS_HOST': '127.0.0.1',
+        'REDIS_PORT': 6379,
+        'REDIS_DB': 15,  # 使用测试数据库
+        'PROJECT_NAME': 'distributed_test'
+    }
+    settings = initialize_framework(custom_settings)
+    
+    # 创建爬虫进程
+    from crawlo.crawler import CrawlerProcess
+    process = CrawlerProcess(settings=settings)
+    
+    # 运行爬虫
+    await process.crawl(DistributedTestSpider)
+    
+    print("分布式模式测试完成！")
+
+
+def main():
+    """主函数"""
+    print("开始分布式模式测试...")
+    asyncio.run(test_distributed_mode())
+
+
+if __name__ == "__main__":
+    main()

tests/distributed_test_debug.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+分布式模式调试测试脚本
+"""
+
+import sys
+import os
+import asyncio
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class DistributedTestSpider(Spider):
+    """分布式测试爬虫"""
+    name = 'distributed_test_spider'
+    
+    def start_requests(self):
+        """发起测试请求"""
+        # 生成一些测试请求
+        for i in range(3):
+            yield Request(f'https://httpbin.org/get?page={i}', callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功获取响应: {response.url}")
+        print(f"状态码: {response.status_code}")
+        return []
+
+
+async def test_distributed_mode():
+    """测试分布式模式"""
+    print("开始测试分布式模式...")
+    
+    # 初始化框架，设置为分布式模式
+    from crawlo.initialization import initialize_framework
+    custom_settings = {
+        'RUN_MODE': 'distributed',
+        'QUEUE_TYPE': 'redis',
+        'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
+        'REDIS_HOST': '127.0.0.1',
+        'REDIS_PORT': 6379,
+        'REDIS_DB': 15,  # 使用测试数据库
+        'PROJECT_NAME': 'distributed_test'
+    }
+    
+    print("自定义配置:")
+    for key, value in custom_settings.items():
+        print(f"  {key}: {value}")
+    
+    settings = initialize_framework(custom_settings)
+    
+    print("初始化后的配置:")
+    print(f"  RUN_MODE: {settings.get('RUN_MODE')}")
+    print(f"  QUEUE_TYPE: {settings.get('QUEUE_TYPE')}")
+    print(f"  FILTER_CLASS: {settings.get('FILTER_CLASS')}")
+    
+    # 创建爬虫进程
+    from crawlo.crawler import CrawlerProcess
+    process = CrawlerProcess(settings=settings)
+    
+    # 运行爬虫
+    await process.crawl(DistributedTestSpider)
+    
+    print("分布式模式测试完成！")
+
+
+def main():
+    """主函数"""
+    print("开始分布式模式调试测试...")
+    asyncio.run(test_distributed_mode())
+
+
+if __name__ == "__main__":
+    main()

tests/final_comprehensive_test.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+最终综合测试脚本
+全面测试框架的所有核心功能，特别是我们修改的部分
+"""
+
+import sys
+import os
+import asyncio
+import time
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class TestSpider(Spider):
+    """测试爬虫"""
+    name = 'final_test_spider'
+    
+    def start_requests(self):
+        """发起测试请求"""
+        # 生成一些测试请求
+        for i in range(5):
+            yield Request(f'https://httpbin.org/get?page={i}', callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功获取响应: {response.url}")
+        print(f"状态码: {response.status_code}")
+        return []
+
+
+async def test_queue_blocking_behavior():
+    """测试队列阻塞行为"""
+    print("测试队列阻塞行为...")
+    
+    from crawlo.queue.queue_manager import QueueConfig, QueueManager
+    
+    # 创建小队列配置
+    queue_config = QueueConfig(
+        queue_type='memory',
+        max_queue_size=3  # 非常小的队列
+    )
+    
+    # 创建队列管理器
+    queue_manager = QueueManager(queue_config)
+    await queue_manager.initialize()
+    
+    # 测试添加超过队列大小的请求
+    print("添加6个请求到大小为3的队列...")
+    start_time = time.time()
+    
+    # 创建生产者任务
+    async def producer():
+        for i in range(6):
+            request = Request(f'https://example.com/test{i}')
+            await queue_manager.put(request)
+            print(f"添加请求 {i}")
+    
+    # 创建消费者任务
+    async def consumer():
+        retrieved = 0
+        while retrieved < 6:
+            request = await queue_manager.get(timeout=2.0)
+            if request:
+                print(f"获取请求: {request.url}")
+                retrieved += 1
+                await asyncio.sleep(0.1)  # 模拟处理时间
+    
+    # 并发运行生产者和消费者
+    await asyncio.gather(producer(), consumer())
+    
+    end_time = time.time()
+    print(f"队列测试完成，耗时 {end_time - start_time:.2f} 秒")
+    
+    # 关闭队列
+    await queue_manager.close()
+
+
+async def test_framework_initialization():
+    """测试框架初始化"""
+    print("测试框架初始化...")
+    
+    from crawlo.initialization import initialize_framework
+    
+    # 测试默认配置
+    settings = initialize_framework()
+    print(f"默认配置 - RUN_MODE: {settings.get('RUN_MODE')}")
+    print(f"默认配置 - QUEUE_TYPE: {settings.get('QUEUE_TYPE')}")
+    
+    # 测试自定义配置
+    custom_settings = {
+        'PROJECT_NAME': 'final_test',
+        'SCHEDULER_MAX_QUEUE_SIZE': 100
+    }
+    
+    settings = initialize_framework(custom_settings)
+    print(f"自定义配置 - PROJECT_NAME: {settings.get('PROJECT_NAME')}")
+    print(f"自定义配置 - SCHEDULER_MAX_QUEUE_SIZE: {settings.get('SCHEDULER_MAX_QUEUE_SIZE')}")
+
+
+async def test_crawler_execution():
+    """测试爬虫执行"""
+    print("测试爬虫执行...")
+    
+    from crawlo.initialization import initialize_framework
+    from crawlo.crawler import CrawlerProcess
+    
+    # 初始化框架
+    settings = initialize_framework({
+        'PROJECT_NAME': 'final_test'
+    })
+    
+    # 创建爬虫进程
+    process = CrawlerProcess(settings=settings)
+    
+    # 运行爬虫
+    await process.crawl(TestSpider)
+
+
+async def main():
+    """主函数"""
+    print("开始最终综合测试...")
+    print("=" * 50)
+    
+    try:
+        # 1. 测试框架初始化
+        await test_framework_initialization()
+        print()
+        
+        # 2. 测试队列阻塞行为
+        await test_queue_blocking_behavior()
+        print()
+        
+        # 3. 测试爬虫执行
+        await test_crawler_execution()
+        print()
+        
+        print("=" * 50)
+        print("所有测试通过！框架工作正常。")
+        
+    except Exception as e:
+        print("=" * 50)
+        print(f"测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/final_validation_test.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+最终验证测试脚本
+验证所有修改的功能是否正常工作
+"""
+
+import sys
+import os
+import asyncio
+import time
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class ValidationTestSpider(Spider):
+    """验证测试爬虫"""
+    name = 'validation_test_spider'
+    
+    def start_requests(self):
+        """发起测试请求"""
+        # 生成一些测试请求
+        for i in range(3):
+            yield Request(f'https://httpbin.org/get?page={i}', callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功获取响应: {response.url}")
+        print(f"状态码: {response.status_code}")
+        return []
+
+
+async def test_framework_startup_logging():
+    """测试框架启动日志"""
+    print("测试框架启动日志...")
+    
+    from crawlo.initialization import initialize_framework
+    from crawlo.logging import get_logger
+    
+    # 初始化框架
+    settings = initialize_framework({
+        'PROJECT_NAME': 'validation_test'
+    })
+    
+    # 获取框架logger并检查启动日志
+    logger = get_logger('crawlo.framework')
+    print("框架启动日志已记录")
+
+
+async def test_queue_blocking_behavior():
+    """测试队列阻塞行为"""
+    print("测试队列阻塞行为...")
+    
+    from crawlo.queue.queue_manager import QueueConfig, QueueManager
+    from crawlo import Request
+    
+    # 创建小队列配置进行测试
+    queue_config = QueueConfig(
+        queue_type='memory',
+        max_queue_size=3  # 非常小的队列
+    )
+    
+    # 创建队列管理器
+    queue_manager = QueueManager(queue_config)
+    await queue_manager.initialize()
+    
+    # 测试添加超过队列大小的请求
+    print("添加5个请求到大小为3的队列...")
+    start_time = time.time()
+    
+    # 创建生产者任务
+    async def producer():
+        for i in range(5):
+            request = Request(f'https://example.com/test{i}')
+            await queue_manager.put(request)
+            print(f"添加请求 {i}")
+    
+    # 创建消费者任务
+    async def consumer():
+        retrieved = 0
+        while retrieved < 5:
+            request = await queue_manager.get(timeout=2.0)
+            if request:
+                print(f"获取请求: {request.url}")
+                retrieved += 1
+                await asyncio.sleep(0.1)  # 模拟处理时间
+    
+    # 并发运行生产者和消费者
+    await asyncio.gather(producer(), consumer())
+    
+    end_time = time.time()
+    print(f"队列测试完成，耗时 {end_time - start_time:.2f} 秒")
+    
+    # 关闭队列
+    await queue_manager.close()
+
+
+async def test_stats_output():
+    """测试统计信息输出"""
+    print("测试统计信息输出...")
+    
+    from crawlo.initialization import initialize_framework
+    from crawlo.crawler import CrawlerProcess
+    
+    # 初始化框架
+    settings = initialize_framework({
+        'PROJECT_NAME': 'stats_test'
+    })
+    
+    # 创建爬虫进程
+    process = CrawlerProcess(settings=settings)
+    
+    # 运行爬虫
+    await process.crawl(ValidationTestSpider)
+    
+    print("统计信息输出测试完成")
+
+
+async def test_spider_auto_import():
+    """测试爬虫自动导入"""
+    print("测试爬虫自动导入...")
+    
+    from crawlo.initialization import initialize_framework
+    from crawlo.crawler import CrawlerProcess
+    
+    # 初始化框架
+    settings = initialize_framework({
+        'PROJECT_NAME': 'auto_import_test'
+    })
+    
+    # 创建爬虫进程，指定spider_modules
+    spider_modules = ['crawlo.spider']  # 使用框架的spider模块进行测试
+    process = CrawlerProcess(settings=settings, spider_modules=spider_modules)
+    
+    # 检查爬虫是否已注册
+    spider_name = ValidationTestSpider.name
+    is_registered = process.is_spider_registered(spider_name)
+    print(f"爬虫 '{spider_name}' 是否已注册: {is_registered}")
+    
+    if is_registered:
+        print("爬虫自动导入功能正常")
+    else:
+        print("爬虫自动导入功能异常")
+
+
+async def main():
+    """主函数"""
+    print("开始最终验证测试...")
+    print("=" * 50)
+    
+    try:
+        # 1. 测试框架启动日志
+        await test_framework_startup_logging()
+        print()
+        
+        # 2. 测试队列阻塞行为
+        await test_queue_blocking_behavior()
+        print()
+        
+        # 3. 测试统计信息输出
+        await test_stats_output()
+        print()
+        
+        # 4. 测试爬虫自动导入
+        await test_spider_auto_import()
+        print()
+        
+        print("=" * 50)
+        print("所有验证测试通过！框架修改的功能工作正常。")
+        
+    except Exception as e:
+        print("=" * 50)
+        print(f"验证测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())