crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

tests/simple_log_test2.py

@@ -1,138 +1,138 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-"""
-简单的日志系统测试
-"""
-
-import os
-import sys
-import tempfile
-import shutil
-from pathlib import Path
-
-# 添加项目根目录到Python路径
-project_root = Path(__file__).parent.parent
-sys.path.insert(0, str(project_root))
-
-from crawlo.logging import configure_logging as configure, get_logger, LogManager
-from crawlo.logging.config import LogConfig
-
-
-def test_file_logging_simple():
-    """简单测试文件日志功能"""
-    print("=== 简单测试文件日志功能 ===")
-    
-    # 创建临时目录
-    temp_dir = tempfile.mkdtemp()
-    log_file = os.path.join(temp_dir, 'simple_test.log')
-    
-    try:
-        print(f"日志文件路径: {log_file}")
-        
-        # 配置文件日志
-        configure(
-            LOG_LEVEL='INFO',
-            LOG_FILE=log_file,
-            LOG_FILE_ENABLED=True
-        )
-        
-        config = LogManager().config
-        print(f"配置文件路径: {config.file_path}")
-        print(f"文件启用: {config.file_enabled}")
-        
-        # 获取logger并测试输出
-        logger = get_logger('test.simple')
-        print(f"Logger handlers数量: {len(logger.handlers)}")
-        
-        for handler in logger.handlers:
-            print(f"Handler类型: {type(handler).__name__}")
-            if hasattr(handler, 'baseFilename'):
-                print(f"Handler文件名: {handler.baseFilename}")
-        
-        logger.info("这是一条测试日志")
-        logger.warning("这是一条警告日志")
-        
-        # 检查日志文件
-        print("检查日志文件...")
-        if os.path.exists(log_file):
-            with open(log_file, 'r', encoding='utf-8') as f:
-                content = f.read()
-                print(f"日志文件内容:\n{content}")
-        else:
-            print("日志文件不存在!")
-            
-    finally:
-        # 清理临时目录
-        shutil.rmtree(temp_dir, ignore_errors=True)
-
-
-def test_console_only():
-    """测试仅控制台日志"""
-    print("\n=== 测试仅控制台日志 ===")
-    
-    # 配置仅控制台日志
-    configure(
-        LOG_LEVEL='INFO',
-        LOG_CONSOLE_ENABLED=True,
-        LOG_FILE_ENABLED=False
-    )
-    
-    config = LogManager().config
-    print(f"控制台启用: {config.console_enabled}")
-    print(f"文件启用: {config.file_enabled}")
-    
-    # 获取logger并测试输出
-    logger = get_logger('test.console_only')
-    logger.info("这条日志应该只在控制台显示")
-
-
-def test_module_levels():
-    """测试模块级别日志"""
-    print("\n=== 测试模块级别日志 ===")
-    
-    # 配置模块级别
-    configure(
-        LOG_LEVEL='WARNING',
-        LOG_LEVELS={
-            'module.debug': 'DEBUG',
-            'module.error': 'ERROR'
-        }
-    )
-    
-    config = LogManager().config
-    print(f"默认级别: {config.level}")
-    print(f"模块级别: {config.module_levels}")
-    
-    # 测试不同模块的日志级别
-    debug_logger = get_logger('module.debug')
-    debug_logger.debug("这条DEBUG日志应该显示")
-    debug_logger.info("这条INFO日志应该显示")
-    
-    error_logger = get_logger('module.error')
-    error_logger.info("这条INFO日志不应该显示")
-    error_logger.error("这条ERROR日志应该显示")
-
-
-def main():
-    """主测试函数"""
-    print("开始简单测试Crawlo框架日志系统...")
-    
-    try:
-        # 运行测试
-        test_file_logging_simple()
-        test_console_only()
-        test_module_levels()
-        
-        print("\n=== 简单测试完成 ===")
-        
-    except Exception as e:
-        print(f"\n测试过程中出现错误: {e}")
-        import traceback
-        traceback.print_exc()
-        return 1
-    
-    return 0
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+简单的日志系统测试
+"""
+
+import os
+import sys
+import tempfile
+import shutil
+from pathlib import Path
+
+# 添加项目根目录到Python路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.logging import configure_logging as configure, get_logger, LogManager
+from crawlo.logging.config import LogConfig
+
+
+def test_file_logging_simple():
+    """简单测试文件日志功能"""
+    print("=== 简单测试文件日志功能 ===")
+    
+    # 创建临时目录
+    temp_dir = tempfile.mkdtemp()
+    log_file = os.path.join(temp_dir, 'simple_test.log')
+    
+    try:
+        print(f"日志文件路径: {log_file}")
+        
+        # 配置文件日志
+        configure(
+            LOG_LEVEL='INFO',
+            LOG_FILE=log_file,
+            LOG_FILE_ENABLED=True
+        )
+        
+        config = LogManager().config
+        print(f"配置文件路径: {config.file_path}")
+        print(f"文件启用: {config.file_enabled}")
+        
+        # 获取logger并测试输出
+        logger = get_logger('test.simple')
+        print(f"Logger handlers数量: {len(logger.handlers)}")
+        
+        for handler in logger.handlers:
+            print(f"Handler类型: {type(handler).__name__}")
+            if hasattr(handler, 'baseFilename'):
+                print(f"Handler文件名: {handler.baseFilename}")
+        
+        logger.info("这是一条测试日志")
+        logger.warning("这是一条警告日志")
+        
+        # 检查日志文件
+        print("检查日志文件...")
+        if os.path.exists(log_file):
+            with open(log_file, 'r', encoding='utf-8') as f:
+                content = f.read()
+                print(f"日志文件内容:\n{content}")
+        else:
+            print("日志文件不存在!")
+            
+    finally:
+        # 清理临时目录
+        shutil.rmtree(temp_dir, ignore_errors=True)
+
+
+def test_console_only():
+    """测试仅控制台日志"""
+    print("\n=== 测试仅控制台日志 ===")
+    
+    # 配置仅控制台日志
+    configure(
+        LOG_LEVEL='INFO',
+        LOG_CONSOLE_ENABLED=True,
+        LOG_FILE_ENABLED=False
+    )
+    
+    config = LogManager().config
+    print(f"控制台启用: {config.console_enabled}")
+    print(f"文件启用: {config.file_enabled}")
+    
+    # 获取logger并测试输出
+    logger = get_logger('test.console_only')
+    logger.info("这条日志应该只在控制台显示")
+
+
+def test_module_levels():
+    """测试模块级别日志"""
+    print("\n=== 测试模块级别日志 ===")
+    
+    # 配置模块级别
+    configure(
+        LOG_LEVEL='WARNING',
+        LOG_LEVELS={
+            'module.debug': 'DEBUG',
+            'module.error': 'ERROR'
+        }
+    )
+    
+    config = LogManager().config
+    print(f"默认级别: {config.level}")
+    print(f"模块级别: {config.module_levels}")
+    
+    # 测试不同模块的日志级别
+    debug_logger = get_logger('module.debug')
+    debug_logger.debug("这条DEBUG日志应该显示")
+    debug_logger.info("这条INFO日志应该显示")
+    
+    error_logger = get_logger('module.error')
+    error_logger.info("这条INFO日志不应该显示")
+    error_logger.error("这条ERROR日志应该显示")
+
+
+def main():
+    """主测试函数"""
+    print("开始简单测试Crawlo框架日志系统...")
+    
+    try:
+        # 运行测试
+        test_file_logging_simple()
+        test_console_only()
+        test_module_levels()
+        
+        print("\n=== 简单测试完成 ===")
+        
+    except Exception as e:
+        print(f"\n测试过程中出现错误: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == '__main__':
     sys.exit(main())

tests/simple_optimization_test.py

@@ -1,129 +1,129 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-简单的Crawlo框架优化验证测试
-用于快速验证优化效果
-"""
-
-import asyncio
-import time
-import sys
-import os
-from pathlib import Path
-
-# 添加项目根目录到 Python 路径
-project_root = Path(__file__).parent.parent
-sys.path.insert(0, str(project_root))
-
-from crawlo.crawler import CrawlerProcess
-from crawlo.spider import Spider
-from crawlo import Request, Item
-
-
-class SimpleTestSpider(Spider):
-    """简单的测试爬虫"""
-    name = 'simple_test'
-    
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # 使用较少的测试页面以加快测试速度
-        self.start_urls = ['https://httpbin.org/delay/0' for _ in range(10)]
-    
-    def parse(self, response):
-        """简单解析响应"""
-        yield {'url': response.url, 'status': response.status_code}
-
-
-async def test_concurrent_performance():
-    """测试并发性能"""
-    print("开始并发性能测试...")
-    
-    # 配置设置
-    settings = {
-        'CONCURRENT_REQUESTS': 10,
-        'DOWNLOAD_DELAY': 0.1,
-        'RANDOMIZE_DOWNLOAD_DELAY': False,
-        'SCHEDULER_MAX_QUEUE_SIZE': 5000,
-        'BACKPRESSURE_RATIO': 0.9,
-    }
-    
-    # 创建爬虫进程
-    process = CrawlerProcess(settings=settings)
-    
-    # 记录开始时间
-    start_time = time.time()
-    
-    # 添加测试爬虫
-    crawler = await process.crawl(SimpleTestSpider)
-    
-    # 计算性能指标
-    metrics = crawler.metrics
-    duration = metrics.get_total_duration()
-    pages = 10
-    rps = pages / duration if duration > 0 else 0
-    
-    print(f"完成时间: {duration:.2f} 秒")
-    print(f"每秒请求数: {rps:.2f} RPS")
-    
-    return duration, rps
-
-
-async def test_sequential_performance():
-    """测试顺序执行性能"""
-    print("\n开始顺序执行性能测试...")
-    
-    # 配置设置
-    settings = {
-        'CONCURRENT_REQUESTS': 1,  # 顺序执行
-        'DOWNLOAD_DELAY': 0.1,
-        'RANDOMIZE_DOWNLOAD_DELAY': False,
-        'SCHEDULER_MAX_QUEUE_SIZE': 5000,
-        'BACKPRESSURE_RATIO': 0.9,
-    }
-    
-    # 创建爬虫进程
-    process = CrawlerProcess(settings=settings)
-    
-    # 记录开始时间
-    start_time = time.time()
-    
-    # 添加测试爬虫
-    crawler = await process.crawl(SimpleTestSpider)
-    
-    # 计算性能指标
-    metrics = crawler.metrics
-    duration = metrics.get_total_duration()
-    pages = 10
-    rps = pages / duration if duration > 0 else 0
-    
-    print(f"完成时间: {duration:.2f} 秒")
-    print(f"每秒请求数: {rps:.2f} RPS")
-    
-    return duration, rps
-
-
-async def main():
-    """主函数"""
-    print("Crawlo 框架优化效果验证测试")
-    print("=" * 40)
-    
-    # 测试并发性能
-    concurrent_duration, concurrent_rps = await test_concurrent_performance()
-    
-    # 测试顺序执行性能
-    sequential_duration, sequential_rps = await test_sequential_performance()
-    
-    # 输出比较结果
-    print("\n=== 性能对比 ===")
-    print(f"并发执行: {concurrent_duration:.2f}s, {concurrent_rps:.2f} RPS")
-    print(f"顺序执行: {sequential_duration:.2f}s, {sequential_rps:.2f} RPS")
-    
-    if concurrent_duration < sequential_duration:
-        improvement = (sequential_duration - concurrent_duration) / sequential_duration * 100
-        print(f"并发执行比顺序执行快 {improvement:.1f}%")
-    else:
-        print("并发执行性能未达到预期，请检查优化效果")
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+简单的Crawlo框架优化验证测试
+用于快速验证优化效果
+"""
+
+import asyncio
+import time
+import sys
+import os
+from pathlib import Path
+
+# 添加项目根目录到 Python 路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import Spider
+from crawlo import Request, Item
+
+
+class SimpleTestSpider(Spider):
+    """简单的测试爬虫"""
+    name = 'simple_test'
+    
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # 使用较少的测试页面以加快测试速度
+        self.start_urls = ['https://httpbin.org/delay/0' for _ in range(10)]
+    
+    def parse(self, response):
+        """简单解析响应"""
+        yield {'url': response.url, 'status': response.status_code}
+
+
+async def test_concurrent_performance():
+    """测试并发性能"""
+    print("开始并发性能测试...")
+    
+    # 配置设置
+    settings = {
+        'CONCURRENT_REQUESTS': 10,
+        'DOWNLOAD_DELAY': 0.1,
+        'RANDOMIZE_DOWNLOAD_DELAY': False,
+        'SCHEDULER_MAX_QUEUE_SIZE': 5000,
+        'BACKPRESSURE_RATIO': 0.9,
+    }
+    
+    # 创建爬虫进程
+    process = CrawlerProcess(settings=settings)
+    
+    # 记录开始时间
+    start_time = time.time()
+    
+    # 添加测试爬虫
+    crawler = await process.crawl(SimpleTestSpider)
+    
+    # 计算性能指标
+    metrics = crawler.metrics
+    duration = metrics.get_total_duration()
+    pages = 10
+    rps = pages / duration if duration > 0 else 0
+    
+    print(f"完成时间: {duration:.2f} 秒")
+    print(f"每秒请求数: {rps:.2f} RPS")
+    
+    return duration, rps
+
+
+async def test_sequential_performance():
+    """测试顺序执行性能"""
+    print("\n开始顺序执行性能测试...")
+    
+    # 配置设置
+    settings = {
+        'CONCURRENT_REQUESTS': 1,  # 顺序执行
+        'DOWNLOAD_DELAY': 0.1,
+        'RANDOMIZE_DOWNLOAD_DELAY': False,
+        'SCHEDULER_MAX_QUEUE_SIZE': 5000,
+        'BACKPRESSURE_RATIO': 0.9,
+    }
+    
+    # 创建爬虫进程
+    process = CrawlerProcess(settings=settings)
+    
+    # 记录开始时间
+    start_time = time.time()
+    
+    # 添加测试爬虫
+    crawler = await process.crawl(SimpleTestSpider)
+    
+    # 计算性能指标
+    metrics = crawler.metrics
+    duration = metrics.get_total_duration()
+    pages = 10
+    rps = pages / duration if duration > 0 else 0
+    
+    print(f"完成时间: {duration:.2f} 秒")
+    print(f"每秒请求数: {rps:.2f} RPS")
+    
+    return duration, rps
+
+
+async def main():
+    """主函数"""
+    print("Crawlo 框架优化效果验证测试")
+    print("=" * 40)
+    
+    # 测试并发性能
+    concurrent_duration, concurrent_rps = await test_concurrent_performance()
+    
+    # 测试顺序执行性能
+    sequential_duration, sequential_rps = await test_sequential_performance()
+    
+    # 输出比较结果
+    print("\n=== 性能对比 ===")
+    print(f"并发执行: {concurrent_duration:.2f}s, {concurrent_rps:.2f} RPS")
+    print(f"顺序执行: {sequential_duration:.2f}s, {sequential_rps:.2f} RPS")
+    
+    if concurrent_duration < sequential_duration:
+        improvement = (sequential_duration - concurrent_duration) / sequential_duration * 100
+        print(f"并发执行比顺序执行快 {improvement:.1f}%")
+    else:
+        print("并发执行性能未达到预期，请检查优化效果")
+
+
+if __name__ == '__main__':
     asyncio.run(main())

tests/simple_queue_type_test.py

@@ -1,42 +1,42 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-简单测试 QUEUE_TYPE 配置获取
-验证我们的日志格式修改是否正常工作
-"""
-
-import sys
-import os
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.config import CrawloConfig
-from crawlo.framework import CrawloFramework
-
-
-def test_log_format():
-    """测试日志格式修改是否正常工作"""
-    print("=== 测试日志格式修改 ===")
-    
-    # 创建单机模式配置
-    config = CrawloConfig.standalone(concurrency=4)
-    
-    # 创建框架实例，这会触发日志输出
-    framework = CrawloFramework(config.to_dict())
-    
-    # 获取配置信息
-    run_mode = framework.settings.get('RUN_MODE', 'not found')
-    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
-    
-    print(f"从配置中获取到的信息:")
-    print(f"  RunMode: {run_mode}")
-    print(f"  QueueType: {queue_type}")
-    
-    print("\n✅ 日志格式修改测试完成")
-
-
-if __name__ == "__main__":
-    print("开始简单测试 QUEUE_TYPE 配置获取...")
-    test_log_format()
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+简单测试 QUEUE_TYPE 配置获取
+验证我们的日志格式修改是否正常工作
+"""
+
+import sys
+import os
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.config import CrawloConfig
+from crawlo.framework import CrawloFramework
+
+
+def test_log_format():
+    """测试日志格式修改是否正常工作"""
+    print("=== 测试日志格式修改 ===")
+    
+    # 创建单机模式配置
+    config = CrawloConfig.standalone(concurrency=4)
+    
+    # 创建框架实例，这会触发日志输出
+    framework = CrawloFramework(config.to_dict())
+    
+    # 获取配置信息
+    run_mode = framework.settings.get('RUN_MODE', 'not found')
+    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
+    
+    print(f"从配置中获取到的信息:")
+    print(f"  RunMode: {run_mode}")
+    print(f"  QueueType: {queue_type}")
+    
+    print("\n✅ 日志格式修改测试完成")
+
+
+if __name__ == "__main__":
+    print("开始简单测试 QUEUE_TYPE 配置获取...")
+    test_log_format()
     print("\n测试结束！")