crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

tests/test_fingerprint_simple.py

@@ -1,52 +1,52 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-简化版指纹一致性测试
-==============
-验证框架中各组件对相同数据生成一致的指纹
-"""
-
-import sys
-import os
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.utils.fingerprint import FingerprintGenerator
-
-
-def test_fingerprint_consistency():
-    """测试指纹一致性"""
-    # 测试数据
-    test_data = {
-        "title": "Test Title",
-        "url": "https://example.com",
-        "content": "Test content",
-        "price": 99.99
-    }
-    
-    # 使用指纹生成器生成指纹
-    fingerprint1 = FingerprintGenerator.data_fingerprint(test_data)
-    fingerprint2 = FingerprintGenerator.data_fingerprint(test_data)
-    
-    # 验证相同数据生成相同指纹
-    print(f"First fingerprint: {fingerprint1}")
-    print(f"Second fingerprint: {fingerprint2}")
-    print(f"指纹一致: {fingerprint1 == fingerprint2}")
-    
-    # 测试请求指纹
-    method = "GET"
-    url = "https://example.com"
-    body = b""
-    headers = {"User-Agent": "test-agent"}
-    
-    request_fingerprint1 = FingerprintGenerator.request_fingerprint(method, url, body, headers)
-    request_fingerprint2 = FingerprintGenerator.request_fingerprint(method, url, body, headers)
-    
-    print(f"\nRequest fingerprint 1: {request_fingerprint1}")
-    print(f"Request fingerprint 2: {request_fingerprint2}")
-    print(f"请求指纹一致: {request_fingerprint1 == request_fingerprint2}")
-
-
-if __name__ == '__main__':
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+简化版指纹一致性测试
+==============
+验证框架中各组件对相同数据生成一致的指纹
+"""
+
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.fingerprint import FingerprintGenerator
+
+
+def test_fingerprint_consistency():
+    """测试指纹一致性"""
+    # 测试数据
+    test_data = {
+        "title": "Test Title",
+        "url": "https://example.com",
+        "content": "Test content",
+        "price": 99.99
+    }
+    
+    # 使用指纹生成器生成指纹
+    fingerprint1 = FingerprintGenerator.data_fingerprint(test_data)
+    fingerprint2 = FingerprintGenerator.data_fingerprint(test_data)
+    
+    # 验证相同数据生成相同指纹
+    print(f"First fingerprint: {fingerprint1}")
+    print(f"Second fingerprint: {fingerprint2}")
+    print(f"指纹一致: {fingerprint1 == fingerprint2}")
+    
+    # 测试请求指纹
+    method = "GET"
+    url = "https://example.com"
+    body = b""
+    headers = {"User-Agent": "test-agent"}
+    
+    request_fingerprint1 = FingerprintGenerator.request_fingerprint(method, url, body, headers)
+    request_fingerprint2 = FingerprintGenerator.request_fingerprint(method, url, body, headers)
+    
+    print(f"\nRequest fingerprint 1: {request_fingerprint1}")
+    print(f"Request fingerprint 2: {request_fingerprint2}")
+    print(f"请求指纹一致: {request_fingerprint1 == request_fingerprint2}")
+
+
+if __name__ == '__main__':
     test_fingerprint_consistency()

tests/test_framework_env_usage.py

@@ -1,104 +1,104 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-框架环境变量使用测试
-验证整个框架中环境变量的正确使用
-"""
-import sys
-import os
-import unittest
-from unittest.mock import patch, MagicMock
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.utils.env_config import get_env_var, get_redis_config, get_runtime_config
-from crawlo.settings.setting_manager import SettingManager
-from crawlo.settings import default_settings
-from crawlo.mode_manager import from_env
-
-
-class TestFrameworkEnvUsage(unittest.TestCase):
-    """框架环境变量使用测试"""
-
-    def test_default_settings_env_usage(self):
-        """测试 default_settings.py 中的环境变量使用"""
-        # 验证 default_settings.py 不直接使用 os.getenv
-        import inspect
-        import crawlo.settings.default_settings as default_settings_module
-        
-        source_code = inspect.getsource(default_settings_module)
-        # 检查是否还有直接使用 os.getenv 的地方
-        self.assertNotIn('os.getenv', source_code, 
-                         "default_settings.py 不应该直接使用 os.getenv")
-        
-        # 但应该使用 env_config 工具
-        self.assertIn('get_redis_config', source_code,
-                      "default_settings.py 应该使用 get_redis_config")
-        self.assertIn('get_runtime_config', source_code,
-                      "default_settings.py 应该使用 get_runtime_config")
-
-    def test_env_config_tool(self):
-        """测试环境变量配置工具"""
-        # 测试获取Redis配置
-        with patch.dict(os.environ, {
-            'REDIS_HOST': 'test.redis.com',
-            'REDIS_PORT': '6380',
-            'REDIS_PASSWORD': 'test_pass',
-            'REDIS_DB': '2'
-        }):
-            redis_config = get_redis_config()
-            self.assertEqual(redis_config['REDIS_HOST'], 'test.redis.com')
-            self.assertEqual(redis_config['REDIS_PORT'], 6380)
-            self.assertEqual(redis_config['REDIS_PASSWORD'], 'test_pass')
-            self.assertEqual(redis_config['REDIS_DB'], 2)
-        
-        # 测试获取运行时配置
-        with patch.dict(os.environ, {
-            'PROJECT_NAME': 'test_project',
-            'CRAWLO_MODE': 'distributed',
-            'CONCURRENCY': '16'
-        }):
-            runtime_config = get_runtime_config()
-            self.assertEqual(runtime_config['PROJECT_NAME'], 'test_project')
-            self.assertEqual(runtime_config['CRAWLO_MODE'], 'distributed')
-            self.assertEqual(runtime_config['CONCURRENCY'], 16)
-
-    def test_settings_manager_with_env(self):
-        """测试设置管理器与环境变量的集成"""
-        # 设置环境变量
-        env_vars = {
-            'PROJECT_NAME': 'env_test_project',
-            'CONCURRENCY': '12',
-            'REDIS_HOST': 'env.redis.test',
-            'REDIS_PORT': '6381'
-        }
-        
-        with patch.dict(os.environ, env_vars):
-            # 重新导入 default_settings 模块以获取最新的环境变量
-            import importlib
-            import crawlo.settings.default_settings
-            importlib.reload(crawlo.settings.default_settings)
-            
-            # 创建设置管理器
-            settings = SettingManager()
-            settings.set_settings(crawlo.settings.default_settings)
-            
-            # 验证环境变量被正确使用
-            redis_config = get_redis_config()
-            self.assertEqual(settings.get('REDIS_HOST'), redis_config['REDIS_HOST'])
-            
-            runtime_config = get_runtime_config()
-            self.assertEqual(settings.get('PROJECT_NAME'), runtime_config['PROJECT_NAME'])
-
-    def test_mode_manager_env_usage(self):
-        """测试 mode_manager.py 中的环境变量使用"""
-        # 验证 from_env 函数现在会抛出异常
-        with self.assertRaises(RuntimeError) as context:
-            from_env()
-        
-        self.assertIn("环境变量配置已移除", str(context.exception))
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+框架环境变量使用测试
+验证整个框架中环境变量的正确使用
+"""
+import sys
+import os
+import unittest
+from unittest.mock import patch, MagicMock
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.env_config import get_env_var, get_redis_config, get_runtime_config
+from crawlo.settings.setting_manager import SettingManager
+from crawlo.settings import default_settings
+from crawlo.mode_manager import from_env
+
+
+class TestFrameworkEnvUsage(unittest.TestCase):
+    """框架环境变量使用测试"""
+
+    def test_default_settings_env_usage(self):
+        """测试 default_settings.py 中的环境变量使用"""
+        # 验证 default_settings.py 不直接使用 os.getenv
+        import inspect
+        import crawlo.settings.default_settings as default_settings_module
+        
+        source_code = inspect.getsource(default_settings_module)
+        # 检查是否还有直接使用 os.getenv 的地方
+        self.assertNotIn('os.getenv', source_code, 
+                         "default_settings.py 不应该直接使用 os.getenv")
+        
+        # 但应该使用 env_config 工具
+        self.assertIn('get_redis_config', source_code,
+                      "default_settings.py 应该使用 get_redis_config")
+        self.assertIn('get_runtime_config', source_code,
+                      "default_settings.py 应该使用 get_runtime_config")
+
+    def test_env_config_tool(self):
+        """测试环境变量配置工具"""
+        # 测试获取Redis配置
+        with patch.dict(os.environ, {
+            'REDIS_HOST': 'test.redis.com',
+            'REDIS_PORT': '6380',
+            'REDIS_PASSWORD': 'test_pass',
+            'REDIS_DB': '2'
+        }):
+            redis_config = get_redis_config()
+            self.assertEqual(redis_config['REDIS_HOST'], 'test.redis.com')
+            self.assertEqual(redis_config['REDIS_PORT'], 6380)
+            self.assertEqual(redis_config['REDIS_PASSWORD'], 'test_pass')
+            self.assertEqual(redis_config['REDIS_DB'], 2)
+        
+        # 测试获取运行时配置
+        with patch.dict(os.environ, {
+            'PROJECT_NAME': 'test_project',
+            'CRAWLO_MODE': 'distributed',
+            'CONCURRENCY': '16'
+        }):
+            runtime_config = get_runtime_config()
+            self.assertEqual(runtime_config['PROJECT_NAME'], 'test_project')
+            self.assertEqual(runtime_config['CRAWLO_MODE'], 'distributed')
+            self.assertEqual(runtime_config['CONCURRENCY'], 16)
+
+    def test_settings_manager_with_env(self):
+        """测试设置管理器与环境变量的集成"""
+        # 设置环境变量
+        env_vars = {
+            'PROJECT_NAME': 'env_test_project',
+            'CONCURRENCY': '12',
+            'REDIS_HOST': 'env.redis.test',
+            'REDIS_PORT': '6381'
+        }
+        
+        with patch.dict(os.environ, env_vars):
+            # 重新导入 default_settings 模块以获取最新的环境变量
+            import importlib
+            import crawlo.settings.default_settings
+            importlib.reload(crawlo.settings.default_settings)
+            
+            # 创建设置管理器
+            settings = SettingManager()
+            settings.set_settings(crawlo.settings.default_settings)
+            
+            # 验证环境变量被正确使用
+            redis_config = get_redis_config()
+            self.assertEqual(settings.get('REDIS_HOST'), redis_config['REDIS_HOST'])
+            
+            runtime_config = get_runtime_config()
+            self.assertEqual(settings.get('PROJECT_NAME'), runtime_config['PROJECT_NAME'])
+
+    def test_mode_manager_env_usage(self):
+        """测试 mode_manager.py 中的环境变量使用"""
+        # 验证 from_env 函数现在会抛出异常
+        with self.assertRaises(RuntimeError) as context:
+            from_env()
+        
+        self.assertIn("环境变量配置已移除", str(context.exception))
+
+
+if __name__ == '__main__':
     unittest.main()

tests/test_framework_logger.py

@@ -1,67 +1,67 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-"""
-测试框架日志系统
-"""
-import sys
-import os
-sys.path.insert(0, '/')
-
-from crawlo.initialization import initialize_framework, get_framework_initializer
-from crawlo.utils.log import get_logger, LoggerManager
-
-def test_framework_logger():
-    print("=== 测试框架日志系统 ===")
-    
-    # 1. 初始化框架，传入基本配置
-    print("1. 初始化框架...")
-    custom_settings = {
-        'LOG_LEVEL': 'INFO',
-        'LOG_FILE': 'test_framework.log',
-        'PROJECT_NAME': 'test_framework',
-        'RUN_MODE': 'standalone'
-    }
-    settings = initialize_framework(custom_settings)
-    print(f"   LOG_LEVEL: {settings.get('LOG_LEVEL')}")
-    print(f"   LOG_FILE: {settings.get('LOG_FILE')}")
-    
-    # 2. 获取框架初始化管理器
-    init_manager = get_framework_initializer()
-    print(f"   框架是否就绪: {init_manager.is_ready}")
-    print(f"   初始化阶段: {init_manager.phase}")
-    
-    # 3. 测试框架logger
-    framework_logger = init_manager.logger
-    if framework_logger:
-        print(f"   框架logger名称: {framework_logger.name}")
-        print(f"   框架logger级别: {framework_logger.level}")
-        print(f"   框架logger处理器数量: {len(framework_logger.handlers)}")
-        
-        for i, handler in enumerate(framework_logger.handlers):
-            handler_type = type(handler).__name__
-            print(f"     处理器{i}: {handler_type}, 级别: {handler.level}")
-    else:
-        print("   框架logger为None!")
-        framework_logger = get_logger('crawlo.framework')
-        print(f"   手动创建的框架logger: {framework_logger.name}")
-    
-    # 4. 测试日志输出
-    print("2. 测试日志输出...")
-    framework_logger.info("Crawlo框架初始化完成")
-    framework_logger.info("Crawlo Framework Started 1.3.3")
-    framework_logger.info("使用单机模式 - 简单快速，适合开发和中小规模爬取")
-    framework_logger.info("Run Mode: standalone")
-    framework_logger.info("Starting running test_spider")
-    
-    # 5. 测试其他logger
-    print("3. 测试其他组件logger...")
-    queue_logger = get_logger('QueueManager')
-    queue_logger.info("Queue initialized successfully Type: memory")
-    
-    scheduler_logger = get_logger('Scheduler')
-    scheduler_logger.info("enabled filters: crawlo.filters.memory_filter.MemoryFilter")
-    
-    print("=== 测试完成 ===")
-
-if __name__ == "__main__":
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+测试框架日志系统
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+from crawlo.initialization import initialize_framework, get_framework_initializer
+from crawlo.utils.log import get_logger, LoggerManager
+
+def test_framework_logger():
+    print("=== 测试框架日志系统 ===")
+    
+    # 1. 初始化框架，传入基本配置
+    print("1. 初始化框架...")
+    custom_settings = {
+        'LOG_LEVEL': 'INFO',
+        'LOG_FILE': 'test_framework.log',
+        'PROJECT_NAME': 'test_framework',
+        'RUN_MODE': 'standalone'
+    }
+    settings = initialize_framework(custom_settings)
+    print(f"   LOG_LEVEL: {settings.get('LOG_LEVEL')}")
+    print(f"   LOG_FILE: {settings.get('LOG_FILE')}")
+    
+    # 2. 获取框架初始化管理器
+    init_manager = get_framework_initializer()
+    print(f"   框架是否就绪: {init_manager.is_ready}")
+    print(f"   初始化阶段: {init_manager.phase}")
+    
+    # 3. 测试框架logger
+    framework_logger = init_manager.logger
+    if framework_logger:
+        print(f"   框架logger名称: {framework_logger.name}")
+        print(f"   框架logger级别: {framework_logger.level}")
+        print(f"   框架logger处理器数量: {len(framework_logger.handlers)}")
+        
+        for i, handler in enumerate(framework_logger.handlers):
+            handler_type = type(handler).__name__
+            print(f"     处理器{i}: {handler_type}, 级别: {handler.level}")
+    else:
+        print("   框架logger为None!")
+        framework_logger = get_logger('crawlo.framework')
+        print(f"   手动创建的框架logger: {framework_logger.name}")
+    
+    # 4. 测试日志输出
+    print("2. 测试日志输出...")
+    framework_logger.info("Crawlo框架初始化完成")
+    framework_logger.info("Crawlo Framework Started 1.3.3")
+    framework_logger.info("使用单机模式 - 简单快速，适合开发和中小规模爬取")
+    framework_logger.info("Run Mode: standalone")
+    framework_logger.info("Starting running test_spider")
+    
+    # 5. 测试其他logger
+    print("3. 测试其他组件logger...")
+    queue_logger = get_logger('QueueManager')
+    queue_logger.info("Queue initialized successfully Type: memory")
+    
+    scheduler_logger = get_logger('Scheduler')
+    scheduler_logger.info("enabled filters: crawlo.filters.memory_filter.MemoryFilter")
+    
+    print("=== 测试完成 ===")
+
+if __name__ == "__main__":
     test_framework_logger()

tests/test_framework_startup.py

@@ -1,65 +1,65 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-测试框架启动信息是否正确输出到日志文件
-"""
-import sys
-import os
-sys.path.insert(0, '/')
-
-def test_framework_startup():
-    print("=== 测试框架启动信息输出 ===")
-    
-    # 删除旧的日志文件
-    test_log_file = '/Users/oscar/projects/Crawlo/test_startup.log'
-    if os.path.exists(test_log_file):
-        os.remove(test_log_file)
-    
-    # 准备测试设置
-    test_settings = {
-        'PROJECT_NAME': 'test_startup',
-        'LOG_LEVEL': 'INFO',
-        'LOG_FILE': test_log_file,
-        'RUN_MODE': 'standalone'
-    }
-    
-    # 初始化框架
-    from crawlo.initialization import initialize_framework
-    settings = initialize_framework(test_settings)
-    
-    print(f"设置初始化完成: {settings.get('PROJECT_NAME')}")
-    
-    # 检查日志文件是否包含框架启动信息
-    if os.path.exists(test_log_file):
-        with open(test_log_file, 'r', encoding='utf-8') as f:
-            content = f.read()
-            print(f"日志文件内容长度: {len(content)} 字符")
-            
-            # 检查关键的框架启动信息
-            if "Crawlo框架初始化完成" in content:
-                print("✅ 发现: Crawlo框架初始化完成")
-            else:
-                print("❌ 未找到: Crawlo框架初始化完成")
-                
-            if "Crawlo Framework Started" in content:
-                print("✅ 发现: Crawlo Framework Started")
-            else:
-                print("❌ 未找到: Crawlo Framework Started")
-                
-            if "使用单机模式" in content:
-                print("✅ 发现: 使用单机模式")
-            else:
-                print("❌ 未找到: 使用单机模式")
-                
-            print("\n前50行日志内容:")
-            lines = content.split('\n')[:50]
-            for i, line in enumerate(lines, 1):
-                if any(keyword in line for keyword in ["框架", "Framework", "Started"]):
-                    print(f"{i:3d}: {line}")
-    else:
-        print("❌ 日志文件未创建")
-    
-    print("=== 测试完成 ===")
-
-if __name__ == "__main__":
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+测试框架启动信息是否正确输出到日志文件
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+def test_framework_startup():
+    print("=== 测试框架启动信息输出 ===")
+    
+    # 删除旧的日志文件
+    test_log_file = '/Users/oscar/projects/Crawlo/test_startup.log'
+    if os.path.exists(test_log_file):
+        os.remove(test_log_file)
+    
+    # 准备测试设置
+    test_settings = {
+        'PROJECT_NAME': 'test_startup',
+        'LOG_LEVEL': 'INFO',
+        'LOG_FILE': test_log_file,
+        'RUN_MODE': 'standalone'
+    }
+    
+    # 初始化框架
+    from crawlo.initialization import initialize_framework
+    settings = initialize_framework(test_settings)
+    
+    print(f"设置初始化完成: {settings.get('PROJECT_NAME')}")
+    
+    # 检查日志文件是否包含框架启动信息
+    if os.path.exists(test_log_file):
+        with open(test_log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"日志文件内容长度: {len(content)} 字符")
+            
+            # 检查关键的框架启动信息
+            if "Crawlo框架初始化完成" in content:
+                print("✅ 发现: Crawlo框架初始化完成")
+            else:
+                print("❌ 未找到: Crawlo框架初始化完成")
+                
+            if "Crawlo Framework Started" in content:
+                print("✅ 发现: Crawlo Framework Started")
+            else:
+                print("❌ 未找到: Crawlo Framework Started")
+                
+            if "使用单机模式" in content:
+                print("✅ 发现: 使用单机模式")
+            else:
+                print("❌ 未找到: 使用单机模式")
+                
+            print("\n前50行日志内容:")
+            lines = content.split('\n')[:50]
+            for i, line in enumerate(lines, 1):
+                if any(keyword in line for keyword in ["框架", "Framework", "Started"]):
+                    print(f"{i:3d}: {line}")
+    else:
+        print("❌ 日志文件未创建")
+    
+    print("=== 测试完成 ===")
+
+if __name__ == "__main__":
     test_framework_startup()