crawlo 1.4.7__py3-none-any.whl → 1.4.8__py3-none-any.whl

tests/test_all_pipeline_fingerprints.py

@@ -1,134 +1,134 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-所有去重管道指纹一致性测试
-====================
-验证所有去重管道对相同数据生成一致的指纹
-"""
-
-import sys
-import os
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.utils.fingerprint import FingerprintGenerator
-
-
-class MockItem:
-    """模拟数据项类"""
-    
-    def __init__(self, **kwargs):
-        for key, value in kwargs.items():
-            setattr(self, key, value)
-    
-    def to_dict(self):
-        """转换为字典"""
-        return {k: v for k, v in self.__dict__.items() if not k.startswith('_')}
-
-
-def test_all_pipeline_fingerprints():
-    """测试所有管道指纹一致性"""
-    # 创建测试数据项
-    test_item = MockItem(
-        title="Test Title",
-        url="https://example.com",
-        content="Test content",
-        price=99.99
-    )
-    
-    # 使用指纹生成器生成指纹
-    expected_fingerprint = FingerprintGenerator.item_fingerprint(test_item)
-    
-    print(f"Expected fingerprint: {expected_fingerprint}")
-    
-    # 测试内存去重管道指纹生成方法
-    try:
-        from crawlo.pipelines.memory_dedup_pipeline import MemoryDedupPipeline
-        memory_pipeline = MemoryDedupPipeline()
-        memory_fingerprint = memory_pipeline._generate_item_fingerprint(test_item)
-        print(f"Memory pipeline fingerprint: {memory_fingerprint}")
-        assert memory_fingerprint == expected_fingerprint, "Memory pipeline fingerprint mismatch"
-        print("✓ Memory pipeline fingerprint一致")
-    except Exception as e:
-        print(f"✗ Memory pipeline test failed: {e}")
-    
-    # 测试Redis去重管道指纹生成方法
-    try:
-        from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
-        redis_pipeline = RedisDedupPipeline()
-        redis_fingerprint = redis_pipeline._generate_item_fingerprint(test_item)
-        print(f"Redis pipeline fingerprint: {redis_fingerprint}")
-        assert redis_fingerprint == expected_fingerprint, "Redis pipeline fingerprint mismatch"
-        print("✓ Redis pipeline fingerprint一致")
-    except Exception as e:
-        print(f"✗ Redis pipeline test failed: {e}")
-    
-    # 测试Bloom去重管道指纹生成方法
-    try:
-        from crawlo.pipelines.bloom_dedup_pipeline import BloomDedupPipeline
-        bloom_pipeline = BloomDedupPipeline()
-        bloom_fingerprint = bloom_pipeline._generate_item_fingerprint(test_item)
-        print(f"Bloom pipeline fingerprint: {bloom_fingerprint}")
-        assert bloom_fingerprint == expected_fingerprint, "Bloom pipeline fingerprint mismatch"
-        print("✓ Bloom pipeline fingerprint一致")
-    except Exception as e:
-        print(f"✗ Bloom pipeline test failed: {e}")
-    
-    # 测试数据库去重管道指纹生成方法
-    try:
-        from crawlo.pipelines.database_dedup_pipeline import DatabaseDedupPipeline
-        database_pipeline = DatabaseDedupPipeline()
-        database_fingerprint = database_pipeline._generate_item_fingerprint(test_item)
-        print(f"Database pipeline fingerprint: {database_fingerprint}")
-        assert database_fingerprint == expected_fingerprint, "Database pipeline fingerprint mismatch"
-        print("✓ Database pipeline fingerprint一致")
-    except Exception as e:
-        print(f"✗ Database pipeline test failed: {e}")
-    
-    # 测试分布式协调工具指纹生成方法
-    try:
-        from crawlo.tools.distributed_coordinator import DeduplicationTool
-        dedup_tool = DeduplicationTool()
-        tool_fingerprint = dedup_tool.generate_fingerprint(test_item.to_dict())
-        print(f"Deduplication tool fingerprint: {tool_fingerprint}")
-        # 注意：这里我们传入的是字典，因为工具类的generate_fingerprint方法直接处理数据
-        expected_tool_fingerprint = FingerprintGenerator.data_fingerprint(test_item.to_dict())
-        assert tool_fingerprint == expected_tool_fingerprint, "Deduplication tool fingerprint mismatch"
-        print("✓ Deduplication tool fingerprint一致")
-    except Exception as e:
-        print(f"✗ Deduplication tool test failed: {e}")
-
-
-def test_fingerprint_stability():
-    """测试指纹稳定性"""
-    # 创建相同的测试数据项多次
-    item1 = MockItem(
-        title="Test Title",
-        url="https://example.com",
-        content="Test content",
-        price=99.99
-    )
-    
-    item2 = MockItem(
-        title="Test Title",
-        url="https://example.com",
-        content="Test content",
-        price=99.99
-    )
-    
-    # 生成指纹
-    fingerprint1 = FingerprintGenerator.item_fingerprint(item1)
-    fingerprint2 = FingerprintGenerator.item_fingerprint(item2)
-    
-    # 验证相同数据生成相同指纹
-    print(f"\nFirst fingerprint: {fingerprint1}")
-    print(f"Second fingerprint: {fingerprint2}")
-    assert fingerprint1 == fingerprint2, "Same items should generate same fingerprints"
-    print("✓ 相同数据生成相同指纹")
-
-
-if __name__ == '__main__':
-    test_all_pipeline_fingerprints()
-    test_fingerprint_stability()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+所有去重管道指纹一致性测试
+====================
+验证所有去重管道对相同数据生成一致的指纹
+"""
+
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.fingerprint import FingerprintGenerator
+
+
+class MockItem:
+    """模拟数据项类"""
+    
+    def __init__(self, **kwargs):
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+    
+    def to_dict(self):
+        """转换为字典"""
+        return {k: v for k, v in self.__dict__.items() if not k.startswith('_')}
+
+
+def test_all_pipeline_fingerprints():
+    """测试所有管道指纹一致性"""
+    # 创建测试数据项
+    test_item = MockItem(
+        title="Test Title",
+        url="https://example.com",
+        content="Test content",
+        price=99.99
+    )
+    
+    # 使用指纹生成器生成指纹
+    expected_fingerprint = FingerprintGenerator.item_fingerprint(test_item)
+    
+    print(f"Expected fingerprint: {expected_fingerprint}")
+    
+    # 测试内存去重管道指纹生成方法
+    try:
+        from crawlo.pipelines.memory_dedup_pipeline import MemoryDedupPipeline
+        memory_pipeline = MemoryDedupPipeline()
+        memory_fingerprint = memory_pipeline._generate_item_fingerprint(test_item)
+        print(f"Memory pipeline fingerprint: {memory_fingerprint}")
+        assert memory_fingerprint == expected_fingerprint, "Memory pipeline fingerprint mismatch"
+        print("✓ Memory pipeline fingerprint一致")
+    except Exception as e:
+        print(f"✗ Memory pipeline test failed: {e}")
+    
+    # 测试Redis去重管道指纹生成方法
+    try:
+        from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
+        redis_pipeline = RedisDedupPipeline()
+        redis_fingerprint = redis_pipeline._generate_item_fingerprint(test_item)
+        print(f"Redis pipeline fingerprint: {redis_fingerprint}")
+        assert redis_fingerprint == expected_fingerprint, "Redis pipeline fingerprint mismatch"
+        print("✓ Redis pipeline fingerprint一致")
+    except Exception as e:
+        print(f"✗ Redis pipeline test failed: {e}")
+    
+    # 测试Bloom去重管道指纹生成方法
+    try:
+        from crawlo.pipelines.bloom_dedup_pipeline import BloomDedupPipeline
+        bloom_pipeline = BloomDedupPipeline()
+        bloom_fingerprint = bloom_pipeline._generate_item_fingerprint(test_item)
+        print(f"Bloom pipeline fingerprint: {bloom_fingerprint}")
+        assert bloom_fingerprint == expected_fingerprint, "Bloom pipeline fingerprint mismatch"
+        print("✓ Bloom pipeline fingerprint一致")
+    except Exception as e:
+        print(f"✗ Bloom pipeline test failed: {e}")
+    
+    # 测试数据库去重管道指纹生成方法
+    try:
+        from crawlo.pipelines.database_dedup_pipeline import DatabaseDedupPipeline
+        database_pipeline = DatabaseDedupPipeline()
+        database_fingerprint = database_pipeline._generate_item_fingerprint(test_item)
+        print(f"Database pipeline fingerprint: {database_fingerprint}")
+        assert database_fingerprint == expected_fingerprint, "Database pipeline fingerprint mismatch"
+        print("✓ Database pipeline fingerprint一致")
+    except Exception as e:
+        print(f"✗ Database pipeline test failed: {e}")
+    
+    # 测试分布式协调工具指纹生成方法
+    try:
+        from crawlo.tools.distributed_coordinator import DeduplicationTool
+        dedup_tool = DeduplicationTool()
+        tool_fingerprint = dedup_tool.generate_fingerprint(test_item.to_dict())
+        print(f"Deduplication tool fingerprint: {tool_fingerprint}")
+        # 注意：这里我们传入的是字典，因为工具类的generate_fingerprint方法直接处理数据
+        expected_tool_fingerprint = FingerprintGenerator.data_fingerprint(test_item.to_dict())
+        assert tool_fingerprint == expected_tool_fingerprint, "Deduplication tool fingerprint mismatch"
+        print("✓ Deduplication tool fingerprint一致")
+    except Exception as e:
+        print(f"✗ Deduplication tool test failed: {e}")
+
+
+def test_fingerprint_stability():
+    """测试指纹稳定性"""
+    # 创建相同的测试数据项多次
+    item1 = MockItem(
+        title="Test Title",
+        url="https://example.com",
+        content="Test content",
+        price=99.99
+    )
+    
+    item2 = MockItem(
+        title="Test Title",
+        url="https://example.com",
+        content="Test content",
+        price=99.99
+    )
+    
+    # 生成指纹
+    fingerprint1 = FingerprintGenerator.item_fingerprint(item1)
+    fingerprint2 = FingerprintGenerator.item_fingerprint(item2)
+    
+    # 验证相同数据生成相同指纹
+    print(f"\nFirst fingerprint: {fingerprint1}")
+    print(f"Second fingerprint: {fingerprint2}")
+    assert fingerprint1 == fingerprint2, "Same items should generate same fingerprints"
+    print("✓ 相同数据生成相同指纹")
+
+
+if __name__ == '__main__':
+    test_all_pipeline_fingerprints()
+    test_fingerprint_stability()
     print("\n🎉 所有测试通过!")

tests/test_all_redis_key_configs.py

@@ -1,146 +1,146 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-所有Redis Key配置测试脚本
-用于验证所有配置文件是否符合新的Redis key命名规范
-"""
-import sys
-import os
-import re
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-
-def test_all_redis_key_configs():
-    """测试所有Redis key配置"""
-    print("测试所有Redis key配置...")
-    
-    try:
-        # 检查示例项目配置文件
-        example_projects = [
-            "examples/books_distributed/books_distributed/settings.py",
-            "examples/api_data_collection/api_data_collection/settings.py",
-            "examples/telecom_licenses_distributed/telecom_licenses_distributed/settings.py"
-        ]
-        
-        for project_config in example_projects:
-            print(f"   检查 {project_config}...")
-            if not os.path.exists(project_config):
-                print(f"配置文件不存在: {project_config}")
-                return False
-                
-            with open(project_config, 'r', encoding='utf-8') as f:
-                content = f.read()
-            
-            # 检查是否移除了旧的REDIS_KEY配置
-            if re.search(r'REDIS_KEY\s*=', content) and 'crawlo:{PROJECT_NAME}:filter:fingerprint' not in content:
-                print(f"{project_config}中仍然存在旧的REDIS_KEY配置")
-                return False
-                
-            # 检查是否添加了新的注释
-            if 'crawlo:{PROJECT_NAME}:filter:fingerprint' not in content:
-                print(f"{project_config}中缺少新的Redis key命名规范注释")
-                return False
-                
-            print(f"      {project_config}符合新的Redis key命名规范")
-        
-        # 检查模板文件
-        template_file = "crawlo/templates/project/settings.py.tmpl"
-        print(f"   检查 {template_file}...")
-        if not os.path.exists(template_file):
-            print(f"模板文件不存在: {template_file}")
-            return False
-            
-        with open(template_file, 'r', encoding='utf-8') as f:
-            template_content = f.read()
-        
-        # 检查是否移除了旧的REDIS_KEY配置
-        if "REDIS_KEY = f'{{project_name}}:fingerprint'" in template_content:
-            print("模板文件中仍然存在旧的REDIS_KEY配置")
-            return False
-            
-        # 检查是否添加了新的注释
-        if '# crawlo:{project_name}:filter:fingerprint (请求去重)' not in template_content:
-            print("模板文件中缺少请求去重的Redis key命名规范注释")
-            return False
-            
-        if '# crawlo:{project_name}:item:fingerprint (数据项去重)' not in template_content:
-            print("模板文件中缺少数据项去重的Redis key命名规范注释")
-            return False
-            
-        print(f"      {template_file}符合新的Redis key命名规范")
-        
-        # 检查mode_manager.py
-        mode_manager_file = "crawlo/mode_manager.py"
-        print(f"   检查 {mode_manager_file}...")
-        if not os.path.exists(mode_manager_file):
-            print(f"文件不存在: {mode_manager_file}")
-            return False
-            
-        with open(mode_manager_file, 'r', encoding='utf-8') as f:
-            mode_manager_content = f.read()
-        
-        # 检查是否移除了旧的REDIS_KEY配置
-        if "'REDIS_KEY': f'{project_name}:fingerprint'" in mode_manager_content:
-            print("mode_manager.py中仍然存在旧的REDIS_KEY配置")
-            return False
-            
-        # 检查是否添加了新的注释
-        if 'crawlo:{project_name}:filter:fingerprint (请求去重)' not in mode_manager_content:
-            print("mode_manager.py中缺少新的Redis key命名规范注释")
-            return False
-            
-        print(f"      {mode_manager_file}符合新的Redis key命名规范")
-        
-        # 检查默认设置文件
-        default_settings_file = "crawlo/settings/default_settings.py"
-        print(f"   检查 {default_settings_file}...")
-        if not os.path.exists(default_settings_file):
-            print(f"文件不存在: {default_settings_file}")
-            return False
-            
-        with open(default_settings_file, 'r', encoding='utf-8') as f:
-            default_settings_content = f.read()
-        
-        # 检查是否移除了旧的REDIS_KEY配置
-        if re.search(r'REDIS_KEY\s*=\s*.*fingerprint', default_settings_content):
-            print("默认设置文件中仍然存在旧的REDIS_KEY配置")
-            return False
-            
-        print(f"      {default_settings_file}符合新的Redis key命名规范")
-        
-        print("所有Redis key配置测试通过！")
-        return True
-        
-    except Exception as e:
-        print(f"测试过程中发生错误: {e}")
-        return False
-
-
-def main():
-    """主测试函数"""
-    print("开始所有Redis key配置测试...")
-    print("=" * 50)
-    
-    try:
-        success = test_all_redis_key_configs()
-        
-        print("=" * 50)
-        if success:
-            print("所有测试通过！所有配置文件符合新的Redis key命名规范")
-        else:
-            print("测试失败，请检查配置文件")
-            return 1
-            
-    except Exception as e:
-        print("=" * 50)
-        print(f"测试过程中发生异常: {e}")
-        return 1
-    
-    return 0
-
-
-if __name__ == "__main__":
-    exit_code = main()
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+所有Redis Key配置测试脚本
+用于验证所有配置文件是否符合新的Redis key命名规范
+"""
+import sys
+import os
+import re
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+
+def test_all_redis_key_configs():
+    """测试所有Redis key配置"""
+    print("测试所有Redis key配置...")
+    
+    try:
+        # 检查示例项目配置文件
+        example_projects = [
+            "examples/books_distributed/books_distributed/settings.py",
+            "examples/api_data_collection/api_data_collection/settings.py",
+            "examples/telecom_licenses_distributed/telecom_licenses_distributed/settings.py"
+        ]
+        
+        for project_config in example_projects:
+            print(f"   检查 {project_config}...")
+            if not os.path.exists(project_config):
+                print(f"配置文件不存在: {project_config}")
+                return False
+                
+            with open(project_config, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            # 检查是否移除了旧的REDIS_KEY配置
+            if re.search(r'REDIS_KEY\s*=', content) and 'crawlo:{PROJECT_NAME}:filter:fingerprint' not in content:
+                print(f"{project_config}中仍然存在旧的REDIS_KEY配置")
+                return False
+                
+            # 检查是否添加了新的注释
+            if 'crawlo:{PROJECT_NAME}:filter:fingerprint' not in content:
+                print(f"{project_config}中缺少新的Redis key命名规范注释")
+                return False
+                
+            print(f"      {project_config}符合新的Redis key命名规范")
+        
+        # 检查模板文件
+        template_file = "crawlo/templates/project/settings.py.tmpl"
+        print(f"   检查 {template_file}...")
+        if not os.path.exists(template_file):
+            print(f"模板文件不存在: {template_file}")
+            return False
+            
+        with open(template_file, 'r', encoding='utf-8') as f:
+            template_content = f.read()
+        
+        # 检查是否移除了旧的REDIS_KEY配置
+        if "REDIS_KEY = f'{{project_name}}:fingerprint'" in template_content:
+            print("模板文件中仍然存在旧的REDIS_KEY配置")
+            return False
+            
+        # 检查是否添加了新的注释
+        if '# crawlo:{project_name}:filter:fingerprint (请求去重)' not in template_content:
+            print("模板文件中缺少请求去重的Redis key命名规范注释")
+            return False
+            
+        if '# crawlo:{project_name}:item:fingerprint (数据项去重)' not in template_content:
+            print("模板文件中缺少数据项去重的Redis key命名规范注释")
+            return False
+            
+        print(f"      {template_file}符合新的Redis key命名规范")
+        
+        # 检查mode_manager.py
+        mode_manager_file = "crawlo/mode_manager.py"
+        print(f"   检查 {mode_manager_file}...")
+        if not os.path.exists(mode_manager_file):
+            print(f"文件不存在: {mode_manager_file}")
+            return False
+            
+        with open(mode_manager_file, 'r', encoding='utf-8') as f:
+            mode_manager_content = f.read()
+        
+        # 检查是否移除了旧的REDIS_KEY配置
+        if "'REDIS_KEY': f'{project_name}:fingerprint'" in mode_manager_content:
+            print("mode_manager.py中仍然存在旧的REDIS_KEY配置")
+            return False
+            
+        # 检查是否添加了新的注释
+        if 'crawlo:{project_name}:filter:fingerprint (请求去重)' not in mode_manager_content:
+            print("mode_manager.py中缺少新的Redis key命名规范注释")
+            return False
+            
+        print(f"      {mode_manager_file}符合新的Redis key命名规范")
+        
+        # 检查默认设置文件
+        default_settings_file = "crawlo/settings/default_settings.py"
+        print(f"   检查 {default_settings_file}...")
+        if not os.path.exists(default_settings_file):
+            print(f"文件不存在: {default_settings_file}")
+            return False
+            
+        with open(default_settings_file, 'r', encoding='utf-8') as f:
+            default_settings_content = f.read()
+        
+        # 检查是否移除了旧的REDIS_KEY配置
+        if re.search(r'REDIS_KEY\s*=\s*.*fingerprint', default_settings_content):
+            print("默认设置文件中仍然存在旧的REDIS_KEY配置")
+            return False
+            
+        print(f"      {default_settings_file}符合新的Redis key命名规范")
+        
+        print("所有Redis key配置测试通过！")
+        return True
+        
+    except Exception as e:
+        print(f"测试过程中发生错误: {e}")
+        return False
+
+
+def main():
+    """主测试函数"""
+    print("开始所有Redis key配置测试...")
+    print("=" * 50)
+    
+    try:
+        success = test_all_redis_key_configs()
+        
+        print("=" * 50)
+        if success:
+            print("所有测试通过！所有配置文件符合新的Redis key命名规范")
+        else:
+            print("测试失败，请检查配置文件")
+            return 1
+            
+    except Exception as e:
+        print("=" * 50)
+        print(f"测试过程中发生异常: {e}")
+        return 1
+    
+    return 0
+
+
+if __name__ == "__main__":
+    exit_code = main()
     sys.exit(exit_code)