crawlo 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl

tests/test_advanced_tools.py

@@ -0,0 +1,149 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+高级工具测试
+"""
+import unittest
+from crawlo.tools import (
+    # 数据处理工具
+    clean_text,
+    format_currency,
+    validate_email,
+    validate_url,
+    check_data_integrity,
+    
+    # 重试机制
+    RetryMechanism,
+    should_retry,
+    exponential_backoff,
+    
+    # 反爬虫应对工具
+    AntiCrawler,
+    rotate_proxy,
+    handle_captcha,
+    detect_rate_limiting,
+    
+    # 分布式协调工具
+    generate_pagination_tasks,
+    distribute_tasks,
+    DistributedCoordinator,
+    TaskDistributor,
+    DeduplicationTool
+)
+
+
+class TestAdvancedTools(unittest.TestCase):
+    """高级工具测试类"""
+
+    def test_data_processing_tools(self):
+        """测试数据处理工具"""
+        # 测试数据清洗
+        dirty_text = "<p>这是一个&nbsp;<b>测试</b>&amp;文本</p>"
+        clean_result = clean_text(dirty_text)
+        self.assertEqual(clean_result, "这是一个 测试&文本")
+        
+        # 测试数据格式化
+        price = 1234.567
+        formatted_price = format_currency(price, "¥", 2)
+        self.assertEqual(formatted_price, "¥1,234.57")
+        
+        # 测试字段验证
+        self.assertTrue(validate_email("test@example.com"))
+        self.assertFalse(validate_email("invalid-email"))
+        
+        self.assertTrue(validate_url("https://example.com"))
+        self.assertFalse(validate_url("invalid-url"))
+        
+        # 测试数据完整性检查
+        data = {
+            "name": "张三",
+            "email": "test@example.com",
+            "phone": "13812345678"
+        }
+        required_fields = ["name", "email", "phone"]
+        integrity_result = check_data_integrity(data, required_fields)
+        self.assertTrue(integrity_result["is_valid"])
+
+    def test_retry_mechanism(self):
+        """测试重试机制"""
+        # 测试指数退避
+        delay = exponential_backoff(0)
+        self.assertGreater(delay, 0)
+        
+        # 测试是否应该重试
+        self.assertTrue(should_retry(status_code=500))
+        self.assertTrue(should_retry(exception=ConnectionError()))
+        self.assertFalse(should_retry(status_code=200))
+
+    def test_anti_crawler_tools(self):
+        """测试反爬虫应对工具"""
+        # 测试反爬虫工具
+        anti_crawler = AntiCrawler()
+        
+        # 测试随机User-Agent
+        user_agent = anti_crawler.get_random_user_agent()
+        self.assertIsInstance(user_agent, str)
+        self.assertGreater(len(user_agent), 0)
+        
+        # 测试代理轮换
+        proxy = anti_crawler.rotate_proxy()
+        self.assertIsInstance(proxy, dict)
+        
+        # 测试验证码检测
+        self.assertTrue(anti_crawler.handle_captcha("请输入验证码进行验证"))
+        self.assertFalse(anti_crawler.handle_captcha("正常页面内容"))
+        
+        # 测试频率限制检测
+        self.assertTrue(anti_crawler.detect_rate_limiting(429, {}))
+        self.assertFalse(anti_crawler.detect_rate_limiting(200, {}))
+
+    def test_distributed_coordinator_tools(self):
+        """测试分布式协调工具"""
+        # 测试任务分发器
+        distributor = TaskDistributor()
+        
+        # 测试分页任务生成
+        base_url = "https://example.com/products"
+        pagination_tasks = distributor.generate_pagination_tasks(base_url, 1, 5)
+        self.assertEqual(len(pagination_tasks), 5)
+        
+        # 测试任务分发
+        tasks = list(range(1, 21))  # 20个任务
+        distributed = distributor.distribute_tasks(tasks, 4)  # 分发给4个工作节点
+        self.assertEqual(len(distributed), 4)
+        self.assertEqual(sum(len(worker_tasks) for worker_tasks in distributed), 20)
+        
+        # 测试去重工具
+        dedup_tool = DeduplicationTool()
+        
+        # 测试数据指纹生成
+        fingerprint1 = dedup_tool.generate_fingerprint({"name": "test", "value": 123})
+        fingerprint2 = dedup_tool.generate_fingerprint({"name": "test", "value": 123})
+        self.assertEqual(fingerprint1, fingerprint2)
+        
+        # 测试去重功能
+        self.assertFalse(dedup_tool.is_duplicate({"name": "test", "value": 123}))
+        self.assertTrue(dedup_tool.add_to_dedup({"name": "test", "value": 123}))
+        self.assertTrue(dedup_tool.is_duplicate({"name": "test", "value": 123}))
+        self.assertFalse(dedup_tool.add_to_dedup({"name": "test", "value": 123}))
+        
+        # 测试分布式协调器
+        coordinator = DistributedCoordinator()
+        
+        # 测试任务ID生成
+        task_id = coordinator.generate_task_id("https://example.com", "test_spider")
+        self.assertIsInstance(task_id, str)
+        self.assertEqual(len(task_id), 32)  # MD5 hash长度
+        
+        # 测试分页任务生成
+        pagination_tasks = coordinator.generate_pagination_tasks("https://example.com/products", 1, 5)
+        self.assertEqual(len(pagination_tasks), 5)
+        
+        # 测试任务分发
+        tasks = list(range(1, 21))  # 20个任务
+        distributed = coordinator.distribute_tasks(tasks, 4)  # 分发给4个工作节点
+        self.assertEqual(len(distributed), 4)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_all_redis_key_configs.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+所有Redis Key配置测试脚本
+用于验证所有配置文件是否符合新的Redis key命名规范
+"""
+import sys
+import os
+import re
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+
+def test_all_redis_key_configs():
+    """测试所有Redis key配置"""
+    print("🔍 测试所有Redis key配置...")
+    
+    try:
+        # 检查示例项目配置文件
+        example_projects = [
+            "examples/books_distributed/books_distributed/settings.py",
+            "examples/api_data_collection/api_data_collection/settings.py",
+            "examples/telecom_licenses_distributed/telecom_licenses_distributed/settings.py"
+        ]
+        
+        for project_config in example_projects:
+            print(f"   检查 {project_config}...")
+            if not os.path.exists(project_config):
+                print(f"❌ 配置文件不存在: {project_config}")
+                return False
+                
+            with open(project_config, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            # 检查是否移除了旧的REDIS_KEY配置
+            if re.search(r'REDIS_KEY\s*=', content) and 'crawlo:{PROJECT_NAME}:filter:fingerprint' not in content:
+                print(f"❌ {project_config}中仍然存在旧的REDIS_KEY配置")
+                return False
+                
+            # 检查是否添加了新的注释
+            if 'crawlo:{PROJECT_NAME}:filter:fingerprint' not in content:
+                print(f"❌ {project_config}中缺少新的Redis key命名规范注释")
+                return False
+                
+            print(f"      ✅ {project_config}符合新的Redis key命名规范")
+        
+        # 检查模板文件
+        template_file = "crawlo/templates/project/settings.py.tmpl"
+        print(f"   检查 {template_file}...")
+        if not os.path.exists(template_file):
+            print(f"❌ 模板文件不存在: {template_file}")
+            return False
+            
+        with open(template_file, 'r', encoding='utf-8') as f:
+            template_content = f.read()
+        
+        # 检查是否移除了旧的REDIS_KEY配置
+        if "REDIS_KEY = f'{{project_name}}:fingerprint'" in template_content:
+            print("❌ 模板文件中仍然存在旧的REDIS_KEY配置")
+            return False
+            
+        # 检查是否添加了新的注释
+        if '# crawlo:{project_name}:filter:fingerprint (请求去重)' not in template_content:
+            print("❌ 模板文件中缺少请求去重的Redis key命名规范注释")
+            return False
+            
+        if '# crawlo:{project_name}:item:fingerprint (数据项去重)' not in template_content:
+            print("❌ 模板文件中缺少数据项去重的Redis key命名规范注释")
+            return False
+            
+        print(f"      ✅ {template_file}符合新的Redis key命名规范")
+        
+        # 检查mode_manager.py
+        mode_manager_file = "crawlo/mode_manager.py"
+        print(f"   检查 {mode_manager_file}...")
+        if not os.path.exists(mode_manager_file):
+            print(f"❌ 文件不存在: {mode_manager_file}")
+            return False
+            
+        with open(mode_manager_file, 'r', encoding='utf-8') as f:
+            mode_manager_content = f.read()
+        
+        # 检查是否移除了旧的REDIS_KEY配置
+        if "'REDIS_KEY': f'{project_name}:fingerprint'" in mode_manager_content:
+            print("❌ mode_manager.py中仍然存在旧的REDIS_KEY配置")
+            return False
+            
+        # 检查是否添加了新的注释
+        if 'crawlo:{project_name}:filter:fingerprint (请求去重)' not in mode_manager_content:
+            print("❌ mode_manager.py中缺少新的Redis key命名规范注释")
+            return False
+            
+        print(f"      ✅ {mode_manager_file}符合新的Redis key命名规范")
+        
+        # 检查默认设置文件
+        default_settings_file = "crawlo/settings/default_settings.py"
+        print(f"   检查 {default_settings_file}...")
+        if not os.path.exists(default_settings_file):
+            print(f"❌ 文件不存在: {default_settings_file}")
+            return False
+            
+        with open(default_settings_file, 'r', encoding='utf-8') as f:
+            default_settings_content = f.read()
+        
+        # 检查是否移除了旧的REDIS_KEY配置
+        if re.search(r'REDIS_KEY\s*=\s*.*fingerprint', default_settings_content):
+            print("❌ 默认设置文件中仍然存在旧的REDIS_KEY配置")
+            return False
+            
+        print(f"      ✅ {default_settings_file}符合新的Redis key命名规范")
+        
+        print("✅ 所有Redis key配置测试通过！")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 测试过程中发生错误: {e}")
+        return False
+
+
+def main():
+    """主测试函数"""
+    print("🚀 开始所有Redis key配置测试...")
+    print("=" * 50)
+    
+    try:
+        success = test_all_redis_key_configs()
+        
+        print("=" * 50)
+        if success:
+            print("🎉 所有测试通过！所有配置文件符合新的Redis key命名规范")
+        else:
+            print("❌ 测试失败，请检查配置文件")
+            return 1
+            
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 测试过程中发生异常: {e}")
+        return 1
+    
+    return 0
+
+
+if __name__ == "__main__":
+    exit_code = main()
+    sys.exit(exit_code)

tests/test_authenticated_proxy.py

@@ -0,0 +1,142 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+测试带认证代理的功能
+"""
+
+import asyncio
+import aiohttp
+import httpx
+from crawlo.network.request import Request
+from crawlo.tools import AuthenticatedProxy
+
+
+async def test_proxy_with_aiohttp():
+    """测试AioHttp与认证代理"""
+    print("=== 测试AioHttp与认证代理 ===")
+    
+    # 代理配置
+    proxy_config = {
+        "http": "http://dwe20241014:Dwe0101014@182.201.243.186:58111",
+        "https": "http://dwe20241014:Dwe0101014@182.201.243.186:58111"
+    }
+    
+    # 创建代理对象
+    proxy_url = proxy_config["http"]
+    proxy = AuthenticatedProxy(proxy_url)
+    
+    print(f"原始代理URL: {proxy_url}")
+    print(f"清洁URL: {proxy.clean_url}")
+    print(f"认证信息: {proxy.get_auth_credentials()}")
+    
+    # 使用aiohttp直接测试
+    try:
+        auth = proxy.get_auth_credentials()
+        if auth:
+            basic_auth = aiohttp.BasicAuth(auth['username'], auth['password'])
+        else:
+            basic_auth = None
+            
+        async with aiohttp.ClientSession() as session:
+            async with session.get(
+                "https://httpbin.org/ip",
+                proxy=proxy.clean_url,
+                proxy_auth=basic_auth
+            ) as response:
+                print(f"AioHttp测试成功!")
+                print(f"状态码: {response.status}")
+                content = await response.text()
+                print(f"响应内容: {content[:200]}...")
+                
+    except Exception as e:
+        print(f"AioHttp测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+def test_proxy_with_httpx():
+    """测试HttpX与认证代理"""
+    print("\n=== 测试HttpX与认证代理 ===")
+    
+    # 代理配置
+    proxy_config = {
+        "http": "http://dwe20241014:Dwe0101014@182.201.243.186:58111",
+        "https": "http://dwe20241014:Dwe0101014@182.201.243.186:58111"
+    }
+    
+    # 使用httpx直接测试
+    try:
+        # HttpX可以直接使用带认证的URL作为proxy参数
+        proxy_url = proxy_config["http"]
+        
+        with httpx.Client(proxy=proxy_url) as client:
+            response = client.get("https://httpbin.org/ip")
+            print(f"HttpX测试成功!")
+            print(f"状态码: {response.status_code}")
+            print(f"响应内容: {response.text[:200]}...")
+                
+    except Exception as e:
+        print(f"HttpX测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+async def test_proxy_with_curl_cffi():
+    """测试CurlCffi与认证代理"""
+    print("\n=== 测试CurlCffi与认证代理 ===")
+    
+    # 代理配置
+    proxy_config = {
+        "http": "http://dwe20241014:Dwe0101014@182.201.243.186:58111",
+        "https": "http://dwe20241014:Dwe0101014@182.201.243.186:58111"
+    }
+    
+    # 创建代理对象
+    proxy_url = proxy_config["http"]
+    proxy = AuthenticatedProxy(proxy_url)
+    
+    print(f"原始代理URL: {proxy_url}")
+    print(f"代理字典: {proxy.proxy_dict}")
+    print(f"认证头: {proxy.get_auth_header()}")
+    
+    # 使用curl-cffi直接测试
+    try:
+        from curl_cffi import requests as curl_requests
+        
+        # 设置代理和认证头
+        proxies = proxy.proxy_dict
+        headers = {}
+        auth_header = proxy.get_auth_header()
+        if auth_header:
+            headers["Proxy-Authorization"] = auth_header
+            
+        response = curl_requests.get(
+            "https://httpbin.org/ip",
+            proxies=proxies,
+            headers=headers
+        )
+        
+        print(f"CurlCffi测试成功!")
+        print(f"状态码: {response.status_code}")
+        print(f"响应内容: {response.text[:200]}...")
+                
+    except Exception as e:
+        print(f"CurlCffi测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+async def main():
+    """主测试函数"""
+    print("开始测试带认证代理的功能...\n")
+    
+    # 测试各个库
+    await test_proxy_with_aiohttp()
+    test_proxy_with_httpx()
+    await test_proxy_with_curl_cffi()
+    
+    print("\n所有测试完成!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/test_cleaners.py

@@ -0,0 +1,55 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+数据清洗工具测试
+"""
+import unittest
+from crawlo.cleaners import (
+    TextCleaner,
+    DataFormatter,
+    remove_html_tags,
+    decode_html_entities,
+    clean_text,
+    format_number,
+    format_currency,
+    format_phone_number
+)
+
+
+class TestCleaners(unittest.TestCase):
+    """数据清洗工具测试类"""
+
+    def test_text_cleaner(self):
+        """测试文本清洗功能"""
+        # 测试移除HTML标签
+        html_text = "<p>这是一个<b>测试</b>文本</p>"
+        clean_text_result = remove_html_tags(html_text)
+        self.assertEqual(clean_text_result, "这是一个测试文本")
+        
+        # 测试解码HTML实体
+        entity_text = "这是一个&nbsp;测试&amp;文本"
+        decoded_text = decode_html_entities(entity_text)
+        self.assertEqual(decoded_text, "这是一个 测试&文本")
+        
+        # 测试综合清洗
+        complex_text = "<p>这是一个&nbsp;<b>测试</b>&amp;文本</p>"
+        cleaned = clean_text(complex_text)
+        self.assertEqual(cleaned, "这是一个 测试&文本")
+
+    def test_data_formatter(self):
+        """测试数据格式化功能"""
+        # 测试数字格式化
+        formatted_num = format_number(1234.567, precision=2, thousand_separator=True)
+        self.assertEqual(formatted_num, "1,234.57")
+        
+        # 测试货币格式化
+        formatted_currency = format_currency(1234.567, "¥", 2)
+        self.assertEqual(formatted_currency, "¥1,234.57")
+        
+        # 测试电话号码格式化
+        formatted_phone = format_phone_number("13812345678", "+86", "international")
+        self.assertEqual(formatted_phone, "+86 138 1234 5678")
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_comprehensive.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+综合测试
+验证所有改进的集成效果
+"""
+import sys
+import os
+import asyncio
+import unittest
+from unittest.mock import patch, MagicMock
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.env_config import get_env_var, get_redis_config, get_runtime_config
+from crawlo.utils.error_handler import ErrorHandler, handle_exception
+from crawlo.core.engine import Engine
+from crawlo.settings.setting_manager import SettingManager
+from crawlo.settings import default_settings
+from crawlo.queue.queue_manager import QueueManager, QueueConfig, QueueType
+
+
+class TestComprehensiveIntegration(unittest.TestCase):
+    """综合集成测试"""
+
+    def setUp(self):
+        """测试前准备"""
+        # 设置测试环境变量
+        self.test_env = {
+            'PROJECT_NAME': 'test_project',
+            'CONCURRENCY': '4',
+            'REDIS_HOST': 'localhost',
+            'REDIS_PORT': '6379'
+        }
+        self.original_env = {}
+        for key, value in self.test_env.items():
+            self.original_env[key] = os.environ.get(key)
+            os.environ[key] = value
+
+    def tearDown(self):
+        """测试后清理"""
+        # 恢复原始环境变量
+        for key, value in self.original_env.items():
+            if value is None:
+                os.environ.pop(key, None)
+            else:
+                os.environ[key] = value
+
+    def test_env_config_integration(self):
+        """测试环境变量配置集成"""
+        # 验证环境变量工具正常工作
+        project_name = get_env_var('PROJECT_NAME', 'default', str)
+        self.assertEqual(project_name, 'test_project')
+        
+        concurrency = get_env_var('CONCURRENCY', 1, int)
+        self.assertEqual(concurrency, 4)
+        
+        # 验证Redis配置工具
+        redis_config = get_redis_config()
+        self.assertEqual(redis_config['REDIS_HOST'], 'localhost')
+        self.assertEqual(redis_config['REDIS_PORT'], 6379)
+
+    def test_error_handler_integration(self):
+        """测试错误处理集成"""
+        # 验证错误处理模块正常工作
+        error_handler = ErrorHandler("test")
+        
+        # 测试错误处理
+        try:
+            error_handler.handle_error(ValueError("Test error"), raise_error=False)
+        except Exception:
+            self.fail("Error handler should not raise exception when raise_error=False")
+        
+        # 测试安全调用
+        result = error_handler.safe_call(lambda x: x*2, 5, default_return=0)
+        self.assertEqual(result, 10)
+        
+        # 测试装饰器
+        @handle_exception(raise_error=False)
+        def failing_function():
+            raise RuntimeError("Test")
+        
+        try:
+            failing_function()
+        except Exception:
+            self.fail("Decorated function should not raise exception")
+
+    def test_settings_integration(self):
+        """测试设置管理器集成"""
+        # 重新加载默认设置以获取环境变量
+        import importlib
+        import crawlo.settings.default_settings
+        importlib.reload(crawlo.settings.default_settings)
+        
+        # 创建设置管理器
+        settings = SettingManager()
+        settings.set_settings(crawlo.settings.default_settings)
+        
+        # 验证设置正确加载
+        self.assertEqual(settings.get('PROJECT_NAME'), 'test_project')
+        self.assertEqual(settings.get_int('CONCURRENCY'), 4)
+        self.assertEqual(settings.get('REDIS_HOST'), 'localhost')
+
+    def test_queue_manager_config(self):
+        """测试队列管理器配置"""
+        # 重新加载默认设置
+        import importlib
+        import crawlo.settings.default_settings
+        importlib.reload(crawlo.settings.default_settings)
+        
+        # 创建设置管理器
+        settings = SettingManager()
+        settings.set_settings(crawlo.settings.default_settings)
+        
+        # 从设置创建队列配置
+        queue_config = QueueConfig.from_settings(settings)
+        
+        # 验证配置正确
+        self.assertEqual(queue_config.queue_type, QueueType.AUTO)
+        self.assertIn('test_project', queue_config.queue_name)
+
+    async def test_async_components(self):
+        """测试异步组件"""
+        # 测试异步错误处理装饰器
+        @handle_exception(raise_error=False)
+        async def async_failing_function():
+            raise RuntimeError("Async test")
+        
+        try:
+            await async_failing_function()
+        except Exception:
+            self.fail("Async decorated function should not raise exception")
+
+
+if __name__ == '__main__':
+    # 运行同步测试
+    unittest.main(exit=False)
+    
+    # 运行异步测试
+    async def run_async_tests():
+        test_instance = TestComprehensiveIntegration()
+        test_instance.setUp()
+        await test_instance.test_async_components()
+        test_instance.tearDown()
+    
+    asyncio.run(run_async_tests())