crawlo 1.4.4__py3-none-any.whl → 1.4.6__py3-none-any.whl

tests/test_downloader_proxy_compatibility.py

@@ -35,7 +35,7 @@ class MockCrawler:
         self.spider = MockSpider(self)  # 添加spider属性
 
 
-def create_test_settings(proxy_url=None):
+def create_test_settings(proxy_url=None, proxy_list=None):
     """创建测试设置"""
     settings = SettingManager()
     settings.set("LOG_LEVEL", "DEBUG")
@@ -47,12 +47,13 @@ def create_test_settings(proxy_url=None):
     
     # 代理相关设置
     if proxy_url:
-        settings.set("PROXY_ENABLED", True)
+        # 高级代理配置（适用于ProxyMiddleware）
+        # 只要配置了代理API URL，中间件就会自动启用
         settings.set("PROXY_API_URL", proxy_url)
-        settings.set("PROXY_REFRESH_INTERVAL", 60)
-        settings.set("PROXY_POOL_SIZE", 5)
-    else:
-        settings.set("PROXY_ENABLED", False)
+    elif proxy_list:
+        # 代理配置（适用于ProxyMiddleware）
+        # 只要配置了代理列表，中间件就会自动启用
+        settings.set("PROXY_LIST", proxy_list)
         
     return settings
 
@@ -65,7 +66,7 @@ async def test_aiohttp_with_proxy(proxy_url, target_url):
     
     try:
         # 创建设置
-        settings = create_test_settings(proxy_url)
+        settings = create_test_settings(proxy_url=proxy_url)
         crawler = MockCrawler(settings)
         
         # 创建下载器
@@ -73,6 +74,7 @@ async def test_aiohttp_with_proxy(proxy_url, target_url):
         downloader.open()
         
         # 创建代理中间件
+        from crawlo.middleware.proxy import ProxyMiddleware
         proxy_middleware = ProxyMiddleware(settings, "DEBUG")
         
         # 创建请求
@@ -115,15 +117,15 @@ async def test_aiohttp_with_proxy(proxy_url, target_url):
             pass
 
 
-async def test_httpx_with_proxy_async(proxy_url, target_url):
+async def test_httpx_with_proxy_async(proxy_list, target_url):
     """测试httpx下载器与代理的适配性"""
     print(f"\n=== 测试 httpx 下载器与代理 ===")
-    print(f"代理URL: {proxy_url}")
+    print(f"代理列表: {proxy_list}")
     print(f"目标URL: {target_url}")
     
     try:
         # 创建设置
-        settings = create_test_settings(proxy_url)
+        settings = create_test_settings(proxy_list=proxy_list)
         crawler = MockCrawler(settings)
         
         # 创建下载器
@@ -131,7 +133,8 @@ async def test_httpx_with_proxy_async(proxy_url, target_url):
         downloader.open()
         
         # 创建代理中间件
-        proxy_middleware = ProxyMiddleware(settings, "DEBUG")
+        from crawlo.middleware.simple_proxy import SimpleProxyMiddleware
+        proxy_middleware = SimpleProxyMiddleware(settings, "DEBUG")
         
         # 创建请求
         request = Request(url=target_url)
@@ -168,7 +171,6 @@ async def test_httpx_with_proxy_async(proxy_url, target_url):
         # 清理资源
         try:
             await downloader.close()
-            await proxy_middleware.close()
         except:
             pass
 
@@ -181,7 +183,7 @@ async def test_curl_cffi_with_proxy_async(proxy_url, target_url):
     
     try:
         # 创建设置
-        settings = create_test_settings(proxy_url)
+        settings = create_test_settings(proxy_url=proxy_url)
         crawler = MockCrawler(settings)
         
         # 创建下载器
@@ -238,26 +240,28 @@ async def main():
     # 使用测试代理URL（这里使用一个公开的测试代理）
     # 注意：在实际使用中，您需要替换为有效的代理URL
     test_proxy_url = "http://test.proxy.api:8080/proxy/getitem/"
+    test_proxy_list = ["http://proxy1:8080", "http://proxy2:8080"]
     test_target_url = "https://httpbin.org/ip"  # 一个返回IP信息的测试站点
     
     print(f"测试代理API: {test_proxy_url}")
+    print(f"测试代理列表: {test_proxy_list}")
     print(f"测试目标URL: {test_target_url}")
     
-    # 测试aiohttp下载器
+    # 测试aiohttp下载器（使用高级代理）
     aiohttp_result = await test_aiohttp_with_proxy(test_proxy_url, test_target_url)
     
-    # 测试httpx下载器
-    httpx_result = await test_httpx_with_proxy_async(test_proxy_url, test_target_url)
+    # 测试httpx下载器（使用简化代理）
+    httpx_result = await test_httpx_with_proxy_async(test_proxy_list, test_target_url)
     
-    # 测试curl-cffi下载器
+    # 测试curl-cffi下载器（使用高级代理）
     curl_cffi_result = await test_curl_cffi_with_proxy_async(test_proxy_url, test_target_url)
     
     # 汇总结果
     print("\n" + "="*50)
     print("测试结果汇总:")
-    print(f"aiohttp 下载器: {'✓ 通过' if aiohttp_result else '✗ 失败'}")
-    print(f"httpx 下载器: {'✓ 通过' if httpx_result else '✗ 失败'}")
-    print(f"curl-cffi 下载器: {'✓ 通过' if curl_cffi_result else '✗ 失败'}")
+    print(f"aiohttp 下载器 (高级代理): {'✓ 通过' if aiohttp_result else '✗ 失败'}")
+    print(f"httpx 下载器 (简化代理): {'✓ 通过' if httpx_result else '✗ 失败'}")
+    print(f"curl-cffi 下载器 (高级代理): {'✓ 通过' if curl_cffi_result else '✗ 失败'}")
     
     overall_result = all([aiohttp_result, httpx_result, curl_cffi_result])
     print(f"\n总体结果: {'✓ 所有下载器都适配代理中间件' if overall_result else '✗ 部分下载器不兼容'}")

tests/test_edge_cases.py

@@ -112,13 +112,15 @@ async def test_redis_queue_edge_cases():
         print("   特殊字符 URL 测试通过")
         
         # 4. 测试优先级（高优先级值应该先出队）
-        high_priority_request = Request(url="https://high-priority.com", priority=1000)
-        low_priority_request = Request(url="https://low-priority.com", priority=-1000)
+        # 注意：Request构造函数会将传入的priority值取反存储
+        # 所以priority=1000的请求实际存储为-1000，priority=-1000的请求实际存储为1000
+        high_priority_request = Request(url="https://high-priority.com", priority=1000)  # 实际存储为-1000
+        low_priority_request = Request(url="https://low-priority.com", priority=-1000)   # 实际存储为1000
         
-        await queue.put(high_priority_request)  # 高优先级值
-        await queue.put(low_priority_request)   # 低优先级值
+        await queue.put(high_priority_request, priority=high_priority_request.priority)  # 使用实际存储的priority值
+        await queue.put(low_priority_request, priority=low_priority_request.priority)    # 使用实际存储的priority值
         
-        # 高优先级值应该先出队
+        # 高优先级值应该先出队（因为score = priority，score小的先出队）
         first = await queue.get(timeout=1.0)
         assert first is not None and first.url == "https://high-priority.com", "高优先级值应该先出队"
         print("   优先级测试通过")

tests/test_encoding_core.py

@@ -0,0 +1,57 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+编码检测核心功能测试
+"""
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.network.response import Response
+
+
+def test_encoding_detection():
+    """测试编码检测核心功能"""
+    print("测试编码检测核心功能...")
+    
+    # 测试 Request 编码优先级
+    class MockRequest:
+        encoding = 'gbk'
+    
+    response1 = Response(
+        url="https://example.com",
+        body=b'',
+        request=MockRequest()
+    )
+    print(f"Request 编码优先级: {response1.encoding}")
+    
+    # 测试 Content-Type 头部编码
+    response2 = Response(
+        url="https://example.com",
+        body=b'',
+        headers={"content-type": "text/html; charset=iso-8859-1"}
+    )
+    print(f"Content-Type 编码: {response2.encoding}")
+    
+    # 测试声明编码方法
+    declared_enc = response2._declared_encoding()
+    print(f"声明编码: {declared_enc}")
+    
+    # 测试默认编码
+    response3 = Response(
+        url="https://example.com",
+        body=b''
+    )
+    print(f"默认编码: {response3.encoding}")
+    
+    # 验证结果
+    assert response1.encoding == 'gbk', f"Expected 'gbk', got {response1.encoding}"
+    assert response2.encoding == 'iso-8859-1', f"Expected 'iso-8859-1', got {response2.encoding}"
+    assert declared_enc == 'iso-8859-1', f"Expected 'iso-8859-1', got {declared_enc}"
+    assert response3.encoding == 'utf-8', f"Expected 'utf-8', got {response3.encoding}"
+    
+    print("所有测试通过！")
+
+
+if __name__ == '__main__':
+    test_encoding_detection()

tests/test_encoding_detection.py

@@ -0,0 +1,127 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response 编码检测优化测试
+"""
+import unittest
+
+# 模拟 Response 类的部分功能用于测试
+class MockResponse:
+    def __init__(self, body, headers=None, request=None):
+        self.body = body
+        self.headers = headers or {}
+        self.request = request
+        self._DEFAULT_ENCODING = "ascii"
+    
+    def _determine_encoding(self):
+        """简化版编码检测"""
+        # 1. 优先使用声明的编码
+        declared_encoding = self._declared_encoding()
+        if declared_encoding:
+            return declared_encoding
+            
+        # 2. 默认使用 utf-8
+        return 'utf-8'
+        
+    def _declared_encoding(self):
+        """获取声明的编码"""
+        # 1. Request 中指定的编码
+        if self.request and getattr(self.request, 'encoding', None):
+            return self.request.encoding
+            
+        # 2. 从 Content-Type 头中检测
+        content_type = self.headers.get("content-type", "") or self.headers.get("Content-Type", "")
+        if content_type:
+            import re
+            charset_match = re.search(r"charset=([\w-]+)", content_type, re.I)
+            if charset_match:
+                return charset_match.group(1).lower()
+        
+        return None
+
+
+class TestDetermineEncoding(unittest.TestCase):
+    """编码检测测试类"""
+
+    def test_request_encoding_priority(self):
+        """测试 Request 编码优先级"""
+        class MockRequest:
+            encoding = 'gbk'
+        
+        response = MockResponse(b'', request=MockRequest())
+        encoding = response._determine_encoding()
+        self.assertEqual(encoding, 'gbk')
+
+    def test_content_type_encoding(self):
+        """测试 Content-Type 头部编码检测"""
+        response = MockResponse(
+            b'',
+            headers={"content-type": "text/html; charset=iso-8859-1"}
+        )
+        encoding = response._determine_encoding()
+        self.assertEqual(encoding, 'iso-8859-1')
+
+    def test_default_encoding(self):
+        """测试默认编码"""
+        response = MockResponse(b'')
+        encoding = response._determine_encoding()
+        self.assertEqual(encoding, 'utf-8')
+
+    def test_case_insensitive_content_type(self):
+        """测试 Content-Type 头部大小写不敏感"""
+        response = MockResponse(
+            b'',
+            headers={"Content-Type": "text/html; CHARSET=UTF-8"}
+        )
+        encoding = response._determine_encoding()
+        self.assertEqual(encoding, 'utf-8')
+        
+    def test_declared_encoding_with_request(self):
+        """测试声明编码 - Request优先级"""
+        class MockRequest:
+            encoding = 'gbk'
+        
+        response = MockResponse(b'', request=MockRequest())
+        declared_encoding = response._declared_encoding()
+        self.assertEqual(declared_encoding, 'gbk')
+        
+    def test_declared_encoding_with_content_type(self):
+        """测试声明编码 - Content-Type"""
+        response = MockResponse(
+            b'',
+            headers={"content-type": "text/html; charset=iso-8859-1"}
+        )
+        declared_encoding = response._declared_encoding()
+        self.assertEqual(declared_encoding, 'iso-8859-1')
+
+
+def test_encoding_detection():
+    """简单测试编码检测功能"""
+    print("测试编码检测功能...")
+    
+    # 测试 Request 编码优先级
+    class MockRequest:
+        encoding = 'gbk'
+    
+    response1 = MockResponse(b'', request=MockRequest())
+    encoding1 = response1._determine_encoding()
+    print(f"Request 编码优先级: {encoding1}")
+    
+    # 测试 Content-Type 头部编码
+    response2 = MockResponse(
+        b'',
+        headers={"content-type": "text/html; charset=iso-8859-1"}
+    )
+    encoding2 = response2._determine_encoding()
+    print(f"Content-Type 编码: {encoding2}")
+    
+    # 测试默认编码
+    response3 = MockResponse(b'')
+    encoding3 = response3._determine_encoding()
+    print(f"默认编码: {encoding3}")
+    
+    print("编码检测测试完成！")
+
+
+if __name__ == '__main__':
+    test_encoding_detection()

tests/test_factory_compatibility.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+测试CrawloConfig工厂模式兼容性
+"""
+
+import sys
+import os
+import traceback
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.config import CrawloConfig
+
+
+def test_standalone_factory():
+    """测试单机模式工厂函数"""
+    print("测试单机模式工厂函数...")
+    
+    try:
+        # 创建单机模式配置
+        config = CrawloConfig.standalone(
+            project_name='ofweek_standalone',
+            concurrency=8,
+            download_delay=1.0
+        )
+        
+        print(f"配置创建成功")
+        print(f"RUN_MODE: {config.get('RUN_MODE')}")
+        print(f"QUEUE_TYPE: {config.get('QUEUE_TYPE')}")
+        print(f"PROJECT_NAME: {config.get('PROJECT_NAME')}")
+        print(f"CONCURRENCY: {config.get('CONCURRENCY')}")
+        print(f"DOWNLOAD_DELAY: {config.get('DOWNLOAD_DELAY')}")
+        
+        # 验证配置是否正确
+        assert config.get('RUN_MODE') == 'standalone'
+        assert config.get('QUEUE_TYPE') == 'memory'
+        assert config.get('PROJECT_NAME') == 'ofweek_standalone'
+        assert config.get('CONCURRENCY') == 8
+        assert config.get('DOWNLOAD_DELAY') == 1.0
+        
+        print("✅ 单机模式工厂函数测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 单机模式工厂函数测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def test_distributed_factory():
+    """测试分布式模式工厂函数"""
+    print("\n测试分布式模式工厂函数...")
+    
+    try:
+        # 创建分布式模式配置
+        config = CrawloConfig.distributed(
+            redis_host='127.0.0.1',
+            redis_port=6379,
+            project_name='ofweek_distributed',
+            concurrency=16,
+            download_delay=0.5
+        )
+        
+        print(f"配置创建成功")
+        print(f"RUN_MODE: {config.get('RUN_MODE')}")
+        print(f"QUEUE_TYPE: {config.get('QUEUE_TYPE')}")
+        print(f"PROJECT_NAME: {config.get('PROJECT_NAME')}")
+        print(f"CONCURRENCY: {config.get('CONCURRENCY')}")
+        print(f"DOWNLOAD_DELAY: {config.get('DOWNLOAD_DELAY')}")
+        print(f"REDIS_HOST: {config.get('REDIS_HOST')}")
+        print(f"REDIS_PORT: {config.get('REDIS_PORT')}")
+        
+        # 验证配置是否正确
+        assert config.get('RUN_MODE') == 'distributed'
+        assert config.get('QUEUE_TYPE') == 'redis'
+        assert config.get('PROJECT_NAME') == 'ofweek_distributed'
+        assert config.get('CONCURRENCY') == 16
+        assert config.get('DOWNLOAD_DELAY') == 0.5
+        assert config.get('REDIS_HOST') == '127.0.0.1'
+        assert config.get('REDIS_PORT') == 6379
+        
+        print("✅ 分布式模式工厂函数测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 分布式模式工厂函数测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def test_auto_factory():
+    """测试自动模式工厂函数"""
+    print("\n测试自动模式工厂函数...")
+    
+    try:
+        # 创建自动模式配置
+        config = CrawloConfig.auto(
+            project_name='ofweek_auto',
+            concurrency=12,
+            download_delay=0.8
+        )
+        
+        print(f"配置创建成功")
+        print(f"RUN_MODE: {config.get('RUN_MODE')}")
+        print(f"QUEUE_TYPE: {config.get('QUEUE_TYPE')}")
+        print(f"PROJECT_NAME: {config.get('PROJECT_NAME')}")
+        print(f"CONCURRENCY: {config.get('CONCURRENCY')}")
+        print(f"DOWNLOAD_DELAY: {config.get('DOWNLOAD_DELAY')}")
+        
+        # 验证配置是否正确
+        assert config.get('RUN_MODE') == 'auto'
+        assert config.get('QUEUE_TYPE') == 'auto'
+        assert config.get('PROJECT_NAME') == 'ofweek_auto'
+        assert config.get('CONCURRENCY') == 12
+        assert config.get('DOWNLOAD_DELAY') == 0.8
+        
+        print("✅ 自动模式工厂函数测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 自动模式工厂函数测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def test_config_to_dict():
+    """测试配置转换为字典"""
+    print("\n测试配置转换为字典...")
+    
+    try:
+        # 创建配置
+        config = CrawloConfig.standalone(
+            project_name='test_project',
+            concurrency=4
+        )
+        
+        # 转换为字典
+        config_dict = config.to_dict()
+        
+        print(f"字典转换成功")
+        print(f"字典键数量: {len(config_dict)}")
+        
+        # 验证关键配置项
+        assert 'RUN_MODE' in config_dict
+        assert 'QUEUE_TYPE' in config_dict
+        assert 'PROJECT_NAME' in config_dict
+        assert 'CONCURRENCY' in config_dict
+        
+        print("✅ 配置转换为字典测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 配置转换为字典测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def main():
+    """主函数"""
+    print("开始测试CrawloConfig工厂模式兼容性...")
+    print("=" * 50)
+    
+    tests = [
+        test_standalone_factory,
+        test_distributed_factory,
+        test_auto_factory,
+        test_config_to_dict,
+    ]
+    
+    passed = 0
+    total = len(tests)
+    
+    for test_func in tests:
+        try:
+            if test_func():
+                passed += 1
+                print(f"✓ {test_func.__name__} 通过")
+            else:
+                print(f"✗ {test_func.__name__} 失败")
+        except Exception as e:
+            print(f"✗ {test_func.__name__} 异常: {e}")
+        print()
+    
+    print("=" * 50)
+    print(f"测试结果: {passed}/{total} 通过")
+    
+    if passed == total:
+        print("所有测试通过！CrawloConfig工厂模式兼容性正常。")
+        return 0
+    else:
+        print("部分测试失败，请检查实现。")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = main()
+    exit(exit_code)