crawlo 1.2.0__py3-none-any.whl → 1.2.2__py3-none-any.whl

tests/test_proxy_api.py

@@ -0,0 +1,265 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+代理API测试脚本
+================
+测试指定的代理API接口是否能正常工作
+"""
+
+import asyncio
+import aiohttp
+import sys
+import os
+from urllib.parse import urlparse
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.middleware.proxy import ProxyMiddleware
+from crawlo.network.request import Request
+from crawlo.settings.setting_manager import SettingManager
+
+
+async def test_proxy_api(proxy_api_url):
+    """测试代理API接口"""
+    print(f"=== 测试代理API接口 ===")
+    print(f"API地址: {proxy_api_url}")
+    
+    try:
+        timeout = aiohttp.ClientTimeout(total=10)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.get(proxy_api_url) as response:
+                print(f"状态码: {response.status}")
+                print(f"响应头: {response.headers.get('content-type', 'Unknown')}")
+                
+                # 尝试解析JSON响应
+                try:
+                    data = await response.json()
+                    print(f"响应数据: {data}")
+                    return data
+                except Exception as e:
+                    # 如果不是JSON，尝试获取文本
+                    try:
+                        text = await response.text()
+                        print(f"响应文本: {text[:200]}{'...' if len(text) > 200 else ''}")
+                        return text
+                    except Exception as e2:
+                        print(f"无法解析响应内容: {e2}")
+                        return None
+                        
+    except asyncio.TimeoutError:
+        print("请求超时")
+        return None
+    except Exception as e:
+        print(f"请求失败: {e}")
+        return None
+
+
+def extract_proxy_url(proxy_data):
+    """从API响应中提取代理URL"""
+    proxy_url = None
+    
+    if isinstance(proxy_data, dict):
+        # 检查是否有status字段且为成功状态
+        if proxy_data.get('status') == 0:
+            # 获取proxy字段
+            proxy_info = proxy_data.get('proxy', {})
+            if isinstance(proxy_info, dict):
+                # 优先使用https代理，否则使用http代理
+                proxy_url = proxy_info.get('https') or proxy_info.get('http')
+            elif isinstance(proxy_info, str):
+                proxy_url = proxy_info
+        else:
+            # 直接尝试常见的字段名
+            for key in ['proxy', 'data', 'url', 'http', 'https']:
+                if key in proxy_data:
+                    value = proxy_data[key]
+                    if isinstance(value, str):
+                        proxy_url = value
+                        break
+                    elif isinstance(value, dict):
+                        proxy_url = value.get('https') or value.get('http')
+                        break
+        
+        # 如果还是没有找到，尝试更深层的嵌套
+        if not proxy_url:
+            for key, value in proxy_data.items():
+                if isinstance(value, str) and (value.startswith('http://') or value.startswith('https://')):
+                    proxy_url = value
+                    break
+                elif isinstance(value, dict):
+                    # 递归查找
+                    for sub_key, sub_value in value.items():
+                        if isinstance(sub_value, str) and (sub_value.startswith('http://') or sub_value.startswith('https://')):
+                            proxy_url = sub_value
+                            break
+                    if proxy_url:
+                        break
+    
+    elif isinstance(proxy_data, str):
+        # 如果响应是字符串，直接使用
+        if proxy_data.startswith('http://') or proxy_data.startswith('https://'):
+            proxy_url = proxy_data
+    
+    return proxy_url
+
+
+async def test_target_url_without_proxy(target_url):
+    """不使用代理直接测试访问目标URL"""
+    print(f"\n=== 直接访问目标URL（不使用代理） ===")
+    print(f"目标URL: {target_url}")
+    
+    try:
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            # 添加用户代理头，避免被反爬虫机制拦截
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
+            }
+            async with session.get(target_url, headers=headers) as response:
+                print(f"状态码: {response.status}")
+                print(f"响应头: {response.headers.get('content-type', 'Unknown')}")
+                
+                # 只读取响应状态，不尝试解码内容
+                return response.status == 200
+                
+    except asyncio.TimeoutError:
+        print("请求超时")
+        return False
+    except Exception as e:
+        print(f"请求失败: {e}")
+        return False
+
+
+async def test_target_url_with_proxy(proxy_url, target_url, max_retries=3):
+    """使用代理测试访问目标URL"""
+    print(f"\n=== 使用代理测试访问目标URL ===")
+    print(f"代理地址: {proxy_url}")
+    print(f"目标URL: {target_url}")
+    
+    # 添加用户代理头，避免被反爬虫机制拦截
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
+    }
+    
+    for attempt in range(max_retries):
+        if attempt > 0:
+            print(f"\n第 {attempt + 1} 次重试...")
+        
+        try:
+            # 创建aiohttp客户端会话
+            timeout = aiohttp.ClientTimeout(total=15)
+            async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
+                # 处理代理URL，支持带认证的代理
+                if isinstance(proxy_url, str) and "@" in proxy_url and "://" in proxy_url:
+                    parsed = urlparse(proxy_url)
+                    if parsed.username and parsed.password:
+                        # 提取认证信息
+                        auth = aiohttp.BasicAuth(parsed.username, parsed.password)
+                        # 清理代理URL，移除认证信息
+                        clean_proxy = f"{parsed.scheme}://{parsed.hostname}"
+                        if parsed.port:
+                            clean_proxy += f":{parsed.port}"
+                        
+                        print(f"使用带认证的代理: {clean_proxy}")
+                        async with session.get(target_url, proxy=clean_proxy, proxy_auth=auth) as response:
+                            print(f"状态码: {response.status}")
+                            print(f"响应头: {response.headers.get('content-type', 'Unknown')}")
+                            return response.status == 200
+                    else:
+                        # 没有认证信息的代理
+                        print(f"使用普通代理: {proxy_url}")
+                        async with session.get(target_url, proxy=proxy_url) as response:
+                            print(f"状态码: {response.status}")
+                            print(f"响应头: {response.headers.get('content-type', 'Unknown')}")
+                            return response.status == 200
+                else:
+                    # 直接使用代理URL
+                    print(f"使用代理: {proxy_url}")
+                    async with session.get(target_url, proxy=proxy_url) as response:
+                        print(f"状态码: {response.status}")
+                        print(f"响应头: {response.headers.get('content-type', 'Unknown')}")
+                        return response.status == 200
+                        
+        except asyncio.TimeoutError:
+            print("请求超时")
+            if attempt < max_retries - 1:
+                await asyncio.sleep(2)  # 等待2秒后重试
+            continue
+        except aiohttp.ClientConnectorError as e:
+            print(f"连接错误: {e}")
+            if attempt < max_retries - 1:
+                await asyncio.sleep(2)  # 等待2秒后重试
+            continue
+        except aiohttp.ClientHttpProxyError as e:
+            print(f"代理HTTP错误: {e}")
+            if attempt < max_retries - 1:
+                await asyncio.sleep(2)  # 等待2秒后重试
+            continue
+        except aiohttp.ServerDisconnectedError as e:
+            print(f"服务器断开连接: {e}")
+            if attempt < max_retries - 1:
+                await asyncio.sleep(2)  # 等待2秒后重试
+            continue
+        except Exception as e:
+            print(f"请求失败: {e}")
+            if attempt < max_retries - 1:
+                await asyncio.sleep(2)  # 等待2秒后重试
+            continue
+    
+    return False
+
+
+async def main():
+    """主测试函数"""
+    # 指定的代理API和测试链接
+    proxy_api = 'http://test.proxy.api:8080/proxy/getitem/'
+    target_url = 'https://stock.10jqka.com.cn/20240315/c655957791.shtml'
+    
+    print("开始测试代理接口和目标链接访问...\n")
+    
+    # 1. 测试代理API接口
+    proxy_data = await test_proxy_api(proxy_api)
+    
+    if not proxy_data:
+        print("代理API测试失败，无法获取代理信息")
+        return
+    
+    # 2. 从API响应中提取代理URL
+    proxy_url = extract_proxy_url(proxy_data)
+    
+    if not proxy_url:
+        print("无法从API响应中提取代理URL")
+        print(f"API响应内容: {proxy_data}")
+        return
+    
+    print(f"\n提取到的代理URL: {proxy_url}")
+    
+    # 3. 首先尝试直接访问，确认目标URL是否可访问
+    print("\n=== 测试直接访问目标URL ===")
+    direct_success = await test_target_url_without_proxy(target_url)
+    
+    if direct_success:
+        print("✅ 直接访问目标URL成功")
+    else:
+        print("❌ 直接访问目标URL失败")
+    
+    # 4. 使用代理访问目标URL
+    print("\n=== 测试使用代理访问目标URL ===")
+    proxy_success = await test_target_url_with_proxy(proxy_url, target_url)
+    
+    if proxy_success:
+        print(f"✅ 代理测试成功！代理 {proxy_url} 可以正常访问目标链接")
+    else:
+        print(f"❌ 代理测试失败！代理 {proxy_url} 无法访问目标链接")
+        
+    # 5. 总结
+    print(f"\n=== 测试总结 ===")
+    print(f"代理API访问: {'成功' if proxy_data else '失败'}")
+    print(f"代理提取: {'成功' if proxy_url else '失败'}")
+    print(f"直接访问: {'成功' if direct_success else '失败'}")
+    print(f"代理访问: {'成功' if proxy_success else '失败'}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/test_proxy_health_check.py

@@ -1,33 +1,33 @@
-# tests/test_proxy_health_check.py
-import pytest
-from unittest.mock import AsyncMock, patch
-from crawlo.proxy.health_check import check_single_proxy
-import httpx
-
-
-@pytest.mark.asyncio
-@patch('httpx.AsyncClient')
-async def test_health_check_success(mock_client_class):
-    """测试健康检查：成功"""
-    mock_resp = AsyncMock()
-    mock_resp.status_code = 200
-    mock_client_class.return_value.__aenter__.return_value.get.return_value = mock_resp
-
-    proxy_info = {'url': 'http://good:8080', 'healthy': False}
-    await check_single_proxy(proxy_info)
-
-    assert proxy_info['healthy'] is True
-    assert proxy_info['failures'] == 0
-
-
-@pytest.mark.asyncio
-@patch('httpx.AsyncClient')
-async def test_health_check_failure(mock_client_class):
-    """测试健康检查：失败"""
-    mock_client_class.return_value.__aenter__.return_value.get.side_effect = httpx.ConnectError("Failed")
-
-    proxy_info = {'url': 'http://bad:8080', 'healthy': True, 'failures': 0}
-    await check_single_proxy(proxy_info)
-
-    assert proxy_info['healthy'] is False
+# tests/test_proxy_health_check.py
+import pytest
+from unittest.mock import AsyncMock, patch
+from crawlo.proxy.health_check import check_single_proxy
+import httpx
+
+
+@pytest.mark.asyncio
+@patch('httpx.AsyncClient')
+async def test_health_check_success(mock_client_class):
+    """测试健康检查：成功"""
+    mock_resp = AsyncMock()
+    mock_resp.status_code = 200
+    mock_client_class.return_value.__aenter__.return_value.get.return_value = mock_resp
+
+    proxy_info = {'url': 'http://good:8080', 'healthy': False}
+    await check_single_proxy(proxy_info)
+
+    assert proxy_info['healthy'] is True
+    assert proxy_info['failures'] == 0
+
+
+@pytest.mark.asyncio
+@patch('httpx.AsyncClient')
+async def test_health_check_failure(mock_client_class):
+    """测试健康检查：失败"""
+    mock_client_class.return_value.__aenter__.return_value.get.side_effect = httpx.ConnectError("Failed")
+
+    proxy_info = {'url': 'http://bad:8080', 'healthy': True, 'failures': 0}
+    await check_single_proxy(proxy_info)
+
+    assert proxy_info['healthy'] is False
     assert proxy_info['failures'] == 1

tests/test_proxy_middleware.py

@@ -0,0 +1,122 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+ProxyMiddleware 测试文件
+用于测试代理中间件的功能
+"""
+
+import asyncio
+import unittest
+from unittest.mock import Mock, patch
+
+from crawlo.middleware.proxy import ProxyMiddleware
+from crawlo.exceptions import NotConfiguredError
+from crawlo.settings.setting_manager import SettingManager
+
+
+class MockLogger:
+    """Mock Logger 类，用于测试日志输出"""
+    def __init__(self, name, level=None):
+        self.name = name
+        self.level = level
+        self.logs = []
+
+    def debug(self, msg):
+        self.logs.append(('debug', msg))
+
+    def info(self, msg):
+        self.logs.append(('info', msg))
+
+    def warning(self, msg):
+        self.logs.append(('warning', msg))
+
+    def error(self, msg):
+        self.logs.append(('error', msg))
+
+
+class TestProxyMiddleware(unittest.TestCase):
+    """ProxyMiddleware 测试类"""
+
+    def setUp(self):
+        """测试前准备"""
+        # 创建设置管理器
+        self.settings = SettingManager()
+        
+        # 创建爬虫模拟对象
+        self.crawler = Mock()
+        self.crawler.settings = self.settings
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_middleware_initialization_without_api_url(self, mock_get_logger):
+        """测试没有配置API URL时中间件初始化"""
+        self.settings.set('PROXY_ENABLED', True)
+        self.settings.set('PROXY_API_URL', None)
+        self.settings.set('LOG_LEVEL', 'INFO')
+        
+        mock_get_logger.return_value = MockLogger('ProxyMiddleware')
+        
+        # 应该抛出NotConfiguredError异常
+        with self.assertRaises(NotConfiguredError):
+            ProxyMiddleware.create_instance(self.crawler)
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_middleware_initialization_with_disabled_proxy(self, mock_get_logger):
+        """测试禁用代理时中间件初始化"""
+        self.settings.set('PROXY_ENABLED', False)
+        self.settings.set('LOG_LEVEL', 'INFO')
+        
+        mock_get_logger.return_value = MockLogger('ProxyMiddleware')
+        
+        # 应该正常创建实例
+        middleware = ProxyMiddleware.create_instance(self.crawler)
+        self.assertIsInstance(middleware, ProxyMiddleware)
+        self.assertFalse(middleware.enabled)
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_middleware_initialization_with_api_url(self, mock_get_logger):
+        """测试配置API URL时中间件初始化"""
+        self.settings.set('PROXY_ENABLED', True)
+        self.settings.set('PROXY_API_URL', 'http://proxy-api.example.com')
+        self.settings.set('LOG_LEVEL', 'INFO')
+        
+        mock_get_logger.return_value = MockLogger('ProxyMiddleware')
+        
+        # 应该正常创建实例
+        middleware = ProxyMiddleware.create_instance(self.crawler)
+        self.assertIsInstance(middleware, ProxyMiddleware)
+        self.assertTrue(middleware.enabled)
+        self.assertEqual(middleware.api_url, 'http://proxy-api.example.com')
+
+    def test_is_https_with_https_url(self):
+        """测试HTTPS URL判断"""
+        # 创建中间件实例
+        middleware = ProxyMiddleware(
+            settings=self.settings,
+            log_level='INFO'
+        )
+        
+        # 创建请求对象
+        request = Mock()
+        request.url = 'https://example.com/page'
+        
+        # 应该返回True
+        self.assertTrue(middleware._is_https(request))
+
+    def test_is_https_with_http_url(self):
+        """测试HTTP URL判断"""
+        # 创建中间件实例
+        middleware = ProxyMiddleware(
+            settings=self.settings,
+            log_level='INFO'
+        )
+        
+        # 创建请求对象
+        request = Mock()
+        request.url = 'http://example.com/page'
+        
+        # 应该返回False
+        self.assertFalse(middleware._is_https(request))
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_proxy_middleware_enhanced.py

@@ -0,0 +1,217 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+增强版代理中间件测试
+==================
+测试ProxyMiddleware的代理池和健康检查功能
+"""
+
+import asyncio
+import json
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from unittest.mock import AsyncMock, Mock, patch
+
+from crawlo.middleware.proxy import ProxyMiddleware, Proxy
+from crawlo.network.request import Request
+from crawlo.network.response import Response
+from crawlo.settings.setting_manager import SettingManager
+
+
+def test_proxy_class():
+    """测试Proxy类的基本功能"""
+    print("=== 测试Proxy类 ===")
+    
+    # 创建代理对象
+    proxy = Proxy("http://127.0.0.1:8080")
+    print(f"初始代理: {proxy.proxy_str}")
+    print(f"初始成功率: {proxy.success_rate}")
+    print(f"是否健康: {proxy.is_healthy}")
+    
+    # 测试成功标记
+    proxy.mark_success()
+    print(f"标记成功后 - 成功率: {proxy.success_rate}, 成功次数: {proxy.success_count}")
+    
+    # 测试失败标记
+    proxy.mark_failure()
+    print(f"标记失败后 - 成功率: {proxy.success_rate}, 失败次数: {proxy.failure_count}")
+    print(f"是否健康: {proxy.is_healthy}")
+    
+    # 测试多次失败后健康状态
+    for _ in range(5):
+        proxy.mark_failure()
+    print(f"多次失败后 - 成功率: {proxy.success_rate}, 是否健康: {proxy.is_healthy}")
+    
+    print("Proxy类测试完成\n")
+
+
+def create_mock_settings():
+    """创建模拟设置"""
+    settings = SettingManager()
+    settings.set("PROXY_ENABLED", True)
+    settings.set("PROXY_API_URL", "http://test.proxy.api/get")
+    settings.set("PROXY_EXTRACTOR", "proxy")
+    settings.set("PROXY_REFRESH_INTERVAL", 10)
+    settings.set("PROXY_POOL_SIZE", 3)
+    settings.set("PROXY_HEALTH_CHECK_THRESHOLD", 0.5)
+    settings.set("LOG_LEVEL", "DEBUG")
+    return settings
+
+
+async def test_proxy_middleware_initialization():
+    """测试代理中间件初始化"""
+    print("=== 测试代理中间件初始化 ===")
+    
+    settings = create_mock_settings()
+    middleware = ProxyMiddleware(settings, "DEBUG")
+    
+    print(f"代理中间件已启用: {middleware.enabled}")
+    print(f"API URL: {middleware.api_url}")
+    print(f"代理池大小: {middleware.proxy_pool_size}")
+    print(f"健康检查阈值: {middleware.health_check_threshold}")
+    print(f"刷新间隔: {middleware.refresh_interval}")
+    
+    print("代理中间件初始化测试完成\n")
+
+
+async def test_proxy_pool_management():
+    """测试代理池管理功能"""
+    print("=== 测试代理池管理 ===")
+    
+    settings = create_mock_settings()
+    middleware = ProxyMiddleware(settings, "DEBUG")
+    
+    # 模拟API响应
+    mock_proxies = [
+        "http://proxy1.example.com:8080",
+        "http://proxy2.example.com:8080", 
+        "http://proxy3.example.com:8080"
+    ]
+    
+    # 测试更新代理池
+    with patch.object(middleware, '_get_proxy_from_api', AsyncMock(return_value=mock_proxies[0])):
+        await middleware._update_proxy_pool()
+        print(f"代理池大小: {len(middleware._proxy_pool)}")
+        if middleware._proxy_pool:
+            print(f"第一个代理: {middleware._proxy_pool[0].proxy_str}")
+    
+    # 测试获取健康代理
+    healthy_proxy = await middleware._get_healthy_proxy()
+    if healthy_proxy:
+        print(f"获取到健康代理: {healthy_proxy.proxy_str}")
+    else:
+        print("未获取到健康代理")
+        
+    print("代理池管理测试完成\n")
+
+
+async def test_process_request():
+    """测试请求处理"""
+    print("=== 测试请求处理 ===")
+    
+    settings = create_mock_settings()
+    middleware = ProxyMiddleware(settings, "DEBUG")
+    
+    # 创建模拟请求
+    request = Request(url="http://example.com")
+    
+    # 创建模拟爬虫对象
+    mock_spider = Mock()
+    mock_spider.crawler.settings.get.return_value = "aiohttp"
+    
+    # 添加一些测试代理到池中
+    middleware._proxy_pool = [
+        Proxy("http://proxy1.example.com:8080"),
+        Proxy("http://proxy2.example.com:8080")
+    ]
+    
+    # 处理请求
+    result = await middleware.process_request(request, mock_spider)
+    print(f"处理结果: {result}")
+    print(f"请求代理: {request.proxy}")
+    if "_used_proxy" in request.meta:
+        print(f"使用的代理对象: {request.meta['_used_proxy'].proxy_str}")
+    
+    print("请求处理测试完成\n")
+
+
+def test_process_response():
+    """测试响应处理"""
+    print("=== 测试响应处理 ===")
+    
+    settings = create_mock_settings()
+    middleware = ProxyMiddleware(settings, "DEBUG")
+    
+    # 创建带代理信息的请求
+    request = Request(url="http://example.com")
+    proxy_obj = Proxy("http://proxy1.example.com:8080")
+    request.meta["_used_proxy"] = proxy_obj
+    
+    # 创建响应
+    response = Response(
+        url="http://example.com",
+        status_code=200,
+        body=b"test response",
+        request=request
+    )
+    
+    # 处理响应前
+    print(f"处理前 - 代理成功次数: {proxy_obj.success_count}, 失败次数: {proxy_obj.failure_count}")
+    
+    # 处理响应
+    result = middleware.process_response(request, response, None)
+    
+    # 处理后
+    print(f"处理后 - 代理成功次数: {proxy_obj.success_count}, 失败次数: {proxy_obj.failure_count}")
+    print(f"成功率: {proxy_obj.success_rate}")
+    
+    print("响应处理测试完成\n")
+
+
+def test_process_exception():
+    """测试异常处理"""
+    print("=== 测试异常处理 ===")
+    
+    settings = create_mock_settings()
+    middleware = ProxyMiddleware(settings, "DEBUG")
+    
+    # 创建带代理信息的请求
+    request = Request(url="http://example.com")
+    proxy_obj = Proxy("http://proxy1.example.com:8080")
+    request.meta["_used_proxy"] = proxy_obj
+    
+    # 处理异常前
+    print(f"处理前 - 代理成功次数: {proxy_obj.success_count}, 失败次数: {proxy_obj.failure_count}")
+    
+    # 处理异常
+    result = middleware.process_exception(request, Exception("Test error"), None)
+    
+    # 处理后
+    print(f"处理后 - 代理成功次数: {proxy_obj.success_count}, 失败次数: {proxy_obj.failure_count}")
+    print(f"成功率: {proxy_obj.success_rate}")
+    print(f"是否健康: {proxy_obj.is_healthy}")
+    
+    print("异常处理测试完成\n")
+
+
+async def main():
+    """主测试函数"""
+    print("开始测试增强版代理中间件...\n")
+    
+    # 运行各个测试
+    test_proxy_class()
+    await test_proxy_middleware_initialization()
+    await test_proxy_pool_management()
+    await test_process_request()
+    test_process_response()
+    test_process_exception()
+    
+    print("所有测试完成!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())