crawlo 1.1.0__py3-none-any.whl → 1.1.2__py3-none-any.whl

tests/test_final_validation.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+最终验证测试：确认分布式队列的 logger 序列化问题已完全解决
+"""
+import asyncio
+import pickle
+import sys
+sys.path.insert(0, "..")
+
+from crawlo.network.request import Request
+from crawlo.spider import Spider
+from crawlo.core.scheduler import Scheduler
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+from crawlo.utils.log import get_logger
+from unittest.mock import Mock
+
+
+class TestSpider(Spider):
+    """测试爬虫"""
+    name = "validation_spider"
+    
+    def __init__(self):
+        super().__init__()
+        # 故意添加多个 logger 来测试清理
+        self.custom_logger = get_logger("custom")
+        self.debug_logger = get_logger("debug")
+        self.nested_data = {
+            'logger': get_logger("nested"),
+            'sub': {
+                'logger_ref': get_logger("sub_logger")
+            }
+        }
+    
+    def parse(self, response):
+        # 验证主 logger 还在
+        self.logger.info(f"✅ 主 logger 工作正常: {response.url}")
+        return {"url": response.url, "status": "success"}
+
+
+def test_scheduler_cleaning():
+    """测试调度器的 logger 清理"""
+    print("🔍 测试调度器 logger 清理...")
+    
+    spider = TestSpider()
+    request = Request(
+        url="https://scheduler-test.com",
+        callback=spider.parse,
+        meta={"logger": get_logger("meta_logger")}
+    )
+    
+    # Mock crawler 和 scheduler
+    class MockCrawler:
+        def __init__(self):
+            self.spider = spider
+    
+    class MockScheduler(Scheduler):
+        def __init__(self):
+            self.crawler = MockCrawler()
+            self.logger = get_logger("MockScheduler")
+    
+    scheduler = MockScheduler()
+    
+    # 清理前检查
+    print(f"   🔧 清理前 - spider.logger: {spider.logger is not None}")
+    print(f"   🔧 清理前 - spider.custom_logger: {spider.custom_logger is not None}")
+    print(f"   🔧 清理前 - request.callback: {request.callback is not None}")
+    
+    # 执行清理
+    cleaned_request = scheduler._deep_clean_loggers(request)
+    
+    # 清理后检查
+    print(f"   ✅ 清理后 - spider.logger: {spider.logger is not None}")
+    print(f"   ✅ 清理后 - spider.custom_logger: {spider.custom_logger is None}")
+    print(f"   ✅ 清理后 - request.callback: {cleaned_request.callback is None}")
+    
+    # 序列化测试
+    try:
+        serialized = pickle.dumps(cleaned_request)
+        print(f"   ✅ 调度器清理后序列化成功，大小: {len(serialized)} bytes")
+        return True
+    except Exception as e:
+        print(f"   ❌ 调度器清理后序列化失败: {e}")
+        return False
+
+
+async def test_redis_queue_cleaning():
+    """测试 Redis 队列的 logger 清理"""
+    print("\\n🔍 测试 Redis 队列 logger 清理...")
+    
+    spider = TestSpider()
+    request = Request(
+        url="https://redis-test.com",
+        callback=spider.parse,
+        meta={"logger": get_logger("meta_logger")}
+    )
+    
+    try:
+        queue = RedisPriorityQueue(redis_url="redis://127.0.0.1:6379/0")
+        await queue.connect()
+        
+        # 入队测试
+        success = await queue.put(request, priority=0)
+        print(f"   ✅ Redis 队列入队成功: {success}")
+        
+        if success:
+            # 出队测试
+            retrieved = await queue.get(timeout=2.0)
+            if retrieved:
+                print(f"   ✅ Redis 队列出队成功: {retrieved.url}")
+                print(f"   ✅ callback 信息保存: {'_callback_info' in retrieved.meta}")
+                await queue.close()
+                return True
+            else:
+                print("   ❌ 出队失败")
+                await queue.close()
+                return False
+        else:
+            await queue.close()
+            return False
+            
+    except Exception as e:
+        print(f"   ❌ Redis 队列测试失败: {e}")
+        return False
+
+
+async def main():
+    """主测试函数"""
+    print("🚀 开始最终验证测试...")
+    print("=" * 60)
+    
+    # 测试 1: 调度器清理
+    scheduler_ok = test_scheduler_cleaning()
+    
+    # 测试 2: Redis 队列清理
+    redis_ok = await test_redis_queue_cleaning()
+    
+    print("\\n" + "=" * 60)
+    print("📊 测试结果汇总:")
+    print(f"   调度器 logger 清理: {'✅ 通过' if scheduler_ok else '❌ 失败'}")
+    print(f"   Redis 队列清理: {'✅ 通过' if redis_ok else '❌ 失败'}")
+    
+    if scheduler_ok and redis_ok:
+        print("\\n🎉 所有测试通过！")
+        print("✅ 分布式队列的 logger 序列化问题已完全修复！")
+        print("✅ Crawlo 现在可以正常使用 Redis 分布式队列了！")
+        return True
+    else:
+        print("\\n❌ 部分测试失败，需要进一步修复")
+        return False
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/test_proxy_health_check.py

@@ -1,33 +1,33 @@
-# tests/test_proxy_health_check.py
-import pytest
-from unittest.mock import AsyncMock, patch
-from crawlo.proxy.health_check import check_single_proxy
-import httpx
-
-
-@pytest.mark.asyncio
-@patch('httpx.AsyncClient')
-async def test_health_check_success(mock_client_class):
-    """测试健康检查：成功"""
-    mock_resp = AsyncMock()
-    mock_resp.status_code = 200
-    mock_client_class.return_value.__aenter__.return_value.get.return_value = mock_resp
-
-    proxy_info = {'url': 'http://good:8080', 'healthy': False}
-    await check_single_proxy(proxy_info)
-
-    assert proxy_info['healthy'] is True
-    assert proxy_info['failures'] == 0
-
-
-@pytest.mark.asyncio
-@patch('httpx.AsyncClient')
-async def test_health_check_failure(mock_client_class):
-    """测试健康检查：失败"""
-    mock_client_class.return_value.__aenter__.return_value.get.side_effect = httpx.ConnectError("Failed")
-
-    proxy_info = {'url': 'http://bad:8080', 'healthy': True, 'failures': 0}
-    await check_single_proxy(proxy_info)
-
-    assert proxy_info['healthy'] is False
+# tests/test_proxy_health_check.py
+import pytest
+from unittest.mock import AsyncMock, patch
+from crawlo.proxy.health_check import check_single_proxy
+import httpx
+
+
+@pytest.mark.asyncio
+@patch('httpx.AsyncClient')
+async def test_health_check_success(mock_client_class):
+    """测试健康检查：成功"""
+    mock_resp = AsyncMock()
+    mock_resp.status_code = 200
+    mock_client_class.return_value.__aenter__.return_value.get.return_value = mock_resp
+
+    proxy_info = {'url': 'http://good:8080', 'healthy': False}
+    await check_single_proxy(proxy_info)
+
+    assert proxy_info['healthy'] is True
+    assert proxy_info['failures'] == 0
+
+
+@pytest.mark.asyncio
+@patch('httpx.AsyncClient')
+async def test_health_check_failure(mock_client_class):
+    """测试健康检查：失败"""
+    mock_client_class.return_value.__aenter__.return_value.get.side_effect = httpx.ConnectError("Failed")
+
+    proxy_info = {'url': 'http://bad:8080', 'healthy': True, 'failures': 0}
+    await check_single_proxy(proxy_info)
+
+    assert proxy_info['healthy'] is False
     assert proxy_info['failures'] == 1

tests/test_proxy_middleware_integration.py

@@ -1,137 +1,137 @@
-# tests/test_proxy_middleware_integration.py
-import pytest
-import asyncio
-import time
-from unittest.mock import Mock, AsyncMock, patch
-from crawlo import Request, Response, Spider
-from crawlo.proxy.middleware import ProxyMiddleware
-from crawlo.proxy.stats import ProxyStats
-
-
-@pytest.fixture
-def crawler():
-    class MockSettings:
-        def get(self, key, default=None):
-            defaults = {
-                'PROXY_ENABLED': True,
-                'PROXIES': ['http://p1:8080', 'http://p2:8080'],
-                'PROXY_SELECTION_STRATEGY': 'random',
-                'PROXY_REQUEST_DELAY_ENABLED': False,
-                'PROXY_MAX_RETRY_COUNT': 1,
-            }
-            return defaults.get(key, default)
-
-        def get_bool(self, key, default=None):
-            return self.get(key, default)
-
-        def get_int(self, key, default=None):
-            return self.get(key, default)
-
-        def get_float(self, key, default=None):
-            return self.get(key, default)
-
-        def get_list(self, key, default=None):
-            return self.get(key, default)
-
-    class MockCrawler:
-        def __init__(self):
-            self.settings = MockSettings()
-
-    return MockCrawler()
-
-
-@pytest.fixture
-def middleware(crawler):
-    mw = ProxyMiddleware.create_instance(crawler)
-    mw._load_providers = Mock()
-    mw._update_proxy_pool = AsyncMock()
-    mw._health_check = AsyncMock()
-    mw.scheduler = None
-
-    mw.proxies = [
-        {
-            'url': 'http://p1:8080',
-            'healthy': True,
-            'failures': 0,
-            'last_health_check': 0,
-            'unhealthy_since': 0
-        },
-        {
-            'url': 'http://p2:8080',
-            'healthy': True,
-            'failures': 0,
-            'last_health_check': 0,
-            'unhealthy_since': 0
-        },
-    ]
-    mw.stats = ProxyStats()
-    for p in mw.proxies:
-        mw.stats.record(p['url'], 'total')
-
-    asyncio.get_event_loop().run_until_complete(mw._initial_setup())
-    return mw
-
-
-@pytest.fixture
-def spider():
-    return Mock(spec=Spider, logger=Mock())
-
-
-def test_process_request_sets_proxy(middleware, spider):
-    request = Request("https://example.com")
-    result = asyncio.get_event_loop().run_until_complete(
-        middleware.process_request(request, spider)
-    )
-    assert result is None
-    assert hasattr(request, 'proxy')
-    assert request.proxy in ['http://p1:8080', 'http://p2:8080']
-
-
-def test_process_response_records_success(middleware, spider):
-    request = Request("https://example.com")
-    request.proxy = 'http://p1:8080'
-    response = Response("https://example.com", body=b"ok", headers={})
-    middleware.stats.record(request.proxy, 'total')
-    middleware.process_response(request, response, spider)
-    assert middleware.stats.get(request.proxy)['success'] == 1
-
-
-def test_process_exception_switches_proxy(middleware, spider):
-    request = Request("https://example.com")
-    request.proxy = 'http://p1:8080'
-    request.meta['proxy_retry_count'] = 0
-
-    result = middleware.process_exception(request, Exception("Timeout"), spider)
-    assert result is not None
-    assert result.proxy != 'http://p1:8080'
-    assert result.meta['proxy_retry_count'] == 1
-
-    final = middleware.process_exception(result, Exception("Timeout"), spider)
-    assert final is None
-
-
-def test_mark_failure_disables_proxy(middleware):
-    proxy_url = 'http://p1:8080'
-    p = next(p for p in middleware.proxies if p['url'] == proxy_url)
-    p['failures'] = 2
-
-    middleware._mark_failure(proxy_url)
-    assert p['failures'] == 3
-    assert p['healthy'] is False
-    assert p['unhealthy_since'] > 0
-
-
-@pytest.mark.asyncio
-async def test_request_delay(middleware, spider):
-    """测试请求延迟功能：验证是否调用了 asyncio.sleep"""
-    with patch("crawlo.proxy.middleware.asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
-        middleware.delay_enabled = True  # 注意：这里应该是 delay_enabled 而不是 request_delay_enabled
-        middleware.request_delay = 0.1
-        middleware._last_req_time = time.time() - 0.05  # 50ms 前
-
-        request = Request("https://a.com")
-        await middleware.process_request(request, spider)
-
-        mock_sleep.assert_called_once()
-        delay = mock_sleep.call_args[0][0]
+# tests/test_proxy_middleware_integration.py
+import pytest
+import asyncio
+import time
+from unittest.mock import Mock, AsyncMock, patch
+from crawlo import Request, Response, Spider
+from crawlo.proxy.middleware import ProxyMiddleware
+from crawlo.proxy.stats import ProxyStats
+
+
+@pytest.fixture
+def crawler():
+    class MockSettings:
+        def get(self, key, default=None):
+            defaults = {
+                'PROXY_ENABLED': True,
+                'PROXIES': ['http://p1:8080', 'http://p2:8080'],
+                'PROXY_SELECTION_STRATEGY': 'random',
+                'PROXY_REQUEST_DELAY_ENABLED': False,
+                'PROXY_MAX_RETRY_COUNT': 1,
+            }
+            return defaults.get(key, default)
+
+        def get_bool(self, key, default=None):
+            return self.get(key, default)
+
+        def get_int(self, key, default=None):
+            return self.get(key, default)
+
+        def get_float(self, key, default=None):
+            return self.get(key, default)
+
+        def get_list(self, key, default=None):
+            return self.get(key, default)
+
+    class MockCrawler:
+        def __init__(self):
+            self.settings = MockSettings()
+
+    return MockCrawler()
+
+
+@pytest.fixture
+def middleware(crawler):
+    mw = ProxyMiddleware.create_instance(crawler)
+    mw._load_providers = Mock()
+    mw._update_proxy_pool = AsyncMock()
+    mw._health_check = AsyncMock()
+    mw.scheduler = None
+
+    mw.proxies = [
+        {
+            'url': 'http://p1:8080',
+            'healthy': True,
+            'failures': 0,
+            'last_health_check': 0,
+            'unhealthy_since': 0
+        },
+        {
+            'url': 'http://p2:8080',
+            'healthy': True,
+            'failures': 0,
+            'last_health_check': 0,
+            'unhealthy_since': 0
+        },
+    ]
+    mw.stats = ProxyStats()
+    for p in mw.proxies:
+        mw.stats.record(p['url'], 'total')
+
+    asyncio.get_event_loop().run_until_complete(mw._initial_setup())
+    return mw
+
+
+@pytest.fixture
+def spider():
+    return Mock(spec=Spider, logger=Mock())
+
+
+def test_process_request_sets_proxy(middleware, spider):
+    request = Request("https://example.com")
+    result = asyncio.get_event_loop().run_until_complete(
+        middleware.process_request(request, spider)
+    )
+    assert result is None
+    assert hasattr(request, 'proxy')
+    assert request.proxy in ['http://p1:8080', 'http://p2:8080']
+
+
+def test_process_response_records_success(middleware, spider):
+    request = Request("https://example.com")
+    request.proxy = 'http://p1:8080'
+    response = Response("https://example.com", body=b"ok", headers={})
+    middleware.stats.record(request.proxy, 'total')
+    middleware.process_response(request, response, spider)
+    assert middleware.stats.get(request.proxy)['success'] == 1
+
+
+def test_process_exception_switches_proxy(middleware, spider):
+    request = Request("https://example.com")
+    request.proxy = 'http://p1:8080'
+    request.meta['proxy_retry_count'] = 0
+
+    result = middleware.process_exception(request, Exception("Timeout"), spider)
+    assert result is not None
+    assert result.proxy != 'http://p1:8080'
+    assert result.meta['proxy_retry_count'] == 1
+
+    final = middleware.process_exception(result, Exception("Timeout"), spider)
+    assert final is None
+
+
+def test_mark_failure_disables_proxy(middleware):
+    proxy_url = 'http://p1:8080'
+    p = next(p for p in middleware.proxies if p['url'] == proxy_url)
+    p['failures'] = 2
+
+    middleware._mark_failure(proxy_url)
+    assert p['failures'] == 3
+    assert p['healthy'] is False
+    assert p['unhealthy_since'] > 0
+
+
+@pytest.mark.asyncio
+async def test_request_delay(middleware, spider):
+    """测试请求延迟功能：验证是否调用了 asyncio.sleep"""
+    with patch("crawlo.proxy.middleware.asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
+        middleware.delay_enabled = True  # 注意：这里应该是 delay_enabled 而不是 request_delay_enabled
+        middleware.request_delay = 0.1
+        middleware._last_req_time = time.time() - 0.05  # 50ms 前
+
+        request = Request("https://a.com")
+        await middleware.process_request(request, spider)
+
+        mock_sleep.assert_called_once()
+        delay = mock_sleep.call_args[0][0]
         assert 0.04 <= delay <= 0.06

tests/test_proxy_providers.py

@@ -1,57 +1,57 @@
-# tests/test_proxy_providers.py
-import pytest
-import pytest
-import respx
-from httpx import Response
-from crawlo.proxy.providers import StaticProxyProvider, FileProxyProvider, APIProxyProvider
-import tempfile
-import os
-
-
-@pytest.mark.asyncio
-async def test_static_provider():
-    """测试静态代理提供者"""
-    provider = StaticProxyProvider(['http://1.1.1.1:8080', 'http://2.2.2.2:8080'])
-    proxies = await provider.fetch_proxies()
-    assert len(proxies) == 2
-    assert 'http://1.1.1.1:8080' in proxies
-    assert 'http://2.2.2.2:8080' in proxies
-
-
-@pytest.mark.asyncio
-async def test_file_provider():
-    """测试文件代理提供者"""
-    with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
-        f.write("http://a.com:8080\nhttp://b.com:8080\n")
-        temp_path = f.name
-    try:
-        provider = FileProxyProvider(temp_path)
-        proxies = await provider.fetch_proxies()
-        assert len(proxies) == 2
-        assert 'http://a.com:8080' in proxies
-        assert 'http://b.com:8080' in proxies
-    finally:
-        os.unlink(temp_path)
-
-
-@pytest.mark.asyncio
-@respx.mock
-async def test_api_provider():
-    """使用 respx 拦截 HTTP 请求，更简洁可靠"""
-    # 拦截 GET 请求
-    respx.get("https://api.example.com").mock(
-        return_value=Response(
-            200,
-            json=[
-                {"ip": "1.1.1.1", "port": 8080},
-                {"ip": "2.2.2.2", "port": 8080}
-            ]
-        )
-    )
-
-    provider = APIProxyProvider(url="https://api.example.com")
-    proxies = await provider.fetch_proxies()
-
-    assert len(proxies) == 2
-    assert "http://1.1.1.1:8080" in proxies
+# tests/test_proxy_providers.py
+import pytest
+import pytest
+import respx
+from httpx import Response
+from crawlo.proxy.providers import StaticProxyProvider, FileProxyProvider, APIProxyProvider
+import tempfile
+import os
+
+
+@pytest.mark.asyncio
+async def test_static_provider():
+    """测试静态代理提供者"""
+    provider = StaticProxyProvider(['http://1.1.1.1:8080', 'http://2.2.2.2:8080'])
+    proxies = await provider.fetch_proxies()
+    assert len(proxies) == 2
+    assert 'http://1.1.1.1:8080' in proxies
+    assert 'http://2.2.2.2:8080' in proxies
+
+
+@pytest.mark.asyncio
+async def test_file_provider():
+    """测试文件代理提供者"""
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
+        f.write("http://a.com:8080\nhttp://b.com:8080\n")
+        temp_path = f.name
+    try:
+        provider = FileProxyProvider(temp_path)
+        proxies = await provider.fetch_proxies()
+        assert len(proxies) == 2
+        assert 'http://a.com:8080' in proxies
+        assert 'http://b.com:8080' in proxies
+    finally:
+        os.unlink(temp_path)
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_api_provider():
+    """使用 respx 拦截 HTTP 请求，更简洁可靠"""
+    # 拦截 GET 请求
+    respx.get("https://api.example.com").mock(
+        return_value=Response(
+            200,
+            json=[
+                {"ip": "1.1.1.1", "port": 8080},
+                {"ip": "2.2.2.2", "port": 8080}
+            ]
+        )
+    )
+
+    provider = APIProxyProvider(url="https://api.example.com")
+    proxies = await provider.fetch_proxies()
+
+    assert len(proxies) == 2
+    assert "http://1.1.1.1:8080" in proxies
     assert "http://2.2.2.2:8080" in proxies

tests/test_proxy_stats.py

@@ -1,20 +1,20 @@
-# tests/test_proxy_stats.py
-from crawlo.proxy.stats import ProxyStats
-
-
-def test_proxy_stats():
-    """测试代理统计功能"""
-    stats = ProxyStats()
-    url = 'http://proxy1:8080'
-
-    stats.record(url, 'success')
-    stats.record(url, 'success')
-    stats.record(url, 'failure')
-
-    assert stats.get(url)['success'] == 2
-    assert stats.get(url)['failure'] == 1
-    assert stats.get(url)['total'] == 3
-
-    all_data = stats.all()
-    assert url in all_data
+# tests/test_proxy_stats.py
+from crawlo.proxy.stats import ProxyStats
+
+
+def test_proxy_stats():
+    """测试代理统计功能"""
+    stats = ProxyStats()
+    url = 'http://proxy1:8080'
+
+    stats.record(url, 'success')
+    stats.record(url, 'success')
+    stats.record(url, 'failure')
+
+    assert stats.get(url)['success'] == 2
+    assert stats.get(url)['failure'] == 1
+    assert stats.get(url)['total'] == 3
+
+    all_data = stats.all()
+    assert url in all_data
     assert all_data[url]['success'] == 2