crawlo 1.4.7__py3-none-any.whl → 1.4.8__py3-none-any.whl

tests/test_middleware_debug.py

@@ -1,142 +1,142 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-调试中间件User-Agent随机性问题
-"""
-
-import sys
-import os
-import random
-from unittest.mock import Mock, patch
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.middleware.default_header import DefaultHeaderMiddleware
-from crawlo.settings.setting_manager import SettingManager
-from crawlo.data.user_agents import get_user_agents
-
-
-class MockLogger:
-    """Mock Logger 类，用于测试日志输出"""
-    def __init__(self, name, level=None):
-        self.name = name
-        self.level = level
-        self.logs = []
-
-    def debug(self, msg):
-        self.logs.append(('debug', msg))
-        print(f"DEBUG: {msg}")
-
-    def info(self, msg):
-        self.logs.append(('info', msg))
-
-    def warning(self, msg):
-        self.logs.append(('warning', msg))
-
-    def error(self, msg):
-        self.logs.append(('error', msg))
-
-    def isEnabledFor(self, level):
-        return True
-
-
-def debug_middleware_initialization():
-    """调试中间件初始化过程"""
-    print("=== 调试中间件初始化过程 ===")
-    
-    settings = SettingManager()
-    settings.set('DEFAULT_REQUEST_HEADERS', {
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    })
-    settings.set('RANDOM_USER_AGENT_ENABLED', True)
-    settings.set('LOG_LEVEL', 'DEBUG')
-    settings.set('RANDOMNESS', True)
-    
-    crawler = Mock()
-    crawler.settings = settings
-    
-    logger = MockLogger('DefaultHeaderMiddleware')
-    with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
-        middleware = DefaultHeaderMiddleware.create_instance(crawler)
-        
-        print(f"随机User-Agent启用: {middleware.random_user_agent_enabled}")
-        print(f"User-Agent列表数量: {len(middleware.user_agents)}")
-        print(f"User-Agent设备类型: {middleware.user_agent_device_type}")
-        
-        # 检查前几个User-Agent
-        print("前5个User-Agent:")
-        for i, ua in enumerate(middleware.user_agents[:5]):
-            print(f"  {i+1}. {ua[:50]}...")
-        
-        # 测试_get_random_user_agent方法
-        print("\n测试_get_random_user_agent方法:")
-        for i in range(10):
-            ua = middleware._get_random_user_agent()
-            print(f"  {i+1}. {ua[:50]}...")
-
-
-def test_multiple_middleware_instances():
-    """测试多个中间件实例的随机性"""
-    print("\n=== 测试多个中间件实例的随机性 ===")
-    
-    ua_values = []
-    
-    for i in range(10):
-        settings = SettingManager()
-        settings.set('DEFAULT_REQUEST_HEADERS', {
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-        })
-        settings.set('RANDOM_USER_AGENT_ENABLED', True)
-        settings.set('LOG_LEVEL', 'DEBUG')
-        settings.set('RANDOMNESS', True)
-        
-        crawler = Mock()
-        crawler.settings = settings
-        
-        logger = MockLogger('DefaultHeaderMiddleware')
-        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
-            middleware = DefaultHeaderMiddleware.create_instance(crawler)
-            
-            # 获取随机User-Agent
-            ua = middleware._get_random_user_agent()
-            if ua:
-                ua_values.append(ua)
-                print(f"  实例{i+1}: {ua[:50]}...")
-    
-    unique_uas = set(ua_values)
-    print(f"\n生成了 {len(ua_values)} 个User-Agent，其中 {len(unique_uas)} 个不同")
-
-
-def check_user_agents_module():
-    """检查user_agents模块"""
-    print("\n=== 检查user_agents模块 ===")
-    
-    # 获取不同类型的User-Agent
-    device_types = ["all", "desktop", "mobile", "chrome", "firefox"]
-    
-    for device_type in device_types:
-        uas = get_user_agents(device_type)
-        print(f"{device_type}类型User-Agent数量: {len(uas)}")
-        if uas:
-            print(f"  示例: {uas[0][:50]}...")
-
-
-def main():
-    print("开始调试中间件User-Agent随机性问题...")
-    
-    try:
-        debug_middleware_initialization()
-        test_multiple_middleware_instances()
-        check_user_agents_module()
-        
-        print("\n调试完成！")
-        
-    except Exception as e:
-        print(f"\n调试过程中发生错误: {e}")
-        import traceback
-        traceback.print_exc()
-
-
-if __name__ == "__main__":
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+调试中间件User-Agent随机性问题
+"""
+
+import sys
+import os
+import random
+from unittest.mock import Mock, patch
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.middleware.default_header import DefaultHeaderMiddleware
+from crawlo.settings.setting_manager import SettingManager
+from crawlo.data.user_agents import get_user_agents
+
+
+class MockLogger:
+    """Mock Logger 类，用于测试日志输出"""
+    def __init__(self, name, level=None):
+        self.name = name
+        self.level = level
+        self.logs = []
+
+    def debug(self, msg):
+        self.logs.append(('debug', msg))
+        print(f"DEBUG: {msg}")
+
+    def info(self, msg):
+        self.logs.append(('info', msg))
+
+    def warning(self, msg):
+        self.logs.append(('warning', msg))
+
+    def error(self, msg):
+        self.logs.append(('error', msg))
+
+    def isEnabledFor(self, level):
+        return True
+
+
+def debug_middleware_initialization():
+    """调试中间件初始化过程"""
+    print("=== 调试中间件初始化过程 ===")
+    
+    settings = SettingManager()
+    settings.set('DEFAULT_REQUEST_HEADERS', {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    })
+    settings.set('RANDOM_USER_AGENT_ENABLED', True)
+    settings.set('LOG_LEVEL', 'DEBUG')
+    settings.set('RANDOMNESS', True)
+    
+    crawler = Mock()
+    crawler.settings = settings
+    
+    logger = MockLogger('DefaultHeaderMiddleware')
+    with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+        middleware = DefaultHeaderMiddleware.create_instance(crawler)
+        
+        print(f"随机User-Agent启用: {middleware.random_user_agent_enabled}")
+        print(f"User-Agent列表数量: {len(middleware.user_agents)}")
+        print(f"User-Agent设备类型: {middleware.user_agent_device_type}")
+        
+        # 检查前几个User-Agent
+        print("前5个User-Agent:")
+        for i, ua in enumerate(middleware.user_agents[:5]):
+            print(f"  {i+1}. {ua[:50]}...")
+        
+        # 测试_get_random_user_agent方法
+        print("\n测试_get_random_user_agent方法:")
+        for i in range(10):
+            ua = middleware._get_random_user_agent()
+            print(f"  {i+1}. {ua[:50]}...")
+
+
+def test_multiple_middleware_instances():
+    """测试多个中间件实例的随机性"""
+    print("\n=== 测试多个中间件实例的随机性 ===")
+    
+    ua_values = []
+    
+    for i in range(10):
+        settings = SettingManager()
+        settings.set('DEFAULT_REQUEST_HEADERS', {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        })
+        settings.set('RANDOM_USER_AGENT_ENABLED', True)
+        settings.set('LOG_LEVEL', 'DEBUG')
+        settings.set('RANDOMNESS', True)
+        
+        crawler = Mock()
+        crawler.settings = settings
+        
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            
+            # 获取随机User-Agent
+            ua = middleware._get_random_user_agent()
+            if ua:
+                ua_values.append(ua)
+                print(f"  实例{i+1}: {ua[:50]}...")
+    
+    unique_uas = set(ua_values)
+    print(f"\n生成了 {len(ua_values)} 个User-Agent，其中 {len(unique_uas)} 个不同")
+
+
+def check_user_agents_module():
+    """检查user_agents模块"""
+    print("\n=== 检查user_agents模块 ===")
+    
+    # 获取不同类型的User-Agent
+    device_types = ["all", "desktop", "mobile", "chrome", "firefox"]
+    
+    for device_type in device_types:
+        uas = get_user_agents(device_type)
+        print(f"{device_type}类型User-Agent数量: {len(uas)}")
+        if uas:
+            print(f"  示例: {uas[0][:50]}...")
+
+
+def main():
+    print("开始调试中间件User-Agent随机性问题...")
+    
+    try:
+        debug_middleware_initialization()
+        test_multiple_middleware_instances()
+        check_user_agents_module()
+        
+        print("\n调试完成！")
+        
+    except Exception as e:
+        print(f"\n调试过程中发生错误: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
     main()

tests/test_mode_consistency.py

@@ -1,52 +1,52 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-测试模式一致性提示
-"""
-import asyncio
-import sys
-import os
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.crawler import CrawlerProcess
-from crawlo.spider import Spider
-from crawlo import Request
-
-
-class TestSpider(Spider):
-    name = "test_mode_spider"
-    
-    def start_requests(self):
-        yield Request("https://httpbin.org/get")
-    
-    def parse(self, response):
-        yield {"url": response.url, "status": response.status_code}  # 修复：使用status_code而不是status
-
-
-async def test_mode_consistency():
-    """测试模式一致性提示"""
-    print("测试模式一致性提示...")
-    
-    try:
-        # 创建爬虫进程
-        process = CrawlerProcess()
-        
-        # 添加爬虫
-        await process.crawl(TestSpider)
-        
-        print("模式一致性测试完成")
-        
-    except Exception as e:
-        print(f"测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-
-
-if __name__ == "__main__":
-    # 设置日志级别
-    import logging
-    logging.basicConfig(level=logging.INFO)
-    
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试模式一致性提示
+"""
+import asyncio
+import sys
+import os
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class TestSpider(Spider):
+    name = "test_mode_spider"
+    
+    def start_requests(self):
+        yield Request("https://httpbin.org/get")
+    
+    def parse(self, response):
+        yield {"url": response.url, "status": response.status_code}  # 修复：使用status_code而不是status
+
+
+async def test_mode_consistency():
+    """测试模式一致性提示"""
+    print("测试模式一致性提示...")
+    
+    try:
+        # 创建爬虫进程
+        process = CrawlerProcess()
+        
+        # 添加爬虫
+        await process.crawl(TestSpider)
+        
+        print("模式一致性测试完成")
+        
+    except Exception as e:
+        print(f"测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    # 设置日志级别
+    import logging
+    logging.basicConfig(level=logging.INFO)
+    
     asyncio.run(test_mode_consistency())

tests/test_multi_directory.py

@@ -1,68 +1,68 @@
-#!/usr/bin/env python3
-# -*- coding: UTF-8 -*-
-"""
-测试多个爬虫目录的支持
-"""
-import sys
-import os
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.dirname(__file__))
-
-# 添加ofweek_standalone到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'examples', 'ofweek_standalone'))
-
-def test_multiple_spider_directories():
-    """测试多个爬虫目录的支持"""
-    print("测试多个爬虫目录的支持...")
-    
-    # 导入设置
-    import examples.ofweek_standalone.ofweek_standalone.settings as settings_module
-    
-    # 创建设置管理器
-    from crawlo.settings.setting_manager import SettingManager
-    settings = SettingManager()
-    settings.set_settings(settings_module)
-    
-    # 检查SPIDER_MODULES配置
-    spider_modules = settings.get('SPIDER_MODULES')
-    print(f"SPIDER_MODULES配置: {spider_modules}")
-    
-    # 创建CrawlerProcess实例
-    from crawlo.crawler import CrawlerProcess
-    process = CrawlerProcess(settings=settings)
-    
-    # 检查是否注册了爬虫
-    spider_names = process.get_spider_names()
-    print(f"已注册的爬虫: {spider_names}")
-    
-    # 验证期望的爬虫是否已注册
-    expected_spiders = ['of_week_standalone', 'test_spider']
-    registered_spiders = []
-    
-    for spider_name in expected_spiders:
-        if spider_name in spider_names:
-            print(f"✅ 成功: 爬虫 '{spider_name}' 已注册")
-            registered_spiders.append(spider_name)
-        else:
-            print(f"❌ 失败: 爬虫 '{spider_name}' 未找到")
-    
-    if len(registered_spiders) == len(expected_spiders):
-        print(f"🎉 所有爬虫都已成功注册!")
-        return True
-    else:
-        print(f"⚠️  部分爬虫未注册: {set(expected_spiders) - set(registered_spiders)}")
-        return False
-
-
-if __name__ == '__main__':
-    print("开始测试多个爬虫目录的支持...\n")
-    
-    success = test_multiple_spider_directories()
-    
-    if success:
-        print("\n🎉 测试通过!")
-        sys.exit(0)
-    else:
-        print("\n❌ 测试失败!")
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+"""
+测试多个爬虫目录的支持
+"""
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.dirname(__file__))
+
+# 添加ofweek_standalone到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'examples', 'ofweek_standalone'))
+
+def test_multiple_spider_directories():
+    """测试多个爬虫目录的支持"""
+    print("测试多个爬虫目录的支持...")
+    
+    # 导入设置
+    import examples.ofweek_standalone.ofweek_standalone.settings as settings_module
+    
+    # 创建设置管理器
+    from crawlo.settings.setting_manager import SettingManager
+    settings = SettingManager()
+    settings.set_settings(settings_module)
+    
+    # 检查SPIDER_MODULES配置
+    spider_modules = settings.get('SPIDER_MODULES')
+    print(f"SPIDER_MODULES配置: {spider_modules}")
+    
+    # 创建CrawlerProcess实例
+    from crawlo.crawler import CrawlerProcess
+    process = CrawlerProcess(settings=settings)
+    
+    # 检查是否注册了爬虫
+    spider_names = process.get_spider_names()
+    print(f"已注册的爬虫: {spider_names}")
+    
+    # 验证期望的爬虫是否已注册
+    expected_spiders = ['of_week_standalone', 'test_spider']
+    registered_spiders = []
+    
+    for spider_name in expected_spiders:
+        if spider_name in spider_names:
+            print(f"✅ 成功: 爬虫 '{spider_name}' 已注册")
+            registered_spiders.append(spider_name)
+        else:
+            print(f"❌ 失败: 爬虫 '{spider_name}' 未找到")
+    
+    if len(registered_spiders) == len(expected_spiders):
+        print(f"🎉 所有爬虫都已成功注册!")
+        return True
+    else:
+        print(f"⚠️  部分爬虫未注册: {set(expected_spiders) - set(registered_spiders)}")
+        return False
+
+
+if __name__ == '__main__':
+    print("开始测试多个爬虫目录的支持...\n")
+    
+    success = test_multiple_spider_directories()
+    
+    if success:
+        print("\n🎉 测试通过!")
+        sys.exit(0)
+    else:
+        print("\n❌ 测试失败!")
         sys.exit(1)