crawlo 1.4.6__py3-none-any.whl → 1.4.7__py3-none-any.whl

tests/test_spider_loader.py

@@ -1,50 +1,50 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-测试SpiderLoader功能
-"""
-
-import sys
-import os
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.utils.spider_loader import SpiderLoader
-from crawlo.settings.setting_manager import SettingManager
-
-
-def test_spider_loader():
-    """测试SpiderLoader基本功能"""
-    print("测试SpiderLoader基本功能...")
-    
-    # 创建一个简单的设置
-    settings = SettingManager({
-        'SPIDER_MODULES': ['tests.test_spiders'],
-        'SPIDER_LOADER_WARN_ONLY': True
-    })
-    
-    # 创建SpiderLoader实例
-    loader = SpiderLoader.from_settings(settings)
-    
-    # 测试list方法
-    spider_names = loader.list()
-    print(f"发现的爬虫: {spider_names}")
-    
-    # 测试load方法
-    if spider_names:
-        spider_name = spider_names[0]
-        try:
-            spider_class = loader.load(spider_name)
-            print(f"成功加载爬虫: {spider_name} -> {spider_class}")
-        except KeyError as e:
-            print(f"加载爬虫失败: {e}")
-    
-    # 测试get_all方法
-    all_spiders = loader.get_all()
-    print(f"所有爬虫: {list(all_spiders.keys())}")
-    
-    print("测试完成!")
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+测试SpiderLoader功能
+"""
+
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.spider_loader import SpiderLoader
+from crawlo.settings.setting_manager import SettingManager
+
+
+def test_spider_loader():
+    """测试SpiderLoader基本功能"""
+    print("测试SpiderLoader基本功能...")
+    
+    # 创建一个简单的设置
+    settings = SettingManager({
+        'SPIDER_MODULES': ['tests.test_spiders'],
+        'SPIDER_LOADER_WARN_ONLY': True
+    })
+    
+    # 创建SpiderLoader实例
+    loader = SpiderLoader.from_settings(settings)
+    
+    # 测试list方法
+    spider_names = loader.list()
+    print(f"发现的爬虫: {spider_names}")
+    
+    # 测试load方法
+    if spider_names:
+        spider_name = spider_names[0]
+        try:
+            spider_class = loader.load(spider_name)
+            print(f"成功加载爬虫: {spider_name} -> {spider_class}")
+        except KeyError as e:
+            print(f"加载爬虫失败: {e}")
+    
+    # 测试get_all方法
+    all_spiders = loader.get_all()
+    print(f"所有爬虫: {list(all_spiders.keys())}")
+    
+    print("测试完成!")
+
+
+if __name__ == '__main__':
     test_spider_loader()

tests/test_spider_loader_comprehensive.py

@@ -1,70 +1,70 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-综合测试SpiderLoader功能
-"""
-
-import sys
-import os
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.utils.spider_loader import SpiderLoader
-from crawlo.crawler import CrawlerProcess
-from crawlo.settings.setting_manager import SettingManager
-
-
-def test_spider_loader_comprehensive():
-    """综合测试SpiderLoader功能"""
-    print("综合测试SpiderLoader功能...")
-    
-    # 1. 测试基本的SpiderLoader功能
-    print("\n1. 测试基本的SpiderLoader功能")
-    settings = SettingManager({
-        'SPIDER_MODULES': ['tests.test_spiders'],
-        'SPIDER_LOADER_WARN_ONLY': True
-    })
-    
-    loader = SpiderLoader.from_settings(settings)
-    spider_names = loader.list()
-    print(f"  发现的爬虫: {spider_names}")
-    
-    if spider_names:
-        spider_name = spider_names[0]
-        spider_class = loader.load(spider_name)
-        print(f"  成功加载爬虫: {spider_name} -> {spider_class}")
-    
-    # 2. 测试CrawlerProcess与SPIDER_MODULES的集成
-    print("\n2. 测试CrawlerProcess与SPIDER_MODULES的集成")
-    process = CrawlerProcess(settings=settings)
-    process_spider_names = process.get_spider_names()
-    print(f"  CrawlerProcess发现的爬虫: {process_spider_names}")
-    
-    is_registered = process.is_spider_registered('test_spider')
-    print(f"  爬虫'test_spider'是否已注册: {is_registered}")
-    
-    spider_class = process.get_spider_class('test_spider')
-    print(f"  爬虫'test_spider'的类: {spider_class}")
-    
-    # 3. 测试接口规范
-    print("\n3. 测试接口规范")
-    # 检查SpiderLoader是否实现了ISpiderLoader接口所需的方法
-    from crawlo.interfaces import ISpiderLoader
-    # 由于ISpiderLoader是Protocol，我们不能直接使用isinstance检查
-    # 而是检查是否实现了所需的方法
-    required_methods = ['load', 'list', 'find_by_request']
-    implements_interface = all(hasattr(loader, method) for method in required_methods)
-    print(f"  SpiderLoader是否实现了ISpiderLoader接口: {implements_interface}")
-    
-    # 4. 测试方法存在性
-    print("\n4. 测试方法存在性")
-    required_methods = ['load', 'list', 'find_by_request', 'get_all']
-    for method in required_methods:
-        has_method = hasattr(loader, method)
-        print(f"  SpiderLoader是否有{method}方法: {has_method}")
-    
-    print("\n综合测试完成!")
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+综合测试SpiderLoader功能
+"""
+
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.spider_loader import SpiderLoader
+from crawlo.crawler import CrawlerProcess
+from crawlo.settings.setting_manager import SettingManager
+
+
+def test_spider_loader_comprehensive():
+    """综合测试SpiderLoader功能"""
+    print("综合测试SpiderLoader功能...")
+    
+    # 1. 测试基本的SpiderLoader功能
+    print("\n1. 测试基本的SpiderLoader功能")
+    settings = SettingManager({
+        'SPIDER_MODULES': ['tests.test_spiders'],
+        'SPIDER_LOADER_WARN_ONLY': True
+    })
+    
+    loader = SpiderLoader.from_settings(settings)
+    spider_names = loader.list()
+    print(f"  发现的爬虫: {spider_names}")
+    
+    if spider_names:
+        spider_name = spider_names[0]
+        spider_class = loader.load(spider_name)
+        print(f"  成功加载爬虫: {spider_name} -> {spider_class}")
+    
+    # 2. 测试CrawlerProcess与SPIDER_MODULES的集成
+    print("\n2. 测试CrawlerProcess与SPIDER_MODULES的集成")
+    process = CrawlerProcess(settings=settings)
+    process_spider_names = process.get_spider_names()
+    print(f"  CrawlerProcess发现的爬虫: {process_spider_names}")
+    
+    is_registered = process.is_spider_registered('test_spider')
+    print(f"  爬虫'test_spider'是否已注册: {is_registered}")
+    
+    spider_class = process.get_spider_class('test_spider')
+    print(f"  爬虫'test_spider'的类: {spider_class}")
+    
+    # 3. 测试接口规范
+    print("\n3. 测试接口规范")
+    # 检查SpiderLoader是否实现了ISpiderLoader接口所需的方法
+    from crawlo.interfaces import ISpiderLoader
+    # 由于ISpiderLoader是Protocol，我们不能直接使用isinstance检查
+    # 而是检查是否实现了所需的方法
+    required_methods = ['load', 'list', 'find_by_request']
+    implements_interface = all(hasattr(loader, method) for method in required_methods)
+    print(f"  SpiderLoader是否实现了ISpiderLoader接口: {implements_interface}")
+    
+    # 4. 测试方法存在性
+    print("\n4. 测试方法存在性")
+    required_methods = ['load', 'list', 'find_by_request', 'get_all']
+    for method in required_methods:
+        has_method = hasattr(loader, method)
+        print(f"  SpiderLoader是否有{method}方法: {has_method}")
+    
+    print("\n综合测试完成!")
+
+
+if __name__ == '__main__':
     test_spider_loader_comprehensive()

tests/test_spider_modules.py

@@ -1,85 +1,85 @@
-#!/usr/bin/env python3
-# -*- coding: UTF-8 -*-
-"""
-测试SPIDER_MODULES配置的自动读取功能
-"""
-import sys
-import os
-import asyncio
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.dirname(__file__))
-
-# 添加ofweek_standalone到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'examples', 'ofweek_standalone'))
-
-from crawlo.crawler import CrawlerProcess
-from crawlo.spider import get_spider_names
-
-
-def test_spider_modules_auto_discovery():
-    """测试SPIDER_MODULES配置的自动读取功能"""
-    print("测试SPIDER_MODULES配置的自动读取功能...")
-    
-    # 导入设置
-    import examples.ofweek_standalone.ofweek_standalone.settings as settings_module
-    
-    # 创建设置管理器
-    from crawlo.settings.setting_manager import SettingManager
-    settings = SettingManager()
-    settings.set_settings(settings_module)
-    
-    # 创建CrawlerProcess实例，不显式传递spider_modules
-    process = CrawlerProcess(settings=settings)
-    
-    # 检查是否自动注册了爬虫
-    spider_names = process.get_spider_names()
-    print(f"已注册的爬虫: {spider_names}")
-    
-    # 验证期望的爬虫是否已注册
-    expected_spider = 'of_week_standalone'
-    if expected_spider in spider_names:
-        print(f"✅ 成功: 爬虫 '{expected_spider}' 已自动注册")
-        return True
-    else:
-        print(f"❌ 失败: 爬虫 '{expected_spider}' 未找到")
-        return False
-
-
-def test_crawler_process_with_explicit_spider_modules():
-    """测试显式传递spider_modules参数的功能"""
-    print("\n测试显式传递spider_modules参数的功能...")
-    
-    # 显式传递spider_modules参数
-    spider_modules = ['ofweek_standalone.spiders']
-    process = CrawlerProcess(spider_modules=spider_modules)
-    
-    # 检查是否注册了爬虫
-    spider_names = process.get_spider_names()
-    print(f"已注册的爬虫: {spider_names}")
-    
-    # 验证期望的爬虫是否已注册
-    expected_spider = 'of_week_standalone'
-    if expected_spider in spider_names:
-        print(f"✅ 成功: 爬虫 '{expected_spider}' 已注册")
-        return True
-    else:
-        print(f"❌ 失败: 爬虫 '{expected_spider}' 未找到")
-        return False
-
-
-if __name__ == '__main__':
-    print("开始测试SPIDER_MODULES配置功能...\n")
-    
-    # 测试自动发现功能
-    success1 = test_spider_modules_auto_discovery()
-    
-    # 测试显式传递参数功能
-    success2 = test_crawler_process_with_explicit_spider_modules()
-    
-    if success1 and success2:
-        print("\n🎉 所有测试通过!")
-        sys.exit(0)
-    else:
-        print("\n❌ 部分测试失败!")
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+"""
+测试SPIDER_MODULES配置的自动读取功能
+"""
+import sys
+import os
+import asyncio
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.dirname(__file__))
+
+# 添加ofweek_standalone到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'examples', 'ofweek_standalone'))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import get_spider_names
+
+
+def test_spider_modules_auto_discovery():
+    """测试SPIDER_MODULES配置的自动读取功能"""
+    print("测试SPIDER_MODULES配置的自动读取功能...")
+    
+    # 导入设置
+    import examples.ofweek_standalone.ofweek_standalone.settings as settings_module
+    
+    # 创建设置管理器
+    from crawlo.settings.setting_manager import SettingManager
+    settings = SettingManager()
+    settings.set_settings(settings_module)
+    
+    # 创建CrawlerProcess实例，不显式传递spider_modules
+    process = CrawlerProcess(settings=settings)
+    
+    # 检查是否自动注册了爬虫
+    spider_names = process.get_spider_names()
+    print(f"已注册的爬虫: {spider_names}")
+    
+    # 验证期望的爬虫是否已注册
+    expected_spider = 'of_week_standalone'
+    if expected_spider in spider_names:
+        print(f"✅ 成功: 爬虫 '{expected_spider}' 已自动注册")
+        return True
+    else:
+        print(f"❌ 失败: 爬虫 '{expected_spider}' 未找到")
+        return False
+
+
+def test_crawler_process_with_explicit_spider_modules():
+    """测试显式传递spider_modules参数的功能"""
+    print("\n测试显式传递spider_modules参数的功能...")
+    
+    # 显式传递spider_modules参数
+    spider_modules = ['ofweek_standalone.spiders']
+    process = CrawlerProcess(spider_modules=spider_modules)
+    
+    # 检查是否注册了爬虫
+    spider_names = process.get_spider_names()
+    print(f"已注册的爬虫: {spider_names}")
+    
+    # 验证期望的爬虫是否已注册
+    expected_spider = 'of_week_standalone'
+    if expected_spider in spider_names:
+        print(f"✅ 成功: 爬虫 '{expected_spider}' 已注册")
+        return True
+    else:
+        print(f"❌ 失败: 爬虫 '{expected_spider}' 未找到")
+        return False
+
+
+if __name__ == '__main__':
+    print("开始测试SPIDER_MODULES配置功能...\n")
+    
+    # 测试自动发现功能
+    success1 = test_spider_modules_auto_discovery()
+    
+    # 测试显式传递参数功能
+    success2 = test_crawler_process_with_explicit_spider_modules()
+    
+    if success1 and success2:
+        print("\n🎉 所有测试通过!")
+        sys.exit(0)
+    else:
+        print("\n❌ 部分测试失败!")
         sys.exit(1)

tests/test_spiders/test_spider.py

@@ -1,10 +1,10 @@
-# -*- coding: utf-8 -*-
-
-from crawlo.spider import Spider
-
-
-class TestSpider(Spider):
-    name = 'test_spider'
-    
-    def parse(self, response):
+# -*- coding: utf-8 -*-
+
+from crawlo.spider import Spider
+
+
+class TestSpider(Spider):
+    name = 'test_spider'
+    
+    def parse(self, response):
         pass