crawlo 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

tests/debug_pipelines.py

@@ -1,67 +1,67 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-调试管道配置
-查看实际的管道配置合并结果
-"""
-
-import sys
-import os
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.settings.setting_manager import SettingManager
-
-def debug_pipelines():
-    """调试管道配置"""
-    print("调试管道配置合并...")
-    print("=" * 50)
-    
-    # 用户自定义配置
-    user_config = {
-        'PIPELINES': [
-            'myproject.pipelines.CustomPipeline',
-        ]
-    }
-    
-    settings = SettingManager(user_config)
-    
-    # 获取合并后的管道列表
-    pipelines = settings.get('PIPELINES')
-    
-    print("合并后的管道列表:")
-    for i, pipeline in enumerate(pipelines):
-        print(f"  {i}: {pipeline}")
-    
-    print()
-    print("默认去重管道:")
-    dedup_pipeline = settings.get('DEFAULT_DEDUP_PIPELINE')
-    print(f"  {dedup_pipeline}")
-    
-    print()
-    print("框架默认管道:")
-    default_pipelines = settings.get('PIPELINES', [])  # 直接获取PIPELINES，它已经包含了默认管道
-    # 从合并后的管道中移除去重管道，得到框架默认管道
-    if dedup_pipeline:
-        default_pipelines_without_dedup = [p for p in default_pipelines if p != dedup_pipeline]
-        for i, pipeline in enumerate(default_pipelines_without_dedup):
-            print(f"  {i}: {pipeline}")
-    else:
-        for i, pipeline in enumerate(default_pipelines):
-            print(f"  {i}: {pipeline}")
-    
-    print()
-    print("自定义管道:")
-    custom_pipelines = settings.get('PIPELINES')
-    # 从合并后的管道中移除默认管道，得到自定义管道
-    default_pipelines_list = [
-        'crawlo.pipelines.console_pipeline.ConsolePipeline',
-        'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline'
-    ]
-    custom_pipelines_list = [p for p in custom_pipelines if p not in default_pipelines_list]
-    for i, pipeline in enumerate(custom_pipelines_list):
-        print(f"  {i}: {pipeline}")
-
-if __name__ == "__main__":
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+调试管道配置
+查看实际的管道配置合并结果
+"""
+
+import sys
+import os
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.settings.setting_manager import SettingManager
+
+def debug_pipelines():
+    """调试管道配置"""
+    print("调试管道配置合并...")
+    print("=" * 50)
+    
+    # 用户自定义配置
+    user_config = {
+        'PIPELINES': [
+            'myproject.pipelines.CustomPipeline',
+        ]
+    }
+    
+    settings = SettingManager(user_config)
+    
+    # 获取合并后的管道列表
+    pipelines = settings.get('PIPELINES')
+    
+    print("合并后的管道列表:")
+    for i, pipeline in enumerate(pipelines):
+        print(f"  {i}: {pipeline}")
+    
+    print()
+    print("默认去重管道:")
+    dedup_pipeline = settings.get('DEFAULT_DEDUP_PIPELINE')
+    print(f"  {dedup_pipeline}")
+    
+    print()
+    print("框架默认管道:")
+    default_pipelines = settings.get('PIPELINES', [])  # 直接获取PIPELINES，它已经包含了默认管道
+    # 从合并后的管道中移除去重管道，得到框架默认管道
+    if dedup_pipeline:
+        default_pipelines_without_dedup = [p for p in default_pipelines if p != dedup_pipeline]
+        for i, pipeline in enumerate(default_pipelines_without_dedup):
+            print(f"  {i}: {pipeline}")
+    else:
+        for i, pipeline in enumerate(default_pipelines):
+            print(f"  {i}: {pipeline}")
+    
+    print()
+    print("自定义管道:")
+    custom_pipelines = settings.get('PIPELINES')
+    # 从合并后的管道中移除默认管道，得到自定义管道
+    default_pipelines_list = [
+        'crawlo.pipelines.console_pipeline.ConsolePipeline',
+        'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline'
+    ]
+    custom_pipelines_list = [p for p in custom_pipelines if p not in default_pipelines_list]
+    for i, pipeline in enumerate(custom_pipelines_list):
+        print(f"  {i}: {pipeline}")
+
+if __name__ == "__main__":
     debug_pipelines()

tests/distributed_test.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+分布式模式测试脚本
+"""
+
+import sys
+import os
+import asyncio
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class DistributedTestSpider(Spider):
+    """分布式测试爬虫"""
+    name = 'distributed_test_spider'
+    
+    def start_requests(self):
+        """发起测试请求"""
+        # 生成一些测试请求
+        for i in range(5):
+            yield Request(f'https://httpbin.org/get?page={i}', callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功获取响应: {response.url}")
+        print(f"状态码: {response.status_code}")
+        return []
+
+
+async def test_distributed_mode():
+    """测试分布式模式"""
+    print("开始测试分布式模式...")
+    
+    # 初始化框架，设置为分布式模式
+    from crawlo.initialization import initialize_framework
+    custom_settings = {
+        'RUN_MODE': 'distributed',
+        'QUEUE_TYPE': 'redis',
+        'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
+        'REDIS_HOST': '127.0.0.1',
+        'REDIS_PORT': 6379,
+        'REDIS_DB': 15,  # 使用测试数据库
+        'PROJECT_NAME': 'distributed_test'
+    }
+    settings = initialize_framework(custom_settings)
+    
+    # 创建爬虫进程
+    from crawlo.crawler import CrawlerProcess
+    process = CrawlerProcess(settings=settings)
+    
+    # 运行爬虫
+    await process.crawl(DistributedTestSpider)
+    
+    print("分布式模式测试完成！")
+
+
+def main():
+    """主函数"""
+    print("开始分布式模式测试...")
+    asyncio.run(test_distributed_mode())
+
+
+if __name__ == "__main__":
+    main()

tests/distributed_test_debug.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+分布式模式调试测试脚本
+"""
+
+import sys
+import os
+import asyncio
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class DistributedTestSpider(Spider):
+    """分布式测试爬虫"""
+    name = 'distributed_test_spider'
+    
+    def start_requests(self):
+        """发起测试请求"""
+        # 生成一些测试请求
+        for i in range(3):
+            yield Request(f'https://httpbin.org/get?page={i}', callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功获取响应: {response.url}")
+        print(f"状态码: {response.status_code}")
+        return []
+
+
+async def test_distributed_mode():
+    """测试分布式模式"""
+    print("开始测试分布式模式...")
+    
+    # 初始化框架，设置为分布式模式
+    from crawlo.initialization import initialize_framework
+    custom_settings = {
+        'RUN_MODE': 'distributed',
+        'QUEUE_TYPE': 'redis',
+        'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
+        'REDIS_HOST': '127.0.0.1',
+        'REDIS_PORT': 6379,
+        'REDIS_DB': 15,  # 使用测试数据库
+        'PROJECT_NAME': 'distributed_test'
+    }
+    
+    print("自定义配置:")
+    for key, value in custom_settings.items():
+        print(f"  {key}: {value}")
+    
+    settings = initialize_framework(custom_settings)
+    
+    print("初始化后的配置:")
+    print(f"  RUN_MODE: {settings.get('RUN_MODE')}")
+    print(f"  QUEUE_TYPE: {settings.get('QUEUE_TYPE')}")
+    print(f"  FILTER_CLASS: {settings.get('FILTER_CLASS')}")
+    
+    # 创建爬虫进程
+    from crawlo.crawler import CrawlerProcess
+    process = CrawlerProcess(settings=settings)
+    
+    # 运行爬虫
+    await process.crawl(DistributedTestSpider)
+    
+    print("分布式模式测试完成！")
+
+
+def main():
+    """主函数"""
+    print("开始分布式模式调试测试...")
+    asyncio.run(test_distributed_mode())
+
+
+if __name__ == "__main__":
+    main()