crawlo 1.4.6__py3-none-any.whl → 1.4.7__py3-none-any.whl

tests/dynamic_loading_test.py

@@ -1,105 +1,105 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-动态加载功能测试示例
-==================
-测试新添加的动态加载功能是否符合预期
-"""
-import asyncio
-from crawlo import Spider, Request
-
-
-class TabControlTestSpider(Spider):
-    """标签页控制测试爬虫"""
-    name = 'tab_control_test_spider'
-    
-    custom_settings = {
-        'DOWNLOADER_TYPE': 'playwright',
-        'PLAYWRIGHT_BROWSER_TYPE': 'chromium',
-        'PLAYWRIGHT_HEADLESS': True,
-        'PLAYWRIGHT_SINGLE_BROWSER_MODE': True,
-        'PLAYWRIGHT_MAX_PAGES_PER_BROWSER': 3,  # 限制最大页面数为3
-    }
-    
-    def start_requests(self):
-        # 测试多个URL，验证标签页控制功能
-        urls = [
-            'https://httpbin.org/html',  # 简单HTML页面
-            'https://httpbin.org/json',  # JSON页面
-            'https://httpbin.org/xml',   # XML页面
-            'https://httpbin.org/robots.txt',  # 文本页面
-            'https://httpbin.org/headers',  # 请求头页面
-        ]
-        
-        for url in urls:
-            yield Request(url=url, callback=self.parse)
-    
-    def parse(self, response):
-        """解析响应"""
-        print(f"成功下载页面: {response.url}")
-        print(f"响应状态码: {response.status_code}")
-        print(f"响应内容长度: {len(response.text)}")
-        yield {'url': response.url, 'status': response.status_code}
-
-
-class PaginationTestSpider(Spider):
-    """翻页功能测试爬虫"""
-    name = 'pagination_test_spider'
-    
-    custom_settings = {
-        'DOWNLOADER_TYPE': 'playwright',
-        'PLAYWRIGHT_BROWSER_TYPE': 'chromium',
-        'PLAYWRIGHT_HEADLESS': True,
-    }
-    
-    def start_requests(self):
-        # 测试带有翻页操作的请求
-        request = Request(
-            url='https://httpbin.org/html',
-            callback=self.parse_with_pagination
-        ).set_dynamic_loader(
-            True,
-            {
-                "pagination_actions": [
-                    # 模拟鼠标滑动
-                    {
-                        "type": "scroll",
-                        "params": {
-                            "count": 2,
-                            "distance": 300,
-                            "delay": 500
-                        }
-                    }
-                ]
-            }
-        )
-        yield request
-    
-    def parse_with_pagination(self, response):
-        """解析带翻页操作的响应"""
-        print(f"成功下载页面: {response.url}")
-        yield {'url': response.url, 'status': response.status_code}
-
-
-# 运行测试
-async def run_tests():
-    """运行测试"""
-    print("=== 动态加载功能测试 ===\n")
-    
-    # 测试1: 标签页控制功能
-    print("1. 测试标签页控制功能:")
-    print("   配置最大页面数为3，请求5个URL")
-    print("   预期结果: 正常下载所有页面，但只创建3个标签页\n")
-    
-    # 测试2: 翻页功能
-    print("2. 测试翻页功能:")
-    print("   配置鼠标滑动翻页操作")
-    print("   预期结果: 页面加载完成后执行滑动操作\n")
-    
-    print("请运行以下命令来执行测试:")
-    print("crawlo crawl tab_control_test_spider")
-    print("crawlo crawl pagination_test_spider")
-
-
-if __name__ == '__main__':
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+动态加载功能测试示例
+==================
+测试新添加的动态加载功能是否符合预期
+"""
+import asyncio
+from crawlo import Spider, Request
+
+
+class TabControlTestSpider(Spider):
+    """标签页控制测试爬虫"""
+    name = 'tab_control_test_spider'
+    
+    custom_settings = {
+        'DOWNLOADER_TYPE': 'playwright',
+        'PLAYWRIGHT_BROWSER_TYPE': 'chromium',
+        'PLAYWRIGHT_HEADLESS': True,
+        'PLAYWRIGHT_SINGLE_BROWSER_MODE': True,
+        'PLAYWRIGHT_MAX_PAGES_PER_BROWSER': 3,  # 限制最大页面数为3
+    }
+    
+    def start_requests(self):
+        # 测试多个URL，验证标签页控制功能
+        urls = [
+            'https://httpbin.org/html',  # 简单HTML页面
+            'https://httpbin.org/json',  # JSON页面
+            'https://httpbin.org/xml',   # XML页面
+            'https://httpbin.org/robots.txt',  # 文本页面
+            'https://httpbin.org/headers',  # 请求头页面
+        ]
+        
+        for url in urls:
+            yield Request(url=url, callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功下载页面: {response.url}")
+        print(f"响应状态码: {response.status_code}")
+        print(f"响应内容长度: {len(response.text)}")
+        yield {'url': response.url, 'status': response.status_code}
+
+
+class PaginationTestSpider(Spider):
+    """翻页功能测试爬虫"""
+    name = 'pagination_test_spider'
+    
+    custom_settings = {
+        'DOWNLOADER_TYPE': 'playwright',
+        'PLAYWRIGHT_BROWSER_TYPE': 'chromium',
+        'PLAYWRIGHT_HEADLESS': True,
+    }
+    
+    def start_requests(self):
+        # 测试带有翻页操作的请求
+        request = Request(
+            url='https://httpbin.org/html',
+            callback=self.parse_with_pagination
+        ).set_dynamic_loader(
+            True,
+            {
+                "pagination_actions": [
+                    # 模拟鼠标滑动
+                    {
+                        "type": "scroll",
+                        "params": {
+                            "count": 2,
+                            "distance": 300,
+                            "delay": 500
+                        }
+                    }
+                ]
+            }
+        )
+        yield request
+    
+    def parse_with_pagination(self, response):
+        """解析带翻页操作的响应"""
+        print(f"成功下载页面: {response.url}")
+        yield {'url': response.url, 'status': response.status_code}
+
+
+# 运行测试
+async def run_tests():
+    """运行测试"""
+    print("=== 动态加载功能测试 ===\n")
+    
+    # 测试1: 标签页控制功能
+    print("1. 测试标签页控制功能:")
+    print("   配置最大页面数为3，请求5个URL")
+    print("   预期结果: 正常下载所有页面，但只创建3个标签页\n")
+    
+    # 测试2: 翻页功能
+    print("2. 测试翻页功能:")
+    print("   配置鼠标滑动翻页操作")
+    print("   预期结果: 页面加载完成后执行滑动操作\n")
+    
+    print("请运行以下命令来执行测试:")
+    print("crawlo crawl tab_control_test_spider")
+    print("crawlo crawl pagination_test_spider")
+
+
+if __name__ == '__main__':
     asyncio.run(run_tests())

tests/error_handling_example.py

@@ -1,172 +1,172 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-错误处理使用示例
-展示如何在实际项目中使用增强版错误处理工具
-"""
-import sys
-import os
-import asyncio
-import time
-from typing import Optional
-
-# 添加项目根目录到 Python 路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.utils.enhanced_error_handler import (
-    EnhancedErrorHandler, 
-    ErrorContext, 
-    DetailedException, 
-    handle_exception
-)
-
-
-# 创建错误处理器实例
-error_handler = EnhancedErrorHandler("example_module")
-
-
-class DatabaseConnectionError(DetailedException):
-    """数据库连接错误"""
-    pass
-
-
-class NetworkTimeoutError(DetailedException):
-    """网络超时错误"""
-    pass
-
-
-@handle_exception(context="数据库连接", module="database", function="connect_to_db", error_code="DB001")
-def connect_to_db(host: str, port: int) -> bool:
-    """模拟数据库连接"""
-    print(f"正在连接数据库 {host}:{port}...")
-    
-    # 模拟连接失败
-    if host == "invalid.host":
-        raise DatabaseConnectionError(
-            "无法连接到数据库服务器",
-            context=ErrorContext(context="数据库连接失败", module="database", function="connect_to_db"),
-            error_code="DB001",
-            host=host,
-            port=port
-        )
-    
-    # 模拟连接成功
-    print("数据库连接成功")
-    return True
-
-
-@error_handler.retry_on_failure(max_retries=3, delay=0.5, backoff_factor=2.0)
-async def fetch_data_from_api(url: str) -> dict:
-    """模拟从API获取数据（带重试机制）"""
-    print(f"正在从 {url} 获取数据...")
-    
-    # 模拟网络问题
-    if url == "https://slow.api.com/data":
-        time.sleep(2)  # 模拟慢速响应
-        raise NetworkTimeoutError(
-            "API响应超时",
-            context=ErrorContext(context="API调用超时", module="api", function="fetch_data_from_api"),
-            error_code="API001",
-            url=url,
-            timeout=1
-        )
-    
-    if url == "https://error.api.com/data":
-        raise NetworkTimeoutError(
-            "API服务器错误",
-            context=ErrorContext(context="API服务器错误", module="api", function="fetch_data_from_api"),
-            error_code="API002",
-            url=url,
-            status_code=500
-        )
-    
-    # 模拟成功响应
-    print("API数据获取成功")
-    return {"data": "sample data", "status": "success"}
-
-
-def process_data(data: dict) -> Optional[str]:
-    """处理数据"""
-    try:
-        print("正在处理数据...")
-        
-        # 模拟处理错误
-        if not data:
-            raise ValueError("数据为空")
-        
-        if "error" in data:
-            raise RuntimeError(f"数据处理失败: {data['error']}")
-        
-        # 模拟处理成功
-        result = f"处理完成: {data.get('data', 'no data')}"
-        print(f"{result}")
-        return result
-        
-    except Exception as e:
-        # 使用错误处理器处理异常
-        context = ErrorContext(context="数据处理", module="data_processor", function="process_data")
-        error_handler.handle_error(e, context=context, raise_error=False)
-        return None
-
-
-async def main():
-    """主函数"""
-    print("错误处理使用示例")
-    print("=" * 50)
-    
-    # 1. 测试数据库连接（成功情况）
-    print("1. 测试数据库连接（成功情况）")
-    try:
-        connect_to_db("localhost", 5432)
-    except Exception as e:
-        print(f"意外错误: {e}")
-    print()
-    
-    # 2. 测试数据库连接（失败情况）
-    print("2. 测试数据库连接（失败情况）")
-    try:
-        connect_to_db("invalid.host", 5432)
-    except Exception as e:
-        print(f"预期的数据库连接错误: {e}")
-    print()
-    
-    # 3. 测试API调用（成功情况）
-    print("3. 测试API调用（成功情况）")
-    try:
-        data = await fetch_data_from_api("https://api.com/data")
-        process_data(data)
-    except Exception as e:
-        print(f"API调用错误: {e}")
-    print()
-    
-    # 4. 测试API调用（失败情况，带重试）
-    print("4. 测试API调用（失败情况，带重试）")
-    try:
-        data = await fetch_data_from_api("https://error.api.com/data")
-        process_data(data)
-    except Exception as e:
-        print(f"API调用错误（重试后仍然失败）: {e}")
-    print()
-    
-    # 5. 测试数据处理（失败情况）
-    print("5. 测试数据处理（失败情况）")
-    process_data(None)  # 空数据
-    process_data({"error": "invalid format"})  # 错误数据
-    print()
-    
-    # 6. 查看错误历史
-    print("6. 错误历史记录")
-    history = error_handler.get_error_history()
-    print(f"共记录 {len(history)} 个错误:")
-    for i, record in enumerate(history, 1):
-        print(f"  {i}. {record['exception_type']}: {record['message']}")
-        if record['context']:
-            print(f"     上下文: {record['context']}")
-    print()
-    
-    print("=" * 50)
-    print("示例运行完成")
-
-
-if __name__ == "__main__":
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+错误处理使用示例
+展示如何在实际项目中使用增强版错误处理工具
+"""
+import sys
+import os
+import asyncio
+import time
+from typing import Optional
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.enhanced_error_handler import (
+    EnhancedErrorHandler, 
+    ErrorContext, 
+    DetailedException, 
+    handle_exception
+)
+
+
+# 创建错误处理器实例
+error_handler = EnhancedErrorHandler("example_module")
+
+
+class DatabaseConnectionError(DetailedException):
+    """数据库连接错误"""
+    pass
+
+
+class NetworkTimeoutError(DetailedException):
+    """网络超时错误"""
+    pass
+
+
+@handle_exception(context="数据库连接", module="database", function="connect_to_db", error_code="DB001")
+def connect_to_db(host: str, port: int) -> bool:
+    """模拟数据库连接"""
+    print(f"正在连接数据库 {host}:{port}...")
+    
+    # 模拟连接失败
+    if host == "invalid.host":
+        raise DatabaseConnectionError(
+            "无法连接到数据库服务器",
+            context=ErrorContext(context="数据库连接失败", module="database", function="connect_to_db"),
+            error_code="DB001",
+            host=host,
+            port=port
+        )
+    
+    # 模拟连接成功
+    print("数据库连接成功")
+    return True
+
+
+@error_handler.retry_on_failure(max_retries=3, delay=0.5, backoff_factor=2.0)
+async def fetch_data_from_api(url: str) -> dict:
+    """模拟从API获取数据（带重试机制）"""
+    print(f"正在从 {url} 获取数据...")
+    
+    # 模拟网络问题
+    if url == "https://slow.api.com/data":
+        time.sleep(2)  # 模拟慢速响应
+        raise NetworkTimeoutError(
+            "API响应超时",
+            context=ErrorContext(context="API调用超时", module="api", function="fetch_data_from_api"),
+            error_code="API001",
+            url=url,
+            timeout=1
+        )
+    
+    if url == "https://error.api.com/data":
+        raise NetworkTimeoutError(
+            "API服务器错误",
+            context=ErrorContext(context="API服务器错误", module="api", function="fetch_data_from_api"),
+            error_code="API002",
+            url=url,
+            status_code=500
+        )
+    
+    # 模拟成功响应
+    print("API数据获取成功")
+    return {"data": "sample data", "status": "success"}
+
+
+def process_data(data: dict) -> Optional[str]:
+    """处理数据"""
+    try:
+        print("正在处理数据...")
+        
+        # 模拟处理错误
+        if not data:
+            raise ValueError("数据为空")
+        
+        if "error" in data:
+            raise RuntimeError(f"数据处理失败: {data['error']}")
+        
+        # 模拟处理成功
+        result = f"处理完成: {data.get('data', 'no data')}"
+        print(f"{result}")
+        return result
+        
+    except Exception as e:
+        # 使用错误处理器处理异常
+        context = ErrorContext(context="数据处理", module="data_processor", function="process_data")
+        error_handler.handle_error(e, context=context, raise_error=False)
+        return None
+
+
+async def main():
+    """主函数"""
+    print("错误处理使用示例")
+    print("=" * 50)
+    
+    # 1. 测试数据库连接（成功情况）
+    print("1. 测试数据库连接（成功情况）")
+    try:
+        connect_to_db("localhost", 5432)
+    except Exception as e:
+        print(f"意外错误: {e}")
+    print()
+    
+    # 2. 测试数据库连接（失败情况）
+    print("2. 测试数据库连接（失败情况）")
+    try:
+        connect_to_db("invalid.host", 5432)
+    except Exception as e:
+        print(f"预期的数据库连接错误: {e}")
+    print()
+    
+    # 3. 测试API调用（成功情况）
+    print("3. 测试API调用（成功情况）")
+    try:
+        data = await fetch_data_from_api("https://api.com/data")
+        process_data(data)
+    except Exception as e:
+        print(f"API调用错误: {e}")
+    print()
+    
+    # 4. 测试API调用（失败情况，带重试）
+    print("4. 测试API调用（失败情况，带重试）")
+    try:
+        data = await fetch_data_from_api("https://error.api.com/data")
+        process_data(data)
+    except Exception as e:
+        print(f"API调用错误（重试后仍然失败）: {e}")
+    print()
+    
+    # 5. 测试数据处理（失败情况）
+    print("5. 测试数据处理（失败情况）")
+    process_data(None)  # 空数据
+    process_data({"error": "invalid format"})  # 错误数据
+    print()
+    
+    # 6. 查看错误历史
+    print("6. 错误历史记录")
+    history = error_handler.get_error_history()
+    print(f"共记录 {len(history)} 个错误:")
+    for i, record in enumerate(history, 1):
+        print(f"  {i}. {record['exception_type']}: {record['message']}")
+        if record['context']:
+            print(f"     上下文: {record['context']}")
+    print()
+    
+    print("=" * 50)
+    print("示例运行完成")
+
+
+if __name__ == "__main__":
     asyncio.run(main())