crawlo 1.1.4__py3-none-any.whl → 1.1.5__py3-none-any.whl

tests/env_config_example.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+环境变量配置工具使用示例
+展示如何在 Crawlo 项目中正确使用环境变量配置工具
+"""
+import os
+import sys
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.env_config import get_env_var, get_redis_config, get_runtime_config
+from crawlo.settings.setting_manager import SettingManager
+from crawlo.settings import default_settings
+
+
+def example_basic_usage():
+    """基本使用示例"""
+    print("=== 基本环境变量使用示例 ===")
+    
+    # 获取字符串环境变量
+    project_name = get_env_var('PROJECT_NAME', 'my_crawler', str)
+    print(f"项目名称: {project_name}")
+    
+    # 获取整数环境变量
+    concurrency = get_env_var('CONCURRENCY', 8, int)
+    print(f"并发数: {concurrency}")
+    
+    # 获取布尔环境变量
+    debug_mode = get_env_var('DEBUG_MODE', False, bool)
+    print(f"调试模式: {debug_mode}")
+
+
+def example_redis_config():
+    """Redis配置示例"""
+    print("\n=== Redis配置示例 ===")
+    
+    # 获取Redis配置
+    redis_config = get_redis_config()
+    print(f"Redis主机: {redis_config['REDIS_HOST']}")
+    print(f"Redis端口: {redis_config['REDIS_PORT']}")
+    print(f"Redis密码: {'*' * len(redis_config['REDIS_PASSWORD']) if redis_config['REDIS_PASSWORD'] else '无'}")
+    print(f"Redis数据库: {redis_config['REDIS_DB']}")
+    
+    # 生成Redis URL
+    if redis_config['REDIS_PASSWORD']:
+        redis_url = f"redis://:{redis_config['REDIS_PASSWORD']}@{redis_config['REDIS_HOST']}:{redis_config['REDIS_PORT']}/{redis_config['REDIS_DB']}"
+    else:
+        redis_url = f"redis://{redis_config['REDIS_HOST']}:{redis_config['REDIS_PORT']}/{redis_config['REDIS_DB']}"
+    
+    print(f"Redis URL: {redis_url}")
+
+
+def example_runtime_config():
+    """运行时配置示例"""
+    print("\n=== 运行时配置示例 ===")
+    
+    # 获取运行时配置
+    runtime_config = get_runtime_config()
+    print(f"运行模式: {runtime_config['CRAWLO_MODE']}")
+    print(f"项目名称: {runtime_config['PROJECT_NAME']}")
+    print(f"并发数: {runtime_config['CONCURRENCY']}")
+
+
+def example_settings_integration():
+    """与Settings集成示例"""
+    print("\n=== 与Settings集成示例 ===")
+    
+    # 创建设置管理器
+    settings = SettingManager()
+    
+    # 更新Redis相关设置
+    redis_config = get_redis_config()
+    settings.set('REDIS_HOST', redis_config['REDIS_HOST'])
+    settings.set('REDIS_PORT', redis_config['REDIS_PORT'])
+    settings.set('REDIS_PASSWORD', redis_config['REDIS_PASSWORD'])
+    settings.set('REDIS_DB', redis_config['REDIS_DB'])
+    
+    # 更新运行时设置
+    runtime_config = get_runtime_config()
+    settings.set('PROJECT_NAME', runtime_config['PROJECT_NAME'])
+    settings.set('RUN_MODE', runtime_config['CRAWLO_MODE'])
+    settings.set('CONCURRENCY', runtime_config['CONCURRENCY'])
+    
+    # 显示一些关键设置
+    print(f"项目名称: {settings.get('PROJECT_NAME')}")
+    print(f"运行模式: {settings.get('RUN_MODE')}")
+    print(f"并发数: {settings.get_int('CONCURRENCY')}")
+    print(f"Redis主机: {settings.get('REDIS_HOST')}")
+    print(f"Redis端口: {settings.get_int('REDIS_PORT')}")
+
+
+def example_env_setup():
+    """环境变量设置示例"""
+    print("\n=== 环境变量设置示例 ===")
+    print("在命令行中设置环境变量的示例:")
+    print("  Windows (PowerShell):")
+    print("    $env:PROJECT_NAME = \"my_distributed_crawler\"")
+    print("    $env:REDIS_HOST = \"redis.example.com\"")
+    print("    $env:REDIS_PORT = \"6380\"")
+    print("    $env:CONCURRENCY = \"16\"")
+    print("    $env:CRAWLO_MODE = \"distributed\"")
+    print()
+    print("  Linux/macOS:")
+    print("    export PROJECT_NAME=\"my_distributed_crawler\"")
+    print("    export REDIS_HOST=\"redis.example.com\"")
+    print("    export REDIS_PORT=\"6380\"")
+    print("    export CONCURRENCY=\"16\"")
+    print("    export CRAWLO_MODE=\"distributed\"")
+
+
+if __name__ == '__main__':
+    # 设置一些测试环境变量
+    os.environ['PROJECT_NAME'] = 'test_crawler'
+    os.environ['CONCURRENCY'] = '12'
+    os.environ['DEBUG_MODE'] = 'true'
+    os.environ['REDIS_HOST'] = 'redis.test.com'
+    os.environ['REDIS_PORT'] = '6380'
+    os.environ['REDIS_PASSWORD'] = 'test_password'
+    os.environ['CRAWLO_MODE'] = 'distributed'
+    
+    # 运行示例
+    example_basic_usage()
+    example_redis_config()
+    example_runtime_config()
+    example_settings_integration()
+    example_env_setup()
+    
+    # 清理测试环境变量
+    for var in ['PROJECT_NAME', 'CONCURRENCY', 'DEBUG_MODE', 'REDIS_HOST', 
+                'REDIS_PORT', 'REDIS_PASSWORD', 'CRAWLO_MODE']:
+        if var in os.environ:
+            del os.environ[var]

tests/error_handling_example.py

@@ -0,0 +1,172 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+错误处理使用示例
+展示如何在实际项目中使用增强版错误处理工具
+"""
+import sys
+import os
+import asyncio
+import time
+from typing import Optional
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.enhanced_error_handler import (
+    EnhancedErrorHandler, 
+    ErrorContext, 
+    DetailedException, 
+    handle_exception
+)
+
+
+# 创建错误处理器实例
+error_handler = EnhancedErrorHandler("example_module")
+
+
+class DatabaseConnectionError(DetailedException):
+    """数据库连接错误"""
+    pass
+
+
+class NetworkTimeoutError(DetailedException):
+    """网络超时错误"""
+    pass
+
+
+@handle_exception(context="数据库连接", module="database", function="connect_to_db", error_code="DB001")
+def connect_to_db(host: str, port: int) -> bool:
+    """模拟数据库连接"""
+    print(f"正在连接数据库 {host}:{port}...")
+    
+    # 模拟连接失败
+    if host == "invalid.host":
+        raise DatabaseConnectionError(
+            "无法连接到数据库服务器",
+            context=ErrorContext(context="数据库连接失败", module="database", function="connect_to_db"),
+            error_code="DB001",
+            host=host,
+            port=port
+        )
+    
+    # 模拟连接成功
+    print("✅ 数据库连接成功")
+    return True
+
+
+@error_handler.retry_on_failure(max_retries=3, delay=0.5, backoff_factor=2.0)
+async def fetch_data_from_api(url: str) -> dict:
+    """模拟从API获取数据（带重试机制）"""
+    print(f"正在从 {url} 获取数据...")
+    
+    # 模拟网络问题
+    if url == "https://slow.api.com/data":
+        time.sleep(2)  # 模拟慢速响应
+        raise NetworkTimeoutError(
+            "API响应超时",
+            context=ErrorContext(context="API调用超时", module="api", function="fetch_data_from_api"),
+            error_code="API001",
+            url=url,
+            timeout=1
+        )
+    
+    if url == "https://error.api.com/data":
+        raise NetworkTimeoutError(
+            "API服务器错误",
+            context=ErrorContext(context="API服务器错误", module="api", function="fetch_data_from_api"),
+            error_code="API002",
+            url=url,
+            status_code=500
+        )
+    
+    # 模拟成功响应
+    print("✅ API数据获取成功")
+    return {"data": "sample data", "status": "success"}
+
+
+def process_data(data: dict) -> Optional[str]:
+    """处理数据"""
+    try:
+        print("正在处理数据...")
+        
+        # 模拟处理错误
+        if not data:
+            raise ValueError("数据为空")
+        
+        if "error" in data:
+            raise RuntimeError(f"数据处理失败: {data['error']}")
+        
+        # 模拟处理成功
+        result = f"处理完成: {data.get('data', 'no data')}"
+        print(f"✅ {result}")
+        return result
+        
+    except Exception as e:
+        # 使用错误处理器处理异常
+        context = ErrorContext(context="数据处理", module="data_processor", function="process_data")
+        error_handler.handle_error(e, context=context, raise_error=False)
+        return None
+
+
+async def main():
+    """主函数"""
+    print("🚀 错误处理使用示例")
+    print("=" * 50)
+    
+    # 1. 测试数据库连接（成功情况）
+    print("1. 测试数据库连接（成功情况）")
+    try:
+        connect_to_db("localhost", 5432)
+    except Exception as e:
+        print(f"❌ 意外错误: {e}")
+    print()
+    
+    # 2. 测试数据库连接（失败情况）
+    print("2. 测试数据库连接（失败情况）")
+    try:
+        connect_to_db("invalid.host", 5432)
+    except Exception as e:
+        print(f"❌ 预期的数据库连接错误: {e}")
+    print()
+    
+    # 3. 测试API调用（成功情况）
+    print("3. 测试API调用（成功情况）")
+    try:
+        data = await fetch_data_from_api("https://api.com/data")
+        process_data(data)
+    except Exception as e:
+        print(f"❌ API调用错误: {e}")
+    print()
+    
+    # 4. 测试API调用（失败情况，带重试）
+    print("4. 测试API调用（失败情况，带重试）")
+    try:
+        data = await fetch_data_from_api("https://error.api.com/data")
+        process_data(data)
+    except Exception as e:
+        print(f"❌ API调用错误（重试后仍然失败）: {e}")
+    print()
+    
+    # 5. 测试数据处理（失败情况）
+    print("5. 测试数据处理（失败情况）")
+    process_data(None)  # 空数据
+    process_data({"error": "invalid format"})  # 错误数据
+    print()
+    
+    # 6. 查看错误历史
+    print("6. 错误历史记录")
+    history = error_handler.get_error_history()
+    print(f"共记录 {len(history)} 个错误:")
+    for i, record in enumerate(history, 1):
+        print(f"  {i}. {record['exception_type']}: {record['message']}")
+        if record['context']:
+            print(f"     上下文: {record['context']}")
+    print()
+    
+    print("=" * 50)
+    print("🎉 示例运行完成")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/redis_key_validation_demo.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Redis Key 验证演示脚本
+演示如何使用Redis Key验证工具
+"""
+import sys
+import os
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.redis_key_validator import (
+    RedisKeyValidator, 
+    validate_redis_key_naming, 
+    validate_multiple_redis_keys,
+    get_redis_key_info,
+    print_validation_report
+)
+
+
+def demonstrate_redis_key_validation():
+    """演示Redis Key验证功能"""
+    print("=== Redis Key 验证功能演示 ===\n")
+    
+    # 1. 验证单个有效的Redis Key
+    print("1. 验证单个有效的Redis Key:")
+    valid_keys = [
+        "crawlo:books_distributed:filter:fingerprint",
+        "crawlo:books_distributed:queue:requests",
+        "crawlo:books_distributed:queue:processing",
+        "crawlo:books_distributed:queue:failed",
+        "crawlo:books_distributed:item:fingerprint"
+    ]
+    
+    for key in valid_keys:
+        is_valid = validate_redis_key_naming(key, "books_distributed")
+        print(f"  ✅ {key} - {'通过' if is_valid else '失败'}")
+    
+    print()
+    
+    # 2. 验证无效的Redis Key
+    print("2. 验证无效的Redis Key:")
+    invalid_keys = [
+        "invalid_format",  # 不以crawlo开头
+        "crawlo:books_distributed",  # 部分缺失
+        "crawlo:books_distributed:invalid_component:fingerprint",  # 无效组件
+        "crawlo:books_distributed:queue:invalid_subcomponent",  # 无效子组件
+        "crawlo:wrong_project:filter:fingerprint"  # 项目名称不匹配
+    ]
+    
+    for key in invalid_keys:
+        is_valid = validate_redis_key_naming(key, "books_distributed")
+        print(f"  ❌ {key} - {'通过' if is_valid else '失败'}")
+    
+    print()
+    
+    # 3. 批量验证Redis Key
+    print("3. 批量验证Redis Key:")
+    all_keys = valid_keys + invalid_keys
+    is_valid, invalid_keys_list = validate_multiple_redis_keys(all_keys, "books_distributed")
+    
+    print(f"  批量验证结果: {'全部通过' if is_valid else '存在无效Key'}")
+    if not is_valid:
+        print("  无效的Key:")
+        for key in invalid_keys_list:
+            print(f"    - {key}")
+    
+    print()
+    
+    # 4. 获取Redis Key信息
+    print("4. 获取Redis Key信息:")
+    sample_keys = [
+        "crawlo:api_data_collection:filter:fingerprint",
+        "crawlo:api_data_collection:queue:requests",
+        "crawlo:api_data_collection:item:fingerprint"
+    ]
+    
+    for key in sample_keys:
+        info = get_redis_key_info(key)
+        print(f"  Key: {key}")
+        if info['valid']:
+            print(f"    框架: {info['framework']}")
+            print(f"    项目: {info['project']}")
+            print(f"    组件: {info['component']}")
+            if 'sub_component' in info:
+                print(f"    子组件: {info['sub_component']}")
+        else:
+            print(f"    错误: {info.get('error', '无效')}")
+        print()
+    
+    # 5. 打印验证报告
+    print("5. 打印验证报告:")
+    print_validation_report(all_keys, "books_distributed")
+
+
+def demonstrate_advanced_validation():
+    """演示高级验证功能"""
+    print("\n=== 高级验证功能演示 ===\n")
+    
+    validator = RedisKeyValidator()
+    
+    # 1. 不指定项目名称的验证
+    print("1. 不指定项目名称的验证:")
+    key = "crawlo:any_project:filter:fingerprint"
+    is_valid = validator.validate_key_naming(key)  # 不指定项目名称
+    print(f"  {key} - {'通过' if is_valid else '失败'}")
+    
+    # 2. 验证不同项目的Key
+    print("\n2. 验证不同项目的Key:")
+    project_keys = {
+        "books_distributed": [
+            "crawlo:books_distributed:filter:fingerprint",
+            "crawlo:books_distributed:queue:requests"
+        ],
+        "api_data_collection": [
+            "crawlo:api_data_collection:filter:fingerprint",
+            "crawlo:api_data_collection:queue:requests"
+        ]
+    }
+    
+    for project_name, keys in project_keys.items():
+        print(f"  项目: {project_name}")
+        for key in keys:
+            is_valid = validator.validate_key_naming(key, project_name)
+            print(f"    {key} - {'通过' if is_valid else '失败'}")
+
+
+if __name__ == "__main__":
+    demonstrate_redis_key_validation()
+    demonstrate_advanced_validation()

tests/response_improvements_example.py

@@ -0,0 +1,145 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response 改进功能使用示例
+"""
+from crawlo.network.response import Response
+
+
+def demo_response_improvements():
+    """演示Response改进功能"""
+    print("=== Response 改进功能演示 ===\n")
+    
+    # 创建一个示例HTML响应
+    html_content = """
+    <html>
+    <head>
+        <title>Crawlo框架示例页面</title>
+    </head>
+    <body>
+        <div class="container">
+            <h1>产品列表</h1>
+            <div class="product-list">
+                <div class="product-item" data-id="1">
+                    <h2>产品A</h2>
+                    <p class="price">¥99.99</p>
+                    <p class="description">这是产品A的描述信息</p>
+                    <a href="/product/1" class="details-link">查看详情</a>
+                </div>
+                <div class="product-item" data-id="2">
+                    <h2>产品B</h2>
+                    <p class="price">¥149.99</p>
+                    <p class="description">这是产品B的描述信息</p>
+                    <a href="/product/2" class="details-link">查看详情</a>
+                </div>
+                <div class="product-item" data-id="3">
+                    <h2>产品C</h2>
+                    <p class="price">¥199.99</p>
+                    <p class="description">这是产品C的描述信息</p>
+                    <a href="/product/3" class="details-link">查看详情</a>
+                </div>
+            </div>
+            <div class="pagination">
+                <a href="/page/1" class="page-link">1</a>
+                <a href="/page/2" class="page-link active">2</a>
+                <a href="/page/3" class="page-link">3</a>
+            </div>
+        </div>
+    </body>
+    </html>
+    """
+    
+    # 创建Response对象
+    response = Response(
+        url="https://example.com/products",
+        body=html_content.encode('utf-8'),
+        headers={"content-type": "text/html; charset=utf-8"}
+    )
+    
+    # 1. 演示 extract_text 方法（支持CSS和XPath）
+    print("1. 提取文本内容:")
+    title = response.extract_text('title')
+    print(f"   页面标题: {title}")
+    
+    # 使用CSS选择器提取第一个产品名称
+    first_product_name = response.extract_text('.product-item:first-child h2')
+    print(f"   第一个产品名称 (CSS): {first_product_name}")
+    
+    # 使用XPath选择器
+    first_product_name_xpath = response.extract_text('//div[@class="product-item"][1]/h2')
+    print(f"   第一个产品名称 (XPath): {first_product_name_xpath}")
+    
+    # 使用默认值处理不存在的元素
+    non_exist = response.extract_text('.non-exist', default='未找到')
+    print(f"   不存在的元素: {non_exist}")
+    
+    print()
+    
+    # 2. 演示 extract_texts 方法（提取多个元素的文本）
+    print("2. 提取多个元素的文本:")
+    # 提取所有产品名称
+    product_names = response.extract_texts('.product-item h2')
+    print(f"   所有产品名称: {product_names}")
+    
+    # 提取所有价格
+    prices = response.extract_texts('.price')
+    print(f"   所有价格: {prices}")
+    
+    # 使用XPath提取所有产品名称
+    product_names_xpath = response.extract_texts('//div[@class="product-item"]/h2')
+    print(f"   所有产品名称 (XPath): {product_names_xpath}")
+    
+    print()
+    
+    # 3. 演示 extract_attr 方法（提取元素属性）
+    print("3. 提取元素属性:")
+    # 提取第一个产品项的data-id属性
+    first_product_id = response.extract_attr('.product-item', 'data-id')
+    print(f"   第一个产品ID: {first_product_id}")
+    
+    # 提取详情链接的href属性
+    first_detail_link = response.extract_attr('.details-link', 'href')
+    print(f"   第一个详情链接: {first_detail_link}")
+    
+    # 提取不存在属性的默认值
+    non_exist_attr = response.extract_attr('.product-item', 'non-exist', default='默认值')
+    print(f"   不存在的属性: {non_exist_attr}")
+    
+    print()
+    
+    # 4. 演示 extract_attrs 方法（提取多个元素的属性）
+    print("4. 提取多个元素的属性:")
+    # 提取所有产品项的data-id属性
+    product_ids = response.extract_attrs('.product-item', 'data-id')
+    print(f"   所有产品ID: {product_ids}")
+    
+    # 提取所有详情链接的href属性
+    detail_links = response.extract_attrs('.details-link', 'href')
+    print(f"   所有详情链接: {detail_links}")
+    
+    # 提取分页链接的href属性
+    page_links = response.extract_attrs('.page-link', 'href')
+    print(f"   分页链接: {page_links}")
+    
+    print()
+    
+    # 5. 演示复杂文本提取
+    print("5. 复杂文本提取:")
+    # 提取所有产品描述
+    descriptions = response.extract_texts('.description')
+    print(f"   所有产品描述: {descriptions}")
+    
+    print()
+    
+    # 6. 边界情况处理
+    print("6. 边界情况处理:")
+    # 空响应测试
+    empty_response = Response(url="https://example.com/empty", body=b"")
+    empty_text = empty_response.extract_text('title', default='默认标题')
+    print(f"   空响应默认值: {empty_text}")
+    
+    print("\n=== 演示完成 ===")
+
+
+if __name__ == '__main__':
+    demo_response_improvements()