crawlo 1.2.3__py3-none-any.whl → 1.2.5__py3-none-any.whl

examples/download_delay_middleware_example.py

@@ -1,160 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-DownloadDelayMiddleware 使用示例
-展示如何配置和使用下载延迟中间件
-"""
-
-import asyncio
-from crawlo.settings.setting_manager import SettingManager
-from crawlo.middleware.download_delay import DownloadDelayMiddleware
-
-
-def example_with_fixed_delay():
-    """固定延迟示例"""
-    print("=== 固定延迟示例 ===")
-    
-    # 创建设置管理器
-    settings = SettingManager()
-    
-    # 配置固定延迟
-    settings.set('DOWNLOAD_DELAY', 2.0)  # 2秒固定延迟
-    settings.set('RANDOMNESS', False)    # 不启用随机延迟
-    settings.set('LOG_LEVEL', 'INFO')   # 设置日志级别
-    
-    # 创建爬虫模拟对象
-    class MockCrawler:
-        def __init__(self, settings):
-            self.settings = settings
-            self.stats = None
-    
-    crawler = MockCrawler(settings)
-    
-    # 创建中间件实例
-    middleware = DownloadDelayMiddleware.create_instance(crawler)
-    
-    print(f"延迟设置: {middleware.delay}秒")
-    print(f"是否启用随机延迟: {middleware.randomness}")
-    print("中间件创建成功！")
-
-
-def example_with_random_delay():
-    """随机延迟示例"""
-    print("\n=== 随机延迟示例 ===")
-    
-    # 创建设置管理器
-    settings = SettingManager()
-    
-    # 配置随机延迟
-    settings.set('DOWNLOAD_DELAY', 1.0)        # 基础延迟1秒
-    settings.set('RANDOMNESS', True)           # 启用随机延迟
-    settings.set('RANDOM_RANGE', [0.5, 2.0])   # 随机范围因子
-    settings.set('LOG_LEVEL', 'INFO')         # 设置日志级别
-    
-    # 创建爬虫模拟对象
-    class MockCrawler:
-        def __init__(self, settings):
-            self.settings = settings
-            self.stats = None
-    
-    crawler = MockCrawler(settings)
-    
-    # 创建中间件实例
-    middleware = DownloadDelayMiddleware.create_instance(crawler)
-    
-    print(f"基础延迟设置: {middleware.delay}秒")
-    print(f"是否启用随机延迟: {middleware.randomness}")
-    print(f"随机范围: {middleware.floor} - {middleware.upper}")
-    print(f"实际延迟范围: {middleware.delay * middleware.floor} - {middleware.delay * middleware.upper}秒")
-    print("中间件创建成功！")
-
-
-def example_with_invalid_config():
-    """无效配置示例"""
-    print("\n=== 无效配置示例 ===")
-    
-    # 创建设置管理器
-    settings = SettingManager()
-    
-    # 配置无效的延迟（0值）
-    settings.set('DOWNLOAD_DELAY', 0)      # 无效延迟
-    settings.set('LOG_LEVEL', 'INFO')     # 设置日志级别
-    
-    # 创建爬虫模拟对象
-    class MockCrawler:
-        def __init__(self, settings):
-            self.settings = settings
-            self.stats = None
-    
-    crawler = MockCrawler(settings)
-    
-    try:
-        # 尝试创建中间件实例
-        middleware = DownloadDelayMiddleware.create_instance(crawler)
-        print("中间件创建成功！")
-    except Exception as e:
-        print(f"中间件创建失败: {e}")
-
-
-def example_with_stats():
-    """带统计信息的示例"""
-    print("\n=== 带统计信息的示例 ===")
-    
-    # 创建设置管理器
-    settings = SettingManager()
-    
-    # 配置固定延迟
-    settings.set('DOWNLOAD_DELAY', 1.0)  # 1秒固定延迟
-    settings.set('RANDOMNESS', False)    # 不启用随机延迟
-    settings.set('LOG_LEVEL', 'INFO')   # 设置日志级别
-    
-    # 创建统计收集器模拟对象
-    class MockStats:
-        def __init__(self):
-            self.stats = {}
-            
-        def inc_value(self, key, value=1):
-            if key in self.stats:
-                self.stats[key] += value
-            else:
-                self.stats[key] = value
-                
-        def __str__(self):
-            return str(self.stats)
-    
-    # 创建爬虫模拟对象
-    class MockCrawler:
-        def __init__(self, settings):
-            self.settings = settings
-            self.stats = MockStats()
-    
-    crawler = MockCrawler(settings)
-    
-    # 创建中间件实例
-    middleware = DownloadDelayMiddleware.create_instance(crawler)
-    
-    print(f"延迟设置: {middleware.delay}秒")
-    print("中间件创建成功！")
-    
-    # 模拟处理请求
-    class MockRequest:
-        pass
-        
-    class MockSpider:
-        pass
-        
-    request = MockRequest()
-    spider = MockSpider()
-    
-    # 执行请求处理
-    asyncio.run(middleware.process_request(request, spider))
-    
-    print(f"统计信息: {crawler.stats}")
-
-
-if __name__ == '__main__':
-    # 运行所有示例
-    example_with_fixed_delay()
-    example_with_random_delay()
-    example_with_invalid_config()
-    example_with_stats()

examples/httpx_settings.py

@@ -1,42 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-httpx下载器配置示例
-"""
-
-# 基础配置
-SETTINGS = {
-    # 下载器配置
-    'DOWNLOADER': 'crawlo.downloader.httpx_downloader.HttpXDownloader',
-    'DOWNLOADER_TYPE': 'httpx',
-    
-    # httpx特定配置
-    'HTTPX_HTTP2': True,
-    'HTTPX_FOLLOW_REDIRECTS': True,
-    
-    # 代理配置
-    'PROXY_ENABLED': True,
-    'PROXY_API_URL': 'http://test.proxy.api:8080/proxy/getitem/',
-    'PROXY_EXTRACTOR': 'proxy',
-    'PROXY_REFRESH_INTERVAL': 60,
-    'PROXY_POOL_SIZE': 5,
-    
-    # 通用下载配置
-    'DOWNLOAD_TIMEOUT': 30,
-    'CONNECTION_POOL_LIMIT': 100,
-    'CONNECTION_POOL_LIMIT_PER_HOST': 20,
-    'DOWNLOAD_MAXSIZE': 10 * 1024 * 1024,  # 10MB
-    'VERIFY_SSL': True,
-    
-    # 日志配置
-    'LOG_LEVEL': 'INFO',
-}
-
-def get_settings():
-    """获取配置"""
-    return SETTINGS
-
-if __name__ == "__main__":
-    print("httpx下载器配置:")
-    for key, value in SETTINGS.items():
-        print(f"  {key}: {value}")

examples/multi_downloader_proxy_example.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-Crawlo框架多下载器代理配置示例
-展示如何在Crawlo中配置不同下载器并使用代理功能
-"""
-
-# aiohttp下载器配置
-DOWNLOADER_CONFIGS = {
-    "aiohttp": {
-        'DOWNLOADER': 'crawlo.downloader.aiohttp_downloader.AioHttpDownloader',
-        'DOWNLOADER_TYPE': 'aiohttp',
-        # aiohttp特定配置
-        'AIOHTTP_AUTO_DECOMPRESS': True,
-        'AIOHTTP_FORCE_CLOSE': False,
-    },
-    
-    "httpx": {
-        'DOWNLOADER': 'crawlo.downloader.httpx_downloader.HttpXDownloader',
-        'DOWNLOADER_TYPE': 'httpx',
-        # httpx特定配置
-        'HTTPX_HTTP2': True,
-        'HTTPX_FOLLOW_REDIRECTS': True,
-    },
-    
-    "curl_cffi": {
-        'DOWNLOADER': 'crawlo.downloader.cffi_downloader.CurlCffiDownloader',
-        'DOWNLOADER_TYPE': 'curl_cffi',
-        # curl-cffi特定配置
-        'CURL_BROWSER_TYPE': 'chrome',
-    }
-}
-
-# 通用配置（适用于所有下载器）
-COMMON_SETTINGS = {
-    # 代理配置
-    'PROXY_ENABLED': True,
-    'PROXY_API_URL': 'http://test.proxy.api:8080/proxy/getitem/',
-    'PROXY_EXTRACTOR': 'proxy',
-    'PROXY_REFRESH_INTERVAL': 60,
-    'PROXY_POOL_SIZE': 5,
-    
-    # 下载器通用配置
-    'DOWNLOAD_TIMEOUT': 30,
-    'CONNECTION_POOL_LIMIT': 100,
-    'CONNECTION_POOL_LIMIT_PER_HOST': 20,
-    'DOWNLOAD_MAXSIZE': 10 * 1024 * 1024,  # 10MB
-    'VERIFY_SSL': True,
-    
-    # 日志配置
-    'LOG_LEVEL': 'INFO',
-}
-
-def get_downloader_settings(downloader_type):
-    """
-    获取指定下载器的完整配置
-    """
-    if downloader_type not in DOWNLOADER_CONFIGS:
-        raise ValueError(f"不支持的下载器类型: {downloader_type}")
-    
-    # 合并通用配置和特定下载器配置
-    settings = COMMON_SETTINGS.copy()
-    settings.update(DOWNLOADER_CONFIGS[downloader_type])
-    return settings
-
-# 使用示例
-if __name__ == "__main__":
-    print("Crawlo框架多下载器代理配置示例")
-    print("=" * 50)
-    
-    for downloader_type in DOWNLOADER_CONFIGS.keys():
-        print(f"\n{downloader_type.upper()} 下载器配置:")
-        settings = get_downloader_settings(downloader_type)
-        for key, value in settings.items():
-            print(f"  {key}: {value}")
-    
-    print("\n" + "=" * 50)
-    print("所有下载器均已适配代理中间件:")
-    print("✓ aiohttp: 通过 meta 传递代理认证信息")
-    print("✓ httpx: 直接使用代理URL")
-    print("✓ curl-cffi: 支持 str 和 dict 格式代理")

examples/offsite_middleware_example.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-OffsiteMiddleware 使用示例
-展示如何配置和使用OffsiteMiddleware来限制爬虫只爬取指定域名
-"""
-
-# 基础配置
-SETTINGS = {
-    # 允许的域名列表（OffsiteMiddleware会使用这个配置）
-    'ALLOWED_DOMAINS': [
-        'example.com',
-        'www.example.com',
-        'subdomain.example.com'
-    ],
-    
-    # 中间件配置（OffsiteMiddleware已默认启用）
-    'MIDDLEWARES': [
-        # === 请求预处理阶段 ===
-        'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',  # 1. 忽略无效请求
-        'crawlo.middleware.download_delay.DownloadDelayMiddleware',  # 2. 控制请求频率
-        'crawlo.middleware.default_header.DefaultHeaderMiddleware',  # 3. 添加默认请求头
-        'crawlo.middleware.proxy.ProxyMiddleware',  # 4. 设置代理
-        'crawlo.middleware.offsite.OffsiteMiddleware',  # 5. 站外请求过滤
-        
-        # === 响应处理阶段 ===
-        'crawlo.middleware.retry.RetryMiddleware',  # 6. 失败请求重试
-        'crawlo.middleware.response_code.ResponseCodeMiddleware',  # 7. 处理特殊状态码
-        'crawlo.middleware.response_filter.ResponseFilterMiddleware',  # 8. 响应内容过滤
-    ],
-    
-    # 其他常用配置
-    'DOWNLOAD_DELAY': 1,
-    'CONCURRENCY': 8,
-    'LOG_LEVEL': 'INFO',
-}
-
-def get_settings():
-    """获取配置"""
-    return SETTINGS
-
-if __name__ == "__main__":
-    print("OffsiteMiddleware配置示例:")
-    print("=" * 40)
-    print(f"允许的域名: {SETTINGS['ALLOWED_DOMAINS']}")
-    print("\n中间件列表:")
-    for i, middleware in enumerate(SETTINGS['MIDDLEWARES'], 1):
-        print(f"  {i}. {middleware}")
-    
-    print("\n" + "=" * 40)
-    print("OffsiteMiddleware功能说明:")
-    print("✓ 自动过滤不在ALLOWED_DOMAINS中的请求")
-    print("✓ 支持子域名匹配")
-    print("✓ 记录被过滤的请求统计信息")
-    print("✓ 可通过设置ALLOWED_DOMAINS=[]来禁用此中间件")

examples/offsite_spider_example.py

@@ -1,107 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-使用OffsiteMiddleware的爬虫示例
-展示如何在实际爬虫中使用OffsiteMiddleware限制爬取范围
-"""
-
-from crawlo.spider import Spider
-from crawlo.network.request import Request
-
-
-class ExampleSpider(Spider):
-    """
-    示例爬虫，演示OffsiteMiddleware的使用
-    """
-    
-    # 爬虫名称
-    name = "example_offsite_spider"
-    
-    # 自定义设置
-    custom_settings = {
-        # 允许的域名列表
-        'ALLOWED_DOMAINS': [
-            'httpbin.org',
-            'example.com',
-            'www.example.com'
-        ],
-        
-        # 请求延迟（秒）
-        'DOWNLOAD_DELAY': 1,
-        
-        # 并发数
-        'CONCURRENCY': 4,
-        
-        # 日志级别
-        'LOG_LEVEL': 'INFO',
-    }
-    
-    def start_requests(self):
-        """
-        开始请求
-        """
-        # 这些URL会被允许
-        allowed_urls = [
-            'https://httpbin.org/ip',
-            'https://httpbin.org/user-agent',
-            'https://example.com/page1',
-            'https://www.example.com/page2'
-        ]
-        
-        # 这些URL会被过滤（站外请求）
-        offsite_urls = [
-            'https://google.com',
-            'https://github.com',
-            'https://stackoverflow.com'
-        ]
-        
-        # 生成允许的请求
-        for url in allowed_urls:
-            yield Request(url=url, callback=self.parse_allowed)
-            
-        # 生成站外请求（会被OffsiteMiddleware过滤）
-        for url in offsite_urls:
-            yield Request(url=url, callback=self.parse_offsite)
-    
-    async def parse_allowed(self, response):
-        """
-        处理允许的请求响应
-        """
-        self.logger.info(f"成功处理允许的请求: {response.url}")
-        self.logger.info(f"状态码: {response.status_code}")
-        # 这里可以添加解析逻辑
-        
-    async def parse_offsite(self, response):
-        """
-        这个方法实际上不会被调用，因为站外请求会被过滤
-        """
-        self.logger.info(f"这个消息不应该出现: {response.url}")
-
-
-# 运行爬虫的示例代码
-if __name__ == "__main__":
-    """
-    运行说明:
-    
-    1. 确保已在项目根目录下安装了crawlo:
-       pip install -e .
-       
-    2. 运行爬虫:
-       crawlo run example_offsite_spider
-       
-    3. 观察日志输出:
-       - 允许的域名请求会被正常处理
-       - 站外请求会被OffsiteMiddleware过滤，并在日志中显示过滤信息
-       - 统计信息会记录被过滤的请求数量
-       
-    OffsiteMiddleware的优势:
-    ✓ 防止爬虫意外爬取到无关网站
-    ✓ 节省带宽和服务器资源
-    ✓ 提高爬取效率，专注于目标网站
-    ✓ 可配置的域名白名单，灵活控制爬取范围
-    """
-    print("OffsiteSpider示例")
-    print("=" * 30)
-    print("此爬虫演示了OffsiteMiddleware的使用方法")
-    print("请使用以下命令运行:")
-    print("  crawlo run example_offsite_spider")

examples/proxy_spider_example.py

@@ -1,166 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-代理爬虫示例
-==============
-展示如何在Crawlo框架中使用代理API爬取网站
-"""
-
-import asyncio
-import sys
-import os
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo import Spider, Request
-from crawlo.network.response import Response
-
-
-class ProxyExampleSpider(Spider):
-    """使用代理的示例爬虫"""
-    name = 'proxy_example_spider'
-    
-    def __init__(self):
-        super().__init__()
-        # 要爬取的URL列表
-        self.urls = [
-            'https://httpbin.org/ip',  # 查看当前IP
-            'https://httpbin.org/headers',  # 查看请求头
-            'https://stock.10jqka.com.cn/20240315/c655957791.shtml',  # 测试目标链接
-        ]
-    
-    def start_requests(self):
-        """生成初始请求"""
-        for i, url in enumerate(self.urls):
-            # 为每个请求添加一些元数据
-            request = Request(
-                url=url,
-                callback=self.parse,
-                meta={'request_id': i}
-            )
-            yield request
-    
-    def parse(self, response: Response):
-        """解析响应"""
-        request_id = response.request.meta.get('request_id', 'unknown')
-        
-        print(f"\n{'='*50}")
-        print(f"请求 #{request_id}: {response.url}")
-        print(f"状态码: {response.status_code}")
-        print(f"{'='*50}")
-        
-        # 特殊处理httpbin.org的响应
-        if 'httpbin.org/ip' in response.url:
-            print("当前IP信息:")
-            print(response.text[:500])
-            
-        elif 'httpbin.org/headers' in response.url:
-            print("请求头信息:")
-            print(response.text[:500])
-            
-        else:
-            # 处理目标网站
-            print("页面标题:")
-            title = response.css('title::text').get()
-            if title:
-                print(f"  {title}")
-            else:
-                print("  未找到标题")
-            
-            print("\n页面内容预览:")
-            # 清理HTML标签，只显示文本内容
-            text_content = response.css('*::text').getall()
-            if text_content:
-                # 合并前几个文本片段
-                content = ''.join(text_content[:10])
-                print(f"  {content[:200]}{'...' if len(content) > 200 else ''}")
-            else:
-                print("  无文本内容")
-        
-        # 返回结果
-        return {
-            'request_id': request_id,
-            'url': response.url,
-            'status_code': response.status_code,
-            'title': response.css('title::text').get(),
-        }
-
-
-# 配置说明
-SETTINGS = {
-    # 基础配置
-    'LOG_LEVEL': 'INFO',
-    'CONCURRENCY': 2,
-    
-    # 代理配置
-    'PROXY_ENABLED': True,
-    'PROXY_API_URL': 'http://test.proxy.api:8080/proxy/getitem/',
-    'PROXY_EXTRACTOR': 'proxy',
-    'PROXY_REFRESH_INTERVAL': 60,  # 1分钟刷新一次
-    'PROXY_API_TIMEOUT': 10,
-    'PROXY_POOL_SIZE': 5,
-    'PROXY_HEALTH_CHECK_THRESHOLD': 0.5,
-    
-    # 下载延迟
-    'DOWNLOAD_DELAY': 1,
-    'RANDOMNESS': True,
-    
-    # 中间件
-    'MIDDLEWARES': [
-        'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
-        'crawlo.middleware.download_delay.DownloadDelayMiddleware',
-        'crawlo.middleware.default_header.DefaultHeaderMiddleware',
-        'crawlo.middleware.proxy.ProxyMiddleware',
-        'crawlo.middleware.retry.RetryMiddleware',
-        'crawlo.middleware.response_code.ResponseCodeMiddleware',
-        'crawlo.middleware.response_filter.ResponseFilterMiddleware',
-    ],
-    
-    # 管道
-    'PIPELINES': [
-        'crawlo.pipelines.console_pipeline.ConsolePipeline',
-    ],
-    
-    # 默认请求头
-    'DEFAULT_REQUEST_HEADERS': {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
-        'Accept-Encoding': 'gzip, deflate, br',
-        'Connection': 'keep-alive',
-        'Upgrade-Insecure-Requests': '1',
-    }
-}
-
-
-def main():
-    """主函数"""
-    print("代理爬虫示例")
-    print("=" * 50)
-    print("此示例展示如何在Crawlo框架中使用代理API")
-    print("代理API: http://test.proxy.api:8080/proxy/getitem/")
-    print("目标网站: https://stock.10jqka.com.cn/20240315/c655957791.shtml")
-    print("=" * 50)
-    
-    print("\n使用方法:")
-    print("1. 确保在settings.py中配置了代理参数")
-    print("2. 运行爬虫: crawlo run proxy_example_spider")
-    print("3. 爬虫会自动使用代理API获取代理并应用到请求中")
-    
-    print("\n配置示例:")
-    for key, value in SETTINGS.items():
-        if key in ['MIDDLEWARES', 'PIPELINES', 'DEFAULT_REQUEST_HEADERS']:
-            print(f"{key}:")
-            if isinstance(value, list):
-                for item in value:
-                    print(f"  - {item}")
-            elif isinstance(value, dict):
-                for k, v in value.items():
-                    print(f"  {k}: {v}")
-        else:
-            print(f"{key}: {value}")
-
-
-if __name__ == '__main__':
-    main()

examples/request_ignore_middleware_example.py

@@ -1,51 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-RequestIgnoreMiddleware 使用示例
-展示如何使用RequestIgnoreMiddleware处理和记录被忽略的请求
-"""
-
-# RequestIgnoreMiddleware是默认启用的中间件，无需特殊配置
-# 它会自动处理IgnoreRequestError异常并记录相关统计信息
-
-# 中间件配置（RequestIgnoreMiddleware已默认启用）
-SETTINGS = {
-    'MIDDLEWARES': [
-        # === 请求预处理阶段 ===
-        'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',  # 1. 忽略无效请求
-        'crawlo.middleware.download_delay.DownloadDelayMiddleware',  # 2. 控制请求频率
-        'crawlo.middleware.default_header.DefaultHeaderMiddleware',  # 3. 添加默认请求头
-        'crawlo.middleware.proxy.ProxyMiddleware',  # 4. 设置代理
-        'crawlo.middleware.offsite.OffsiteMiddleware',  # 5. 站外请求过滤
-        
-        # === 响应处理阶段 ===
-        'crawlo.middleware.retry.RetryMiddleware',  # 6. 失败请求重试
-        'crawlo.middleware.response_code.ResponseCodeMiddleware',  # 7. 处理特殊状态码
-        'crawlo.middleware.response_filter.ResponseFilterMiddleware',  # 8. 响应内容过滤
-    ],
-    
-    # 其他常用配置
-    'DOWNLOAD_DELAY': 1,
-    'CONCURRENCY': 8,
-    'LOG_LEVEL': 'INFO',
-}
-
-def get_settings():
-    """获取配置"""
-    return SETTINGS
-
-if __name__ == "__main__":
-    print("RequestIgnoreMiddleware配置示例:")
-    print("=" * 40)
-    print("中间件列表:")
-    for i, middleware in enumerate(SETTINGS['MIDDLEWARES'], 1):
-        print(f"  {i}. {middleware}")
-    
-    print("\n" + "=" * 40)
-    print("RequestIgnoreMiddleware功能说明:")
-    print("✓ 自动处理IgnoreRequestError异常")
-    print("✓ 记录被忽略请求的详细统计信息")
-    print("✓ 按忽略原因分类统计")
-    print("✓ 按域名分布统计")
-    print("✓ 提供详细的日志信息")
-    print("✓ 无需特殊配置，默认启用")