crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

crawlo/templates/project/settings_distributed.py.tmpl

@@ -0,0 +1,120 @@
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}} 项目配置文件（分布式版）
+=============================
+基于 Crawlo 框架的分布式爬虫项目配置。
+适合大规模数据采集和多节点部署。
+"""
+
+import os
+from crawlo.config import CrawloConfig
+
+# ============================== 项目基本信息 ==============================
+PROJECT_NAME = '{{project_name}}'
+VERSION = '1.0.0'
+
+# ============================== 分布式配置 ==============================
+# 使用配置工厂创建分布式配置
+CONFIG = CrawloConfig.distributed(
+    redis_host=os.getenv('REDIS_HOST', '127.0.0.1'),
+    redis_port=int(os.getenv('REDIS_PORT', 6379)),
+    redis_password=os.getenv('REDIS_PASSWORD', ''),
+    project_name='{{project_name}}',
+    concurrency=16,
+    download_delay=1.0
+)
+
+# 获取配置
+locals().update(CONFIG.to_dict())
+
+# ============================== 网络请求配置 ==============================
+DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader"
+DOWNLOAD_TIMEOUT = 60
+VERIFY_SSL = True
+
+# ============================== 并发配置 ==============================
+CONCURRENCY = 16
+MAX_RUNNING_SPIDERS = 5
+DOWNLOAD_DELAY = 1.0
+
+# ============================== 队列配置 ==============================
+SCHEDULER_MAX_QUEUE_SIZE = 5000
+QUEUE_MAX_RETRIES = 5
+QUEUE_TIMEOUT = 300
+
+# ============================== Redis 配置 ==============================
+REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')
+REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
+REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
+REDIS_DB = int(os.getenv('REDIS_DB', 0))
+
+# 根据是否有密码生成 URL
+if REDIS_PASSWORD:
+    REDIS_URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+else:
+    REDIS_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+
+# ============================== 数据存储配置 ==============================
+# MySQL 配置
+MYSQL_HOST = os.getenv('MYSQL_HOST', '127.0.0.1')
+MYSQL_PORT = int(os.getenv('MYSQL_PORT', 3306))
+MYSQL_USER = os.getenv('MYSQL_USER', 'root')
+MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', '123456')
+MYSQL_DB = os.getenv('MYSQL_DB', '{{project_name}}')
+MYSQL_TABLE = '{{project_name}}_data'
+MYSQL_BATCH_SIZE = 100
+MYSQL_USE_BATCH = True
+
+# MongoDB 配置
+MONGO_URI = os.getenv('MONGO_URI', 'mongodb://localhost:27017')
+MONGO_DATABASE = '{{project_name}}_db'
+MONGO_COLLECTION = '{{project_name}}_items'
+MONGO_BATCH_SIZE = 100
+MONGO_USE_BATCH = True
+
+# ============================== 去重配置 ==============================
+REDIS_TTL = 0
+CLEANUP_FP = 0
+FILTER_DEBUG = True
+
+# ============================== 中间件与管道 ==============================
+MIDDLEWARES = [
+    'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
+    'crawlo.middleware.download_delay.DownloadDelayMiddleware',
+    'crawlo.middleware.default_header.DefaultHeaderMiddleware',
+    'crawlo.middleware.proxy.ProxyMiddleware',
+    'crawlo.middleware.retry.RetryMiddleware',
+    'crawlo.middleware.response_code.ResponseCodeMiddleware',
+    'crawlo.middleware.response_filter.ResponseFilterMiddleware',
+]
+
+PIPELINES = [
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
+    # '{{project_name}}.pipelines.DatabasePipeline',
+    # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',
+    # 'crawlo.pipelines.mongo_pipeline.MongoPipeline',
+]
+
+# ============================== 扩展组件 ==============================
+EXTENSIONS = [
+    'crawlo.extension.log_interval.LogIntervalExtension',
+    'crawlo.extension.log_stats.LogStats',
+    'crawlo.extension.logging_extension.CustomLoggerExtension',
+    # 'crawlo.extension.memory_monitor.MemoryMonitorExtension',
+    # 'crawlo.extension.request_recorder.RequestRecorderExtension',
+]
+
+# ============================== 日志配置 ==============================
+LOG_LEVEL = 'INFO'
+LOG_FILE = f'logs/{{project_name}}.log'
+STATS_DUMP = True
+
+# ============================== 代理配置 ==============================
+PROXY_ENABLED = False
+PROXY_API_URL = ""
+PROXY_EXTRACTOR = "proxy"
+PROXY_REFRESH_INTERVAL = 60
+PROXY_API_TIMEOUT = 10
+
+# ============================== 自定义配置 ==============================
+# 在此处添加项目特定的配置项

crawlo/templates/project/settings_gentle.py.tmpl

@@ -0,0 +1,95 @@
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}} 项目配置文件（温和版）
+=============================
+基于 Crawlo 框架的温和爬虫项目配置。
+适合对目标网站友好的低负载爬取。
+"""
+
+import os
+from crawlo.config import CrawloConfig
+
+# ============================== 项目基本信息 ==============================
+PROJECT_NAME = '{{project_name}}'
+VERSION = '1.0.0'
+
+# ============================== 温和模式配置 ==============================
+# 使用配置工厂创建温和模式配置
+CONFIG = CrawloConfig.presets().gentle()
+
+# 获取配置
+locals().update(CONFIG.to_dict())
+
+# ============================== 网络请求配置 ==============================
+DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader"
+DOWNLOAD_TIMEOUT = 60
+VERIFY_SSL = True
+
+# ============================== 低并发配置 ==============================
+CONCURRENCY = 2
+MAX_RUNNING_SPIDERS = 1
+DOWNLOAD_DELAY = 3.0
+RANDOMNESS = True
+RANDOM_RANGE = (2.0, 5.0)
+
+# ============================== 连接池配置 ==============================
+CONNECTION_POOL_LIMIT = 10
+CONNECTION_POOL_LIMIT_PER_HOST = 5
+
+# ============================== 重试配置 ==============================
+MAX_RETRY_TIMES = 3
+RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524]
+IGNORE_HTTP_CODES = [403, 404]
+
+# ============================== 队列配置 ==============================
+SCHEDULER_MAX_QUEUE_SIZE = 1000
+QUEUE_MAX_RETRIES = 3
+QUEUE_TIMEOUT = 300
+
+# ============================== 数据存储配置 ==============================
+# MySQL 配置
+MYSQL_HOST = os.getenv('MYSQL_HOST', '127.0.0.1')
+MYSQL_PORT = int(os.getenv('MYSQL_PORT', 3306))
+MYSQL_USER = os.getenv('MYSQL_USER', 'root')
+MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', '123456')
+MYSQL_DB = os.getenv('MYSQL_DB', '{{project_name}}')
+MYSQL_TABLE = '{{project_name}}_data'
+
+# MongoDB 配置
+MONGO_URI = os.getenv('MONGO_URI', 'mongodb://localhost:27017')
+MONGO_DATABASE = '{{project_name}}_db'
+MONGO_COLLECTION = '{{project_name}}_items'
+
+# ============================== 去重配置 ==============================
+REDIS_TTL = 0
+CLEANUP_FP = 0
+FILTER_DEBUG = True
+
+# ============================== 中间件与管道 ==============================
+MIDDLEWARES = [
+    'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
+    'crawlo.middleware.download_delay.DownloadDelayMiddleware',
+    'crawlo.middleware.default_header.DefaultHeaderMiddleware',
+    'crawlo.middleware.retry.RetryMiddleware',
+    'crawlo.middleware.response_code.ResponseCodeMiddleware',
+]
+
+PIPELINES = [
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
+    # '{{project_name}}.pipelines.DatabasePipeline',
+]
+
+# ============================== 扩展组件 ==============================
+EXTENSIONS = [
+    'crawlo.extension.log_interval.LogIntervalExtension',
+    'crawlo.extension.log_stats.LogStats',
+    'crawlo.extension.logging_extension.CustomLoggerExtension',
+]
+
+# ============================== 日志配置 ==============================
+LOG_LEVEL = 'INFO'
+LOG_FILE = f'logs/{{project_name}}.log'
+STATS_DUMP = True
+
+# ============================== 自定义配置 ==============================
+# 在此处添加项目特定的配置项

crawlo/templates/project/settings_high_performance.py.tmpl

@@ -0,0 +1,152 @@
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}} 项目配置文件（高性能版）
+=============================
+基于 Crawlo 框架的高性能爬虫项目配置。
+针对大规模、高并发场景优化。
+"""
+
+import os
+from crawlo.config import CrawloConfig
+
+# ============================== 项目基本信息 ==============================
+PROJECT_NAME = '{{project_name}}'
+VERSION = '1.0.0'
+
+# ============================== 高性能配置 ==============================
+# 使用配置工厂创建高性能配置
+CONFIG = CrawloConfig.presets().large_scale(
+    redis_host=os.getenv('REDIS_HOST', '127.0.0.1'),
+    project_name='{{project_name}}'
+)
+
+# 获取配置
+locals().update(CONFIG.to_dict())
+
+# ============================== 网络请求配置 ==============================
+DOWNLOADER = "crawlo.downloader.cffi_downloader.CurlCffiDownloader"
+DOWNLOAD_TIMEOUT = 30
+VERIFY_SSL = True
+USE_SESSION = True
+
+# ============================== 高并发配置 ==============================
+CONCURRENCY = 32
+MAX_RUNNING_SPIDERS = 10
+DOWNLOAD_DELAY = 0.5
+RANDOMNESS = True
+RANDOM_RANGE = (0.8, 1.2)
+
+# ============================== 连接池配置 ==============================
+CONNECTION_POOL_LIMIT = 100
+CONNECTION_POOL_LIMIT_PER_HOST = 50
+
+# ============================== 重试配置 ==============================
+MAX_RETRY_TIMES = 5
+RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524]
+IGNORE_HTTP_CODES = [403, 404]
+
+# ============================== 队列配置 ==============================
+SCHEDULER_MAX_QUEUE_SIZE = 10000
+SCHEDULER_QUEUE_NAME = f'crawlo:{{project_name}}:queue:requests'
+QUEUE_MAX_RETRIES = 5
+QUEUE_TIMEOUT = 300
+LARGE_SCALE_BATCH_SIZE = 2000
+LARGE_SCALE_CHECKPOINT_INTERVAL = 5000
+
+# ============================== Redis 配置 ==============================
+REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')
+REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
+REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
+REDIS_DB = int(os.getenv('REDIS_DB', 0))
+
+# 根据是否有密码生成 URL
+if REDIS_PASSWORD:
+    REDIS_URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+else:
+    REDIS_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+
+# ============================== 数据存储配置 ==============================
+# MySQL 配置
+MYSQL_HOST = os.getenv('MYSQL_HOST', '127.0.0.1')
+MYSQL_PORT = int(os.getenv('MYSQL_PORT', 3306))
+MYSQL_USER = os.getenv('MYSQL_USER', 'root')
+MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', '123456')
+MYSQL_DB = os.getenv('MYSQL_DB', '{{project_name}}')
+MYSQL_TABLE = '{{project_name}}_data'
+MYSQL_BATCH_SIZE = 200
+MYSQL_USE_BATCH = True
+MYSQL_POOL_MIN = 10
+MYSQL_POOL_MAX = 50
+
+# MongoDB 配置
+MONGO_URI = os.getenv('MONGO_URI', 'mongodb://localhost:27017')
+MONGO_DATABASE = '{{project_name}}_db'
+MONGO_COLLECTION = '{{project_name}}_items'
+MONGO_BATCH_SIZE = 200
+MONGO_USE_BATCH = True
+MONGO_MAX_POOL_SIZE = 300
+MONGO_MIN_POOL_SIZE = 50
+
+# ============================== 去重配置 ==============================
+REDIS_TTL = 0
+CLEANUP_FP = 0
+FILTER_DEBUG = False  # 生产环境关闭调试日志
+
+# ============================== 中间件与管道 ==============================
+MIDDLEWARES = [
+    'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
+    'crawlo.middleware.download_delay.DownloadDelayMiddleware',
+    'crawlo.middleware.default_header.DefaultHeaderMiddleware',
+    'crawlo.middleware.proxy.ProxyMiddleware',
+    'crawlo.middleware.retry.RetryMiddleware',
+    'crawlo.middleware.response_code.ResponseCodeMiddleware',
+    'crawlo.middleware.response_filter.ResponseFilterMiddleware',
+]
+
+PIPELINES = [
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
+    # '{{project_name}}.pipelines.DatabasePipeline',
+    # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',
+    # 'crawlo.pipelines.mongo_pipeline.MongoPipeline',
+]
+
+# ============================== 扩展组件 ==============================
+EXTENSIONS = [
+    'crawlo.extension.log_interval.LogIntervalExtension',
+    'crawlo.extension.log_stats.LogStats',
+    'crawlo.extension.logging_extension.CustomLoggerExtension',
+    # 'crawlo.extension.memory_monitor.MemoryMonitorExtension',
+    # 'crawlo.extension.request_recorder.RequestRecorderExtension',
+    # 'crawlo.extension.performance_profiler.PerformanceProfilerExtension',
+]
+
+# ============================== 日志配置 ==============================
+LOG_LEVEL = 'INFO'
+LOG_FILE = f'logs/{{project_name}}.log'
+STATS_DUMP = True
+
+# ============================== 代理配置 ==============================
+PROXY_ENABLED = False
+PROXY_API_URL = ""
+PROXY_EXTRACTOR = "proxy"
+PROXY_REFRESH_INTERVAL = 30
+PROXY_API_TIMEOUT = 5
+
+# ============================== 浏览器指纹配置 ==============================
+CURL_BROWSER_TYPE = "chrome"
+CURL_BROWSER_VERSION_MAP = {
+    "chrome": "chrome136",
+    "edge": "edge101",
+    "safari": "safari184",
+    "firefox": "firefox135",
+}
+
+# ============================== 下载器优化配置 ==============================
+HTTPX_HTTP2 = True
+HTTPX_FOLLOW_REDIRECTS = True
+AIOHTTP_AUTO_DECOMPRESS = True
+CONNECTION_TTL_DNS_CACHE = 300
+CONNECTION_KEEPALIVE_TIMEOUT = 15
+
+# ============================== 自定义配置 ==============================
+# 在此处添加项目特定的配置项

crawlo/templates/project/settings_simple.py.tmpl

@@ -0,0 +1,69 @@
+# -*- coding: UTF-8 -*-
+"""
+{{project_name}} 项目配置文件（简化版）
+=============================
+基于 Crawlo 框架的简化爬虫项目配置。
+适合快速开始和小型项目。
+"""
+
+import os
+from crawlo.config import CrawloConfig
+
+# ============================== 项目基本信息 ==============================
+PROJECT_NAME = '{{project_name}}'
+VERSION = '1.0.0'
+
+# ============================== 基本配置 ==============================
+# 使用配置工厂创建基本配置
+CONFIG = CrawloConfig.standalone(
+    concurrency=4,
+    download_delay=1.0
+)
+
+# 获取配置
+locals().update(CONFIG.to_dict())
+
+# ============================== 网络请求配置 ==============================
+DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader"
+DOWNLOAD_TIMEOUT = 30
+VERIFY_SSL = True
+
+# ============================== 并发配置 ==============================
+CONCURRENCY = 4
+DOWNLOAD_DELAY = 1.0
+
+# ============================== 数据存储配置 ==============================
+# MySQL 配置
+MYSQL_HOST = os.getenv('MYSQL_HOST', '127.0.0.1')
+MYSQL_PORT = int(os.getenv('MYSQL_PORT', 3306))
+MYSQL_USER = os.getenv('MYSQL_USER', 'root')
+MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', '123456')
+MYSQL_DB = os.getenv('MYSQL_DB', '{{project_name}}')
+MYSQL_TABLE = '{{project_name}}_data'
+
+# MongoDB 配置
+MONGO_URI = os.getenv('MONGO_URI', 'mongodb://localhost:27017')
+MONGO_DATABASE = '{{project_name}}_db'
+MONGO_COLLECTION = '{{project_name}}_items'
+
+# ============================== 中间件与管道 ==============================
+MIDDLEWARES = [
+    'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
+    'crawlo.middleware.download_delay.DownloadDelayMiddleware',
+    'crawlo.middleware.default_header.DefaultHeaderMiddleware',
+    'crawlo.middleware.retry.RetryMiddleware',
+    'crawlo.middleware.response_code.ResponseCodeMiddleware',
+]
+
+PIPELINES = [
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
+    # '{{project_name}}.pipelines.DatabasePipeline',
+]
+
+# ============================== 日志配置 ==============================
+LOG_LEVEL = 'INFO'
+LOG_FILE = f'logs/{{project_name}}.log'
+STATS_DUMP = True
+
+# ============================== 自定义配置 ==============================
+# 在此处添加项目特定的配置项

crawlo/templates/spider/spider.py.tmpl

@@ -11,7 +11,7 @@
 
 from crawlo.spider import Spider
 from crawlo import Request
-# from {{project_name}}.items import {{item_class}}  # 可选：导入数据项
+from ..items import ExampleItem
 
 
 class {{class_name}}(Spider):
@@ -139,40 +139,4 @@ class {{class_name}}(Spider):
         #     'publish_time': response.xpath('//time/@datetime').get(),
         # }
         
-        pass
-
-    def parse_error(self, failure):
-        """
-        处理请求失败的方法（可选）。
-        
-        当请求失败时会被调用。
-        """
-        self.logger.error(f'请求失败: {failure.request.url} - {failure.value}')
-        
-        # 可以选择重试或记录失败信息
-        # yield {
-        #     'error_url': failure.request.url,
-        #     'error_message': str(failure.value),
-        #     'error_type': failure.type.__name__,
-        # }
-
-    def spider_opened(self, spider):
-        """
-        爬虫启动时的回调方法（可选）。
-        """
-        self.logger.info(f'爬虫 {spider.name} 已启动')
-        
-        # 初始化操作，例如连接数据库、加载配置等
-        # self.database = self.connect_database()
-        # self.cookies = self.load_cookies()
-
-    def spider_closed(self, spider, reason):
-        """
-        爬虫关闭时的回调方法（可选）。
-        """
-        self.logger.info(f'爬虫 {spider.name} 已关闭，原因: {reason}')
-        
-        # 清理操作，例如关闭数据库连接、保存状态等
-        # if hasattr(self, 'database'):
-        #     self.database.close()
-        # self.save_cookies()
+        pass

crawlo/tools/__init__.py

@@ -0,0 +1,183 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-10 22:00
+# @Author  : crawl-coder
+# @Desc    : Crawlo框架工具包集合
+"""
+
+# 日期工具封装
+from .date_tools import (
+    TimeUtils,
+    parse_time,
+    format_time,
+    time_diff,
+    to_timestamp,
+    to_datetime,
+    now,
+    to_timezone,
+    to_utc,
+    to_local,
+    from_timestamp_with_tz
+)
+
+# 数据清洗工具封装
+from ..cleaners import (
+    TextCleaner,
+    DataFormatter,
+    EncodingConverter,
+    remove_html_tags,
+    decode_html_entities,
+    remove_extra_whitespace,
+    remove_special_chars,
+    normalize_unicode,
+    clean_text,
+    extract_numbers,
+    extract_emails,
+    extract_urls,
+    format_number,
+    format_currency,
+    format_percentage,
+    format_phone_number,
+    format_chinese_id_card,
+    capitalize_words,
+    detect_encoding,
+    to_utf8,
+    convert_encoding
+)
+
+# 数据验证工具封装
+from .data_validator import (
+    DataValidator,
+    validate_email,
+    validate_phone,
+    validate_url,
+    validate_chinese_id_card,
+    validate_date,
+    validate_number_range,
+    check_data_integrity
+)
+
+# 重试机制封装
+from .retry_mechanism import (
+    RetryMechanism,
+    retry,
+    should_retry,
+    exponential_backoff
+)
+
+# 反爬虫应对工具
+from .anti_crawler import (
+    ProxyPoolManager,
+    CaptchaHandler,
+    AntiCrawler,
+    get_random_user_agent,
+    rotate_proxy,
+    handle_captcha,
+    detect_rate_limiting
+)
+
+# 带认证代理工具
+from .authenticated_proxy import (
+    AuthenticatedProxy,
+    create_proxy_config,
+    format_proxy_for_request,
+    parse_proxy_url,
+    validate_proxy_url,
+    get_proxy_info
+)
+
+# 分布式协调工具
+from .distributed_coordinator import (
+    TaskDistributor,
+    DeduplicationTool,
+    DistributedCoordinator,
+    generate_task_id,
+    claim_task,
+    report_task_status,
+    get_cluster_info,
+    generate_pagination_tasks,
+    distribute_tasks
+)
+
+__all__ = [
+    # 日期工具
+    "TimeUtils",
+    "parse_time",
+    "format_time",
+    "time_diff",
+    "to_timestamp",
+    "to_datetime",
+    "now",
+    "to_timezone",
+    "to_utc",
+    "to_local",
+    "from_timestamp_with_tz",
+    
+    # 数据清洗工具
+    "TextCleaner",
+    "DataFormatter",
+    "EncodingConverter",
+    "remove_html_tags",
+    "decode_html_entities",
+    "remove_extra_whitespace",
+    "remove_special_chars",
+    "normalize_unicode",
+    "clean_text",
+    "extract_numbers",
+    "extract_emails",
+    "extract_urls",
+    "format_number",
+    "format_currency",
+    "format_percentage",
+    "format_phone_number",
+    "format_chinese_id_card",
+    "capitalize_words",
+    "detect_encoding",
+    "to_utf8",
+    "convert_encoding",
+    
+    # 数据验证工具
+    "DataValidator",
+    "validate_email",
+    "validate_phone",
+    "validate_url",
+    "validate_chinese_id_card",
+    "validate_date",
+    "validate_number_range",
+    "check_data_integrity",
+    
+    # 重试机制封装
+    "RetryMechanism",
+    "retry",
+    "should_retry",
+    "exponential_backoff",
+    
+    # 反爬虫应对工具
+    "ProxyPoolManager",
+    "CaptchaHandler",
+    "AntiCrawler",
+    "get_random_user_agent",
+    "rotate_proxy",
+    "handle_captcha",
+    "detect_rate_limiting",
+    
+    # 带认证代理工具
+    "AuthenticatedProxy",
+    "create_proxy_config",
+    "format_proxy_for_request",
+    "parse_proxy_url",
+    "validate_proxy_url",
+    "get_proxy_info",
+    
+    # 分布式协调工具
+    "TaskDistributor",
+    "DeduplicationTool",
+    "DistributedCoordinator",
+    "generate_task_id",
+    "claim_task",
+    "report_task_status",
+    "get_cluster_info",
+    "generate_pagination_tasks",
+    "distribute_tasks"
+]