crawlo 1.2.4__py3-none-any.whl → 1.2.6__py3-none-any.whl

crawlo/templates/project/settings.py.tmpl

@@ -1,35 +1,9 @@
 # -*- coding: UTF-8 -*-
-"""
-{{project_name}} 项目配置文件
-=============================
-基于 Crawlo 框架的爬虫项目配置。
-
-🎯 快速开始：
-
-# 方式1：使用默认配置（推荐）
-from crawlo.crawler import CrawlerProcess
-process = CrawlerProcess()  # 无需任何配置
-
-# 方式2：使用配置工厂
-from crawlo.config import CrawloConfig
-config = CrawloConfig.standalone()  # 单机模式
-config = CrawloConfig.distributed(redis_host='192.168.1.100')  # 分布式模式
-process = CrawlerProcess(settings=config.to_dict())
-
-# 方式3：使用环境变量
-from crawlo.config import CrawloConfig
-config = CrawloConfig.from_env()  # 从环境变量读取
-"""
 import os
 from crawlo.config import CrawloConfig
 
 # ============================== 项目基本信息 ==============================
 PROJECT_NAME = '{{project_name}}'
-try:
-    from crawlo import __version__
-    VERSION = __version__
-except ImportError:
-    VERSION = '1.0.0'
 
 # ============================== 运行模式选择 ==============================
 
@@ -101,43 +75,6 @@ INTERVAL = 5
 DEPTH_PRIORITY = 1
 MAX_RUNNING_SPIDERS = 3
 
-# ============================== 运行模式选择 ==============================
-# 运行模式：'standalone'(单机), 'distributed'(分布式), 'auto'(自动检测)
-# 
-# 三种运行模式的最佳使用场景：
-# 
-# 1. standalone（单机模式）：
-#    - 适用场景：开发调试、小规模数据采集、个人项目
-#    - 特点：简单易用，资源占用少，无需额外依赖
-#    - 配置建议：
-#      * QUEUE_TYPE = 'auto'（自动选择队列类型）
-#      * FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'（内存过滤器）
-#      * DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline'（内存去重）
-#    - 混合配置（推荐）：
-#      * QUEUE_TYPE = 'auto'（自动选择）
-#      * FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'（Redis过滤器）
-#      * DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline'（Redis去重）
-#      * 优势：享受Redis去重的持久性，同时保持部署简单
-#
-# 2. distributed（分布式模式）：
-#    - 适用场景：大规模数据采集、多节点协同工作、高并发需求
-#    - 特点：支持多节点扩展，高并发处理能力，需要Redis支持
-#    - 配置建议：
-#      * QUEUE_TYPE = 'redis'（Redis队列）
-#      * FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'（Redis过滤器）
-#      * DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline'（Redis去重）
-#    - 部署要求：需要配置Redis服务器连接参数
-#
-# 3. auto（自动检测模式）：
-#    - 适用场景：希望根据环境自动选择最佳运行方式
-#    - 特点：智能检测环境配置，自动选择运行模式
-#    - 配置建议：
-#      * 框架会根据Redis可用性自动选择队列类型
-#      * 默认使用内存过滤器和去重管道
-#    - 适用情况：需要在不同环境中使用同一套配置
-
-RUN_MODE = 'standalone'  # 默认单机模式，简单易用
-
 # ============================== 队列配置（支持分布式） ==============================
 
 # 队列类型：'auto'（自动选择）, 'memory'（内存队列）, 'redis'（分布式队列）
@@ -183,8 +120,9 @@ MONGO_USE_BATCH = False  # 是否启用批量插入
 REQUEST_DIR = '.'
 
 # 明确配置默认去重管道和过滤器，避免冗余的if-else判断
-DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline'
-FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
+# 在单机模式下，如果Redis可用则使用Redis去重，否则使用内存去重
+DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline'
+FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'
 
 # --- Redis 配置（用于分布式去重和队列） ---
 REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')

crawlo/templates/project/settings_distributed.py.tmpl

@@ -5,20 +5,17 @@
 基于 Crawlo 框架的分布式爬虫项目配置。
 适合大规模数据采集和多节点部署。
 """
-
 import os
 from crawlo.config import CrawloConfig
 
 # ============================== 项目基本信息 ==============================
 PROJECT_NAME = '{{project_name}}'
-try:
-    from crawlo import __version__
-    VERSION = __version__
-except ImportError:
-    VERSION = '1.0.0'
 
 # ============================== 分布式配置说明 ==============================
-# 
+RUN_MODE = 'distributed'  
+QUEUE_TYPE = 'redis'  
+FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'  
+DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline'
 # 本模板专为分布式部署设计，适用于以下场景：
 # - 大规模数据采集任务
 # - 需要多节点协同工作的项目

crawlo/templates/project/settings_gentle.py.tmpl

@@ -1,124 +1,91 @@
 # -*- coding: UTF-8 -*-
 """
-{{project_name}} 项目配置文件（温和版）
-=============================
-基于 Crawlo 框架的温和爬虫项目配置。
-适合对目标网站友好的低负载爬取。
+温和模式配置模板
+低负载配置，对目标网站友好
 """
 
-import os
-from crawlo.config import CrawloConfig
-
 # ============================== 项目基本信息 ==============================
 PROJECT_NAME = '{{project_name}}'
-try:
-    from crawlo import __version__
-    VERSION = __version__
-except ImportError:
-    VERSION = '1.0.0'
-
-# ============================== 温和模式配置说明 ==============================
-# 
-# 本模板专为对目标网站友好的低负载爬取设计，适用于以下场景：
-# - 需要避免对目标网站造成过大压力
-# - 长时间运行的监控类爬虫
-# - 对目标网站有友好性要求的项目
-# 
-# 运行模式特点：
-# - RUN_MODE = 'standalone'（单机模式）
-# - QUEUE_TYPE = 'memory'（使用内存队列）
-# - FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'（内存过滤器）
-# - DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline'（内存去重）
-# 
-# 配置特点：
-# - 低并发：CONCURRENCY = 2
-# - 高延迟：DOWNLOAD_DELAY = 3.0秒
-# - 随机化：启用RANDOMNESS增加随机性
-# - 连接池限制：减少连接数避免给服务器造成压力
-# 
-# 扩展建议：
-# - 如需跨会话去重，可将FILTER_CLASS和DEFAULT_DEDUP_PIPELINE改为Redis实现
-# - 可根据目标网站特性调整DOWNLOAD_DELAY和RANDOM_RANGE参数
-#
-# 🎯 最佳使用方式：
-# 推荐使用配置工厂方式创建温和模式配置：
-# from crawlo.config import CrawloConfig
-# config = CrawloConfig.presets().gentle()
-# process = CrawlerProcess(settings=config.to_dict())
-
-# ============================== 温和模式配置 ==============================
-# 使用配置工厂创建温和模式配置
-CONFIG = CrawloConfig.presets().gentle()
-
-# 获取配置
-locals().update(CONFIG.to_dict())
-
-# ============================== 网络请求配置 ==============================
-DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader"
-DOWNLOAD_TIMEOUT = 60
-VERIFY_SSL = True
-
-# ============================== 低并发配置 ==============================
-CONCURRENCY = 2
-MAX_RUNNING_SPIDERS = 1
-DOWNLOAD_DELAY = 3.0
-RANDOMNESS = True
-RANDOM_RANGE = (2.0, 5.0)
-
-# ============================== 连接池配置 ==============================
-CONNECTION_POOL_LIMIT = 10
-CONNECTION_POOL_LIMIT_PER_HOST = 5
-
-# ============================== 重试配置 ==============================
-MAX_RETRY_TIMES = 3
-RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524]
-IGNORE_HTTP_CODES = [403, 404]
+
+# ============================== 温和运行模式 ==============================
+# 运行模式：'standalone'(单机), 'distributed'(分布式), 'auto'(自动检测)
+RUN_MODE = 'standalone'  # 单机模式 - 适用于开发和小规模数据采集
+
+# 并发配置
+CONCURRENCY = 2  # 极低并发数以减少目标网站压力
+DOWNLOAD_DELAY = 3.0  # 增加延迟以降低目标网站压力
+RANDOMNESS = True  # 启用随机延迟
+RANDOM_RANGE = (0.5, 2.0)  # 随机延迟范围
 
 # ============================== 队列配置 ==============================
-SCHEDULER_MAX_QUEUE_SIZE = 1000
+
+# 队列类型：'auto'（自动选择）, 'memory'（内存队列）, 'redis'（分布式队列）
+QUEUE_TYPE = 'auto'  # 自动检测，如果Redis可用则使用Redis队列
+SCHEDULER_MAX_QUEUE_SIZE = 500
+SCHEDULER_QUEUE_NAME = f'crawlo:{{project_name}}:queue:requests'
 QUEUE_MAX_RETRIES = 3
 QUEUE_TIMEOUT = 300
 
-# ============================== 数据存储配置 ==============================
-# MySQL 配置
-MYSQL_HOST = os.getenv('MYSQL_HOST', '127.0.0.1')
-MYSQL_PORT = int(os.getenv('MYSQL_PORT', 3306))
-MYSQL_USER = os.getenv('MYSQL_USER', 'root')
-MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', '123456')
-MYSQL_DB = os.getenv('MYSQL_DB', '{{project_name}}')
-MYSQL_TABLE = '{{project_name}}_data'
-
-# MongoDB 配置
-MONGO_URI = os.getenv('MONGO_URI', 'mongodb://localhost:27017')
-MONGO_DATABASE = '{{project_name}}_db'
-MONGO_COLLECTION = '{{project_name}}_items'
-
-# ============================== 去重配置 ==============================
-# 明确指定温和模式下使用内存去重管道
+# ============================== 去重过滤配置 ==============================
+
+# 温和模式下使用内存去重管道和过滤器
 DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline'
 FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
+
+# --- Redis 配置（用于分布式去重和队列） ---
+REDIS_HOST = '127.0.0.1'
+REDIS_PORT = 6379
+REDIS_PASSWORD = ''  # 如果有密码，请填写
+
+# 根据是否有密码生成 URL
+if REDIS_PASSWORD:
+    REDIS_URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/0'
+else:
+    REDIS_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}/0'
+
+# Redis key配置已移至各组件中，使用统一的命名规范
+# crawlo:{project_name}:filter:fingerprint (请求去重)
+# crawlo:{project_name}:item:fingerprint (数据项去重)
+# crawlo:{project_name}:queue:requests (请求队列)
+# crawlo:{project_name}:queue:processing (处理中队列)
+# crawlo:{project_name}:queue:failed (失败队列)
+
 REDIS_TTL = 0
 CLEANUP_FP = 0
 FILTER_DEBUG = True
+DECODE_RESPONSES = True
+
+# ============================== 中间件配置 ==============================
 
-# ============================== 中间件与管道 ==============================
 MIDDLEWARES = [
+    # === 请求预处理阶段 ===
     'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
     'crawlo.middleware.download_delay.DownloadDelayMiddleware',
     'crawlo.middleware.default_header.DefaultHeaderMiddleware',
+    'crawlo.middleware.proxy.ProxyMiddleware',
+    'crawlo.middleware.offsite.OffsiteMiddleware', 
+    
+    # === 响应处理阶段 ===
     'crawlo.middleware.retry.RetryMiddleware',
     'crawlo.middleware.response_code.ResponseCodeMiddleware',
+    'crawlo.middleware.response_filter.ResponseFilterMiddleware',
 ]
 
+# ============================== 数据管道配置 ==============================
+
+# 数据处理管道（启用的存储方式）
 PIPELINES = [
     'crawlo.pipelines.console_pipeline.ConsolePipeline',
-    # '{{project_name}}.pipelines.DatabasePipeline',
+    # '{{project_name}}.pipelines.DatabasePipeline',        # 自定义数据库管道
+    # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',  # MySQL 存储
+    # 'crawlo.pipelines.mongo_pipeline.MongoPipeline',      # MongoDB 存储
 ]
 
-# 明确添加内存去重管道到管道列表开头
+# 明确添加默认去重管道到管道列表开头
 PIPELINES.insert(0, DEFAULT_DEDUP_PIPELINE)
 
 # ============================== 扩展组件 ==============================
+
 EXTENSIONS = [
     'crawlo.extension.log_interval.LogIntervalExtension',
     'crawlo.extension.log_stats.LogStats',
@@ -126,9 +93,9 @@ EXTENSIONS = [
 ]
 
 # ============================== 日志配置 ==============================
+
 LOG_LEVEL = 'INFO'
-LOG_FILE = f'logs/{{project_name}}.log'
 STATS_DUMP = True
-
-# ============================== 自定义配置 ==============================
-# 在此处添加项目特定的配置项
+LOG_FILE = f'logs/{{project_name}}.log'
+LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s： %(message)s'
+LOG_ENCODING = 'utf-8'

crawlo/templates/project/settings_high_performance.py.tmpl

@@ -1,156 +1,135 @@
 # -*- coding: UTF-8 -*-
 """
-{{project_name}} 项目配置文件（高性能版）
-=============================
-基于 Crawlo 框架的高性能爬虫项目配置。
-针对大规模、高并发场景优化。
+高性能模式配置模板
+针对大规模高并发优化
 """
 
-import os
-from crawlo.config import CrawloConfig
-
 # ============================== 项目基本信息 ==============================
 PROJECT_NAME = '{{project_name}}'
-try:
-    from crawlo import __version__
-    VERSION = __version__
-except ImportError:
-    VERSION = '1.0.0'
-
-# ============================== 高性能配置 ==============================
-# 使用配置工厂创建高性能配置
-CONFIG = CrawloConfig.presets().large_scale(
-    redis_host=os.getenv('REDIS_HOST', '127.0.0.1'),
-    project_name='{{project_name}}'
-)
-
-# 获取配置
-locals().update(CONFIG.to_dict())
-
-# ============================== 网络请求配置 ==============================
-DOWNLOADER = "crawlo.downloader.cffi_downloader.CurlCffiDownloader"
-DOWNLOAD_TIMEOUT = 30
-VERIFY_SSL = True
-USE_SESSION = True
-
-# ============================== 高并发配置 ==============================
-CONCURRENCY = 32
-MAX_RUNNING_SPIDERS = 10
-DOWNLOAD_DELAY = 0.5
-RANDOMNESS = True
-RANDOM_RANGE = (0.8, 1.2)
-
-# ============================== 连接池配置 ==============================
-CONNECTION_POOL_LIMIT = 100
-CONNECTION_POOL_LIMIT_PER_HOST = 50
 
-# ============================== 重试配置 ==============================
-MAX_RETRY_TIMES = 5
-RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524]
-IGNORE_HTTP_CODES = [403, 404]
+# ============================== 高性能运行模式 ==============================
+# 运行模式：'standalone'(单机), 'distributed'(分布式), 'auto'(自动检测)
+RUN_MODE = 'standalone'  # 单机模式 - 适用于开发和小规模数据采集
+
+# 并发配置
+CONCURRENCY = 32  # 高并发数以充分利用系统资源
+DOWNLOAD_DELAY = 0.1  # 极小延迟以提高吞吐量
+RANDOMNESS = False  # 禁用随机延迟以保证性能
 
 # ============================== 队列配置 ==============================
-SCHEDULER_MAX_QUEUE_SIZE = 10000
+
+# 队列类型：'auto'（自动选择）, 'memory'（内存队列）, 'redis'（分布式队列）
+QUEUE_TYPE = 'auto'  # 自动检测，如果Redis可用则使用Redis队列
+SCHEDULER_MAX_QUEUE_SIZE = 5000
 SCHEDULER_QUEUE_NAME = f'crawlo:{{project_name}}:queue:requests'
-QUEUE_MAX_RETRIES = 5
+QUEUE_MAX_RETRIES = 3
 QUEUE_TIMEOUT = 300
-LARGE_SCALE_BATCH_SIZE = 2000
-LARGE_SCALE_CHECKPOINT_INTERVAL = 5000
 
-# ============================== Redis 配置 ==============================
-REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')
-REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
-REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
-REDIS_DB = int(os.getenv('REDIS_DB', 0))
+# ============================== 去重过滤配置 ==============================
+
+# 高性能模式下，如果Redis可用则使用Redis去重，否则使用内存去重
+DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline'
+FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'
+
+# --- Redis 配置（用于分布式去重和队列） ---
+REDIS_HOST = '127.0.0.1'
+REDIS_PORT = 6379
+REDIS_PASSWORD = ''  # 如果有密码，请填写
 
 # 根据是否有密码生成 URL
 if REDIS_PASSWORD:
-    REDIS_URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+    REDIS_URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/0'
 else:
-    REDIS_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
-
-# ============================== 数据存储配置 ==============================
-# MySQL 配置
-MYSQL_HOST = os.getenv('MYSQL_HOST', '127.0.0.1')
-MYSQL_PORT = int(os.getenv('MYSQL_PORT', 3306))
-MYSQL_USER = os.getenv('MYSQL_USER', 'root')
-MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', '123456')
-MYSQL_DB = os.getenv('MYSQL_DB', '{{project_name}}')
-MYSQL_TABLE = '{{project_name}}_data'
-MYSQL_BATCH_SIZE = 200
-MYSQL_USE_BATCH = True
-MYSQL_POOL_MIN = 10
-MYSQL_POOL_MAX = 50
-
-# MongoDB 配置
-MONGO_URI = os.getenv('MONGO_URI', 'mongodb://localhost:27017')
-MONGO_DATABASE = '{{project_name}}_db'
-MONGO_COLLECTION = '{{project_name}}_items'
-MONGO_BATCH_SIZE = 200
-MONGO_USE_BATCH = True
-MONGO_MAX_POOL_SIZE = 300
-MONGO_MIN_POOL_SIZE = 50
-
-# ============================== 去重配置 ==============================
+    REDIS_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}/0'
+
+# Redis key配置已移至各组件中，使用统一的命名规范
+# crawlo:{project_name}:filter:fingerprint (请求去重)
+# crawlo:{project_name}:item:fingerprint (数据项去重)
+# crawlo:{project_name}:queue:requests (请求队列)
+# crawlo:{project_name}:queue:processing (处理中队列)
+# crawlo:{project_name}:queue:failed (失败队列)
+
 REDIS_TTL = 0
 CLEANUP_FP = 0
-FILTER_DEBUG = False  # 生产环境关闭调试日志
+FILTER_DEBUG = True
+DECODE_RESPONSES = True
+
+# ============================== 中间件配置 ==============================
 
-# ============================== 中间件与管道 ==============================
 MIDDLEWARES = [
+    # === 请求预处理阶段 ===
     'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
     'crawlo.middleware.download_delay.DownloadDelayMiddleware',
     'crawlo.middleware.default_header.DefaultHeaderMiddleware',
     'crawlo.middleware.proxy.ProxyMiddleware',
+    'crawlo.middleware.offsite.OffsiteMiddleware', 
+    
+    # === 响应处理阶段 ===
     'crawlo.middleware.retry.RetryMiddleware',
     'crawlo.middleware.response_code.ResponseCodeMiddleware',
     'crawlo.middleware.response_filter.ResponseFilterMiddleware',
 ]
 
+# ============================== 数据管道配置 ==============================
+
+# 数据处理管道（启用的存储方式）
 PIPELINES = [
     'crawlo.pipelines.console_pipeline.ConsolePipeline',
-    # '{{project_name}}.pipelines.DatabasePipeline',
-    # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',
-    # 'crawlo.pipelines.mongo_pipeline.MongoPipeline',
+    # '{{project_name}}.pipelines.DatabasePipeline',        # 自定义数据库管道
+    # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',  # MySQL 存储
+    # 'crawlo.pipelines.mongo_pipeline.MongoPipeline',      # MongoDB 存储
 ]
 
+# 明确添加默认去重管道到管道列表开头
+PIPELINES.insert(0, DEFAULT_DEDUP_PIPELINE)
+
 # ============================== 扩展组件 ==============================
+
 EXTENSIONS = [
     'crawlo.extension.log_interval.LogIntervalExtension',
     'crawlo.extension.log_stats.LogStats',
     'crawlo.extension.logging_extension.CustomLoggerExtension',
-    # 'crawlo.extension.memory_monitor.MemoryMonitorExtension',
-    # 'crawlo.extension.request_recorder.RequestRecorderExtension',
-    # 'crawlo.extension.performance_profiler.PerformanceProfilerExtension',
+    # 'crawlo.extension.memory_monitor.MemoryMonitorExtension',  # 内存监控
+    # 'crawlo.extension.request_recorder.RequestRecorderExtension',  # 请求记录
+    # 'crawlo.extension.performance_profiler.PerformanceProfilerExtension',  # 性能分析
+    # 'crawlo.extension.health_check.HealthCheckExtension',  # 健康检查
 ]
 
 # ============================== 日志配置 ==============================
+
 LOG_LEVEL = 'INFO'
-LOG_FILE = f'logs/{{project_name}}.log'
 STATS_DUMP = True
+LOG_FILE = f'logs/{{project_name}}.log'
+LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s： %(message)s'
+LOG_ENCODING = 'utf-8'
+
+# ============================== 性能优化配置 ==============================
 
-# ============================== 代理配置 ==============================
-PROXY_ENABLED = False
-PROXY_API_URL = ""
-PROXY_EXTRACTOR = "proxy"
-PROXY_REFRESH_INTERVAL = 30
-PROXY_API_TIMEOUT = 5
-
-# ============================== 浏览器指纹配置 ==============================
-CURL_BROWSER_TYPE = "chrome"
-CURL_BROWSER_VERSION_MAP = {
-    "chrome": "chrome136",
-    "edge": "edge101",
-    "safari": "safari184",
-    "firefox": "firefox135",
-}
-
-# ============================== 下载器优化配置 ==============================
+# 连接池配置
+CONNECTION_POOL_LIMIT = 100
+DOWNLOAD_MAXSIZE = 10 * 1024 * 1024    # 10MB
+DOWNLOAD_WARN_SIZE = 1024 * 1024       # 1MB
+
+# 下载器优化配置
+DOWNLOADER_HEALTH_CHECK = True
+HEALTH_CHECK_INTERVAL = 30
+
+# 请求统计配置
+REQUEST_STATS_ENABLED = True
+STATS_RESET_ON_START = False
+
+# HttpX 下载器专用配置
 HTTPX_HTTP2 = True
 HTTPX_FOLLOW_REDIRECTS = True
+
+# AioHttp 下载器专用配置
 AIOHTTP_AUTO_DECOMPRESS = True
+AIOHTTP_FORCE_CLOSE = False
+
+# 通用优化配置
 CONNECTION_TTL_DNS_CACHE = 300
 CONNECTION_KEEPALIVE_TIMEOUT = 15
 
-# ============================== 自定义配置 ==============================
-# 在此处添加项目特定的配置项
+# 性能监控
+ENABLE_PERFORMANCE_MONITORING = True
+MEMORY_USAGE_WARNING_THRESHOLD = 800  # MB