crawlo 1.1.2__py3-none-any.whl → 1.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of crawlo might be problematic. Click here for more details.

Files changed (113) hide show
  1. crawlo/__init__.py +34 -34
  2. crawlo/__version__.py +1 -1
  3. crawlo/cli.py +40 -40
  4. crawlo/commands/__init__.py +13 -13
  5. crawlo/commands/check.py +594 -594
  6. crawlo/commands/genspider.py +151 -151
  7. crawlo/commands/list.py +155 -155
  8. crawlo/commands/run.py +285 -285
  9. crawlo/commands/startproject.py +196 -196
  10. crawlo/commands/stats.py +188 -188
  11. crawlo/commands/utils.py +186 -186
  12. crawlo/config.py +279 -279
  13. crawlo/core/__init__.py +2 -2
  14. crawlo/core/engine.py +171 -171
  15. crawlo/core/enhanced_engine.py +189 -189
  16. crawlo/core/processor.py +40 -40
  17. crawlo/core/scheduler.py +166 -162
  18. crawlo/crawler.py +1027 -1027
  19. crawlo/downloader/__init__.py +242 -242
  20. crawlo/downloader/aiohttp_downloader.py +212 -212
  21. crawlo/downloader/cffi_downloader.py +251 -251
  22. crawlo/downloader/httpx_downloader.py +259 -257
  23. crawlo/event.py +11 -11
  24. crawlo/exceptions.py +82 -78
  25. crawlo/extension/__init__.py +31 -31
  26. crawlo/extension/log_interval.py +49 -49
  27. crawlo/extension/log_stats.py +44 -44
  28. crawlo/extension/logging_extension.py +34 -34
  29. crawlo/filters/__init__.py +154 -154
  30. crawlo/filters/aioredis_filter.py +242 -242
  31. crawlo/filters/memory_filter.py +269 -269
  32. crawlo/items/__init__.py +23 -23
  33. crawlo/items/base.py +21 -21
  34. crawlo/items/fields.py +53 -53
  35. crawlo/items/items.py +104 -104
  36. crawlo/middleware/__init__.py +21 -21
  37. crawlo/middleware/default_header.py +32 -32
  38. crawlo/middleware/download_delay.py +28 -28
  39. crawlo/middleware/middleware_manager.py +135 -135
  40. crawlo/middleware/proxy.py +248 -248
  41. crawlo/middleware/request_ignore.py +30 -30
  42. crawlo/middleware/response_code.py +18 -18
  43. crawlo/middleware/response_filter.py +26 -26
  44. crawlo/middleware/retry.py +125 -125
  45. crawlo/mode_manager.py +200 -200
  46. crawlo/network/__init__.py +21 -21
  47. crawlo/network/request.py +311 -311
  48. crawlo/network/response.py +271 -269
  49. crawlo/pipelines/__init__.py +22 -13
  50. crawlo/pipelines/bloom_dedup_pipeline.py +157 -0
  51. crawlo/pipelines/console_pipeline.py +39 -39
  52. crawlo/pipelines/csv_pipeline.py +316 -316
  53. crawlo/pipelines/database_dedup_pipeline.py +225 -0
  54. crawlo/pipelines/json_pipeline.py +218 -218
  55. crawlo/pipelines/memory_dedup_pipeline.py +116 -0
  56. crawlo/pipelines/mongo_pipeline.py +116 -116
  57. crawlo/pipelines/mysql_pipeline.py +195 -195
  58. crawlo/pipelines/pipeline_manager.py +56 -56
  59. crawlo/pipelines/redis_dedup_pipeline.py +163 -0
  60. crawlo/project.py +153 -153
  61. crawlo/queue/pqueue.py +37 -37
  62. crawlo/queue/queue_manager.py +307 -303
  63. crawlo/queue/redis_priority_queue.py +208 -191
  64. crawlo/settings/__init__.py +7 -7
  65. crawlo/settings/default_settings.py +245 -226
  66. crawlo/settings/setting_manager.py +99 -99
  67. crawlo/spider/__init__.py +639 -639
  68. crawlo/stats_collector.py +59 -59
  69. crawlo/subscriber.py +106 -106
  70. crawlo/task_manager.py +30 -30
  71. crawlo/templates/crawlo.cfg.tmpl +10 -10
  72. crawlo/templates/project/__init__.py.tmpl +3 -3
  73. crawlo/templates/project/items.py.tmpl +17 -17
  74. crawlo/templates/project/middlewares.py.tmpl +86 -86
  75. crawlo/templates/project/pipelines.py.tmpl +341 -335
  76. crawlo/templates/project/run.py.tmpl +251 -238
  77. crawlo/templates/project/settings.py.tmpl +250 -247
  78. crawlo/templates/project/spiders/__init__.py.tmpl +5 -5
  79. crawlo/templates/spider/spider.py.tmpl +177 -177
  80. crawlo/utils/__init__.py +7 -7
  81. crawlo/utils/controlled_spider_mixin.py +439 -335
  82. crawlo/utils/date_tools.py +233 -233
  83. crawlo/utils/db_helper.py +343 -343
  84. crawlo/utils/func_tools.py +82 -82
  85. crawlo/utils/large_scale_config.py +286 -286
  86. crawlo/utils/large_scale_helper.py +343 -343
  87. crawlo/utils/log.py +128 -128
  88. crawlo/utils/queue_helper.py +175 -175
  89. crawlo/utils/request.py +267 -267
  90. crawlo/utils/request_serializer.py +219 -219
  91. crawlo/utils/spider_loader.py +62 -62
  92. crawlo/utils/system.py +11 -11
  93. crawlo/utils/tools.py +4 -4
  94. crawlo/utils/url.py +39 -39
  95. {crawlo-1.1.2.dist-info → crawlo-1.1.3.dist-info}/METADATA +635 -567
  96. crawlo-1.1.3.dist-info/RECORD +113 -0
  97. examples/__init__.py +7 -7
  98. examples/controlled_spider_example.py +205 -0
  99. tests/__init__.py +7 -7
  100. tests/test_final_validation.py +153 -153
  101. tests/test_proxy_health_check.py +32 -32
  102. tests/test_proxy_middleware_integration.py +136 -136
  103. tests/test_proxy_providers.py +56 -56
  104. tests/test_proxy_stats.py +19 -19
  105. tests/test_proxy_strategies.py +59 -59
  106. tests/test_redis_config.py +28 -28
  107. tests/test_redis_queue.py +224 -224
  108. tests/test_request_serialization.py +70 -70
  109. tests/test_scheduler.py +241 -241
  110. crawlo-1.1.2.dist-info/RECORD +0 -108
  111. {crawlo-1.1.2.dist-info → crawlo-1.1.3.dist-info}/WHEEL +0 -0
  112. {crawlo-1.1.2.dist-info → crawlo-1.1.3.dist-info}/entry_points.txt +0 -0
  113. {crawlo-1.1.2.dist-info → crawlo-1.1.3.dist-info}/top_level.txt +0 -0
@@ -1,226 +1,245 @@
1
- #!/usr/bin/python
2
- # -*- coding: UTF-8 -*-
3
- """
4
- ==================================
5
- Crawlo 项目配置文件
6
- ==================================
7
- 说明:
8
- - 所有配置项均已按功能模块分类。
9
- - 支持通过环境变量覆盖部分敏感配置(如 Redis、MySQL 密码等)。
10
- - 可根据需求启用/禁用组件(如 MySQL、Redis、Proxy 等)。
11
- """
12
- import os
13
-
14
- # ============================== 核心信息 ==============================
15
- PROJECT_NAME = 'crawlo'
16
-
17
- # ============================== 网络请求配置 ==============================
18
-
19
- # 下载器选择(支持三种方式)
20
- # 方式1: 直接指定类路径
21
- DOWNLOADER = "crawlo.downloader.aiohttp_downloader.AioHttpDownloader"
22
- # DOWNLOADER = "crawlo.downloader.cffi_downloader.CurlCffiDownloader" # 支持浏览器指纹
23
- # DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader" # 支持HTTP/2
24
-
25
- # 方式2: 使用简化名称(推荐)
26
- # DOWNLOADER_TYPE = 'aiohttp' # 可选: aiohttp, httpx, curl_cffi, cffi
27
-
28
- # 方式3: 在Spider中动态选择
29
- # 可以在Spider类中设置 custom_settings = {'DOWNLOADER_TYPE': 'httpx'}
30
-
31
- # 请求超时与安全
32
- DOWNLOAD_TIMEOUT = 30 # 下载超时时间(秒)
33
- VERIFY_SSL = True # 是否验证 SSL 证书
34
- USE_SESSION = True # 是否使用持久化会话(aiohttp 特有)
35
-
36
- # 请求延迟控制
37
- DOWNLOAD_DELAY = 1.0 # 基础延迟(秒)
38
- RANDOM_RANGE = (0.8, 1.2) # 随机延迟系数范围
39
- RANDOMNESS = True # 是否启用随机延迟
40
-
41
- # 重试策略
42
- MAX_RETRY_TIMES = 3 # 最大重试次数
43
- RETRY_PRIORITY = -1 # 重试请求的优先级调整
44
- RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524] # 触发重试的状态码
45
- IGNORE_HTTP_CODES = [403, 404] # 直接标记成功、不重试的状态码
46
- ALLOWED_CODES = [] # 允许的状态码(空表示不限制)
47
-
48
- # 连接与响应大小限制
49
- CONNECTION_POOL_LIMIT = 50 # 最大并发连接数(连接池大小)
50
- CONNECTION_POOL_LIMIT_PER_HOST = 20 # 每个主机的连接池大小
51
- DOWNLOAD_MAXSIZE = 10 * 1024 * 1024 # 最大响应体大小(10MB)
52
- DOWNLOAD_WARN_SIZE = 1024 * 1024 # 响应体警告阈值(1MB)
53
- DOWNLOAD_RETRY_TIMES = MAX_RETRY_TIMES # 下载器内部重试次数(复用全局)
54
-
55
- # 下载统计配置
56
- DOWNLOADER_STATS = True # 是否启用下载器统计功能
57
- DOWNLOAD_STATS = True # 是否记录下载时间和大小统计
58
-
59
- # ============================== 并发与调度 ==============================
60
-
61
- CONCURRENCY = 8 # 单个爬虫的并发请求数
62
- INTERVAL = 5 # 日志统计输出间隔(秒)
63
- DEPTH_PRIORITY = 1 # 深度优先策略优先级
64
- MAX_RUNNING_SPIDERS = 3 # 最大同时运行的爬虫数
65
-
66
- # ============================== 队列配置 ==============================
67
-
68
- # 🎯 运行模式选择:'standalone'(单机), 'distributed'(分布式), 'auto'(自动检测)
69
- RUN_MODE = 'standalone' # 默认单机模式,简单易用
70
-
71
- # 队列类型选择:'memory'(内存), 'redis'(分布式), 'auto'(自动选择)
72
- QUEUE_TYPE = 'memory' # 默认内存队列,无需外部依赖
73
- SCHEDULER_MAX_QUEUE_SIZE = 2000 # 调度器队列最大容量
74
- SCHEDULER_QUEUE_NAME = 'crawlo:requests' # Redis 队列名称
75
- QUEUE_MAX_RETRIES = 3 # 队列操作最大重试次数
76
- QUEUE_TIMEOUT = 300 # 队列操作超时时间(秒)
77
-
78
- # 大规模爬取优化配置
79
- LARGE_SCALE_BATCH_SIZE = 1000 # 批处理大小
80
- LARGE_SCALE_CHECKPOINT_INTERVAL = 5000 # 进度保存间隔
81
- LARGE_SCALE_MAX_MEMORY_USAGE = 500 # 最大内存使用量(MB)
82
-
83
- # ============================== 数据存储配置 ==============================
84
-
85
- # --- MySQL 配置 ---
86
- MYSQL_HOST = '127.0.0.1'
87
- MYSQL_PORT = 3306
88
- MYSQL_USER = 'root'
89
- MYSQL_PASSWORD = '123456'
90
- MYSQL_DB = 'crawl'
91
- MYSQL_TABLE = 'crawlo'
92
- MYSQL_BATCH_SIZE = 100 # 批量插入条数
93
-
94
- # MySQL 连接池
95
- MYSQL_FLUSH_INTERVAL = 5 # 缓存刷新间隔(秒)
96
- MYSQL_POOL_MIN = 5
97
- MYSQL_POOL_MAX = 20
98
- MYSQL_ECHO = False # 是否打印 SQL 日志
99
-
100
- # --- MongoDB 配置 ---
101
- MONGO_URI = 'mongodb://user:password@host:27017'
102
- MONGO_DATABASE = 'scrapy_data'
103
- MONGO_COLLECTION = 'crawled_items'
104
- MONGO_MAX_POOL_SIZE = 200
105
- MONGO_MIN_POOL_SIZE = 20
106
-
107
- # ============================== 去重过滤配置 ==============================
108
-
109
- # 请求指纹存储目录(文件过滤器使用)
110
- REQUEST_DIR = '.'
111
-
112
- # 去重过滤器类(二选一)
113
- FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
114
- # FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter' # 分布式去重
115
-
116
- # --- Redis 过滤器配置 ---
117
- REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')
118
- REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
119
- REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '') # 默认无密码
120
- # 🔧 根据是否有密码生成不同的 URL 格式
121
- if REDIS_PASSWORD:
122
- REDIS_URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/0'
123
- else:
124
- REDIS_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}/0'
125
- REDIS_KEY = 'request_fingerprint' # Redis 中存储指纹的键名
126
- REDIS_TTL = 0 # 指纹过期时间(0 表示永不过期)
127
- CLEANUP_FP = 0 # 程序结束时是否清理指纹(0=不清理)
128
- FILTER_DEBUG = True # 是否开启去重调试日志
129
- DECODE_RESPONSES = True # Redis 返回是否解码为字符串
130
-
131
- # ============================== 中间件配置 ==============================
132
-
133
- MIDDLEWARES = [
134
- # === 请求预处理阶段 ===
135
- 'crawlo.middleware.request_ignore.RequestIgnoreMiddleware', # 1. 忽略无效请求
136
- 'crawlo.middleware.download_delay.DownloadDelayMiddleware', # 2. 控制请求频率
137
- 'crawlo.middleware.default_header.DefaultHeaderMiddleware', # 3. 添加默认请求头
138
- 'crawlo.middleware.proxy.ProxyMiddleware', # 4. 设置代理
139
-
140
- # === 响应处理阶段 ===
141
- 'crawlo.middleware.retry.RetryMiddleware', # 5. 失败请求重试
142
- 'crawlo.middleware.response_code.ResponseCodeMiddleware', # 6. 处理特殊状态码
143
- 'crawlo.middleware.response_filter.ResponseFilterMiddleware', # 7. 响应内容过滤
144
- ]
145
-
146
- # ============================== 扩展与管道 ==============================
147
-
148
- # 数据处理管道(启用的存储方式)
149
- PIPELINES = [
150
- 'crawlo.pipelines.console_pipeline.ConsolePipeline', # 控制台输出
151
- # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline', # MySQL 存储(可选)
152
- ]
153
-
154
- # 扩展组件(监控与日志)
155
- EXTENSIONS = [
156
- 'crawlo.extension.log_interval.LogIntervalExtension', # 定时日志
157
- 'crawlo.extension.log_stats.LogStats', # 统计信息
158
- 'crawlo.extension.logging_extension.CustomLoggerExtension', # 自定义日志
159
- ]
160
-
161
- # ============================== 日志与监控 ==============================
162
-
163
- LOG_LEVEL = 'INFO' # 日志级别: DEBUG/INFO/WARNING/ERROR
164
- STATS_DUMP = True # 是否周期性输出统计信息
165
- LOG_FILE = f'logs/{PROJECT_NAME}.log' # 日志文件路径
166
- LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s'
167
- LOG_ENCODING = 'utf-8'
168
-
169
- # ============================== 代理配置 ==============================
170
-
171
- PROXY_ENABLED = False # 是否启用代理
172
- PROXY_API_URL = "https://api.proxyprovider.com/get" # 代理获取接口(请替换为真实地址)
173
-
174
- # 代理提取方式(支持字段路径或函数)
175
- PROXY_EXTRACTOR = "proxy" # 如返回 {"proxy": "http://1.1.1.1:8080"}
176
-
177
- # 代理刷新控制
178
- PROXY_REFRESH_INTERVAL = 60 # 代理刷新间隔(秒)
179
- PROXY_API_TIMEOUT = 10 # 请求代理 API 超时时间
180
-
181
- # ============================== Curl-Cffi 特有配置 ==============================
182
-
183
- # 浏览器指纹模拟(仅 CurlCffi 下载器有效)
184
- CURL_BROWSER_TYPE = "chrome" # 可选: chrome, edge, safari, firefox 或版本如 chrome136
185
-
186
- # 自定义浏览器版本映射(可覆盖默认行为)
187
- CURL_BROWSER_VERSION_MAP = {
188
- "chrome": "chrome136",
189
- "edge": "edge101",
190
- "safari": "safari184",
191
- "firefox": "firefox135",
192
- # 示例:旧版本测试
193
- # "chrome_legacy": "chrome110",
194
- }
195
-
196
- # Curl-Cffi 优化配置
197
- CURL_RANDOMIZE_DELAY = False # 是否启用随机延迟
198
- CURL_RETRY_BACKOFF = True # 是否启用指数退避重试
199
-
200
- # 默认请求头(可被 Spider 覆盖)
201
- DEFAULT_REQUEST_HEADERS = {
202
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
203
- '(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
204
- }
205
-
206
- # ============================== 下载器优化配置 ==============================
207
-
208
- # 下载器健康检查
209
- DOWNLOADER_HEALTH_CHECK = True # 是否启用下载器健康检查
210
- HEALTH_CHECK_INTERVAL = 60 # 健康检查间隔(秒)
211
-
212
- # 请求统计配置
213
- REQUEST_STATS_ENABLED = True # 是否启用请求统计
214
- STATS_RESET_ON_START = False # 启动时是否重置统计
215
-
216
- # HttpX 下载器专用配置
217
- HTTPX_HTTP2 = True # 是否启用HTTP/2支持
218
- HTTPX_FOLLOW_REDIRECTS = True # 是否自动跟随重定向
219
-
220
- # AioHttp 下载器专用配置
221
- AIOHTTP_AUTO_DECOMPRESS = True # 是否自动解压响应
222
- AIOHTTP_FORCE_CLOSE = False # 是否强制关闭连接
223
-
224
- # 通用优化配置
225
- CONNECTION_TTL_DNS_CACHE = 300 # DNS缓存TTL(秒)
226
- CONNECTION_KEEPALIVE_TIMEOUT = 15 # Keep-Alive超时(秒)
1
+ #!/usr/bin/python
2
+ # -*- coding: UTF-8 -*-
3
+ """
4
+ ==================================
5
+ Crawlo 项目配置文件
6
+ ==================================
7
+ 说明:
8
+ - 所有配置项均已按功能模块分类。
9
+ - 支持通过环境变量覆盖部分敏感配置(如 Redis、MySQL 密码等)。
10
+ - 可根据需求启用/禁用组件(如 MySQL、Redis、Proxy 等)。
11
+ """
12
+ import os
13
+
14
+ # ============================== 核心信息 ==============================
15
+ PROJECT_NAME = 'crawlo'
16
+
17
+ # ============================== 网络请求配置 ==============================
18
+
19
+ # 下载器选择(支持三种方式)
20
+ # 方式1: 直接指定类路径
21
+ DOWNLOADER = "crawlo.downloader.aiohttp_downloader.AioHttpDownloader"
22
+ # DOWNLOADER = "crawlo.downloader.cffi_downloader.CurlCffiDownloader" # 支持浏览器指纹
23
+ # DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader" # 支持HTTP/2
24
+
25
+ # 方式2: 使用简化名称(推荐)
26
+ # DOWNLOADER_TYPE = 'aiohttp' # 可选: aiohttp, httpx, curl_cffi, cffi
27
+
28
+ # 方式3: 在Spider中动态选择
29
+ # 可以在Spider类中设置 custom_settings = {'DOWNLOADER_TYPE': 'httpx'}
30
+
31
+ # 请求超时与安全
32
+ DOWNLOAD_TIMEOUT = 30 # 下载超时时间(秒)
33
+ VERIFY_SSL = True # 是否验证 SSL 证书
34
+ USE_SESSION = True # 是否使用持久化会话(aiohttp 特有)
35
+
36
+ # 请求延迟控制
37
+ DOWNLOAD_DELAY = 1.0 # 基础延迟(秒)
38
+ RANDOM_RANGE = (0.8, 1.2) # 随机延迟系数范围
39
+ RANDOMNESS = True # 是否启用随机延迟
40
+
41
+ # 重试策略
42
+ MAX_RETRY_TIMES = 3 # 最大重试次数
43
+ RETRY_PRIORITY = -1 # 重试请求的优先级调整
44
+ RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524] # 触发重试的状态码
45
+ IGNORE_HTTP_CODES = [403, 404] # 直接标记成功、不重试的状态码
46
+ ALLOWED_CODES = [] # 允许的状态码(空表示不限制)
47
+
48
+ # 连接与响应大小限制
49
+ CONNECTION_POOL_LIMIT = 50 # 最大并发连接数(连接池大小)
50
+ CONNECTION_POOL_LIMIT_PER_HOST = 20 # 每个主机的连接池大小
51
+ DOWNLOAD_MAXSIZE = 10 * 1024 * 1024 # 最大响应体大小(10MB)
52
+ DOWNLOAD_WARN_SIZE = 1024 * 1024 # 响应体警告阈值(1MB)
53
+ DOWNLOAD_RETRY_TIMES = MAX_RETRY_TIMES # 下载器内部重试次数(复用全局)
54
+
55
+ # 下载统计配置
56
+ DOWNLOADER_STATS = True # 是否启用下载器统计功能
57
+ DOWNLOAD_STATS = True # 是否记录下载时间和大小统计
58
+
59
+ # ============================== 并发与调度 ==============================
60
+
61
+ CONCURRENCY = 8 # 单个爬虫的并发请求数
62
+ INTERVAL = 5 # 日志统计输出间隔(秒)
63
+ DEPTH_PRIORITY = 1 # 深度优先策略优先级
64
+ MAX_RUNNING_SPIDERS = 3 # 最大同时运行的爬虫数
65
+
66
+ # ============================== 队列配置 ==============================
67
+
68
+ # 🎯 运行模式选择:'standalone'(单机), 'distributed'(分布式), 'auto'(自动检测)
69
+ RUN_MODE = 'standalone' # 默认单机模式,简单易用
70
+
71
+ # 队列类型选择:'memory'(内存), 'redis'(分布式), 'auto'(自动选择)
72
+ QUEUE_TYPE = 'memory' # 默认内存队列,无需外部依赖
73
+ SCHEDULER_MAX_QUEUE_SIZE = 2000 # 调度器队列最大容量
74
+ SCHEDULER_QUEUE_NAME = 'crawlo:requests' # Redis 队列名称
75
+ QUEUE_MAX_RETRIES = 3 # 队列操作最大重试次数
76
+ QUEUE_TIMEOUT = 300 # 队列操作超时时间(秒)
77
+
78
+ # 大规模爬取优化配置
79
+ LARGE_SCALE_BATCH_SIZE = 1000 # 批处理大小
80
+ LARGE_SCALE_CHECKPOINT_INTERVAL = 5000 # 进度保存间隔
81
+ LARGE_SCALE_MAX_MEMORY_USAGE = 500 # 最大内存使用量(MB)
82
+
83
+ # ============================== 数据存储配置 ==============================
84
+
85
+ # --- MySQL 配置 ---
86
+ MYSQL_HOST = '127.0.0.1'
87
+ MYSQL_PORT = 3306
88
+ MYSQL_USER = 'root'
89
+ MYSQL_PASSWORD = '123456'
90
+ MYSQL_DB = 'crawl'
91
+ MYSQL_TABLE = 'crawlo'
92
+ MYSQL_BATCH_SIZE = 100 # 批量插入条数
93
+
94
+ # MySQL 连接池
95
+ MYSQL_FLUSH_INTERVAL = 5 # 缓存刷新间隔(秒)
96
+ MYSQL_POOL_MIN = 5
97
+ MYSQL_POOL_MAX = 20
98
+ MYSQL_ECHO = False # 是否打印 SQL 日志
99
+
100
+ # --- MongoDB 配置 ---
101
+ MONGO_URI = 'mongodb://user:password@host:27017'
102
+ MONGO_DATABASE = 'scrapy_data'
103
+ MONGO_COLLECTION = 'crawled_items'
104
+ MONGO_MAX_POOL_SIZE = 200
105
+ MONGO_MIN_POOL_SIZE = 20
106
+
107
+ # ============================== 去重过滤配置 ==============================
108
+
109
+ # 请求指纹存储目录(文件过滤器使用)
110
+ REQUEST_DIR = '.'
111
+
112
+ # 根据运行模式自动选择去重管道
113
+ # 单机模式默认使用内存去重管道
114
+ # 分布式模式默认使用Redis去重管道
115
+ if RUN_MODE == 'distributed':
116
+ # 分布式模式下默认使用Redis去重管道
117
+ DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.RedisDedupPipeline'
118
+ else:
119
+ # 单机模式下默认使用内存去重管道
120
+ DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.MemoryDedupPipeline'
121
+
122
+ # 去重过滤器类(二选一)
123
+ FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
124
+ # FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter' # 分布式去重
125
+
126
+ # --- Redis 过滤器配置 ---
127
+ REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')
128
+ REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
129
+ REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '') # 默认无密码
130
+ REDIS_DB = int(os.getenv('REDIS_DB', 0)) # Redis 数据库编号,默认为 0
131
+ # 🔧 根据是否有密码生成不同的 URL 格式
132
+ if REDIS_PASSWORD:
133
+ REDIS_URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
134
+ else:
135
+ REDIS_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
136
+ REDIS_KEY = 'request_fingerprint' # Redis 中存储指纹的键名
137
+ REDIS_TTL = 0 # 指纹过期时间(0 表示永不过期)
138
+ CLEANUP_FP = 0 # 程序结束时是否清理指纹(0=不清理)
139
+ FILTER_DEBUG = True # 是否开启去重调试日志
140
+ DECODE_RESPONSES = True # Redis 返回是否解码为字符串
141
+
142
+ # ============================== 中间件配置 ==============================
143
+
144
+ MIDDLEWARES = [
145
+ # === 请求预处理阶段 ===
146
+ 'crawlo.middleware.request_ignore.RequestIgnoreMiddleware', # 1. 忽略无效请求
147
+ 'crawlo.middleware.download_delay.DownloadDelayMiddleware', # 2. 控制请求频率
148
+ 'crawlo.middleware.default_header.DefaultHeaderMiddleware', # 3. 添加默认请求头
149
+ 'crawlo.middleware.proxy.ProxyMiddleware', # 4. 设置代理
150
+
151
+ # === 响应处理阶段 ===
152
+ 'crawlo.middleware.retry.RetryMiddleware', # 5. 失败请求重试
153
+ 'crawlo.middleware.response_code.ResponseCodeMiddleware', # 6. 处理特殊状态码
154
+ 'crawlo.middleware.response_filter.ResponseFilterMiddleware', # 7. 响应内容过滤
155
+ ]
156
+
157
+ # ============================== 扩展与管道 ==============================
158
+
159
+ # 数据处理管道(启用的存储方式)
160
+ PIPELINES = [
161
+ 'crawlo.pipelines.console_pipeline.ConsolePipeline', # 控制台输出
162
+ # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline', # MySQL 存储(可选)
163
+ ]
164
+
165
+ # 根据运行模式自动配置默认去重管道
166
+ if RUN_MODE == 'distributed':
167
+ # 分布式模式下添加Redis去重管道
168
+ PIPELINES.insert(0, DEFAULT_DEDUP_PIPELINE)
169
+ else:
170
+ # 单机模式下添加内存去重管道
171
+ PIPELINES.insert(0, DEFAULT_DEDUP_PIPELINE)
172
+
173
+ # 扩展组件(监控与日志)
174
+ EXTENSIONS = [
175
+ 'crawlo.extension.log_interval.LogIntervalExtension', # 定时日志
176
+ 'crawlo.extension.log_stats.LogStats', # 统计信息
177
+ 'crawlo.extension.logging_extension.CustomLoggerExtension', # 自定义日志
178
+ ]
179
+
180
+ # ============================== 日志与监控 ==============================
181
+
182
+ LOG_LEVEL = 'INFO' # 日志级别: DEBUG/INFO/WARNING/ERROR
183
+ STATS_DUMP = True # 是否周期性输出统计信息
184
+ LOG_FILE = f'logs/{PROJECT_NAME}.log' # 日志文件路径
185
+ LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s'
186
+ LOG_ENCODING = 'utf-8'
187
+
188
+ # ============================== 代理配置 ==============================
189
+
190
+ PROXY_ENABLED = False # 是否启用代理
191
+ PROXY_API_URL = "https://api.proxyprovider.com/get" # 代理获取接口(请替换为真实地址)
192
+
193
+ # 代理提取方式(支持字段路径或函数)
194
+ PROXY_EXTRACTOR = "proxy" # 如返回 {"proxy": "http://1.1.1.1:8080"}
195
+
196
+ # 代理刷新控制
197
+ PROXY_REFRESH_INTERVAL = 60 # 代理刷新间隔(秒)
198
+ PROXY_API_TIMEOUT = 10 # 请求代理 API 超时时间
199
+
200
+ # ============================== Curl-Cffi 特有配置 ==============================
201
+
202
+ # 浏览器指纹模拟(仅 CurlCffi 下载器有效)
203
+ CURL_BROWSER_TYPE = "chrome" # 可选: chrome, edge, safari, firefox 或版本如 chrome136
204
+
205
+ # 自定义浏览器版本映射(可覆盖默认行为)
206
+ CURL_BROWSER_VERSION_MAP = {
207
+ "chrome": "chrome136",
208
+ "edge": "edge101",
209
+ "safari": "safari184",
210
+ "firefox": "firefox135",
211
+ # 示例:旧版本测试
212
+ # "chrome_legacy": "chrome110",
213
+ }
214
+
215
+ # Curl-Cffi 优化配置
216
+ CURL_RANDOMIZE_DELAY = False # 是否启用随机延迟
217
+ CURL_RETRY_BACKOFF = True # 是否启用指数退避重试
218
+
219
+ # 默认请求头(可被 Spider 覆盖)
220
+ DEFAULT_REQUEST_HEADERS = {
221
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
222
+ '(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
223
+ }
224
+
225
+ # ============================== 下载器优化配置 ==============================
226
+
227
+ # 下载器健康检查
228
+ DOWNLOADER_HEALTH_CHECK = True # 是否启用下载器健康检查
229
+ HEALTH_CHECK_INTERVAL = 60 # 健康检查间隔(秒)
230
+
231
+ # 请求统计配置
232
+ REQUEST_STATS_ENABLED = True # 是否启用请求统计
233
+ STATS_RESET_ON_START = False # 启动时是否重置统计
234
+
235
+ # HttpX 下载器专用配置
236
+ HTTPX_HTTP2 = True # 是否启用HTTP/2支持
237
+ HTTPX_FOLLOW_REDIRECTS = True # 是否自动跟随重定向
238
+
239
+ # AioHttp 下载器专用配置
240
+ AIOHTTP_AUTO_DECOMPRESS = True # 是否自动解压响应
241
+ AIOHTTP_FORCE_CLOSE = False # 是否强制关闭连接
242
+
243
+ # 通用优化配置
244
+ CONNECTION_TTL_DNS_CACHE = 300 # DNS缓存TTL(秒)
245
+ CONNECTION_KEEPALIVE_TIMEOUT = 15 # Keep-Alive超时(秒)