crawlo 1.1.9__py3-none-any.whl → 1.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of crawlo might be problematic. Click here for more details.

Files changed (221) hide show
  1. crawlo/__init__.py +61 -61
  2. crawlo/__version__.py +1 -1
  3. crawlo/cleaners/__init__.py +60 -60
  4. crawlo/cleaners/data_formatter.py +225 -225
  5. crawlo/cleaners/encoding_converter.py +125 -125
  6. crawlo/cleaners/text_cleaner.py +232 -232
  7. crawlo/cli.py +65 -65
  8. crawlo/commands/__init__.py +14 -14
  9. crawlo/commands/check.py +594 -594
  10. crawlo/commands/genspider.py +151 -151
  11. crawlo/commands/help.py +142 -132
  12. crawlo/commands/list.py +155 -155
  13. crawlo/commands/run.py +292 -292
  14. crawlo/commands/startproject.py +418 -418
  15. crawlo/commands/stats.py +188 -188
  16. crawlo/commands/utils.py +186 -186
  17. crawlo/config.py +312 -312
  18. crawlo/config_validator.py +252 -252
  19. crawlo/core/__init__.py +2 -2
  20. crawlo/core/engine.py +354 -345
  21. crawlo/core/processor.py +40 -40
  22. crawlo/core/scheduler.py +143 -136
  23. crawlo/crawler.py +1027 -1027
  24. crawlo/downloader/__init__.py +266 -266
  25. crawlo/downloader/aiohttp_downloader.py +220 -220
  26. crawlo/downloader/cffi_downloader.py +256 -256
  27. crawlo/downloader/httpx_downloader.py +259 -259
  28. crawlo/downloader/hybrid_downloader.py +213 -213
  29. crawlo/downloader/playwright_downloader.py +402 -402
  30. crawlo/downloader/selenium_downloader.py +472 -472
  31. crawlo/event.py +11 -11
  32. crawlo/exceptions.py +81 -81
  33. crawlo/extension/__init__.py +37 -37
  34. crawlo/extension/health_check.py +141 -141
  35. crawlo/extension/log_interval.py +57 -57
  36. crawlo/extension/log_stats.py +81 -81
  37. crawlo/extension/logging_extension.py +43 -43
  38. crawlo/extension/memory_monitor.py +104 -104
  39. crawlo/extension/performance_profiler.py +133 -133
  40. crawlo/extension/request_recorder.py +107 -107
  41. crawlo/filters/__init__.py +154 -154
  42. crawlo/filters/aioredis_filter.py +280 -280
  43. crawlo/filters/memory_filter.py +269 -269
  44. crawlo/items/__init__.py +23 -23
  45. crawlo/items/base.py +21 -21
  46. crawlo/items/fields.py +53 -53
  47. crawlo/items/items.py +104 -104
  48. crawlo/middleware/__init__.py +21 -21
  49. crawlo/middleware/default_header.py +132 -32
  50. crawlo/middleware/download_delay.py +105 -28
  51. crawlo/middleware/middleware_manager.py +135 -135
  52. crawlo/middleware/offsite.py +116 -0
  53. crawlo/middleware/proxy.py +366 -272
  54. crawlo/middleware/request_ignore.py +88 -30
  55. crawlo/middleware/response_code.py +164 -18
  56. crawlo/middleware/response_filter.py +138 -26
  57. crawlo/middleware/retry.py +124 -124
  58. crawlo/mode_manager.py +211 -211
  59. crawlo/network/__init__.py +21 -21
  60. crawlo/network/request.py +338 -338
  61. crawlo/network/response.py +359 -359
  62. crawlo/pipelines/__init__.py +21 -21
  63. crawlo/pipelines/bloom_dedup_pipeline.py +156 -156
  64. crawlo/pipelines/console_pipeline.py +39 -39
  65. crawlo/pipelines/csv_pipeline.py +316 -316
  66. crawlo/pipelines/database_dedup_pipeline.py +224 -224
  67. crawlo/pipelines/json_pipeline.py +218 -218
  68. crawlo/pipelines/memory_dedup_pipeline.py +115 -115
  69. crawlo/pipelines/mongo_pipeline.py +131 -131
  70. crawlo/pipelines/mysql_pipeline.py +316 -316
  71. crawlo/pipelines/pipeline_manager.py +61 -61
  72. crawlo/pipelines/redis_dedup_pipeline.py +167 -167
  73. crawlo/project.py +187 -187
  74. crawlo/queue/pqueue.py +37 -37
  75. crawlo/queue/queue_manager.py +337 -334
  76. crawlo/queue/redis_priority_queue.py +298 -298
  77. crawlo/settings/__init__.py +7 -7
  78. crawlo/settings/default_settings.py +226 -219
  79. crawlo/settings/setting_manager.py +122 -122
  80. crawlo/spider/__init__.py +639 -639
  81. crawlo/stats_collector.py +59 -59
  82. crawlo/subscriber.py +130 -130
  83. crawlo/task_manager.py +30 -30
  84. crawlo/templates/crawlo.cfg.tmpl +10 -10
  85. crawlo/templates/project/__init__.py.tmpl +3 -3
  86. crawlo/templates/project/items.py.tmpl +17 -17
  87. crawlo/templates/project/middlewares.py.tmpl +118 -109
  88. crawlo/templates/project/pipelines.py.tmpl +96 -96
  89. crawlo/templates/project/run.py.tmpl +45 -45
  90. crawlo/templates/project/settings.py.tmpl +327 -326
  91. crawlo/templates/project/settings_distributed.py.tmpl +119 -119
  92. crawlo/templates/project/settings_gentle.py.tmpl +94 -94
  93. crawlo/templates/project/settings_high_performance.py.tmpl +151 -151
  94. crawlo/templates/project/settings_simple.py.tmpl +68 -68
  95. crawlo/templates/project/spiders/__init__.py.tmpl +5 -5
  96. crawlo/templates/spider/spider.py.tmpl +143 -141
  97. crawlo/tools/__init__.py +182 -182
  98. crawlo/tools/anti_crawler.py +268 -268
  99. crawlo/tools/authenticated_proxy.py +240 -240
  100. crawlo/tools/data_validator.py +180 -180
  101. crawlo/tools/date_tools.py +35 -35
  102. crawlo/tools/distributed_coordinator.py +386 -386
  103. crawlo/tools/retry_mechanism.py +220 -220
  104. crawlo/tools/scenario_adapter.py +262 -262
  105. crawlo/utils/__init__.py +35 -35
  106. crawlo/utils/batch_processor.py +260 -260
  107. crawlo/utils/controlled_spider_mixin.py +439 -439
  108. crawlo/utils/date_tools.py +290 -290
  109. crawlo/utils/db_helper.py +343 -343
  110. crawlo/utils/enhanced_error_handler.py +359 -359
  111. crawlo/utils/env_config.py +105 -105
  112. crawlo/utils/error_handler.py +125 -125
  113. crawlo/utils/func_tools.py +82 -82
  114. crawlo/utils/large_scale_config.py +286 -286
  115. crawlo/utils/large_scale_helper.py +343 -343
  116. crawlo/utils/log.py +128 -128
  117. crawlo/utils/performance_monitor.py +284 -284
  118. crawlo/utils/queue_helper.py +175 -175
  119. crawlo/utils/redis_connection_pool.py +334 -334
  120. crawlo/utils/redis_key_validator.py +199 -199
  121. crawlo/utils/request.py +267 -267
  122. crawlo/utils/request_serializer.py +219 -219
  123. crawlo/utils/spider_loader.py +62 -62
  124. crawlo/utils/system.py +11 -11
  125. crawlo/utils/tools.py +4 -4
  126. crawlo/utils/url.py +39 -39
  127. crawlo-1.2.1.dist-info/METADATA +692 -0
  128. crawlo-1.2.1.dist-info/RECORD +220 -0
  129. examples/__init__.py +7 -7
  130. examples/aiohttp_settings.py +42 -0
  131. examples/curl_cffi_settings.py +41 -0
  132. examples/default_header_middleware_example.py +107 -0
  133. examples/default_header_spider_example.py +129 -0
  134. examples/download_delay_middleware_example.py +160 -0
  135. examples/httpx_settings.py +42 -0
  136. examples/multi_downloader_proxy_example.py +81 -0
  137. examples/offsite_middleware_example.py +55 -0
  138. examples/offsite_spider_example.py +107 -0
  139. examples/proxy_spider_example.py +166 -0
  140. examples/request_ignore_middleware_example.py +51 -0
  141. examples/request_ignore_spider_example.py +99 -0
  142. examples/response_code_middleware_example.py +52 -0
  143. examples/response_filter_middleware_example.py +67 -0
  144. examples/tong_hua_shun_settings.py +62 -0
  145. examples/tong_hua_shun_spider.py +170 -0
  146. tests/DOUBLE_CRAWLO_PREFIX_FIX_REPORT.md +81 -81
  147. tests/__init__.py +7 -7
  148. tests/advanced_tools_example.py +275 -275
  149. tests/authenticated_proxy_example.py +236 -236
  150. tests/cleaners_example.py +160 -160
  151. tests/config_validation_demo.py +102 -102
  152. tests/controlled_spider_example.py +205 -205
  153. tests/date_tools_example.py +180 -180
  154. tests/dynamic_loading_example.py +523 -523
  155. tests/dynamic_loading_test.py +104 -104
  156. tests/env_config_example.py +133 -133
  157. tests/error_handling_example.py +171 -171
  158. tests/redis_key_validation_demo.py +130 -130
  159. tests/response_improvements_example.py +144 -144
  160. tests/test_advanced_tools.py +148 -148
  161. tests/test_all_redis_key_configs.py +145 -145
  162. tests/test_authenticated_proxy.py +141 -141
  163. tests/test_cleaners.py +54 -54
  164. tests/test_comprehensive.py +146 -146
  165. tests/test_config_validator.py +193 -193
  166. tests/test_crawlo_proxy_integration.py +173 -0
  167. tests/test_date_tools.py +123 -123
  168. tests/test_default_header_middleware.py +159 -0
  169. tests/test_double_crawlo_fix.py +207 -207
  170. tests/test_double_crawlo_fix_simple.py +124 -124
  171. tests/test_download_delay_middleware.py +222 -0
  172. tests/test_downloader_proxy_compatibility.py +269 -0
  173. tests/test_dynamic_downloaders_proxy.py +124 -124
  174. tests/test_dynamic_proxy.py +92 -92
  175. tests/test_dynamic_proxy_config.py +146 -146
  176. tests/test_dynamic_proxy_real.py +109 -109
  177. tests/test_edge_cases.py +303 -303
  178. tests/test_enhanced_error_handler.py +270 -270
  179. tests/test_env_config.py +121 -121
  180. tests/test_error_handler_compatibility.py +112 -112
  181. tests/test_final_validation.py +153 -153
  182. tests/test_framework_env_usage.py +103 -103
  183. tests/test_integration.py +356 -356
  184. tests/test_item_dedup_redis_key.py +122 -122
  185. tests/test_offsite_middleware.py +222 -0
  186. tests/test_parsel.py +29 -29
  187. tests/test_performance.py +327 -327
  188. tests/test_proxy_api.py +265 -0
  189. tests/test_proxy_health_check.py +32 -32
  190. tests/test_proxy_middleware.py +122 -0
  191. tests/test_proxy_middleware_enhanced.py +217 -0
  192. tests/test_proxy_middleware_integration.py +136 -136
  193. tests/test_proxy_providers.py +56 -56
  194. tests/test_proxy_stats.py +19 -19
  195. tests/test_proxy_strategies.py +59 -59
  196. tests/test_queue_manager_double_crawlo.py +174 -231
  197. tests/test_queue_manager_redis_key.py +176 -176
  198. tests/test_real_scenario_proxy.py +196 -0
  199. tests/test_redis_config.py +28 -28
  200. tests/test_redis_connection_pool.py +294 -294
  201. tests/test_redis_key_naming.py +181 -181
  202. tests/test_redis_key_validator.py +123 -123
  203. tests/test_redis_queue.py +224 -224
  204. tests/test_request_ignore_middleware.py +183 -0
  205. tests/test_request_serialization.py +70 -70
  206. tests/test_response_code_middleware.py +350 -0
  207. tests/test_response_filter_middleware.py +428 -0
  208. tests/test_response_improvements.py +152 -152
  209. tests/test_retry_middleware.py +242 -0
  210. tests/test_scheduler.py +241 -241
  211. tests/test_simple_response.py +61 -61
  212. tests/test_telecom_spider_redis_key.py +205 -205
  213. tests/test_template_content.py +87 -87
  214. tests/test_template_redis_key.py +134 -134
  215. tests/test_tools.py +153 -153
  216. tests/tools_example.py +257 -257
  217. crawlo-1.1.9.dist-info/METADATA +0 -626
  218. crawlo-1.1.9.dist-info/RECORD +0 -190
  219. {crawlo-1.1.9.dist-info → crawlo-1.2.1.dist-info}/WHEEL +0 -0
  220. {crawlo-1.1.9.dist-info → crawlo-1.2.1.dist-info}/entry_points.txt +0 -0
  221. {crawlo-1.1.9.dist-info → crawlo-1.2.1.dist-info}/top_level.txt +0 -0
@@ -1,419 +1,419 @@
1
- #!/usr/bin/python
2
- # -*- coding: UTF-8 -*-
3
- """
4
- # @Time : 2025-08-31 22:36
5
- # @Author : crawl-coder
6
- # @Desc : 命令行入口:crawlo startproject baidu,创建项目。
7
- """
8
- import shutil
9
- import re
10
- import sys
11
- import os
12
- from pathlib import Path
13
- from typing import Optional, List
14
-
15
- # 添加项目根目录到路径,以便能够导入utils模块
16
- sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
17
-
18
- try:
19
- from rich.console import Console
20
- from rich.panel import Panel
21
- from rich.text import Text
22
- from rich.table import Table
23
- RICH_AVAILABLE = True
24
- except ImportError:
25
- RICH_AVAILABLE = False
26
-
27
- try:
28
- from .utils import show_error_panel, show_success_panel
29
- UTILS_AVAILABLE = True
30
- except ImportError:
31
- # 如果相对导入失败,尝试绝对导入
32
- try:
33
- from crawlo.commands.utils import show_error_panel, show_success_panel
34
- UTILS_AVAILABLE = True
35
- except ImportError:
36
- UTILS_AVAILABLE = False
37
-
38
- # 初始化 rich 控制台(如果可用)
39
- if RICH_AVAILABLE:
40
- console = Console()
41
- else:
42
- # 简单的控制台输出替代
43
- class Console:
44
- def print(self, text):
45
- print(text)
46
- console = Console()
47
-
48
- TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
49
-
50
- # 可用的模板类型
51
- TEMPLATE_TYPES = {
52
- 'default': '默认模板 - 通用配置,适合大多数项目',
53
- 'simple': '简化模板 - 最小配置,适合快速开始',
54
- 'distributed': '分布式模板 - 针对分布式爬取优化',
55
- 'high-performance': '高性能模板 - 针对大规模高并发优化',
56
- 'gentle': '温和模板 - 低负载配置,对目标网站友好'
57
- }
58
-
59
- # 可选的模块组件
60
- OPTIONAL_MODULES = {
61
- 'mysql': 'MySQL数据库支持',
62
- 'mongodb': 'MongoDB数据库支持',
63
- 'redis': 'Redis支持(分布式队列和去重)',
64
- 'proxy': '代理支持',
65
- 'monitoring': '监控和性能分析',
66
- 'dedup': '去重功能',
67
- 'httpx': 'HttpX下载器',
68
- 'aiohttp': 'AioHttp下载器',
69
- 'curl': 'CurlCffi下载器'
70
- }
71
-
72
-
73
- def show_error_panel(title, content):
74
- """显示错误面板的简单实现"""
75
- if RICH_AVAILABLE:
76
- from rich.panel import Panel
77
- console.print(Panel(content, title=title, border_style="red"))
78
- else:
79
- print(f"❌ {title}")
80
- print(content)
81
-
82
- def show_success_panel(title, content):
83
- """显示成功面板的简单实现"""
84
- if RICH_AVAILABLE:
85
- from rich.panel import Panel
86
- console.print(Panel(content, title=title, border_style="green"))
87
- else:
88
- print(f"✅ {title}")
89
- print(content)
90
-
91
- def _render_template(tmpl_path, context):
92
- """读取模板文件,替换 {{key}} 为 context 中的值"""
93
- with open(tmpl_path, 'r', encoding='utf-8') as f:
94
- content = f.read()
95
- for key, value in context.items():
96
- content = content.replace(f'{{{{{key}}}}}', str(value))
97
- return content
98
-
99
-
100
- def _copytree_with_templates(src, dst, context, template_type='default', modules: List[str] = None):
101
- """
102
- 递归复制目录,将 .tmpl 文件渲染后复制(去除 .tmpl 后缀),其他文件直接复制。
103
- 支持选择性模块复制。
104
- """
105
- src_path = Path(src)
106
- dst_path = Path(dst)
107
- dst_path.mkdir(parents=True, exist_ok=True)
108
-
109
- for item in src_path.rglob('*'):
110
- rel_path = item.relative_to(src_path)
111
- dst_item = dst_path / rel_path
112
-
113
- # 检查是否应该包含此文件(基于模块选择)
114
- if not _should_include_file(rel_path, modules):
115
- continue
116
-
117
- if item.is_dir():
118
- dst_item.mkdir(parents=True, exist_ok=True)
119
- else:
120
- if item.suffix == '.tmpl':
121
- rendered_content = None
122
- # 处理特定模板类型的设置文件
123
- if item.name == 'settings.py.tmpl':
124
- # 对于设置文件,根据模板类型选择相应的内容模板
125
- if template_type != 'default':
126
- # 使用特定模板类型的设置文件
127
- template_file_name = f'settings_{template_type}.py.tmpl'
128
- template_file_path = src_path / template_file_name
129
- if template_file_path.exists():
130
- rendered_content = _render_template(template_file_path, context)
131
- else:
132
- # 如果特定模板不存在,使用默认模板
133
- rendered_content = _render_template(item, context)
134
- else:
135
- # 使用默认模板
136
- rendered_content = _render_template(item, context)
137
- # 跳过其他以 settings_ 开头的模板文件,避免重复处理
138
- elif item.name.startswith('settings_') and item.name.endswith('.py.tmpl'):
139
- continue
140
- else:
141
- rendered_content = _render_template(item, context)
142
-
143
- # 确保设置文件始终命名为 settings.py
144
- if item.name == 'settings.py.tmpl':
145
- # 特殊处理设置模板文件,统一生成为 settings.py
146
- final_dst = dst_item.parent / 'settings.py'
147
- else:
148
- final_dst = dst_item.with_suffix('')
149
-
150
- final_dst.parent.mkdir(parents=True, exist_ok=True)
151
- with open(final_dst, 'w', encoding='utf-8') as f:
152
- f.write(rendered_content)
153
- else:
154
- shutil.copy2(item, dst_item)
155
-
156
-
157
- def _should_include_file(rel_path, modules: List[str]) -> bool:
158
- """
159
- 根据选择的模块决定是否包含文件
160
- """
161
- if modules is None:
162
- # 如果没有指定模块,则包含所有文件
163
- return True
164
-
165
- # 基础文件始终包含
166
- basic_files = [
167
- '__init__.py.tmpl',
168
- 'settings.py.tmpl',
169
- 'spiders/__init__.py.tmpl',
170
- 'items.py.tmpl',
171
- 'middlewares.py.tmpl',
172
- 'run.py.tmpl'
173
- ]
174
-
175
- path_str = str(rel_path).replace('\\', '/')
176
-
177
- # 始终包含基础文件
178
- if path_str in basic_files:
179
- return True
180
-
181
- # 根据模块选择包含特定文件
182
- if 'mysql' in modules and 'mysql' in path_str:
183
- return True
184
- if 'mongodb' in modules and 'mongo' in path_str:
185
- return True
186
- if 'redis' in modules and 'redis' in path_str:
187
- return True
188
- if 'proxy' in modules and 'proxy' in path_str:
189
- return True
190
- if 'monitoring' in modules and ('monitor' in path_str or 'stats' in path_str):
191
- return True
192
- if 'dedup' in modules and 'dedup' in path_str:
193
- return True
194
- if 'httpx' in modules and 'httpx' in path_str:
195
- return True
196
- if 'aiohttp' in modules and 'aiohttp' in path_str:
197
- return True
198
- if 'curl' in modules and 'cffi' in path_str:
199
- return True
200
-
201
- # 默认不包含特定模块文件
202
- return False
203
-
204
-
205
- def validate_project_name(project_name: str) -> tuple[bool, str]:
206
- """
207
- 验证项目名称是否有效
208
-
209
- Returns:
210
- tuple[bool, str]: (是否有效, 错误信息)
211
- """
212
- # 检查是否为空
213
- if not project_name or not project_name.strip():
214
- return False, "Project name cannot be empty"
215
-
216
- project_name = project_name.strip()
217
-
218
- # 检查长度
219
- if len(project_name) > 50:
220
- return False, "Project name too long (max 50 characters)"
221
-
222
- # 检查是否为Python关键字
223
- python_keywords = {
224
- 'False', 'None', 'True', 'and', 'as', 'assert', 'break', 'class',
225
- 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally',
226
- 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda',
227
- 'nonlocal', 'not', 'or', 'pass', 'raise', 'return', 'try',
228
- 'while', 'with', 'yield'
229
- }
230
- if project_name in python_keywords:
231
- return False, f"'{project_name}' is a Python keyword and cannot be used as project name"
232
-
233
- # 检查是否为有效的Python标识符
234
- if not project_name.isidentifier():
235
- return False, "Project name must be a valid Python identifier"
236
-
237
- # 检查格式(建议使用snake_case)
238
- if not re.match(r'^[a-z][a-z0-9_]*$', project_name):
239
- return False, (
240
- "Project name should start with lowercase letter and "
241
- "contain only lowercase letters, numbers, and underscores"
242
- )
243
-
244
- # 检查是否以数字结尾(不推荐)
245
- if project_name[-1].isdigit():
246
- return False, "Project name should not end with a number"
247
-
248
- return True, ""
249
-
250
-
251
- def show_template_options():
252
- """显示可用的模板选项"""
253
- if RICH_AVAILABLE:
254
- table = Table(title="可用模板类型", show_header=True, header_style="bold magenta")
255
- table.add_column("模板类型", style="cyan", no_wrap=True)
256
- table.add_column("描述", style="green")
257
-
258
- for template_type, description in TEMPLATE_TYPES.items():
259
- table.add_row(template_type, description)
260
-
261
- console.print(table)
262
- else:
263
- print("可用模板类型:")
264
- for template_type, description in TEMPLATE_TYPES.items():
265
- print(f" {template_type}: {description}")
266
-
267
-
268
- def show_module_options():
269
- """显示可用的模块选项"""
270
- if RICH_AVAILABLE:
271
- table = Table(title="可选模块组件", show_header=True, header_style="bold magenta")
272
- table.add_column("模块", style="cyan", no_wrap=True)
273
- table.add_column("描述", style="green")
274
-
275
- for module, description in OPTIONAL_MODULES.items():
276
- table.add_row(module, description)
277
-
278
- console.print(table)
279
- else:
280
- print("可选模块组件:")
281
- for module, description in OPTIONAL_MODULES.items():
282
- print(f" {module}: {description}")
283
-
284
-
285
- def main(args):
286
- if len(args) < 1:
287
- console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo startproject[/blue] <project_name> [template_type] [--modules module1,module2]")
288
- console.print("💡 Examples:")
289
- console.print(" [blue]crawlo startproject[/blue] my_spider_project")
290
- console.print(" [blue]crawlo startproject[/blue] news_crawler simple")
291
- console.print(" [blue]crawlo startproject[/blue] ecommerce_spider distributed --modules mysql,proxy")
292
- show_template_options()
293
- show_module_options()
294
- return 1
295
-
296
- # 解析参数
297
- project_name = args[0]
298
- template_type = 'default'
299
- modules = None
300
-
301
- # 解析可选参数
302
- if len(args) > 1:
303
- for i, arg in enumerate(args[1:], 1):
304
- if arg.startswith('--modules='):
305
- modules_str = arg.split('=', 1)[1]
306
- modules = [m.strip() for m in modules_str.split(',') if m.strip()]
307
- elif arg.startswith('--modules'):
308
- # 处理 --modules module1,module2 格式
309
- if i + 1 < len(args):
310
- modules_str = args[i + 1]
311
- modules = [m.strip() for m in modules_str.split(',') if m.strip()]
312
- elif not arg.startswith('--') and arg in TEMPLATE_TYPES:
313
- template_type = arg
314
-
315
- # 验证模板类型
316
- if template_type not in TEMPLATE_TYPES:
317
- show_error_panel(
318
- "Invalid Template Type",
319
- f"Template type '[cyan]{template_type}[/cyan]' is not supported.\n"
320
- )
321
- show_template_options()
322
- return 1
323
-
324
- # 验证项目名称
325
- is_valid, error_msg = validate_project_name(project_name)
326
- if not is_valid:
327
- show_error_panel(
328
- "Invalid Project Name",
329
- f"[cyan]{project_name}[/cyan] is not a valid project name.\n"
330
- f"❌ {error_msg}\n\n"
331
- "💡 Project name should:\n"
332
- " • Start with lowercase letter\n"
333
- " • Contain only lowercase letters, numbers, and underscores\n"
334
- " • Be a valid Python identifier\n"
335
- " • Not be a Python keyword"
336
- )
337
- return 1
338
-
339
- project_dir = Path(project_name)
340
-
341
- if project_dir.exists():
342
- show_error_panel(
343
- "Directory Exists",
344
- f"Directory '[cyan]{project_dir}[/cyan]' already exists.\n"
345
- "💡 Choose a different project name or remove the existing directory."
346
- )
347
- return 1
348
-
349
- context = {'project_name': project_name}
350
- template_dir = TEMPLATES_DIR / 'project'
351
-
352
- try:
353
- # 1. 创建项目根目录
354
- project_dir.mkdir()
355
-
356
- # 2. 渲染 crawlo.cfg.tmpl
357
- cfg_template = TEMPLATES_DIR / 'crawlo.cfg.tmpl'
358
- if cfg_template.exists():
359
- cfg_content = _render_template(cfg_template, context)
360
- (project_dir / 'crawlo.cfg').write_text(cfg_content, encoding='utf-8')
361
- console.print(f":white_check_mark: Created [green]{project_dir / 'crawlo.cfg'}[/green]")
362
- else:
363
- console.print("[yellow]⚠ Warning:[/yellow] Template 'crawlo.cfg.tmpl' not found.")
364
-
365
- # 3. 复制并渲染项目包内容
366
- package_dir = project_dir / project_name
367
- _copytree_with_templates(template_dir, package_dir, context, template_type, modules)
368
- console.print(f":white_check_mark: Created project package: [green]{package_dir}[/green]")
369
-
370
- # 4. 创建 logs 目录
371
- (project_dir / 'logs').mkdir(exist_ok=True)
372
- console.print(":white_check_mark: Created logs directory")
373
-
374
- # 5. 创建 output 目录(用于数据输出)
375
- (project_dir / 'output').mkdir(exist_ok=True)
376
- console.print(":white_check_mark: Created output directory")
377
-
378
- # 成功面板
379
- success_text = Text.from_markup(f"Project '[bold cyan]{project_name}[/bold cyan]' created successfully!")
380
- console.print(Panel(success_text, title=":rocket: Success", border_style="green", padding=(1, 2)))
381
-
382
- # 显示使用的模板类型
383
- if template_type != 'default':
384
- console.print(f":information: 使用模板类型: [bold blue]{template_type}[/bold blue] - {TEMPLATE_TYPES[template_type]}")
385
-
386
- # 显示选择的模块
387
- if modules:
388
- console.print(f":information: 选择的模块: [bold blue]{', '.join(modules)}[/bold blue]")
389
-
390
- # 下一步操作提示(对齐美观 + 语法高亮)
391
- next_steps = f"""
392
- [bold]🚀 Next steps:[/bold]
393
- [blue]cd[/blue] {project_name}
394
- [blue]crawlo genspider[/blue] example example.com
395
- [blue]crawlo run[/blue] example
396
-
397
- [bold]📚 Learn more:[/bold]
398
- [blue]crawlo list[/blue] # List all spiders
399
- [blue]crawlo check[/blue] example # Check spider validity
400
- [blue]crawlo stats[/blue] # View statistics
401
- """.strip()
402
- console.print(next_steps)
403
-
404
- return 0
405
-
406
- except Exception as e:
407
- show_error_panel(
408
- "Creation Failed",
409
- f"Failed to create project: {e}"
410
- )
411
- if project_dir.exists():
412
- shutil.rmtree(project_dir, ignore_errors=True)
413
- console.print("[red]:cross_mark: Cleaned up partially created project.[/red]")
414
- return 1
415
-
416
- if __name__ == "__main__":
417
- import sys
418
- exit_code = main(sys.argv[1:])
1
+ #!/usr/bin/python
2
+ # -*- coding: UTF-8 -*-
3
+ """
4
+ # @Time : 2025-08-31 22:36
5
+ # @Author : crawl-coder
6
+ # @Desc : 命令行入口:crawlo startproject baidu,创建项目。
7
+ """
8
+ import shutil
9
+ import re
10
+ import sys
11
+ import os
12
+ from pathlib import Path
13
+ from typing import Optional, List
14
+
15
+ # 添加项目根目录到路径,以便能够导入utils模块
16
+ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
17
+
18
+ try:
19
+ from rich.console import Console
20
+ from rich.panel import Panel
21
+ from rich.text import Text
22
+ from rich.table import Table
23
+ RICH_AVAILABLE = True
24
+ except ImportError:
25
+ RICH_AVAILABLE = False
26
+
27
+ try:
28
+ from .utils import show_error_panel, show_success_panel
29
+ UTILS_AVAILABLE = True
30
+ except ImportError:
31
+ # 如果相对导入失败,尝试绝对导入
32
+ try:
33
+ from crawlo.commands.utils import show_error_panel, show_success_panel
34
+ UTILS_AVAILABLE = True
35
+ except ImportError:
36
+ UTILS_AVAILABLE = False
37
+
38
+ # 初始化 rich 控制台(如果可用)
39
+ if RICH_AVAILABLE:
40
+ console = Console()
41
+ else:
42
+ # 简单的控制台输出替代
43
+ class Console:
44
+ def print(self, text):
45
+ print(text)
46
+ console = Console()
47
+
48
+ TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
49
+
50
+ # 可用的模板类型
51
+ TEMPLATE_TYPES = {
52
+ 'default': '默认模板 - 通用配置,适合大多数项目',
53
+ 'simple': '简化模板 - 最小配置,适合快速开始',
54
+ 'distributed': '分布式模板 - 针对分布式爬取优化',
55
+ 'high-performance': '高性能模板 - 针对大规模高并发优化',
56
+ 'gentle': '温和模板 - 低负载配置,对目标网站友好'
57
+ }
58
+
59
+ # 可选的模块组件
60
+ OPTIONAL_MODULES = {
61
+ 'mysql': 'MySQL数据库支持',
62
+ 'mongodb': 'MongoDB数据库支持',
63
+ 'redis': 'Redis支持(分布式队列和去重)',
64
+ 'proxy': '代理支持',
65
+ 'monitoring': '监控和性能分析',
66
+ 'dedup': '去重功能',
67
+ 'httpx': 'HttpX下载器',
68
+ 'aiohttp': 'AioHttp下载器',
69
+ 'curl': 'CurlCffi下载器'
70
+ }
71
+
72
+
73
+ def show_error_panel(title, content):
74
+ """显示错误面板的简单实现"""
75
+ if RICH_AVAILABLE:
76
+ from rich.panel import Panel
77
+ console.print(Panel(content, title=title, border_style="red"))
78
+ else:
79
+ print(f"❌ {title}")
80
+ print(content)
81
+
82
+ def show_success_panel(title, content):
83
+ """显示成功面板的简单实现"""
84
+ if RICH_AVAILABLE:
85
+ from rich.panel import Panel
86
+ console.print(Panel(content, title=title, border_style="green"))
87
+ else:
88
+ print(f"✅ {title}")
89
+ print(content)
90
+
91
+ def _render_template(tmpl_path, context):
92
+ """读取模板文件,替换 {{key}} 为 context 中的值"""
93
+ with open(tmpl_path, 'r', encoding='utf-8') as f:
94
+ content = f.read()
95
+ for key, value in context.items():
96
+ content = content.replace(f'{{{{{key}}}}}', str(value))
97
+ return content
98
+
99
+
100
+ def _copytree_with_templates(src, dst, context, template_type='default', modules: List[str] = None):
101
+ """
102
+ 递归复制目录,将 .tmpl 文件渲染后复制(去除 .tmpl 后缀),其他文件直接复制。
103
+ 支持选择性模块复制。
104
+ """
105
+ src_path = Path(src)
106
+ dst_path = Path(dst)
107
+ dst_path.mkdir(parents=True, exist_ok=True)
108
+
109
+ for item in src_path.rglob('*'):
110
+ rel_path = item.relative_to(src_path)
111
+ dst_item = dst_path / rel_path
112
+
113
+ # 检查是否应该包含此文件(基于模块选择)
114
+ if not _should_include_file(rel_path, modules):
115
+ continue
116
+
117
+ if item.is_dir():
118
+ dst_item.mkdir(parents=True, exist_ok=True)
119
+ else:
120
+ if item.suffix == '.tmpl':
121
+ rendered_content = None
122
+ # 处理特定模板类型的设置文件
123
+ if item.name == 'settings.py.tmpl':
124
+ # 对于设置文件,根据模板类型选择相应的内容模板
125
+ if template_type != 'default':
126
+ # 使用特定模板类型的设置文件
127
+ template_file_name = f'settings_{template_type}.py.tmpl'
128
+ template_file_path = src_path / template_file_name
129
+ if template_file_path.exists():
130
+ rendered_content = _render_template(template_file_path, context)
131
+ else:
132
+ # 如果特定模板不存在,使用默认模板
133
+ rendered_content = _render_template(item, context)
134
+ else:
135
+ # 使用默认模板
136
+ rendered_content = _render_template(item, context)
137
+ # 跳过其他以 settings_ 开头的模板文件,避免重复处理
138
+ elif item.name.startswith('settings_') and item.name.endswith('.py.tmpl'):
139
+ continue
140
+ else:
141
+ rendered_content = _render_template(item, context)
142
+
143
+ # 确保设置文件始终命名为 settings.py
144
+ if item.name == 'settings.py.tmpl':
145
+ # 特殊处理设置模板文件,统一生成为 settings.py
146
+ final_dst = dst_item.parent / 'settings.py'
147
+ else:
148
+ final_dst = dst_item.with_suffix('')
149
+
150
+ final_dst.parent.mkdir(parents=True, exist_ok=True)
151
+ with open(final_dst, 'w', encoding='utf-8') as f:
152
+ f.write(rendered_content)
153
+ else:
154
+ shutil.copy2(item, dst_item)
155
+
156
+
157
+ def _should_include_file(rel_path, modules: List[str]) -> bool:
158
+ """
159
+ 根据选择的模块决定是否包含文件
160
+ """
161
+ if modules is None:
162
+ # 如果没有指定模块,则包含所有文件
163
+ return True
164
+
165
+ # 基础文件始终包含
166
+ basic_files = [
167
+ '__init__.py.tmpl',
168
+ 'settings.py.tmpl',
169
+ 'spiders/__init__.py.tmpl',
170
+ 'items.py.tmpl',
171
+ 'middlewares.py.tmpl',
172
+ 'run.py.tmpl'
173
+ ]
174
+
175
+ path_str = str(rel_path).replace('\\', '/')
176
+
177
+ # 始终包含基础文件
178
+ if path_str in basic_files:
179
+ return True
180
+
181
+ # 根据模块选择包含特定文件
182
+ if 'mysql' in modules and 'mysql' in path_str:
183
+ return True
184
+ if 'mongodb' in modules and 'mongo' in path_str:
185
+ return True
186
+ if 'redis' in modules and 'redis' in path_str:
187
+ return True
188
+ if 'proxy' in modules and 'proxy' in path_str:
189
+ return True
190
+ if 'monitoring' in modules and ('monitor' in path_str or 'stats' in path_str):
191
+ return True
192
+ if 'dedup' in modules and 'dedup' in path_str:
193
+ return True
194
+ if 'httpx' in modules and 'httpx' in path_str:
195
+ return True
196
+ if 'aiohttp' in modules and 'aiohttp' in path_str:
197
+ return True
198
+ if 'curl' in modules and 'cffi' in path_str:
199
+ return True
200
+
201
+ # 默认不包含特定模块文件
202
+ return False
203
+
204
+
205
+ def validate_project_name(project_name: str) -> tuple[bool, str]:
206
+ """
207
+ 验证项目名称是否有效
208
+
209
+ Returns:
210
+ tuple[bool, str]: (是否有效, 错误信息)
211
+ """
212
+ # 检查是否为空
213
+ if not project_name or not project_name.strip():
214
+ return False, "Project name cannot be empty"
215
+
216
+ project_name = project_name.strip()
217
+
218
+ # 检查长度
219
+ if len(project_name) > 50:
220
+ return False, "Project name too long (max 50 characters)"
221
+
222
+ # 检查是否为Python关键字
223
+ python_keywords = {
224
+ 'False', 'None', 'True', 'and', 'as', 'assert', 'break', 'class',
225
+ 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally',
226
+ 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda',
227
+ 'nonlocal', 'not', 'or', 'pass', 'raise', 'return', 'try',
228
+ 'while', 'with', 'yield'
229
+ }
230
+ if project_name in python_keywords:
231
+ return False, f"'{project_name}' is a Python keyword and cannot be used as project name"
232
+
233
+ # 检查是否为有效的Python标识符
234
+ if not project_name.isidentifier():
235
+ return False, "Project name must be a valid Python identifier"
236
+
237
+ # 检查格式(建议使用snake_case)
238
+ if not re.match(r'^[a-z][a-z0-9_]*$', project_name):
239
+ return False, (
240
+ "Project name should start with lowercase letter and "
241
+ "contain only lowercase letters, numbers, and underscores"
242
+ )
243
+
244
+ # 检查是否以数字结尾(不推荐)
245
+ if project_name[-1].isdigit():
246
+ return False, "Project name should not end with a number"
247
+
248
+ return True, ""
249
+
250
+
251
+ def show_template_options():
252
+ """显示可用的模板选项"""
253
+ if RICH_AVAILABLE:
254
+ table = Table(title="可用模板类型", show_header=True, header_style="bold magenta")
255
+ table.add_column("模板类型", style="cyan", no_wrap=True)
256
+ table.add_column("描述", style="green")
257
+
258
+ for template_type, description in TEMPLATE_TYPES.items():
259
+ table.add_row(template_type, description)
260
+
261
+ console.print(table)
262
+ else:
263
+ print("可用模板类型:")
264
+ for template_type, description in TEMPLATE_TYPES.items():
265
+ print(f" {template_type}: {description}")
266
+
267
+
268
+ def show_module_options():
269
+ """显示可用的模块选项"""
270
+ if RICH_AVAILABLE:
271
+ table = Table(title="可选模块组件", show_header=True, header_style="bold magenta")
272
+ table.add_column("模块", style="cyan", no_wrap=True)
273
+ table.add_column("描述", style="green")
274
+
275
+ for module, description in OPTIONAL_MODULES.items():
276
+ table.add_row(module, description)
277
+
278
+ console.print(table)
279
+ else:
280
+ print("可选模块组件:")
281
+ for module, description in OPTIONAL_MODULES.items():
282
+ print(f" {module}: {description}")
283
+
284
+
285
+ def main(args):
286
+ if len(args) < 1:
287
+ console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo startproject[/blue] <project_name> [template_type] [--modules module1,module2]")
288
+ console.print("💡 Examples:")
289
+ console.print(" [blue]crawlo startproject[/blue] my_spider_project")
290
+ console.print(" [blue]crawlo startproject[/blue] news_crawler simple")
291
+ console.print(" [blue]crawlo startproject[/blue] ecommerce_spider distributed --modules mysql,proxy")
292
+ show_template_options()
293
+ show_module_options()
294
+ return 1
295
+
296
+ # 解析参数
297
+ project_name = args[0]
298
+ template_type = 'default'
299
+ modules = None
300
+
301
+ # 解析可选参数
302
+ if len(args) > 1:
303
+ for i, arg in enumerate(args[1:], 1):
304
+ if arg.startswith('--modules='):
305
+ modules_str = arg.split('=', 1)[1]
306
+ modules = [m.strip() for m in modules_str.split(',') if m.strip()]
307
+ elif arg.startswith('--modules'):
308
+ # 处理 --modules module1,module2 格式
309
+ if i + 1 < len(args):
310
+ modules_str = args[i + 1]
311
+ modules = [m.strip() for m in modules_str.split(',') if m.strip()]
312
+ elif not arg.startswith('--') and arg in TEMPLATE_TYPES:
313
+ template_type = arg
314
+
315
+ # 验证模板类型
316
+ if template_type not in TEMPLATE_TYPES:
317
+ show_error_panel(
318
+ "Invalid Template Type",
319
+ f"Template type '[cyan]{template_type}[/cyan]' is not supported.\n"
320
+ )
321
+ show_template_options()
322
+ return 1
323
+
324
+ # 验证项目名称
325
+ is_valid, error_msg = validate_project_name(project_name)
326
+ if not is_valid:
327
+ show_error_panel(
328
+ "Invalid Project Name",
329
+ f"[cyan]{project_name}[/cyan] is not a valid project name.\n"
330
+ f"❌ {error_msg}\n\n"
331
+ "💡 Project name should:\n"
332
+ " • Start with lowercase letter\n"
333
+ " • Contain only lowercase letters, numbers, and underscores\n"
334
+ " • Be a valid Python identifier\n"
335
+ " • Not be a Python keyword"
336
+ )
337
+ return 1
338
+
339
+ project_dir = Path(project_name)
340
+
341
+ if project_dir.exists():
342
+ show_error_panel(
343
+ "Directory Exists",
344
+ f"Directory '[cyan]{project_dir}[/cyan]' already exists.\n"
345
+ "💡 Choose a different project name or remove the existing directory."
346
+ )
347
+ return 1
348
+
349
+ context = {'project_name': project_name}
350
+ template_dir = TEMPLATES_DIR / 'project'
351
+
352
+ try:
353
+ # 1. 创建项目根目录
354
+ project_dir.mkdir()
355
+
356
+ # 2. 渲染 crawlo.cfg.tmpl
357
+ cfg_template = TEMPLATES_DIR / 'crawlo.cfg.tmpl'
358
+ if cfg_template.exists():
359
+ cfg_content = _render_template(cfg_template, context)
360
+ (project_dir / 'crawlo.cfg').write_text(cfg_content, encoding='utf-8')
361
+ console.print(f":white_check_mark: Created [green]{project_dir / 'crawlo.cfg'}[/green]")
362
+ else:
363
+ console.print("[yellow]⚠ Warning:[/yellow] Template 'crawlo.cfg.tmpl' not found.")
364
+
365
+ # 3. 复制并渲染项目包内容
366
+ package_dir = project_dir / project_name
367
+ _copytree_with_templates(template_dir, package_dir, context, template_type, modules)
368
+ console.print(f":white_check_mark: Created project package: [green]{package_dir}[/green]")
369
+
370
+ # 4. 创建 logs 目录
371
+ (project_dir / 'logs').mkdir(exist_ok=True)
372
+ console.print(":white_check_mark: Created logs directory")
373
+
374
+ # 5. 创建 output 目录(用于数据输出)
375
+ (project_dir / 'output').mkdir(exist_ok=True)
376
+ console.print(":white_check_mark: Created output directory")
377
+
378
+ # 成功面板
379
+ success_text = Text.from_markup(f"Project '[bold cyan]{project_name}[/bold cyan]' created successfully!")
380
+ console.print(Panel(success_text, title=":rocket: Success", border_style="green", padding=(1, 2)))
381
+
382
+ # 显示使用的模板类型
383
+ if template_type != 'default':
384
+ console.print(f":information: 使用模板类型: [bold blue]{template_type}[/bold blue] - {TEMPLATE_TYPES[template_type]}")
385
+
386
+ # 显示选择的模块
387
+ if modules:
388
+ console.print(f":information: 选择的模块: [bold blue]{', '.join(modules)}[/bold blue]")
389
+
390
+ # 下一步操作提示(对齐美观 + 语法高亮)
391
+ next_steps = f"""
392
+ [bold]🚀 Next steps:[/bold]
393
+ [blue]cd[/blue] {project_name}
394
+ [blue]crawlo genspider[/blue] example example.com
395
+ [blue]crawlo run[/blue] example
396
+
397
+ [bold]📚 Learn more:[/bold]
398
+ [blue]crawlo list[/blue] # List all spiders
399
+ [blue]crawlo check[/blue] example # Check spider validity
400
+ [blue]crawlo stats[/blue] # View statistics
401
+ """.strip()
402
+ console.print(next_steps)
403
+
404
+ return 0
405
+
406
+ except Exception as e:
407
+ show_error_panel(
408
+ "Creation Failed",
409
+ f"Failed to create project: {e}"
410
+ )
411
+ if project_dir.exists():
412
+ shutil.rmtree(project_dir, ignore_errors=True)
413
+ console.print("[red]:cross_mark: Cleaned up partially created project.[/red]")
414
+ return 1
415
+
416
+ if __name__ == "__main__":
417
+ import sys
418
+ exit_code = main(sys.argv[1:])
419
419
  sys.exit(exit_code)