crawlo 1.4.5__py3-none-any.whl → 1.4.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of crawlo might be problematic. Click here for more details.

Files changed (375) hide show
  1. crawlo/__init__.py +90 -89
  2. crawlo/__version__.py +1 -1
  3. crawlo/cli.py +75 -75
  4. crawlo/commands/__init__.py +14 -14
  5. crawlo/commands/check.py +594 -594
  6. crawlo/commands/genspider.py +186 -186
  7. crawlo/commands/help.py +140 -138
  8. crawlo/commands/list.py +155 -155
  9. crawlo/commands/run.py +379 -341
  10. crawlo/commands/startproject.py +460 -460
  11. crawlo/commands/stats.py +187 -187
  12. crawlo/commands/utils.py +196 -196
  13. crawlo/config.py +320 -312
  14. crawlo/config_validator.py +277 -277
  15. crawlo/core/__init__.py +52 -52
  16. crawlo/core/engine.py +451 -438
  17. crawlo/core/processor.py +47 -47
  18. crawlo/core/scheduler.py +290 -291
  19. crawlo/crawler.py +698 -657
  20. crawlo/data/__init__.py +5 -5
  21. crawlo/data/user_agents.py +194 -194
  22. crawlo/downloader/__init__.py +280 -276
  23. crawlo/downloader/aiohttp_downloader.py +233 -233
  24. crawlo/downloader/cffi_downloader.py +250 -245
  25. crawlo/downloader/httpx_downloader.py +265 -259
  26. crawlo/downloader/hybrid_downloader.py +212 -212
  27. crawlo/downloader/playwright_downloader.py +425 -402
  28. crawlo/downloader/selenium_downloader.py +486 -472
  29. crawlo/event.py +45 -11
  30. crawlo/exceptions.py +215 -82
  31. crawlo/extension/__init__.py +65 -64
  32. crawlo/extension/health_check.py +141 -141
  33. crawlo/extension/log_interval.py +94 -94
  34. crawlo/extension/log_stats.py +70 -70
  35. crawlo/extension/logging_extension.py +53 -61
  36. crawlo/extension/memory_monitor.py +104 -104
  37. crawlo/extension/performance_profiler.py +133 -133
  38. crawlo/extension/request_recorder.py +107 -107
  39. crawlo/factories/__init__.py +27 -27
  40. crawlo/factories/base.py +68 -68
  41. crawlo/factories/crawler.py +104 -103
  42. crawlo/factories/registry.py +84 -84
  43. crawlo/factories/utils.py +135 -0
  44. crawlo/filters/__init__.py +170 -153
  45. crawlo/filters/aioredis_filter.py +348 -264
  46. crawlo/filters/memory_filter.py +261 -276
  47. crawlo/framework.py +306 -292
  48. crawlo/initialization/__init__.py +44 -44
  49. crawlo/initialization/built_in.py +391 -434
  50. crawlo/initialization/context.py +141 -141
  51. crawlo/initialization/core.py +240 -194
  52. crawlo/initialization/phases.py +230 -149
  53. crawlo/initialization/registry.py +143 -145
  54. crawlo/initialization/utils.py +49 -0
  55. crawlo/interfaces.py +23 -23
  56. crawlo/items/__init__.py +23 -23
  57. crawlo/items/base.py +23 -23
  58. crawlo/items/fields.py +52 -52
  59. crawlo/items/items.py +104 -104
  60. crawlo/logging/__init__.py +42 -46
  61. crawlo/logging/config.py +277 -197
  62. crawlo/logging/factory.py +175 -171
  63. crawlo/logging/manager.py +104 -112
  64. crawlo/middleware/__init__.py +87 -24
  65. crawlo/middleware/default_header.py +132 -132
  66. crawlo/middleware/download_delay.py +104 -104
  67. crawlo/middleware/middleware_manager.py +142 -142
  68. crawlo/middleware/offsite.py +123 -123
  69. crawlo/middleware/proxy.py +209 -386
  70. crawlo/middleware/request_ignore.py +86 -86
  71. crawlo/middleware/response_code.py +150 -150
  72. crawlo/middleware/response_filter.py +136 -136
  73. crawlo/middleware/retry.py +124 -124
  74. crawlo/mode_manager.py +287 -253
  75. crawlo/network/__init__.py +21 -21
  76. crawlo/network/request.py +375 -379
  77. crawlo/network/response.py +569 -664
  78. crawlo/pipelines/__init__.py +53 -22
  79. crawlo/pipelines/base_pipeline.py +452 -0
  80. crawlo/pipelines/bloom_dedup_pipeline.py +146 -146
  81. crawlo/pipelines/console_pipeline.py +39 -39
  82. crawlo/pipelines/csv_pipeline.py +316 -316
  83. crawlo/pipelines/database_dedup_pipeline.py +197 -197
  84. crawlo/pipelines/json_pipeline.py +218 -218
  85. crawlo/pipelines/memory_dedup_pipeline.py +105 -105
  86. crawlo/pipelines/mongo_pipeline.py +140 -132
  87. crawlo/pipelines/mysql_pipeline.py +470 -326
  88. crawlo/pipelines/pipeline_manager.py +100 -100
  89. crawlo/pipelines/redis_dedup_pipeline.py +155 -156
  90. crawlo/project.py +347 -347
  91. crawlo/queue/__init__.py +10 -0
  92. crawlo/queue/pqueue.py +38 -38
  93. crawlo/queue/queue_manager.py +591 -525
  94. crawlo/queue/redis_priority_queue.py +519 -370
  95. crawlo/settings/__init__.py +7 -7
  96. crawlo/settings/default_settings.py +285 -270
  97. crawlo/settings/setting_manager.py +219 -219
  98. crawlo/spider/__init__.py +657 -657
  99. crawlo/stats_collector.py +82 -73
  100. crawlo/subscriber.py +129 -129
  101. crawlo/task_manager.py +138 -138
  102. crawlo/templates/crawlo.cfg.tmpl +10 -10
  103. crawlo/templates/project/__init__.py.tmpl +2 -4
  104. crawlo/templates/project/items.py.tmpl +13 -17
  105. crawlo/templates/project/middlewares.py.tmpl +38 -38
  106. crawlo/templates/project/pipelines.py.tmpl +35 -36
  107. crawlo/templates/project/settings.py.tmpl +110 -157
  108. crawlo/templates/project/settings_distributed.py.tmpl +156 -161
  109. crawlo/templates/project/settings_gentle.py.tmpl +170 -171
  110. crawlo/templates/project/settings_high_performance.py.tmpl +171 -172
  111. crawlo/templates/project/settings_minimal.py.tmpl +99 -77
  112. crawlo/templates/project/settings_simple.py.tmpl +168 -169
  113. crawlo/templates/project/spiders/__init__.py.tmpl +9 -9
  114. crawlo/templates/run.py.tmpl +23 -30
  115. crawlo/templates/spider/spider.py.tmpl +33 -144
  116. crawlo/templates/spiders_init.py.tmpl +5 -10
  117. crawlo/tools/__init__.py +86 -189
  118. crawlo/tools/date_tools.py +289 -289
  119. crawlo/tools/distributed_coordinator.py +384 -384
  120. crawlo/tools/scenario_adapter.py +262 -262
  121. crawlo/tools/text_cleaner.py +232 -232
  122. crawlo/utils/__init__.py +50 -50
  123. crawlo/utils/batch_processor.py +276 -259
  124. crawlo/utils/config_manager.py +442 -0
  125. crawlo/utils/controlled_spider_mixin.py +439 -439
  126. crawlo/utils/db_helper.py +250 -244
  127. crawlo/utils/error_handler.py +410 -410
  128. crawlo/utils/fingerprint.py +121 -121
  129. crawlo/utils/func_tools.py +82 -82
  130. crawlo/utils/large_scale_helper.py +344 -344
  131. crawlo/utils/leak_detector.py +335 -0
  132. crawlo/utils/log.py +79 -79
  133. crawlo/utils/misc.py +81 -81
  134. crawlo/utils/mongo_connection_pool.py +157 -0
  135. crawlo/utils/mysql_connection_pool.py +197 -0
  136. crawlo/utils/performance_monitor.py +285 -285
  137. crawlo/utils/queue_helper.py +175 -175
  138. crawlo/utils/redis_checker.py +91 -0
  139. crawlo/utils/redis_connection_pool.py +578 -388
  140. crawlo/utils/redis_key_validator.py +198 -198
  141. crawlo/utils/request.py +278 -256
  142. crawlo/utils/request_serializer.py +225 -225
  143. crawlo/utils/resource_manager.py +337 -0
  144. crawlo/utils/selector_helper.py +137 -137
  145. crawlo/utils/singleton.py +70 -0
  146. crawlo/utils/spider_loader.py +201 -201
  147. crawlo/utils/text_helper.py +94 -94
  148. crawlo/utils/{url.py → url_utils.py} +39 -39
  149. crawlo-1.4.7.dist-info/METADATA +689 -0
  150. crawlo-1.4.7.dist-info/RECORD +347 -0
  151. examples/__init__.py +7 -7
  152. tests/__init__.py +7 -7
  153. tests/advanced_tools_example.py +217 -275
  154. tests/authenticated_proxy_example.py +110 -106
  155. tests/baidu_performance_test.py +108 -108
  156. tests/baidu_test.py +59 -59
  157. tests/bug_check_test.py +250 -250
  158. tests/cleaners_example.py +160 -160
  159. tests/comprehensive_framework_test.py +212 -212
  160. tests/comprehensive_test.py +81 -81
  161. tests/comprehensive_testing_summary.md +186 -186
  162. tests/config_validation_demo.py +142 -142
  163. tests/controlled_spider_example.py +205 -205
  164. tests/date_tools_example.py +180 -180
  165. tests/debug_configure.py +69 -69
  166. tests/debug_framework_logger.py +84 -84
  167. tests/debug_log_config.py +126 -126
  168. tests/debug_log_levels.py +63 -63
  169. tests/debug_pipelines.py +66 -66
  170. tests/detailed_log_test.py +233 -233
  171. tests/direct_selector_helper_test.py +96 -96
  172. tests/distributed_dedup_test.py +467 -0
  173. tests/distributed_test.py +66 -66
  174. tests/distributed_test_debug.py +76 -76
  175. tests/dynamic_loading_example.py +523 -523
  176. tests/dynamic_loading_test.py +104 -104
  177. tests/error_handling_example.py +171 -171
  178. tests/explain_mysql_update_behavior.py +77 -0
  179. tests/final_comprehensive_test.py +151 -151
  180. tests/final_log_test.py +260 -260
  181. tests/final_validation_test.py +182 -182
  182. tests/fix_log_test.py +142 -142
  183. tests/framework_performance_test.py +202 -202
  184. tests/log_buffering_test.py +111 -111
  185. tests/log_generation_timing_test.py +153 -153
  186. tests/monitor_redis_dedup.sh +72 -0
  187. tests/ofweek_scrapy/ofweek_scrapy/items.py +12 -12
  188. tests/ofweek_scrapy/ofweek_scrapy/middlewares.py +100 -100
  189. tests/ofweek_scrapy/ofweek_scrapy/pipelines.py +13 -13
  190. tests/ofweek_scrapy/ofweek_scrapy/settings.py +84 -84
  191. tests/ofweek_scrapy/scrapy.cfg +11 -11
  192. tests/optimized_performance_test.py +211 -211
  193. tests/performance_comparison.py +244 -244
  194. tests/queue_blocking_test.py +113 -113
  195. tests/queue_test.py +89 -89
  196. tests/redis_key_validation_demo.py +130 -130
  197. tests/request_params_example.py +150 -150
  198. tests/response_improvements_example.py +144 -144
  199. tests/scrapy_comparison/ofweek_scrapy.py +138 -138
  200. tests/scrapy_comparison/scrapy_test.py +133 -133
  201. tests/simple_cli_test.py +55 -0
  202. tests/simple_command_test.py +119 -119
  203. tests/simple_crawlo_test.py +126 -126
  204. tests/simple_follow_test.py +38 -38
  205. tests/simple_log_test2.py +137 -137
  206. tests/simple_optimization_test.py +128 -128
  207. tests/simple_queue_type_test.py +41 -41
  208. tests/simple_response_selector_test.py +94 -94
  209. tests/simple_selector_helper_test.py +154 -154
  210. tests/simple_selector_test.py +207 -207
  211. tests/simple_spider_test.py +49 -49
  212. tests/simple_url_test.py +73 -73
  213. tests/simulate_mysql_update_test.py +140 -0
  214. tests/spider_log_timing_test.py +177 -177
  215. tests/test_advanced_tools.py +148 -148
  216. tests/test_all_commands.py +230 -230
  217. tests/test_all_pipeline_fingerprints.py +133 -133
  218. tests/test_all_redis_key_configs.py +145 -145
  219. tests/test_asyncmy_usage.py +57 -0
  220. tests/test_batch_processor.py +178 -178
  221. tests/test_cleaners.py +54 -54
  222. tests/test_cli_arguments.py +119 -0
  223. tests/test_component_factory.py +174 -174
  224. tests/test_config_consistency.py +80 -80
  225. tests/test_config_merge.py +152 -152
  226. tests/test_config_validator.py +182 -182
  227. tests/test_controlled_spider_mixin.py +79 -79
  228. tests/test_crawler_process_import.py +38 -38
  229. tests/test_crawler_process_spider_modules.py +47 -47
  230. tests/test_crawlo_proxy_integration.py +114 -108
  231. tests/test_date_tools.py +123 -123
  232. tests/test_dedup_fix.py +220 -220
  233. tests/test_dedup_pipeline_consistency.py +124 -124
  234. tests/test_default_header_middleware.py +313 -313
  235. tests/test_distributed.py +65 -65
  236. tests/test_double_crawlo_fix.py +204 -204
  237. tests/test_double_crawlo_fix_simple.py +124 -124
  238. tests/test_download_delay_middleware.py +221 -221
  239. tests/test_downloader_proxy_compatibility.py +272 -268
  240. tests/test_edge_cases.py +305 -305
  241. tests/test_encoding_core.py +56 -56
  242. tests/test_encoding_detection.py +126 -126
  243. tests/test_enhanced_error_handler.py +270 -270
  244. tests/test_enhanced_error_handler_comprehensive.py +245 -245
  245. tests/test_error_handler_compatibility.py +112 -112
  246. tests/test_factories.py +252 -252
  247. tests/test_factory_compatibility.py +196 -196
  248. tests/test_final_validation.py +153 -153
  249. tests/test_fingerprint_consistency.py +135 -135
  250. tests/test_fingerprint_simple.py +51 -51
  251. tests/test_get_component_logger.py +83 -83
  252. tests/test_hash_performance.py +99 -99
  253. tests/test_integration.py +169 -169
  254. tests/test_item_dedup_redis_key.py +122 -122
  255. tests/test_large_scale_helper.py +235 -235
  256. tests/test_logging_enhancements.py +374 -374
  257. tests/test_logging_final.py +184 -184
  258. tests/test_logging_integration.py +312 -312
  259. tests/test_logging_system.py +282 -282
  260. tests/test_middleware_debug.py +141 -141
  261. tests/test_mode_consistency.py +51 -51
  262. tests/test_multi_directory.py +67 -67
  263. tests/test_multiple_spider_modules.py +80 -80
  264. tests/test_mysql_pipeline_config.py +165 -0
  265. tests/test_mysql_pipeline_error.py +99 -0
  266. tests/test_mysql_pipeline_init_log.py +83 -0
  267. tests/test_mysql_pipeline_integration.py +133 -0
  268. tests/test_mysql_pipeline_refactor.py +144 -0
  269. tests/test_mysql_pipeline_refactor_simple.py +86 -0
  270. tests/test_mysql_pipeline_robustness.py +196 -0
  271. tests/test_mysql_pipeline_types.py +89 -0
  272. tests/test_mysql_update_columns.py +94 -0
  273. tests/test_offsite_middleware.py +244 -244
  274. tests/test_offsite_middleware_simple.py +203 -203
  275. tests/test_optimized_selector_naming.py +100 -100
  276. tests/test_parsel.py +29 -29
  277. tests/test_performance.py +327 -327
  278. tests/test_performance_monitor.py +115 -115
  279. tests/test_pipeline_fingerprint_consistency.py +86 -86
  280. tests/test_priority_behavior.py +211 -211
  281. tests/test_priority_consistency.py +151 -151
  282. tests/test_priority_consistency_fixed.py +249 -249
  283. tests/test_proxy_health_check.py +32 -32
  284. tests/test_proxy_middleware.py +217 -121
  285. tests/test_proxy_middleware_enhanced.py +212 -216
  286. tests/test_proxy_middleware_integration.py +142 -137
  287. tests/test_proxy_middleware_refactored.py +207 -184
  288. tests/test_proxy_only.py +84 -0
  289. tests/test_proxy_providers.py +56 -56
  290. tests/test_proxy_stats.py +19 -19
  291. tests/test_proxy_strategies.py +59 -59
  292. tests/test_proxy_with_downloader.py +153 -0
  293. tests/test_queue_empty_check.py +41 -41
  294. tests/test_queue_manager_double_crawlo.py +173 -173
  295. tests/test_queue_manager_redis_key.py +179 -179
  296. tests/test_queue_naming.py +154 -154
  297. tests/test_queue_type.py +106 -106
  298. tests/test_queue_type_redis_config_consistency.py +130 -130
  299. tests/test_random_headers_default.py +322 -322
  300. tests/test_random_headers_necessity.py +308 -308
  301. tests/test_random_user_agent.py +72 -72
  302. tests/test_redis_config.py +28 -28
  303. tests/test_redis_connection_pool.py +294 -294
  304. tests/test_redis_key_naming.py +181 -181
  305. tests/test_redis_key_validator.py +123 -123
  306. tests/test_redis_queue.py +224 -224
  307. tests/test_redis_queue_name_fix.py +175 -175
  308. tests/test_redis_queue_type_fallback.py +129 -129
  309. tests/test_request_ignore_middleware.py +182 -182
  310. tests/test_request_params.py +111 -111
  311. tests/test_request_serialization.py +70 -70
  312. tests/test_response_code_middleware.py +349 -349
  313. tests/test_response_filter_middleware.py +427 -427
  314. tests/test_response_follow.py +104 -104
  315. tests/test_response_improvements.py +152 -152
  316. tests/test_response_selector_methods.py +92 -92
  317. tests/test_response_url_methods.py +70 -70
  318. tests/test_response_urljoin.py +86 -86
  319. tests/test_retry_middleware.py +333 -333
  320. tests/test_retry_middleware_realistic.py +273 -273
  321. tests/test_scheduler.py +252 -252
  322. tests/test_scheduler_config_update.py +133 -133
  323. tests/test_scrapy_style_encoding.py +112 -112
  324. tests/test_selector_helper.py +100 -100
  325. tests/test_selector_optimizations.py +146 -146
  326. tests/test_simple_response.py +61 -61
  327. tests/test_spider_loader.py +49 -49
  328. tests/test_spider_loader_comprehensive.py +69 -69
  329. tests/test_spider_modules.py +84 -84
  330. tests/test_spiders/test_spider.py +9 -9
  331. tests/test_telecom_spider_redis_key.py +205 -205
  332. tests/test_template_content.py +87 -87
  333. tests/test_template_redis_key.py +134 -134
  334. tests/test_tools.py +159 -159
  335. tests/test_user_agent_randomness.py +176 -176
  336. tests/test_user_agents.py +96 -96
  337. tests/untested_features_report.md +138 -138
  338. tests/verify_debug.py +51 -51
  339. tests/verify_distributed.py +117 -117
  340. tests/verify_log_fix.py +111 -111
  341. tests/verify_mysql_warnings.py +110 -0
  342. crawlo/logging/async_handler.py +0 -181
  343. crawlo/logging/monitor.py +0 -153
  344. crawlo/logging/sampler.py +0 -167
  345. crawlo/middleware/simple_proxy.py +0 -65
  346. crawlo/tools/authenticated_proxy.py +0 -241
  347. crawlo/tools/data_formatter.py +0 -226
  348. crawlo/tools/data_validator.py +0 -181
  349. crawlo/tools/encoding_converter.py +0 -127
  350. crawlo/tools/network_diagnostic.py +0 -365
  351. crawlo/tools/request_tools.py +0 -83
  352. crawlo/tools/retry_mechanism.py +0 -224
  353. crawlo/utils/env_config.py +0 -143
  354. crawlo/utils/large_scale_config.py +0 -287
  355. crawlo/utils/system.py +0 -11
  356. crawlo/utils/tools.py +0 -5
  357. crawlo-1.4.5.dist-info/METADATA +0 -329
  358. crawlo-1.4.5.dist-info/RECORD +0 -347
  359. tests/env_config_example.py +0 -134
  360. tests/ofweek_scrapy/ofweek_scrapy/spiders/ofweek_spider.py +0 -162
  361. tests/test_authenticated_proxy.py +0 -142
  362. tests/test_comprehensive.py +0 -147
  363. tests/test_dynamic_downloaders_proxy.py +0 -125
  364. tests/test_dynamic_proxy.py +0 -93
  365. tests/test_dynamic_proxy_config.py +0 -147
  366. tests/test_dynamic_proxy_real.py +0 -110
  367. tests/test_env_config.py +0 -122
  368. tests/test_framework_env_usage.py +0 -104
  369. tests/test_large_scale_config.py +0 -113
  370. tests/test_proxy_api.py +0 -265
  371. tests/test_real_scenario_proxy.py +0 -196
  372. tests/tools_example.py +0 -261
  373. {crawlo-1.4.5.dist-info → crawlo-1.4.7.dist-info}/WHEEL +0 -0
  374. {crawlo-1.4.5.dist-info → crawlo-1.4.7.dist-info}/entry_points.txt +0 -0
  375. {crawlo-1.4.5.dist-info → crawlo-1.4.7.dist-info}/top_level.txt +0 -0
@@ -1,246 +1,246 @@
1
- #!/usr/bin/env python3
2
- # -*- coding: utf-8 -*-
3
- """
4
- 增强错误处理工具综合测试
5
- 测试 ErrorContext, DetailedException, EnhancedErrorHandler 的更多功能
6
- """
7
- import sys
8
- import os
9
- import unittest
10
- from unittest.mock import Mock, patch, MagicMock
11
- import asyncio
12
- import traceback
13
-
14
- # 添加项目根目录到 Python 路径
15
- sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
16
-
17
- from crawlo.utils.enhanced_error_handler import (
18
- EnhancedErrorHandler,
19
- ErrorContext,
20
- DetailedException,
21
- handle_exception
22
- )
23
-
24
-
25
- class TestErrorContext(unittest.TestCase):
26
- """错误上下文测试"""
27
-
28
- def test_error_context_initialization(self):
29
- """测试错误上下文初始化"""
30
- context = ErrorContext(
31
- context="测试上下文",
32
- module="test_module",
33
- function="test_function"
34
- )
35
-
36
- self.assertEqual(context.context, "测试上下文")
37
- self.assertEqual(context.module, "test_module")
38
- self.assertEqual(context.function, "test_function")
39
- self.assertIsNotNone(context.timestamp)
40
-
41
- def test_error_context_string_representation(self):
42
- """测试错误上下文字符串表示"""
43
- context = ErrorContext(
44
- context="测试上下文",
45
- module="test_module",
46
- function="test_function"
47
- )
48
-
49
- context_str = str(context)
50
- self.assertIn("Module: test_module", context_str)
51
- self.assertIn("Function: test_function", context_str)
52
- self.assertIn("Context: 测试上下文", context_str)
53
- self.assertIn("Time:", context_str)
54
-
55
-
56
- class TestDetailedException(unittest.TestCase):
57
- """详细异常测试"""
58
-
59
- def test_detailed_exception_initialization(self):
60
- """测试详细异常初始化"""
61
- context = ErrorContext(context="测试上下文")
62
-
63
- exception = DetailedException(
64
- message="测试异常消息",
65
- context=context,
66
- error_code="TEST_001",
67
- detail1="详细信息1",
68
- detail2="详细信息2"
69
- )
70
-
71
- self.assertIn("测试异常消息", str(exception))
72
- self.assertEqual(exception.context, context)
73
- self.assertEqual(exception.error_code, "TEST_001")
74
- self.assertEqual(exception.details["detail1"], "详细信息1")
75
- self.assertEqual(exception.details["detail2"], "详细信息2")
76
- self.assertIsNotNone(exception.timestamp)
77
-
78
- def test_detailed_exception_without_context(self):
79
- """测试没有上下文的详细异常"""
80
- exception = DetailedException(
81
- message="测试异常消息",
82
- error_code="TEST_002"
83
- )
84
-
85
- self.assertEqual(str(exception), "测试异常消息")
86
- self.assertIsNone(exception.context)
87
- self.assertEqual(exception.error_code, "TEST_002")
88
- self.assertIsNotNone(exception.timestamp)
89
-
90
- def test_detailed_exception_string_with_context(self):
91
- """测试带上下文的详细异常字符串表示"""
92
- context = ErrorContext(context="测试上下文")
93
- exception = DetailedException(
94
- message="测试异常消息",
95
- context=context
96
- )
97
-
98
- exception_str = str(exception)
99
- self.assertIn("测试异常消息", exception_str)
100
- self.assertIn("测试上下文", exception_str)
101
-
102
- def test_get_full_details(self):
103
- """测试获取完整详情"""
104
- context = ErrorContext(context="测试上下文")
105
- exception = DetailedException(
106
- message="测试异常消息",
107
- context=context,
108
- error_code="TEST_003",
109
- detail1="详细信息1"
110
- )
111
-
112
- details = exception.get_full_details()
113
- self.assertIn("测试异常消息", details["message"])
114
- self.assertEqual(details["error_code"], "TEST_003")
115
- self.assertIn("测试上下文", details["context"])
116
- self.assertEqual(details["details"]["detail1"], "详细信息1")
117
- self.assertEqual(details["exception_type"], "DetailedException")
118
-
119
-
120
- class TestEnhancedErrorHandler(unittest.TestCase):
121
- """增强错误处理器测试"""
122
-
123
- def setUp(self):
124
- """测试前准备"""
125
- self.handler = EnhancedErrorHandler("test_logger", "ERROR")
126
-
127
- def test_handler_initialization(self):
128
- """测试处理器初始化"""
129
- self.assertEqual(len(self.handler.error_history), 0)
130
- self.assertEqual(self.handler.max_history_size, 100)
131
-
132
- def test_handle_error_without_raising(self):
133
- """测试不抛出异常的错误处理"""
134
- try:
135
- raise ValueError("测试错误")
136
- except Exception as e:
137
- # 处理错误但不重新抛出
138
- error_details = self.handler.handle_error(e, raise_error=False, log_error=False)
139
-
140
- # 验证返回的错误详情
141
- self.assertIsInstance(error_details, dict)
142
- self.assertEqual(error_details["exception_type"], "ValueError")
143
- self.assertEqual(error_details["message"], "测试错误")
144
-
145
- def test_safe_call_success(self):
146
- """测试安全调用成功"""
147
- def normal_function(x, y):
148
- return x + y
149
-
150
- result = self.handler.safe_call(normal_function, 1, 2, default_return=0)
151
- self.assertEqual(result, 3)
152
-
153
- def test_safe_call_with_exception(self):
154
- """测试安全调用异常"""
155
- def failing_function():
156
- raise RuntimeError("函数执行失败")
157
-
158
- result = self.handler.safe_call(failing_function, default_return="默认值")
159
- self.assertEqual(result, "默认值")
160
-
161
- def test_get_and_clear_error_history(self):
162
- """测试获取和清空错误历史"""
163
- # 产生一些错误
164
- try:
165
- raise ValueError("错误1")
166
- except Exception as e:
167
- self.handler.handle_error(e, raise_error=False, log_error=False)
168
-
169
- try:
170
- raise RuntimeError("错误2")
171
- except Exception as e:
172
- self.handler.handle_error(e, raise_error=False, log_error=False)
173
-
174
- # 检查历史记录
175
- history = self.handler.get_error_history()
176
- self.assertEqual(len(history), 2)
177
-
178
- # 清空历史记录
179
- self.handler.clear_error_history()
180
- history = self.handler.get_error_history()
181
- self.assertEqual(len(history), 0)
182
-
183
- def test_error_history_size_limit(self):
184
- """测试错误历史大小限制"""
185
- # 产生超过100个错误
186
- for i in range(110):
187
- try:
188
- raise ValueError(f"错误{i}")
189
- except Exception as e:
190
- self.handler.handle_error(e, raise_error=False, log_error=False)
191
-
192
- # 检查历史记录大小限制
193
- history = self.handler.get_error_history()
194
- self.assertEqual(len(history), 100) # 应该限制在100个
195
-
196
-
197
- class TestHandleExceptionDecorator(unittest.TestCase):
198
- """异常处理装饰器测试"""
199
-
200
- def test_handle_exception_decorator_sync(self):
201
- """测试同步函数的异常处理装饰器"""
202
- @handle_exception(context="测试装饰器", module="test_module", raise_error=False)
203
- def failing_function():
204
- raise ValueError("装饰器测试错误")
205
-
206
- # 调用应该捕获异常但不抛出
207
- try:
208
- result = failing_function()
209
- self.assertIsNone(result) # 默认返回None
210
- except Exception:
211
- self.fail("异常应该被捕获")
212
-
213
- def test_handle_exception_decorator_async(self):
214
- """测试异步函数的异常处理装饰器"""
215
- @handle_exception(context="异步测试装饰器", module="test_module", raise_error=False)
216
- async def async_failing_function():
217
- raise RuntimeError("异步装饰器测试错误")
218
-
219
- # 异步调用
220
- async def test_async():
221
- try:
222
- result = await async_failing_function()
223
- self.assertIsNone(result) # 默认返回None
224
- except Exception:
225
- self.fail("异常应该被捕获")
226
-
227
- # 使用事件循环运行异步函数
228
- asyncio.run(test_async())
229
-
230
- def test_handle_exception_decorator_with_detailed_exception(self):
231
- """测试装饰器处理详细异常"""
232
- @handle_exception(context="详细异常测试", module="test_module", raise_error=False)
233
- def function_with_detailed_exception():
234
- context = ErrorContext(context="函数内部上下文")
235
- raise DetailedException("详细异常消息", context=context, error_code="DETAIL_001")
236
-
237
- # 调用应该捕获异常
238
- try:
239
- result = function_with_detailed_exception()
240
- self.assertIsNone(result)
241
- except Exception:
242
- self.fail("详细异常应该被捕获")
243
-
244
-
245
- if __name__ == '__main__':
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ 增强错误处理工具综合测试
5
+ 测试 ErrorContext, DetailedException, EnhancedErrorHandler 的更多功能
6
+ """
7
+ import sys
8
+ import os
9
+ import unittest
10
+ from unittest.mock import Mock, patch, MagicMock
11
+ import asyncio
12
+ import traceback
13
+
14
+ # 添加项目根目录到 Python 路径
15
+ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
16
+
17
+ from crawlo.utils.enhanced_error_handler import (
18
+ EnhancedErrorHandler,
19
+ ErrorContext,
20
+ DetailedException,
21
+ handle_exception
22
+ )
23
+
24
+
25
+ class TestErrorContext(unittest.TestCase):
26
+ """错误上下文测试"""
27
+
28
+ def test_error_context_initialization(self):
29
+ """测试错误上下文初始化"""
30
+ context = ErrorContext(
31
+ context="测试上下文",
32
+ module="test_module",
33
+ function="test_function"
34
+ )
35
+
36
+ self.assertEqual(context.context, "测试上下文")
37
+ self.assertEqual(context.module, "test_module")
38
+ self.assertEqual(context.function, "test_function")
39
+ self.assertIsNotNone(context.timestamp)
40
+
41
+ def test_error_context_string_representation(self):
42
+ """测试错误上下文字符串表示"""
43
+ context = ErrorContext(
44
+ context="测试上下文",
45
+ module="test_module",
46
+ function="test_function"
47
+ )
48
+
49
+ context_str = str(context)
50
+ self.assertIn("Module: test_module", context_str)
51
+ self.assertIn("Function: test_function", context_str)
52
+ self.assertIn("Context: 测试上下文", context_str)
53
+ self.assertIn("Time:", context_str)
54
+
55
+
56
+ class TestDetailedException(unittest.TestCase):
57
+ """详细异常测试"""
58
+
59
+ def test_detailed_exception_initialization(self):
60
+ """测试详细异常初始化"""
61
+ context = ErrorContext(context="测试上下文")
62
+
63
+ exception = DetailedException(
64
+ message="测试异常消息",
65
+ context=context,
66
+ error_code="TEST_001",
67
+ detail1="详细信息1",
68
+ detail2="详细信息2"
69
+ )
70
+
71
+ self.assertIn("测试异常消息", str(exception))
72
+ self.assertEqual(exception.context, context)
73
+ self.assertEqual(exception.error_code, "TEST_001")
74
+ self.assertEqual(exception.details["detail1"], "详细信息1")
75
+ self.assertEqual(exception.details["detail2"], "详细信息2")
76
+ self.assertIsNotNone(exception.timestamp)
77
+
78
+ def test_detailed_exception_without_context(self):
79
+ """测试没有上下文的详细异常"""
80
+ exception = DetailedException(
81
+ message="测试异常消息",
82
+ error_code="TEST_002"
83
+ )
84
+
85
+ self.assertEqual(str(exception), "测试异常消息")
86
+ self.assertIsNone(exception.context)
87
+ self.assertEqual(exception.error_code, "TEST_002")
88
+ self.assertIsNotNone(exception.timestamp)
89
+
90
+ def test_detailed_exception_string_with_context(self):
91
+ """测试带上下文的详细异常字符串表示"""
92
+ context = ErrorContext(context="测试上下文")
93
+ exception = DetailedException(
94
+ message="测试异常消息",
95
+ context=context
96
+ )
97
+
98
+ exception_str = str(exception)
99
+ self.assertIn("测试异常消息", exception_str)
100
+ self.assertIn("测试上下文", exception_str)
101
+
102
+ def test_get_full_details(self):
103
+ """测试获取完整详情"""
104
+ context = ErrorContext(context="测试上下文")
105
+ exception = DetailedException(
106
+ message="测试异常消息",
107
+ context=context,
108
+ error_code="TEST_003",
109
+ detail1="详细信息1"
110
+ )
111
+
112
+ details = exception.get_full_details()
113
+ self.assertIn("测试异常消息", details["message"])
114
+ self.assertEqual(details["error_code"], "TEST_003")
115
+ self.assertIn("测试上下文", details["context"])
116
+ self.assertEqual(details["details"]["detail1"], "详细信息1")
117
+ self.assertEqual(details["exception_type"], "DetailedException")
118
+
119
+
120
+ class TestEnhancedErrorHandler(unittest.TestCase):
121
+ """增强错误处理器测试"""
122
+
123
+ def setUp(self):
124
+ """测试前准备"""
125
+ self.handler = EnhancedErrorHandler("test_logger", "ERROR")
126
+
127
+ def test_handler_initialization(self):
128
+ """测试处理器初始化"""
129
+ self.assertEqual(len(self.handler.error_history), 0)
130
+ self.assertEqual(self.handler.max_history_size, 100)
131
+
132
+ def test_handle_error_without_raising(self):
133
+ """测试不抛出异常的错误处理"""
134
+ try:
135
+ raise ValueError("测试错误")
136
+ except Exception as e:
137
+ # 处理错误但不重新抛出
138
+ error_details = self.handler.handle_error(e, raise_error=False, log_error=False)
139
+
140
+ # 验证返回的错误详情
141
+ self.assertIsInstance(error_details, dict)
142
+ self.assertEqual(error_details["exception_type"], "ValueError")
143
+ self.assertEqual(error_details["message"], "测试错误")
144
+
145
+ def test_safe_call_success(self):
146
+ """测试安全调用成功"""
147
+ def normal_function(x, y):
148
+ return x + y
149
+
150
+ result = self.handler.safe_call(normal_function, 1, 2, default_return=0)
151
+ self.assertEqual(result, 3)
152
+
153
+ def test_safe_call_with_exception(self):
154
+ """测试安全调用异常"""
155
+ def failing_function():
156
+ raise RuntimeError("函数执行失败")
157
+
158
+ result = self.handler.safe_call(failing_function, default_return="默认值")
159
+ self.assertEqual(result, "默认值")
160
+
161
+ def test_get_and_clear_error_history(self):
162
+ """测试获取和清空错误历史"""
163
+ # 产生一些错误
164
+ try:
165
+ raise ValueError("错误1")
166
+ except Exception as e:
167
+ self.handler.handle_error(e, raise_error=False, log_error=False)
168
+
169
+ try:
170
+ raise RuntimeError("错误2")
171
+ except Exception as e:
172
+ self.handler.handle_error(e, raise_error=False, log_error=False)
173
+
174
+ # 检查历史记录
175
+ history = self.handler.get_error_history()
176
+ self.assertEqual(len(history), 2)
177
+
178
+ # 清空历史记录
179
+ self.handler.clear_error_history()
180
+ history = self.handler.get_error_history()
181
+ self.assertEqual(len(history), 0)
182
+
183
+ def test_error_history_size_limit(self):
184
+ """测试错误历史大小限制"""
185
+ # 产生超过100个错误
186
+ for i in range(110):
187
+ try:
188
+ raise ValueError(f"错误{i}")
189
+ except Exception as e:
190
+ self.handler.handle_error(e, raise_error=False, log_error=False)
191
+
192
+ # 检查历史记录大小限制
193
+ history = self.handler.get_error_history()
194
+ self.assertEqual(len(history), 100) # 应该限制在100个
195
+
196
+
197
+ class TestHandleExceptionDecorator(unittest.TestCase):
198
+ """异常处理装饰器测试"""
199
+
200
+ def test_handle_exception_decorator_sync(self):
201
+ """测试同步函数的异常处理装饰器"""
202
+ @handle_exception(context="测试装饰器", module="test_module", raise_error=False)
203
+ def failing_function():
204
+ raise ValueError("装饰器测试错误")
205
+
206
+ # 调用应该捕获异常但不抛出
207
+ try:
208
+ result = failing_function()
209
+ self.assertIsNone(result) # 默认返回None
210
+ except Exception:
211
+ self.fail("异常应该被捕获")
212
+
213
+ def test_handle_exception_decorator_async(self):
214
+ """测试异步函数的异常处理装饰器"""
215
+ @handle_exception(context="异步测试装饰器", module="test_module", raise_error=False)
216
+ async def async_failing_function():
217
+ raise RuntimeError("异步装饰器测试错误")
218
+
219
+ # 异步调用
220
+ async def test_async():
221
+ try:
222
+ result = await async_failing_function()
223
+ self.assertIsNone(result) # 默认返回None
224
+ except Exception:
225
+ self.fail("异常应该被捕获")
226
+
227
+ # 使用事件循环运行异步函数
228
+ asyncio.run(test_async())
229
+
230
+ def test_handle_exception_decorator_with_detailed_exception(self):
231
+ """测试装饰器处理详细异常"""
232
+ @handle_exception(context="详细异常测试", module="test_module", raise_error=False)
233
+ def function_with_detailed_exception():
234
+ context = ErrorContext(context="函数内部上下文")
235
+ raise DetailedException("详细异常消息", context=context, error_code="DETAIL_001")
236
+
237
+ # 调用应该捕获异常
238
+ try:
239
+ result = function_with_detailed_exception()
240
+ self.assertIsNone(result)
241
+ except Exception:
242
+ self.fail("详细异常应该被捕获")
243
+
244
+
245
+ if __name__ == '__main__':
246
246
  unittest.main()