crawlo 1.4.5__py3-none-any.whl → 1.4.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of crawlo might be problematic. Click here for more details.
- crawlo/__init__.py +90 -89
- crawlo/__version__.py +1 -1
- crawlo/cli.py +75 -75
- crawlo/commands/__init__.py +14 -14
- crawlo/commands/check.py +594 -594
- crawlo/commands/genspider.py +186 -186
- crawlo/commands/help.py +140 -138
- crawlo/commands/list.py +155 -155
- crawlo/commands/run.py +379 -341
- crawlo/commands/startproject.py +460 -460
- crawlo/commands/stats.py +187 -187
- crawlo/commands/utils.py +196 -196
- crawlo/config.py +320 -312
- crawlo/config_validator.py +277 -277
- crawlo/core/__init__.py +52 -52
- crawlo/core/engine.py +451 -438
- crawlo/core/processor.py +47 -47
- crawlo/core/scheduler.py +290 -291
- crawlo/crawler.py +698 -657
- crawlo/data/__init__.py +5 -5
- crawlo/data/user_agents.py +194 -194
- crawlo/downloader/__init__.py +280 -276
- crawlo/downloader/aiohttp_downloader.py +233 -233
- crawlo/downloader/cffi_downloader.py +250 -245
- crawlo/downloader/httpx_downloader.py +265 -259
- crawlo/downloader/hybrid_downloader.py +212 -212
- crawlo/downloader/playwright_downloader.py +425 -402
- crawlo/downloader/selenium_downloader.py +486 -472
- crawlo/event.py +45 -11
- crawlo/exceptions.py +215 -82
- crawlo/extension/__init__.py +65 -64
- crawlo/extension/health_check.py +141 -141
- crawlo/extension/log_interval.py +94 -94
- crawlo/extension/log_stats.py +70 -70
- crawlo/extension/logging_extension.py +53 -61
- crawlo/extension/memory_monitor.py +104 -104
- crawlo/extension/performance_profiler.py +133 -133
- crawlo/extension/request_recorder.py +107 -107
- crawlo/factories/__init__.py +27 -27
- crawlo/factories/base.py +68 -68
- crawlo/factories/crawler.py +104 -103
- crawlo/factories/registry.py +84 -84
- crawlo/factories/utils.py +135 -0
- crawlo/filters/__init__.py +170 -153
- crawlo/filters/aioredis_filter.py +348 -264
- crawlo/filters/memory_filter.py +261 -276
- crawlo/framework.py +306 -292
- crawlo/initialization/__init__.py +44 -44
- crawlo/initialization/built_in.py +391 -434
- crawlo/initialization/context.py +141 -141
- crawlo/initialization/core.py +240 -194
- crawlo/initialization/phases.py +230 -149
- crawlo/initialization/registry.py +143 -145
- crawlo/initialization/utils.py +49 -0
- crawlo/interfaces.py +23 -23
- crawlo/items/__init__.py +23 -23
- crawlo/items/base.py +23 -23
- crawlo/items/fields.py +52 -52
- crawlo/items/items.py +104 -104
- crawlo/logging/__init__.py +42 -46
- crawlo/logging/config.py +277 -197
- crawlo/logging/factory.py +175 -171
- crawlo/logging/manager.py +104 -112
- crawlo/middleware/__init__.py +87 -24
- crawlo/middleware/default_header.py +132 -132
- crawlo/middleware/download_delay.py +104 -104
- crawlo/middleware/middleware_manager.py +142 -142
- crawlo/middleware/offsite.py +123 -123
- crawlo/middleware/proxy.py +209 -386
- crawlo/middleware/request_ignore.py +86 -86
- crawlo/middleware/response_code.py +150 -150
- crawlo/middleware/response_filter.py +136 -136
- crawlo/middleware/retry.py +124 -124
- crawlo/mode_manager.py +287 -253
- crawlo/network/__init__.py +21 -21
- crawlo/network/request.py +375 -379
- crawlo/network/response.py +569 -664
- crawlo/pipelines/__init__.py +53 -22
- crawlo/pipelines/base_pipeline.py +452 -0
- crawlo/pipelines/bloom_dedup_pipeline.py +146 -146
- crawlo/pipelines/console_pipeline.py +39 -39
- crawlo/pipelines/csv_pipeline.py +316 -316
- crawlo/pipelines/database_dedup_pipeline.py +197 -197
- crawlo/pipelines/json_pipeline.py +218 -218
- crawlo/pipelines/memory_dedup_pipeline.py +105 -105
- crawlo/pipelines/mongo_pipeline.py +140 -132
- crawlo/pipelines/mysql_pipeline.py +470 -326
- crawlo/pipelines/pipeline_manager.py +100 -100
- crawlo/pipelines/redis_dedup_pipeline.py +155 -156
- crawlo/project.py +347 -347
- crawlo/queue/__init__.py +10 -0
- crawlo/queue/pqueue.py +38 -38
- crawlo/queue/queue_manager.py +591 -525
- crawlo/queue/redis_priority_queue.py +519 -370
- crawlo/settings/__init__.py +7 -7
- crawlo/settings/default_settings.py +285 -270
- crawlo/settings/setting_manager.py +219 -219
- crawlo/spider/__init__.py +657 -657
- crawlo/stats_collector.py +82 -73
- crawlo/subscriber.py +129 -129
- crawlo/task_manager.py +138 -138
- crawlo/templates/crawlo.cfg.tmpl +10 -10
- crawlo/templates/project/__init__.py.tmpl +2 -4
- crawlo/templates/project/items.py.tmpl +13 -17
- crawlo/templates/project/middlewares.py.tmpl +38 -38
- crawlo/templates/project/pipelines.py.tmpl +35 -36
- crawlo/templates/project/settings.py.tmpl +110 -157
- crawlo/templates/project/settings_distributed.py.tmpl +156 -161
- crawlo/templates/project/settings_gentle.py.tmpl +170 -171
- crawlo/templates/project/settings_high_performance.py.tmpl +171 -172
- crawlo/templates/project/settings_minimal.py.tmpl +99 -77
- crawlo/templates/project/settings_simple.py.tmpl +168 -169
- crawlo/templates/project/spiders/__init__.py.tmpl +9 -9
- crawlo/templates/run.py.tmpl +23 -30
- crawlo/templates/spider/spider.py.tmpl +33 -144
- crawlo/templates/spiders_init.py.tmpl +5 -10
- crawlo/tools/__init__.py +86 -189
- crawlo/tools/date_tools.py +289 -289
- crawlo/tools/distributed_coordinator.py +384 -384
- crawlo/tools/scenario_adapter.py +262 -262
- crawlo/tools/text_cleaner.py +232 -232
- crawlo/utils/__init__.py +50 -50
- crawlo/utils/batch_processor.py +276 -259
- crawlo/utils/config_manager.py +442 -0
- crawlo/utils/controlled_spider_mixin.py +439 -439
- crawlo/utils/db_helper.py +250 -244
- crawlo/utils/error_handler.py +410 -410
- crawlo/utils/fingerprint.py +121 -121
- crawlo/utils/func_tools.py +82 -82
- crawlo/utils/large_scale_helper.py +344 -344
- crawlo/utils/leak_detector.py +335 -0
- crawlo/utils/log.py +79 -79
- crawlo/utils/misc.py +81 -81
- crawlo/utils/mongo_connection_pool.py +157 -0
- crawlo/utils/mysql_connection_pool.py +197 -0
- crawlo/utils/performance_monitor.py +285 -285
- crawlo/utils/queue_helper.py +175 -175
- crawlo/utils/redis_checker.py +91 -0
- crawlo/utils/redis_connection_pool.py +578 -388
- crawlo/utils/redis_key_validator.py +198 -198
- crawlo/utils/request.py +278 -256
- crawlo/utils/request_serializer.py +225 -225
- crawlo/utils/resource_manager.py +337 -0
- crawlo/utils/selector_helper.py +137 -137
- crawlo/utils/singleton.py +70 -0
- crawlo/utils/spider_loader.py +201 -201
- crawlo/utils/text_helper.py +94 -94
- crawlo/utils/{url.py → url_utils.py} +39 -39
- crawlo-1.4.7.dist-info/METADATA +689 -0
- crawlo-1.4.7.dist-info/RECORD +347 -0
- examples/__init__.py +7 -7
- tests/__init__.py +7 -7
- tests/advanced_tools_example.py +217 -275
- tests/authenticated_proxy_example.py +110 -106
- tests/baidu_performance_test.py +108 -108
- tests/baidu_test.py +59 -59
- tests/bug_check_test.py +250 -250
- tests/cleaners_example.py +160 -160
- tests/comprehensive_framework_test.py +212 -212
- tests/comprehensive_test.py +81 -81
- tests/comprehensive_testing_summary.md +186 -186
- tests/config_validation_demo.py +142 -142
- tests/controlled_spider_example.py +205 -205
- tests/date_tools_example.py +180 -180
- tests/debug_configure.py +69 -69
- tests/debug_framework_logger.py +84 -84
- tests/debug_log_config.py +126 -126
- tests/debug_log_levels.py +63 -63
- tests/debug_pipelines.py +66 -66
- tests/detailed_log_test.py +233 -233
- tests/direct_selector_helper_test.py +96 -96
- tests/distributed_dedup_test.py +467 -0
- tests/distributed_test.py +66 -66
- tests/distributed_test_debug.py +76 -76
- tests/dynamic_loading_example.py +523 -523
- tests/dynamic_loading_test.py +104 -104
- tests/error_handling_example.py +171 -171
- tests/explain_mysql_update_behavior.py +77 -0
- tests/final_comprehensive_test.py +151 -151
- tests/final_log_test.py +260 -260
- tests/final_validation_test.py +182 -182
- tests/fix_log_test.py +142 -142
- tests/framework_performance_test.py +202 -202
- tests/log_buffering_test.py +111 -111
- tests/log_generation_timing_test.py +153 -153
- tests/monitor_redis_dedup.sh +72 -0
- tests/ofweek_scrapy/ofweek_scrapy/items.py +12 -12
- tests/ofweek_scrapy/ofweek_scrapy/middlewares.py +100 -100
- tests/ofweek_scrapy/ofweek_scrapy/pipelines.py +13 -13
- tests/ofweek_scrapy/ofweek_scrapy/settings.py +84 -84
- tests/ofweek_scrapy/scrapy.cfg +11 -11
- tests/optimized_performance_test.py +211 -211
- tests/performance_comparison.py +244 -244
- tests/queue_blocking_test.py +113 -113
- tests/queue_test.py +89 -89
- tests/redis_key_validation_demo.py +130 -130
- tests/request_params_example.py +150 -150
- tests/response_improvements_example.py +144 -144
- tests/scrapy_comparison/ofweek_scrapy.py +138 -138
- tests/scrapy_comparison/scrapy_test.py +133 -133
- tests/simple_cli_test.py +55 -0
- tests/simple_command_test.py +119 -119
- tests/simple_crawlo_test.py +126 -126
- tests/simple_follow_test.py +38 -38
- tests/simple_log_test2.py +137 -137
- tests/simple_optimization_test.py +128 -128
- tests/simple_queue_type_test.py +41 -41
- tests/simple_response_selector_test.py +94 -94
- tests/simple_selector_helper_test.py +154 -154
- tests/simple_selector_test.py +207 -207
- tests/simple_spider_test.py +49 -49
- tests/simple_url_test.py +73 -73
- tests/simulate_mysql_update_test.py +140 -0
- tests/spider_log_timing_test.py +177 -177
- tests/test_advanced_tools.py +148 -148
- tests/test_all_commands.py +230 -230
- tests/test_all_pipeline_fingerprints.py +133 -133
- tests/test_all_redis_key_configs.py +145 -145
- tests/test_asyncmy_usage.py +57 -0
- tests/test_batch_processor.py +178 -178
- tests/test_cleaners.py +54 -54
- tests/test_cli_arguments.py +119 -0
- tests/test_component_factory.py +174 -174
- tests/test_config_consistency.py +80 -80
- tests/test_config_merge.py +152 -152
- tests/test_config_validator.py +182 -182
- tests/test_controlled_spider_mixin.py +79 -79
- tests/test_crawler_process_import.py +38 -38
- tests/test_crawler_process_spider_modules.py +47 -47
- tests/test_crawlo_proxy_integration.py +114 -108
- tests/test_date_tools.py +123 -123
- tests/test_dedup_fix.py +220 -220
- tests/test_dedup_pipeline_consistency.py +124 -124
- tests/test_default_header_middleware.py +313 -313
- tests/test_distributed.py +65 -65
- tests/test_double_crawlo_fix.py +204 -204
- tests/test_double_crawlo_fix_simple.py +124 -124
- tests/test_download_delay_middleware.py +221 -221
- tests/test_downloader_proxy_compatibility.py +272 -268
- tests/test_edge_cases.py +305 -305
- tests/test_encoding_core.py +56 -56
- tests/test_encoding_detection.py +126 -126
- tests/test_enhanced_error_handler.py +270 -270
- tests/test_enhanced_error_handler_comprehensive.py +245 -245
- tests/test_error_handler_compatibility.py +112 -112
- tests/test_factories.py +252 -252
- tests/test_factory_compatibility.py +196 -196
- tests/test_final_validation.py +153 -153
- tests/test_fingerprint_consistency.py +135 -135
- tests/test_fingerprint_simple.py +51 -51
- tests/test_get_component_logger.py +83 -83
- tests/test_hash_performance.py +99 -99
- tests/test_integration.py +169 -169
- tests/test_item_dedup_redis_key.py +122 -122
- tests/test_large_scale_helper.py +235 -235
- tests/test_logging_enhancements.py +374 -374
- tests/test_logging_final.py +184 -184
- tests/test_logging_integration.py +312 -312
- tests/test_logging_system.py +282 -282
- tests/test_middleware_debug.py +141 -141
- tests/test_mode_consistency.py +51 -51
- tests/test_multi_directory.py +67 -67
- tests/test_multiple_spider_modules.py +80 -80
- tests/test_mysql_pipeline_config.py +165 -0
- tests/test_mysql_pipeline_error.py +99 -0
- tests/test_mysql_pipeline_init_log.py +83 -0
- tests/test_mysql_pipeline_integration.py +133 -0
- tests/test_mysql_pipeline_refactor.py +144 -0
- tests/test_mysql_pipeline_refactor_simple.py +86 -0
- tests/test_mysql_pipeline_robustness.py +196 -0
- tests/test_mysql_pipeline_types.py +89 -0
- tests/test_mysql_update_columns.py +94 -0
- tests/test_offsite_middleware.py +244 -244
- tests/test_offsite_middleware_simple.py +203 -203
- tests/test_optimized_selector_naming.py +100 -100
- tests/test_parsel.py +29 -29
- tests/test_performance.py +327 -327
- tests/test_performance_monitor.py +115 -115
- tests/test_pipeline_fingerprint_consistency.py +86 -86
- tests/test_priority_behavior.py +211 -211
- tests/test_priority_consistency.py +151 -151
- tests/test_priority_consistency_fixed.py +249 -249
- tests/test_proxy_health_check.py +32 -32
- tests/test_proxy_middleware.py +217 -121
- tests/test_proxy_middleware_enhanced.py +212 -216
- tests/test_proxy_middleware_integration.py +142 -137
- tests/test_proxy_middleware_refactored.py +207 -184
- tests/test_proxy_only.py +84 -0
- tests/test_proxy_providers.py +56 -56
- tests/test_proxy_stats.py +19 -19
- tests/test_proxy_strategies.py +59 -59
- tests/test_proxy_with_downloader.py +153 -0
- tests/test_queue_empty_check.py +41 -41
- tests/test_queue_manager_double_crawlo.py +173 -173
- tests/test_queue_manager_redis_key.py +179 -179
- tests/test_queue_naming.py +154 -154
- tests/test_queue_type.py +106 -106
- tests/test_queue_type_redis_config_consistency.py +130 -130
- tests/test_random_headers_default.py +322 -322
- tests/test_random_headers_necessity.py +308 -308
- tests/test_random_user_agent.py +72 -72
- tests/test_redis_config.py +28 -28
- tests/test_redis_connection_pool.py +294 -294
- tests/test_redis_key_naming.py +181 -181
- tests/test_redis_key_validator.py +123 -123
- tests/test_redis_queue.py +224 -224
- tests/test_redis_queue_name_fix.py +175 -175
- tests/test_redis_queue_type_fallback.py +129 -129
- tests/test_request_ignore_middleware.py +182 -182
- tests/test_request_params.py +111 -111
- tests/test_request_serialization.py +70 -70
- tests/test_response_code_middleware.py +349 -349
- tests/test_response_filter_middleware.py +427 -427
- tests/test_response_follow.py +104 -104
- tests/test_response_improvements.py +152 -152
- tests/test_response_selector_methods.py +92 -92
- tests/test_response_url_methods.py +70 -70
- tests/test_response_urljoin.py +86 -86
- tests/test_retry_middleware.py +333 -333
- tests/test_retry_middleware_realistic.py +273 -273
- tests/test_scheduler.py +252 -252
- tests/test_scheduler_config_update.py +133 -133
- tests/test_scrapy_style_encoding.py +112 -112
- tests/test_selector_helper.py +100 -100
- tests/test_selector_optimizations.py +146 -146
- tests/test_simple_response.py +61 -61
- tests/test_spider_loader.py +49 -49
- tests/test_spider_loader_comprehensive.py +69 -69
- tests/test_spider_modules.py +84 -84
- tests/test_spiders/test_spider.py +9 -9
- tests/test_telecom_spider_redis_key.py +205 -205
- tests/test_template_content.py +87 -87
- tests/test_template_redis_key.py +134 -134
- tests/test_tools.py +159 -159
- tests/test_user_agent_randomness.py +176 -176
- tests/test_user_agents.py +96 -96
- tests/untested_features_report.md +138 -138
- tests/verify_debug.py +51 -51
- tests/verify_distributed.py +117 -117
- tests/verify_log_fix.py +111 -111
- tests/verify_mysql_warnings.py +110 -0
- crawlo/logging/async_handler.py +0 -181
- crawlo/logging/monitor.py +0 -153
- crawlo/logging/sampler.py +0 -167
- crawlo/middleware/simple_proxy.py +0 -65
- crawlo/tools/authenticated_proxy.py +0 -241
- crawlo/tools/data_formatter.py +0 -226
- crawlo/tools/data_validator.py +0 -181
- crawlo/tools/encoding_converter.py +0 -127
- crawlo/tools/network_diagnostic.py +0 -365
- crawlo/tools/request_tools.py +0 -83
- crawlo/tools/retry_mechanism.py +0 -224
- crawlo/utils/env_config.py +0 -143
- crawlo/utils/large_scale_config.py +0 -287
- crawlo/utils/system.py +0 -11
- crawlo/utils/tools.py +0 -5
- crawlo-1.4.5.dist-info/METADATA +0 -329
- crawlo-1.4.5.dist-info/RECORD +0 -347
- tests/env_config_example.py +0 -134
- tests/ofweek_scrapy/ofweek_scrapy/spiders/ofweek_spider.py +0 -162
- tests/test_authenticated_proxy.py +0 -142
- tests/test_comprehensive.py +0 -147
- tests/test_dynamic_downloaders_proxy.py +0 -125
- tests/test_dynamic_proxy.py +0 -93
- tests/test_dynamic_proxy_config.py +0 -147
- tests/test_dynamic_proxy_real.py +0 -110
- tests/test_env_config.py +0 -122
- tests/test_framework_env_usage.py +0 -104
- tests/test_large_scale_config.py +0 -113
- tests/test_proxy_api.py +0 -265
- tests/test_real_scenario_proxy.py +0 -196
- tests/tools_example.py +0 -261
- {crawlo-1.4.5.dist-info → crawlo-1.4.7.dist-info}/WHEEL +0 -0
- {crawlo-1.4.5.dist-info → crawlo-1.4.7.dist-info}/entry_points.txt +0 -0
- {crawlo-1.4.5.dist-info → crawlo-1.4.7.dist-info}/top_level.txt +0 -0
|
@@ -1,435 +1,392 @@
|
|
|
1
|
-
#!/usr/bin/python
|
|
2
|
-
# -*- coding: UTF-8 -*-
|
|
3
|
-
"""
|
|
4
|
-
内置初始化器 - 提供框架核心组件的初始化实现
|
|
5
|
-
"""
|
|
6
|
-
|
|
7
|
-
import time
|
|
8
|
-
from
|
|
9
|
-
|
|
10
|
-
from .
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
context.add_shared_data('
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
import
|
|
148
|
-
import
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
config_parser.
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
parent
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
from crawlo.
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
context.settings
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
#
|
|
255
|
-
#
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
def
|
|
281
|
-
"""
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
except Exception as e:
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
# 获取框架版本
|
|
393
|
-
version = self._get_framework_version()
|
|
394
|
-
|
|
395
|
-
# 记录框架启动信息(符合规范要求)
|
|
396
|
-
logger.info(f"Crawlo Framework Started {version}")
|
|
397
|
-
|
|
398
|
-
# 获取运行模式
|
|
399
|
-
run_mode = "unknown"
|
|
400
|
-
if context.settings:
|
|
401
|
-
run_mode = context.settings.get('RUN_MODE', 'standalone')
|
|
402
|
-
logger.info(f"Run mode: {run_mode}")
|
|
403
|
-
|
|
404
|
-
# 注意:爬虫名称信息将在实际启动爬虫时记录,而不是在框架初始化时
|
|
405
|
-
|
|
406
|
-
return self._create_result(
|
|
407
|
-
success=True,
|
|
408
|
-
duration=time.time() - start_time,
|
|
409
|
-
artifacts={}
|
|
410
|
-
)
|
|
411
|
-
|
|
412
|
-
except Exception as e:
|
|
413
|
-
# 即使日志记录失败,也不应该影响框架初始化
|
|
414
|
-
return self._create_result(
|
|
415
|
-
success=True, # 不影响初始化成功与否
|
|
416
|
-
duration=time.time() - start_time,
|
|
417
|
-
error=e
|
|
418
|
-
)
|
|
419
|
-
|
|
420
|
-
def _get_framework_version(self):
|
|
421
|
-
"""获取框架版本"""
|
|
422
|
-
try:
|
|
423
|
-
from crawlo import __version__
|
|
424
|
-
return __version__
|
|
425
|
-
except Exception:
|
|
426
|
-
return "unknown"
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
def register_built_in_initializers():
|
|
430
|
-
"""注册所有内置初始化器"""
|
|
431
|
-
register_initializer(LoggingInitializer())
|
|
432
|
-
register_initializer(SettingsInitializer())
|
|
433
|
-
register_initializer(CoreComponentsInitializer())
|
|
434
|
-
register_initializer(ExtensionsInitializer())
|
|
1
|
+
#!/usr/bin/python
|
|
2
|
+
# -*- coding: UTF-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
内置初始化器 - 提供框架核心组件的初始化实现
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import time
|
|
8
|
+
from typing import TYPE_CHECKING
|
|
9
|
+
|
|
10
|
+
from .registry import BaseInitializer, register_initializer
|
|
11
|
+
from .phases import InitializationPhase, PhaseResult
|
|
12
|
+
from .context import InitializationContext
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from crawlo.logging import LogConfig
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class LoggingInitializer(BaseInitializer):
|
|
19
|
+
"""日志系统初始化器"""
|
|
20
|
+
|
|
21
|
+
def __init__(self):
|
|
22
|
+
super().__init__(InitializationPhase.LOGGING)
|
|
23
|
+
|
|
24
|
+
def initialize(self, context: InitializationContext) -> PhaseResult:
|
|
25
|
+
"""初始化日志系统"""
|
|
26
|
+
start_time = time.time()
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
# 导入日志模块
|
|
30
|
+
from crawlo.logging import configure_logging, LogConfig
|
|
31
|
+
|
|
32
|
+
# 获取日志配置
|
|
33
|
+
log_config = self._get_log_config(context)
|
|
34
|
+
|
|
35
|
+
# 确保日志目录存在
|
|
36
|
+
if log_config and log_config.file_path and log_config.file_enabled:
|
|
37
|
+
import os
|
|
38
|
+
log_dir = os.path.dirname(log_config.file_path)
|
|
39
|
+
if log_dir and not os.path.exists(log_dir):
|
|
40
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
41
|
+
|
|
42
|
+
# 配置日志系统
|
|
43
|
+
configure_logging(log_config)
|
|
44
|
+
|
|
45
|
+
# 存储到共享数据
|
|
46
|
+
context.add_shared_data('log_config', log_config)
|
|
47
|
+
|
|
48
|
+
# 创建框架logger
|
|
49
|
+
from crawlo.logging import get_logger
|
|
50
|
+
framework_logger = get_logger('crawlo.framework')
|
|
51
|
+
context.add_shared_data('framework_logger', framework_logger)
|
|
52
|
+
|
|
53
|
+
return self._create_result(
|
|
54
|
+
success=True,
|
|
55
|
+
duration=time.time() - start_time,
|
|
56
|
+
artifacts={'log_config': log_config}
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
except Exception as e:
|
|
60
|
+
return self._create_result(
|
|
61
|
+
success=False,
|
|
62
|
+
duration=time.time() - start_time,
|
|
63
|
+
error=e
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
def _get_log_config(self, context: InitializationContext) -> 'LogConfig | None':
|
|
67
|
+
"""
|
|
68
|
+
获取日志配置
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
context: 初始化上下文
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
LogConfig: 日志配置对象
|
|
75
|
+
"""
|
|
76
|
+
# 导入日志配置类
|
|
77
|
+
from crawlo.logging import LogConfig
|
|
78
|
+
from crawlo.utils.config_manager import ConfigUtils
|
|
79
|
+
|
|
80
|
+
# 按优先级获取配置:自定义配置 > 上下文配置 > 项目配置 > 默认配置
|
|
81
|
+
config_sources = [
|
|
82
|
+
context.custom_settings,
|
|
83
|
+
context.settings,
|
|
84
|
+
self._load_project_config()
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
# 遍历配置源
|
|
88
|
+
for config_source in config_sources:
|
|
89
|
+
if config_source and ConfigUtils.has_config_prefix(config_source, 'LOG_'):
|
|
90
|
+
log_config = self._create_log_config_from_source(config_source)
|
|
91
|
+
if log_config:
|
|
92
|
+
return log_config
|
|
93
|
+
|
|
94
|
+
# 使用默认配置
|
|
95
|
+
return LogConfig()
|
|
96
|
+
|
|
97
|
+
def _create_log_config_from_source(self, config_source) -> 'LogConfig | None':
|
|
98
|
+
"""
|
|
99
|
+
从配置源创建日志配置
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
config_source: 配置源
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
LogConfig: 日志配置对象,如果配置源无效则返回None
|
|
106
|
+
"""
|
|
107
|
+
# 导入日志配置类
|
|
108
|
+
from crawlo.logging import LogConfig
|
|
109
|
+
from crawlo.utils.config_manager import ConfigUtils
|
|
110
|
+
|
|
111
|
+
# 检查配置源是否有效
|
|
112
|
+
if not config_source:
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
# 检查是否有日志相关配置
|
|
116
|
+
if not ConfigUtils.has_config_prefix(config_source, 'LOG_'):
|
|
117
|
+
return None
|
|
118
|
+
|
|
119
|
+
# 从配置源获取日志配置
|
|
120
|
+
log_level = ConfigUtils.get_config_value([config_source], 'LOG_LEVEL', 'INFO')
|
|
121
|
+
log_file = ConfigUtils.get_config_value([config_source], 'LOG_FILE')
|
|
122
|
+
log_format = ConfigUtils.get_config_value([config_source], 'LOG_FORMAT', '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s')
|
|
123
|
+
log_encoding = ConfigUtils.get_config_value([config_source], 'LOG_ENCODING', 'utf-8')
|
|
124
|
+
log_max_bytes = ConfigUtils.get_config_value([config_source], 'LOG_MAX_BYTES', 10 * 1024 * 1024, int)
|
|
125
|
+
log_backup_count = ConfigUtils.get_config_value([config_source], 'LOG_BACKUP_COUNT', 5, int)
|
|
126
|
+
log_console_enabled = ConfigUtils.get_config_value([config_source], 'LOG_CONSOLE_ENABLED', True, bool)
|
|
127
|
+
log_file_enabled = ConfigUtils.get_config_value([config_source], 'LOG_FILE_ENABLED', True, bool)
|
|
128
|
+
|
|
129
|
+
# 创建日志配置
|
|
130
|
+
return LogConfig(
|
|
131
|
+
level=log_level,
|
|
132
|
+
format=log_format,
|
|
133
|
+
encoding=log_encoding,
|
|
134
|
+
file_path=log_file,
|
|
135
|
+
max_bytes=log_max_bytes,
|
|
136
|
+
backup_count=log_backup_count,
|
|
137
|
+
console_enabled=log_console_enabled,
|
|
138
|
+
file_enabled=log_file_enabled
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
def _load_project_config(self):
|
|
142
|
+
"""
|
|
143
|
+
自动加载项目配置以获取日志设置
|
|
144
|
+
"""
|
|
145
|
+
try:
|
|
146
|
+
# 查找项目根目录
|
|
147
|
+
import os
|
|
148
|
+
import sys
|
|
149
|
+
import configparser
|
|
150
|
+
|
|
151
|
+
current_path = os.getcwd()
|
|
152
|
+
|
|
153
|
+
# 向上查找直到找到crawlo.cfg
|
|
154
|
+
checked_paths = set()
|
|
155
|
+
path = current_path
|
|
156
|
+
|
|
157
|
+
while path not in checked_paths:
|
|
158
|
+
checked_paths.add(path)
|
|
159
|
+
|
|
160
|
+
# 检查crawlo.cfg
|
|
161
|
+
cfg_file = os.path.join(path, "crawlo.cfg")
|
|
162
|
+
if os.path.exists(cfg_file):
|
|
163
|
+
# 读取配置文件
|
|
164
|
+
config_parser = configparser.ConfigParser()
|
|
165
|
+
config_parser.read(cfg_file, encoding="utf-8")
|
|
166
|
+
|
|
167
|
+
if config_parser.has_section("settings") and config_parser.has_option("settings", "default"):
|
|
168
|
+
# 获取settings模块路径
|
|
169
|
+
settings_module_path = config_parser.get("settings", "default")
|
|
170
|
+
|
|
171
|
+
# 添加项目根目录到Python路径
|
|
172
|
+
if path not in sys.path:
|
|
173
|
+
sys.path.insert(0, path)
|
|
174
|
+
|
|
175
|
+
# 导入项目配置模块
|
|
176
|
+
import importlib
|
|
177
|
+
settings_module = importlib.import_module(settings_module_path)
|
|
178
|
+
|
|
179
|
+
# 创建配置字典
|
|
180
|
+
from crawlo.utils.config_manager import ConfigUtils
|
|
181
|
+
project_config = ConfigUtils.merge_config_sources([settings_module])
|
|
182
|
+
|
|
183
|
+
return project_config
|
|
184
|
+
|
|
185
|
+
# 向上一级目录
|
|
186
|
+
parent = os.path.dirname(path)
|
|
187
|
+
if parent == path:
|
|
188
|
+
break
|
|
189
|
+
path = parent
|
|
190
|
+
|
|
191
|
+
return {}
|
|
192
|
+
|
|
193
|
+
except Exception as e:
|
|
194
|
+
return {}
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class SettingsInitializer(BaseInitializer):
|
|
198
|
+
"""配置系统初始化器"""
|
|
199
|
+
|
|
200
|
+
def __init__(self):
|
|
201
|
+
super().__init__(InitializationPhase.SETTINGS)
|
|
202
|
+
|
|
203
|
+
def initialize(self, context: InitializationContext) -> PhaseResult:
|
|
204
|
+
"""初始化配置系统"""
|
|
205
|
+
start_time = time.time()
|
|
206
|
+
|
|
207
|
+
try:
|
|
208
|
+
# 导入配置管理器
|
|
209
|
+
from crawlo.settings.setting_manager import SettingManager
|
|
210
|
+
from crawlo.project import _load_project_settings
|
|
211
|
+
|
|
212
|
+
# 如果上下文中已有设置,则使用它作为基础配置
|
|
213
|
+
if context.settings:
|
|
214
|
+
# 使用用户传递的设置作为基础配置
|
|
215
|
+
settings = context.settings
|
|
216
|
+
# 加载项目配置并合并
|
|
217
|
+
project_settings = _load_project_settings(context.custom_settings)
|
|
218
|
+
# 合并配置,用户配置优先
|
|
219
|
+
settings.update_attributes(project_settings.attributes)
|
|
220
|
+
else:
|
|
221
|
+
# 创建配置管理器并加载项目配置
|
|
222
|
+
settings = _load_project_settings(context.custom_settings)
|
|
223
|
+
|
|
224
|
+
# 存储到上下文
|
|
225
|
+
context.settings = settings
|
|
226
|
+
context.add_shared_data('settings', settings)
|
|
227
|
+
|
|
228
|
+
return self._create_result(
|
|
229
|
+
success=True,
|
|
230
|
+
duration=time.time() - start_time,
|
|
231
|
+
artifacts={'settings': settings}
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
except Exception as e:
|
|
235
|
+
return self._create_result(
|
|
236
|
+
success=False,
|
|
237
|
+
duration=time.time() - start_time,
|
|
238
|
+
error=e
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
class CoreComponentsInitializer(BaseInitializer):
|
|
243
|
+
"""核心组件初始化器"""
|
|
244
|
+
|
|
245
|
+
def __init__(self):
|
|
246
|
+
super().__init__(InitializationPhase.CORE_COMPONENTS)
|
|
247
|
+
|
|
248
|
+
def initialize(self, context: InitializationContext) -> PhaseResult:
|
|
249
|
+
"""初始化核心组件"""
|
|
250
|
+
start_time = time.time()
|
|
251
|
+
|
|
252
|
+
try:
|
|
253
|
+
# 在核心组件初始化阶段,大多数组件需要crawler参数
|
|
254
|
+
# 因此我们只初始化那些完全独立的组件
|
|
255
|
+
# 或者只记录需要初始化的组件类型,实际初始化在crawler创建时进行
|
|
256
|
+
|
|
257
|
+
return self._create_result(
|
|
258
|
+
success=True,
|
|
259
|
+
duration=time.time() - start_time,
|
|
260
|
+
artifacts={}
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
except Exception as e:
|
|
264
|
+
return self._create_result(
|
|
265
|
+
success=False,
|
|
266
|
+
duration=time.time() - start_time,
|
|
267
|
+
error=e
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
# 注意:核心组件需要crawler参数,不能在此阶段初始化
|
|
271
|
+
# 实际初始化将在crawler创建时进行
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
class ExtensionsInitializer(BaseInitializer):
|
|
275
|
+
"""扩展组件初始化器"""
|
|
276
|
+
|
|
277
|
+
def __init__(self):
|
|
278
|
+
super().__init__(InitializationPhase.EXTENSIONS)
|
|
279
|
+
|
|
280
|
+
def initialize(self, context: InitializationContext) -> PhaseResult:
|
|
281
|
+
"""初始化扩展组件"""
|
|
282
|
+
start_time = time.time()
|
|
283
|
+
|
|
284
|
+
try:
|
|
285
|
+
# 初始化扩展组件
|
|
286
|
+
self._initialize_extensions(context)
|
|
287
|
+
|
|
288
|
+
return self._create_result(
|
|
289
|
+
success=True,
|
|
290
|
+
duration=time.time() - start_time,
|
|
291
|
+
artifacts={}
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
except Exception as e:
|
|
295
|
+
return self._create_result(
|
|
296
|
+
success=False,
|
|
297
|
+
duration=time.time() - start_time,
|
|
298
|
+
error=e
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
def _initialize_extensions(self, context: InitializationContext):
|
|
302
|
+
"""初始化扩展组件"""
|
|
303
|
+
try:
|
|
304
|
+
# 获取扩展配置
|
|
305
|
+
extensions = []
|
|
306
|
+
if context.settings:
|
|
307
|
+
extensions = context.settings.get('EXTENSIONS', [])
|
|
308
|
+
elif context.custom_settings:
|
|
309
|
+
extensions = context.custom_settings.get('EXTENSIONS', [])
|
|
310
|
+
|
|
311
|
+
# 初始化每个扩展
|
|
312
|
+
initialized_extensions = []
|
|
313
|
+
for extension_path in extensions:
|
|
314
|
+
try:
|
|
315
|
+
from crawlo.utils.misc import load_object
|
|
316
|
+
extension_class = load_object(extension_path)
|
|
317
|
+
extension_instance = extension_class()
|
|
318
|
+
initialized_extensions.append(extension_instance)
|
|
319
|
+
except Exception as e:
|
|
320
|
+
if context.settings and context.settings.get('EXTENSIONS_STRICT', False):
|
|
321
|
+
raise
|
|
322
|
+
else:
|
|
323
|
+
# 非严格模式下记录警告但继续
|
|
324
|
+
context.add_warning(f"Failed to initialize extension {extension_path}: {e}")
|
|
325
|
+
|
|
326
|
+
# 存储到上下文
|
|
327
|
+
context.add_shared_data('extensions', initialized_extensions)
|
|
328
|
+
except Exception as e:
|
|
329
|
+
context.add_error(f"Failed to initialize extensions: {e}")
|
|
330
|
+
raise
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
class FrameworkStartupLogger(BaseInitializer):
|
|
334
|
+
"""框架启动日志记录器"""
|
|
335
|
+
|
|
336
|
+
def __init__(self):
|
|
337
|
+
# 使用新的FRAMEWORK_STARTUP_LOG阶段
|
|
338
|
+
super().__init__(InitializationPhase.FRAMEWORK_STARTUP_LOG)
|
|
339
|
+
|
|
340
|
+
def initialize(self, context: InitializationContext) -> PhaseResult:
|
|
341
|
+
"""记录框架启动日志"""
|
|
342
|
+
start_time = time.time()
|
|
343
|
+
|
|
344
|
+
try:
|
|
345
|
+
# 获取框架logger
|
|
346
|
+
from crawlo.logging import get_logger
|
|
347
|
+
logger = get_logger('crawlo.framework')
|
|
348
|
+
|
|
349
|
+
# 获取框架版本
|
|
350
|
+
version = self._get_framework_version()
|
|
351
|
+
|
|
352
|
+
# 记录框架启动信息(符合规范要求)
|
|
353
|
+
logger.info(f"Crawlo Framework Started {version}")
|
|
354
|
+
|
|
355
|
+
# 获取运行模式
|
|
356
|
+
run_mode = "unknown"
|
|
357
|
+
if context.settings:
|
|
358
|
+
run_mode = context.settings.get('RUN_MODE', 'standalone')
|
|
359
|
+
logger.info(f"Run mode: {run_mode}")
|
|
360
|
+
|
|
361
|
+
# 注意:爬虫名称信息将在实际启动爬虫时记录,而不是在框架初始化时
|
|
362
|
+
|
|
363
|
+
return self._create_result(
|
|
364
|
+
success=True,
|
|
365
|
+
duration=time.time() - start_time,
|
|
366
|
+
artifacts={}
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
except Exception as e:
|
|
370
|
+
# 即使日志记录失败,也不应该影响框架初始化
|
|
371
|
+
return self._create_result(
|
|
372
|
+
success=True, # 不影响初始化成功与否
|
|
373
|
+
duration=time.time() - start_time,
|
|
374
|
+
error=e
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
def _get_framework_version(self):
|
|
378
|
+
"""获取框架版本"""
|
|
379
|
+
try:
|
|
380
|
+
from crawlo import __version__
|
|
381
|
+
return __version__
|
|
382
|
+
except Exception:
|
|
383
|
+
return "unknown"
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def register_built_in_initializers():
|
|
387
|
+
"""注册所有内置初始化器"""
|
|
388
|
+
register_initializer(LoggingInitializer())
|
|
389
|
+
register_initializer(SettingsInitializer())
|
|
390
|
+
register_initializer(CoreComponentsInitializer())
|
|
391
|
+
register_initializer(ExtensionsInitializer())
|
|
435
392
|
register_initializer(FrameworkStartupLogger()) # 添加框架启动日志记录器
|