crawlo 1.3.4__py3-none-any.whl → 1.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of crawlo might be problematic. Click here for more details.
- crawlo/__version__.py +1 -1
- crawlo/pipelines/mysql_pipeline.py +7 -0
- crawlo/templates/run.py.tmpl +2 -2
- crawlo/utils/log.py +42 -6
- {crawlo-1.3.4.dist-info → crawlo-1.3.5.dist-info}/METADATA +1 -1
- {crawlo-1.3.4.dist-info → crawlo-1.3.5.dist-info}/RECORD +19 -9
- tests/debug_log_config.py +127 -0
- tests/detailed_log_test.py +234 -0
- tests/final_log_test.py +261 -0
- tests/fix_log_test.py +143 -0
- tests/log_buffering_test.py +112 -0
- tests/log_generation_timing_test.py +154 -0
- tests/simple_log_test2.py +138 -0
- tests/spider_log_timing_test.py +178 -0
- tests/test_get_component_logger.py +84 -0
- tests/test_logging_system.py +283 -0
- {crawlo-1.3.4.dist-info → crawlo-1.3.5.dist-info}/WHEEL +0 -0
- {crawlo-1.3.4.dist-info → crawlo-1.3.5.dist-info}/entry_points.txt +0 -0
- {crawlo-1.3.4.dist-info → crawlo-1.3.5.dist-info}/top_level.txt +0 -0
crawlo/__version__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = '1.3.
|
|
1
|
+
__version__ = '1.3.5'
|
|
@@ -20,7 +20,14 @@ class AsyncmyMySQLPipeline:
|
|
|
20
20
|
self._pool_lock = asyncio.Lock()
|
|
21
21
|
self._pool_initialized = False
|
|
22
22
|
self.pool = None
|
|
23
|
+
|
|
24
|
+
# 优先从爬虫的custom_settings中获取表名,如果没有则使用默认值
|
|
25
|
+
spider_table_name = None
|
|
26
|
+
if hasattr(crawler, 'spider') and crawler.spider and hasattr(crawler.spider, 'custom_settings'):
|
|
27
|
+
spider_table_name = crawler.spider.custom_settings.get('MYSQL_TABLE')
|
|
28
|
+
|
|
23
29
|
self.table_name = (
|
|
30
|
+
spider_table_name or
|
|
24
31
|
self.settings.get('MYSQL_TABLE') or
|
|
25
32
|
getattr(crawler.spider, 'mysql_table', None) or
|
|
26
33
|
f"{crawler.spider.name}_items"
|
crawlo/templates/run.py.tmpl
CHANGED
|
@@ -21,8 +21,8 @@ def main():
|
|
|
21
21
|
spider_modules = ['{{project_name}}.spiders']
|
|
22
22
|
process = CrawlerProcess(spider_modules=spider_modules)
|
|
23
23
|
|
|
24
|
-
# 运行指定的爬虫
|
|
25
|
-
asyncio.run(process.crawl('
|
|
24
|
+
# TODO 运行指定的爬虫
|
|
25
|
+
asyncio.run(process.crawl('spider_name'))
|
|
26
26
|
|
|
27
27
|
except Exception as e:
|
|
28
28
|
print(f"❌ 运行失败: {e}")
|
crawlo/utils/log.py
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
# ==================== 向后兼容的日志接口 ====================
|
|
2
2
|
# 主要功能已迁移到 crawlo.logging 模块
|
|
3
|
-
#
|
|
3
|
+
# 本文件提供向后兼容接口,同时支持新的日志系统功能
|
|
4
4
|
|
|
5
5
|
import logging
|
|
6
|
-
from typing import Optional
|
|
6
|
+
from typing import Optional, Any
|
|
7
7
|
|
|
8
8
|
# 向后兼容:导入新的日志系统
|
|
9
9
|
try:
|
|
10
10
|
from crawlo.logging import get_logger as new_get_logger, configure_logging
|
|
11
|
+
|
|
11
12
|
_NEW_LOGGING_AVAILABLE = True
|
|
12
13
|
except ImportError:
|
|
13
14
|
_NEW_LOGGING_AVAILABLE = False
|
|
@@ -16,6 +17,7 @@ except ImportError:
|
|
|
16
17
|
|
|
17
18
|
LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s'
|
|
18
19
|
|
|
20
|
+
|
|
19
21
|
# 向后兼容的日志函数
|
|
20
22
|
def get_logger(name: str = 'default', level: Optional[int] = None):
|
|
21
23
|
"""获取Logger实例 - 向后兼容函数"""
|
|
@@ -33,12 +35,46 @@ def get_logger(name: str = 'default', level: Optional[int] = None):
|
|
|
33
35
|
logger.setLevel(level or logging.INFO)
|
|
34
36
|
return logger
|
|
35
37
|
|
|
36
|
-
|
|
37
|
-
def get_component_logger(component_class, settings=None, level=None):
|
|
38
|
-
"""
|
|
38
|
+
|
|
39
|
+
def get_component_logger(component_class: Any, settings: Optional[Any] = None, level: Optional[str] = None):
|
|
40
|
+
"""
|
|
41
|
+
获取组件Logger - 推荐的组件日志创建方式
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
component_class: 组件类
|
|
45
|
+
settings: 配置对象,用于读取日志级别配置
|
|
46
|
+
level: 日志级别(优先级低于settings中的配置)
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
logging.Logger: 配置好的Logger实例
|
|
50
|
+
"""
|
|
51
|
+
# 获取组件名称
|
|
39
52
|
if hasattr(component_class, '__name__'):
|
|
40
53
|
component_name = component_class.__name__
|
|
41
54
|
else:
|
|
42
55
|
component_name = str(component_class)
|
|
43
56
|
|
|
44
|
-
|
|
57
|
+
# 如果新日志系统可用,使用新系统
|
|
58
|
+
if _NEW_LOGGING_AVAILABLE and new_get_logger:
|
|
59
|
+
return new_get_logger(component_name)
|
|
60
|
+
|
|
61
|
+
# 否则使用向后兼容方式
|
|
62
|
+
# 从settings中获取日志级别(如果提供)
|
|
63
|
+
if settings is not None:
|
|
64
|
+
# 尝试从settings获取组件特定的日志级别
|
|
65
|
+
if hasattr(settings, 'get'):
|
|
66
|
+
# 检查是否有组件特定的日志级别配置
|
|
67
|
+
component_level = settings.get(f'LOG_LEVEL_{component_name}')
|
|
68
|
+
if component_level is not None:
|
|
69
|
+
level = component_level
|
|
70
|
+
else:
|
|
71
|
+
# 检查通用日志级别
|
|
72
|
+
general_level = settings.get('LOG_LEVEL')
|
|
73
|
+
if general_level is not None:
|
|
74
|
+
level = general_level
|
|
75
|
+
|
|
76
|
+
# 转换日志级别
|
|
77
|
+
if isinstance(level, str):
|
|
78
|
+
level = getattr(logging, level.upper(), logging.INFO)
|
|
79
|
+
|
|
80
|
+
return get_logger(component_name, level)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
crawlo/__init__.py,sha256=rCeDq1OoX6mmcBxuK60eUpEp1cIg5T8Zgic3FUQAOkA,2318
|
|
2
|
-
crawlo/__version__.py,sha256=
|
|
2
|
+
crawlo/__version__.py,sha256=AraH1WB67qbGfdcq6TC7ARcEi7zORceoTxsCTwF_h8g,22
|
|
3
3
|
crawlo/cli.py,sha256=OXprmcTUbFK02ptw_Gq8Gk4-ZCU-WEMJgzU1ztgP6Bk,2327
|
|
4
4
|
crawlo/config.py,sha256=dNoNyTkXLe2msQ7bZx3YTQItk1m49nIg5-g89FQDNwE,9486
|
|
5
5
|
crawlo/config_validator.py,sha256=gsiLqf5swWd9ISDvoLqCdG7iSXr-ZdBPD4iT6ug1ua4,11239
|
|
@@ -85,7 +85,7 @@ crawlo/pipelines/database_dedup_pipeline.py,sha256=Ao_5jvVPl5QikxXhPeIrcB7_3tinR
|
|
|
85
85
|
crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
|
|
86
86
|
crawlo/pipelines/memory_dedup_pipeline.py,sha256=oIksbIrmSw9s9jMh6JJMfVbv6hzseVMV_g9S8UHQUP4,3837
|
|
87
87
|
crawlo/pipelines/mongo_pipeline.py,sha256=k7gNqAO-g2MtIfArphC6z5ZzkKVRkBKcv-2ImziPFA0,5706
|
|
88
|
-
crawlo/pipelines/mysql_pipeline.py,sha256=
|
|
88
|
+
crawlo/pipelines/mysql_pipeline.py,sha256=_oRfIvlEiOsTKkr4v-yPTcL8nG9O9coRmke2ZSkkKII,13871
|
|
89
89
|
crawlo/pipelines/pipeline_manager.py,sha256=rtKZEgDc9oMDYaTrSSQYCc7rVJ-a65TQw4p3dWHF1SM,3116
|
|
90
90
|
crawlo/pipelines/redis_dedup_pipeline.py,sha256=POYRiWAOp1pqDW9iTPJ8h3VcpLALeLrpw74MvJJqPiM,6342
|
|
91
91
|
crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -97,7 +97,7 @@ crawlo/settings/default_settings.py,sha256=vmacY04PZqumteQn7URMo0r3JWwJCctXaJcox
|
|
|
97
97
|
crawlo/settings/setting_manager.py,sha256=AWPvvhOGo04Yv_q3jqEMyzhEpbxOX_Wr8tSHmI2sUnA,8109
|
|
98
98
|
crawlo/spider/__init__.py,sha256=oi9LEYq9xaCSjktAIRUgjpGQQI7rTtN61ESdHeWb1x4,21224
|
|
99
99
|
crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
|
|
100
|
-
crawlo/templates/run.py.tmpl,sha256=
|
|
100
|
+
crawlo/templates/run.py.tmpl,sha256=j8xRI4IEI_YiDFvMyfYG0XD2GgBM9eXgSqpvBniferU,927
|
|
101
101
|
crawlo/templates/spiders_init.py.tmpl,sha256=pDB5X9NO7KIko3V5X0qz38JHy_k-UbEEqRFgCSJHvUU,345
|
|
102
102
|
crawlo/templates/project/__init__.py.tmpl,sha256=f3ETIXw_O6K-lkL6lXM5znMPJW1FZYGFrwDs2BnHcnQ,58
|
|
103
103
|
crawlo/templates/project/items.py.tmpl,sha256=mt1Mm--H2Ouos3r7JPkYh0r33rgYJf1YOMz0OZy8TYs,297
|
|
@@ -135,7 +135,7 @@ crawlo/utils/error_handler.py,sha256=nDfDA99q2sirE2pe7OT2bcA54GqUiAYgtdAh38uFEX4
|
|
|
135
135
|
crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
|
|
136
136
|
crawlo/utils/large_scale_config.py,sha256=j7wQ5ty7pQlBRygw2vhRJ7OI19RYBZKPfYMP3WeF2WI,8154
|
|
137
137
|
crawlo/utils/large_scale_helper.py,sha256=Kxdy3WMuqjzQTyCc6z4xEYxXDi4xnYKJzsVwaBYZrrg,12108
|
|
138
|
-
crawlo/utils/log.py,sha256=
|
|
138
|
+
crawlo/utils/log.py,sha256=nGAyU1Mg2N1A9D9d0PgYkBiHTRyTqlFcgrdvr9qlPnc,2835
|
|
139
139
|
crawlo/utils/performance_monitor.py,sha256=Q9xxuXBIfFoig_U-FQPOUuPAh1axO3MzYgpielDyku0,9547
|
|
140
140
|
crawlo/utils/queue_helper.py,sha256=xpUUTOqlU1xz2Pb9NKAVGo3AfAO-7Xvx8Lm1q65Dgck,4743
|
|
141
141
|
crawlo/utils/redis_connection_pool.py,sha256=wh_qYYeYAW3a3hfgq41PS7Lo2CPvugi7t6PXGafEDyk,12187
|
|
@@ -161,8 +161,10 @@ tests/controlled_spider_example.py,sha256=SP_k4mdKPvD1JCPs9UCm68jcy2Frg84vvXv9-1
|
|
|
161
161
|
tests/date_tools_example.py,sha256=x_-duqnVZ-Hrk-SaNplIfcIV6W3c6u6MTxW35u1i0F0,4862
|
|
162
162
|
tests/debug_configure.py,sha256=E-6Djz8kk7tf2pzEqrGdekW2W20vrJeZN7iNm9ArWKk,2144
|
|
163
163
|
tests/debug_framework_logger.py,sha256=l2OX6igGu-pCUGrlwdWqcenqSSK9wMDheZ47XhEUqPg,3341
|
|
164
|
+
tests/debug_log_config.py,sha256=F5MrUmpkaD9Dc1eSbajaOrw3nKo9Sp55BTnNxmQkUWc,3588
|
|
164
165
|
tests/debug_log_levels.py,sha256=yPyKRNwz9kNWU1QMVLRD989Wh2sb6CrH4GAsMO0PHW8,2117
|
|
165
166
|
tests/debug_pipelines.py,sha256=VpUmoYlt6Ci7foIGuQIotUu42xp6TzoA1cBDeagBzDk,2098
|
|
167
|
+
tests/detailed_log_test.py,sha256=M6yXjQypKmIjihgTSBffkgOzC7Nn0_4ZdQLrBN-L8i0,7268
|
|
166
168
|
tests/distributed_test.py,sha256=qZpFAMQTFcg0KUEdp2RUpkuYauSCf4C3lbbosyIDqgw,1759
|
|
167
169
|
tests/distributed_test_debug.py,sha256=XOX8UlH0sQiuicoAqrSmAwteBfgTyGaOA5TNNMDFrH8,2105
|
|
168
170
|
tests/dynamic_loading_example.py,sha256=NI0SCg4lPME0RCcNpDDw1HjErjmCgJntCN0ahAEw61g,18263
|
|
@@ -171,8 +173,12 @@ tests/env_config_example.py,sha256=sKE8DvMBhM3uy439LpgLHd4wF7MGUrUc-X6E7g9qsz0,4
|
|
|
171
173
|
tests/error_handling_example.py,sha256=goF8fnTXxU3CgHcX4ALEcidVPd-zACn2tDqqQislRPA,5123
|
|
172
174
|
tests/final_command_test_report.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
173
175
|
tests/final_comprehensive_test.py,sha256=XhOjHBbetZIf4PcT3sgFSCUa913U93tta2MQuxXBwks,4163
|
|
176
|
+
tests/final_log_test.py,sha256=vSJ0KOPX6PEWkEKgozaPYt1CWDxE3LGeuKoEsrpQNCY,9107
|
|
174
177
|
tests/final_validation_test.py,sha256=p5V2bpRBT1CA1l74nggwk6Is4roaRQSR5K7lNqZ3GBc,5062
|
|
178
|
+
tests/fix_log_test.py,sha256=r222qYV69qxuWas8vaQ_ZRizgCZXg_YbtvZXkIhrGJ8,4298
|
|
175
179
|
tests/framework_performance_test.py,sha256=B-s-w5iKcxDDamJICIQP8UZXZ0ZryvfKu5k33S6b7EQ,6783
|
|
180
|
+
tests/log_buffering_test.py,sha256=jRWq5fGxYH08kSAoL-58atyHGzZco1dZ5GDYiNw0YjM,3115
|
|
181
|
+
tests/log_generation_timing_test.py,sha256=Nku-2SQRtxJmEQwwIgHi2XxEI4sdAmyiLauGA1JEIBQ,4528
|
|
176
182
|
tests/optimized_performance_test.py,sha256=m1wRrhQM6d5UhG2dwCqurNdi-kU5hk7Znz6y_vq-BN4,7168
|
|
177
183
|
tests/performance_comparison.py,sha256=2amQ_nnWxuXQDCFNUnFlNNJ4cPwTrCp9ZAwG9LBkpPg,9057
|
|
178
184
|
tests/queue_blocking_test.py,sha256=xgIgo3Itj7ndFL5tsdc2uWjWQJkaP7jGDvWhbs_3TS0,3842
|
|
@@ -183,9 +189,11 @@ tests/response_improvements_example.py,sha256=wnYGJO6MKj5_jbwKLDlbXu_Dli5XC7vlWd
|
|
|
183
189
|
tests/simple_command_test.py,sha256=GJ4KfxKxAZ8JJFuccJQn4SMPzWJcApaVVSvhz9SzvM8,3569
|
|
184
190
|
tests/simple_crawlo_test.py,sha256=8x8DNL7O_1DNtOQ_K7YsOFIZoWeGmpeEP9mKWHlkbHg,4721
|
|
185
191
|
tests/simple_log_test.py,sha256=4daRH0bqTViz-BmyPcAZY9xKGks7G5kb39MH5W7v2XI,1700
|
|
192
|
+
tests/simple_log_test2.py,sha256=Z2xcCiT_-sCd1Sd-SK7hINcn6WcH_-7Bq0TWAei-XIg,3807
|
|
186
193
|
tests/simple_optimization_test.py,sha256=CyhyzW9lhPlTDAwrJu7gTWwcEQuCBL_Bnm9mkS_-iFo,3550
|
|
187
194
|
tests/simple_spider_test.py,sha256=X5oFRV02mkOXUd5lpzOBF7gX8K62j4ZwAUXoBEZ0KKE,1119
|
|
188
195
|
tests/simple_test.py,sha256=kzMspCmfJxdnAIXXJv9tmDW1gpodkD9pznW5vA_gL84,1211
|
|
196
|
+
tests/spider_log_timing_test.py,sha256=ngZQ_v3o9oHYcs_BtZgxH1N-N2tZUDPu-cnTnsHEpP8,5396
|
|
189
197
|
tests/test_advanced_tools.py,sha256=3R8EfKVyBHEb6FA5TP3ieaWeHZhobVgSx8t3phipCrE,5250
|
|
190
198
|
tests/test_all_commands.py,sha256=yGPw8zMrB5Z5w5LkaymSzKRLOcZsBPBXLvllCkgEY4I,7488
|
|
191
199
|
tests/test_all_redis_key_configs.py,sha256=SGoip8M7oB2LNWC_31aJ4ECcDRmx0psr7i7DGzuaH7c,5565
|
|
@@ -220,10 +228,12 @@ tests/test_final_validation.py,sha256=aAiWLzhDCcv-GEXg9sauaVIfq5rz3s2vm67Gk2_lmB
|
|
|
220
228
|
tests/test_framework_env_usage.py,sha256=HYpTwORXeaJHMffCYAGHGvc_a6ax4lo28xP8BYOaKxk,4098
|
|
221
229
|
tests/test_framework_logger.py,sha256=HNkOlyA-dQKEdE6H4VaUHfF3aeVkKRoISSr53Hw90qQ,2506
|
|
222
230
|
tests/test_framework_startup.py,sha256=I0zUfJUjkM7JgUBChO2w9cIL-tDJwUHdzKm3QjuEEJM,2215
|
|
231
|
+
tests/test_get_component_logger.py,sha256=f7nmmSGxoD1i3d17FlSicOmMGLTcyJxcujoS5eJFbAI,2202
|
|
223
232
|
tests/test_integration.py,sha256=OCkjyv76Wop7CrXEko6rfoDsIK6SESA18KgCaTwL7Q4,4670
|
|
224
233
|
tests/test_item_dedup_redis_key.py,sha256=QxLuXHUx0xqT6y7lQzOWcrLkRui7Qs7C6NgRvjzIypA,3720
|
|
225
234
|
tests/test_large_scale_config.py,sha256=wyeMOMjGYhbZ6mrcnLH3Eh6GfspJwhavwWoyOy1y90c,4184
|
|
226
235
|
tests/test_large_scale_helper.py,sha256=spvL0MPyXMAUDpzI2fY6-OQdSxOHtgJ1yuSUIbydyHY,8136
|
|
236
|
+
tests/test_logging_system.py,sha256=_LRdgprZFrChA26JJgkjVyf6S6qRIyi6BRajK13l_Q8,8924
|
|
227
237
|
tests/test_mode_change.py,sha256=kh5C4ut7T5dZ8b2dDot4RbLWMXJidv4FHzuTIgDxMBI,2605
|
|
228
238
|
tests/test_mode_consistency.py,sha256=YJXf0SqAYVnFXy8eeBLC-zGTFAyO2fnsR4qLB76gZts,1225
|
|
229
239
|
tests/test_offsite_middleware.py,sha256=L5YT9ZqcQwBunUv0Ddj-sLZcW4IMlAlgaJCwICHFWxI,7543
|
|
@@ -271,8 +281,8 @@ tests/verify_distributed.py,sha256=krnYYA5Qx9xXDMWc9YF5DxPSplGvawDg2n0l-3CAqoM,3
|
|
|
271
281
|
tests/verify_log_fix.py,sha256=TD7M1R22NxLqQPufvgE-H33u9tUjyz-rSR2ayIXozRU,4225
|
|
272
282
|
tests/scrapy_comparison/ofweek_scrapy.py,sha256=2Hvpi6DRTubUxBy6RyJApQxMQONPLc1zWjKTQO_i5U4,5652
|
|
273
283
|
tests/scrapy_comparison/scrapy_test.py,sha256=5sw7jOHhaTmQ8bsUd1TiolAUTRQYQOe-f49HPfysqbI,5466
|
|
274
|
-
crawlo-1.3.
|
|
275
|
-
crawlo-1.3.
|
|
276
|
-
crawlo-1.3.
|
|
277
|
-
crawlo-1.3.
|
|
278
|
-
crawlo-1.3.
|
|
284
|
+
crawlo-1.3.5.dist-info/METADATA,sha256=RDqsfzstoxtnrY_AR26XXQ3MbZA-W7sdCx4_CPReUGo,29742
|
|
285
|
+
crawlo-1.3.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
286
|
+
crawlo-1.3.5.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
|
|
287
|
+
crawlo-1.3.5.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
|
|
288
|
+
crawlo-1.3.5.dist-info/RECORD,,
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: UTF-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
调试日志配置问题
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
# 添加项目根目录到Python路径
|
|
12
|
+
project_root = Path(__file__).parent.parent
|
|
13
|
+
sys.path.insert(0, str(project_root))
|
|
14
|
+
|
|
15
|
+
from crawlo.logging import configure_logging as configure, get_logger, LogManager
|
|
16
|
+
from crawlo.logging.config import LogConfig
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def debug_log_configuration():
|
|
20
|
+
"""调试日志配置"""
|
|
21
|
+
print("=== 调试日志配置 ===")
|
|
22
|
+
|
|
23
|
+
# 重置配置
|
|
24
|
+
LogManager().reset()
|
|
25
|
+
|
|
26
|
+
# 1. 检查初始状态
|
|
27
|
+
print("1. 检查初始状态...")
|
|
28
|
+
print(f" 初始配置状态: {LogManager().is_configured}")
|
|
29
|
+
if LogManager().config:
|
|
30
|
+
print(f" 初始配置: {LogManager().config}")
|
|
31
|
+
|
|
32
|
+
# 2. 配置日志系统
|
|
33
|
+
print("2. 配置日志系统...")
|
|
34
|
+
config = configure(
|
|
35
|
+
LOG_LEVEL='DEBUG',
|
|
36
|
+
LOG_FILE='debug_test.log',
|
|
37
|
+
LOG_MAX_BYTES=1024,
|
|
38
|
+
LOG_BACKUP_COUNT=2,
|
|
39
|
+
LOG_CONSOLE_ENABLED=True,
|
|
40
|
+
LOG_FILE_ENABLED=True
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
print(f" 配置返回值: {config}")
|
|
44
|
+
print(f" 配置类型: {type(config)}")
|
|
45
|
+
print(f" 配置级别: {config.level}")
|
|
46
|
+
print(f" 配置文件路径: {config.file_path}")
|
|
47
|
+
print(f" 轮转大小: {config.max_bytes}")
|
|
48
|
+
print(f" 备份数量: {config.backup_count}")
|
|
49
|
+
print(f" 控制台启用: {config.console_enabled}")
|
|
50
|
+
print(f" 文件启用: {config.file_enabled}")
|
|
51
|
+
|
|
52
|
+
# 3. 检查管理器状态
|
|
53
|
+
print("3. 检查管理器状态...")
|
|
54
|
+
manager = LogManager()
|
|
55
|
+
print(f" 管理器配置状态: {manager.is_configured}")
|
|
56
|
+
if manager.config:
|
|
57
|
+
print(f" 管理器配置: {manager.config}")
|
|
58
|
+
print(f" 管理器配置文件路径: {manager.config.file_path}")
|
|
59
|
+
|
|
60
|
+
# 4. 测试Logger创建
|
|
61
|
+
print("4. 测试Logger创建...")
|
|
62
|
+
logger = get_logger('test.debug')
|
|
63
|
+
print(f" Logger handlers数量: {len(logger.handlers)}")
|
|
64
|
+
|
|
65
|
+
for i, handler in enumerate(logger.handlers):
|
|
66
|
+
handler_type = type(handler).__name__
|
|
67
|
+
print(f" Handler {i}: {handler_type}")
|
|
68
|
+
if hasattr(handler, 'baseFilename'):
|
|
69
|
+
print(f" 文件名: {handler.baseFilename}")
|
|
70
|
+
|
|
71
|
+
# 5. 测试日志输出
|
|
72
|
+
print("5. 测试日志输出...")
|
|
73
|
+
logger.info("调试测试消息")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_config_from_dict():
|
|
77
|
+
"""测试从字典创建配置"""
|
|
78
|
+
print("\n=== 测试从字典创建配置 ===")
|
|
79
|
+
|
|
80
|
+
LogManager().reset()
|
|
81
|
+
|
|
82
|
+
# 使用字典配置
|
|
83
|
+
config_dict = {
|
|
84
|
+
'level': 'DEBUG',
|
|
85
|
+
'file_path': 'dict_test.log',
|
|
86
|
+
'max_bytes': 1024,
|
|
87
|
+
'backup_count': 2,
|
|
88
|
+
'console_enabled': True,
|
|
89
|
+
'file_enabled': True
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
config = LogConfig.from_dict(config_dict)
|
|
93
|
+
print(f" 字典配置: {config}")
|
|
94
|
+
print(f" 验证结果: {config.validate()}")
|
|
95
|
+
|
|
96
|
+
# 应用配置
|
|
97
|
+
LogManager().configure(config)
|
|
98
|
+
|
|
99
|
+
logger = get_logger('test.dict')
|
|
100
|
+
print(f" Logger handlers数量: {len(logger.handlers)}")
|
|
101
|
+
|
|
102
|
+
for i, handler in enumerate(logger.handlers):
|
|
103
|
+
handler_type = type(handler).__name__
|
|
104
|
+
print(f" Handler {i}: {handler_type}")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def main():
|
|
108
|
+
"""主函数"""
|
|
109
|
+
print("开始调试日志配置问题...")
|
|
110
|
+
|
|
111
|
+
try:
|
|
112
|
+
debug_log_configuration()
|
|
113
|
+
test_config_from_dict()
|
|
114
|
+
|
|
115
|
+
print("\n=== 调试完成 ===")
|
|
116
|
+
|
|
117
|
+
except Exception as e:
|
|
118
|
+
print(f"\n调试过程中出现错误: {e}")
|
|
119
|
+
import traceback
|
|
120
|
+
traceback.print_exc()
|
|
121
|
+
return 1
|
|
122
|
+
|
|
123
|
+
return 0
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
if __name__ == '__main__':
|
|
127
|
+
sys.exit(main())
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: UTF-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
详细的日志系统功能测试
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
import tempfile
|
|
10
|
+
import shutil
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
# 添加项目根目录到Python路径
|
|
14
|
+
project_root = Path(__file__).parent.parent
|
|
15
|
+
sys.path.insert(0, str(project_root))
|
|
16
|
+
|
|
17
|
+
from crawlo.logging import configure_logging as configure, get_logger, LogManager
|
|
18
|
+
from crawlo.logging.config import LogConfig
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_log_config_creation():
|
|
22
|
+
"""测试日志配置创建"""
|
|
23
|
+
print("=== 测试日志配置创建 ===")
|
|
24
|
+
|
|
25
|
+
# 重置日志管理器
|
|
26
|
+
LogManager().reset()
|
|
27
|
+
|
|
28
|
+
# 1. 测试通过关键字参数创建配置
|
|
29
|
+
print("1. 测试通过关键字参数创建配置...")
|
|
30
|
+
config = configure(
|
|
31
|
+
LOG_LEVEL='DEBUG',
|
|
32
|
+
LOG_FILE='test.log',
|
|
33
|
+
LOG_MAX_BYTES=1024,
|
|
34
|
+
LOG_BACKUP_COUNT=3
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
print(f" 配置级别: {config.level}")
|
|
38
|
+
print(f" 配置文件路径: {config.file_path}")
|
|
39
|
+
print(f" 轮转大小: {config.max_bytes}")
|
|
40
|
+
print(f" 备份数量: {config.backup_count}")
|
|
41
|
+
|
|
42
|
+
# 2. 测试通过字典创建配置
|
|
43
|
+
print("2. 测试通过字典创建配置...")
|
|
44
|
+
LogManager().reset()
|
|
45
|
+
config_dict = {
|
|
46
|
+
'LOG_LEVEL': 'WARNING',
|
|
47
|
+
'LOG_FILE': 'dict_test.log',
|
|
48
|
+
'LOG_CONSOLE_ENABLED': False
|
|
49
|
+
}
|
|
50
|
+
config = configure(**config_dict)
|
|
51
|
+
|
|
52
|
+
print(f" 配置级别: {config.level}")
|
|
53
|
+
print(f" 配置文件路径: {config.file_path}")
|
|
54
|
+
print(f" 控制台启用: {config.console_enabled}")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_logger_factory():
|
|
58
|
+
"""测试Logger工厂"""
|
|
59
|
+
print("\n=== 测试Logger工厂 ===")
|
|
60
|
+
|
|
61
|
+
# 重置并配置
|
|
62
|
+
LogManager().reset()
|
|
63
|
+
configure(LOG_LEVEL='INFO')
|
|
64
|
+
|
|
65
|
+
# 1. 测试获取Logger
|
|
66
|
+
print("1. 测试获取Logger...")
|
|
67
|
+
logger1 = get_logger('test.factory1')
|
|
68
|
+
logger2 = get_logger('test.factory2')
|
|
69
|
+
logger3 = get_logger('test.factory1') # 应该是同一个实例
|
|
70
|
+
|
|
71
|
+
print(f" Logger1 ID: {id(logger1)}")
|
|
72
|
+
print(f" Logger2 ID: {id(logger2)}")
|
|
73
|
+
print(f" Logger3 ID: {id(logger3)}")
|
|
74
|
+
print(f" Logger1和Logger3是同一对象: {logger1 is logger3}")
|
|
75
|
+
|
|
76
|
+
# 2. 测试Logger配置
|
|
77
|
+
print("2. 测试Logger配置...")
|
|
78
|
+
print(f" Logger1名称: {logger1.name}")
|
|
79
|
+
print(f" Logger1 handlers数量: {len(logger1.handlers)}")
|
|
80
|
+
|
|
81
|
+
for i, handler in enumerate(logger1.handlers):
|
|
82
|
+
print(f" Handler {i}: {type(handler).__name__}")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_file_and_console_handlers():
|
|
86
|
+
"""测试文件和控制台处理器"""
|
|
87
|
+
print("\n=== 测试文件和控制台处理器 ===")
|
|
88
|
+
|
|
89
|
+
# 创建临时目录
|
|
90
|
+
temp_dir = tempfile.mkdtemp()
|
|
91
|
+
log_file = os.path.join(temp_dir, 'handler_test.log')
|
|
92
|
+
|
|
93
|
+
try:
|
|
94
|
+
# 1. 测试文件和控制台都启用
|
|
95
|
+
print("1. 测试文件和控制台都启用...")
|
|
96
|
+
LogManager().reset()
|
|
97
|
+
configure(
|
|
98
|
+
LOG_LEVEL='INFO',
|
|
99
|
+
LOG_FILE=log_file,
|
|
100
|
+
LOG_CONSOLE_ENABLED=True,
|
|
101
|
+
LOG_FILE_ENABLED=True
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
logger = get_logger('test.handlers')
|
|
105
|
+
print(f" Handlers数量: {len(logger.handlers)}")
|
|
106
|
+
|
|
107
|
+
has_file_handler = False
|
|
108
|
+
has_console_handler = False
|
|
109
|
+
|
|
110
|
+
for handler in logger.handlers:
|
|
111
|
+
handler_type = type(handler).__name__
|
|
112
|
+
print(f" Handler类型: {handler_type}")
|
|
113
|
+
if 'FileHandler' in handler_type:
|
|
114
|
+
has_file_handler = True
|
|
115
|
+
print(f" 文件路径: {getattr(handler, 'baseFilename', 'N/A')}")
|
|
116
|
+
elif 'StreamHandler' in handler_type:
|
|
117
|
+
has_console_handler = True
|
|
118
|
+
|
|
119
|
+
print(f" 有文件处理器: {has_file_handler}")
|
|
120
|
+
print(f" 有控制台处理器: {has_console_handler}")
|
|
121
|
+
|
|
122
|
+
# 输出日志
|
|
123
|
+
logger.info("测试文件和控制台处理器")
|
|
124
|
+
|
|
125
|
+
# 检查文件是否存在
|
|
126
|
+
if os.path.exists(log_file):
|
|
127
|
+
with open(log_file, 'r', encoding='utf-8') as f:
|
|
128
|
+
content = f.read()
|
|
129
|
+
print(f" 文件内容行数: {len(content.splitlines())}")
|
|
130
|
+
else:
|
|
131
|
+
print(" 文件不存在!")
|
|
132
|
+
|
|
133
|
+
finally:
|
|
134
|
+
# 清理
|
|
135
|
+
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def test_log_levels():
|
|
139
|
+
"""测试日志级别"""
|
|
140
|
+
print("\n=== 测试日志级别 ===")
|
|
141
|
+
|
|
142
|
+
# 1. 测试默认级别
|
|
143
|
+
print("1. 测试默认级别...")
|
|
144
|
+
LogManager().reset()
|
|
145
|
+
configure(LOG_LEVEL='WARNING')
|
|
146
|
+
|
|
147
|
+
logger = get_logger('test.levels')
|
|
148
|
+
logger.debug("DEBUG消息 - 不应该显示")
|
|
149
|
+
logger.info("INFO消息 - 不应该显示")
|
|
150
|
+
logger.warning("WARNING消息 - 应该显示")
|
|
151
|
+
logger.error("ERROR消息 - 应该显示")
|
|
152
|
+
|
|
153
|
+
# 2. 测试模块特定级别
|
|
154
|
+
print("2. 测试模块特定级别...")
|
|
155
|
+
LogManager().reset()
|
|
156
|
+
configure(
|
|
157
|
+
LOG_LEVEL='ERROR',
|
|
158
|
+
LOG_LEVELS={
|
|
159
|
+
'test.debug_module': 'DEBUG',
|
|
160
|
+
'test.info_module': 'INFO'
|
|
161
|
+
}
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# 默认模块(ERROR级别)
|
|
165
|
+
default_logger = get_logger('test.default')
|
|
166
|
+
default_logger.info("默认模块INFO消息 - 不应该显示")
|
|
167
|
+
default_logger.error("默认模块ERROR消息 - 应该显示")
|
|
168
|
+
|
|
169
|
+
# DEBUG模块(DEBUG级别)
|
|
170
|
+
debug_logger = get_logger('test.debug_module')
|
|
171
|
+
debug_logger.debug("DEBUG模块DEBUG消息 - 应该显示")
|
|
172
|
+
debug_logger.info("DEBUG模块INFO消息 - 应该显示")
|
|
173
|
+
|
|
174
|
+
# INFO模块(INFO级别)
|
|
175
|
+
info_logger = get_logger('test.info_module')
|
|
176
|
+
info_logger.debug("INFO模块DEBUG消息 - 不应该显示")
|
|
177
|
+
info_logger.info("INFO模块INFO消息 - 应该显示")
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def test_log_config_validation():
|
|
181
|
+
"""测试日志配置验证"""
|
|
182
|
+
print("\n=== 测试日志配置验证 ===")
|
|
183
|
+
|
|
184
|
+
# 1. 测试有效配置
|
|
185
|
+
print("1. 测试有效配置...")
|
|
186
|
+
valid_config = LogConfig(level='INFO', file_path='valid.log')
|
|
187
|
+
is_valid = valid_config.validate()
|
|
188
|
+
print(f" 有效配置验证: {is_valid}")
|
|
189
|
+
|
|
190
|
+
# 2. 测试无效级别
|
|
191
|
+
print("2. 测试无效级别...")
|
|
192
|
+
invalid_config = LogConfig(level='INVALID', file_path='invalid.log')
|
|
193
|
+
is_valid = invalid_config.validate()
|
|
194
|
+
print(f" 无效级别验证: {is_valid}")
|
|
195
|
+
|
|
196
|
+
# 3. 测试目录创建
|
|
197
|
+
print("3. 测试目录创建...")
|
|
198
|
+
temp_dir = tempfile.mkdtemp()
|
|
199
|
+
nested_path = os.path.join(temp_dir, 'subdir', 'nested.log')
|
|
200
|
+
|
|
201
|
+
nested_config = LogConfig(level='INFO', file_path=nested_path)
|
|
202
|
+
is_valid = nested_config.validate()
|
|
203
|
+
print(f" 嵌套目录验证: {is_valid}")
|
|
204
|
+
print(f" 目录存在: {os.path.exists(os.path.dirname(nested_path))}")
|
|
205
|
+
|
|
206
|
+
# 清理
|
|
207
|
+
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def main():
|
|
211
|
+
"""主测试函数"""
|
|
212
|
+
print("开始详细测试Crawlo框架日志系统...")
|
|
213
|
+
|
|
214
|
+
try:
|
|
215
|
+
# 运行所有测试
|
|
216
|
+
test_log_config_creation()
|
|
217
|
+
test_logger_factory()
|
|
218
|
+
test_file_and_console_handlers()
|
|
219
|
+
test_log_levels()
|
|
220
|
+
test_log_config_validation()
|
|
221
|
+
|
|
222
|
+
print("\n=== 详细测试完成 ===")
|
|
223
|
+
|
|
224
|
+
except Exception as e:
|
|
225
|
+
print(f"\n测试过程中出现错误: {e}")
|
|
226
|
+
import traceback
|
|
227
|
+
traceback.print_exc()
|
|
228
|
+
return 1
|
|
229
|
+
|
|
230
|
+
return 0
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
if __name__ == '__main__':
|
|
234
|
+
sys.exit(main())
|