crawlo 1.3.4__py3-none-any.whl → 1.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of crawlo might be problematic. Click here for more details.

crawlo/__version__.py CHANGED
@@ -1 +1 @@
1
- __version__ = '1.3.4'
1
+ __version__ = '1.3.5'
@@ -20,7 +20,14 @@ class AsyncmyMySQLPipeline:
20
20
  self._pool_lock = asyncio.Lock()
21
21
  self._pool_initialized = False
22
22
  self.pool = None
23
+
24
+ # 优先从爬虫的custom_settings中获取表名,如果没有则使用默认值
25
+ spider_table_name = None
26
+ if hasattr(crawler, 'spider') and crawler.spider and hasattr(crawler.spider, 'custom_settings'):
27
+ spider_table_name = crawler.spider.custom_settings.get('MYSQL_TABLE')
28
+
23
29
  self.table_name = (
30
+ spider_table_name or
24
31
  self.settings.get('MYSQL_TABLE') or
25
32
  getattr(crawler.spider, 'mysql_table', None) or
26
33
  f"{crawler.spider.name}_items"
@@ -21,8 +21,8 @@ def main():
21
21
  spider_modules = ['{{project_name}}.spiders']
22
22
  process = CrawlerProcess(spider_modules=spider_modules)
23
23
 
24
- # 运行指定的爬虫
25
- asyncio.run(process.crawl('{{spider_name}}'))
24
+ # TODO 运行指定的爬虫
25
+ asyncio.run(process.crawl('spider_name'))
26
26
 
27
27
  except Exception as e:
28
28
  print(f"❌ 运行失败: {e}")
crawlo/utils/log.py CHANGED
@@ -1,13 +1,14 @@
1
1
  # ==================== 向后兼容的日志接口 ====================
2
2
  # 主要功能已迁移到 crawlo.logging 模块
3
- # 本文件仅保留最基本的兼容性接口
3
+ # 本文件提供向后兼容接口,同时支持新的日志系统功能
4
4
 
5
5
  import logging
6
- from typing import Optional
6
+ from typing import Optional, Any
7
7
 
8
8
  # 向后兼容:导入新的日志系统
9
9
  try:
10
10
  from crawlo.logging import get_logger as new_get_logger, configure_logging
11
+
11
12
  _NEW_LOGGING_AVAILABLE = True
12
13
  except ImportError:
13
14
  _NEW_LOGGING_AVAILABLE = False
@@ -16,6 +17,7 @@ except ImportError:
16
17
 
17
18
  LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s'
18
19
 
20
+
19
21
  # 向后兼容的日志函数
20
22
  def get_logger(name: str = 'default', level: Optional[int] = None):
21
23
  """获取Logger实例 - 向后兼容函数"""
@@ -33,12 +35,46 @@ def get_logger(name: str = 'default', level: Optional[int] = None):
33
35
  logger.setLevel(level or logging.INFO)
34
36
  return logger
35
37
 
36
- # 兼容性函数
37
- def get_component_logger(component_class, settings=None, level=None):
38
- """获取组件Logger - 向后兼容"""
38
+
39
+ def get_component_logger(component_class: Any, settings: Optional[Any] = None, level: Optional[str] = None):
40
+ """
41
+ 获取组件Logger - 推荐的组件日志创建方式
42
+
43
+ Args:
44
+ component_class: 组件类
45
+ settings: 配置对象,用于读取日志级别配置
46
+ level: 日志级别(优先级低于settings中的配置)
47
+
48
+ Returns:
49
+ logging.Logger: 配置好的Logger实例
50
+ """
51
+ # 获取组件名称
39
52
  if hasattr(component_class, '__name__'):
40
53
  component_name = component_class.__name__
41
54
  else:
42
55
  component_name = str(component_class)
43
56
 
44
- return get_logger(component_name)
57
+ # 如果新日志系统可用,使用新系统
58
+ if _NEW_LOGGING_AVAILABLE and new_get_logger:
59
+ return new_get_logger(component_name)
60
+
61
+ # 否则使用向后兼容方式
62
+ # 从settings中获取日志级别(如果提供)
63
+ if settings is not None:
64
+ # 尝试从settings获取组件特定的日志级别
65
+ if hasattr(settings, 'get'):
66
+ # 检查是否有组件特定的日志级别配置
67
+ component_level = settings.get(f'LOG_LEVEL_{component_name}')
68
+ if component_level is not None:
69
+ level = component_level
70
+ else:
71
+ # 检查通用日志级别
72
+ general_level = settings.get('LOG_LEVEL')
73
+ if general_level is not None:
74
+ level = general_level
75
+
76
+ # 转换日志级别
77
+ if isinstance(level, str):
78
+ level = getattr(logging, level.upper(), logging.INFO)
79
+
80
+ return get_logger(component_name, level)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: crawlo
3
- Version: 1.3.4
3
+ Version: 1.3.5
4
4
  Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架,支持分布式抓取。
5
5
  Home-page: https://github.com/crawl-coder/Crawlo.git
6
6
  Author: crawl-coder
@@ -1,5 +1,5 @@
1
1
  crawlo/__init__.py,sha256=rCeDq1OoX6mmcBxuK60eUpEp1cIg5T8Zgic3FUQAOkA,2318
2
- crawlo/__version__.py,sha256=znOUWqTRUyFzytrxffOUq80wt0j_tYutMKHTUCSPrAo,22
2
+ crawlo/__version__.py,sha256=AraH1WB67qbGfdcq6TC7ARcEi7zORceoTxsCTwF_h8g,22
3
3
  crawlo/cli.py,sha256=OXprmcTUbFK02ptw_Gq8Gk4-ZCU-WEMJgzU1ztgP6Bk,2327
4
4
  crawlo/config.py,sha256=dNoNyTkXLe2msQ7bZx3YTQItk1m49nIg5-g89FQDNwE,9486
5
5
  crawlo/config_validator.py,sha256=gsiLqf5swWd9ISDvoLqCdG7iSXr-ZdBPD4iT6ug1ua4,11239
@@ -85,7 +85,7 @@ crawlo/pipelines/database_dedup_pipeline.py,sha256=Ao_5jvVPl5QikxXhPeIrcB7_3tinR
85
85
  crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
86
86
  crawlo/pipelines/memory_dedup_pipeline.py,sha256=oIksbIrmSw9s9jMh6JJMfVbv6hzseVMV_g9S8UHQUP4,3837
87
87
  crawlo/pipelines/mongo_pipeline.py,sha256=k7gNqAO-g2MtIfArphC6z5ZzkKVRkBKcv-2ImziPFA0,5706
88
- crawlo/pipelines/mysql_pipeline.py,sha256=CKll3rNFXc0-BGQ_0A6QOSm2-ymHtdjdybX6bSB8i2g,13500
88
+ crawlo/pipelines/mysql_pipeline.py,sha256=_oRfIvlEiOsTKkr4v-yPTcL8nG9O9coRmke2ZSkkKII,13871
89
89
  crawlo/pipelines/pipeline_manager.py,sha256=rtKZEgDc9oMDYaTrSSQYCc7rVJ-a65TQw4p3dWHF1SM,3116
90
90
  crawlo/pipelines/redis_dedup_pipeline.py,sha256=POYRiWAOp1pqDW9iTPJ8h3VcpLALeLrpw74MvJJqPiM,6342
91
91
  crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -97,7 +97,7 @@ crawlo/settings/default_settings.py,sha256=vmacY04PZqumteQn7URMo0r3JWwJCctXaJcox
97
97
  crawlo/settings/setting_manager.py,sha256=AWPvvhOGo04Yv_q3jqEMyzhEpbxOX_Wr8tSHmI2sUnA,8109
98
98
  crawlo/spider/__init__.py,sha256=oi9LEYq9xaCSjktAIRUgjpGQQI7rTtN61ESdHeWb1x4,21224
99
99
  crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
100
- crawlo/templates/run.py.tmpl,sha256=yZuY8Sd0Vv8KLsneE2eY5s8iFPKIECpRZIJOIIu1k8U,926
100
+ crawlo/templates/run.py.tmpl,sha256=j8xRI4IEI_YiDFvMyfYG0XD2GgBM9eXgSqpvBniferU,927
101
101
  crawlo/templates/spiders_init.py.tmpl,sha256=pDB5X9NO7KIko3V5X0qz38JHy_k-UbEEqRFgCSJHvUU,345
102
102
  crawlo/templates/project/__init__.py.tmpl,sha256=f3ETIXw_O6K-lkL6lXM5znMPJW1FZYGFrwDs2BnHcnQ,58
103
103
  crawlo/templates/project/items.py.tmpl,sha256=mt1Mm--H2Ouos3r7JPkYh0r33rgYJf1YOMz0OZy8TYs,297
@@ -135,7 +135,7 @@ crawlo/utils/error_handler.py,sha256=nDfDA99q2sirE2pe7OT2bcA54GqUiAYgtdAh38uFEX4
135
135
  crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
136
136
  crawlo/utils/large_scale_config.py,sha256=j7wQ5ty7pQlBRygw2vhRJ7OI19RYBZKPfYMP3WeF2WI,8154
137
137
  crawlo/utils/large_scale_helper.py,sha256=Kxdy3WMuqjzQTyCc6z4xEYxXDi4xnYKJzsVwaBYZrrg,12108
138
- crawlo/utils/log.py,sha256=Q9AO7GGWZlA86fjhRz_Fb9MluCx9yihYmzsFIcK-0-w,1532
138
+ crawlo/utils/log.py,sha256=nGAyU1Mg2N1A9D9d0PgYkBiHTRyTqlFcgrdvr9qlPnc,2835
139
139
  crawlo/utils/performance_monitor.py,sha256=Q9xxuXBIfFoig_U-FQPOUuPAh1axO3MzYgpielDyku0,9547
140
140
  crawlo/utils/queue_helper.py,sha256=xpUUTOqlU1xz2Pb9NKAVGo3AfAO-7Xvx8Lm1q65Dgck,4743
141
141
  crawlo/utils/redis_connection_pool.py,sha256=wh_qYYeYAW3a3hfgq41PS7Lo2CPvugi7t6PXGafEDyk,12187
@@ -161,8 +161,10 @@ tests/controlled_spider_example.py,sha256=SP_k4mdKPvD1JCPs9UCm68jcy2Frg84vvXv9-1
161
161
  tests/date_tools_example.py,sha256=x_-duqnVZ-Hrk-SaNplIfcIV6W3c6u6MTxW35u1i0F0,4862
162
162
  tests/debug_configure.py,sha256=E-6Djz8kk7tf2pzEqrGdekW2W20vrJeZN7iNm9ArWKk,2144
163
163
  tests/debug_framework_logger.py,sha256=l2OX6igGu-pCUGrlwdWqcenqSSK9wMDheZ47XhEUqPg,3341
164
+ tests/debug_log_config.py,sha256=F5MrUmpkaD9Dc1eSbajaOrw3nKo9Sp55BTnNxmQkUWc,3588
164
165
  tests/debug_log_levels.py,sha256=yPyKRNwz9kNWU1QMVLRD989Wh2sb6CrH4GAsMO0PHW8,2117
165
166
  tests/debug_pipelines.py,sha256=VpUmoYlt6Ci7foIGuQIotUu42xp6TzoA1cBDeagBzDk,2098
167
+ tests/detailed_log_test.py,sha256=M6yXjQypKmIjihgTSBffkgOzC7Nn0_4ZdQLrBN-L8i0,7268
166
168
  tests/distributed_test.py,sha256=qZpFAMQTFcg0KUEdp2RUpkuYauSCf4C3lbbosyIDqgw,1759
167
169
  tests/distributed_test_debug.py,sha256=XOX8UlH0sQiuicoAqrSmAwteBfgTyGaOA5TNNMDFrH8,2105
168
170
  tests/dynamic_loading_example.py,sha256=NI0SCg4lPME0RCcNpDDw1HjErjmCgJntCN0ahAEw61g,18263
@@ -171,8 +173,12 @@ tests/env_config_example.py,sha256=sKE8DvMBhM3uy439LpgLHd4wF7MGUrUc-X6E7g9qsz0,4
171
173
  tests/error_handling_example.py,sha256=goF8fnTXxU3CgHcX4ALEcidVPd-zACn2tDqqQislRPA,5123
172
174
  tests/final_command_test_report.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
173
175
  tests/final_comprehensive_test.py,sha256=XhOjHBbetZIf4PcT3sgFSCUa913U93tta2MQuxXBwks,4163
176
+ tests/final_log_test.py,sha256=vSJ0KOPX6PEWkEKgozaPYt1CWDxE3LGeuKoEsrpQNCY,9107
174
177
  tests/final_validation_test.py,sha256=p5V2bpRBT1CA1l74nggwk6Is4roaRQSR5K7lNqZ3GBc,5062
178
+ tests/fix_log_test.py,sha256=r222qYV69qxuWas8vaQ_ZRizgCZXg_YbtvZXkIhrGJ8,4298
175
179
  tests/framework_performance_test.py,sha256=B-s-w5iKcxDDamJICIQP8UZXZ0ZryvfKu5k33S6b7EQ,6783
180
+ tests/log_buffering_test.py,sha256=jRWq5fGxYH08kSAoL-58atyHGzZco1dZ5GDYiNw0YjM,3115
181
+ tests/log_generation_timing_test.py,sha256=Nku-2SQRtxJmEQwwIgHi2XxEI4sdAmyiLauGA1JEIBQ,4528
176
182
  tests/optimized_performance_test.py,sha256=m1wRrhQM6d5UhG2dwCqurNdi-kU5hk7Znz6y_vq-BN4,7168
177
183
  tests/performance_comparison.py,sha256=2amQ_nnWxuXQDCFNUnFlNNJ4cPwTrCp9ZAwG9LBkpPg,9057
178
184
  tests/queue_blocking_test.py,sha256=xgIgo3Itj7ndFL5tsdc2uWjWQJkaP7jGDvWhbs_3TS0,3842
@@ -183,9 +189,11 @@ tests/response_improvements_example.py,sha256=wnYGJO6MKj5_jbwKLDlbXu_Dli5XC7vlWd
183
189
  tests/simple_command_test.py,sha256=GJ4KfxKxAZ8JJFuccJQn4SMPzWJcApaVVSvhz9SzvM8,3569
184
190
  tests/simple_crawlo_test.py,sha256=8x8DNL7O_1DNtOQ_K7YsOFIZoWeGmpeEP9mKWHlkbHg,4721
185
191
  tests/simple_log_test.py,sha256=4daRH0bqTViz-BmyPcAZY9xKGks7G5kb39MH5W7v2XI,1700
192
+ tests/simple_log_test2.py,sha256=Z2xcCiT_-sCd1Sd-SK7hINcn6WcH_-7Bq0TWAei-XIg,3807
186
193
  tests/simple_optimization_test.py,sha256=CyhyzW9lhPlTDAwrJu7gTWwcEQuCBL_Bnm9mkS_-iFo,3550
187
194
  tests/simple_spider_test.py,sha256=X5oFRV02mkOXUd5lpzOBF7gX8K62j4ZwAUXoBEZ0KKE,1119
188
195
  tests/simple_test.py,sha256=kzMspCmfJxdnAIXXJv9tmDW1gpodkD9pznW5vA_gL84,1211
196
+ tests/spider_log_timing_test.py,sha256=ngZQ_v3o9oHYcs_BtZgxH1N-N2tZUDPu-cnTnsHEpP8,5396
189
197
  tests/test_advanced_tools.py,sha256=3R8EfKVyBHEb6FA5TP3ieaWeHZhobVgSx8t3phipCrE,5250
190
198
  tests/test_all_commands.py,sha256=yGPw8zMrB5Z5w5LkaymSzKRLOcZsBPBXLvllCkgEY4I,7488
191
199
  tests/test_all_redis_key_configs.py,sha256=SGoip8M7oB2LNWC_31aJ4ECcDRmx0psr7i7DGzuaH7c,5565
@@ -220,10 +228,12 @@ tests/test_final_validation.py,sha256=aAiWLzhDCcv-GEXg9sauaVIfq5rz3s2vm67Gk2_lmB
220
228
  tests/test_framework_env_usage.py,sha256=HYpTwORXeaJHMffCYAGHGvc_a6ax4lo28xP8BYOaKxk,4098
221
229
  tests/test_framework_logger.py,sha256=HNkOlyA-dQKEdE6H4VaUHfF3aeVkKRoISSr53Hw90qQ,2506
222
230
  tests/test_framework_startup.py,sha256=I0zUfJUjkM7JgUBChO2w9cIL-tDJwUHdzKm3QjuEEJM,2215
231
+ tests/test_get_component_logger.py,sha256=f7nmmSGxoD1i3d17FlSicOmMGLTcyJxcujoS5eJFbAI,2202
223
232
  tests/test_integration.py,sha256=OCkjyv76Wop7CrXEko6rfoDsIK6SESA18KgCaTwL7Q4,4670
224
233
  tests/test_item_dedup_redis_key.py,sha256=QxLuXHUx0xqT6y7lQzOWcrLkRui7Qs7C6NgRvjzIypA,3720
225
234
  tests/test_large_scale_config.py,sha256=wyeMOMjGYhbZ6mrcnLH3Eh6GfspJwhavwWoyOy1y90c,4184
226
235
  tests/test_large_scale_helper.py,sha256=spvL0MPyXMAUDpzI2fY6-OQdSxOHtgJ1yuSUIbydyHY,8136
236
+ tests/test_logging_system.py,sha256=_LRdgprZFrChA26JJgkjVyf6S6qRIyi6BRajK13l_Q8,8924
227
237
  tests/test_mode_change.py,sha256=kh5C4ut7T5dZ8b2dDot4RbLWMXJidv4FHzuTIgDxMBI,2605
228
238
  tests/test_mode_consistency.py,sha256=YJXf0SqAYVnFXy8eeBLC-zGTFAyO2fnsR4qLB76gZts,1225
229
239
  tests/test_offsite_middleware.py,sha256=L5YT9ZqcQwBunUv0Ddj-sLZcW4IMlAlgaJCwICHFWxI,7543
@@ -271,8 +281,8 @@ tests/verify_distributed.py,sha256=krnYYA5Qx9xXDMWc9YF5DxPSplGvawDg2n0l-3CAqoM,3
271
281
  tests/verify_log_fix.py,sha256=TD7M1R22NxLqQPufvgE-H33u9tUjyz-rSR2ayIXozRU,4225
272
282
  tests/scrapy_comparison/ofweek_scrapy.py,sha256=2Hvpi6DRTubUxBy6RyJApQxMQONPLc1zWjKTQO_i5U4,5652
273
283
  tests/scrapy_comparison/scrapy_test.py,sha256=5sw7jOHhaTmQ8bsUd1TiolAUTRQYQOe-f49HPfysqbI,5466
274
- crawlo-1.3.4.dist-info/METADATA,sha256=MaE6HSo6UIybOmQrY0SUMnUXvPAdmclmsPtcWAil3bY,29742
275
- crawlo-1.3.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
276
- crawlo-1.3.4.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
277
- crawlo-1.3.4.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
278
- crawlo-1.3.4.dist-info/RECORD,,
284
+ crawlo-1.3.5.dist-info/METADATA,sha256=RDqsfzstoxtnrY_AR26XXQ3MbZA-W7sdCx4_CPReUGo,29742
285
+ crawlo-1.3.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
286
+ crawlo-1.3.5.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
287
+ crawlo-1.3.5.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
288
+ crawlo-1.3.5.dist-info/RECORD,,
@@ -0,0 +1,127 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: UTF-8 -*-
3
+ """
4
+ 调试日志配置问题
5
+ """
6
+
7
+ import os
8
+ import sys
9
+ from pathlib import Path
10
+
11
+ # 添加项目根目录到Python路径
12
+ project_root = Path(__file__).parent.parent
13
+ sys.path.insert(0, str(project_root))
14
+
15
+ from crawlo.logging import configure_logging as configure, get_logger, LogManager
16
+ from crawlo.logging.config import LogConfig
17
+
18
+
19
+ def debug_log_configuration():
20
+ """调试日志配置"""
21
+ print("=== 调试日志配置 ===")
22
+
23
+ # 重置配置
24
+ LogManager().reset()
25
+
26
+ # 1. 检查初始状态
27
+ print("1. 检查初始状态...")
28
+ print(f" 初始配置状态: {LogManager().is_configured}")
29
+ if LogManager().config:
30
+ print(f" 初始配置: {LogManager().config}")
31
+
32
+ # 2. 配置日志系统
33
+ print("2. 配置日志系统...")
34
+ config = configure(
35
+ LOG_LEVEL='DEBUG',
36
+ LOG_FILE='debug_test.log',
37
+ LOG_MAX_BYTES=1024,
38
+ LOG_BACKUP_COUNT=2,
39
+ LOG_CONSOLE_ENABLED=True,
40
+ LOG_FILE_ENABLED=True
41
+ )
42
+
43
+ print(f" 配置返回值: {config}")
44
+ print(f" 配置类型: {type(config)}")
45
+ print(f" 配置级别: {config.level}")
46
+ print(f" 配置文件路径: {config.file_path}")
47
+ print(f" 轮转大小: {config.max_bytes}")
48
+ print(f" 备份数量: {config.backup_count}")
49
+ print(f" 控制台启用: {config.console_enabled}")
50
+ print(f" 文件启用: {config.file_enabled}")
51
+
52
+ # 3. 检查管理器状态
53
+ print("3. 检查管理器状态...")
54
+ manager = LogManager()
55
+ print(f" 管理器配置状态: {manager.is_configured}")
56
+ if manager.config:
57
+ print(f" 管理器配置: {manager.config}")
58
+ print(f" 管理器配置文件路径: {manager.config.file_path}")
59
+
60
+ # 4. 测试Logger创建
61
+ print("4. 测试Logger创建...")
62
+ logger = get_logger('test.debug')
63
+ print(f" Logger handlers数量: {len(logger.handlers)}")
64
+
65
+ for i, handler in enumerate(logger.handlers):
66
+ handler_type = type(handler).__name__
67
+ print(f" Handler {i}: {handler_type}")
68
+ if hasattr(handler, 'baseFilename'):
69
+ print(f" 文件名: {handler.baseFilename}")
70
+
71
+ # 5. 测试日志输出
72
+ print("5. 测试日志输出...")
73
+ logger.info("调试测试消息")
74
+
75
+
76
+ def test_config_from_dict():
77
+ """测试从字典创建配置"""
78
+ print("\n=== 测试从字典创建配置 ===")
79
+
80
+ LogManager().reset()
81
+
82
+ # 使用字典配置
83
+ config_dict = {
84
+ 'level': 'DEBUG',
85
+ 'file_path': 'dict_test.log',
86
+ 'max_bytes': 1024,
87
+ 'backup_count': 2,
88
+ 'console_enabled': True,
89
+ 'file_enabled': True
90
+ }
91
+
92
+ config = LogConfig.from_dict(config_dict)
93
+ print(f" 字典配置: {config}")
94
+ print(f" 验证结果: {config.validate()}")
95
+
96
+ # 应用配置
97
+ LogManager().configure(config)
98
+
99
+ logger = get_logger('test.dict')
100
+ print(f" Logger handlers数量: {len(logger.handlers)}")
101
+
102
+ for i, handler in enumerate(logger.handlers):
103
+ handler_type = type(handler).__name__
104
+ print(f" Handler {i}: {handler_type}")
105
+
106
+
107
+ def main():
108
+ """主函数"""
109
+ print("开始调试日志配置问题...")
110
+
111
+ try:
112
+ debug_log_configuration()
113
+ test_config_from_dict()
114
+
115
+ print("\n=== 调试完成 ===")
116
+
117
+ except Exception as e:
118
+ print(f"\n调试过程中出现错误: {e}")
119
+ import traceback
120
+ traceback.print_exc()
121
+ return 1
122
+
123
+ return 0
124
+
125
+
126
+ if __name__ == '__main__':
127
+ sys.exit(main())
@@ -0,0 +1,234 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: UTF-8 -*-
3
+ """
4
+ 详细的日志系统功能测试
5
+ """
6
+
7
+ import os
8
+ import sys
9
+ import tempfile
10
+ import shutil
11
+ from pathlib import Path
12
+
13
+ # 添加项目根目录到Python路径
14
+ project_root = Path(__file__).parent.parent
15
+ sys.path.insert(0, str(project_root))
16
+
17
+ from crawlo.logging import configure_logging as configure, get_logger, LogManager
18
+ from crawlo.logging.config import LogConfig
19
+
20
+
21
+ def test_log_config_creation():
22
+ """测试日志配置创建"""
23
+ print("=== 测试日志配置创建 ===")
24
+
25
+ # 重置日志管理器
26
+ LogManager().reset()
27
+
28
+ # 1. 测试通过关键字参数创建配置
29
+ print("1. 测试通过关键字参数创建配置...")
30
+ config = configure(
31
+ LOG_LEVEL='DEBUG',
32
+ LOG_FILE='test.log',
33
+ LOG_MAX_BYTES=1024,
34
+ LOG_BACKUP_COUNT=3
35
+ )
36
+
37
+ print(f" 配置级别: {config.level}")
38
+ print(f" 配置文件路径: {config.file_path}")
39
+ print(f" 轮转大小: {config.max_bytes}")
40
+ print(f" 备份数量: {config.backup_count}")
41
+
42
+ # 2. 测试通过字典创建配置
43
+ print("2. 测试通过字典创建配置...")
44
+ LogManager().reset()
45
+ config_dict = {
46
+ 'LOG_LEVEL': 'WARNING',
47
+ 'LOG_FILE': 'dict_test.log',
48
+ 'LOG_CONSOLE_ENABLED': False
49
+ }
50
+ config = configure(**config_dict)
51
+
52
+ print(f" 配置级别: {config.level}")
53
+ print(f" 配置文件路径: {config.file_path}")
54
+ print(f" 控制台启用: {config.console_enabled}")
55
+
56
+
57
+ def test_logger_factory():
58
+ """测试Logger工厂"""
59
+ print("\n=== 测试Logger工厂 ===")
60
+
61
+ # 重置并配置
62
+ LogManager().reset()
63
+ configure(LOG_LEVEL='INFO')
64
+
65
+ # 1. 测试获取Logger
66
+ print("1. 测试获取Logger...")
67
+ logger1 = get_logger('test.factory1')
68
+ logger2 = get_logger('test.factory2')
69
+ logger3 = get_logger('test.factory1') # 应该是同一个实例
70
+
71
+ print(f" Logger1 ID: {id(logger1)}")
72
+ print(f" Logger2 ID: {id(logger2)}")
73
+ print(f" Logger3 ID: {id(logger3)}")
74
+ print(f" Logger1和Logger3是同一对象: {logger1 is logger3}")
75
+
76
+ # 2. 测试Logger配置
77
+ print("2. 测试Logger配置...")
78
+ print(f" Logger1名称: {logger1.name}")
79
+ print(f" Logger1 handlers数量: {len(logger1.handlers)}")
80
+
81
+ for i, handler in enumerate(logger1.handlers):
82
+ print(f" Handler {i}: {type(handler).__name__}")
83
+
84
+
85
+ def test_file_and_console_handlers():
86
+ """测试文件和控制台处理器"""
87
+ print("\n=== 测试文件和控制台处理器 ===")
88
+
89
+ # 创建临时目录
90
+ temp_dir = tempfile.mkdtemp()
91
+ log_file = os.path.join(temp_dir, 'handler_test.log')
92
+
93
+ try:
94
+ # 1. 测试文件和控制台都启用
95
+ print("1. 测试文件和控制台都启用...")
96
+ LogManager().reset()
97
+ configure(
98
+ LOG_LEVEL='INFO',
99
+ LOG_FILE=log_file,
100
+ LOG_CONSOLE_ENABLED=True,
101
+ LOG_FILE_ENABLED=True
102
+ )
103
+
104
+ logger = get_logger('test.handlers')
105
+ print(f" Handlers数量: {len(logger.handlers)}")
106
+
107
+ has_file_handler = False
108
+ has_console_handler = False
109
+
110
+ for handler in logger.handlers:
111
+ handler_type = type(handler).__name__
112
+ print(f" Handler类型: {handler_type}")
113
+ if 'FileHandler' in handler_type:
114
+ has_file_handler = True
115
+ print(f" 文件路径: {getattr(handler, 'baseFilename', 'N/A')}")
116
+ elif 'StreamHandler' in handler_type:
117
+ has_console_handler = True
118
+
119
+ print(f" 有文件处理器: {has_file_handler}")
120
+ print(f" 有控制台处理器: {has_console_handler}")
121
+
122
+ # 输出日志
123
+ logger.info("测试文件和控制台处理器")
124
+
125
+ # 检查文件是否存在
126
+ if os.path.exists(log_file):
127
+ with open(log_file, 'r', encoding='utf-8') as f:
128
+ content = f.read()
129
+ print(f" 文件内容行数: {len(content.splitlines())}")
130
+ else:
131
+ print(" 文件不存在!")
132
+
133
+ finally:
134
+ # 清理
135
+ shutil.rmtree(temp_dir, ignore_errors=True)
136
+
137
+
138
+ def test_log_levels():
139
+ """测试日志级别"""
140
+ print("\n=== 测试日志级别 ===")
141
+
142
+ # 1. 测试默认级别
143
+ print("1. 测试默认级别...")
144
+ LogManager().reset()
145
+ configure(LOG_LEVEL='WARNING')
146
+
147
+ logger = get_logger('test.levels')
148
+ logger.debug("DEBUG消息 - 不应该显示")
149
+ logger.info("INFO消息 - 不应该显示")
150
+ logger.warning("WARNING消息 - 应该显示")
151
+ logger.error("ERROR消息 - 应该显示")
152
+
153
+ # 2. 测试模块特定级别
154
+ print("2. 测试模块特定级别...")
155
+ LogManager().reset()
156
+ configure(
157
+ LOG_LEVEL='ERROR',
158
+ LOG_LEVELS={
159
+ 'test.debug_module': 'DEBUG',
160
+ 'test.info_module': 'INFO'
161
+ }
162
+ )
163
+
164
+ # 默认模块(ERROR级别)
165
+ default_logger = get_logger('test.default')
166
+ default_logger.info("默认模块INFO消息 - 不应该显示")
167
+ default_logger.error("默认模块ERROR消息 - 应该显示")
168
+
169
+ # DEBUG模块(DEBUG级别)
170
+ debug_logger = get_logger('test.debug_module')
171
+ debug_logger.debug("DEBUG模块DEBUG消息 - 应该显示")
172
+ debug_logger.info("DEBUG模块INFO消息 - 应该显示")
173
+
174
+ # INFO模块(INFO级别)
175
+ info_logger = get_logger('test.info_module')
176
+ info_logger.debug("INFO模块DEBUG消息 - 不应该显示")
177
+ info_logger.info("INFO模块INFO消息 - 应该显示")
178
+
179
+
180
+ def test_log_config_validation():
181
+ """测试日志配置验证"""
182
+ print("\n=== 测试日志配置验证 ===")
183
+
184
+ # 1. 测试有效配置
185
+ print("1. 测试有效配置...")
186
+ valid_config = LogConfig(level='INFO', file_path='valid.log')
187
+ is_valid = valid_config.validate()
188
+ print(f" 有效配置验证: {is_valid}")
189
+
190
+ # 2. 测试无效级别
191
+ print("2. 测试无效级别...")
192
+ invalid_config = LogConfig(level='INVALID', file_path='invalid.log')
193
+ is_valid = invalid_config.validate()
194
+ print(f" 无效级别验证: {is_valid}")
195
+
196
+ # 3. 测试目录创建
197
+ print("3. 测试目录创建...")
198
+ temp_dir = tempfile.mkdtemp()
199
+ nested_path = os.path.join(temp_dir, 'subdir', 'nested.log')
200
+
201
+ nested_config = LogConfig(level='INFO', file_path=nested_path)
202
+ is_valid = nested_config.validate()
203
+ print(f" 嵌套目录验证: {is_valid}")
204
+ print(f" 目录存在: {os.path.exists(os.path.dirname(nested_path))}")
205
+
206
+ # 清理
207
+ shutil.rmtree(temp_dir, ignore_errors=True)
208
+
209
+
210
+ def main():
211
+ """主测试函数"""
212
+ print("开始详细测试Crawlo框架日志系统...")
213
+
214
+ try:
215
+ # 运行所有测试
216
+ test_log_config_creation()
217
+ test_logger_factory()
218
+ test_file_and_console_handlers()
219
+ test_log_levels()
220
+ test_log_config_validation()
221
+
222
+ print("\n=== 详细测试完成 ===")
223
+
224
+ except Exception as e:
225
+ print(f"\n测试过程中出现错误: {e}")
226
+ import traceback
227
+ traceback.print_exc()
228
+ return 1
229
+
230
+ return 0
231
+
232
+
233
+ if __name__ == '__main__':
234
+ sys.exit(main())