crawlo 1.4.7__py3-none-any.whl → 1.4.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of crawlo might be problematic. Click here for more details.
- crawlo/__init__.py +90 -90
- crawlo/__version__.py +1 -1
- crawlo/cli.py +75 -75
- crawlo/commands/__init__.py +14 -14
- crawlo/commands/check.py +594 -594
- crawlo/commands/genspider.py +186 -186
- crawlo/commands/help.py +140 -140
- crawlo/commands/list.py +155 -155
- crawlo/commands/run.py +379 -379
- crawlo/commands/startproject.py +460 -460
- crawlo/commands/stats.py +187 -187
- crawlo/commands/utils.py +196 -196
- crawlo/config.py +320 -320
- crawlo/config_validator.py +277 -277
- crawlo/core/__init__.py +52 -52
- crawlo/core/engine.py +451 -451
- crawlo/core/processor.py +47 -47
- crawlo/core/scheduler.py +290 -290
- crawlo/crawler.py +698 -698
- crawlo/data/__init__.py +5 -5
- crawlo/data/user_agents.py +194 -194
- crawlo/downloader/__init__.py +280 -280
- crawlo/downloader/aiohttp_downloader.py +233 -233
- crawlo/downloader/cffi_downloader.py +250 -250
- crawlo/downloader/httpx_downloader.py +265 -265
- crawlo/downloader/hybrid_downloader.py +212 -212
- crawlo/downloader/playwright_downloader.py +425 -425
- crawlo/downloader/selenium_downloader.py +486 -486
- crawlo/event.py +45 -45
- crawlo/exceptions.py +214 -214
- crawlo/extension/__init__.py +64 -64
- crawlo/extension/health_check.py +141 -141
- crawlo/extension/log_interval.py +94 -94
- crawlo/extension/log_stats.py +70 -70
- crawlo/extension/logging_extension.py +53 -53
- crawlo/extension/memory_monitor.py +104 -104
- crawlo/extension/performance_profiler.py +133 -133
- crawlo/extension/request_recorder.py +107 -107
- crawlo/factories/__init__.py +27 -27
- crawlo/factories/base.py +68 -68
- crawlo/factories/crawler.py +104 -104
- crawlo/factories/registry.py +84 -84
- crawlo/factories/utils.py +134 -134
- crawlo/filters/__init__.py +170 -170
- crawlo/filters/aioredis_filter.py +347 -347
- crawlo/filters/memory_filter.py +261 -261
- crawlo/framework.py +306 -306
- crawlo/initialization/__init__.py +44 -44
- crawlo/initialization/built_in.py +391 -391
- crawlo/initialization/context.py +141 -141
- crawlo/initialization/core.py +240 -240
- crawlo/initialization/phases.py +229 -229
- crawlo/initialization/registry.py +143 -143
- crawlo/initialization/utils.py +48 -48
- crawlo/interfaces.py +23 -23
- crawlo/items/__init__.py +23 -23
- crawlo/items/base.py +23 -23
- crawlo/items/fields.py +52 -52
- crawlo/items/items.py +104 -104
- crawlo/logging/__init__.py +42 -42
- crawlo/logging/config.py +280 -276
- crawlo/logging/factory.py +175 -175
- crawlo/logging/manager.py +104 -104
- crawlo/middleware/__init__.py +87 -87
- crawlo/middleware/default_header.py +132 -132
- crawlo/middleware/download_delay.py +104 -104
- crawlo/middleware/middleware_manager.py +142 -142
- crawlo/middleware/offsite.py +123 -123
- crawlo/middleware/proxy.py +209 -209
- crawlo/middleware/request_ignore.py +86 -86
- crawlo/middleware/response_code.py +150 -150
- crawlo/middleware/response_filter.py +136 -136
- crawlo/middleware/retry.py +124 -124
- crawlo/mode_manager.py +287 -287
- crawlo/network/__init__.py +21 -21
- crawlo/network/request.py +408 -376
- crawlo/network/response.py +598 -569
- crawlo/pipelines/__init__.py +52 -52
- crawlo/pipelines/base_pipeline.py +452 -452
- crawlo/pipelines/bloom_dedup_pipeline.py +145 -146
- crawlo/pipelines/console_pipeline.py +39 -39
- crawlo/pipelines/csv_pipeline.py +316 -316
- crawlo/pipelines/database_dedup_pipeline.py +196 -197
- crawlo/pipelines/json_pipeline.py +218 -218
- crawlo/pipelines/memory_dedup_pipeline.py +104 -105
- crawlo/pipelines/mongo_pipeline.py +140 -139
- crawlo/pipelines/mysql_pipeline.py +468 -469
- crawlo/pipelines/pipeline_manager.py +100 -100
- crawlo/pipelines/redis_dedup_pipeline.py +155 -155
- crawlo/project.py +347 -347
- crawlo/queue/__init__.py +9 -9
- crawlo/queue/pqueue.py +38 -38
- crawlo/queue/queue_manager.py +591 -591
- crawlo/queue/redis_priority_queue.py +518 -518
- crawlo/settings/__init__.py +7 -7
- crawlo/settings/default_settings.py +287 -284
- crawlo/settings/setting_manager.py +219 -219
- crawlo/spider/__init__.py +658 -657
- crawlo/stats_collector.py +81 -81
- crawlo/subscriber.py +129 -129
- crawlo/task_manager.py +138 -138
- crawlo/templates/crawlo.cfg.tmpl +10 -10
- crawlo/templates/project/__init__.py.tmpl +1 -1
- crawlo/templates/project/items.py.tmpl +13 -13
- crawlo/templates/project/middlewares.py.tmpl +38 -38
- crawlo/templates/project/pipelines.py.tmpl +35 -35
- crawlo/templates/project/settings.py.tmpl +113 -109
- crawlo/templates/project/settings_distributed.py.tmpl +160 -156
- crawlo/templates/project/settings_gentle.py.tmpl +174 -170
- crawlo/templates/project/settings_high_performance.py.tmpl +175 -171
- crawlo/templates/project/settings_minimal.py.tmpl +102 -98
- crawlo/templates/project/settings_simple.py.tmpl +172 -168
- crawlo/templates/project/spiders/__init__.py.tmpl +9 -9
- crawlo/templates/run.py.tmpl +23 -23
- crawlo/templates/spider/spider.py.tmpl +32 -32
- crawlo/templates/spiders_init.py.tmpl +4 -4
- crawlo/tools/__init__.py +86 -86
- crawlo/tools/date_tools.py +289 -289
- crawlo/tools/distributed_coordinator.py +384 -384
- crawlo/tools/scenario_adapter.py +262 -262
- crawlo/tools/text_cleaner.py +232 -232
- crawlo/utils/__init__.py +74 -50
- crawlo/utils/batch_processor.py +276 -276
- crawlo/utils/config_manager.py +442 -442
- crawlo/utils/controlled_spider_mixin.py +439 -439
- crawlo/utils/db_helper.py +250 -250
- crawlo/utils/encoding_helper.py +190 -0
- crawlo/utils/error_handler.py +410 -410
- crawlo/utils/fingerprint.py +121 -121
- crawlo/utils/func_tools.py +82 -82
- crawlo/utils/large_scale_helper.py +344 -344
- crawlo/utils/leak_detector.py +335 -335
- crawlo/utils/misc.py +81 -81
- crawlo/utils/mongo_connection_pool.py +157 -157
- crawlo/utils/mysql_connection_pool.py +197 -197
- crawlo/utils/performance_monitor.py +285 -285
- crawlo/utils/queue_helper.py +175 -175
- crawlo/utils/redis_checker.py +90 -90
- crawlo/utils/redis_connection_pool.py +578 -578
- crawlo/utils/redis_key_validator.py +198 -198
- crawlo/utils/request.py +278 -278
- crawlo/utils/request_serializer.py +225 -225
- crawlo/utils/resource_manager.py +337 -337
- crawlo/utils/response_helper.py +113 -0
- crawlo/utils/selector_helper.py +138 -137
- crawlo/utils/singleton.py +69 -69
- crawlo/utils/spider_loader.py +201 -201
- crawlo/utils/text_helper.py +94 -94
- {crawlo-1.4.7.dist-info → crawlo-1.4.8.dist-info}/METADATA +831 -689
- crawlo-1.4.8.dist-info/RECORD +347 -0
- examples/__init__.py +7 -7
- tests/__init__.py +7 -7
- tests/advanced_tools_example.py +217 -217
- tests/authenticated_proxy_example.py +110 -110
- tests/baidu_performance_test.py +108 -108
- tests/baidu_test.py +59 -59
- tests/bug_check_test.py +250 -250
- tests/cleaners_example.py +160 -160
- tests/comprehensive_framework_test.py +212 -212
- tests/comprehensive_test.py +81 -81
- tests/comprehensive_testing_summary.md +186 -186
- tests/config_validation_demo.py +142 -142
- tests/controlled_spider_example.py +205 -205
- tests/date_tools_example.py +180 -180
- tests/debug_configure.py +69 -69
- tests/debug_framework_logger.py +84 -84
- tests/debug_log_config.py +126 -126
- tests/debug_log_levels.py +63 -63
- tests/debug_pipelines.py +66 -66
- tests/detailed_log_test.py +233 -233
- tests/direct_selector_helper_test.py +96 -96
- tests/distributed_dedup_test.py +467 -467
- tests/distributed_test.py +66 -66
- tests/distributed_test_debug.py +76 -76
- tests/dynamic_loading_example.py +523 -523
- tests/dynamic_loading_test.py +104 -104
- tests/error_handling_example.py +171 -171
- tests/explain_mysql_update_behavior.py +76 -76
- tests/final_comprehensive_test.py +151 -151
- tests/final_log_test.py +260 -260
- tests/final_validation_test.py +182 -182
- tests/fix_log_test.py +142 -142
- tests/framework_performance_test.py +202 -202
- tests/log_buffering_test.py +111 -111
- tests/log_generation_timing_test.py +153 -153
- tests/monitor_redis_dedup.sh +72 -72
- tests/ofweek_scrapy/ofweek_scrapy/items.py +12 -12
- tests/ofweek_scrapy/ofweek_scrapy/middlewares.py +100 -100
- tests/ofweek_scrapy/ofweek_scrapy/pipelines.py +13 -13
- tests/ofweek_scrapy/ofweek_scrapy/settings.py +84 -84
- tests/ofweek_scrapy/ofweek_scrapy/spiders/__init__.py +4 -4
- tests/ofweek_scrapy/scrapy.cfg +11 -11
- tests/optimized_performance_test.py +211 -211
- tests/performance_comparison.py +244 -244
- tests/queue_blocking_test.py +113 -113
- tests/queue_test.py +89 -89
- tests/redis_key_validation_demo.py +130 -130
- tests/request_params_example.py +150 -150
- tests/response_improvements_example.py +144 -144
- tests/scrapy_comparison/ofweek_scrapy.py +138 -138
- tests/scrapy_comparison/scrapy_test.py +133 -133
- tests/simple_cli_test.py +54 -54
- tests/simple_command_test.py +119 -119
- tests/simple_crawlo_test.py +126 -126
- tests/simple_follow_test.py +38 -38
- tests/simple_log_test2.py +137 -137
- tests/simple_optimization_test.py +128 -128
- tests/simple_queue_type_test.py +41 -41
- tests/simple_response_selector_test.py +94 -94
- tests/simple_selector_helper_test.py +154 -154
- tests/simple_selector_test.py +207 -207
- tests/simple_spider_test.py +49 -49
- tests/simple_url_test.py +73 -73
- tests/simulate_mysql_update_test.py +139 -139
- tests/spider_log_timing_test.py +177 -177
- tests/test_advanced_tools.py +148 -148
- tests/test_all_commands.py +230 -230
- tests/test_all_pipeline_fingerprints.py +133 -133
- tests/test_all_redis_key_configs.py +145 -145
- tests/test_asyncmy_usage.py +56 -56
- tests/test_batch_processor.py +178 -178
- tests/test_cleaners.py +54 -54
- tests/test_cli_arguments.py +118 -118
- tests/test_component_factory.py +174 -174
- tests/test_config_consistency.py +80 -80
- tests/test_config_merge.py +152 -152
- tests/test_config_validator.py +182 -182
- tests/test_controlled_spider_mixin.py +79 -79
- tests/test_crawler_process_import.py +38 -38
- tests/test_crawler_process_spider_modules.py +47 -47
- tests/test_crawlo_proxy_integration.py +114 -114
- tests/test_date_tools.py +123 -123
- tests/test_dedup_fix.py +220 -220
- tests/test_dedup_pipeline_consistency.py +124 -124
- tests/test_default_header_middleware.py +313 -313
- tests/test_distributed.py +65 -65
- tests/test_double_crawlo_fix.py +204 -204
- tests/test_double_crawlo_fix_simple.py +124 -124
- tests/test_download_delay_middleware.py +221 -221
- tests/test_downloader_proxy_compatibility.py +272 -272
- tests/test_edge_cases.py +305 -305
- tests/test_encoding_core.py +56 -56
- tests/test_encoding_detection.py +126 -126
- tests/test_enhanced_error_handler.py +270 -270
- tests/test_enhanced_error_handler_comprehensive.py +245 -245
- tests/test_error_handler_compatibility.py +112 -112
- tests/test_factories.py +252 -252
- tests/test_factory_compatibility.py +196 -196
- tests/test_final_validation.py +153 -153
- tests/test_fingerprint_consistency.py +135 -135
- tests/test_fingerprint_simple.py +51 -51
- tests/test_get_component_logger.py +83 -83
- tests/test_hash_performance.py +99 -99
- tests/test_integration.py +169 -169
- tests/test_item_dedup_redis_key.py +122 -122
- tests/test_large_scale_helper.py +235 -235
- tests/test_logging_enhancements.py +374 -374
- tests/test_logging_final.py +184 -184
- tests/test_logging_integration.py +312 -312
- tests/test_logging_system.py +282 -282
- tests/test_middleware_debug.py +141 -141
- tests/test_mode_consistency.py +51 -51
- tests/test_multi_directory.py +67 -67
- tests/test_multiple_spider_modules.py +80 -80
- tests/test_mysql_pipeline_config.py +164 -164
- tests/test_mysql_pipeline_error.py +98 -98
- tests/test_mysql_pipeline_init_log.py +82 -82
- tests/test_mysql_pipeline_integration.py +132 -132
- tests/test_mysql_pipeline_refactor.py +143 -143
- tests/test_mysql_pipeline_refactor_simple.py +85 -85
- tests/test_mysql_pipeline_robustness.py +195 -195
- tests/test_mysql_pipeline_types.py +88 -88
- tests/test_mysql_update_columns.py +93 -93
- tests/test_offsite_middleware.py +244 -244
- tests/test_offsite_middleware_simple.py +203 -203
- tests/test_optimized_selector_naming.py +100 -100
- tests/test_parsel.py +29 -29
- tests/test_performance.py +327 -327
- tests/test_performance_monitor.py +115 -115
- tests/test_pipeline_fingerprint_consistency.py +86 -86
- tests/test_priority_behavior.py +211 -211
- tests/test_priority_consistency.py +151 -151
- tests/test_priority_consistency_fixed.py +249 -249
- tests/test_proxy_health_check.py +32 -32
- tests/test_proxy_middleware.py +217 -217
- tests/test_proxy_middleware_enhanced.py +212 -212
- tests/test_proxy_middleware_integration.py +142 -142
- tests/test_proxy_middleware_refactored.py +207 -207
- tests/test_proxy_only.py +83 -83
- tests/test_proxy_providers.py +56 -56
- tests/test_proxy_stats.py +19 -19
- tests/test_proxy_strategies.py +59 -59
- tests/test_proxy_with_downloader.py +152 -152
- tests/test_queue_empty_check.py +41 -41
- tests/test_queue_manager_double_crawlo.py +173 -173
- tests/test_queue_manager_redis_key.py +179 -179
- tests/test_queue_naming.py +154 -154
- tests/test_queue_type.py +106 -106
- tests/test_queue_type_redis_config_consistency.py +130 -130
- tests/test_random_headers_default.py +322 -322
- tests/test_random_headers_necessity.py +308 -308
- tests/test_random_user_agent.py +72 -72
- tests/test_redis_config.py +28 -28
- tests/test_redis_connection_pool.py +294 -294
- tests/test_redis_key_naming.py +181 -181
- tests/test_redis_key_validator.py +123 -123
- tests/test_redis_queue.py +224 -224
- tests/test_redis_queue_name_fix.py +175 -175
- tests/test_redis_queue_type_fallback.py +129 -129
- tests/test_request_ignore_middleware.py +182 -182
- tests/test_request_params.py +111 -111
- tests/test_request_serialization.py +70 -70
- tests/test_response_code_middleware.py +349 -349
- tests/test_response_filter_middleware.py +427 -427
- tests/test_response_follow.py +104 -104
- tests/test_response_improvements.py +152 -152
- tests/test_response_selector_methods.py +92 -92
- tests/test_response_url_methods.py +70 -70
- tests/test_response_urljoin.py +86 -86
- tests/test_retry_middleware.py +333 -333
- tests/test_retry_middleware_realistic.py +273 -273
- tests/test_scheduler.py +252 -252
- tests/test_scheduler_config_update.py +133 -133
- tests/test_scrapy_style_encoding.py +112 -112
- tests/test_selector_helper.py +100 -100
- tests/test_selector_optimizations.py +146 -146
- tests/test_simple_response.py +61 -61
- tests/test_spider_loader.py +49 -49
- tests/test_spider_loader_comprehensive.py +69 -69
- tests/test_spider_modules.py +84 -84
- tests/test_spiders/test_spider.py +9 -9
- tests/test_telecom_spider_redis_key.py +205 -205
- tests/test_template_content.py +87 -87
- tests/test_template_redis_key.py +134 -134
- tests/test_tools.py +159 -159
- tests/test_user_agent_randomness.py +176 -176
- tests/test_user_agents.py +96 -96
- tests/untested_features_report.md +138 -138
- tests/verify_debug.py +51 -51
- tests/verify_distributed.py +117 -117
- tests/verify_log_fix.py +111 -111
- tests/verify_mysql_warnings.py +109 -109
- crawlo/utils/log.py +0 -80
- crawlo/utils/url_utils.py +0 -40
- crawlo-1.4.7.dist-info/RECORD +0 -347
- {crawlo-1.4.7.dist-info → crawlo-1.4.8.dist-info}/WHEEL +0 -0
- {crawlo-1.4.7.dist-info → crawlo-1.4.8.dist-info}/entry_points.txt +0 -0
- {crawlo-1.4.7.dist-info → crawlo-1.4.8.dist-info}/top_level.txt +0 -0
crawlo/items/items.py
CHANGED
|
@@ -1,105 +1,105 @@
|
|
|
1
|
-
#!/usr/bin/python
|
|
2
|
-
# -*- coding: UTF-8 -*-
|
|
3
|
-
"""
|
|
4
|
-
Item 类定义
|
|
5
|
-
"""
|
|
6
|
-
from copy import deepcopy
|
|
7
|
-
from pprint import pformat
|
|
8
|
-
from typing import Any, Iterator, Dict
|
|
9
|
-
from collections.abc import MutableMapping
|
|
10
|
-
|
|
11
|
-
from .base import ItemMeta
|
|
12
|
-
from crawlo.exceptions import ItemInitError, ItemAttributeError
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
class Item(MutableMapping, metaclass=ItemMeta):
|
|
16
|
-
"""
|
|
17
|
-
数据项基类,用于定义结构化数据
|
|
18
|
-
"""
|
|
19
|
-
FIELDS: Dict[str, Any] = {}
|
|
20
|
-
|
|
21
|
-
def __init__(self, *args, **kwargs):
|
|
22
|
-
if args:
|
|
23
|
-
raise ItemInitError(f"{self.__class__.__name__} 不支持位置参数:{args},请使用关键字参数初始化。")
|
|
24
|
-
|
|
25
|
-
self._values: Dict[str, Any] = {}
|
|
26
|
-
|
|
27
|
-
# 初始化字段,默认值填充
|
|
28
|
-
for field_name, field_obj in self.FIELDS.items():
|
|
29
|
-
if field_obj.default is not None:
|
|
30
|
-
self._values[field_name] = field_obj.default
|
|
31
|
-
|
|
32
|
-
# 覆盖默认值或设置新值
|
|
33
|
-
for key, value in kwargs.items():
|
|
34
|
-
self[key] = value
|
|
35
|
-
|
|
36
|
-
def __getitem__(self, item: str) -> Any:
|
|
37
|
-
return self._values[item]
|
|
38
|
-
|
|
39
|
-
def __setitem__(self, key: str, value: Any) -> None:
|
|
40
|
-
if key not in self.FIELDS:
|
|
41
|
-
raise KeyError(f"{self.__class__.__name__} 不包含字段:{key}")
|
|
42
|
-
|
|
43
|
-
field = self.FIELDS[key]
|
|
44
|
-
try:
|
|
45
|
-
validated_value = field.validate(value, field_name=key)
|
|
46
|
-
self._values[key] = validated_value
|
|
47
|
-
except Exception as e:
|
|
48
|
-
error_lines = [
|
|
49
|
-
"",
|
|
50
|
-
"【字段校验失败】",
|
|
51
|
-
f"字段名称: {key}",
|
|
52
|
-
f"数据类型: {type(value)}",
|
|
53
|
-
f"原始值: {repr(value)}",
|
|
54
|
-
f"是否允许空值: {field.nullable}",
|
|
55
|
-
f"错误原因: {str(e)}",
|
|
56
|
-
""
|
|
57
|
-
]
|
|
58
|
-
detailed_error = "\n".join(error_lines)
|
|
59
|
-
raise type(e)(detailed_error) from e
|
|
60
|
-
|
|
61
|
-
def __delitem__(self, key: str) -> None:
|
|
62
|
-
del self._values[key]
|
|
63
|
-
|
|
64
|
-
def __setattr__(self, key: str, value: Any) -> None:
|
|
65
|
-
if not key.startswith("_"):
|
|
66
|
-
raise AttributeError(
|
|
67
|
-
f"设置字段值请使用 item[{key!r}] = {value!r}"
|
|
68
|
-
)
|
|
69
|
-
super().__setattr__(key, value)
|
|
70
|
-
|
|
71
|
-
def __getattr__(self, item: str) -> Any:
|
|
72
|
-
raise AttributeError(
|
|
73
|
-
f"{self.__class__.__name__} 不支持字段:{item}。"
|
|
74
|
-
f"请先在 `{self.__class__.__name__}` 中声明该字段,再通过 item[{item!r}] 获取。"
|
|
75
|
-
)
|
|
76
|
-
|
|
77
|
-
def __getattribute__(self, item: str) -> Any:
|
|
78
|
-
try:
|
|
79
|
-
field = super().__getattribute__("FIELDS")
|
|
80
|
-
if isinstance(field, dict) and item in field:
|
|
81
|
-
raise ItemAttributeError(
|
|
82
|
-
f"获取字段值请使用 item[{item!r}]"
|
|
83
|
-
)
|
|
84
|
-
except AttributeError:
|
|
85
|
-
pass # 如果 FIELDS 尚未定义,继续执行后续逻辑
|
|
86
|
-
return super().__getattribute__(item)
|
|
87
|
-
|
|
88
|
-
def __repr__(self) -> str:
|
|
89
|
-
return pformat(dict(self))
|
|
90
|
-
|
|
91
|
-
__str__ = __repr__
|
|
92
|
-
|
|
93
|
-
def __iter__(self) -> Iterator[str]:
|
|
94
|
-
return iter(self._values)
|
|
95
|
-
|
|
96
|
-
def __len__(self) -> int:
|
|
97
|
-
return len(self._values)
|
|
98
|
-
|
|
99
|
-
def to_dict(self) -> Dict[str, Any]:
|
|
100
|
-
"""转换为字典"""
|
|
101
|
-
return dict(self)
|
|
102
|
-
|
|
103
|
-
def copy(self) -> "Item":
|
|
104
|
-
"""深拷贝当前 Item"""
|
|
1
|
+
#!/usr/bin/python
|
|
2
|
+
# -*- coding: UTF-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Item 类定义
|
|
5
|
+
"""
|
|
6
|
+
from copy import deepcopy
|
|
7
|
+
from pprint import pformat
|
|
8
|
+
from typing import Any, Iterator, Dict
|
|
9
|
+
from collections.abc import MutableMapping
|
|
10
|
+
|
|
11
|
+
from .base import ItemMeta
|
|
12
|
+
from crawlo.exceptions import ItemInitError, ItemAttributeError
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Item(MutableMapping, metaclass=ItemMeta):
|
|
16
|
+
"""
|
|
17
|
+
数据项基类,用于定义结构化数据
|
|
18
|
+
"""
|
|
19
|
+
FIELDS: Dict[str, Any] = {}
|
|
20
|
+
|
|
21
|
+
def __init__(self, *args, **kwargs):
|
|
22
|
+
if args:
|
|
23
|
+
raise ItemInitError(f"{self.__class__.__name__} 不支持位置参数:{args},请使用关键字参数初始化。")
|
|
24
|
+
|
|
25
|
+
self._values: Dict[str, Any] = {}
|
|
26
|
+
|
|
27
|
+
# 初始化字段,默认值填充
|
|
28
|
+
for field_name, field_obj in self.FIELDS.items():
|
|
29
|
+
if field_obj.default is not None:
|
|
30
|
+
self._values[field_name] = field_obj.default
|
|
31
|
+
|
|
32
|
+
# 覆盖默认值或设置新值
|
|
33
|
+
for key, value in kwargs.items():
|
|
34
|
+
self[key] = value
|
|
35
|
+
|
|
36
|
+
def __getitem__(self, item: str) -> Any:
|
|
37
|
+
return self._values[item]
|
|
38
|
+
|
|
39
|
+
def __setitem__(self, key: str, value: Any) -> None:
|
|
40
|
+
if key not in self.FIELDS:
|
|
41
|
+
raise KeyError(f"{self.__class__.__name__} 不包含字段:{key}")
|
|
42
|
+
|
|
43
|
+
field = self.FIELDS[key]
|
|
44
|
+
try:
|
|
45
|
+
validated_value = field.validate(value, field_name=key)
|
|
46
|
+
self._values[key] = validated_value
|
|
47
|
+
except Exception as e:
|
|
48
|
+
error_lines = [
|
|
49
|
+
"",
|
|
50
|
+
"【字段校验失败】",
|
|
51
|
+
f"字段名称: {key}",
|
|
52
|
+
f"数据类型: {type(value)}",
|
|
53
|
+
f"原始值: {repr(value)}",
|
|
54
|
+
f"是否允许空值: {field.nullable}",
|
|
55
|
+
f"错误原因: {str(e)}",
|
|
56
|
+
""
|
|
57
|
+
]
|
|
58
|
+
detailed_error = "\n".join(error_lines)
|
|
59
|
+
raise type(e)(detailed_error) from e
|
|
60
|
+
|
|
61
|
+
def __delitem__(self, key: str) -> None:
|
|
62
|
+
del self._values[key]
|
|
63
|
+
|
|
64
|
+
def __setattr__(self, key: str, value: Any) -> None:
|
|
65
|
+
if not key.startswith("_"):
|
|
66
|
+
raise AttributeError(
|
|
67
|
+
f"设置字段值请使用 item[{key!r}] = {value!r}"
|
|
68
|
+
)
|
|
69
|
+
super().__setattr__(key, value)
|
|
70
|
+
|
|
71
|
+
def __getattr__(self, item: str) -> Any:
|
|
72
|
+
raise AttributeError(
|
|
73
|
+
f"{self.__class__.__name__} 不支持字段:{item}。"
|
|
74
|
+
f"请先在 `{self.__class__.__name__}` 中声明该字段,再通过 item[{item!r}] 获取。"
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
def __getattribute__(self, item: str) -> Any:
|
|
78
|
+
try:
|
|
79
|
+
field = super().__getattribute__("FIELDS")
|
|
80
|
+
if isinstance(field, dict) and item in field:
|
|
81
|
+
raise ItemAttributeError(
|
|
82
|
+
f"获取字段值请使用 item[{item!r}]"
|
|
83
|
+
)
|
|
84
|
+
except AttributeError:
|
|
85
|
+
pass # 如果 FIELDS 尚未定义,继续执行后续逻辑
|
|
86
|
+
return super().__getattribute__(item)
|
|
87
|
+
|
|
88
|
+
def __repr__(self) -> str:
|
|
89
|
+
return pformat(dict(self))
|
|
90
|
+
|
|
91
|
+
__str__ = __repr__
|
|
92
|
+
|
|
93
|
+
def __iter__(self) -> Iterator[str]:
|
|
94
|
+
return iter(self._values)
|
|
95
|
+
|
|
96
|
+
def __len__(self) -> int:
|
|
97
|
+
return len(self._values)
|
|
98
|
+
|
|
99
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
100
|
+
"""转换为字典"""
|
|
101
|
+
return dict(self)
|
|
102
|
+
|
|
103
|
+
def copy(self) -> "Item":
|
|
104
|
+
"""深拷贝当前 Item"""
|
|
105
105
|
return deepcopy(self)
|
crawlo/logging/__init__.py
CHANGED
|
@@ -1,42 +1,42 @@
|
|
|
1
|
-
#!/usr/bin/python
|
|
2
|
-
# -*- coding: UTF-8 -*-
|
|
3
|
-
"""
|
|
4
|
-
Crawlo统一日志系统
|
|
5
|
-
=================
|
|
6
|
-
|
|
7
|
-
设计原则:
|
|
8
|
-
1. 简单优先 - 避免过度设计
|
|
9
|
-
2. 性能优先 - 减少锁竞争和复杂逻辑
|
|
10
|
-
3. 一致性 - 统一的日志接口
|
|
11
|
-
4. 可靠性 - 确保日志始终可用
|
|
12
|
-
"""
|
|
13
|
-
|
|
14
|
-
from .manager import LogManager
|
|
15
|
-
from .factory import LoggerFactory
|
|
16
|
-
from .config import LogConfig
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
# 统一的公共接口
|
|
20
|
-
def get_logger(name: str = 'default'):
|
|
21
|
-
"""获取logger实例"""
|
|
22
|
-
return LoggerFactory.get_logger(name)
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
def configure_logging(settings=None, **kwargs):
|
|
26
|
-
"""配置日志系统"""
|
|
27
|
-
return LogManager().configure(settings, **kwargs)
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def is_configured() -> bool:
|
|
31
|
-
"""检查日志系统是否已配置"""
|
|
32
|
-
return LogManager().is_configured
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
__all__ = [
|
|
36
|
-
'LogManager',
|
|
37
|
-
'LoggerFactory',
|
|
38
|
-
'LogConfig',
|
|
39
|
-
'get_logger',
|
|
40
|
-
'configure_logging',
|
|
41
|
-
'is_configured'
|
|
42
|
-
]
|
|
1
|
+
#!/usr/bin/python
|
|
2
|
+
# -*- coding: UTF-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Crawlo统一日志系统
|
|
5
|
+
=================
|
|
6
|
+
|
|
7
|
+
设计原则:
|
|
8
|
+
1. 简单优先 - 避免过度设计
|
|
9
|
+
2. 性能优先 - 减少锁竞争和复杂逻辑
|
|
10
|
+
3. 一致性 - 统一的日志接口
|
|
11
|
+
4. 可靠性 - 确保日志始终可用
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from .manager import LogManager
|
|
15
|
+
from .factory import LoggerFactory
|
|
16
|
+
from .config import LogConfig
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# 统一的公共接口
|
|
20
|
+
def get_logger(name: str = 'default'):
|
|
21
|
+
"""获取logger实例"""
|
|
22
|
+
return LoggerFactory.get_logger(name)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def configure_logging(settings=None, **kwargs):
|
|
26
|
+
"""配置日志系统"""
|
|
27
|
+
return LogManager().configure(settings, **kwargs)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def is_configured() -> bool:
|
|
31
|
+
"""检查日志系统是否已配置"""
|
|
32
|
+
return LogManager().is_configured
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
'LogManager',
|
|
37
|
+
'LoggerFactory',
|
|
38
|
+
'LogConfig',
|
|
39
|
+
'get_logger',
|
|
40
|
+
'configure_logging',
|
|
41
|
+
'is_configured'
|
|
42
|
+
]
|