crawlo 1.2.0__py3-none-any.whl → 1.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of crawlo might be problematic. Click here for more details.
- crawlo/__init__.py +61 -61
- crawlo/__version__.py +1 -1
- crawlo/cleaners/__init__.py +60 -60
- crawlo/cleaners/data_formatter.py +225 -225
- crawlo/cleaners/encoding_converter.py +125 -125
- crawlo/cleaners/text_cleaner.py +232 -232
- crawlo/cli.py +81 -65
- crawlo/commands/__init__.py +14 -14
- crawlo/commands/check.py +594 -594
- crawlo/commands/genspider.py +151 -151
- crawlo/commands/help.py +143 -133
- crawlo/commands/list.py +155 -155
- crawlo/commands/run.py +292 -292
- crawlo/commands/startproject.py +418 -418
- crawlo/commands/stats.py +188 -188
- crawlo/commands/utils.py +186 -186
- crawlo/config.py +312 -312
- crawlo/config_validator.py +252 -252
- crawlo/core/__init__.py +2 -2
- crawlo/core/engine.py +354 -354
- crawlo/core/processor.py +40 -40
- crawlo/core/scheduler.py +143 -143
- crawlo/crawler.py +1027 -1027
- crawlo/downloader/__init__.py +266 -266
- crawlo/downloader/aiohttp_downloader.py +220 -220
- crawlo/downloader/cffi_downloader.py +256 -256
- crawlo/downloader/httpx_downloader.py +259 -259
- crawlo/downloader/hybrid_downloader.py +213 -213
- crawlo/downloader/playwright_downloader.py +402 -402
- crawlo/downloader/selenium_downloader.py +472 -472
- crawlo/event.py +11 -11
- crawlo/exceptions.py +81 -81
- crawlo/extension/__init__.py +37 -37
- crawlo/extension/health_check.py +141 -141
- crawlo/extension/log_interval.py +57 -57
- crawlo/extension/log_stats.py +81 -81
- crawlo/extension/logging_extension.py +43 -43
- crawlo/extension/memory_monitor.py +104 -104
- crawlo/extension/performance_profiler.py +133 -133
- crawlo/extension/request_recorder.py +107 -107
- crawlo/filters/__init__.py +154 -154
- crawlo/filters/aioredis_filter.py +280 -280
- crawlo/filters/memory_filter.py +269 -269
- crawlo/items/__init__.py +23 -23
- crawlo/items/base.py +21 -21
- crawlo/items/fields.py +53 -53
- crawlo/items/items.py +104 -104
- crawlo/middleware/__init__.py +21 -21
- crawlo/middleware/default_header.py +132 -32
- crawlo/middleware/download_delay.py +105 -28
- crawlo/middleware/middleware_manager.py +135 -135
- crawlo/middleware/offsite.py +116 -0
- crawlo/middleware/proxy.py +366 -272
- crawlo/middleware/request_ignore.py +88 -30
- crawlo/middleware/response_code.py +164 -18
- crawlo/middleware/response_filter.py +138 -26
- crawlo/middleware/retry.py +124 -124
- crawlo/mode_manager.py +211 -211
- crawlo/network/__init__.py +21 -21
- crawlo/network/request.py +338 -338
- crawlo/network/response.py +359 -359
- crawlo/pipelines/__init__.py +21 -21
- crawlo/pipelines/bloom_dedup_pipeline.py +156 -156
- crawlo/pipelines/console_pipeline.py +39 -39
- crawlo/pipelines/csv_pipeline.py +316 -316
- crawlo/pipelines/database_dedup_pipeline.py +224 -224
- crawlo/pipelines/json_pipeline.py +218 -218
- crawlo/pipelines/memory_dedup_pipeline.py +115 -115
- crawlo/pipelines/mongo_pipeline.py +131 -131
- crawlo/pipelines/mysql_pipeline.py +316 -316
- crawlo/pipelines/pipeline_manager.py +61 -61
- crawlo/pipelines/redis_dedup_pipeline.py +167 -167
- crawlo/project.py +187 -187
- crawlo/queue/pqueue.py +37 -37
- crawlo/queue/queue_manager.py +337 -337
- crawlo/queue/redis_priority_queue.py +298 -298
- crawlo/settings/__init__.py +7 -7
- crawlo/settings/default_settings.py +226 -219
- crawlo/settings/setting_manager.py +122 -122
- crawlo/spider/__init__.py +639 -639
- crawlo/stats_collector.py +59 -59
- crawlo/subscriber.py +130 -130
- crawlo/task_manager.py +30 -30
- crawlo/templates/crawlo.cfg.tmpl +10 -10
- crawlo/templates/project/__init__.py.tmpl +3 -3
- crawlo/templates/project/items.py.tmpl +17 -17
- crawlo/templates/project/middlewares.py.tmpl +118 -109
- crawlo/templates/project/pipelines.py.tmpl +96 -96
- crawlo/templates/project/run.py.tmpl +45 -45
- crawlo/templates/project/settings.py.tmpl +327 -326
- crawlo/templates/project/settings_distributed.py.tmpl +119 -119
- crawlo/templates/project/settings_gentle.py.tmpl +94 -94
- crawlo/templates/project/settings_high_performance.py.tmpl +151 -151
- crawlo/templates/project/settings_simple.py.tmpl +68 -68
- crawlo/templates/project/spiders/__init__.py.tmpl +5 -5
- crawlo/templates/spider/spider.py.tmpl +143 -141
- crawlo/tools/__init__.py +182 -182
- crawlo/tools/anti_crawler.py +268 -268
- crawlo/tools/authenticated_proxy.py +240 -240
- crawlo/tools/data_validator.py +180 -180
- crawlo/tools/date_tools.py +35 -35
- crawlo/tools/distributed_coordinator.py +386 -386
- crawlo/tools/retry_mechanism.py +220 -220
- crawlo/tools/scenario_adapter.py +262 -262
- crawlo/utils/__init__.py +35 -35
- crawlo/utils/batch_processor.py +260 -260
- crawlo/utils/controlled_spider_mixin.py +439 -439
- crawlo/utils/date_tools.py +290 -290
- crawlo/utils/db_helper.py +343 -343
- crawlo/utils/enhanced_error_handler.py +359 -359
- crawlo/utils/env_config.py +105 -105
- crawlo/utils/error_handler.py +125 -125
- crawlo/utils/func_tools.py +82 -82
- crawlo/utils/large_scale_config.py +286 -286
- crawlo/utils/large_scale_helper.py +343 -343
- crawlo/utils/log.py +128 -128
- crawlo/utils/performance_monitor.py +284 -284
- crawlo/utils/queue_helper.py +175 -175
- crawlo/utils/redis_connection_pool.py +334 -334
- crawlo/utils/redis_key_validator.py +199 -199
- crawlo/utils/request.py +267 -267
- crawlo/utils/request_serializer.py +219 -219
- crawlo/utils/spider_loader.py +62 -62
- crawlo/utils/system.py +11 -11
- crawlo/utils/tools.py +4 -4
- crawlo/utils/url.py +39 -39
- {crawlo-1.2.0.dist-info → crawlo-1.2.2.dist-info}/METADATA +692 -697
- crawlo-1.2.2.dist-info/RECORD +220 -0
- examples/__init__.py +7 -7
- examples/aiohttp_settings.py +42 -0
- examples/curl_cffi_settings.py +41 -0
- examples/default_header_middleware_example.py +107 -0
- examples/default_header_spider_example.py +129 -0
- examples/download_delay_middleware_example.py +160 -0
- examples/httpx_settings.py +42 -0
- examples/multi_downloader_proxy_example.py +81 -0
- examples/offsite_middleware_example.py +55 -0
- examples/offsite_spider_example.py +107 -0
- examples/proxy_spider_example.py +166 -0
- examples/request_ignore_middleware_example.py +51 -0
- examples/request_ignore_spider_example.py +99 -0
- examples/response_code_middleware_example.py +52 -0
- examples/response_filter_middleware_example.py +67 -0
- examples/tong_hua_shun_settings.py +62 -0
- examples/tong_hua_shun_spider.py +170 -0
- tests/DOUBLE_CRAWLO_PREFIX_FIX_REPORT.md +81 -81
- tests/__init__.py +7 -7
- tests/advanced_tools_example.py +275 -275
- tests/authenticated_proxy_example.py +236 -236
- tests/cleaners_example.py +160 -160
- tests/config_validation_demo.py +102 -102
- tests/controlled_spider_example.py +205 -205
- tests/date_tools_example.py +180 -180
- tests/dynamic_loading_example.py +523 -523
- tests/dynamic_loading_test.py +104 -104
- tests/env_config_example.py +133 -133
- tests/error_handling_example.py +171 -171
- tests/redis_key_validation_demo.py +130 -130
- tests/response_improvements_example.py +144 -144
- tests/test_advanced_tools.py +148 -148
- tests/test_all_redis_key_configs.py +145 -145
- tests/test_authenticated_proxy.py +141 -141
- tests/test_cleaners.py +54 -54
- tests/test_comprehensive.py +146 -146
- tests/test_config_validator.py +193 -193
- tests/test_crawlo_proxy_integration.py +173 -0
- tests/test_date_tools.py +123 -123
- tests/test_default_header_middleware.py +159 -0
- tests/test_double_crawlo_fix.py +207 -207
- tests/test_double_crawlo_fix_simple.py +124 -124
- tests/test_download_delay_middleware.py +222 -0
- tests/test_downloader_proxy_compatibility.py +269 -0
- tests/test_dynamic_downloaders_proxy.py +124 -124
- tests/test_dynamic_proxy.py +92 -92
- tests/test_dynamic_proxy_config.py +146 -146
- tests/test_dynamic_proxy_real.py +109 -109
- tests/test_edge_cases.py +303 -303
- tests/test_enhanced_error_handler.py +270 -270
- tests/test_env_config.py +121 -121
- tests/test_error_handler_compatibility.py +112 -112
- tests/test_final_validation.py +153 -153
- tests/test_framework_env_usage.py +103 -103
- tests/test_integration.py +356 -356
- tests/test_item_dedup_redis_key.py +122 -122
- tests/test_offsite_middleware.py +222 -0
- tests/test_parsel.py +29 -29
- tests/test_performance.py +327 -327
- tests/test_proxy_api.py +265 -0
- tests/test_proxy_health_check.py +32 -32
- tests/test_proxy_middleware.py +122 -0
- tests/test_proxy_middleware_enhanced.py +217 -0
- tests/test_proxy_middleware_integration.py +136 -136
- tests/test_proxy_providers.py +56 -56
- tests/test_proxy_stats.py +19 -19
- tests/test_proxy_strategies.py +59 -59
- tests/test_queue_manager_double_crawlo.py +173 -173
- tests/test_queue_manager_redis_key.py +176 -176
- tests/test_real_scenario_proxy.py +196 -0
- tests/test_redis_config.py +28 -28
- tests/test_redis_connection_pool.py +294 -294
- tests/test_redis_key_naming.py +181 -181
- tests/test_redis_key_validator.py +123 -123
- tests/test_redis_queue.py +224 -224
- tests/test_request_ignore_middleware.py +183 -0
- tests/test_request_serialization.py +70 -70
- tests/test_response_code_middleware.py +350 -0
- tests/test_response_filter_middleware.py +428 -0
- tests/test_response_improvements.py +152 -152
- tests/test_retry_middleware.py +242 -0
- tests/test_scheduler.py +241 -241
- tests/test_simple_response.py +61 -61
- tests/test_telecom_spider_redis_key.py +205 -205
- tests/test_template_content.py +87 -87
- tests/test_template_redis_key.py +134 -134
- tests/test_tools.py +153 -153
- tests/tools_example.py +257 -257
- crawlo-1.2.0.dist-info/RECORD +0 -190
- {crawlo-1.2.0.dist-info → crawlo-1.2.2.dist-info}/WHEEL +0 -0
- {crawlo-1.2.0.dist-info → crawlo-1.2.2.dist-info}/entry_points.txt +0 -0
- {crawlo-1.2.0.dist-info → crawlo-1.2.2.dist-info}/top_level.txt +0 -0
tests/date_tools_example.py
CHANGED
|
@@ -1,181 +1,181 @@
|
|
|
1
|
-
#!/usr/bin/python
|
|
2
|
-
# -*- coding: UTF-8 -*-
|
|
3
|
-
"""
|
|
4
|
-
Crawlo框架日期工具使用示例
|
|
5
|
-
"""
|
|
6
|
-
from crawlo.tools import (
|
|
7
|
-
TimeUtils,
|
|
8
|
-
parse_time,
|
|
9
|
-
format_time,
|
|
10
|
-
time_diff,
|
|
11
|
-
to_timestamp,
|
|
12
|
-
to_datetime,
|
|
13
|
-
now,
|
|
14
|
-
to_timezone,
|
|
15
|
-
to_utc,
|
|
16
|
-
to_local
|
|
17
|
-
)
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def demo_date_parsing():
|
|
21
|
-
"""演示日期解析功能"""
|
|
22
|
-
print("=== 日期解析演示 ===\n")
|
|
23
|
-
|
|
24
|
-
# 解析各种格式的日期字符串
|
|
25
|
-
date_strings = [
|
|
26
|
-
"2025-09-10 14:30:00",
|
|
27
|
-
"September 10, 2025 2:30 PM",
|
|
28
|
-
"2025/09/10 14:30:00",
|
|
29
|
-
"10-09-2025 14:30:00",
|
|
30
|
-
"2025年9月10日 14时30分00秒"
|
|
31
|
-
]
|
|
32
|
-
|
|
33
|
-
for date_str in date_strings:
|
|
34
|
-
parsed = parse_time(date_str)
|
|
35
|
-
print(f"解析 '{date_str}' -> {parsed}")
|
|
36
|
-
|
|
37
|
-
print()
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def demo_date_formatting():
|
|
41
|
-
"""演示日期格式化功能"""
|
|
42
|
-
print("=== 日期格式化演示 ===\n")
|
|
43
|
-
|
|
44
|
-
# 获取当前时间
|
|
45
|
-
current_time = now()
|
|
46
|
-
print(f"当前时间: {current_time}")
|
|
47
|
-
|
|
48
|
-
# 使用不同格式进行格式化
|
|
49
|
-
formats = [
|
|
50
|
-
"%Y-%m-%d",
|
|
51
|
-
"%Y年%m月%d日",
|
|
52
|
-
"%Y-%m-%d %H:%M:%S",
|
|
53
|
-
"%B %d, %Y at %I:%M %p",
|
|
54
|
-
"%A, %B %d, %Y"
|
|
55
|
-
]
|
|
56
|
-
|
|
57
|
-
for fmt in formats:
|
|
58
|
-
formatted = format_time(current_time, fmt)
|
|
59
|
-
print(f"格式化为 '{fmt}' -> {formatted}")
|
|
60
|
-
|
|
61
|
-
print()
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
def demo_time_calculations():
|
|
65
|
-
"""演示时间计算功能"""
|
|
66
|
-
print("=== 时间计算演示 ===\n")
|
|
67
|
-
|
|
68
|
-
# 计算时间差
|
|
69
|
-
start_time = "2025-09-10 10:00:00"
|
|
70
|
-
end_time = "2025-09-10 14:30:45"
|
|
71
|
-
|
|
72
|
-
diff_seconds = time_diff(start_time, end_time, "seconds")
|
|
73
|
-
diff_minutes = time_diff(start_time, end_time, "minutes")
|
|
74
|
-
diff_hours = time_diff(start_time, end_time, "hours")
|
|
75
|
-
|
|
76
|
-
print(f"起始时间: {start_time}")
|
|
77
|
-
print(f"结束时间: {end_time}")
|
|
78
|
-
print(f"时间差 - 秒: {diff_seconds}")
|
|
79
|
-
print(f"时间差 - 分钟: {diff_minutes}")
|
|
80
|
-
print(f"时间差 - 小时: {diff_hours}")
|
|
81
|
-
|
|
82
|
-
print()
|
|
83
|
-
|
|
84
|
-
# 日期加减
|
|
85
|
-
base_date = "2025-09-10"
|
|
86
|
-
plus_30_days = TimeUtils.add_days(base_date, 30)
|
|
87
|
-
minus_15_days = TimeUtils.add_days(base_date, -15)
|
|
88
|
-
plus_3_months = TimeUtils.add_months(base_date, 3)
|
|
89
|
-
|
|
90
|
-
print(f"基础日期: {base_date}")
|
|
91
|
-
print(f"加30天: {plus_30_days}")
|
|
92
|
-
print(f"减15天: {minus_15_days}")
|
|
93
|
-
print(f"加3个月: {plus_3_months}")
|
|
94
|
-
|
|
95
|
-
print()
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def demo_timezone_conversion():
|
|
99
|
-
"""演示时区转换功能"""
|
|
100
|
-
print("=== 时区转换演示 ===\n")
|
|
101
|
-
|
|
102
|
-
# 创建一个时间
|
|
103
|
-
dt = parse_time("2025-09-10 14:30:00")
|
|
104
|
-
print(f"原始时间: {dt}")
|
|
105
|
-
|
|
106
|
-
# 转换为UTC时区
|
|
107
|
-
utc_time = to_utc(dt)
|
|
108
|
-
print(f"UTC时间: {utc_time}")
|
|
109
|
-
|
|
110
|
-
# 转换为本地时区
|
|
111
|
-
local_time = to_local(dt)
|
|
112
|
-
print(f"本地时间: {local_time}")
|
|
113
|
-
|
|
114
|
-
# 转换为纽约时区
|
|
115
|
-
ny_time = to_timezone(dt, "America/New_York")
|
|
116
|
-
print(f"纽约时间: {ny_time}")
|
|
117
|
-
|
|
118
|
-
# 转换为伦敦时区
|
|
119
|
-
london_time = to_timezone(dt, "Europe/London")
|
|
120
|
-
print(f"伦敦时间: {london_time}")
|
|
121
|
-
|
|
122
|
-
print()
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
def demo_timestamp_conversion():
|
|
126
|
-
"""演示时间戳转换功能"""
|
|
127
|
-
print("=== 时间戳转换演示 ===\n")
|
|
128
|
-
|
|
129
|
-
# 获取当前时间戳
|
|
130
|
-
current_timestamp = to_timestamp(now())
|
|
131
|
-
print(f"当前时间戳: {current_timestamp}")
|
|
132
|
-
|
|
133
|
-
# 从时间戳转换为datetime
|
|
134
|
-
dt_from_timestamp = to_datetime(current_timestamp)
|
|
135
|
-
print(f"从时间戳转换: {dt_from_timestamp}")
|
|
136
|
-
|
|
137
|
-
# 从时间戳创建带时区的datetime
|
|
138
|
-
dt_with_tz = to_timezone(to_datetime(current_timestamp), "Asia/Shanghai")
|
|
139
|
-
print(f"带时区的时间: {dt_with_tz}")
|
|
140
|
-
|
|
141
|
-
print()
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
def demo_in_spider():
|
|
145
|
-
"""演示在爬虫中使用日期工具"""
|
|
146
|
-
print("=== 在爬虫中使用日期工具 ===\n")
|
|
147
|
-
print("在爬虫项目中,您可以这样使用日期工具:")
|
|
148
|
-
print("""
|
|
149
|
-
from crawlo import Spider
|
|
150
|
-
from crawlo.tools import parse_time, format_time
|
|
151
|
-
|
|
152
|
-
class NewsSpider(Spider):
|
|
153
|
-
def parse(self, response):
|
|
154
|
-
# 提取文章发布时间
|
|
155
|
-
publish_time_str = response.css('.publish-time::text').get()
|
|
156
|
-
|
|
157
|
-
# 解析发布时间
|
|
158
|
-
publish_time = parse_time(publish_time_str)
|
|
159
|
-
|
|
160
|
-
# 格式化时间为标准格式
|
|
161
|
-
formatted_time = format_time(publish_time, "%Y-%m-%d %H:%M:%S")
|
|
162
|
-
|
|
163
|
-
# 计算文章发布多久了
|
|
164
|
-
current_time = self.tools.now()
|
|
165
|
-
hours_since_publish = self.tools.time_diff(publish_time, current_time, "hours")
|
|
166
|
-
|
|
167
|
-
# 根据发布时间决定是否继续处理
|
|
168
|
-
if hours_since_publish < 24: # 只处理24小时内的文章
|
|
169
|
-
# 处理文章...
|
|
170
|
-
pass
|
|
171
|
-
""")
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if __name__ == '__main__':
|
|
175
|
-
# 运行演示
|
|
176
|
-
demo_date_parsing()
|
|
177
|
-
demo_date_formatting()
|
|
178
|
-
demo_time_calculations()
|
|
179
|
-
demo_timezone_conversion()
|
|
180
|
-
demo_timestamp_conversion()
|
|
1
|
+
#!/usr/bin/python
|
|
2
|
+
# -*- coding: UTF-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Crawlo框架日期工具使用示例
|
|
5
|
+
"""
|
|
6
|
+
from crawlo.tools import (
|
|
7
|
+
TimeUtils,
|
|
8
|
+
parse_time,
|
|
9
|
+
format_time,
|
|
10
|
+
time_diff,
|
|
11
|
+
to_timestamp,
|
|
12
|
+
to_datetime,
|
|
13
|
+
now,
|
|
14
|
+
to_timezone,
|
|
15
|
+
to_utc,
|
|
16
|
+
to_local
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def demo_date_parsing():
|
|
21
|
+
"""演示日期解析功能"""
|
|
22
|
+
print("=== 日期解析演示 ===\n")
|
|
23
|
+
|
|
24
|
+
# 解析各种格式的日期字符串
|
|
25
|
+
date_strings = [
|
|
26
|
+
"2025-09-10 14:30:00",
|
|
27
|
+
"September 10, 2025 2:30 PM",
|
|
28
|
+
"2025/09/10 14:30:00",
|
|
29
|
+
"10-09-2025 14:30:00",
|
|
30
|
+
"2025年9月10日 14时30分00秒"
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
for date_str in date_strings:
|
|
34
|
+
parsed = parse_time(date_str)
|
|
35
|
+
print(f"解析 '{date_str}' -> {parsed}")
|
|
36
|
+
|
|
37
|
+
print()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def demo_date_formatting():
|
|
41
|
+
"""演示日期格式化功能"""
|
|
42
|
+
print("=== 日期格式化演示 ===\n")
|
|
43
|
+
|
|
44
|
+
# 获取当前时间
|
|
45
|
+
current_time = now()
|
|
46
|
+
print(f"当前时间: {current_time}")
|
|
47
|
+
|
|
48
|
+
# 使用不同格式进行格式化
|
|
49
|
+
formats = [
|
|
50
|
+
"%Y-%m-%d",
|
|
51
|
+
"%Y年%m月%d日",
|
|
52
|
+
"%Y-%m-%d %H:%M:%S",
|
|
53
|
+
"%B %d, %Y at %I:%M %p",
|
|
54
|
+
"%A, %B %d, %Y"
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
for fmt in formats:
|
|
58
|
+
formatted = format_time(current_time, fmt)
|
|
59
|
+
print(f"格式化为 '{fmt}' -> {formatted}")
|
|
60
|
+
|
|
61
|
+
print()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def demo_time_calculations():
|
|
65
|
+
"""演示时间计算功能"""
|
|
66
|
+
print("=== 时间计算演示 ===\n")
|
|
67
|
+
|
|
68
|
+
# 计算时间差
|
|
69
|
+
start_time = "2025-09-10 10:00:00"
|
|
70
|
+
end_time = "2025-09-10 14:30:45"
|
|
71
|
+
|
|
72
|
+
diff_seconds = time_diff(start_time, end_time, "seconds")
|
|
73
|
+
diff_minutes = time_diff(start_time, end_time, "minutes")
|
|
74
|
+
diff_hours = time_diff(start_time, end_time, "hours")
|
|
75
|
+
|
|
76
|
+
print(f"起始时间: {start_time}")
|
|
77
|
+
print(f"结束时间: {end_time}")
|
|
78
|
+
print(f"时间差 - 秒: {diff_seconds}")
|
|
79
|
+
print(f"时间差 - 分钟: {diff_minutes}")
|
|
80
|
+
print(f"时间差 - 小时: {diff_hours}")
|
|
81
|
+
|
|
82
|
+
print()
|
|
83
|
+
|
|
84
|
+
# 日期加减
|
|
85
|
+
base_date = "2025-09-10"
|
|
86
|
+
plus_30_days = TimeUtils.add_days(base_date, 30)
|
|
87
|
+
minus_15_days = TimeUtils.add_days(base_date, -15)
|
|
88
|
+
plus_3_months = TimeUtils.add_months(base_date, 3)
|
|
89
|
+
|
|
90
|
+
print(f"基础日期: {base_date}")
|
|
91
|
+
print(f"加30天: {plus_30_days}")
|
|
92
|
+
print(f"减15天: {minus_15_days}")
|
|
93
|
+
print(f"加3个月: {plus_3_months}")
|
|
94
|
+
|
|
95
|
+
print()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def demo_timezone_conversion():
|
|
99
|
+
"""演示时区转换功能"""
|
|
100
|
+
print("=== 时区转换演示 ===\n")
|
|
101
|
+
|
|
102
|
+
# 创建一个时间
|
|
103
|
+
dt = parse_time("2025-09-10 14:30:00")
|
|
104
|
+
print(f"原始时间: {dt}")
|
|
105
|
+
|
|
106
|
+
# 转换为UTC时区
|
|
107
|
+
utc_time = to_utc(dt)
|
|
108
|
+
print(f"UTC时间: {utc_time}")
|
|
109
|
+
|
|
110
|
+
# 转换为本地时区
|
|
111
|
+
local_time = to_local(dt)
|
|
112
|
+
print(f"本地时间: {local_time}")
|
|
113
|
+
|
|
114
|
+
# 转换为纽约时区
|
|
115
|
+
ny_time = to_timezone(dt, "America/New_York")
|
|
116
|
+
print(f"纽约时间: {ny_time}")
|
|
117
|
+
|
|
118
|
+
# 转换为伦敦时区
|
|
119
|
+
london_time = to_timezone(dt, "Europe/London")
|
|
120
|
+
print(f"伦敦时间: {london_time}")
|
|
121
|
+
|
|
122
|
+
print()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def demo_timestamp_conversion():
|
|
126
|
+
"""演示时间戳转换功能"""
|
|
127
|
+
print("=== 时间戳转换演示 ===\n")
|
|
128
|
+
|
|
129
|
+
# 获取当前时间戳
|
|
130
|
+
current_timestamp = to_timestamp(now())
|
|
131
|
+
print(f"当前时间戳: {current_timestamp}")
|
|
132
|
+
|
|
133
|
+
# 从时间戳转换为datetime
|
|
134
|
+
dt_from_timestamp = to_datetime(current_timestamp)
|
|
135
|
+
print(f"从时间戳转换: {dt_from_timestamp}")
|
|
136
|
+
|
|
137
|
+
# 从时间戳创建带时区的datetime
|
|
138
|
+
dt_with_tz = to_timezone(to_datetime(current_timestamp), "Asia/Shanghai")
|
|
139
|
+
print(f"带时区的时间: {dt_with_tz}")
|
|
140
|
+
|
|
141
|
+
print()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def demo_in_spider():
|
|
145
|
+
"""演示在爬虫中使用日期工具"""
|
|
146
|
+
print("=== 在爬虫中使用日期工具 ===\n")
|
|
147
|
+
print("在爬虫项目中,您可以这样使用日期工具:")
|
|
148
|
+
print("""
|
|
149
|
+
from crawlo import Spider
|
|
150
|
+
from crawlo.tools import parse_time, format_time
|
|
151
|
+
|
|
152
|
+
class NewsSpider(Spider):
|
|
153
|
+
def parse(self, response):
|
|
154
|
+
# 提取文章发布时间
|
|
155
|
+
publish_time_str = response.css('.publish-time::text').get()
|
|
156
|
+
|
|
157
|
+
# 解析发布时间
|
|
158
|
+
publish_time = parse_time(publish_time_str)
|
|
159
|
+
|
|
160
|
+
# 格式化时间为标准格式
|
|
161
|
+
formatted_time = format_time(publish_time, "%Y-%m-%d %H:%M:%S")
|
|
162
|
+
|
|
163
|
+
# 计算文章发布多久了
|
|
164
|
+
current_time = self.tools.now()
|
|
165
|
+
hours_since_publish = self.tools.time_diff(publish_time, current_time, "hours")
|
|
166
|
+
|
|
167
|
+
# 根据发布时间决定是否继续处理
|
|
168
|
+
if hours_since_publish < 24: # 只处理24小时内的文章
|
|
169
|
+
# 处理文章...
|
|
170
|
+
pass
|
|
171
|
+
""")
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
if __name__ == '__main__':
|
|
175
|
+
# 运行演示
|
|
176
|
+
demo_date_parsing()
|
|
177
|
+
demo_date_formatting()
|
|
178
|
+
demo_time_calculations()
|
|
179
|
+
demo_timezone_conversion()
|
|
180
|
+
demo_timestamp_conversion()
|
|
181
181
|
demo_in_spider()
|