oafuncs 0.0.59__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oafuncs/__init__.py +26 -0
- oafuncs/oa_cmap.py +141 -0
- oafuncs/oa_data.py +205 -0
- oafuncs/oa_down/__init__.py +20 -0
- oafuncs/oa_down/hycom_3hourly.py +882 -0
- oafuncs/oa_down/refs_pdf.py +342 -0
- oafuncs/oa_draw.py +431 -0
- oafuncs/oa_file.py +297 -0
- oafuncs/oa_help.py +40 -0
- oafuncs/oa_nc.py +345 -0
- oafuncs/oa_python.py +106 -0
- oafuncs/oa_sign/__init__.py +21 -0
- oafuncs/oa_sign/meteorological.py +175 -0
- oafuncs/oa_sign/ocean.py +163 -0
- oafuncs/oa_sign/scientific.py +139 -0
- oafuncs/oa_tool/__init__.py +18 -0
- oafuncs/oa_tool/email.py +110 -0
- oafuncs-0.0.59.dist-info/LICENSE.txt +19 -0
- oafuncs-0.0.59.dist-info/METADATA +913 -0
- oafuncs-0.0.59.dist-info/RECORD +22 -0
- oafuncs-0.0.59.dist-info/WHEEL +6 -0
- oafuncs-0.0.59.dist-info/top_level.txt +1 -0
@@ -0,0 +1,882 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# coding=utf-8
|
3
|
+
'''
|
4
|
+
Author: Liu Kun && 16031215@qq.com
|
5
|
+
Date: 2024-11-01 10:31:09
|
6
|
+
LastEditors: Liu Kun && 16031215@qq.com
|
7
|
+
LastEditTime: 2024-11-12 16:22:33
|
8
|
+
FilePath: \\Python\\My_Funcs\\OAFuncs\\OAFuncs\\oa_down\\hycom_3hourly.py
|
9
|
+
Description:
|
10
|
+
EditPlatform: vscode
|
11
|
+
ComputerInfo: XPS 15 9510
|
12
|
+
SystemInfo: Windows 11
|
13
|
+
Python Version: 3.11
|
14
|
+
'''
|
15
|
+
import datetime
|
16
|
+
import os
|
17
|
+
import random
|
18
|
+
import time
|
19
|
+
import warnings
|
20
|
+
from concurrent.futures import ThreadPoolExecutor
|
21
|
+
from pathlib import Path
|
22
|
+
|
23
|
+
import matplotlib.pyplot as plt
|
24
|
+
import numpy as np
|
25
|
+
import pandas as pd
|
26
|
+
import requests
|
27
|
+
from rich import print
|
28
|
+
from rich.progress import Progress
|
29
|
+
|
30
|
+
warnings.filterwarnings("ignore", category=RuntimeWarning,
|
31
|
+
message="Engine '.*' loading failed:.*")
|
32
|
+
|
33
|
+
__all__ = ['draw_time_range', 'download', 'how_to_use', 'get_time_list']
|
34
|
+
|
35
|
+
# time resolution
|
36
|
+
data_info = {'yearly': {}, 'monthly': {}, 'daily': {}, 'hourly': {}}
|
37
|
+
|
38
|
+
# hourly data
|
39
|
+
# dataset: GLBv0.08, GLBu0.08, GLBy0.08
|
40
|
+
data_info['hourly']['dataset'] = {
|
41
|
+
'GLBv0.08': {}, 'GLBu0.08': {}, 'GLBy0.08': {}}
|
42
|
+
|
43
|
+
# version
|
44
|
+
# version of GLBv0.08: 53.X, 56.3, 57.2, 92.8, 57.7, 92.9, 93.0
|
45
|
+
data_info['hourly']['dataset']['GLBv0.08']['version'] = {
|
46
|
+
'53.X': {}, '56.3': {}, '57.2': {}, '92.8': {}, '57.7': {}, '92.9': {}, '93.0': {}}
|
47
|
+
# version of GLBu0.08: 93.0
|
48
|
+
data_info['hourly']['dataset']['GLBu0.08']['version'] = {'93.0': {}}
|
49
|
+
# version of GLBy0.08: 93.0
|
50
|
+
data_info['hourly']['dataset']['GLBy0.08']['version'] = {'93.0': {}}
|
51
|
+
|
52
|
+
# info details
|
53
|
+
# time range
|
54
|
+
# GLBv0.08
|
55
|
+
# 在网页上提交超过范围的时间,会返回该数据集实际时间范围,从而纠正下面的时间范围
|
56
|
+
# 目前只纠正了GLBv0.08 93.0的时间范围,具体到小时了
|
57
|
+
# 其他数据集的时刻暂时默认为00起,21止
|
58
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['53.X']['time_range'] = {
|
59
|
+
'time_start': '19940101', 'time_end': '20151230'}
|
60
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['56.3']['time_range'] = {
|
61
|
+
'time_start': '20140701', 'time_end': '20160430'}
|
62
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['57.2']['time_range'] = {
|
63
|
+
'time_start': '20160501', 'time_end': '20170131'}
|
64
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['92.8']['time_range'] = {
|
65
|
+
'time_start': '20170201', 'time_end': '20170531'}
|
66
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['57.7']['time_range'] = {
|
67
|
+
'time_start': '20170601', 'time_end': '20170930'}
|
68
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['92.9']['time_range'] = {
|
69
|
+
'time_start': '20171001', 'time_end': '20171231'}
|
70
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['93.0']['time_range'] = {
|
71
|
+
'time_start': '2018010112', 'time_end': '2020021909'}
|
72
|
+
# GLBu0.08
|
73
|
+
data_info['hourly']['dataset']['GLBu0.08']['version']['93.0']['time_range'] = {
|
74
|
+
'time_start': '20180919', 'time_end': '20181208'}
|
75
|
+
# GLBy0.08
|
76
|
+
data_info['hourly']['dataset']['GLBy0.08']['version']['93.0']['time_range'] = {
|
77
|
+
'time_start': '20181204', 'time_end': '20300904'}
|
78
|
+
|
79
|
+
# variable
|
80
|
+
variable_info = {
|
81
|
+
'u': {'var_name': 'water_u', 'standard_name': 'eastward_sea_water_velocity'},
|
82
|
+
'v': {'var_name': 'water_v', 'standard_name': 'northward_sea_water_velocity'},
|
83
|
+
'temp': {'var_name': 'water_temp', 'standard_name': 'sea_water_potential_temperature'},
|
84
|
+
'salinity': {'var_name': 'salinity', 'standard_name': 'sea_water_salinity'},
|
85
|
+
'ssh': {'var_name': 'surf_el', 'standard_name': 'sea_surface_elevation'},
|
86
|
+
'u_b': {'var_name': 'water_u_bottom', 'standard_name': 'eastward_sea_water_velocity_at_sea_floor'},
|
87
|
+
'v_b': {'var_name': 'water_v_bottom', 'standard_name': 'northward_sea_water_velocity_at_sea_floor'},
|
88
|
+
'temp_b': {'var_name': 'water_temp_bottom', 'standard_name': 'sea_water_potential_temperature_at_sea_floor'},
|
89
|
+
'salinity_b': {'var_name': 'salinity_bottom', 'standard_name': 'sea_water_salinity_at_sea_floor'},
|
90
|
+
}
|
91
|
+
|
92
|
+
# classification method
|
93
|
+
# year_different: the data of different years is stored in different files
|
94
|
+
# same_path: the data of different years is stored in the same file
|
95
|
+
# var_different: the data of different variables is stored in different files
|
96
|
+
# var_year_different: the data of different variables and years is stored in different files
|
97
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['53.X']['classification'] = 'year_different'
|
98
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['56.3']['classification'] = 'same_path'
|
99
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['57.2']['classification'] = 'same_path'
|
100
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['92.8']['classification'] = 'var_different'
|
101
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['57.7']['classification'] = 'same_path'
|
102
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['92.9']['classification'] = 'var_different'
|
103
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['93.0']['classification'] = 'var_different'
|
104
|
+
data_info['hourly']['dataset']['GLBu0.08']['version']['93.0']['classification'] = 'var_different'
|
105
|
+
data_info['hourly']['dataset']['GLBy0.08']['version']['93.0']['classification'] = 'var_year_different'
|
106
|
+
|
107
|
+
# download info
|
108
|
+
# base url
|
109
|
+
# GLBv0.08 53.X
|
110
|
+
url_53x = {}
|
111
|
+
for y_53x in range(1994, 2016):
|
112
|
+
# r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_53.X/data/2013?'
|
113
|
+
url_53x[str(
|
114
|
+
y_53x)] = rf'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_53.X/data/{y_53x}?'
|
115
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['53.X']['url'] = url_53x
|
116
|
+
# GLBv0.08 56.3
|
117
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['56.3']['url'] = r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_56.3?'
|
118
|
+
# GLBv0.08 57.2
|
119
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['57.2']['url'] = r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_57.2?'
|
120
|
+
# GLBv0.08 92.8
|
121
|
+
url_928 = {
|
122
|
+
'uv3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.8/uv3z?',
|
123
|
+
'ts3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.8/ts3z?',
|
124
|
+
'ssh': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.8/ssh?',
|
125
|
+
}
|
126
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['92.8']['url'] = url_928
|
127
|
+
# GLBv0.08 57.7
|
128
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['57.7']['url'] = r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_57.7?'
|
129
|
+
# GLBv0.08 92.9
|
130
|
+
url_929 = {
|
131
|
+
'uv3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.9/uv3z?',
|
132
|
+
'ts3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.9/ts3z?',
|
133
|
+
'ssh': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.9/ssh?',
|
134
|
+
}
|
135
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['92.9']['url'] = url_929
|
136
|
+
# GLBv0.08 93.0
|
137
|
+
url_930_v = {
|
138
|
+
'uv3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_93.0/uv3z?',
|
139
|
+
'ts3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_93.0/ts3z?',
|
140
|
+
'ssh': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_93.0/ssh?',
|
141
|
+
}
|
142
|
+
data_info['hourly']['dataset']['GLBv0.08']['version']['93.0']['url'] = url_930_v
|
143
|
+
# GLBu0.08 93.0
|
144
|
+
url_930_u = {
|
145
|
+
'uv3z': r'https://ncss.hycom.org/thredds/ncss/GLBu0.08/expt_93.0/uv3z?',
|
146
|
+
'ts3z': r'https://ncss.hycom.org/thredds/ncss/GLBu0.08/expt_93.0/ts3z?',
|
147
|
+
'ssh': r'https://ncss.hycom.org/thredds/ncss/GLBu0.08/expt_93.0/ssh?',
|
148
|
+
}
|
149
|
+
data_info['hourly']['dataset']['GLBu0.08']['version']['93.0']['url'] = url_930_u
|
150
|
+
# GLBy0.08 93.0
|
151
|
+
uv3z_930_y = {}
|
152
|
+
ts3z_930_y = {}
|
153
|
+
ssh_930_y = {}
|
154
|
+
for y_930_y in range(2018, 2025):
|
155
|
+
uv3z_930_y[str(
|
156
|
+
y_930_y)] = rf'https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/uv3z/{y_930_y}?'
|
157
|
+
ts3z_930_y[str(
|
158
|
+
y_930_y)] = rf'https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/ts3z/{y_930_y}?'
|
159
|
+
ssh_930_y[str(
|
160
|
+
y_930_y)] = rf'https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/ssh/{y_930_y}?'
|
161
|
+
url_930_y = {
|
162
|
+
'uv3z': uv3z_930_y,
|
163
|
+
'ts3z': ts3z_930_y,
|
164
|
+
'ssh': ssh_930_y,
|
165
|
+
}
|
166
|
+
data_info['hourly']['dataset']['GLBy0.08']['version']['93.0']['url'] = url_930_y
|
167
|
+
|
168
|
+
|
169
|
+
def draw_time_range(pic_save_folder=None):
|
170
|
+
if pic_save_folder is not None:
|
171
|
+
os.makedirs(pic_save_folder, exist_ok=True)
|
172
|
+
# Converting the data into a format suitable for plotting
|
173
|
+
data = []
|
174
|
+
for dataset, versions in data_info['hourly']['dataset'].items():
|
175
|
+
for version, time_range in versions['version'].items():
|
176
|
+
t_s = time_range['time_range']['time_start']
|
177
|
+
t_e = time_range['time_range']['time_end']
|
178
|
+
if len(t_s) == 8:
|
179
|
+
t_s = t_s + '00'
|
180
|
+
if len(t_e) == 8:
|
181
|
+
t_e = t_e + '21'
|
182
|
+
t_s, t_e = t_s + '0000', t_e + '0000'
|
183
|
+
data.append({
|
184
|
+
'dataset': dataset,
|
185
|
+
'version': version,
|
186
|
+
'start_date': pd.to_datetime(t_s),
|
187
|
+
'end_date': pd.to_datetime(t_e)
|
188
|
+
})
|
189
|
+
|
190
|
+
# Creating a DataFrame
|
191
|
+
df = pd.DataFrame(data)
|
192
|
+
|
193
|
+
# Plotting with combined labels for datasets and versions on the y-axis
|
194
|
+
plt.figure(figsize=(12, 6))
|
195
|
+
|
196
|
+
# Combined labels for datasets and versions
|
197
|
+
combined_labels = [f"{dataset}_{version}" for dataset,
|
198
|
+
version in zip(df['dataset'], df['version'])]
|
199
|
+
|
200
|
+
colors = plt.cm.viridis(np.linspace(0, 1, len(combined_labels)))
|
201
|
+
|
202
|
+
# Assigning a color to each combined label
|
203
|
+
label_colors = {label: colors[i]
|
204
|
+
for i, label in enumerate(combined_labels)}
|
205
|
+
|
206
|
+
# Plotting each time range
|
207
|
+
k = 1
|
208
|
+
for _, row in df.iterrows():
|
209
|
+
plt.plot([row['start_date'], row['end_date']], [k, k],
|
210
|
+
color=label_colors[f"{row['dataset']}_{row['version']}"], linewidth=6)
|
211
|
+
# plt.text(row['end_date'], k,
|
212
|
+
# f"{row['version']}", ha='right', color='black')
|
213
|
+
ymdh_s = row['start_date'].strftime('%Y-%m-%d %H')
|
214
|
+
ymdh_e = row['end_date'].strftime('%Y-%m-%d %H')
|
215
|
+
if k == 1 or k == len(combined_labels):
|
216
|
+
plt.text(row['start_date'], k+0.125,
|
217
|
+
f"{ymdh_s}", ha='left', color='black')
|
218
|
+
plt.text(row['end_date'], k+0.125,
|
219
|
+
f"{ymdh_e}", ha='right', color='black')
|
220
|
+
else:
|
221
|
+
plt.text(row['start_date'], k+0.125,
|
222
|
+
f"{ymdh_s}", ha='right', color='black')
|
223
|
+
plt.text(row['end_date'], k+0.125,
|
224
|
+
f"{ymdh_e}", ha='left', color='black')
|
225
|
+
k += 1
|
226
|
+
|
227
|
+
# Setting the y-axis labels
|
228
|
+
plt.yticks(range(1, len(combined_labels)+1), combined_labels)
|
229
|
+
plt.xlabel('Time')
|
230
|
+
plt.ylabel('Dataset - Version')
|
231
|
+
plt.title('Time Range of Different Versions of Datasets')
|
232
|
+
plt.xticks(rotation=45)
|
233
|
+
plt.grid(True)
|
234
|
+
plt.tight_layout()
|
235
|
+
if pic_save_folder:
|
236
|
+
plt.savefig(Path(pic_save_folder) / 'HYCOM_time_range.png')
|
237
|
+
print(f'[bold green]HYCOM_time_range.png has been saved in {
|
238
|
+
pic_save_folder}')
|
239
|
+
else:
|
240
|
+
plt.savefig('HYCOM_time_range.png')
|
241
|
+
print('[bold green]HYCOM_time_range.png has been saved in the current folder')
|
242
|
+
print(f'Curren folder: {os.getcwd()}')
|
243
|
+
# plt.show()
|
244
|
+
plt.close()
|
245
|
+
|
246
|
+
|
247
|
+
def transform_time(time_str):
|
248
|
+
# old_time = '2023080203'
|
249
|
+
# time_new = '2023-08-02T03%3A00%3A00Z'
|
250
|
+
time_new = f'{time_str[:4]}-{time_str[4:6]
|
251
|
+
}-{time_str[6:8]}T{time_str[8:10]}%3A00%3A00Z'
|
252
|
+
return time_new
|
253
|
+
|
254
|
+
|
255
|
+
def get_time_list(time_s, time_e, delta_hour):
|
256
|
+
'''
|
257
|
+
Description: get a list of time strings from time_s to time_e with delta_hour
|
258
|
+
Args:
|
259
|
+
time_s: start time string, e.g. '2023080203'
|
260
|
+
time_e: end time string, e.g. '2023080303'
|
261
|
+
delta_hour: interval of hours
|
262
|
+
Returns:
|
263
|
+
dt_list: a list of time strings
|
264
|
+
'''
|
265
|
+
dt = datetime.datetime.strptime(time_s, '%Y%m%d%H')
|
266
|
+
dt_list = []
|
267
|
+
while dt.strftime('%Y%m%d%H') <= time_e:
|
268
|
+
dt_list.append(dt.strftime('%Y%m%d%H'))
|
269
|
+
dt = dt + datetime.timedelta(hours=delta_hour)
|
270
|
+
return dt_list
|
271
|
+
|
272
|
+
|
273
|
+
def get_nearest_level_index(depth):
|
274
|
+
level_depth = [0.0, 2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 15.0, 20.0, 25.0, 30.0, 35.0, 40.0, 45.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0, 125.0,
|
275
|
+
150.0, 200.0, 250.0, 300.0, 350.0, 400.0, 500.0, 600.0, 700.0, 800.0, 900.0, 1000.0, 1250.0, 1500.0, 2000.0, 2500.0, 3000.0, 4000.0, 5000]
|
276
|
+
return min(range(len(level_depth)), key=lambda i: abs(level_depth[i]-depth))
|
277
|
+
|
278
|
+
|
279
|
+
def set_query_dict_no_vertical(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh):
|
280
|
+
query_dict = {
|
281
|
+
'var': variable_info[var]['var_name'],
|
282
|
+
'north': lat_max,
|
283
|
+
'west': lon_min,
|
284
|
+
'east': lon_max,
|
285
|
+
'south': lat_min,
|
286
|
+
'horizStride': 1,
|
287
|
+
'time': transform_time(time_str_ymdh),
|
288
|
+
'addLatLon': 'true',
|
289
|
+
'accept': 'netcdf4'
|
290
|
+
}
|
291
|
+
return query_dict
|
292
|
+
|
293
|
+
|
294
|
+
def set_query_dict_depth_or_level(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh):
|
295
|
+
query_dict = {
|
296
|
+
'var': variable_info[var]['var_name'],
|
297
|
+
'north': lat_max,
|
298
|
+
'west': lon_min,
|
299
|
+
'east': lon_max,
|
300
|
+
'south': lat_min,
|
301
|
+
'horizStride': 1,
|
302
|
+
'time': transform_time(time_str_ymdh),
|
303
|
+
'vertCoord': 0,
|
304
|
+
'addLatLon': 'true',
|
305
|
+
'accept': 'netcdf4'
|
306
|
+
}
|
307
|
+
return query_dict
|
308
|
+
|
309
|
+
|
310
|
+
def set_query_dict_full(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh):
|
311
|
+
query_dict = {
|
312
|
+
'var': variable_info[var]['var_name'],
|
313
|
+
'north': lat_max,
|
314
|
+
'west': lon_min,
|
315
|
+
'east': lon_max,
|
316
|
+
'south': lat_min,
|
317
|
+
'horizStride': 1,
|
318
|
+
'time': transform_time(time_str_ymdh),
|
319
|
+
'vertStride': 1,
|
320
|
+
'addLatLon': 'true',
|
321
|
+
'accept': 'netcdf4'
|
322
|
+
}
|
323
|
+
return query_dict
|
324
|
+
|
325
|
+
|
326
|
+
def get_query_dict_single_depth(var, lon_min, lon_max, lat_min, lat_max, depth, time_str_ymdh):
|
327
|
+
if var in ['ssh']:
|
328
|
+
query_dict = set_query_dict_no_vertical(
|
329
|
+
var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
|
330
|
+
else:
|
331
|
+
query_dict = set_query_dict_depth_or_level(
|
332
|
+
var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
|
333
|
+
if var in ['u', 'v', 'temp', 'salinity']:
|
334
|
+
print('Please ensure the depth is in the range of 0-5000 m')
|
335
|
+
query_dict['vertCoord'] = get_nearest_level_index(depth)+1
|
336
|
+
return query_dict
|
337
|
+
|
338
|
+
|
339
|
+
def get_query_dict_single_level(var, lon_min, lon_max, lat_min, lat_max, level_num, time_str_ymdh):
|
340
|
+
if var in ['ssh']:
|
341
|
+
query_dict = set_query_dict_no_vertical(
|
342
|
+
var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
|
343
|
+
else:
|
344
|
+
# level_num: 1-40
|
345
|
+
query_dict = set_query_dict_depth_or_level(
|
346
|
+
var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
|
347
|
+
if var in ['u', 'v', 'temp', 'salinity']:
|
348
|
+
print('Please ensure the level_num is in the range of 1-40')
|
349
|
+
if level_num == 0:
|
350
|
+
level_num = 1
|
351
|
+
print('The level_num is set to 1')
|
352
|
+
if level_num > 40:
|
353
|
+
level_num = 40
|
354
|
+
print('The level_num is set to 40')
|
355
|
+
query_dict['vertCoord'] = level_num
|
356
|
+
return query_dict
|
357
|
+
|
358
|
+
|
359
|
+
def get_query_dict_full_level(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh):
|
360
|
+
if var in ['ssh']:
|
361
|
+
query_dict = set_query_dict_no_vertical(
|
362
|
+
var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
|
363
|
+
else:
|
364
|
+
query_dict = set_query_dict_full(
|
365
|
+
var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
|
366
|
+
return query_dict
|
367
|
+
|
368
|
+
|
369
|
+
def get_query_dict_full_depth(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh):
|
370
|
+
if var in ['ssh']:
|
371
|
+
query_dict = set_query_dict_no_vertical(
|
372
|
+
var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
|
373
|
+
else:
|
374
|
+
query_dict = set_query_dict_full(
|
375
|
+
var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
|
376
|
+
return query_dict
|
377
|
+
|
378
|
+
|
379
|
+
def ymd_in_which_dataset_and_version(time_ymdh):
|
380
|
+
time_ymdh = int(time_ymdh)
|
381
|
+
d_list = []
|
382
|
+
v_list = []
|
383
|
+
trange_list = []
|
384
|
+
have_data = False
|
385
|
+
for dataset_name in data_info['hourly']['dataset'].keys():
|
386
|
+
for version_name in data_info['hourly']['dataset'][dataset_name]['version'].keys():
|
387
|
+
# print(data_info['hourly']['dataset'][dataset_name]
|
388
|
+
# ['version'][version_name]['time_range'].values())
|
389
|
+
time_s, time_e = list(
|
390
|
+
data_info['hourly']['dataset'][dataset_name]['version'][version_name]['time_range'].values())
|
391
|
+
time_s, time_e = str(time_s), str(time_e)
|
392
|
+
if len(time_s) == 8:
|
393
|
+
time_s = time_s + '00'
|
394
|
+
if len(time_e) == 8:
|
395
|
+
time_e = time_e + '21'
|
396
|
+
if time_ymdh >= int(time_s) and time_ymdh <= int(time_e):
|
397
|
+
d_list.append(dataset_name)
|
398
|
+
v_list.append(version_name)
|
399
|
+
trange_list.append(f'{time_s}-{time_e}')
|
400
|
+
have_data = True
|
401
|
+
print(f'[bold red]{time_ymdh} is in the following dataset and version:')
|
402
|
+
if have_data:
|
403
|
+
for d, v, trange in zip(d_list, v_list, trange_list):
|
404
|
+
# print(f'{time_ymd} is in {d} {v} {trange}')
|
405
|
+
print(f'[bold blue]{d} {v} {trange}')
|
406
|
+
else:
|
407
|
+
raise ValueError(
|
408
|
+
f'[bold red]{time_ymdh} is not in any dataset and version')
|
409
|
+
|
410
|
+
|
411
|
+
def direct_choose_dataset_and_version(time_ymdh):
|
412
|
+
time_ymdh = int(time_ymdh)
|
413
|
+
for dataset_name in data_info['hourly']['dataset'].keys():
|
414
|
+
for version_name in data_info['hourly']['dataset'][dataset_name]['version'].keys():
|
415
|
+
[time_s, time_e] = list(data_info['hourly']['dataset'][dataset_name]['version'][version_name]['time_range'].values(
|
416
|
+
))
|
417
|
+
# print(time_s, time_e, time_ymd)
|
418
|
+
time_s, time_e = str(time_s), str(time_e)
|
419
|
+
if len(time_s) == 8:
|
420
|
+
time_s = time_s + '00'
|
421
|
+
if len(time_e) == 8:
|
422
|
+
time_e = time_e + '21'
|
423
|
+
if time_ymdh >= int(time_s) and time_ymdh <= int(time_e):
|
424
|
+
print(
|
425
|
+
f'[bold purple]dataset: {dataset_name}, version: {version_name} is chosen')
|
426
|
+
# print(f'{time_ymdh} is in {dataset_name} {version_name}')
|
427
|
+
return dataset_name, version_name
|
428
|
+
|
429
|
+
|
430
|
+
def get_base_url(dataset_name, version_name, var, year_str):
|
431
|
+
url_dict = data_info['hourly']['dataset'][dataset_name]['version'][version_name]['url']
|
432
|
+
classification_method = data_info['hourly']['dataset'][dataset_name]['version'][version_name]['classification']
|
433
|
+
if classification_method == 'year_different':
|
434
|
+
base_url = url_dict[str(year_str)]
|
435
|
+
elif classification_method == 'same_path':
|
436
|
+
base_url = url_dict
|
437
|
+
elif classification_method == 'var_different':
|
438
|
+
if var in ['u', 'v', 'u_b', 'v_b']:
|
439
|
+
base_url = url_dict['uv3z']
|
440
|
+
elif var in ['temp', 'salinity', 'temp_b', 'salinity_b']:
|
441
|
+
base_url = url_dict['ts3z']
|
442
|
+
elif var in ['ssh']:
|
443
|
+
base_url = url_dict['ssh']
|
444
|
+
else:
|
445
|
+
print(
|
446
|
+
'Please ensure the var is in [u,v,temp,salinity,ssh,u_b,v_b,temp_b,salinity_b]')
|
447
|
+
elif classification_method == 'var_year_different':
|
448
|
+
if var in ['u', 'v', 'u_b', 'v_b']:
|
449
|
+
base_url = url_dict['uv3z'][str(year_str)]
|
450
|
+
elif var in ['temp', 'salinity', 'temp_b', 'salinity_b']:
|
451
|
+
base_url = url_dict['ts3z'][str(year_str)]
|
452
|
+
elif var in ['ssh']:
|
453
|
+
base_url = url_dict['ssh'][str(year_str)]
|
454
|
+
else:
|
455
|
+
print(
|
456
|
+
'Please ensure the var is in [u,v,temp,salinity,ssh,u_b,v_b,temp_b,salinity_b]')
|
457
|
+
return base_url
|
458
|
+
|
459
|
+
|
460
|
+
def get_submit_url(dataset_name, version_name, var, year_str, query_dict):
|
461
|
+
base_url = get_base_url(dataset_name, version_name, var, year_str)
|
462
|
+
query_dict['var'] = [query_dict['var']]
|
463
|
+
target_url = base_url + '&'.join(f"var={var}" for var in query_dict['var']) + '&' + '&'.join(
|
464
|
+
f"{key}={value}" for key, value in query_dict.items() if key != 'var')
|
465
|
+
return target_url
|
466
|
+
|
467
|
+
|
468
|
+
def clear_existing_file(file_full_path):
|
469
|
+
if os.path.exists(file_full_path):
|
470
|
+
os.remove(file_full_path)
|
471
|
+
print(f'{file_full_path} has been removed')
|
472
|
+
|
473
|
+
|
474
|
+
def check_existing_file(file_full_path):
|
475
|
+
if os.path.exists(file_full_path):
|
476
|
+
print(f'{file_full_path} exists')
|
477
|
+
return True
|
478
|
+
else:
|
479
|
+
print(f'{file_full_path} does not exist')
|
480
|
+
return False
|
481
|
+
|
482
|
+
|
483
|
+
def get_ua():
|
484
|
+
ua_list = [
|
485
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60",
|
486
|
+
"Opera/8.0 (Windows NT 5.1; U; en)",
|
487
|
+
"Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50",
|
488
|
+
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50",
|
489
|
+
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
|
490
|
+
"Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
|
491
|
+
"Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
|
492
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
|
493
|
+
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
|
494
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv,2.0.1) Gecko/20100101 Firefox/4.0.1",
|
495
|
+
"Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1",
|
496
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
|
497
|
+
"MAC:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36",
|
498
|
+
"Windows:Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
|
499
|
+
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
|
500
|
+
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
|
501
|
+
"Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
|
502
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36",
|
503
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
|
504
|
+
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
|
505
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
|
506
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36",
|
507
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
|
508
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
|
509
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
|
510
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
|
511
|
+
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
|
512
|
+
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)"
|
513
|
+
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
|
514
|
+
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
|
515
|
+
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0",
|
516
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)",
|
517
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
|
518
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36",
|
519
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
|
520
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36",
|
521
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.4094.1 Safari/537.36",
|
522
|
+
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
|
523
|
+
"Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
|
524
|
+
"Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
|
525
|
+
"Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
|
526
|
+
"Mozilla/5.0 (Linux; U; Android 2.2.1; zh-cn; HTC_Wildfire_A3333 Build/FRG83D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
527
|
+
"Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
528
|
+
"MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
529
|
+
"Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
|
530
|
+
"Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
|
531
|
+
"Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
|
532
|
+
"Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
|
533
|
+
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;",
|
534
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
|
535
|
+
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
|
536
|
+
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
|
537
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
|
538
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
|
539
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
|
540
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)",
|
541
|
+
"Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
542
|
+
"Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
|
543
|
+
"Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
|
544
|
+
"UCWEB7.0.2.37/28/999",
|
545
|
+
"NOKIA5700/UCWEB7.0.2.37/28/999",
|
546
|
+
"Openwave/UCWEB7.0.2.37/28/999",
|
547
|
+
"Openwave/UCWEB7.0.2.37/28/999",
|
548
|
+
]
|
549
|
+
ua_index = random.randint(0, len(ua_list)-1)
|
550
|
+
ua = ua_list[ua_index]
|
551
|
+
# print(f'Using User-Agent: {ua}')
|
552
|
+
return ua
|
553
|
+
|
554
|
+
|
555
|
+
def get_proxy():
|
556
|
+
# 获取当前脚本的绝对路径
|
557
|
+
script_dir = os.path.dirname(os.path.abspath(__file__))
|
558
|
+
# 构建ip.txt的绝对路径
|
559
|
+
ip_file_txt = os.path.join(script_dir, 'ip.txt')
|
560
|
+
with open(ip_file_txt, 'r') as f:
|
561
|
+
ips = f.readlines()
|
562
|
+
ip_list = []
|
563
|
+
for ip in ips:
|
564
|
+
ip_list.append(ip.strip())
|
565
|
+
choose_ip = random.choice(ip_list)
|
566
|
+
proxies = {
|
567
|
+
'http': 'http://' + choose_ip,
|
568
|
+
'https': 'https://' + choose_ip
|
569
|
+
}
|
570
|
+
# print(f'Using proxy: {proxies}')
|
571
|
+
return proxies
|
572
|
+
|
573
|
+
|
574
|
+
def dlownload_file(target_url, store_path, file_name, check=False):
|
575
|
+
print(f'[bold red]Downloading {file_name}...')
|
576
|
+
# 创建会话
|
577
|
+
s = requests.Session()
|
578
|
+
download_success = False
|
579
|
+
request_times = 0
|
580
|
+
filename = Path(store_path) / file_name
|
581
|
+
if check:
|
582
|
+
if check_existing_file(filename):
|
583
|
+
return
|
584
|
+
clear_existing_file(filename)
|
585
|
+
print(f'Download_start_time: {datetime.datetime.now()}')
|
586
|
+
while not download_success:
|
587
|
+
if request_times > 0:
|
588
|
+
print(f'\r正在重试第 {request_times} 次', end="")
|
589
|
+
# 尝试下载文件
|
590
|
+
try:
|
591
|
+
headers = {
|
592
|
+
'User-Agent': get_ua()}
|
593
|
+
# response = s.get(target_url, headers=headers, stream=True)
|
594
|
+
# response = s.get(target_url, headers=headers,
|
595
|
+
# proxies=get_proxy(), timeout=5)
|
596
|
+
response = s.get(target_url, headers=headers,
|
597
|
+
timeout=5)
|
598
|
+
# response = s.get(target_url, headers=headers)
|
599
|
+
# print(target_url)
|
600
|
+
response.raise_for_status() # 如果请求返回的不是200,将抛出HTTPError异常
|
601
|
+
|
602
|
+
# 保存文件
|
603
|
+
with open(filename, 'wb') as f:
|
604
|
+
f.write(response.content)
|
605
|
+
# print(f'\r文件 {filename} 下载成功', end="")
|
606
|
+
print(f'[bold green]文件 {filename} 下载成功')
|
607
|
+
# query_ncfile_time(filename) # 这个函数在linux上目前会出问题
|
608
|
+
download_success = True
|
609
|
+
|
610
|
+
except requests.exceptions.HTTPError as errh:
|
611
|
+
print(f"Http Error: {errh}")
|
612
|
+
except requests.exceptions.ConnectionError as errc:
|
613
|
+
print(f"Error Connecting: {errc}")
|
614
|
+
except requests.exceptions.Timeout as errt:
|
615
|
+
print(f"Timeout Error: {errt}")
|
616
|
+
except requests.exceptions.RequestException as err:
|
617
|
+
print(f"OOps: Something Else: {err}")
|
618
|
+
|
619
|
+
time.sleep(3)
|
620
|
+
request_times += 1
|
621
|
+
print(f'Download_end_time: {datetime.datetime.now()}')
|
622
|
+
|
623
|
+
|
624
|
+
def check_hour_is_valid(ymdh_str):
|
625
|
+
# hour should be 00, 03, 06, 09, 12, 15, 18, 21
|
626
|
+
hh = int(str(ymdh_str[-2:]))
|
627
|
+
if hh in [0, 3, 6, 9, 12, 15, 18, 21]:
|
628
|
+
return True
|
629
|
+
else:
|
630
|
+
return False
|
631
|
+
|
632
|
+
|
633
|
+
def direct_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, download_time='2024083100', depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None, check=False):
|
634
|
+
download_time = str(download_time)
|
635
|
+
if not check_hour_is_valid(download_time):
|
636
|
+
print('Please ensure the hour is 00, 03, 06, 09, 12, 15, 18, 21')
|
637
|
+
raise ValueError('The hour is invalid')
|
638
|
+
if dataset_name is None and version_name is None:
|
639
|
+
print('The dataset_name and version_name are None, so the dataset and version will be chosen according to the download_time.\nIf there is more than one dataset and version in the time range, the first one will be chosen.')
|
640
|
+
print('If you wanna choose the dataset and version by yourself, please set the dataset_name and version_name together.')
|
641
|
+
ymd_in_which_dataset_and_version(download_time)
|
642
|
+
dataset_name, version_name = direct_choose_dataset_and_version(
|
643
|
+
download_time)
|
644
|
+
elif dataset_name is None and version_name is not None:
|
645
|
+
print('Please ensure the dataset_name is not None')
|
646
|
+
print('If you do not add the dataset_name, both the dataset and version will be chosen according to the download_time.')
|
647
|
+
ymd_in_which_dataset_and_version(download_time)
|
648
|
+
dataset_name, version_name = direct_choose_dataset_and_version(
|
649
|
+
download_time)
|
650
|
+
elif dataset_name is not None and version_name is None:
|
651
|
+
print('Please ensure the version_name is not None')
|
652
|
+
print('If you do not add the version_name, both the dataset and version will be chosen according to the download_time.')
|
653
|
+
ymd_in_which_dataset_and_version(download_time)
|
654
|
+
dataset_name, version_name = direct_choose_dataset_and_version(
|
655
|
+
download_time)
|
656
|
+
else:
|
657
|
+
print('The dataset_name and version_name are both set by yourself.')
|
658
|
+
|
659
|
+
year_str = str(download_time)[:4]
|
660
|
+
if depth is not None and level_num is not None:
|
661
|
+
print('Please ensure the depth or level_num is None')
|
662
|
+
elif depth is not None:
|
663
|
+
print(f'Data of single depth ({depth}m) will be downloaded...')
|
664
|
+
query_dict = get_query_dict_single_depth(
|
665
|
+
var, lon_min, lon_max, lat_min, lat_max, depth, download_time)
|
666
|
+
elif level_num is not None:
|
667
|
+
print(f'Data of single level ({level_num}) will be downloaded...')
|
668
|
+
query_dict = get_query_dict_single_level(
|
669
|
+
var, lon_min, lon_max, lat_min, lat_max, level_num, download_time)
|
670
|
+
else:
|
671
|
+
print('Full depth or full level data will be downloaded...')
|
672
|
+
query_dict = get_query_dict_full_level(
|
673
|
+
var, lon_min, lon_max, lat_min, lat_max, download_time)
|
674
|
+
submit_url = get_submit_url(
|
675
|
+
dataset_name, version_name, var, year_str, query_dict)
|
676
|
+
file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
|
677
|
+
if store_path is None:
|
678
|
+
store_path = str(Path.cwd())
|
679
|
+
else:
|
680
|
+
os.makedirs(str(store_path), exist_ok=True)
|
681
|
+
dlownload_file(submit_url, store_path, file_name, check)
|
682
|
+
|
683
|
+
|
684
|
+
def convert_full_name_to_short_name(full_name):
|
685
|
+
for var, info in variable_info.items():
|
686
|
+
if full_name == info['var_name'] or full_name == info['standard_name'] or full_name == var:
|
687
|
+
return var
|
688
|
+
print('[bold #FFE4E1]Please ensure the var is in:\n[bold blue]u,v,temp,salinity,ssh,u_b,v_b,temp_b,salinity_b')
|
689
|
+
print('or')
|
690
|
+
print('[bold blue]water_u, water_v, water_temp, salinity, surf_el, water_u_bottom, water_v_bottom, water_temp_bottom, salinity_bottom')
|
691
|
+
return False
|
692
|
+
|
693
|
+
|
694
|
+
def download_task(var, time_str, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check):
|
695
|
+
'''
|
696
|
+
# 并行下载任务
|
697
|
+
# 这个函数是为了并行下载而设置的,是必须的,直接调用direct_download并行下载会出问题
|
698
|
+
|
699
|
+
任务封装:将每个任务需要的数据和操作封装在一个函数中,这样每个任务都是独立的,不会相互干扰。
|
700
|
+
本情况下,download_task函数的作用是将每个下载任务封装起来,包括它所需的所有参数。
|
701
|
+
这样,每个任务都是独立的,有自己的参数和数据,不会与其他任务共享或修改任何数据。
|
702
|
+
因此,即使多个任务同时执行,也不会出现数据交互错乱的问题。
|
703
|
+
'''
|
704
|
+
direct_download(var, lon_min, lon_max, lat_min, lat_max, time_str,
|
705
|
+
depth, level, store_path, dataset_name, version_name, check)
|
706
|
+
|
707
|
+
|
708
|
+
def download(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False):
|
709
|
+
'''
|
710
|
+
Description:
|
711
|
+
Download the data of single time or a series of time
|
712
|
+
|
713
|
+
Parameters:
|
714
|
+
var: str, the variable name, such as 'u', 'v', 'temp', 'salinity', 'ssh', 'u_b', 'v_b', 'temp_b', 'salinity_b' or 'water_u', 'water_v', 'water_temp', 'salinity', 'surf_el', 'water_u_bottom', 'water_v_bottom', 'water_temp_bottom', 'salinity_bottom'
|
715
|
+
time_s: str, the start time, such as '2024110100' or '20241101', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
|
716
|
+
time_e: str, the end time, such as '2024110221' or '20241102', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
|
717
|
+
lon_min: float, the minimum longitude, default is 0
|
718
|
+
lon_max: float, the maximum longitude, default is 359.92
|
719
|
+
lat_min: float, the minimum latitude, default is -80
|
720
|
+
lat_max: float, the maximum latitude, default is 90
|
721
|
+
depth: float, the depth, default is None
|
722
|
+
level: int, the level number, default is None
|
723
|
+
store_path: str, the path to store the data, default is None
|
724
|
+
dataset_name: str, the dataset name, default is None, example: 'GLBv0.08', 'GLBu0.08', 'GLBy0.08'
|
725
|
+
version_name: str, the version name, default is None, example: '53.X', '56.3'
|
726
|
+
num_workers: int, the number of workers, default is None
|
727
|
+
|
728
|
+
Returns:
|
729
|
+
None
|
730
|
+
'''
|
731
|
+
var = convert_full_name_to_short_name(var)
|
732
|
+
if var is False:
|
733
|
+
raise ValueError('The var is invalid')
|
734
|
+
if lon_min < 0 or lon_min > 359.92 or lon_max < 0 or lon_max > 359.92 or lat_min < -80 or lat_min > 90 or lat_max < -80 or lat_max > 90:
|
735
|
+
print('Please ensure the lon_min, lon_max, lat_min, lat_max are in the range')
|
736
|
+
print('The range of lon_min, lon_max is 0~359.92')
|
737
|
+
print('The range of lat_min, lat_max is -80~90')
|
738
|
+
raise ValueError('The lon or lat is invalid')
|
739
|
+
ymdh_time_s = str(time_s)
|
740
|
+
if len(ymdh_time_s) == 8:
|
741
|
+
ymdh_time_s += '00'
|
742
|
+
ymdh_time_e = str(time_e)
|
743
|
+
if len(ymdh_time_e) == 8:
|
744
|
+
ymdh_time_e += '21'
|
745
|
+
if ymdh_time_s == ymdh_time_e:
|
746
|
+
direct_download(var, lon_min, lon_max, lat_min, lat_max,
|
747
|
+
ymdh_time_s, depth, level, store_path, dataset_name, version_name)
|
748
|
+
elif int(ymdh_time_s) < int(ymdh_time_e):
|
749
|
+
print('Downloading a series of files...')
|
750
|
+
time_list = get_time_list(ymdh_time_s, ymdh_time_e, 3)
|
751
|
+
with Progress() as progress:
|
752
|
+
task = progress.add_task(
|
753
|
+
"[cyan]Downloading...", total=len(time_list))
|
754
|
+
if num_workers is None or num_workers <= 1:
|
755
|
+
# 串行方式
|
756
|
+
for time_str in time_list:
|
757
|
+
direct_download(var, lon_min, lon_max, lat_min, lat_max,
|
758
|
+
time_str, depth, level, store_path, dataset_name, version_name, check)
|
759
|
+
progress.update(task, advance=1)
|
760
|
+
else:
|
761
|
+
# 并行方式
|
762
|
+
if num_workers > 10:
|
763
|
+
print('The number of workers is too large!')
|
764
|
+
print(
|
765
|
+
'In order to avoid the server being blocked, the number of workers is set to 10')
|
766
|
+
num_workers = 10
|
767
|
+
with ThreadPoolExecutor(max_workers=num_workers) as executor:
|
768
|
+
futures = [executor.submit(download_task, var, time_str, lon_min, lon_max, lat_min, lat_max,
|
769
|
+
depth, level, store_path, dataset_name, version_name, check) for time_str in time_list]
|
770
|
+
for future in futures:
|
771
|
+
future.add_done_callback(
|
772
|
+
lambda _: progress.update(task, advance=1))
|
773
|
+
else:
|
774
|
+
print('Please ensure the ymdh_time_s is less than the ymdh_time_e')
|
775
|
+
|
776
|
+
|
777
|
+
def how_to_use():
|
778
|
+
print('''
|
779
|
+
# 1. Choose the dataset and version according to the time:
|
780
|
+
# 1.1 Use function to query
|
781
|
+
You can use the function ymd_in_which_dataset_and_version(time_ymd=20241101) to find the dataset and version according to the time.
|
782
|
+
Then, you can see the dataset and version in the output.
|
783
|
+
# 1.2 Draw a picture to see
|
784
|
+
You can draw a picture to see the time range of each dataset and version.
|
785
|
+
Using the function draw_time_range(pic_save_folder=None) to draw the picture.
|
786
|
+
|
787
|
+
# 2. Get the base url according to the dataset, version, var and year:
|
788
|
+
# 2.1 Dataset and version were found in step 1
|
789
|
+
# 2.2 Var: u, v, temp, salinity, ssh, u_b, v_b, temp_b, salinity_b
|
790
|
+
# 2.3 Year: 1994-2024(current year)
|
791
|
+
|
792
|
+
# 3. Get the query_dict according to the var, lon_min, lon_max, lat_min, lat_max, depth, level_num, time_str_ymdh:
|
793
|
+
# 3.1 Var: u, v, temp, salinity, ssh, u_b, v_b, temp_b, salinity_b
|
794
|
+
# 3.2 Lon_min, lon_max, lat_min, lat_max: float
|
795
|
+
# 3.3 Depth: 0-5000m, if you wanna get single depth data, you can set the depth
|
796
|
+
# 3.4 Level_num: 1-40, if you wanna get single level data, you can set the level_num
|
797
|
+
# 3.5 Time_str_ymdh: '2024110112', the hour normally is 00, 03, 06, 09, 12, 15, 18, 21, besides 1 hourly data
|
798
|
+
# 3.6 Use the function to get the query_dict
|
799
|
+
# 3.7 Note: If you wanna get the full depth or full level data, you can needn't set the depth or level_num
|
800
|
+
|
801
|
+
# 4. Get the submit url according to the dataset, version, var, year, query_dict:
|
802
|
+
# 4.1 Use the function to get the submit url
|
803
|
+
# 4.2 You can use the submit url to download the data
|
804
|
+
|
805
|
+
# 5. Download the data according to the submit url:
|
806
|
+
# 5.1 Use the function to download the data
|
807
|
+
# 5.2 You can download the data of single time or a series of time
|
808
|
+
# 5.3 Note: If you wanna download a series of data, you can set the ymdh_time_s and ymdh_time_e different
|
809
|
+
# 5.4 Note: The time resolution is 3 hours
|
810
|
+
|
811
|
+
# 6. Direct download the data:
|
812
|
+
# 6.1 Use the function to direct download the data
|
813
|
+
# 6.2 You can set the dataset_name and version_name by yourself
|
814
|
+
# 6.3 Note: If you do not set the dataset_name and version_name, the dataset and version will be chosen according to the download_time
|
815
|
+
# 6.4 Note: If you set the dataset_name and version_name, please ensure the dataset_name and version_name are correct
|
816
|
+
# 6.5 Note: If you just set one of the dataset_name and version_name, both the dataset and version will be chosen according to the download_time
|
817
|
+
|
818
|
+
# 7. Simple use:
|
819
|
+
# 7.1 You can use the function: download(var, ymdh_time_s, ymdh_time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None)
|
820
|
+
# 7.2 You can download the data of single time or a series of time
|
821
|
+
# 7.3 The parameters you must set are var, ymdh_time_s, ymdh_time_e
|
822
|
+
# 7.4 Example: download('u', '2024110112', '2024110212', lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None)
|
823
|
+
''')
|
824
|
+
|
825
|
+
|
826
|
+
if __name__ == '__main__':
|
827
|
+
time_s, time_e = '2018010112', '2018010300'
|
828
|
+
merge_name = '201801'
|
829
|
+
root_path = r'I:\hycom_data_2018'
|
830
|
+
location_dict = {'west': 115, 'east': 130, 'south': 33, 'north': 45}
|
831
|
+
download_dict = {
|
832
|
+
'water_u': {'simple_name': 'u', 'download': 1},
|
833
|
+
'water_v': {'simple_name': 'v', 'download': 0},
|
834
|
+
'surf_el': {'simple_name': 'ssh', 'download': 0},
|
835
|
+
'water_temp': {'simple_name': 'temp', 'download': 0},
|
836
|
+
'salinity': {'simple_name': 'salinity', 'download': 1},
|
837
|
+
}
|
838
|
+
|
839
|
+
# set depth or level, only one can be True
|
840
|
+
# if you wanna download all depth or level, set both False
|
841
|
+
set_depth, depth = False, 0 # 0-4000 meters
|
842
|
+
set_level, level = False, 1 # 1-40 levels
|
843
|
+
num_workers = 5
|
844
|
+
check = True
|
845
|
+
|
846
|
+
download_switch = True
|
847
|
+
combine_switch = False
|
848
|
+
|
849
|
+
def download_var(var_name):
|
850
|
+
var = download_dict[var_name]['simple_name']
|
851
|
+
if_download = download_dict[var_name]['download']
|
852
|
+
if if_download == 0:
|
853
|
+
return
|
854
|
+
if set_depth:
|
855
|
+
download(var=var, time_s=time_s, time_e=time_e, store_path=Path(root_path)/var, lon_min=location_dict[
|
856
|
+
'west'], lon_max=location_dict['east'], lat_min=location_dict['south'], lat_max=location_dict['north'], depth=depth, num_workers=num_workers, check=check)
|
857
|
+
elif set_level:
|
858
|
+
download(var=var, time_s=time_s, time_e=time_e, store_path=Path(root_path)/var, lon_min=location_dict[
|
859
|
+
'west'], lon_max=location_dict['east'], lat_min=location_dict['south'], lat_max=location_dict['north'], level=level, num_workers=num_workers, check=check)
|
860
|
+
else:
|
861
|
+
download(var=var, time_s=time_s, time_e=time_e, store_path=Path(root_path)/var, lon_min=location_dict[
|
862
|
+
'west'], lon_max=location_dict['east'], lat_min=location_dict['south'], lat_max=location_dict['north'], num_workers=num_workers, check=check)
|
863
|
+
|
864
|
+
if download_switch:
|
865
|
+
for var_name in download_dict.keys():
|
866
|
+
download_var(var_name)
|
867
|
+
|
868
|
+
# draw_time_range(r'I:\Delete')
|
869
|
+
|
870
|
+
""" if combine_switch:
|
871
|
+
time_list = get_time_list(time_s, time_e, 3)
|
872
|
+
for var_name in download_dict.keys():
|
873
|
+
var = download_dict[var_name]['simple_name']
|
874
|
+
if_download = download_dict[var_name]['download']
|
875
|
+
if if_download == 0:
|
876
|
+
continue
|
877
|
+
file_list = []
|
878
|
+
for time_str in time_list:
|
879
|
+
file_list.append(
|
880
|
+
Path(root_path)/var/f'HYCOM_{var_name}_{time_str}.nc')
|
881
|
+
merge5nc(file_list, var_name, 'time',
|
882
|
+
Path(root_path)/var/f'HYCOM_{var_name}_{merge_name}.nc') """
|