oafuncs 0.0.59__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,882 @@
1
+ #!/usr/bin/env python
2
+ # coding=utf-8
3
+ '''
4
+ Author: Liu Kun && 16031215@qq.com
5
+ Date: 2024-11-01 10:31:09
6
+ LastEditors: Liu Kun && 16031215@qq.com
7
+ LastEditTime: 2024-11-12 16:22:33
8
+ FilePath: \\Python\\My_Funcs\\OAFuncs\\OAFuncs\\oa_down\\hycom_3hourly.py
9
+ Description:
10
+ EditPlatform: vscode
11
+ ComputerInfo: XPS 15 9510
12
+ SystemInfo: Windows 11
13
+ Python Version: 3.11
14
+ '''
15
+ import datetime
16
+ import os
17
+ import random
18
+ import time
19
+ import warnings
20
+ from concurrent.futures import ThreadPoolExecutor
21
+ from pathlib import Path
22
+
23
+ import matplotlib.pyplot as plt
24
+ import numpy as np
25
+ import pandas as pd
26
+ import requests
27
+ from rich import print
28
+ from rich.progress import Progress
29
+
30
+ warnings.filterwarnings("ignore", category=RuntimeWarning,
31
+ message="Engine '.*' loading failed:.*")
32
+
33
+ __all__ = ['draw_time_range', 'download', 'how_to_use', 'get_time_list']
34
+
35
+ # time resolution
36
+ data_info = {'yearly': {}, 'monthly': {}, 'daily': {}, 'hourly': {}}
37
+
38
+ # hourly data
39
+ # dataset: GLBv0.08, GLBu0.08, GLBy0.08
40
+ data_info['hourly']['dataset'] = {
41
+ 'GLBv0.08': {}, 'GLBu0.08': {}, 'GLBy0.08': {}}
42
+
43
+ # version
44
+ # version of GLBv0.08: 53.X, 56.3, 57.2, 92.8, 57.7, 92.9, 93.0
45
+ data_info['hourly']['dataset']['GLBv0.08']['version'] = {
46
+ '53.X': {}, '56.3': {}, '57.2': {}, '92.8': {}, '57.7': {}, '92.9': {}, '93.0': {}}
47
+ # version of GLBu0.08: 93.0
48
+ data_info['hourly']['dataset']['GLBu0.08']['version'] = {'93.0': {}}
49
+ # version of GLBy0.08: 93.0
50
+ data_info['hourly']['dataset']['GLBy0.08']['version'] = {'93.0': {}}
51
+
52
+ # info details
53
+ # time range
54
+ # GLBv0.08
55
+ # 在网页上提交超过范围的时间,会返回该数据集实际时间范围,从而纠正下面的时间范围
56
+ # 目前只纠正了GLBv0.08 93.0的时间范围,具体到小时了
57
+ # 其他数据集的时刻暂时默认为00起,21止
58
+ data_info['hourly']['dataset']['GLBv0.08']['version']['53.X']['time_range'] = {
59
+ 'time_start': '19940101', 'time_end': '20151230'}
60
+ data_info['hourly']['dataset']['GLBv0.08']['version']['56.3']['time_range'] = {
61
+ 'time_start': '20140701', 'time_end': '20160430'}
62
+ data_info['hourly']['dataset']['GLBv0.08']['version']['57.2']['time_range'] = {
63
+ 'time_start': '20160501', 'time_end': '20170131'}
64
+ data_info['hourly']['dataset']['GLBv0.08']['version']['92.8']['time_range'] = {
65
+ 'time_start': '20170201', 'time_end': '20170531'}
66
+ data_info['hourly']['dataset']['GLBv0.08']['version']['57.7']['time_range'] = {
67
+ 'time_start': '20170601', 'time_end': '20170930'}
68
+ data_info['hourly']['dataset']['GLBv0.08']['version']['92.9']['time_range'] = {
69
+ 'time_start': '20171001', 'time_end': '20171231'}
70
+ data_info['hourly']['dataset']['GLBv0.08']['version']['93.0']['time_range'] = {
71
+ 'time_start': '2018010112', 'time_end': '2020021909'}
72
+ # GLBu0.08
73
+ data_info['hourly']['dataset']['GLBu0.08']['version']['93.0']['time_range'] = {
74
+ 'time_start': '20180919', 'time_end': '20181208'}
75
+ # GLBy0.08
76
+ data_info['hourly']['dataset']['GLBy0.08']['version']['93.0']['time_range'] = {
77
+ 'time_start': '20181204', 'time_end': '20300904'}
78
+
79
+ # variable
80
+ variable_info = {
81
+ 'u': {'var_name': 'water_u', 'standard_name': 'eastward_sea_water_velocity'},
82
+ 'v': {'var_name': 'water_v', 'standard_name': 'northward_sea_water_velocity'},
83
+ 'temp': {'var_name': 'water_temp', 'standard_name': 'sea_water_potential_temperature'},
84
+ 'salinity': {'var_name': 'salinity', 'standard_name': 'sea_water_salinity'},
85
+ 'ssh': {'var_name': 'surf_el', 'standard_name': 'sea_surface_elevation'},
86
+ 'u_b': {'var_name': 'water_u_bottom', 'standard_name': 'eastward_sea_water_velocity_at_sea_floor'},
87
+ 'v_b': {'var_name': 'water_v_bottom', 'standard_name': 'northward_sea_water_velocity_at_sea_floor'},
88
+ 'temp_b': {'var_name': 'water_temp_bottom', 'standard_name': 'sea_water_potential_temperature_at_sea_floor'},
89
+ 'salinity_b': {'var_name': 'salinity_bottom', 'standard_name': 'sea_water_salinity_at_sea_floor'},
90
+ }
91
+
92
+ # classification method
93
+ # year_different: the data of different years is stored in different files
94
+ # same_path: the data of different years is stored in the same file
95
+ # var_different: the data of different variables is stored in different files
96
+ # var_year_different: the data of different variables and years is stored in different files
97
+ data_info['hourly']['dataset']['GLBv0.08']['version']['53.X']['classification'] = 'year_different'
98
+ data_info['hourly']['dataset']['GLBv0.08']['version']['56.3']['classification'] = 'same_path'
99
+ data_info['hourly']['dataset']['GLBv0.08']['version']['57.2']['classification'] = 'same_path'
100
+ data_info['hourly']['dataset']['GLBv0.08']['version']['92.8']['classification'] = 'var_different'
101
+ data_info['hourly']['dataset']['GLBv0.08']['version']['57.7']['classification'] = 'same_path'
102
+ data_info['hourly']['dataset']['GLBv0.08']['version']['92.9']['classification'] = 'var_different'
103
+ data_info['hourly']['dataset']['GLBv0.08']['version']['93.0']['classification'] = 'var_different'
104
+ data_info['hourly']['dataset']['GLBu0.08']['version']['93.0']['classification'] = 'var_different'
105
+ data_info['hourly']['dataset']['GLBy0.08']['version']['93.0']['classification'] = 'var_year_different'
106
+
107
+ # download info
108
+ # base url
109
+ # GLBv0.08 53.X
110
+ url_53x = {}
111
+ for y_53x in range(1994, 2016):
112
+ # r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_53.X/data/2013?'
113
+ url_53x[str(
114
+ y_53x)] = rf'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_53.X/data/{y_53x}?'
115
+ data_info['hourly']['dataset']['GLBv0.08']['version']['53.X']['url'] = url_53x
116
+ # GLBv0.08 56.3
117
+ data_info['hourly']['dataset']['GLBv0.08']['version']['56.3']['url'] = r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_56.3?'
118
+ # GLBv0.08 57.2
119
+ data_info['hourly']['dataset']['GLBv0.08']['version']['57.2']['url'] = r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_57.2?'
120
+ # GLBv0.08 92.8
121
+ url_928 = {
122
+ 'uv3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.8/uv3z?',
123
+ 'ts3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.8/ts3z?',
124
+ 'ssh': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.8/ssh?',
125
+ }
126
+ data_info['hourly']['dataset']['GLBv0.08']['version']['92.8']['url'] = url_928
127
+ # GLBv0.08 57.7
128
+ data_info['hourly']['dataset']['GLBv0.08']['version']['57.7']['url'] = r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_57.7?'
129
+ # GLBv0.08 92.9
130
+ url_929 = {
131
+ 'uv3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.9/uv3z?',
132
+ 'ts3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.9/ts3z?',
133
+ 'ssh': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_92.9/ssh?',
134
+ }
135
+ data_info['hourly']['dataset']['GLBv0.08']['version']['92.9']['url'] = url_929
136
+ # GLBv0.08 93.0
137
+ url_930_v = {
138
+ 'uv3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_93.0/uv3z?',
139
+ 'ts3z': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_93.0/ts3z?',
140
+ 'ssh': r'https://ncss.hycom.org/thredds/ncss/GLBv0.08/expt_93.0/ssh?',
141
+ }
142
+ data_info['hourly']['dataset']['GLBv0.08']['version']['93.0']['url'] = url_930_v
143
+ # GLBu0.08 93.0
144
+ url_930_u = {
145
+ 'uv3z': r'https://ncss.hycom.org/thredds/ncss/GLBu0.08/expt_93.0/uv3z?',
146
+ 'ts3z': r'https://ncss.hycom.org/thredds/ncss/GLBu0.08/expt_93.0/ts3z?',
147
+ 'ssh': r'https://ncss.hycom.org/thredds/ncss/GLBu0.08/expt_93.0/ssh?',
148
+ }
149
+ data_info['hourly']['dataset']['GLBu0.08']['version']['93.0']['url'] = url_930_u
150
+ # GLBy0.08 93.0
151
+ uv3z_930_y = {}
152
+ ts3z_930_y = {}
153
+ ssh_930_y = {}
154
+ for y_930_y in range(2018, 2025):
155
+ uv3z_930_y[str(
156
+ y_930_y)] = rf'https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/uv3z/{y_930_y}?'
157
+ ts3z_930_y[str(
158
+ y_930_y)] = rf'https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/ts3z/{y_930_y}?'
159
+ ssh_930_y[str(
160
+ y_930_y)] = rf'https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/ssh/{y_930_y}?'
161
+ url_930_y = {
162
+ 'uv3z': uv3z_930_y,
163
+ 'ts3z': ts3z_930_y,
164
+ 'ssh': ssh_930_y,
165
+ }
166
+ data_info['hourly']['dataset']['GLBy0.08']['version']['93.0']['url'] = url_930_y
167
+
168
+
169
+ def draw_time_range(pic_save_folder=None):
170
+ if pic_save_folder is not None:
171
+ os.makedirs(pic_save_folder, exist_ok=True)
172
+ # Converting the data into a format suitable for plotting
173
+ data = []
174
+ for dataset, versions in data_info['hourly']['dataset'].items():
175
+ for version, time_range in versions['version'].items():
176
+ t_s = time_range['time_range']['time_start']
177
+ t_e = time_range['time_range']['time_end']
178
+ if len(t_s) == 8:
179
+ t_s = t_s + '00'
180
+ if len(t_e) == 8:
181
+ t_e = t_e + '21'
182
+ t_s, t_e = t_s + '0000', t_e + '0000'
183
+ data.append({
184
+ 'dataset': dataset,
185
+ 'version': version,
186
+ 'start_date': pd.to_datetime(t_s),
187
+ 'end_date': pd.to_datetime(t_e)
188
+ })
189
+
190
+ # Creating a DataFrame
191
+ df = pd.DataFrame(data)
192
+
193
+ # Plotting with combined labels for datasets and versions on the y-axis
194
+ plt.figure(figsize=(12, 6))
195
+
196
+ # Combined labels for datasets and versions
197
+ combined_labels = [f"{dataset}_{version}" for dataset,
198
+ version in zip(df['dataset'], df['version'])]
199
+
200
+ colors = plt.cm.viridis(np.linspace(0, 1, len(combined_labels)))
201
+
202
+ # Assigning a color to each combined label
203
+ label_colors = {label: colors[i]
204
+ for i, label in enumerate(combined_labels)}
205
+
206
+ # Plotting each time range
207
+ k = 1
208
+ for _, row in df.iterrows():
209
+ plt.plot([row['start_date'], row['end_date']], [k, k],
210
+ color=label_colors[f"{row['dataset']}_{row['version']}"], linewidth=6)
211
+ # plt.text(row['end_date'], k,
212
+ # f"{row['version']}", ha='right', color='black')
213
+ ymdh_s = row['start_date'].strftime('%Y-%m-%d %H')
214
+ ymdh_e = row['end_date'].strftime('%Y-%m-%d %H')
215
+ if k == 1 or k == len(combined_labels):
216
+ plt.text(row['start_date'], k+0.125,
217
+ f"{ymdh_s}", ha='left', color='black')
218
+ plt.text(row['end_date'], k+0.125,
219
+ f"{ymdh_e}", ha='right', color='black')
220
+ else:
221
+ plt.text(row['start_date'], k+0.125,
222
+ f"{ymdh_s}", ha='right', color='black')
223
+ plt.text(row['end_date'], k+0.125,
224
+ f"{ymdh_e}", ha='left', color='black')
225
+ k += 1
226
+
227
+ # Setting the y-axis labels
228
+ plt.yticks(range(1, len(combined_labels)+1), combined_labels)
229
+ plt.xlabel('Time')
230
+ plt.ylabel('Dataset - Version')
231
+ plt.title('Time Range of Different Versions of Datasets')
232
+ plt.xticks(rotation=45)
233
+ plt.grid(True)
234
+ plt.tight_layout()
235
+ if pic_save_folder:
236
+ plt.savefig(Path(pic_save_folder) / 'HYCOM_time_range.png')
237
+ print(f'[bold green]HYCOM_time_range.png has been saved in {
238
+ pic_save_folder}')
239
+ else:
240
+ plt.savefig('HYCOM_time_range.png')
241
+ print('[bold green]HYCOM_time_range.png has been saved in the current folder')
242
+ print(f'Curren folder: {os.getcwd()}')
243
+ # plt.show()
244
+ plt.close()
245
+
246
+
247
+ def transform_time(time_str):
248
+ # old_time = '2023080203'
249
+ # time_new = '2023-08-02T03%3A00%3A00Z'
250
+ time_new = f'{time_str[:4]}-{time_str[4:6]
251
+ }-{time_str[6:8]}T{time_str[8:10]}%3A00%3A00Z'
252
+ return time_new
253
+
254
+
255
+ def get_time_list(time_s, time_e, delta_hour):
256
+ '''
257
+ Description: get a list of time strings from time_s to time_e with delta_hour
258
+ Args:
259
+ time_s: start time string, e.g. '2023080203'
260
+ time_e: end time string, e.g. '2023080303'
261
+ delta_hour: interval of hours
262
+ Returns:
263
+ dt_list: a list of time strings
264
+ '''
265
+ dt = datetime.datetime.strptime(time_s, '%Y%m%d%H')
266
+ dt_list = []
267
+ while dt.strftime('%Y%m%d%H') <= time_e:
268
+ dt_list.append(dt.strftime('%Y%m%d%H'))
269
+ dt = dt + datetime.timedelta(hours=delta_hour)
270
+ return dt_list
271
+
272
+
273
+ def get_nearest_level_index(depth):
274
+ level_depth = [0.0, 2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 15.0, 20.0, 25.0, 30.0, 35.0, 40.0, 45.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0, 125.0,
275
+ 150.0, 200.0, 250.0, 300.0, 350.0, 400.0, 500.0, 600.0, 700.0, 800.0, 900.0, 1000.0, 1250.0, 1500.0, 2000.0, 2500.0, 3000.0, 4000.0, 5000]
276
+ return min(range(len(level_depth)), key=lambda i: abs(level_depth[i]-depth))
277
+
278
+
279
+ def set_query_dict_no_vertical(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh):
280
+ query_dict = {
281
+ 'var': variable_info[var]['var_name'],
282
+ 'north': lat_max,
283
+ 'west': lon_min,
284
+ 'east': lon_max,
285
+ 'south': lat_min,
286
+ 'horizStride': 1,
287
+ 'time': transform_time(time_str_ymdh),
288
+ 'addLatLon': 'true',
289
+ 'accept': 'netcdf4'
290
+ }
291
+ return query_dict
292
+
293
+
294
+ def set_query_dict_depth_or_level(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh):
295
+ query_dict = {
296
+ 'var': variable_info[var]['var_name'],
297
+ 'north': lat_max,
298
+ 'west': lon_min,
299
+ 'east': lon_max,
300
+ 'south': lat_min,
301
+ 'horizStride': 1,
302
+ 'time': transform_time(time_str_ymdh),
303
+ 'vertCoord': 0,
304
+ 'addLatLon': 'true',
305
+ 'accept': 'netcdf4'
306
+ }
307
+ return query_dict
308
+
309
+
310
+ def set_query_dict_full(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh):
311
+ query_dict = {
312
+ 'var': variable_info[var]['var_name'],
313
+ 'north': lat_max,
314
+ 'west': lon_min,
315
+ 'east': lon_max,
316
+ 'south': lat_min,
317
+ 'horizStride': 1,
318
+ 'time': transform_time(time_str_ymdh),
319
+ 'vertStride': 1,
320
+ 'addLatLon': 'true',
321
+ 'accept': 'netcdf4'
322
+ }
323
+ return query_dict
324
+
325
+
326
+ def get_query_dict_single_depth(var, lon_min, lon_max, lat_min, lat_max, depth, time_str_ymdh):
327
+ if var in ['ssh']:
328
+ query_dict = set_query_dict_no_vertical(
329
+ var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
330
+ else:
331
+ query_dict = set_query_dict_depth_or_level(
332
+ var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
333
+ if var in ['u', 'v', 'temp', 'salinity']:
334
+ print('Please ensure the depth is in the range of 0-5000 m')
335
+ query_dict['vertCoord'] = get_nearest_level_index(depth)+1
336
+ return query_dict
337
+
338
+
339
+ def get_query_dict_single_level(var, lon_min, lon_max, lat_min, lat_max, level_num, time_str_ymdh):
340
+ if var in ['ssh']:
341
+ query_dict = set_query_dict_no_vertical(
342
+ var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
343
+ else:
344
+ # level_num: 1-40
345
+ query_dict = set_query_dict_depth_or_level(
346
+ var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
347
+ if var in ['u', 'v', 'temp', 'salinity']:
348
+ print('Please ensure the level_num is in the range of 1-40')
349
+ if level_num == 0:
350
+ level_num = 1
351
+ print('The level_num is set to 1')
352
+ if level_num > 40:
353
+ level_num = 40
354
+ print('The level_num is set to 40')
355
+ query_dict['vertCoord'] = level_num
356
+ return query_dict
357
+
358
+
359
+ def get_query_dict_full_level(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh):
360
+ if var in ['ssh']:
361
+ query_dict = set_query_dict_no_vertical(
362
+ var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
363
+ else:
364
+ query_dict = set_query_dict_full(
365
+ var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
366
+ return query_dict
367
+
368
+
369
+ def get_query_dict_full_depth(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh):
370
+ if var in ['ssh']:
371
+ query_dict = set_query_dict_no_vertical(
372
+ var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
373
+ else:
374
+ query_dict = set_query_dict_full(
375
+ var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
376
+ return query_dict
377
+
378
+
379
+ def ymd_in_which_dataset_and_version(time_ymdh):
380
+ time_ymdh = int(time_ymdh)
381
+ d_list = []
382
+ v_list = []
383
+ trange_list = []
384
+ have_data = False
385
+ for dataset_name in data_info['hourly']['dataset'].keys():
386
+ for version_name in data_info['hourly']['dataset'][dataset_name]['version'].keys():
387
+ # print(data_info['hourly']['dataset'][dataset_name]
388
+ # ['version'][version_name]['time_range'].values())
389
+ time_s, time_e = list(
390
+ data_info['hourly']['dataset'][dataset_name]['version'][version_name]['time_range'].values())
391
+ time_s, time_e = str(time_s), str(time_e)
392
+ if len(time_s) == 8:
393
+ time_s = time_s + '00'
394
+ if len(time_e) == 8:
395
+ time_e = time_e + '21'
396
+ if time_ymdh >= int(time_s) and time_ymdh <= int(time_e):
397
+ d_list.append(dataset_name)
398
+ v_list.append(version_name)
399
+ trange_list.append(f'{time_s}-{time_e}')
400
+ have_data = True
401
+ print(f'[bold red]{time_ymdh} is in the following dataset and version:')
402
+ if have_data:
403
+ for d, v, trange in zip(d_list, v_list, trange_list):
404
+ # print(f'{time_ymd} is in {d} {v} {trange}')
405
+ print(f'[bold blue]{d} {v} {trange}')
406
+ else:
407
+ raise ValueError(
408
+ f'[bold red]{time_ymdh} is not in any dataset and version')
409
+
410
+
411
+ def direct_choose_dataset_and_version(time_ymdh):
412
+ time_ymdh = int(time_ymdh)
413
+ for dataset_name in data_info['hourly']['dataset'].keys():
414
+ for version_name in data_info['hourly']['dataset'][dataset_name]['version'].keys():
415
+ [time_s, time_e] = list(data_info['hourly']['dataset'][dataset_name]['version'][version_name]['time_range'].values(
416
+ ))
417
+ # print(time_s, time_e, time_ymd)
418
+ time_s, time_e = str(time_s), str(time_e)
419
+ if len(time_s) == 8:
420
+ time_s = time_s + '00'
421
+ if len(time_e) == 8:
422
+ time_e = time_e + '21'
423
+ if time_ymdh >= int(time_s) and time_ymdh <= int(time_e):
424
+ print(
425
+ f'[bold purple]dataset: {dataset_name}, version: {version_name} is chosen')
426
+ # print(f'{time_ymdh} is in {dataset_name} {version_name}')
427
+ return dataset_name, version_name
428
+
429
+
430
+ def get_base_url(dataset_name, version_name, var, year_str):
431
+ url_dict = data_info['hourly']['dataset'][dataset_name]['version'][version_name]['url']
432
+ classification_method = data_info['hourly']['dataset'][dataset_name]['version'][version_name]['classification']
433
+ if classification_method == 'year_different':
434
+ base_url = url_dict[str(year_str)]
435
+ elif classification_method == 'same_path':
436
+ base_url = url_dict
437
+ elif classification_method == 'var_different':
438
+ if var in ['u', 'v', 'u_b', 'v_b']:
439
+ base_url = url_dict['uv3z']
440
+ elif var in ['temp', 'salinity', 'temp_b', 'salinity_b']:
441
+ base_url = url_dict['ts3z']
442
+ elif var in ['ssh']:
443
+ base_url = url_dict['ssh']
444
+ else:
445
+ print(
446
+ 'Please ensure the var is in [u,v,temp,salinity,ssh,u_b,v_b,temp_b,salinity_b]')
447
+ elif classification_method == 'var_year_different':
448
+ if var in ['u', 'v', 'u_b', 'v_b']:
449
+ base_url = url_dict['uv3z'][str(year_str)]
450
+ elif var in ['temp', 'salinity', 'temp_b', 'salinity_b']:
451
+ base_url = url_dict['ts3z'][str(year_str)]
452
+ elif var in ['ssh']:
453
+ base_url = url_dict['ssh'][str(year_str)]
454
+ else:
455
+ print(
456
+ 'Please ensure the var is in [u,v,temp,salinity,ssh,u_b,v_b,temp_b,salinity_b]')
457
+ return base_url
458
+
459
+
460
+ def get_submit_url(dataset_name, version_name, var, year_str, query_dict):
461
+ base_url = get_base_url(dataset_name, version_name, var, year_str)
462
+ query_dict['var'] = [query_dict['var']]
463
+ target_url = base_url + '&'.join(f"var={var}" for var in query_dict['var']) + '&' + '&'.join(
464
+ f"{key}={value}" for key, value in query_dict.items() if key != 'var')
465
+ return target_url
466
+
467
+
468
+ def clear_existing_file(file_full_path):
469
+ if os.path.exists(file_full_path):
470
+ os.remove(file_full_path)
471
+ print(f'{file_full_path} has been removed')
472
+
473
+
474
+ def check_existing_file(file_full_path):
475
+ if os.path.exists(file_full_path):
476
+ print(f'{file_full_path} exists')
477
+ return True
478
+ else:
479
+ print(f'{file_full_path} does not exist')
480
+ return False
481
+
482
+
483
+ def get_ua():
484
+ ua_list = [
485
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60",
486
+ "Opera/8.0 (Windows NT 5.1; U; en)",
487
+ "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50",
488
+ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50",
489
+ "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
490
+ "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
491
+ "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
492
+ "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
493
+ "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
494
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv,2.0.1) Gecko/20100101 Firefox/4.0.1",
495
+ "Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1",
496
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
497
+ "MAC:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36",
498
+ "Windows:Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
499
+ "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
500
+ "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
501
+ "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
502
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36",
503
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
504
+ "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
505
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
506
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36",
507
+ "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
508
+ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
509
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
510
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
511
+ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
512
+ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)"
513
+ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
514
+ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
515
+ "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0",
516
+ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)",
517
+ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
518
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36",
519
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
520
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36",
521
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.4094.1 Safari/537.36",
522
+ "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
523
+ "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
524
+ "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
525
+ "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
526
+ "Mozilla/5.0 (Linux; U; Android 2.2.1; zh-cn; HTC_Wildfire_A3333 Build/FRG83D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
527
+ "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
528
+ "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
529
+ "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
530
+ "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
531
+ "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
532
+ "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
533
+ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;",
534
+ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
535
+ "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
536
+ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
537
+ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
538
+ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
539
+ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
540
+ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)",
541
+ "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
542
+ "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
543
+ "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
544
+ "UCWEB7.0.2.37/28/999",
545
+ "NOKIA5700/UCWEB7.0.2.37/28/999",
546
+ "Openwave/UCWEB7.0.2.37/28/999",
547
+ "Openwave/UCWEB7.0.2.37/28/999",
548
+ ]
549
+ ua_index = random.randint(0, len(ua_list)-1)
550
+ ua = ua_list[ua_index]
551
+ # print(f'Using User-Agent: {ua}')
552
+ return ua
553
+
554
+
555
+ def get_proxy():
556
+ # 获取当前脚本的绝对路径
557
+ script_dir = os.path.dirname(os.path.abspath(__file__))
558
+ # 构建ip.txt的绝对路径
559
+ ip_file_txt = os.path.join(script_dir, 'ip.txt')
560
+ with open(ip_file_txt, 'r') as f:
561
+ ips = f.readlines()
562
+ ip_list = []
563
+ for ip in ips:
564
+ ip_list.append(ip.strip())
565
+ choose_ip = random.choice(ip_list)
566
+ proxies = {
567
+ 'http': 'http://' + choose_ip,
568
+ 'https': 'https://' + choose_ip
569
+ }
570
+ # print(f'Using proxy: {proxies}')
571
+ return proxies
572
+
573
+
574
+ def dlownload_file(target_url, store_path, file_name, check=False):
575
+ print(f'[bold red]Downloading {file_name}...')
576
+ # 创建会话
577
+ s = requests.Session()
578
+ download_success = False
579
+ request_times = 0
580
+ filename = Path(store_path) / file_name
581
+ if check:
582
+ if check_existing_file(filename):
583
+ return
584
+ clear_existing_file(filename)
585
+ print(f'Download_start_time: {datetime.datetime.now()}')
586
+ while not download_success:
587
+ if request_times > 0:
588
+ print(f'\r正在重试第 {request_times} 次', end="")
589
+ # 尝试下载文件
590
+ try:
591
+ headers = {
592
+ 'User-Agent': get_ua()}
593
+ # response = s.get(target_url, headers=headers, stream=True)
594
+ # response = s.get(target_url, headers=headers,
595
+ # proxies=get_proxy(), timeout=5)
596
+ response = s.get(target_url, headers=headers,
597
+ timeout=5)
598
+ # response = s.get(target_url, headers=headers)
599
+ # print(target_url)
600
+ response.raise_for_status() # 如果请求返回的不是200,将抛出HTTPError异常
601
+
602
+ # 保存文件
603
+ with open(filename, 'wb') as f:
604
+ f.write(response.content)
605
+ # print(f'\r文件 {filename} 下载成功', end="")
606
+ print(f'[bold green]文件 {filename} 下载成功')
607
+ # query_ncfile_time(filename) # 这个函数在linux上目前会出问题
608
+ download_success = True
609
+
610
+ except requests.exceptions.HTTPError as errh:
611
+ print(f"Http Error: {errh}")
612
+ except requests.exceptions.ConnectionError as errc:
613
+ print(f"Error Connecting: {errc}")
614
+ except requests.exceptions.Timeout as errt:
615
+ print(f"Timeout Error: {errt}")
616
+ except requests.exceptions.RequestException as err:
617
+ print(f"OOps: Something Else: {err}")
618
+
619
+ time.sleep(3)
620
+ request_times += 1
621
+ print(f'Download_end_time: {datetime.datetime.now()}')
622
+
623
+
624
+ def check_hour_is_valid(ymdh_str):
625
+ # hour should be 00, 03, 06, 09, 12, 15, 18, 21
626
+ hh = int(str(ymdh_str[-2:]))
627
+ if hh in [0, 3, 6, 9, 12, 15, 18, 21]:
628
+ return True
629
+ else:
630
+ return False
631
+
632
+
633
+ def direct_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, download_time='2024083100', depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None, check=False):
634
+ download_time = str(download_time)
635
+ if not check_hour_is_valid(download_time):
636
+ print('Please ensure the hour is 00, 03, 06, 09, 12, 15, 18, 21')
637
+ raise ValueError('The hour is invalid')
638
+ if dataset_name is None and version_name is None:
639
+ print('The dataset_name and version_name are None, so the dataset and version will be chosen according to the download_time.\nIf there is more than one dataset and version in the time range, the first one will be chosen.')
640
+ print('If you wanna choose the dataset and version by yourself, please set the dataset_name and version_name together.')
641
+ ymd_in_which_dataset_and_version(download_time)
642
+ dataset_name, version_name = direct_choose_dataset_and_version(
643
+ download_time)
644
+ elif dataset_name is None and version_name is not None:
645
+ print('Please ensure the dataset_name is not None')
646
+ print('If you do not add the dataset_name, both the dataset and version will be chosen according to the download_time.')
647
+ ymd_in_which_dataset_and_version(download_time)
648
+ dataset_name, version_name = direct_choose_dataset_and_version(
649
+ download_time)
650
+ elif dataset_name is not None and version_name is None:
651
+ print('Please ensure the version_name is not None')
652
+ print('If you do not add the version_name, both the dataset and version will be chosen according to the download_time.')
653
+ ymd_in_which_dataset_and_version(download_time)
654
+ dataset_name, version_name = direct_choose_dataset_and_version(
655
+ download_time)
656
+ else:
657
+ print('The dataset_name and version_name are both set by yourself.')
658
+
659
+ year_str = str(download_time)[:4]
660
+ if depth is not None and level_num is not None:
661
+ print('Please ensure the depth or level_num is None')
662
+ elif depth is not None:
663
+ print(f'Data of single depth ({depth}m) will be downloaded...')
664
+ query_dict = get_query_dict_single_depth(
665
+ var, lon_min, lon_max, lat_min, lat_max, depth, download_time)
666
+ elif level_num is not None:
667
+ print(f'Data of single level ({level_num}) will be downloaded...')
668
+ query_dict = get_query_dict_single_level(
669
+ var, lon_min, lon_max, lat_min, lat_max, level_num, download_time)
670
+ else:
671
+ print('Full depth or full level data will be downloaded...')
672
+ query_dict = get_query_dict_full_level(
673
+ var, lon_min, lon_max, lat_min, lat_max, download_time)
674
+ submit_url = get_submit_url(
675
+ dataset_name, version_name, var, year_str, query_dict)
676
+ file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
677
+ if store_path is None:
678
+ store_path = str(Path.cwd())
679
+ else:
680
+ os.makedirs(str(store_path), exist_ok=True)
681
+ dlownload_file(submit_url, store_path, file_name, check)
682
+
683
+
684
+ def convert_full_name_to_short_name(full_name):
685
+ for var, info in variable_info.items():
686
+ if full_name == info['var_name'] or full_name == info['standard_name'] or full_name == var:
687
+ return var
688
+ print('[bold #FFE4E1]Please ensure the var is in:\n[bold blue]u,v,temp,salinity,ssh,u_b,v_b,temp_b,salinity_b')
689
+ print('or')
690
+ print('[bold blue]water_u, water_v, water_temp, salinity, surf_el, water_u_bottom, water_v_bottom, water_temp_bottom, salinity_bottom')
691
+ return False
692
+
693
+
694
+ def download_task(var, time_str, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check):
695
+ '''
696
+ # 并行下载任务
697
+ # 这个函数是为了并行下载而设置的,是必须的,直接调用direct_download并行下载会出问题
698
+
699
+ 任务封装:将每个任务需要的数据和操作封装在一个函数中,这样每个任务都是独立的,不会相互干扰。
700
+ 本情况下,download_task函数的作用是将每个下载任务封装起来,包括它所需的所有参数。
701
+ 这样,每个任务都是独立的,有自己的参数和数据,不会与其他任务共享或修改任何数据。
702
+ 因此,即使多个任务同时执行,也不会出现数据交互错乱的问题。
703
+ '''
704
+ direct_download(var, lon_min, lon_max, lat_min, lat_max, time_str,
705
+ depth, level, store_path, dataset_name, version_name, check)
706
+
707
+
708
+ def download(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False):
709
+ '''
710
+ Description:
711
+ Download the data of single time or a series of time
712
+
713
+ Parameters:
714
+ var: str, the variable name, such as 'u', 'v', 'temp', 'salinity', 'ssh', 'u_b', 'v_b', 'temp_b', 'salinity_b' or 'water_u', 'water_v', 'water_temp', 'salinity', 'surf_el', 'water_u_bottom', 'water_v_bottom', 'water_temp_bottom', 'salinity_bottom'
715
+ time_s: str, the start time, such as '2024110100' or '20241101', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
716
+ time_e: str, the end time, such as '2024110221' or '20241102', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
717
+ lon_min: float, the minimum longitude, default is 0
718
+ lon_max: float, the maximum longitude, default is 359.92
719
+ lat_min: float, the minimum latitude, default is -80
720
+ lat_max: float, the maximum latitude, default is 90
721
+ depth: float, the depth, default is None
722
+ level: int, the level number, default is None
723
+ store_path: str, the path to store the data, default is None
724
+ dataset_name: str, the dataset name, default is None, example: 'GLBv0.08', 'GLBu0.08', 'GLBy0.08'
725
+ version_name: str, the version name, default is None, example: '53.X', '56.3'
726
+ num_workers: int, the number of workers, default is None
727
+
728
+ Returns:
729
+ None
730
+ '''
731
+ var = convert_full_name_to_short_name(var)
732
+ if var is False:
733
+ raise ValueError('The var is invalid')
734
+ if lon_min < 0 or lon_min > 359.92 or lon_max < 0 or lon_max > 359.92 or lat_min < -80 or lat_min > 90 or lat_max < -80 or lat_max > 90:
735
+ print('Please ensure the lon_min, lon_max, lat_min, lat_max are in the range')
736
+ print('The range of lon_min, lon_max is 0~359.92')
737
+ print('The range of lat_min, lat_max is -80~90')
738
+ raise ValueError('The lon or lat is invalid')
739
+ ymdh_time_s = str(time_s)
740
+ if len(ymdh_time_s) == 8:
741
+ ymdh_time_s += '00'
742
+ ymdh_time_e = str(time_e)
743
+ if len(ymdh_time_e) == 8:
744
+ ymdh_time_e += '21'
745
+ if ymdh_time_s == ymdh_time_e:
746
+ direct_download(var, lon_min, lon_max, lat_min, lat_max,
747
+ ymdh_time_s, depth, level, store_path, dataset_name, version_name)
748
+ elif int(ymdh_time_s) < int(ymdh_time_e):
749
+ print('Downloading a series of files...')
750
+ time_list = get_time_list(ymdh_time_s, ymdh_time_e, 3)
751
+ with Progress() as progress:
752
+ task = progress.add_task(
753
+ "[cyan]Downloading...", total=len(time_list))
754
+ if num_workers is None or num_workers <= 1:
755
+ # 串行方式
756
+ for time_str in time_list:
757
+ direct_download(var, lon_min, lon_max, lat_min, lat_max,
758
+ time_str, depth, level, store_path, dataset_name, version_name, check)
759
+ progress.update(task, advance=1)
760
+ else:
761
+ # 并行方式
762
+ if num_workers > 10:
763
+ print('The number of workers is too large!')
764
+ print(
765
+ 'In order to avoid the server being blocked, the number of workers is set to 10')
766
+ num_workers = 10
767
+ with ThreadPoolExecutor(max_workers=num_workers) as executor:
768
+ futures = [executor.submit(download_task, var, time_str, lon_min, lon_max, lat_min, lat_max,
769
+ depth, level, store_path, dataset_name, version_name, check) for time_str in time_list]
770
+ for future in futures:
771
+ future.add_done_callback(
772
+ lambda _: progress.update(task, advance=1))
773
+ else:
774
+ print('Please ensure the ymdh_time_s is less than the ymdh_time_e')
775
+
776
+
777
+ def how_to_use():
778
+ print('''
779
+ # 1. Choose the dataset and version according to the time:
780
+ # 1.1 Use function to query
781
+ You can use the function ymd_in_which_dataset_and_version(time_ymd=20241101) to find the dataset and version according to the time.
782
+ Then, you can see the dataset and version in the output.
783
+ # 1.2 Draw a picture to see
784
+ You can draw a picture to see the time range of each dataset and version.
785
+ Using the function draw_time_range(pic_save_folder=None) to draw the picture.
786
+
787
+ # 2. Get the base url according to the dataset, version, var and year:
788
+ # 2.1 Dataset and version were found in step 1
789
+ # 2.2 Var: u, v, temp, salinity, ssh, u_b, v_b, temp_b, salinity_b
790
+ # 2.3 Year: 1994-2024(current year)
791
+
792
+ # 3. Get the query_dict according to the var, lon_min, lon_max, lat_min, lat_max, depth, level_num, time_str_ymdh:
793
+ # 3.1 Var: u, v, temp, salinity, ssh, u_b, v_b, temp_b, salinity_b
794
+ # 3.2 Lon_min, lon_max, lat_min, lat_max: float
795
+ # 3.3 Depth: 0-5000m, if you wanna get single depth data, you can set the depth
796
+ # 3.4 Level_num: 1-40, if you wanna get single level data, you can set the level_num
797
+ # 3.5 Time_str_ymdh: '2024110112', the hour normally is 00, 03, 06, 09, 12, 15, 18, 21, besides 1 hourly data
798
+ # 3.6 Use the function to get the query_dict
799
+ # 3.7 Note: If you wanna get the full depth or full level data, you can needn't set the depth or level_num
800
+
801
+ # 4. Get the submit url according to the dataset, version, var, year, query_dict:
802
+ # 4.1 Use the function to get the submit url
803
+ # 4.2 You can use the submit url to download the data
804
+
805
+ # 5. Download the data according to the submit url:
806
+ # 5.1 Use the function to download the data
807
+ # 5.2 You can download the data of single time or a series of time
808
+ # 5.3 Note: If you wanna download a series of data, you can set the ymdh_time_s and ymdh_time_e different
809
+ # 5.4 Note: The time resolution is 3 hours
810
+
811
+ # 6. Direct download the data:
812
+ # 6.1 Use the function to direct download the data
813
+ # 6.2 You can set the dataset_name and version_name by yourself
814
+ # 6.3 Note: If you do not set the dataset_name and version_name, the dataset and version will be chosen according to the download_time
815
+ # 6.4 Note: If you set the dataset_name and version_name, please ensure the dataset_name and version_name are correct
816
+ # 6.5 Note: If you just set one of the dataset_name and version_name, both the dataset and version will be chosen according to the download_time
817
+
818
+ # 7. Simple use:
819
+ # 7.1 You can use the function: download(var, ymdh_time_s, ymdh_time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None)
820
+ # 7.2 You can download the data of single time or a series of time
821
+ # 7.3 The parameters you must set are var, ymdh_time_s, ymdh_time_e
822
+ # 7.4 Example: download('u', '2024110112', '2024110212', lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None)
823
+ ''')
824
+
825
+
826
+ if __name__ == '__main__':
827
+ time_s, time_e = '2018010112', '2018010300'
828
+ merge_name = '201801'
829
+ root_path = r'I:\hycom_data_2018'
830
+ location_dict = {'west': 115, 'east': 130, 'south': 33, 'north': 45}
831
+ download_dict = {
832
+ 'water_u': {'simple_name': 'u', 'download': 1},
833
+ 'water_v': {'simple_name': 'v', 'download': 0},
834
+ 'surf_el': {'simple_name': 'ssh', 'download': 0},
835
+ 'water_temp': {'simple_name': 'temp', 'download': 0},
836
+ 'salinity': {'simple_name': 'salinity', 'download': 1},
837
+ }
838
+
839
+ # set depth or level, only one can be True
840
+ # if you wanna download all depth or level, set both False
841
+ set_depth, depth = False, 0 # 0-4000 meters
842
+ set_level, level = False, 1 # 1-40 levels
843
+ num_workers = 5
844
+ check = True
845
+
846
+ download_switch = True
847
+ combine_switch = False
848
+
849
+ def download_var(var_name):
850
+ var = download_dict[var_name]['simple_name']
851
+ if_download = download_dict[var_name]['download']
852
+ if if_download == 0:
853
+ return
854
+ if set_depth:
855
+ download(var=var, time_s=time_s, time_e=time_e, store_path=Path(root_path)/var, lon_min=location_dict[
856
+ 'west'], lon_max=location_dict['east'], lat_min=location_dict['south'], lat_max=location_dict['north'], depth=depth, num_workers=num_workers, check=check)
857
+ elif set_level:
858
+ download(var=var, time_s=time_s, time_e=time_e, store_path=Path(root_path)/var, lon_min=location_dict[
859
+ 'west'], lon_max=location_dict['east'], lat_min=location_dict['south'], lat_max=location_dict['north'], level=level, num_workers=num_workers, check=check)
860
+ else:
861
+ download(var=var, time_s=time_s, time_e=time_e, store_path=Path(root_path)/var, lon_min=location_dict[
862
+ 'west'], lon_max=location_dict['east'], lat_min=location_dict['south'], lat_max=location_dict['north'], num_workers=num_workers, check=check)
863
+
864
+ if download_switch:
865
+ for var_name in download_dict.keys():
866
+ download_var(var_name)
867
+
868
+ # draw_time_range(r'I:\Delete')
869
+
870
+ """ if combine_switch:
871
+ time_list = get_time_list(time_s, time_e, 3)
872
+ for var_name in download_dict.keys():
873
+ var = download_dict[var_name]['simple_name']
874
+ if_download = download_dict[var_name]['download']
875
+ if if_download == 0:
876
+ continue
877
+ file_list = []
878
+ for time_str in time_list:
879
+ file_list.append(
880
+ Path(root_path)/var/f'HYCOM_{var_name}_{time_str}.nc')
881
+ merge5nc(file_list, var_name, 'time',
882
+ Path(root_path)/var/f'HYCOM_{var_name}_{merge_name}.nc') """