oafuncs 0.0.62__py2.py3-none-any.whl → 0.0.64__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oafuncs/oa_down/hycom_3hourly.py +442 -52
- oafuncs/oa_down/hycom_3hourly_wholeday.py +1173 -0
- oafuncs/oa_down/refs_pdf.py +1 -1
- {oafuncs-0.0.62.dist-info → oafuncs-0.0.64.dist-info}/METADATA +1 -1
- {oafuncs-0.0.62.dist-info → oafuncs-0.0.64.dist-info}/RECORD +8 -7
- {oafuncs-0.0.62.dist-info → oafuncs-0.0.64.dist-info}/LICENSE.txt +0 -0
- {oafuncs-0.0.62.dist-info → oafuncs-0.0.64.dist-info}/WHEEL +0 -0
- {oafuncs-0.0.62.dist-info → oafuncs-0.0.64.dist-info}/top_level.txt +0 -0
oafuncs/oa_down/hycom_3hourly.py
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
Author: Liu Kun && 16031215@qq.com
|
5
5
|
Date: 2024-11-01 10:31:09
|
6
6
|
LastEditors: Liu Kun && 16031215@qq.com
|
7
|
-
LastEditTime: 2024-11-
|
7
|
+
LastEditTime: 2024-11-29 19:00:47
|
8
8
|
FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_down\\hycom_3hourly.py
|
9
9
|
Description:
|
10
10
|
EditPlatform: vscode
|
@@ -29,7 +29,7 @@ from rich.progress import Progress
|
|
29
29
|
|
30
30
|
warnings.filterwarnings("ignore", category=RuntimeWarning, message="Engine '.*' loading failed:.*")
|
31
31
|
|
32
|
-
__all__ = ['draw_time_range', 'download', 'how_to_use', 'get_time_list']
|
32
|
+
__all__ = ['draw_time_range', 'download', 'how_to_use', 'get_time_list', 'get_hour_list', 'get_day_list']
|
33
33
|
|
34
34
|
# time resolution
|
35
35
|
data_info = {'yearly': {}, 'monthly': {}, 'daily': {}, 'hourly': {}}
|
@@ -69,12 +69,12 @@ variable_info = {
|
|
69
69
|
'u': {'var_name': 'water_u', 'standard_name': 'eastward_sea_water_velocity'},
|
70
70
|
'v': {'var_name': 'water_v', 'standard_name': 'northward_sea_water_velocity'},
|
71
71
|
'temp': {'var_name': 'water_temp', 'standard_name': 'sea_water_potential_temperature'},
|
72
|
-
'
|
72
|
+
'salt': {'var_name': 'salinity', 'standard_name': 'sea_water_salinity'},
|
73
73
|
'ssh': {'var_name': 'surf_el', 'standard_name': 'sea_surface_elevation'},
|
74
74
|
'u_b': {'var_name': 'water_u_bottom', 'standard_name': 'eastward_sea_water_velocity_at_sea_floor'},
|
75
75
|
'v_b': {'var_name': 'water_v_bottom', 'standard_name': 'northward_sea_water_velocity_at_sea_floor'},
|
76
76
|
'temp_b': {'var_name': 'water_temp_bottom', 'standard_name': 'sea_water_potential_temperature_at_sea_floor'},
|
77
|
-
'
|
77
|
+
'salt_b': {'var_name': 'salinity_bottom', 'standard_name': 'sea_water_salinity_at_sea_floor'},
|
78
78
|
}
|
79
79
|
|
80
80
|
# classification method
|
@@ -152,6 +152,12 @@ url_930_y = {
|
|
152
152
|
}
|
153
153
|
data_info['hourly']['dataset']['GLBy0.08']['version']['93.0']['url'] = url_930_y
|
154
154
|
|
155
|
+
var_group = {
|
156
|
+
'uv3z': ['u', 'v', 'u_b', 'v_b'],
|
157
|
+
'ts3z': ['temp', 'salt', 'temp_b', 'salt_b'],
|
158
|
+
'ssh': ['ssh'],
|
159
|
+
}
|
160
|
+
|
155
161
|
|
156
162
|
def draw_time_range(pic_save_folder=None):
|
157
163
|
if pic_save_folder is not None:
|
@@ -248,6 +254,44 @@ def get_time_list(time_s, time_e, delta_hour):
|
|
248
254
|
return dt_list
|
249
255
|
|
250
256
|
|
257
|
+
def get_hour_list(time_s, time_e, delta_hour):
|
258
|
+
'''
|
259
|
+
Description: get a list of time strings from time_s to time_e with delta_hour
|
260
|
+
Args:
|
261
|
+
time_s: start time string, e.g. '2023080203'
|
262
|
+
time_e: end time string, e.g. '2023080303'
|
263
|
+
delta_hour: interval of hours
|
264
|
+
Returns:
|
265
|
+
dt_list: a list of time strings
|
266
|
+
'''
|
267
|
+
dt = datetime.datetime.strptime(time_s, '%Y%m%d%H')
|
268
|
+
dt_list = []
|
269
|
+
while dt.strftime('%Y%m%d%H') <= time_e:
|
270
|
+
dt_list.append(dt.strftime('%Y%m%d%H'))
|
271
|
+
dt = dt + datetime.timedelta(hours=delta_hour)
|
272
|
+
return dt_list
|
273
|
+
|
274
|
+
|
275
|
+
def get_day_list(time_s, time_e, delta_day):
|
276
|
+
'''
|
277
|
+
Description: get a list of time strings from time_s to time_e with delta_hour
|
278
|
+
Args:
|
279
|
+
time_s: start time string, e.g. '20230802'
|
280
|
+
time_e: end time string, e.g. '20230803'
|
281
|
+
delta_hour: interval of hours
|
282
|
+
Returns:
|
283
|
+
dt_list: a list of time strings
|
284
|
+
'''
|
285
|
+
time_s = time_s[:8]
|
286
|
+
time_e = time_e[:8]
|
287
|
+
dt = datetime.datetime.strptime(time_s, '%Y%m%d')
|
288
|
+
dt_list = []
|
289
|
+
while dt.strftime('%Y%m%d') <= time_e:
|
290
|
+
dt_list.append(dt.strftime('%Y%m%d'))
|
291
|
+
dt = dt + datetime.timedelta(days=delta_day)
|
292
|
+
return dt_list
|
293
|
+
|
294
|
+
|
251
295
|
def get_nearest_level_index(depth):
|
252
296
|
level_depth = [0.0, 2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 15.0, 20.0, 25.0, 30.0, 35.0, 40.0, 45.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0, 125.0, 150.0, 200.0, 250.0, 300.0, 350.0, 400.0, 500.0, 600.0, 700.0, 800.0, 900.0, 1000.0, 1250.0, 1500.0, 2000.0, 2500.0, 3000.0, 4000.0, 5000]
|
253
297
|
return min(range(len(level_depth)), key=lambda i: abs(level_depth[i]-depth))
|
@@ -305,7 +349,7 @@ def get_query_dict_single_depth(var, lon_min, lon_max, lat_min, lat_max, depth,
|
|
305
349
|
query_dict = set_query_dict_no_vertical(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
|
306
350
|
else:
|
307
351
|
query_dict = set_query_dict_depth_or_level(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
|
308
|
-
if var in ['u', 'v', 'temp', '
|
352
|
+
if var in ['u', 'v', 'temp', 'salt']:
|
309
353
|
print('Please ensure the depth is in the range of 0-5000 m')
|
310
354
|
query_dict['vertCoord'] = get_nearest_level_index(depth)+1
|
311
355
|
return query_dict
|
@@ -317,7 +361,7 @@ def get_query_dict_single_level(var, lon_min, lon_max, lat_min, lat_max, level_n
|
|
317
361
|
else:
|
318
362
|
# level_num: 1-40
|
319
363
|
query_dict = set_query_dict_depth_or_level(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
|
320
|
-
if var in ['u', 'v', 'temp', '
|
364
|
+
if var in ['u', 'v', 'temp', 'salt']:
|
321
365
|
print('Please ensure the level_num is in the range of 1-40')
|
322
366
|
if level_num == 0:
|
323
367
|
level_num = 1
|
@@ -345,7 +389,7 @@ def get_query_dict_full_depth(var, lon_min, lon_max, lat_min, lat_max, time_str_
|
|
345
389
|
return query_dict
|
346
390
|
|
347
391
|
|
348
|
-
def
|
392
|
+
def ymdh_in_which_dataset_and_version(time_ymdh):
|
349
393
|
time_ymdh = int(time_ymdh)
|
350
394
|
d_list = []
|
351
395
|
v_list = []
|
@@ -372,6 +416,33 @@ def ymd_in_which_dataset_and_version(time_ymdh):
|
|
372
416
|
raise ValueError(f'[bold red]{time_ymdh} is not in any dataset and version')
|
373
417
|
|
374
418
|
|
419
|
+
def ymd_in_which_dataset_and_version(time_ymd):
|
420
|
+
time_ymd = int(str(time_ymd)[:8])
|
421
|
+
d_list = []
|
422
|
+
v_list = []
|
423
|
+
trange_list = []
|
424
|
+
have_data = False
|
425
|
+
for dataset_name in data_info['hourly']['dataset'].keys():
|
426
|
+
for version_name in data_info['hourly']['dataset'][dataset_name]['version'].keys():
|
427
|
+
time_s, time_e = list(data_info['hourly']['dataset'][dataset_name]['version'][version_name]['time_range'].values())
|
428
|
+
time_s, time_e = str(time_s), str(time_e)
|
429
|
+
if len(time_s) == 8:
|
430
|
+
time_s = time_s + '00'
|
431
|
+
if len(time_e) == 8:
|
432
|
+
time_e = time_e + '21'
|
433
|
+
if time_ymd*100 >= int(time_s) and time_ymd*100+21 <= int(time_e):
|
434
|
+
d_list.append(dataset_name)
|
435
|
+
v_list.append(version_name)
|
436
|
+
trange_list.append(f'{time_s}-{time_e}')
|
437
|
+
have_data = True
|
438
|
+
print(f'[bold red]{time_ymd} is in the following dataset and version:')
|
439
|
+
if have_data:
|
440
|
+
for d, v, trange in zip(d_list, v_list, trange_list):
|
441
|
+
print(f'[bold blue]{d} {v} {trange}')
|
442
|
+
else:
|
443
|
+
raise ValueError(f'[bold red]{time_ymd} is not in any dataset and version')
|
444
|
+
|
445
|
+
|
375
446
|
def direct_choose_dataset_and_version(time_ymdh):
|
376
447
|
time_ymdh = int(time_ymdh)
|
377
448
|
for dataset_name in data_info['hourly']['dataset'].keys():
|
@@ -387,6 +458,21 @@ def direct_choose_dataset_and_version(time_ymdh):
|
|
387
458
|
return dataset_name, version_name
|
388
459
|
|
389
460
|
|
461
|
+
def direct_choose_dataset_and_version_whole_day(time_ymd):
|
462
|
+
time_ymd = int(str(time_ymd)[:8])
|
463
|
+
for dataset_name in data_info['hourly']['dataset'].keys():
|
464
|
+
for version_name in data_info['hourly']['dataset'][dataset_name]['version'].keys():
|
465
|
+
[time_s, time_e] = list(data_info['hourly']['dataset'][dataset_name]['version'][version_name]['time_range'].values())
|
466
|
+
time_s, time_e = str(time_s), str(time_e)
|
467
|
+
if len(time_s) == 8:
|
468
|
+
time_s = time_s + '00'
|
469
|
+
if len(time_e) == 8:
|
470
|
+
time_e = time_e + '21'
|
471
|
+
if time_ymd*100 >= int(time_s) and time_ymd*100+21 <= int(time_e):
|
472
|
+
print(f'[bold purple]dataset: {dataset_name}, version: {version_name} is chosen')
|
473
|
+
return dataset_name, version_name
|
474
|
+
|
475
|
+
|
390
476
|
def get_base_url(dataset_name, version_name, var, year_str):
|
391
477
|
url_dict = data_info['hourly']['dataset'][dataset_name]['version'][version_name]['url']
|
392
478
|
classification_method = data_info['hourly']['dataset'][dataset_name]['version'][version_name]['classification']
|
@@ -395,29 +481,28 @@ def get_base_url(dataset_name, version_name, var, year_str):
|
|
395
481
|
elif classification_method == 'same_path':
|
396
482
|
base_url = url_dict
|
397
483
|
elif classification_method == 'var_different':
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
print('Please ensure the var is in [u,v,temp,salinity,ssh,u_b,v_b,temp_b,salinity_b]')
|
484
|
+
base_url = None
|
485
|
+
for key, value in var_group.items():
|
486
|
+
if var in value:
|
487
|
+
base_url = url_dict[key]
|
488
|
+
break
|
489
|
+
if base_url is None:
|
490
|
+
print('Please ensure the var is in [u,v,temp,salt,ssh,u_b,v_b,temp_b,salt_b]')
|
406
491
|
elif classification_method == 'var_year_different':
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
print('Please ensure the var is in [u,v,temp,salinity,ssh,u_b,v_b,temp_b,salinity_b]')
|
492
|
+
base_url = None
|
493
|
+
for key, value in var_group.items():
|
494
|
+
if var in value:
|
495
|
+
base_url = url_dict[key][str(year_str)]
|
496
|
+
break
|
497
|
+
if base_url is None:
|
498
|
+
print('Please ensure the var is in [u,v,temp,salt,ssh,u_b,v_b,temp_b,salt_b]')
|
415
499
|
return base_url
|
416
500
|
|
417
501
|
|
418
502
|
def get_submit_url(dataset_name, version_name, var, year_str, query_dict):
|
419
503
|
base_url = get_base_url(dataset_name, version_name, var, year_str)
|
420
|
-
query_dict['var']
|
504
|
+
if isinstance(query_dict['var'], str):
|
505
|
+
query_dict['var'] = [query_dict['var']]
|
421
506
|
target_url = base_url + '&'.join(f"var={var}" for var in query_dict['var']) + '&' + '&'.join(f"{key}={value}" for key, value in query_dict.items() if key != 'var')
|
422
507
|
return target_url
|
423
508
|
|
@@ -581,7 +666,7 @@ def check_hour_is_valid(ymdh_str):
|
|
581
666
|
return False
|
582
667
|
|
583
668
|
|
584
|
-
def
|
669
|
+
def check_dataset_version_single_time(dataset_name, version_name, download_time):
|
585
670
|
download_time = str(download_time)
|
586
671
|
if not check_hour_is_valid(download_time):
|
587
672
|
print('Please ensure the hour is 00, 03, 06, 09, 12, 15, 18, 21')
|
@@ -589,21 +674,25 @@ def direct_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, dow
|
|
589
674
|
if dataset_name is None and version_name is None:
|
590
675
|
print('The dataset_name and version_name are None, so the dataset and version will be chosen according to the download_time.\nIf there is more than one dataset and version in the time range, the first one will be chosen.')
|
591
676
|
print('If you wanna choose the dataset and version by yourself, please set the dataset_name and version_name together.')
|
592
|
-
|
677
|
+
ymdh_in_which_dataset_and_version(download_time)
|
593
678
|
dataset_name, version_name = direct_choose_dataset_and_version(download_time)
|
594
679
|
elif dataset_name is None and version_name is not None:
|
595
680
|
print('Please ensure the dataset_name is not None')
|
596
681
|
print('If you do not add the dataset_name, both the dataset and version will be chosen according to the download_time.')
|
597
|
-
|
682
|
+
ymdh_in_which_dataset_and_version(download_time)
|
598
683
|
dataset_name, version_name = direct_choose_dataset_and_version(download_time)
|
599
684
|
elif dataset_name is not None and version_name is None:
|
600
685
|
print('Please ensure the version_name is not None')
|
601
686
|
print('If you do not add the version_name, both the dataset and version will be chosen according to the download_time.')
|
602
|
-
|
687
|
+
ymdh_in_which_dataset_and_version(download_time)
|
603
688
|
dataset_name, version_name = direct_choose_dataset_and_version(download_time)
|
604
689
|
else:
|
605
690
|
print('The dataset_name and version_name are both set by yourself.')
|
606
691
|
|
692
|
+
return dataset_name, version_name
|
693
|
+
|
694
|
+
|
695
|
+
def get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time):
|
607
696
|
year_str = str(download_time)[:4]
|
608
697
|
if depth is not None and level_num is not None:
|
609
698
|
print('Please ensure the depth or level_num is None')
|
@@ -618,19 +707,135 @@ def direct_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, dow
|
|
618
707
|
query_dict = get_query_dict_full_level(var, lon_min, lon_max, lat_min, lat_max, download_time)
|
619
708
|
submit_url = get_submit_url(
|
620
709
|
dataset_name, version_name, var, year_str, query_dict)
|
621
|
-
|
710
|
+
return submit_url
|
711
|
+
|
712
|
+
|
713
|
+
def direct_download_single_time(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, download_time='2024083100', depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None, check=False):
|
714
|
+
download_time = str(download_time)
|
715
|
+
dataset_name, version_name = check_dataset_version_single_time(dataset_name, version_name, download_time)
|
716
|
+
|
622
717
|
if store_path is None:
|
623
718
|
store_path = str(Path.cwd())
|
624
719
|
else:
|
625
720
|
os.makedirs(str(store_path), exist_ok=True)
|
626
|
-
|
721
|
+
|
722
|
+
if isinstance(var, str):
|
723
|
+
var = [var]
|
724
|
+
|
725
|
+
if isinstance(var, list):
|
726
|
+
if len(var) == 1:
|
727
|
+
var = var[0]
|
728
|
+
submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time)
|
729
|
+
file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
|
730
|
+
dlownload_file(submit_url, store_path, file_name, check)
|
731
|
+
else:
|
732
|
+
varlist = [_ for _ in var]
|
733
|
+
for key, value in var_group.items():
|
734
|
+
current_group = []
|
735
|
+
for v in varlist:
|
736
|
+
if v in value:
|
737
|
+
current_group.append(v)
|
738
|
+
if len(current_group) == 0:
|
739
|
+
continue
|
740
|
+
|
741
|
+
var = current_group[0]
|
742
|
+
submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time)
|
743
|
+
file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
|
744
|
+
old_str = f'var={variable_info[var]["var_name"]}'
|
745
|
+
new_str = f'var={variable_info[var]["var_name"]}'
|
746
|
+
if len(current_group) > 1:
|
747
|
+
for v in current_group[1:]:
|
748
|
+
new_str = f'{new_str}&var={variable_info[v]["var_name"]}'
|
749
|
+
submit_url = submit_url.replace(old_str, new_str)
|
750
|
+
# file_name = f'HYCOM_{'-'.join([variable_info[v]["var_name"] for v in current_group])}_{download_time}.nc'
|
751
|
+
file_name = f'HYCOM_{key}_{download_time}.nc'
|
752
|
+
dlownload_file(submit_url, store_path, file_name, check)
|
753
|
+
|
754
|
+
|
755
|
+
def check_dataset_version_whold_day(dataset_name, version_name, download_time):
|
756
|
+
download_time = str(download_time)
|
757
|
+
if dataset_name is None and version_name is None:
|
758
|
+
print('The dataset_name and version_name are None, so the dataset and version will be chosen according to the download_time.\nIf there is more than one dataset and version in the time range, the first one will be chosen.')
|
759
|
+
print('If you wanna choose the dataset and version by yourself, please set the dataset_name and version_name together.')
|
760
|
+
ymd_in_which_dataset_and_version(download_time)
|
761
|
+
dataset_name, version_name = direct_choose_dataset_and_version_whole_day(download_time)
|
762
|
+
elif dataset_name is None and version_name is not None:
|
763
|
+
print('Please ensure the dataset_name is not None')
|
764
|
+
print('If you do not add the dataset_name, both the dataset and version will be chosen according to the download_time.')
|
765
|
+
ymd_in_which_dataset_and_version(download_time)
|
766
|
+
dataset_name, version_name = direct_choose_dataset_and_version_whole_day(download_time)
|
767
|
+
elif dataset_name is not None and version_name is None:
|
768
|
+
print('Please ensure the version_name is not None')
|
769
|
+
print('If you do not add the version_name, both the dataset and version will be chosen according to the download_time.')
|
770
|
+
ymd_in_which_dataset_and_version(download_time)
|
771
|
+
dataset_name, version_name = direct_choose_dataset_and_version_whole_day(download_time)
|
772
|
+
else:
|
773
|
+
print('The dataset_name and version_name are both set by yourself.')
|
774
|
+
|
775
|
+
return dataset_name, version_name
|
776
|
+
|
777
|
+
|
778
|
+
def direct_download_whole_day(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, download_time='20240831', depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None, check=False):
|
779
|
+
download_time = str(download_time)[:8]+'00'
|
780
|
+
dataset_name, version_name = check_dataset_version_whold_day(dataset_name, version_name, download_time[:8])
|
781
|
+
|
782
|
+
if store_path is None:
|
783
|
+
store_path = str(Path.cwd())
|
784
|
+
else:
|
785
|
+
os.makedirs(str(store_path), exist_ok=True)
|
786
|
+
|
787
|
+
if isinstance(var, str):
|
788
|
+
var = [var]
|
789
|
+
|
790
|
+
if isinstance(var, list):
|
791
|
+
if len(var) == 1:
|
792
|
+
var = var[0]
|
793
|
+
submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time)
|
794
|
+
|
795
|
+
# https://ncss.hycom.org/thredds/ncss/GLBu0.08/expt_93.0/ts3z?var=salinity&disableLLSubset=on&disableProjSubset=on&horizStride=1&time=2018-12-09T09%3A00%3A00Z&vertCoord=&accept=netcdf4
|
796
|
+
# https://ncss.hycom.org/thredds/ncss/GLBu0.08/expt_93.0/ts3z?var=salinity&disableLLSubset=on&disableProjSubset=on&horizStride=1&time_start=2018-09-19T12%3A00%3A00Z&time_end=2018-12-09T09%3A00%3A00Z&timeStride=1&vertCoord=&accept=netcdf4
|
797
|
+
# 将time=2018-12-09T09%3A00%3A00Z替换为time_start=2018-09-19T12%3A00%3A00Z&time_end=2018-12-09T09%3A00%3A00Z&timeStride=1
|
798
|
+
daytime_s = transform_time(str(download_time)[:8]+'00')
|
799
|
+
daytime_e = transform_time(str(download_time)[:8]+'21')
|
800
|
+
submit_url = submit_url.replace(
|
801
|
+
f'time={daytime_s}', f'time_start={daytime_s}&time_end={daytime_e}&timeStride=1')
|
802
|
+
|
803
|
+
file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
|
804
|
+
|
805
|
+
dlownload_file(submit_url, store_path, file_name, check)
|
806
|
+
else:
|
807
|
+
varlist = [_ for _ in var]
|
808
|
+
for key, value in var_group.items():
|
809
|
+
current_group = []
|
810
|
+
for v in varlist:
|
811
|
+
if v in value:
|
812
|
+
current_group.append(v)
|
813
|
+
if len(current_group) == 0:
|
814
|
+
continue
|
815
|
+
|
816
|
+
var = current_group[0]
|
817
|
+
submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time)
|
818
|
+
daytime_s = transform_time(str(download_time)[:8]+'00')
|
819
|
+
daytime_e = transform_time(str(download_time)[:8]+'21')
|
820
|
+
submit_url = submit_url.replace(
|
821
|
+
f'time={daytime_s}', f'time_start={daytime_s}&time_end={daytime_e}&timeStride=1')
|
822
|
+
file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
|
823
|
+
old_str = f'var={variable_info[var]["var_name"]}'
|
824
|
+
new_str = f'var={variable_info[var]["var_name"]}'
|
825
|
+
if len(current_group) > 1:
|
826
|
+
for v in current_group[1:]:
|
827
|
+
new_str = f'{new_str}&var={variable_info[v]["var_name"]}'
|
828
|
+
submit_url = submit_url.replace(old_str, new_str)
|
829
|
+
# file_name = f'HYCOM_{'-'.join([variable_info[v]["var_name"] for v in current_group])}_{download_time}.nc'
|
830
|
+
file_name = f'HYCOM_{key}_{download_time}.nc'
|
831
|
+
dlownload_file(submit_url, store_path, file_name, check)
|
627
832
|
|
628
833
|
|
629
834
|
def convert_full_name_to_short_name(full_name):
|
630
835
|
for var, info in variable_info.items():
|
631
836
|
if full_name == info['var_name'] or full_name == info['standard_name'] or full_name == var:
|
632
837
|
return var
|
633
|
-
print('[bold #FFE4E1]Please ensure the var is in:\n[bold blue]u,v,temp,
|
838
|
+
print('[bold #FFE4E1]Please ensure the var is in:\n[bold blue]u,v,temp,salt,ssh,u_b,v_b,temp_b,salt_b')
|
634
839
|
print('or')
|
635
840
|
print('[bold blue]water_u, water_v, water_temp, salinity, surf_el, water_u_bottom, water_v_bottom, water_temp_bottom, salinity_bottom')
|
636
841
|
return False
|
@@ -646,16 +851,19 @@ def download_task(var, time_str, lon_min, lon_max, lat_min, lat_max, depth, leve
|
|
646
851
|
这样,每个任务都是独立的,有自己的参数和数据,不会与其他任务共享或修改任何数据。
|
647
852
|
因此,即使多个任务同时执行,也不会出现数据交互错乱的问题。
|
648
853
|
'''
|
649
|
-
|
854
|
+
if len(time_str) == 8:
|
855
|
+
direct_download_whole_day(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
|
856
|
+
else:
|
857
|
+
direct_download_single_time(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
|
650
858
|
|
651
859
|
|
652
|
-
def
|
860
|
+
def download_single_hour(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False):
|
653
861
|
'''
|
654
862
|
Description:
|
655
863
|
Download the data of single time or a series of time
|
656
864
|
|
657
865
|
Parameters:
|
658
|
-
var: str, the variable name, such as 'u', 'v', 'temp', '
|
866
|
+
var: str, the variable name, such as 'u', 'v', 'temp', 'salt', 'ssh', 'u_b', 'v_b', 'temp_b', 'salt_b' or 'water_u', 'water_v', 'water_temp', 'salinity', 'surf_el', 'water_u_bottom', 'water_v_bottom', 'water_temp_bottom', 'salinity_bottom'
|
659
867
|
time_s: str, the start time, such as '2024110100' or '20241101', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
|
660
868
|
time_e: str, the end time, such as '2024110221' or '20241102', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
|
661
869
|
lon_min: float, the minimum longitude, default is 0
|
@@ -672,6 +880,146 @@ def download(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_ma
|
|
672
880
|
Returns:
|
673
881
|
None
|
674
882
|
'''
|
883
|
+
if isinstance(var, list):
|
884
|
+
if len(var) == 1:
|
885
|
+
var = convert_full_name_to_short_name(var[0])
|
886
|
+
else:
|
887
|
+
var = [convert_full_name_to_short_name(v) for v in var]
|
888
|
+
elif isinstance(var, str):
|
889
|
+
var = convert_full_name_to_short_name(var)
|
890
|
+
else:
|
891
|
+
raise ValueError('The var is invalid')
|
892
|
+
if var is False:
|
893
|
+
raise ValueError('The var is invalid')
|
894
|
+
if lon_min < 0 or lon_min > 359.92 or lon_max < 0 or lon_max > 359.92 or lat_min < -80 or lat_min > 90 or lat_max < -80 or lat_max > 90:
|
895
|
+
print('Please ensure the lon_min, lon_max, lat_min, lat_max are in the range')
|
896
|
+
print('The range of lon_min, lon_max is 0~359.92')
|
897
|
+
print('The range of lat_min, lat_max is -80~90')
|
898
|
+
raise ValueError('The lon or lat is invalid')
|
899
|
+
ymdh_time_s = str(time_s)
|
900
|
+
if len(ymdh_time_s) == 8:
|
901
|
+
ymdh_time_s += '00'
|
902
|
+
ymdh_time_e = str(time_e)
|
903
|
+
if len(ymdh_time_e) == 8:
|
904
|
+
ymdh_time_e += '21'
|
905
|
+
if ymdh_time_s == ymdh_time_e:
|
906
|
+
direct_download_single_time(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, depth, level, store_path, dataset_name, version_name)
|
907
|
+
elif int(ymdh_time_s) < int(ymdh_time_e):
|
908
|
+
print('Downloading a series of files...')
|
909
|
+
time_list = get_hour_list(ymdh_time_s, ymdh_time_e, 3)
|
910
|
+
with Progress() as progress:
|
911
|
+
task = progress.add_task("[cyan]Downloading...", total=len(time_list))
|
912
|
+
if num_workers is None or num_workers <= 1:
|
913
|
+
# 串行方式
|
914
|
+
for time_str in time_list:
|
915
|
+
direct_download_single_time(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
|
916
|
+
progress.update(task, advance=1)
|
917
|
+
else:
|
918
|
+
# 并行方式
|
919
|
+
if num_workers > 10:
|
920
|
+
print('The number of workers is too large!')
|
921
|
+
print('In order to avoid the server being blocked, the number of workers is set to 10')
|
922
|
+
num_workers = 10
|
923
|
+
with ThreadPoolExecutor(max_workers=num_workers) as executor:
|
924
|
+
futures = [executor.submit(download_task, var, time_str, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check) for time_str in time_list]
|
925
|
+
for future in futures:
|
926
|
+
future.add_done_callback(lambda _: progress.update(task, advance=1))
|
927
|
+
else:
|
928
|
+
print('Please ensure the time_s is less than the time_e')
|
929
|
+
|
930
|
+
|
931
|
+
def download_whole_day(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False):
|
932
|
+
'''
|
933
|
+
Description:
|
934
|
+
Download the data of single time or a series of time
|
935
|
+
|
936
|
+
Parameters:
|
937
|
+
var: str, the variable name, such as 'u', 'v', 'temp', 'salt', 'ssh', 'u_b', 'v_b', 'temp_b', 'salt_b' or 'water_u', 'water_v', 'water_temp', 'salinity', 'surf_el', 'water_u_bottom', 'water_v_bottom', 'water_temp_bottom', 'salinity_bottom'
|
938
|
+
time_s: str, the start time, such as '20241101', without hour
|
939
|
+
time_e: str, the end time, such as '20241102', without hour
|
940
|
+
lon_min: float, the minimum longitude, default is 0
|
941
|
+
lon_max: float, the maximum longitude, default is 359.92
|
942
|
+
lat_min: float, the minimum latitude, default is -80
|
943
|
+
lat_max: float, the maximum latitude, default is 90
|
944
|
+
depth: float, the depth, default is None
|
945
|
+
level: int, the level number, default is None
|
946
|
+
store_path: str, the path to store the data, default is None
|
947
|
+
dataset_name: str, the dataset name, default is None, example: 'GLBv0.08', 'GLBu0.08', 'GLBy0.08'
|
948
|
+
version_name: str, the version name, default is None, example: '53.X', '56.3'
|
949
|
+
num_workers: int, the number of workers, default is None
|
950
|
+
|
951
|
+
Returns:
|
952
|
+
None
|
953
|
+
'''
|
954
|
+
if isinstance(var, list):
|
955
|
+
if len(var) == 1:
|
956
|
+
var = convert_full_name_to_short_name(var[0])
|
957
|
+
else:
|
958
|
+
var = [convert_full_name_to_short_name(v) for v in var]
|
959
|
+
elif isinstance(var, str):
|
960
|
+
var = convert_full_name_to_short_name(var)
|
961
|
+
else:
|
962
|
+
raise ValueError('The var is invalid')
|
963
|
+
if var is False:
|
964
|
+
raise ValueError('The var is invalid')
|
965
|
+
if lon_min < 0 or lon_min > 359.92 or lon_max < 0 or lon_max > 359.92 or lat_min < -80 or lat_min > 90 or lat_max < -80 or lat_max > 90:
|
966
|
+
print('Please ensure the lon_min, lon_max, lat_min, lat_max are in the range')
|
967
|
+
print('The range of lon_min, lon_max is 0~359.92')
|
968
|
+
print('The range of lat_min, lat_max is -80~90')
|
969
|
+
raise ValueError('The lon or lat is invalid')
|
970
|
+
time_s, time_e = str(time_s)[:8], str(time_e)[:8]
|
971
|
+
|
972
|
+
if time_s == time_e:
|
973
|
+
direct_download_whole_day(var, lon_min, lon_max, lat_min, lat_max, time_s, depth, level, store_path, dataset_name, version_name)
|
974
|
+
elif int(time_s) < int(time_e):
|
975
|
+
print('Downloading a series of files...')
|
976
|
+
time_list = get_day_list(time_s, time_e, 1)
|
977
|
+
with Progress() as progress:
|
978
|
+
task = progress.add_task("[cyan]Downloading...", total=len(time_list))
|
979
|
+
if num_workers is None or num_workers <= 1:
|
980
|
+
# 串行方式
|
981
|
+
for time_str in time_list:
|
982
|
+
direct_download_whole_day(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
|
983
|
+
progress.update(task, advance=1)
|
984
|
+
else:
|
985
|
+
# 并行方式
|
986
|
+
if num_workers > 10:
|
987
|
+
print('The number of workers is too large!')
|
988
|
+
print('In order to avoid the server being blocked, the number of workers is set to 10')
|
989
|
+
num_workers = 10
|
990
|
+
with ThreadPoolExecutor(max_workers=num_workers) as executor:
|
991
|
+
futures = [executor.submit(download_task, var, time_str, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check) for time_str in time_list]
|
992
|
+
for future in futures:
|
993
|
+
future.add_done_callback(lambda _: progress.update(task, advance=1))
|
994
|
+
else:
|
995
|
+
print('Please ensure the time_s is less than the time_e')
|
996
|
+
|
997
|
+
|
998
|
+
def download_his1(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False):
|
999
|
+
'''
|
1000
|
+
Description:
|
1001
|
+
Download the data of single time or a series of time
|
1002
|
+
|
1003
|
+
Parameters:
|
1004
|
+
var: str, the variable name, such as 'u', 'v', 'temp', 'salt', 'ssh', 'u_b', 'v_b', 'temp_b', 'salt_b' or 'water_u', 'water_v', 'water_temp', 'salinity', 'surf_el', 'water_u_bottom', 'water_v_bottom', 'water_temp_bottom', 'salinity_bottom'
|
1005
|
+
time_s: str, the start time, such as '2024110100' or '20241101', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
|
1006
|
+
time_e: str, the end time, such as '2024110221' or '20241102', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
|
1007
|
+
lon_min: float, the minimum longitude, default is 0
|
1008
|
+
lon_max: float, the maximum longitude, default is 359.92
|
1009
|
+
lat_min: float, the minimum latitude, default is -80
|
1010
|
+
lat_max: float, the maximum latitude, default is 90
|
1011
|
+
depth: float, the depth, default is None
|
1012
|
+
level: int, the level number, default is None
|
1013
|
+
store_path: str, the path to store the data, default is None
|
1014
|
+
dataset_name: str, the dataset name, default is None, example: 'GLBv0.08', 'GLBu0.08', 'GLBy0.08'
|
1015
|
+
version_name: str, the version name, default is None, example: '53.X', '56.3'
|
1016
|
+
num_workers: int, the number of workers, default is None
|
1017
|
+
|
1018
|
+
Returns:
|
1019
|
+
None
|
1020
|
+
'''
|
1021
|
+
if isinstance(var, list):
|
1022
|
+
var = var[0]
|
675
1023
|
var = convert_full_name_to_short_name(var)
|
676
1024
|
if var is False:
|
677
1025
|
raise ValueError('The var is invalid')
|
@@ -687,16 +1035,16 @@ def download(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_ma
|
|
687
1035
|
if len(ymdh_time_e) == 8:
|
688
1036
|
ymdh_time_e += '21'
|
689
1037
|
if ymdh_time_s == ymdh_time_e:
|
690
|
-
|
1038
|
+
direct_download_single_time(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, depth, level, store_path, dataset_name, version_name)
|
691
1039
|
elif int(ymdh_time_s) < int(ymdh_time_e):
|
692
1040
|
print('Downloading a series of files...')
|
693
|
-
time_list =
|
1041
|
+
time_list = get_hour_list(ymdh_time_s, ymdh_time_e, 3)
|
694
1042
|
with Progress() as progress:
|
695
1043
|
task = progress.add_task("[cyan]Downloading...", total=len(time_list))
|
696
1044
|
if num_workers is None or num_workers <= 1:
|
697
1045
|
# 串行方式
|
698
1046
|
for time_str in time_list:
|
699
|
-
|
1047
|
+
direct_download_single_time(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
|
700
1048
|
progress.update(task, advance=1)
|
701
1049
|
else:
|
702
1050
|
# 并行方式
|
@@ -713,6 +1061,40 @@ def download(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_ma
|
|
713
1061
|
print('Please ensure the ymdh_time_s is less than the ymdh_time_e')
|
714
1062
|
|
715
1063
|
|
1064
|
+
def download(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False, resolution='hour'):
|
1065
|
+
'''
|
1066
|
+
Description:
|
1067
|
+
Download the data of single time or a series of time
|
1068
|
+
|
1069
|
+
Parameters:
|
1070
|
+
var: str, the variable name, such as 'u', 'v', 'temp', 'salt', 'ssh', 'u_b', 'v_b', 'temp_b', 'salt_b' or 'water_u', 'water_v', 'water_temp', 'salinity', 'surf_el', 'water_u_bottom', 'water_v_bottom', 'water_temp_bottom', 'salinity_bottom'
|
1071
|
+
time_s: str, the start time, such as '2024110100' or '20241101', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
|
1072
|
+
time_e: str, the end time, such as '2024110221' or '20241102', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
|
1073
|
+
lon_min: float, the minimum longitude, default is 0
|
1074
|
+
lon_max: float, the maximum longitude, default is 359.92
|
1075
|
+
lat_min: float, the minimum latitude, default is -80
|
1076
|
+
lat_max: float, the maximum latitude, default is 90
|
1077
|
+
depth: float, the depth, default is None
|
1078
|
+
level: int, the level number, default is None
|
1079
|
+
store_path: str, the path to store the data, default is None
|
1080
|
+
dataset_name: str, the dataset name, default is None, example: 'GLBv0.08', 'GLBu0.08', 'GLBy0.08'
|
1081
|
+
version_name: str, the version name, default is None, example: '53.X', '56.3'
|
1082
|
+
num_workers: int, the number of workers, default is None
|
1083
|
+
|
1084
|
+
Returns:
|
1085
|
+
None
|
1086
|
+
'''
|
1087
|
+
if resolution == 'hour':
|
1088
|
+
download_single_hour(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check)
|
1089
|
+
elif resolution == 'day':
|
1090
|
+
print('Currently can not download the data of whole day...')
|
1091
|
+
# download_whole_day(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check)
|
1092
|
+
download_single_hour(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check)
|
1093
|
+
else:
|
1094
|
+
print('Please ensure the resolution is in [hour, day]\n This will be set to hour')
|
1095
|
+
download_single_hour(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check)
|
1096
|
+
|
1097
|
+
|
716
1098
|
def how_to_use():
|
717
1099
|
print('''
|
718
1100
|
# 1. Choose the dataset and version according to the time:
|
@@ -725,11 +1107,11 @@ def how_to_use():
|
|
725
1107
|
|
726
1108
|
# 2. Get the base url according to the dataset, version, var and year:
|
727
1109
|
# 2.1 Dataset and version were found in step 1
|
728
|
-
# 2.2 Var: u, v, temp,
|
1110
|
+
# 2.2 Var: u, v, temp, salt, ssh, u_b, v_b, temp_b, salt_b
|
729
1111
|
# 2.3 Year: 1994-2024(current year)
|
730
1112
|
|
731
1113
|
# 3. Get the query_dict according to the var, lon_min, lon_max, lat_min, lat_max, depth, level_num, time_str_ymdh:
|
732
|
-
# 3.1 Var: u, v, temp,
|
1114
|
+
# 3.1 Var: u, v, temp, salt, ssh, u_b, v_b, temp_b, salt_b
|
733
1115
|
# 3.2 Lon_min, lon_max, lat_min, lat_max: float
|
734
1116
|
# 3.3 Depth: 0-5000m, if you wanna get single depth data, you can set the depth
|
735
1117
|
# 3.4 Level_num: 1-40, if you wanna get single level data, you can set the level_num
|
@@ -763,33 +1145,36 @@ def how_to_use():
|
|
763
1145
|
|
764
1146
|
|
765
1147
|
if __name__ == '__main__':
|
766
|
-
time_s, time_e = '
|
1148
|
+
time_s, time_e = '2018010512', '2018010800'
|
767
1149
|
merge_name = '201801'
|
768
1150
|
root_path = r'I:\hycom_data_2018'
|
769
1151
|
location_dict = {'west': 115, 'east': 130, 'south': 33, 'north': 45}
|
770
1152
|
download_dict = {
|
771
1153
|
'water_u': {'simple_name': 'u', 'download': 1},
|
772
|
-
'water_v': {'simple_name': 'v', 'download':
|
773
|
-
'surf_el': {'simple_name': 'ssh', 'download':
|
774
|
-
'water_temp': {'simple_name': 'temp', 'download':
|
775
|
-
'salinity': {'simple_name': '
|
1154
|
+
'water_v': {'simple_name': 'v', 'download': 1},
|
1155
|
+
'surf_el': {'simple_name': 'ssh', 'download': 1},
|
1156
|
+
'water_temp': {'simple_name': 'temp', 'download': 1},
|
1157
|
+
'salinity': {'simple_name': 'salt', 'download': 1},
|
776
1158
|
}
|
777
1159
|
|
1160
|
+
var_list = []
|
1161
|
+
for var_name in download_dict.keys():
|
1162
|
+
if download_dict[var_name]['download'] == 1:
|
1163
|
+
var_list.append(download_dict[var_name]['simple_name'])
|
1164
|
+
|
778
1165
|
# set depth or level, only one can be True
|
779
1166
|
# if you wanna download all depth or level, set both False
|
780
1167
|
set_depth, depth = False, 0 # 0-4000 meters
|
781
1168
|
set_level, level = False, 1 # 1-40 levels
|
782
|
-
num_workers =
|
1169
|
+
num_workers = 1
|
783
1170
|
check = True
|
784
1171
|
|
785
1172
|
download_switch = True
|
786
1173
|
combine_switch = False
|
1174
|
+
|
1175
|
+
single_var = True
|
787
1176
|
|
788
|
-
def download_var(
|
789
|
-
var = download_dict[var_name]['simple_name']
|
790
|
-
if_download = download_dict[var_name]['download']
|
791
|
-
if if_download == 0:
|
792
|
-
return
|
1177
|
+
def download_var(var):
|
793
1178
|
if set_depth:
|
794
1179
|
download(var=var, time_s=time_s, time_e=time_e, store_path=Path(root_path)/var, lon_min=location_dict[
|
795
1180
|
'west'], lon_max=location_dict['east'], lat_min=location_dict['south'], lat_max=location_dict['north'], depth=depth, num_workers=num_workers, check=check)
|
@@ -801,8 +1186,13 @@ if __name__ == '__main__':
|
|
801
1186
|
'west'], lon_max=location_dict['east'], lat_min=location_dict['south'], lat_max=location_dict['north'], num_workers=num_workers, check=check)
|
802
1187
|
|
803
1188
|
if download_switch:
|
804
|
-
|
805
|
-
|
1189
|
+
if single_var:
|
1190
|
+
# single var
|
1191
|
+
for var_name in var_list:
|
1192
|
+
download_var(var_name)
|
1193
|
+
else:
|
1194
|
+
# multiple var
|
1195
|
+
download_var(var_list)
|
806
1196
|
|
807
1197
|
# draw_time_range(r'I:\Delete')
|
808
1198
|
|