oafuncs 0.0.62__py2.py3-none-any.whl → 0.0.64__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,7 +4,7 @@
4
4
  Author: Liu Kun && 16031215@qq.com
5
5
  Date: 2024-11-01 10:31:09
6
6
  LastEditors: Liu Kun && 16031215@qq.com
7
- LastEditTime: 2024-11-21 13:24:49
7
+ LastEditTime: 2024-11-29 19:00:47
8
8
  FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_down\\hycom_3hourly.py
9
9
  Description:
10
10
  EditPlatform: vscode
@@ -29,7 +29,7 @@ from rich.progress import Progress
29
29
 
30
30
  warnings.filterwarnings("ignore", category=RuntimeWarning, message="Engine '.*' loading failed:.*")
31
31
 
32
- __all__ = ['draw_time_range', 'download', 'how_to_use', 'get_time_list']
32
+ __all__ = ['draw_time_range', 'download', 'how_to_use', 'get_time_list', 'get_hour_list', 'get_day_list']
33
33
 
34
34
  # time resolution
35
35
  data_info = {'yearly': {}, 'monthly': {}, 'daily': {}, 'hourly': {}}
@@ -69,12 +69,12 @@ variable_info = {
69
69
  'u': {'var_name': 'water_u', 'standard_name': 'eastward_sea_water_velocity'},
70
70
  'v': {'var_name': 'water_v', 'standard_name': 'northward_sea_water_velocity'},
71
71
  'temp': {'var_name': 'water_temp', 'standard_name': 'sea_water_potential_temperature'},
72
- 'salinity': {'var_name': 'salinity', 'standard_name': 'sea_water_salinity'},
72
+ 'salt': {'var_name': 'salinity', 'standard_name': 'sea_water_salinity'},
73
73
  'ssh': {'var_name': 'surf_el', 'standard_name': 'sea_surface_elevation'},
74
74
  'u_b': {'var_name': 'water_u_bottom', 'standard_name': 'eastward_sea_water_velocity_at_sea_floor'},
75
75
  'v_b': {'var_name': 'water_v_bottom', 'standard_name': 'northward_sea_water_velocity_at_sea_floor'},
76
76
  'temp_b': {'var_name': 'water_temp_bottom', 'standard_name': 'sea_water_potential_temperature_at_sea_floor'},
77
- 'salinity_b': {'var_name': 'salinity_bottom', 'standard_name': 'sea_water_salinity_at_sea_floor'},
77
+ 'salt_b': {'var_name': 'salinity_bottom', 'standard_name': 'sea_water_salinity_at_sea_floor'},
78
78
  }
79
79
 
80
80
  # classification method
@@ -152,6 +152,12 @@ url_930_y = {
152
152
  }
153
153
  data_info['hourly']['dataset']['GLBy0.08']['version']['93.0']['url'] = url_930_y
154
154
 
155
+ var_group = {
156
+ 'uv3z': ['u', 'v', 'u_b', 'v_b'],
157
+ 'ts3z': ['temp', 'salt', 'temp_b', 'salt_b'],
158
+ 'ssh': ['ssh'],
159
+ }
160
+
155
161
 
156
162
  def draw_time_range(pic_save_folder=None):
157
163
  if pic_save_folder is not None:
@@ -248,6 +254,44 @@ def get_time_list(time_s, time_e, delta_hour):
248
254
  return dt_list
249
255
 
250
256
 
257
+ def get_hour_list(time_s, time_e, delta_hour):
258
+ '''
259
+ Description: get a list of time strings from time_s to time_e with delta_hour
260
+ Args:
261
+ time_s: start time string, e.g. '2023080203'
262
+ time_e: end time string, e.g. '2023080303'
263
+ delta_hour: interval of hours
264
+ Returns:
265
+ dt_list: a list of time strings
266
+ '''
267
+ dt = datetime.datetime.strptime(time_s, '%Y%m%d%H')
268
+ dt_list = []
269
+ while dt.strftime('%Y%m%d%H') <= time_e:
270
+ dt_list.append(dt.strftime('%Y%m%d%H'))
271
+ dt = dt + datetime.timedelta(hours=delta_hour)
272
+ return dt_list
273
+
274
+
275
+ def get_day_list(time_s, time_e, delta_day):
276
+ '''
277
+ Description: get a list of time strings from time_s to time_e with delta_hour
278
+ Args:
279
+ time_s: start time string, e.g. '20230802'
280
+ time_e: end time string, e.g. '20230803'
281
+ delta_hour: interval of hours
282
+ Returns:
283
+ dt_list: a list of time strings
284
+ '''
285
+ time_s = time_s[:8]
286
+ time_e = time_e[:8]
287
+ dt = datetime.datetime.strptime(time_s, '%Y%m%d')
288
+ dt_list = []
289
+ while dt.strftime('%Y%m%d') <= time_e:
290
+ dt_list.append(dt.strftime('%Y%m%d'))
291
+ dt = dt + datetime.timedelta(days=delta_day)
292
+ return dt_list
293
+
294
+
251
295
  def get_nearest_level_index(depth):
252
296
  level_depth = [0.0, 2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 15.0, 20.0, 25.0, 30.0, 35.0, 40.0, 45.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0, 125.0, 150.0, 200.0, 250.0, 300.0, 350.0, 400.0, 500.0, 600.0, 700.0, 800.0, 900.0, 1000.0, 1250.0, 1500.0, 2000.0, 2500.0, 3000.0, 4000.0, 5000]
253
297
  return min(range(len(level_depth)), key=lambda i: abs(level_depth[i]-depth))
@@ -305,7 +349,7 @@ def get_query_dict_single_depth(var, lon_min, lon_max, lat_min, lat_max, depth,
305
349
  query_dict = set_query_dict_no_vertical(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
306
350
  else:
307
351
  query_dict = set_query_dict_depth_or_level(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
308
- if var in ['u', 'v', 'temp', 'salinity']:
352
+ if var in ['u', 'v', 'temp', 'salt']:
309
353
  print('Please ensure the depth is in the range of 0-5000 m')
310
354
  query_dict['vertCoord'] = get_nearest_level_index(depth)+1
311
355
  return query_dict
@@ -317,7 +361,7 @@ def get_query_dict_single_level(var, lon_min, lon_max, lat_min, lat_max, level_n
317
361
  else:
318
362
  # level_num: 1-40
319
363
  query_dict = set_query_dict_depth_or_level(var, lon_min, lon_max, lat_min, lat_max, time_str_ymdh)
320
- if var in ['u', 'v', 'temp', 'salinity']:
364
+ if var in ['u', 'v', 'temp', 'salt']:
321
365
  print('Please ensure the level_num is in the range of 1-40')
322
366
  if level_num == 0:
323
367
  level_num = 1
@@ -345,7 +389,7 @@ def get_query_dict_full_depth(var, lon_min, lon_max, lat_min, lat_max, time_str_
345
389
  return query_dict
346
390
 
347
391
 
348
- def ymd_in_which_dataset_and_version(time_ymdh):
392
+ def ymdh_in_which_dataset_and_version(time_ymdh):
349
393
  time_ymdh = int(time_ymdh)
350
394
  d_list = []
351
395
  v_list = []
@@ -372,6 +416,33 @@ def ymd_in_which_dataset_and_version(time_ymdh):
372
416
  raise ValueError(f'[bold red]{time_ymdh} is not in any dataset and version')
373
417
 
374
418
 
419
+ def ymd_in_which_dataset_and_version(time_ymd):
420
+ time_ymd = int(str(time_ymd)[:8])
421
+ d_list = []
422
+ v_list = []
423
+ trange_list = []
424
+ have_data = False
425
+ for dataset_name in data_info['hourly']['dataset'].keys():
426
+ for version_name in data_info['hourly']['dataset'][dataset_name]['version'].keys():
427
+ time_s, time_e = list(data_info['hourly']['dataset'][dataset_name]['version'][version_name]['time_range'].values())
428
+ time_s, time_e = str(time_s), str(time_e)
429
+ if len(time_s) == 8:
430
+ time_s = time_s + '00'
431
+ if len(time_e) == 8:
432
+ time_e = time_e + '21'
433
+ if time_ymd*100 >= int(time_s) and time_ymd*100+21 <= int(time_e):
434
+ d_list.append(dataset_name)
435
+ v_list.append(version_name)
436
+ trange_list.append(f'{time_s}-{time_e}')
437
+ have_data = True
438
+ print(f'[bold red]{time_ymd} is in the following dataset and version:')
439
+ if have_data:
440
+ for d, v, trange in zip(d_list, v_list, trange_list):
441
+ print(f'[bold blue]{d} {v} {trange}')
442
+ else:
443
+ raise ValueError(f'[bold red]{time_ymd} is not in any dataset and version')
444
+
445
+
375
446
  def direct_choose_dataset_and_version(time_ymdh):
376
447
  time_ymdh = int(time_ymdh)
377
448
  for dataset_name in data_info['hourly']['dataset'].keys():
@@ -387,6 +458,21 @@ def direct_choose_dataset_and_version(time_ymdh):
387
458
  return dataset_name, version_name
388
459
 
389
460
 
461
+ def direct_choose_dataset_and_version_whole_day(time_ymd):
462
+ time_ymd = int(str(time_ymd)[:8])
463
+ for dataset_name in data_info['hourly']['dataset'].keys():
464
+ for version_name in data_info['hourly']['dataset'][dataset_name]['version'].keys():
465
+ [time_s, time_e] = list(data_info['hourly']['dataset'][dataset_name]['version'][version_name]['time_range'].values())
466
+ time_s, time_e = str(time_s), str(time_e)
467
+ if len(time_s) == 8:
468
+ time_s = time_s + '00'
469
+ if len(time_e) == 8:
470
+ time_e = time_e + '21'
471
+ if time_ymd*100 >= int(time_s) and time_ymd*100+21 <= int(time_e):
472
+ print(f'[bold purple]dataset: {dataset_name}, version: {version_name} is chosen')
473
+ return dataset_name, version_name
474
+
475
+
390
476
  def get_base_url(dataset_name, version_name, var, year_str):
391
477
  url_dict = data_info['hourly']['dataset'][dataset_name]['version'][version_name]['url']
392
478
  classification_method = data_info['hourly']['dataset'][dataset_name]['version'][version_name]['classification']
@@ -395,29 +481,28 @@ def get_base_url(dataset_name, version_name, var, year_str):
395
481
  elif classification_method == 'same_path':
396
482
  base_url = url_dict
397
483
  elif classification_method == 'var_different':
398
- if var in ['u', 'v', 'u_b', 'v_b']:
399
- base_url = url_dict['uv3z']
400
- elif var in ['temp', 'salinity', 'temp_b', 'salinity_b']:
401
- base_url = url_dict['ts3z']
402
- elif var in ['ssh']:
403
- base_url = url_dict['ssh']
404
- else:
405
- print('Please ensure the var is in [u,v,temp,salinity,ssh,u_b,v_b,temp_b,salinity_b]')
484
+ base_url = None
485
+ for key, value in var_group.items():
486
+ if var in value:
487
+ base_url = url_dict[key]
488
+ break
489
+ if base_url is None:
490
+ print('Please ensure the var is in [u,v,temp,salt,ssh,u_b,v_b,temp_b,salt_b]')
406
491
  elif classification_method == 'var_year_different':
407
- if var in ['u', 'v', 'u_b', 'v_b']:
408
- base_url = url_dict['uv3z'][str(year_str)]
409
- elif var in ['temp', 'salinity', 'temp_b', 'salinity_b']:
410
- base_url = url_dict['ts3z'][str(year_str)]
411
- elif var in ['ssh']:
412
- base_url = url_dict['ssh'][str(year_str)]
413
- else:
414
- print('Please ensure the var is in [u,v,temp,salinity,ssh,u_b,v_b,temp_b,salinity_b]')
492
+ base_url = None
493
+ for key, value in var_group.items():
494
+ if var in value:
495
+ base_url = url_dict[key][str(year_str)]
496
+ break
497
+ if base_url is None:
498
+ print('Please ensure the var is in [u,v,temp,salt,ssh,u_b,v_b,temp_b,salt_b]')
415
499
  return base_url
416
500
 
417
501
 
418
502
  def get_submit_url(dataset_name, version_name, var, year_str, query_dict):
419
503
  base_url = get_base_url(dataset_name, version_name, var, year_str)
420
- query_dict['var'] = [query_dict['var']]
504
+ if isinstance(query_dict['var'], str):
505
+ query_dict['var'] = [query_dict['var']]
421
506
  target_url = base_url + '&'.join(f"var={var}" for var in query_dict['var']) + '&' + '&'.join(f"{key}={value}" for key, value in query_dict.items() if key != 'var')
422
507
  return target_url
423
508
 
@@ -581,7 +666,7 @@ def check_hour_is_valid(ymdh_str):
581
666
  return False
582
667
 
583
668
 
584
- def direct_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, download_time='2024083100', depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None, check=False):
669
+ def check_dataset_version_single_time(dataset_name, version_name, download_time):
585
670
  download_time = str(download_time)
586
671
  if not check_hour_is_valid(download_time):
587
672
  print('Please ensure the hour is 00, 03, 06, 09, 12, 15, 18, 21')
@@ -589,21 +674,25 @@ def direct_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, dow
589
674
  if dataset_name is None and version_name is None:
590
675
  print('The dataset_name and version_name are None, so the dataset and version will be chosen according to the download_time.\nIf there is more than one dataset and version in the time range, the first one will be chosen.')
591
676
  print('If you wanna choose the dataset and version by yourself, please set the dataset_name and version_name together.')
592
- ymd_in_which_dataset_and_version(download_time)
677
+ ymdh_in_which_dataset_and_version(download_time)
593
678
  dataset_name, version_name = direct_choose_dataset_and_version(download_time)
594
679
  elif dataset_name is None and version_name is not None:
595
680
  print('Please ensure the dataset_name is not None')
596
681
  print('If you do not add the dataset_name, both the dataset and version will be chosen according to the download_time.')
597
- ymd_in_which_dataset_and_version(download_time)
682
+ ymdh_in_which_dataset_and_version(download_time)
598
683
  dataset_name, version_name = direct_choose_dataset_and_version(download_time)
599
684
  elif dataset_name is not None and version_name is None:
600
685
  print('Please ensure the version_name is not None')
601
686
  print('If you do not add the version_name, both the dataset and version will be chosen according to the download_time.')
602
- ymd_in_which_dataset_and_version(download_time)
687
+ ymdh_in_which_dataset_and_version(download_time)
603
688
  dataset_name, version_name = direct_choose_dataset_and_version(download_time)
604
689
  else:
605
690
  print('The dataset_name and version_name are both set by yourself.')
606
691
 
692
+ return dataset_name, version_name
693
+
694
+
695
+ def get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time):
607
696
  year_str = str(download_time)[:4]
608
697
  if depth is not None and level_num is not None:
609
698
  print('Please ensure the depth or level_num is None')
@@ -618,19 +707,135 @@ def direct_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, dow
618
707
  query_dict = get_query_dict_full_level(var, lon_min, lon_max, lat_min, lat_max, download_time)
619
708
  submit_url = get_submit_url(
620
709
  dataset_name, version_name, var, year_str, query_dict)
621
- file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
710
+ return submit_url
711
+
712
+
713
+ def direct_download_single_time(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, download_time='2024083100', depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None, check=False):
714
+ download_time = str(download_time)
715
+ dataset_name, version_name = check_dataset_version_single_time(dataset_name, version_name, download_time)
716
+
622
717
  if store_path is None:
623
718
  store_path = str(Path.cwd())
624
719
  else:
625
720
  os.makedirs(str(store_path), exist_ok=True)
626
- dlownload_file(submit_url, store_path, file_name, check)
721
+
722
+ if isinstance(var, str):
723
+ var = [var]
724
+
725
+ if isinstance(var, list):
726
+ if len(var) == 1:
727
+ var = var[0]
728
+ submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time)
729
+ file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
730
+ dlownload_file(submit_url, store_path, file_name, check)
731
+ else:
732
+ varlist = [_ for _ in var]
733
+ for key, value in var_group.items():
734
+ current_group = []
735
+ for v in varlist:
736
+ if v in value:
737
+ current_group.append(v)
738
+ if len(current_group) == 0:
739
+ continue
740
+
741
+ var = current_group[0]
742
+ submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time)
743
+ file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
744
+ old_str = f'var={variable_info[var]["var_name"]}'
745
+ new_str = f'var={variable_info[var]["var_name"]}'
746
+ if len(current_group) > 1:
747
+ for v in current_group[1:]:
748
+ new_str = f'{new_str}&var={variable_info[v]["var_name"]}'
749
+ submit_url = submit_url.replace(old_str, new_str)
750
+ # file_name = f'HYCOM_{'-'.join([variable_info[v]["var_name"] for v in current_group])}_{download_time}.nc'
751
+ file_name = f'HYCOM_{key}_{download_time}.nc'
752
+ dlownload_file(submit_url, store_path, file_name, check)
753
+
754
+
755
+ def check_dataset_version_whold_day(dataset_name, version_name, download_time):
756
+ download_time = str(download_time)
757
+ if dataset_name is None and version_name is None:
758
+ print('The dataset_name and version_name are None, so the dataset and version will be chosen according to the download_time.\nIf there is more than one dataset and version in the time range, the first one will be chosen.')
759
+ print('If you wanna choose the dataset and version by yourself, please set the dataset_name and version_name together.')
760
+ ymd_in_which_dataset_and_version(download_time)
761
+ dataset_name, version_name = direct_choose_dataset_and_version_whole_day(download_time)
762
+ elif dataset_name is None and version_name is not None:
763
+ print('Please ensure the dataset_name is not None')
764
+ print('If you do not add the dataset_name, both the dataset and version will be chosen according to the download_time.')
765
+ ymd_in_which_dataset_and_version(download_time)
766
+ dataset_name, version_name = direct_choose_dataset_and_version_whole_day(download_time)
767
+ elif dataset_name is not None and version_name is None:
768
+ print('Please ensure the version_name is not None')
769
+ print('If you do not add the version_name, both the dataset and version will be chosen according to the download_time.')
770
+ ymd_in_which_dataset_and_version(download_time)
771
+ dataset_name, version_name = direct_choose_dataset_and_version_whole_day(download_time)
772
+ else:
773
+ print('The dataset_name and version_name are both set by yourself.')
774
+
775
+ return dataset_name, version_name
776
+
777
+
778
+ def direct_download_whole_day(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, download_time='20240831', depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None, check=False):
779
+ download_time = str(download_time)[:8]+'00'
780
+ dataset_name, version_name = check_dataset_version_whold_day(dataset_name, version_name, download_time[:8])
781
+
782
+ if store_path is None:
783
+ store_path = str(Path.cwd())
784
+ else:
785
+ os.makedirs(str(store_path), exist_ok=True)
786
+
787
+ if isinstance(var, str):
788
+ var = [var]
789
+
790
+ if isinstance(var, list):
791
+ if len(var) == 1:
792
+ var = var[0]
793
+ submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time)
794
+
795
+ # https://ncss.hycom.org/thredds/ncss/GLBu0.08/expt_93.0/ts3z?var=salinity&disableLLSubset=on&disableProjSubset=on&horizStride=1&time=2018-12-09T09%3A00%3A00Z&vertCoord=&accept=netcdf4
796
+ # https://ncss.hycom.org/thredds/ncss/GLBu0.08/expt_93.0/ts3z?var=salinity&disableLLSubset=on&disableProjSubset=on&horizStride=1&time_start=2018-09-19T12%3A00%3A00Z&time_end=2018-12-09T09%3A00%3A00Z&timeStride=1&vertCoord=&accept=netcdf4
797
+ # 将time=2018-12-09T09%3A00%3A00Z替换为time_start=2018-09-19T12%3A00%3A00Z&time_end=2018-12-09T09%3A00%3A00Z&timeStride=1
798
+ daytime_s = transform_time(str(download_time)[:8]+'00')
799
+ daytime_e = transform_time(str(download_time)[:8]+'21')
800
+ submit_url = submit_url.replace(
801
+ f'time={daytime_s}', f'time_start={daytime_s}&time_end={daytime_e}&timeStride=1')
802
+
803
+ file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
804
+
805
+ dlownload_file(submit_url, store_path, file_name, check)
806
+ else:
807
+ varlist = [_ for _ in var]
808
+ for key, value in var_group.items():
809
+ current_group = []
810
+ for v in varlist:
811
+ if v in value:
812
+ current_group.append(v)
813
+ if len(current_group) == 0:
814
+ continue
815
+
816
+ var = current_group[0]
817
+ submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time)
818
+ daytime_s = transform_time(str(download_time)[:8]+'00')
819
+ daytime_e = transform_time(str(download_time)[:8]+'21')
820
+ submit_url = submit_url.replace(
821
+ f'time={daytime_s}', f'time_start={daytime_s}&time_end={daytime_e}&timeStride=1')
822
+ file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
823
+ old_str = f'var={variable_info[var]["var_name"]}'
824
+ new_str = f'var={variable_info[var]["var_name"]}'
825
+ if len(current_group) > 1:
826
+ for v in current_group[1:]:
827
+ new_str = f'{new_str}&var={variable_info[v]["var_name"]}'
828
+ submit_url = submit_url.replace(old_str, new_str)
829
+ # file_name = f'HYCOM_{'-'.join([variable_info[v]["var_name"] for v in current_group])}_{download_time}.nc'
830
+ file_name = f'HYCOM_{key}_{download_time}.nc'
831
+ dlownload_file(submit_url, store_path, file_name, check)
627
832
 
628
833
 
629
834
  def convert_full_name_to_short_name(full_name):
630
835
  for var, info in variable_info.items():
631
836
  if full_name == info['var_name'] or full_name == info['standard_name'] or full_name == var:
632
837
  return var
633
- print('[bold #FFE4E1]Please ensure the var is in:\n[bold blue]u,v,temp,salinity,ssh,u_b,v_b,temp_b,salinity_b')
838
+ print('[bold #FFE4E1]Please ensure the var is in:\n[bold blue]u,v,temp,salt,ssh,u_b,v_b,temp_b,salt_b')
634
839
  print('or')
635
840
  print('[bold blue]water_u, water_v, water_temp, salinity, surf_el, water_u_bottom, water_v_bottom, water_temp_bottom, salinity_bottom')
636
841
  return False
@@ -646,16 +851,19 @@ def download_task(var, time_str, lon_min, lon_max, lat_min, lat_max, depth, leve
646
851
  这样,每个任务都是独立的,有自己的参数和数据,不会与其他任务共享或修改任何数据。
647
852
  因此,即使多个任务同时执行,也不会出现数据交互错乱的问题。
648
853
  '''
649
- direct_download(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
854
+ if len(time_str) == 8:
855
+ direct_download_whole_day(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
856
+ else:
857
+ direct_download_single_time(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
650
858
 
651
859
 
652
- def download(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False):
860
+ def download_single_hour(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False):
653
861
  '''
654
862
  Description:
655
863
  Download the data of single time or a series of time
656
864
 
657
865
  Parameters:
658
- var: str, the variable name, such as 'u', 'v', 'temp', 'salinity', 'ssh', 'u_b', 'v_b', 'temp_b', 'salinity_b' or 'water_u', 'water_v', 'water_temp', 'salinity', 'surf_el', 'water_u_bottom', 'water_v_bottom', 'water_temp_bottom', 'salinity_bottom'
866
+ var: str, the variable name, such as 'u', 'v', 'temp', 'salt', 'ssh', 'u_b', 'v_b', 'temp_b', 'salt_b' or 'water_u', 'water_v', 'water_temp', 'salinity', 'surf_el', 'water_u_bottom', 'water_v_bottom', 'water_temp_bottom', 'salinity_bottom'
659
867
  time_s: str, the start time, such as '2024110100' or '20241101', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
660
868
  time_e: str, the end time, such as '2024110221' or '20241102', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
661
869
  lon_min: float, the minimum longitude, default is 0
@@ -672,6 +880,146 @@ def download(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_ma
672
880
  Returns:
673
881
  None
674
882
  '''
883
+ if isinstance(var, list):
884
+ if len(var) == 1:
885
+ var = convert_full_name_to_short_name(var[0])
886
+ else:
887
+ var = [convert_full_name_to_short_name(v) for v in var]
888
+ elif isinstance(var, str):
889
+ var = convert_full_name_to_short_name(var)
890
+ else:
891
+ raise ValueError('The var is invalid')
892
+ if var is False:
893
+ raise ValueError('The var is invalid')
894
+ if lon_min < 0 or lon_min > 359.92 or lon_max < 0 or lon_max > 359.92 or lat_min < -80 or lat_min > 90 or lat_max < -80 or lat_max > 90:
895
+ print('Please ensure the lon_min, lon_max, lat_min, lat_max are in the range')
896
+ print('The range of lon_min, lon_max is 0~359.92')
897
+ print('The range of lat_min, lat_max is -80~90')
898
+ raise ValueError('The lon or lat is invalid')
899
+ ymdh_time_s = str(time_s)
900
+ if len(ymdh_time_s) == 8:
901
+ ymdh_time_s += '00'
902
+ ymdh_time_e = str(time_e)
903
+ if len(ymdh_time_e) == 8:
904
+ ymdh_time_e += '21'
905
+ if ymdh_time_s == ymdh_time_e:
906
+ direct_download_single_time(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, depth, level, store_path, dataset_name, version_name)
907
+ elif int(ymdh_time_s) < int(ymdh_time_e):
908
+ print('Downloading a series of files...')
909
+ time_list = get_hour_list(ymdh_time_s, ymdh_time_e, 3)
910
+ with Progress() as progress:
911
+ task = progress.add_task("[cyan]Downloading...", total=len(time_list))
912
+ if num_workers is None or num_workers <= 1:
913
+ # 串行方式
914
+ for time_str in time_list:
915
+ direct_download_single_time(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
916
+ progress.update(task, advance=1)
917
+ else:
918
+ # 并行方式
919
+ if num_workers > 10:
920
+ print('The number of workers is too large!')
921
+ print('In order to avoid the server being blocked, the number of workers is set to 10')
922
+ num_workers = 10
923
+ with ThreadPoolExecutor(max_workers=num_workers) as executor:
924
+ futures = [executor.submit(download_task, var, time_str, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check) for time_str in time_list]
925
+ for future in futures:
926
+ future.add_done_callback(lambda _: progress.update(task, advance=1))
927
+ else:
928
+ print('Please ensure the time_s is less than the time_e')
929
+
930
+
931
+ def download_whole_day(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False):
932
+ '''
933
+ Description:
934
+ Download the data of single time or a series of time
935
+
936
+ Parameters:
937
+ var: str, the variable name, such as 'u', 'v', 'temp', 'salt', 'ssh', 'u_b', 'v_b', 'temp_b', 'salt_b' or 'water_u', 'water_v', 'water_temp', 'salinity', 'surf_el', 'water_u_bottom', 'water_v_bottom', 'water_temp_bottom', 'salinity_bottom'
938
+ time_s: str, the start time, such as '20241101', without hour
939
+ time_e: str, the end time, such as '20241102', without hour
940
+ lon_min: float, the minimum longitude, default is 0
941
+ lon_max: float, the maximum longitude, default is 359.92
942
+ lat_min: float, the minimum latitude, default is -80
943
+ lat_max: float, the maximum latitude, default is 90
944
+ depth: float, the depth, default is None
945
+ level: int, the level number, default is None
946
+ store_path: str, the path to store the data, default is None
947
+ dataset_name: str, the dataset name, default is None, example: 'GLBv0.08', 'GLBu0.08', 'GLBy0.08'
948
+ version_name: str, the version name, default is None, example: '53.X', '56.3'
949
+ num_workers: int, the number of workers, default is None
950
+
951
+ Returns:
952
+ None
953
+ '''
954
+ if isinstance(var, list):
955
+ if len(var) == 1:
956
+ var = convert_full_name_to_short_name(var[0])
957
+ else:
958
+ var = [convert_full_name_to_short_name(v) for v in var]
959
+ elif isinstance(var, str):
960
+ var = convert_full_name_to_short_name(var)
961
+ else:
962
+ raise ValueError('The var is invalid')
963
+ if var is False:
964
+ raise ValueError('The var is invalid')
965
+ if lon_min < 0 or lon_min > 359.92 or lon_max < 0 or lon_max > 359.92 or lat_min < -80 or lat_min > 90 or lat_max < -80 or lat_max > 90:
966
+ print('Please ensure the lon_min, lon_max, lat_min, lat_max are in the range')
967
+ print('The range of lon_min, lon_max is 0~359.92')
968
+ print('The range of lat_min, lat_max is -80~90')
969
+ raise ValueError('The lon or lat is invalid')
970
+ time_s, time_e = str(time_s)[:8], str(time_e)[:8]
971
+
972
+ if time_s == time_e:
973
+ direct_download_whole_day(var, lon_min, lon_max, lat_min, lat_max, time_s, depth, level, store_path, dataset_name, version_name)
974
+ elif int(time_s) < int(time_e):
975
+ print('Downloading a series of files...')
976
+ time_list = get_day_list(time_s, time_e, 1)
977
+ with Progress() as progress:
978
+ task = progress.add_task("[cyan]Downloading...", total=len(time_list))
979
+ if num_workers is None or num_workers <= 1:
980
+ # 串行方式
981
+ for time_str in time_list:
982
+ direct_download_whole_day(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
983
+ progress.update(task, advance=1)
984
+ else:
985
+ # 并行方式
986
+ if num_workers > 10:
987
+ print('The number of workers is too large!')
988
+ print('In order to avoid the server being blocked, the number of workers is set to 10')
989
+ num_workers = 10
990
+ with ThreadPoolExecutor(max_workers=num_workers) as executor:
991
+ futures = [executor.submit(download_task, var, time_str, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check) for time_str in time_list]
992
+ for future in futures:
993
+ future.add_done_callback(lambda _: progress.update(task, advance=1))
994
+ else:
995
+ print('Please ensure the time_s is less than the time_e')
996
+
997
+
998
+ def download_his1(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False):
999
+ '''
1000
+ Description:
1001
+ Download the data of single time or a series of time
1002
+
1003
+ Parameters:
1004
+ var: str, the variable name, such as 'u', 'v', 'temp', 'salt', 'ssh', 'u_b', 'v_b', 'temp_b', 'salt_b' or 'water_u', 'water_v', 'water_temp', 'salinity', 'surf_el', 'water_u_bottom', 'water_v_bottom', 'water_temp_bottom', 'salinity_bottom'
1005
+ time_s: str, the start time, such as '2024110100' or '20241101', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
1006
+ time_e: str, the end time, such as '2024110221' or '20241102', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
1007
+ lon_min: float, the minimum longitude, default is 0
1008
+ lon_max: float, the maximum longitude, default is 359.92
1009
+ lat_min: float, the minimum latitude, default is -80
1010
+ lat_max: float, the maximum latitude, default is 90
1011
+ depth: float, the depth, default is None
1012
+ level: int, the level number, default is None
1013
+ store_path: str, the path to store the data, default is None
1014
+ dataset_name: str, the dataset name, default is None, example: 'GLBv0.08', 'GLBu0.08', 'GLBy0.08'
1015
+ version_name: str, the version name, default is None, example: '53.X', '56.3'
1016
+ num_workers: int, the number of workers, default is None
1017
+
1018
+ Returns:
1019
+ None
1020
+ '''
1021
+ if isinstance(var, list):
1022
+ var = var[0]
675
1023
  var = convert_full_name_to_short_name(var)
676
1024
  if var is False:
677
1025
  raise ValueError('The var is invalid')
@@ -687,16 +1035,16 @@ def download(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_ma
687
1035
  if len(ymdh_time_e) == 8:
688
1036
  ymdh_time_e += '21'
689
1037
  if ymdh_time_s == ymdh_time_e:
690
- direct_download(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, depth, level, store_path, dataset_name, version_name)
1038
+ direct_download_single_time(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, depth, level, store_path, dataset_name, version_name)
691
1039
  elif int(ymdh_time_s) < int(ymdh_time_e):
692
1040
  print('Downloading a series of files...')
693
- time_list = get_time_list(ymdh_time_s, ymdh_time_e, 3)
1041
+ time_list = get_hour_list(ymdh_time_s, ymdh_time_e, 3)
694
1042
  with Progress() as progress:
695
1043
  task = progress.add_task("[cyan]Downloading...", total=len(time_list))
696
1044
  if num_workers is None or num_workers <= 1:
697
1045
  # 串行方式
698
1046
  for time_str in time_list:
699
- direct_download(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
1047
+ direct_download_single_time(var, lon_min, lon_max, lat_min, lat_max, time_str, depth, level, store_path, dataset_name, version_name, check)
700
1048
  progress.update(task, advance=1)
701
1049
  else:
702
1050
  # 并行方式
@@ -713,6 +1061,40 @@ def download(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_ma
713
1061
  print('Please ensure the ymdh_time_s is less than the ymdh_time_e')
714
1062
 
715
1063
 
1064
+ def download(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False, resolution='hour'):
1065
+ '''
1066
+ Description:
1067
+ Download the data of single time or a series of time
1068
+
1069
+ Parameters:
1070
+ var: str, the variable name, such as 'u', 'v', 'temp', 'salt', 'ssh', 'u_b', 'v_b', 'temp_b', 'salt_b' or 'water_u', 'water_v', 'water_temp', 'salinity', 'surf_el', 'water_u_bottom', 'water_v_bottom', 'water_temp_bottom', 'salinity_bottom'
1071
+ time_s: str, the start time, such as '2024110100' or '20241101', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
1072
+ time_e: str, the end time, such as '2024110221' or '20241102', if add hour, the hour should be 00, 03, 06, 09, 12, 15, 18, 21
1073
+ lon_min: float, the minimum longitude, default is 0
1074
+ lon_max: float, the maximum longitude, default is 359.92
1075
+ lat_min: float, the minimum latitude, default is -80
1076
+ lat_max: float, the maximum latitude, default is 90
1077
+ depth: float, the depth, default is None
1078
+ level: int, the level number, default is None
1079
+ store_path: str, the path to store the data, default is None
1080
+ dataset_name: str, the dataset name, default is None, example: 'GLBv0.08', 'GLBu0.08', 'GLBy0.08'
1081
+ version_name: str, the version name, default is None, example: '53.X', '56.3'
1082
+ num_workers: int, the number of workers, default is None
1083
+
1084
+ Returns:
1085
+ None
1086
+ '''
1087
+ if resolution == 'hour':
1088
+ download_single_hour(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check)
1089
+ elif resolution == 'day':
1090
+ print('Currently can not download the data of whole day...')
1091
+ # download_whole_day(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check)
1092
+ download_single_hour(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check)
1093
+ else:
1094
+ print('Please ensure the resolution is in [hour, day]\n This will be set to hour')
1095
+ download_single_hour(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check)
1096
+
1097
+
716
1098
  def how_to_use():
717
1099
  print('''
718
1100
  # 1. Choose the dataset and version according to the time:
@@ -725,11 +1107,11 @@ def how_to_use():
725
1107
 
726
1108
  # 2. Get the base url according to the dataset, version, var and year:
727
1109
  # 2.1 Dataset and version were found in step 1
728
- # 2.2 Var: u, v, temp, salinity, ssh, u_b, v_b, temp_b, salinity_b
1110
+ # 2.2 Var: u, v, temp, salt, ssh, u_b, v_b, temp_b, salt_b
729
1111
  # 2.3 Year: 1994-2024(current year)
730
1112
 
731
1113
  # 3. Get the query_dict according to the var, lon_min, lon_max, lat_min, lat_max, depth, level_num, time_str_ymdh:
732
- # 3.1 Var: u, v, temp, salinity, ssh, u_b, v_b, temp_b, salinity_b
1114
+ # 3.1 Var: u, v, temp, salt, ssh, u_b, v_b, temp_b, salt_b
733
1115
  # 3.2 Lon_min, lon_max, lat_min, lat_max: float
734
1116
  # 3.3 Depth: 0-5000m, if you wanna get single depth data, you can set the depth
735
1117
  # 3.4 Level_num: 1-40, if you wanna get single level data, you can set the level_num
@@ -763,33 +1145,36 @@ def how_to_use():
763
1145
 
764
1146
 
765
1147
  if __name__ == '__main__':
766
- time_s, time_e = '2018010112', '2018010300'
1148
+ time_s, time_e = '2018010512', '2018010800'
767
1149
  merge_name = '201801'
768
1150
  root_path = r'I:\hycom_data_2018'
769
1151
  location_dict = {'west': 115, 'east': 130, 'south': 33, 'north': 45}
770
1152
  download_dict = {
771
1153
  'water_u': {'simple_name': 'u', 'download': 1},
772
- 'water_v': {'simple_name': 'v', 'download': 0},
773
- 'surf_el': {'simple_name': 'ssh', 'download': 0},
774
- 'water_temp': {'simple_name': 'temp', 'download': 0},
775
- 'salinity': {'simple_name': 'salinity', 'download': 1},
1154
+ 'water_v': {'simple_name': 'v', 'download': 1},
1155
+ 'surf_el': {'simple_name': 'ssh', 'download': 1},
1156
+ 'water_temp': {'simple_name': 'temp', 'download': 1},
1157
+ 'salinity': {'simple_name': 'salt', 'download': 1},
776
1158
  }
777
1159
 
1160
+ var_list = []
1161
+ for var_name in download_dict.keys():
1162
+ if download_dict[var_name]['download'] == 1:
1163
+ var_list.append(download_dict[var_name]['simple_name'])
1164
+
778
1165
  # set depth or level, only one can be True
779
1166
  # if you wanna download all depth or level, set both False
780
1167
  set_depth, depth = False, 0 # 0-4000 meters
781
1168
  set_level, level = False, 1 # 1-40 levels
782
- num_workers = 5
1169
+ num_workers = 1
783
1170
  check = True
784
1171
 
785
1172
  download_switch = True
786
1173
  combine_switch = False
1174
+
1175
+ single_var = True
787
1176
 
788
- def download_var(var_name):
789
- var = download_dict[var_name]['simple_name']
790
- if_download = download_dict[var_name]['download']
791
- if if_download == 0:
792
- return
1177
+ def download_var(var):
793
1178
  if set_depth:
794
1179
  download(var=var, time_s=time_s, time_e=time_e, store_path=Path(root_path)/var, lon_min=location_dict[
795
1180
  'west'], lon_max=location_dict['east'], lat_min=location_dict['south'], lat_max=location_dict['north'], depth=depth, num_workers=num_workers, check=check)
@@ -801,8 +1186,13 @@ if __name__ == '__main__':
801
1186
  'west'], lon_max=location_dict['east'], lat_min=location_dict['south'], lat_max=location_dict['north'], num_workers=num_workers, check=check)
802
1187
 
803
1188
  if download_switch:
804
- for var_name in download_dict.keys():
805
- download_var(var_name)
1189
+ if single_var:
1190
+ # single var
1191
+ for var_name in var_list:
1192
+ download_var(var_name)
1193
+ else:
1194
+ # multiple var
1195
+ download_var(var_list)
806
1196
 
807
1197
  # draw_time_range(r'I:\Delete')
808
1198