oafuncs 0.0.80__py2.py3-none-any.whl → 0.0.82__py2.py3-none-any.whl

oafuncs/oa_down/hycom_3hourly.py

@@ -21,6 +21,7 @@ import warnings
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from threading import Lock
+import re
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -29,6 +30,7 @@ import requests
 from bs4 import BeautifulSoup
 from rich import print
 from rich.progress import Progress
+import glob
 
 warnings.filterwarnings("ignore", category=RuntimeWarning, message="Engine '.*' loading failed:.*")
 
@@ -39,7 +41,7 @@ data_info = {"yearly": {}, "monthly": {}, "daily": {}, "hourly": {}}
 
 # hourly data
 # dataset: GLBv0.08, GLBu0.08, GLBy0.08
-data_info["hourly"]["dataset"] = {"GLBv0.08": {}, "GLBu0.08": {}, "GLBy0.08": {}}
+data_info["hourly"]["dataset"] = {"GLBv0.08": {}, "GLBu0.08": {}, "GLBy0.08": {}, "ESPC_D": {}}
 
 # version
 # version of GLBv0.08: 53.X, 56.3, 57.2, 92.8, 57.7, 92.9, 93.0
@@ -48,6 +50,8 @@ data_info["hourly"]["dataset"]["GLBv0.08"]["version"] = {"53.X": {}, "56.3": {},
 data_info["hourly"]["dataset"]["GLBu0.08"]["version"] = {"93.0": {}}
 # version of GLBy0.08: 93.0
 data_info["hourly"]["dataset"]["GLBy0.08"]["version"] = {"93.0": {}}
+# version of ESPC_D: V02
+data_info["hourly"]["dataset"]["ESPC_D"]["version"] = {"V02": {}}
 
 # info details
 # time range
@@ -65,7 +69,9 @@ data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["93.0"]["time_range"] = {"
 # GLBu0.08
 data_info["hourly"]["dataset"]["GLBu0.08"]["version"]["93.0"]["time_range"] = {"time_start": "2018091912", "time_end": "2018120909"}
 # GLBy0.08
-data_info["hourly"]["dataset"]["GLBy0.08"]["version"]["93.0"]["time_range"] = {"time_start": "2018120412", "time_end": "20300904"}
+data_info["hourly"]["dataset"]["GLBy0.08"]["version"]["93.0"]["time_range"] = {"time_start": "2018120412", "time_end": "2024090509"}
+# ESPC-D
+data_info["hourly"]["dataset"]["ESPC_D"]["version"]["V02"]["time_range"] = {"time_start": "2024081012", "time_end": "2030010100"}
 
 # variable
 variable_info = {
@@ -94,6 +100,7 @@ data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["92.9"]["classification"]
 data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["93.0"]["classification"] = "var_different"
 data_info["hourly"]["dataset"]["GLBu0.08"]["version"]["93.0"]["classification"] = "var_different"
 data_info["hourly"]["dataset"]["GLBy0.08"]["version"]["93.0"]["classification"] = "var_year_different"
+data_info["hourly"]["dataset"]["ESPC_D"]["version"]["V02"]["classification"] = "single_var_year_different"
 
 # download info
 # base url
@@ -141,7 +148,7 @@ data_info["hourly"]["dataset"]["GLBu0.08"]["version"]["93.0"]["url"] = url_930_u
 uv3z_930_y = {}
 ts3z_930_y = {}
 ssh_930_y = {}
-for y_930_y in range(2018, 2030):
+for y_930_y in range(2018, 2025):
     uv3z_930_y[str(y_930_y)] = rf"https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/uv3z/{y_930_y}?"
     ts3z_930_y[str(y_930_y)] = rf"https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/ts3z/{y_930_y}?"
     ssh_930_y[str(y_930_y)] = rf"https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/ssh/{y_930_y}?"
@@ -152,12 +159,39 @@ url_930_y = {
     "ssh": ssh_930_y,
 }
 data_info["hourly"]["dataset"]["GLBy0.08"]["version"]["93.0"]["url"] = url_930_y
+# ESPC-D-V02
+u3z_espc_d_v02_y = {}
+v3z_espc_d_v02_y = {}
+t3z_espc_d_v02_y = {}
+s3z_espc_d_v02_y = {}
+ssh_espc_d_v02_y = {}
+for y_espc_d_v02 in range(2024, 2030):
+    u3z_espc_d_v02_y[str(y_espc_d_v02)] = rf"https://ncss.hycom.org/thredds/ncss/ESPC-D-V02/u3z/{y_espc_d_v02}?"
+    v3z_espc_d_v02_y[str(y_espc_d_v02)] = rf"https://ncss.hycom.org/thredds/ncss/ESPC-D-V02/v3z/{y_espc_d_v02}?"
+    t3z_espc_d_v02_y[str(y_espc_d_v02)] = rf"https://ncss.hycom.org/thredds/ncss/ESPC-D-V02/t3z/{y_espc_d_v02}?"
+    s3z_espc_d_v02_y[str(y_espc_d_v02)] = rf"https://ncss.hycom.org/thredds/ncss/ESPC-D-V02/s3z/{y_espc_d_v02}?"
+    ssh_espc_d_v02_y[str(y_espc_d_v02)] = rf"https://ncss.hycom.org/thredds/ncss/ESPC-D-V02/ssh/{y_espc_d_v02}?"
+url_espc_d_v02_y = {
+    "u3z": u3z_espc_d_v02_y,
+    "v3z": v3z_espc_d_v02_y,
+    "t3z": t3z_espc_d_v02_y,
+    "s3z": s3z_espc_d_v02_y,
+    "ssh": ssh_espc_d_v02_y,
+}
+data_info['hourly']['dataset']['ESPC_D']['version']['V02']['url'] = url_espc_d_v02_y
 
 var_group = {
     "uv3z": ["u", "v", "u_b", "v_b"],
     "ts3z": ["temp", "salt", "temp_b", "salt_b"],
     "ssh": ["ssh"],
 }
+single_var_group = {
+    "u3z": ["u"],
+    "v3z": ["v"],
+    "t3z": ["temp"],
+    "s3z": ["salt"],
+    "ssh": ["ssh"],
+}
 
 
 def draw_time_range(pic_save_folder=None):
@@ -205,7 +239,8 @@ def draw_time_range(pic_save_folder=None):
         #          f"{row['version']}", ha='right', color='black')
         ymdh_s = row["start_date"].strftime("%Y-%m-%d %H")
         ymdh_e = row["end_date"].strftime("%Y-%m-%d %H")
-        if k == 1 or k == len(combined_labels):
+        # if k == 1 or k == len(combined_labels):
+        if k == 1:
             plt.text(row["start_date"], k + 0.125, f"{ymdh_s}", ha="left", color="black")
             plt.text(row["end_date"], k + 0.125, f"{ymdh_e}", ha="right", color="black")
         else:
@@ -496,6 +531,14 @@ def get_base_url(dataset_name, version_name, var, ymdh_str):
                 break
         if base_url is None:
             print("Please ensure the var is in [u,v,temp,salt,ssh,u_b,v_b,temp_b,salt_b]")
+    elif classification_method == "single_var_year_different":
+        base_url = None
+        for key, value in single_var_group.items():
+            if var in value:
+                base_url = url_dict[key][str(year_str)]
+                break
+        if base_url is None:
+            print("Please ensure the var is in [u,v,temp,salt,ssh]")
     return base_url
 
 
@@ -513,7 +556,44 @@ def clear_existing_file(file_full_path):
         print(f"{file_full_path} has been removed")
 
 
-def _get_file_size(file_path, unit="KB"):
+def find_file(parent_path, fname, mode="path"):
+    """
+    description:
+    param {*} parent_path: The parent path where the files are located
+    param {*} fname: The file name pattern to search for
+    param {*} mode: 'path' to return the full path of the files, 'file' to return only the file names
+    return {*} A list of file paths or file names if files are found, None otherwise
+    """
+
+    def natural_sort_key(s):
+        """生成一个用于自然排序的键"""
+        return [int(text) if text.isdigit() else text.lower() for text in re.split("([0-9]+)", s)]
+
+    # 将parent_path和fname结合成完整的搜索路径
+    search_pattern = os.path.join(str(parent_path), fname)
+
+    # 使用glob模块查找所有匹配的文件
+    matched_files = glob.glob(search_pattern)
+
+    # 如果没有找到任何文件，则返回False
+    if not matched_files:
+        return None
+
+    # 在find_files函数中替换natsorted调用
+    matched_files = sorted(matched_files, key=natural_sort_key)
+
+    # 根据mode参数决定返回的内容
+    if mode == "file":
+        # 只返回文件名
+        result = [os.path.basename(file) for file in matched_files]
+    else:  # 默认为'path'
+        # 返回文件的绝对路径
+        result = [os.path.abspath(file) for file in matched_files]
+
+    return result
+
+
+def file_size(file_path, unit="KB"):
     # 检查文件是否存在
     if not os.path.exists(file_path):
         return "文件不存在"
@@ -534,12 +614,34 @@ def _get_file_size(file_path, unit="KB"):
     return converted_size
 
 
-def check_existing_file(file_full_path):
+# ** 计算文件夹下指定相关文件的平均大小
+def mean_size(parent_path, fname):
+    flist = find_file(parent_path, fname)
+    if flist:
+        size_list = [file_size(f) for f in flist if file_size(f) != 0]
+    else:
+        size_list = []
+    if size_list:
+        min_size, max_size = min(size_list), max(size_list)
+        mean_size = sum(size_list) / len(size_list)
+    else:
+        mean_size, min_size, max_size = 0, 0, 0
+    return mean_size, min_size, max_size
+
+
+def check_existing_file(file_full_path, min_size):
     if os.path.exists(file_full_path):
         print(f"[bold #FFA54F]{file_full_path} exists")
-        fsize = _get_file_size(file_full_path)
+        fsize = file_size(file_full_path)
+        if min_size:
+            if fsize < min_size:
+                print(f"[bold #FFA54F]{file_full_path} ({fsize:.2f} KB) may be incomplete")
+                # clear_existing_file(file_full_path)
+                return False
+            else:
+                return True
         if fsize < 5:
-            print(f"[bold #FFA54F]{file_full_path} may be incomplete\nFile size: {fsize:.2f} KB")
+            print(f"[bold #FFA54F]{file_full_path} ({fsize:.2f} KB) may be incomplete")
             # clear_existing_file(file_full_path)
             return False
         else:
@@ -623,8 +725,13 @@ def get_proxy():
 def download_file(target_url, store_path, file_name, check=False):
     # Check if the file exists
     fname = Path(store_path) / file_name
+    file_name_split = file_name.split("_")
+    same_file = f'{file_name_split[0]}_{file_name_split[1]}*nc'
+    fsize_mean, fsize_min, fsize_max = mean_size(store_path, same_file)
+    set_min_size = fsize_mean - 0.5 * (fsize_max - fsize_min)
+    set_min_size = set_min_size if set_min_size > 0 else fsize_min
     if check:
-        if check_existing_file(fname):
+        if check_existing_file(fname, set_min_size*0.8):
             count_dict["skip"] += 1
             return
     clear_existing_file(fname)
@@ -819,29 +926,37 @@ def prepare_url_to_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max
                 file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}_{download_time_end}.nc"
             download_file(submit_url, store_path, file_name, check)
         else:
-            varlist = [_ for _ in var]
-            for key, value in var_group.items():
-                current_group = []
-                for v in varlist:
-                    if v in value:
-                        current_group.append(v)
-                if len(current_group) == 0:
-                    continue
-
-                var = current_group[0]
-                submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time, download_time_end)
-                file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
-                old_str = f'var={variable_info[var]["var_name"]}'
-                new_str = f'var={variable_info[var]["var_name"]}'
-                if len(current_group) > 1:
-                    for v in current_group[1:]:
-                        new_str = f'{new_str}&var={variable_info[v]["var_name"]}'
-                    submit_url = submit_url.replace(old_str, new_str)
-                    # file_name = f'HYCOM_{'-'.join([variable_info[v]["var_name"] for v in current_group])}_{download_time}.nc'
-                    file_name = f"HYCOM_{key}_{download_time}.nc"
+            if download_time < "2024081012":
+                varlist = [_ for _ in var]
+                for key, value in var_group.items():
+                    current_group = []
+                    for v in varlist:
+                        if v in value:
+                            current_group.append(v)
+                    if len(current_group) == 0:
+                        continue
+
+                    var = current_group[0]
+                    submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time, download_time_end)
+                    file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
+                    old_str = f'var={variable_info[var]["var_name"]}'
+                    new_str = f'var={variable_info[var]["var_name"]}'
+                    if len(current_group) > 1:
+                        for v in current_group[1:]:
+                            new_str = f'{new_str}&var={variable_info[v]["var_name"]}'
+                        submit_url = submit_url.replace(old_str, new_str)
+                        # file_name = f'HYCOM_{'-'.join([variable_info[v]["var_name"] for v in current_group])}_{download_time}.nc'
+                        file_name = f"HYCOM_{key}_{download_time}.nc"
+                        if download_time_end is not None:
+                            file_name = f"HYCOM_{key}_{download_time}_{download_time_end}.nc"
+                    download_file(submit_url, store_path, file_name, check)
+            else:
+                for v in var:
+                    submit_url = get_submit_url_var(v, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time, download_time_end)
+                    file_name = f"HYCOM_{variable_info[v]['var_name']}_{download_time}.nc"
                     if download_time_end is not None:
-                        file_name = f"HYCOM_{key}_{download_time}_{download_time_end}.nc"
-                download_file(submit_url, store_path, file_name, check)
+                        file_name = f"HYCOM_{variable_info[v]['var_name']}_{download_time}_{download_time_end}.nc"
+                    download_file(submit_url, store_path, file_name, check)
 
 
 def convert_full_name_to_short_name(full_name):
@@ -913,7 +1028,7 @@ def download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min
         parallel_counter = 0
         counter_lock = Lock()  # 创建一个锁，线程安全的计数器
     if ymdh_time_s == ymdh_time_e:
-        prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, None, depth, level, store_path, dataset_name, version_name)
+        prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, None, depth, level, store_path, dataset_name, version_name, check)
     elif int(ymdh_time_s) < int(ymdh_time_e):
         print("Downloading a series of files...")
         time_list = get_time_list(ymdh_time_s, ymdh_time_e, 3, "hour")
@@ -1111,9 +1226,9 @@ def how_to_use():
 
 if __name__ == "__main__":
     # help(hycom3h.download)
-    time_s, time_e = "2023010100", "2023123121"
+    time_s, time_e = "2024081012", "2024081115"
     merge_name = f"{time_s}_{time_e}"  # 合并后的文件名
-    root_path = r"G:\Data\HYCOM\3hourly"
+    root_path = r"G:\Data\HYCOM\3hourly_test"
     location_dict = {"west": 105, "east": 130, "south": 15, "north": 45}
     download_dict = {
         "water_u": {"simple_name": "u", "download": 1},
@@ -1141,6 +1256,8 @@ if __name__ == "__main__":
     download_switch, single_var = True, False
     combine_switch = False
     copy_switch, copy_dir = False, r"G:\Data\HYCOM\3hourly"
+    
+    # draw_time_range(pic_save_folder=r'I:\Delete')
 
     if download_switch:
         if single_var: