oafuncs 0.0.81__py2.py3-none-any.whl → 0.0.83__py2.py3-none-any.whl

oafuncs/oa_cmap.py

@@ -100,6 +100,7 @@ def create_cmap_rgbtxt(rgbtxt_file,split_mark=','):  # 根据RGB的txt文档制
     return icmap
 
 
+# ** 选择cmap
 def choose_cmap(cmap_name=None, query=False):
     """
     description: Choosing a colormap from the list of available colormaps or a custom colormap

oafuncs/oa_data.py

@@ -1,31 +1,97 @@
 #!/usr/bin/env python
 # coding=utf-8
-'''
+"""
 Author: Liu Kun && 16031215@qq.com
 Date: 2024-09-17 17:12:47
 LastEditors: Liu Kun && 16031215@qq.com
-LastEditTime: 2024-11-21 13:13:20
+LastEditTime: 2024-12-13 19:11:08
 FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_data.py
-Description:  
+Description:
 EditPlatform: vscode
 ComputerInfo: XPS 15 9510
 SystemInfo: Windows 11
 Python Version: 3.11
-'''
-
+"""
 
+import itertools
 import multiprocessing as mp
 from concurrent.futures import ThreadPoolExecutor
 
 import numpy as np
 from scipy.interpolate import griddata
 
-__all__ = ['interp_2d', 'interp_2d_parallel']
+__all__ = ["interp_2d"]
 
-# ** 高维插值函数，插值最后两个维度
 
+def interp_2d(target_x, target_y, origin_x, origin_y, data, method="linear", parallel=True):
+    """
+    Perform 2D interpolation on the last two dimensions of a multi-dimensional array.
+
+    Parameters:
+    - target_x (array-like): 1D array of target grid's x-coordinates.
+    - target_y (array-like): 1D array of target grid's y-coordinates.
+    - origin_x (array-like): 1D array of original grid's x-coordinates.
+    - origin_y (array-like): 1D array of original grid's y-coordinates.
+    - data (numpy.ndarray): Multi-dimensional array where the last two dimensions correspond to the original grid.
+    - method (str, optional): Interpolation method, default is 'linear'. Other options include 'nearest', 'cubic', etc.
+    - parallel (bool, optional): Flag to enable parallel processing. Default is True.
+
+    Returns:
+    - interpolated_data (numpy.ndarray): Interpolated data with the same leading dimensions as the input data, but with the last two dimensions corresponding to the target grid.
+
+    Raises:
+    - ValueError: If the shape of the data does not match the shape of the origin_x or origin_y grids.
+
+    Usage:
+    - Interpolate a 2D array:
+        result = interp_2d(target_x, target_y, origin_x, origin_y, data_2d)
+    - Interpolate a 3D array (where the last two dimensions are spatial):
+        result = interp_2d(target_x, target_y, origin_x, origin_y, data_3d)
+    - Interpolate a 4D array (where the last two dimensions are spatial):
+        result = interp_2d(target_x, target_y, origin_x, origin_y, data_4d)
+    """
+
+    def interp_single(data_slice, target_points, origin_points, method):
+        return griddata(origin_points, data_slice.ravel(), target_points, method=method).reshape(target_y.shape)
+
+    # 确保目标网格和初始网格都是二维的
+    if len(target_y.shape) == 1:
+        target_x, target_y = np.meshgrid(target_x, target_y)
+    if len(origin_y.shape) == 1:
+        origin_x, origin_y = np.meshgrid(origin_x, origin_y)
+
+    # 根据经纬度网格判断输入数据的形状是否匹配
+    if origin_x.shape != data.shape[-2:] or origin_y.shape != data.shape[-2:]:
+        raise ValueError("Shape of data does not match shape of origin_x or origin_y.")
+
+    # 创建网格和展平数据
+    target_points = np.column_stack((target_y.ravel(), target_x.ravel()))
+    origin_points = np.column_stack((origin_y.ravel(), origin_x.ravel()))
+
+    # 根据是否并行选择不同的执行方式
+    if parallel:
+        with ThreadPoolExecutor(max_workers=mp.cpu_count() - 2) as executor:
+            if len(data.shape) == 2:
+                interpolated_data = list(executor.map(interp_single, [data], [target_points], [origin_points], [method]))
+            elif len(data.shape) == 3:
+                interpolated_data = list(executor.map(interp_single, [data[i] for i in range(data.shape[0])], [target_points] * data.shape[0], [origin_points] * data.shape[0], [method] * data.shape[0]))
+            elif len(data.shape) == 4:
+                index_combinations = list(itertools.product(range(data.shape[0]), range(data.shape[1])))
+                interpolated_data = list(executor.map(interp_single, [data[i, j] for i, j in index_combinations], [target_points] * len(index_combinations), [origin_points] * len(index_combinations), [method] * len(index_combinations)))
+                interpolated_data = np.array(interpolated_data).reshape(data.shape[0], data.shape[1], *target_y.shape)
+    else:
+        if len(data.shape) == 2:
+            interpolated_data = interp_single(data, target_points, origin_points, method)
+        elif len(data.shape) == 3:
+            interpolated_data = np.stack([interp_single(data[i], target_points, origin_points, method) for i in range(data.shape[0])])
+        elif len(data.shape) == 4:
+            interpolated_data = np.stack([np.stack([interp_single(data[i, j], target_points, origin_points, method) for j in range(data.shape[1])]) for i in range(data.shape[0])])
+
+    return np.array(interpolated_data)
 
-def interp_2d(target_x, target_y, origin_x, origin_y, data, method='linear'):
+
+# ** 高维插值函数，插值最后两个维度
+def interp_2d_20241213(target_x, target_y, origin_x, origin_y, data, method="linear"):
     """
     高维插值函数，默认插值最后两个维度，传输数据前请确保数据的维度正确
     参数:
@@ -52,7 +118,7 @@ def interp_2d(target_x, target_y, origin_x, origin_y, data, method='linear'):
 
     if origin_x.shape != dims[-2:] or origin_y.shape != dims[-2:]:
         print(origin_x.shape, dims[-2:])
-        raise ValueError('Shape of data does not match shape of origin_x or origin_y.')
+        raise ValueError("Shape of data does not match shape of origin_x or origin_y.")
 
     # 将目标网格展平成一维数组
     target_points = np.column_stack((np.ravel(target_y), np.ravel(target_x)))
@@ -69,7 +135,7 @@ def interp_2d(target_x, target_y, origin_x, origin_y, data, method='linear'):
         for i in range(dims[0]):
             dt = griddata(origin_points, np.ravel(data[i, :, :]), target_points, method=method)
             interpolated_data.append(np.reshape(dt, target_y.shape))
-            print(f'Interpolating {i+1}/{dims[0]}...')
+            print(f"Interpolating {i+1}/{dims[0]}...")
         interpolated_data = np.array(interpolated_data)
     elif len_dims == 4:
         interpolated_data = []
@@ -78,8 +144,8 @@ def interp_2d(target_x, target_y, origin_x, origin_y, data, method='linear'):
             for j in range(dims[1]):
                 dt = griddata(origin_points, np.ravel(data[i, j, :, :]), target_points, method=method)
                 interpolated_data[i].append(np.reshape(dt, target_y.shape))
-                print(f'\rInterpolating {i*dims[1]+j+1}/{dims[0]*dims[1]}...', end='')
-        print('\n')
+                print(f"\rInterpolating {i*dims[1]+j+1}/{dims[0]*dims[1]}...", end="")
+        print("\n")
         interpolated_data = np.array(interpolated_data)
 
     return interpolated_data
@@ -87,8 +153,8 @@ def interp_2d(target_x, target_y, origin_x, origin_y, data, method='linear'):
 
 # ** 高维插值函数，插值最后两个维度，使用多线程进行插值
 # 在本地电脑上可以提速三倍左右，超算上暂时无法加速
-def interp_2d_parallel(target_x, target_y, origin_x, origin_y, data, method='linear'):
-    '''
+def interp_2d_parallel_20241213(target_x, target_y, origin_x, origin_y, data, method="linear"):
+    """
     param        {*} target_x 目标经度网格 1D 或 2D
     param        {*} target_y 目标纬度网格 1D 或 2D
     param        {*} origin_x 初始经度网格 1D 或 2D
@@ -98,22 +164,23 @@ def interp_2d_parallel(target_x, target_y, origin_x, origin_y, data, method='lin
     return       {*} 插值结果
     description : 高维插值函数，默认插值最后两个维度，传输数据前请确保数据的维度正确
     example     : interpolated_data = interp_2d_parallel(target_x, target_y, origin_x, origin_y, data, method='linear')
-    '''
-    def interp_single2d(target_y, target_x, origin_y, origin_x, data, method='linear'):
+    """
+
+    def interp_single2d(target_y, target_x, origin_y, origin_x, data, method="linear"):
         target_points = np.column_stack((np.ravel(target_y), np.ravel(target_x)))
         origin_points = np.column_stack((np.ravel(origin_y), np.ravel(origin_x)))
 
         dt = griddata(origin_points, np.ravel(data[:, :]), target_points, method=method)
         return np.reshape(dt, target_y.shape)
 
-    def interp_single3d(i, target_y, target_x, origin_y, origin_x, data, method='linear'):
+    def interp_single3d(i, target_y, target_x, origin_y, origin_x, data, method="linear"):
         target_points = np.column_stack((np.ravel(target_y), np.ravel(target_x)))
         origin_points = np.column_stack((np.ravel(origin_y), np.ravel(origin_x)))
 
         dt = griddata(origin_points, np.ravel(data[i, :, :]), target_points, method=method)
         return np.reshape(dt, target_y.shape)
 
-    def interp_single4d(i, j, target_y, target_x, origin_y, origin_x, data, method='linear'):
+    def interp_single4d(i, j, target_y, target_x, origin_y, origin_x, data, method="linear"):
         target_points = np.column_stack((np.ravel(target_y), np.ravel(target_x)))
         origin_points = np.column_stack((np.ravel(origin_y), np.ravel(origin_x)))
 
@@ -129,19 +196,31 @@ def interp_2d_parallel(target_x, target_y, origin_x, origin_y, data, method='lin
     len_dims = len(dims)
 
     if origin_x.shape != dims[-2:] or origin_y.shape != dims[-2:]:
-        raise ValueError('数据形状与 origin_x 或 origin_y 的形状不匹配.')
+        raise ValueError("数据形状与 origin_x 或 origin_y 的形状不匹配.")
 
     interpolated_data = []
 
     # 使用多线程进行插值
-    with ThreadPoolExecutor(max_workers=mp.cpu_count()-2) as executor:
-        print(f'Using {mp.cpu_count()-2} threads...')
+    with ThreadPoolExecutor(max_workers=mp.cpu_count() - 2) as executor:
+        print(f"Using {mp.cpu_count()-2} threads...")
         if len_dims == 2:
             interpolated_data = list(executor.map(interp_single2d, [target_y], [target_x], [origin_y], [origin_x], [data], [method]))
         elif len_dims == 3:
-            interpolated_data = list(executor.map(interp_single3d, [i for i in range(dims[0])], [target_y]*dims[0], [target_x]*dims[0], [origin_y]*dims[0], [origin_x]*dims[0], [data]*dims[0], [method]*dims[0]))
+            interpolated_data = list(executor.map(interp_single3d, [i for i in range(dims[0])], [target_y] * dims[0], [target_x] * dims[0], [origin_y] * dims[0], [origin_x] * dims[0], [data] * dims[0], [method] * dims[0]))
         elif len_dims == 4:
-            interpolated_data = list(executor.map(interp_single4d, [i for i in range(dims[0]) for j in range(dims[1])], [j for i in range(dims[0]) for j in range(dims[1])], [target_y]*dims[0]*dims[1], [target_x]*dims[0]*dims[1], [origin_y]*dims[0]*dims[1], [origin_x]*dims[0]*dims[1], [data]*dims[0]*dims[1], [method]*dims[0]*dims[1]))
+            interpolated_data = list(
+                executor.map(
+                    interp_single4d,
+                    [i for i in range(dims[0]) for j in range(dims[1])],
+                    [j for i in range(dims[0]) for j in range(dims[1])],
+                    [target_y] * dims[0] * dims[1],
+                    [target_x] * dims[0] * dims[1],
+                    [origin_y] * dims[0] * dims[1],
+                    [origin_x] * dims[0] * dims[1],
+                    [data] * dims[0] * dims[1],
+                    [method] * dims[0] * dims[1],
+                )
+            )
             interpolated_data = np.array(interpolated_data).reshape(dims[0], dims[1], target_y.shape[0], target_x.shape[1])
 
     interpolated_data = np.array(interpolated_data)
@@ -149,7 +228,7 @@ def interp_2d_parallel(target_x, target_y, origin_x, origin_y, data, method='lin
     return interpolated_data
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     import time
 
     import matplotlib.pyplot as plt
@@ -169,15 +248,15 @@ if __name__ == '__main__':
     data = np.random.rand(10, 10, 11, 11)
 
     start = time.time()
-    interpolated_data = interp_2d(target_x, target_y, origin_x, origin_y, data)
-    print(f'Interpolation time: {time.time()-start:.2f}s')
+    interpolated_data = interp_2d(target_x, target_y, origin_x, origin_y, data, parallel=False)
+    print(f"Interpolation time: {time.time()-start:.2f}s")
 
     print(interpolated_data.shape)
 
     # 高维插值多线程
     start = time.time()
-    interpolated_data = interp_2d_parallel(target_x, target_y, origin_x, origin_y, data)
-    print(f'Interpolation time: {time.time()-start:.2f}s')
+    interpolated_data = interp_2d(target_x, target_y, origin_x, origin_y, data)
+    print(f"Interpolation time: {time.time()-start:.2f}s")
 
     print(interpolated_data.shape)
     print(interpolated_data[0, 0, :, :].shape)

oafuncs/oa_down/__init__.py

@@ -1,20 +1,21 @@
 #!/usr/bin/env python
 # coding=utf-8
-'''
+"""
 Author: Liu Kun && 16031215@qq.com
 Date: 2024-09-17 16:09:20
 LastEditors: Liu Kun && 16031215@qq.com
-LastEditTime: 2024-12-01 19:49:40
+LastEditTime: 2024-12-26 08:09:01
 FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_down\\__init__.py
-Description:  
+Description:
 EditPlatform: vscode
 ComputerInfo: XPS 15 9510
 SystemInfo: Windows 11
 Python Version: 3.11
-'''
+"""
 
 # from .love_ocean import sign as love_ocean
 # from .meteorological_home import sign as meteorological_home
 
 from .hycom_3hourly import *
 from .literature import *
+from .user_agent import *

oafuncs/oa_down/hycom_3hourly.py

@@ -21,6 +21,7 @@ import warnings
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from threading import Lock
+import re
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -29,17 +30,18 @@ import requests
 from bs4 import BeautifulSoup
 from rich import print
 from rich.progress import Progress
+import glob
 
 warnings.filterwarnings("ignore", category=RuntimeWarning, message="Engine '.*' loading failed:.*")
 
-__all__ = ["draw_time_range", "download", "how_to_use", "get_time_list", "get_ua"]
+__all__ = ["draw_time_range", "download", "how_to_use", "get_time_list"]
 
 # time resolution
 data_info = {"yearly": {}, "monthly": {}, "daily": {}, "hourly": {}}
 
 # hourly data
 # dataset: GLBv0.08, GLBu0.08, GLBy0.08
-data_info["hourly"]["dataset"] = {"GLBv0.08": {}, "GLBu0.08": {}, "GLBy0.08": {}}
+data_info["hourly"]["dataset"] = {"GLBv0.08": {}, "GLBu0.08": {}, "GLBy0.08": {}, "ESPC_D": {}}
 
 # version
 # version of GLBv0.08: 53.X, 56.3, 57.2, 92.8, 57.7, 92.9, 93.0
@@ -48,6 +50,8 @@ data_info["hourly"]["dataset"]["GLBv0.08"]["version"] = {"53.X": {}, "56.3": {},
 data_info["hourly"]["dataset"]["GLBu0.08"]["version"] = {"93.0": {}}
 # version of GLBy0.08: 93.0
 data_info["hourly"]["dataset"]["GLBy0.08"]["version"] = {"93.0": {}}
+# version of ESPC_D: V02
+data_info["hourly"]["dataset"]["ESPC_D"]["version"] = {"V02": {}}
 
 # info details
 # time range
@@ -65,7 +69,9 @@ data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["93.0"]["time_range"] = {"
 # GLBu0.08
 data_info["hourly"]["dataset"]["GLBu0.08"]["version"]["93.0"]["time_range"] = {"time_start": "2018091912", "time_end": "2018120909"}
 # GLBy0.08
-data_info["hourly"]["dataset"]["GLBy0.08"]["version"]["93.0"]["time_range"] = {"time_start": "2018120412", "time_end": "20300904"}
+data_info["hourly"]["dataset"]["GLBy0.08"]["version"]["93.0"]["time_range"] = {"time_start": "2018120412", "time_end": "2024090509"}
+# ESPC-D
+data_info["hourly"]["dataset"]["ESPC_D"]["version"]["V02"]["time_range"] = {"time_start": "2024081012", "time_end": "2030010100"}
 
 # variable
 variable_info = {
@@ -94,6 +100,7 @@ data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["92.9"]["classification"]
 data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["93.0"]["classification"] = "var_different"
 data_info["hourly"]["dataset"]["GLBu0.08"]["version"]["93.0"]["classification"] = "var_different"
 data_info["hourly"]["dataset"]["GLBy0.08"]["version"]["93.0"]["classification"] = "var_year_different"
+data_info["hourly"]["dataset"]["ESPC_D"]["version"]["V02"]["classification"] = "single_var_year_different"
 
 # download info
 # base url
@@ -141,7 +148,7 @@ data_info["hourly"]["dataset"]["GLBu0.08"]["version"]["93.0"]["url"] = url_930_u
 uv3z_930_y = {}
 ts3z_930_y = {}
 ssh_930_y = {}
-for y_930_y in range(2018, 2030):
+for y_930_y in range(2018, 2025):
     uv3z_930_y[str(y_930_y)] = rf"https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/uv3z/{y_930_y}?"
     ts3z_930_y[str(y_930_y)] = rf"https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/ts3z/{y_930_y}?"
     ssh_930_y[str(y_930_y)] = rf"https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/ssh/{y_930_y}?"
@@ -152,12 +159,39 @@ url_930_y = {
     "ssh": ssh_930_y,
 }
 data_info["hourly"]["dataset"]["GLBy0.08"]["version"]["93.0"]["url"] = url_930_y
+# ESPC-D-V02
+u3z_espc_d_v02_y = {}
+v3z_espc_d_v02_y = {}
+t3z_espc_d_v02_y = {}
+s3z_espc_d_v02_y = {}
+ssh_espc_d_v02_y = {}
+for y_espc_d_v02 in range(2024, 2030):
+    u3z_espc_d_v02_y[str(y_espc_d_v02)] = rf"https://ncss.hycom.org/thredds/ncss/ESPC-D-V02/u3z/{y_espc_d_v02}?"
+    v3z_espc_d_v02_y[str(y_espc_d_v02)] = rf"https://ncss.hycom.org/thredds/ncss/ESPC-D-V02/v3z/{y_espc_d_v02}?"
+    t3z_espc_d_v02_y[str(y_espc_d_v02)] = rf"https://ncss.hycom.org/thredds/ncss/ESPC-D-V02/t3z/{y_espc_d_v02}?"
+    s3z_espc_d_v02_y[str(y_espc_d_v02)] = rf"https://ncss.hycom.org/thredds/ncss/ESPC-D-V02/s3z/{y_espc_d_v02}?"
+    ssh_espc_d_v02_y[str(y_espc_d_v02)] = rf"https://ncss.hycom.org/thredds/ncss/ESPC-D-V02/ssh/{y_espc_d_v02}?"
+url_espc_d_v02_y = {
+    "u3z": u3z_espc_d_v02_y,
+    "v3z": v3z_espc_d_v02_y,
+    "t3z": t3z_espc_d_v02_y,
+    "s3z": s3z_espc_d_v02_y,
+    "ssh": ssh_espc_d_v02_y,
+}
+data_info['hourly']['dataset']['ESPC_D']['version']['V02']['url'] = url_espc_d_v02_y
 
 var_group = {
     "uv3z": ["u", "v", "u_b", "v_b"],
     "ts3z": ["temp", "salt", "temp_b", "salt_b"],
     "ssh": ["ssh"],
 }
+single_var_group = {
+    "u3z": ["u"],
+    "v3z": ["v"],
+    "t3z": ["temp"],
+    "s3z": ["salt"],
+    "ssh": ["ssh"],
+}
 
 
 def draw_time_range(pic_save_folder=None):
@@ -205,7 +239,8 @@ def draw_time_range(pic_save_folder=None):
         #          f"{row['version']}", ha='right', color='black')
         ymdh_s = row["start_date"].strftime("%Y-%m-%d %H")
         ymdh_e = row["end_date"].strftime("%Y-%m-%d %H")
-        if k == 1 or k == len(combined_labels):
+        # if k == 1 or k == len(combined_labels):
+        if k == 1:
             plt.text(row["start_date"], k + 0.125, f"{ymdh_s}", ha="left", color="black")
             plt.text(row["end_date"], k + 0.125, f"{ymdh_e}", ha="right", color="black")
         else:
@@ -496,6 +531,14 @@ def get_base_url(dataset_name, version_name, var, ymdh_str):
                 break
         if base_url is None:
             print("Please ensure the var is in [u,v,temp,salt,ssh,u_b,v_b,temp_b,salt_b]")
+    elif classification_method == "single_var_year_different":
+        base_url = None
+        for key, value in single_var_group.items():
+            if var in value:
+                base_url = url_dict[key][str(year_str)]
+                break
+        if base_url is None:
+            print("Please ensure the var is in [u,v,temp,salt,ssh]")
     return base_url
 
 
@@ -513,7 +556,44 @@ def clear_existing_file(file_full_path):
         print(f"{file_full_path} has been removed")
 
 
-def _get_file_size(file_path, unit="KB"):
+def find_file(parent_path, fname, mode="path"):
+    """
+    description:
+    param {*} parent_path: The parent path where the files are located
+    param {*} fname: The file name pattern to search for
+    param {*} mode: 'path' to return the full path of the files, 'file' to return only the file names
+    return {*} A list of file paths or file names if files are found, None otherwise
+    """
+
+    def natural_sort_key(s):
+        """生成一个用于自然排序的键"""
+        return [int(text) if text.isdigit() else text.lower() for text in re.split("([0-9]+)", s)]
+
+    # 将parent_path和fname结合成完整的搜索路径
+    search_pattern = os.path.join(str(parent_path), fname)
+
+    # 使用glob模块查找所有匹配的文件
+    matched_files = glob.glob(search_pattern)
+
+    # 如果没有找到任何文件，则返回False
+    if not matched_files:
+        return None
+
+    # 在find_files函数中替换natsorted调用
+    matched_files = sorted(matched_files, key=natural_sort_key)
+
+    # 根据mode参数决定返回的内容
+    if mode == "file":
+        # 只返回文件名
+        result = [os.path.basename(file) for file in matched_files]
+    else:  # 默认为'path'
+        # 返回文件的绝对路径
+        result = [os.path.abspath(file) for file in matched_files]
+
+    return result
+
+
+def file_size(file_path, unit="KB"):
     # 检查文件是否存在
     if not os.path.exists(file_path):
         return "文件不存在"
@@ -534,12 +614,34 @@ def _get_file_size(file_path, unit="KB"):
     return converted_size
 
 
-def check_existing_file(file_full_path):
+# ** 计算文件夹下指定相关文件的平均大小
+def mean_size(parent_path, fname):
+    flist = find_file(parent_path, fname)
+    if flist:
+        size_list = [file_size(f) for f in flist if file_size(f) != 0]
+    else:
+        size_list = []
+    if size_list:
+        min_size, max_size = min(size_list), max(size_list)
+        mean_size = sum(size_list) / len(size_list)
+    else:
+        mean_size, min_size, max_size = 0, 0, 0
+    return mean_size, min_size, max_size
+
+
+def check_existing_file(file_full_path, min_size):
     if os.path.exists(file_full_path):
         print(f"[bold #FFA54F]{file_full_path} exists")
-        fsize = _get_file_size(file_full_path)
+        fsize = file_size(file_full_path)
+        if min_size:
+            if fsize < min_size:
+                print(f"[bold #FFA54F]{file_full_path} ({fsize:.2f} KB) may be incomplete")
+                # clear_existing_file(file_full_path)
+                return False
+            else:
+                return True
         if fsize < 5:
-            print(f"[bold #FFA54F]{file_full_path} may be incomplete\nFile size: {fsize:.2f} KB")
+            print(f"[bold #FFA54F]{file_full_path} ({fsize:.2f} KB) may be incomplete")
             # clear_existing_file(file_full_path)
             return False
         else:
@@ -623,8 +725,13 @@ def get_proxy():
 def download_file(target_url, store_path, file_name, check=False):
     # Check if the file exists
     fname = Path(store_path) / file_name
+    file_name_split = file_name.split("_")
+    same_file = f'{file_name_split[0]}_{file_name_split[1]}*nc'
+    fsize_mean, fsize_min, fsize_max = mean_size(store_path, same_file)
+    set_min_size = fsize_mean - 0.5 * (fsize_max - fsize_min)
+    set_min_size = set_min_size if set_min_size > 0 else fsize_min
     if check:
-        if check_existing_file(fname):
+        if check_existing_file(fname, set_min_size*0.8):
             count_dict["skip"] += 1
             return
     clear_existing_file(fname)
@@ -819,29 +926,37 @@ def prepare_url_to_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max
                 file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}_{download_time_end}.nc"
             download_file(submit_url, store_path, file_name, check)
         else:
-            varlist = [_ for _ in var]
-            for key, value in var_group.items():
-                current_group = []
-                for v in varlist:
-                    if v in value:
-                        current_group.append(v)
-                if len(current_group) == 0:
-                    continue
-
-                var = current_group[0]
-                submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time, download_time_end)
-                file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
-                old_str = f'var={variable_info[var]["var_name"]}'
-                new_str = f'var={variable_info[var]["var_name"]}'
-                if len(current_group) > 1:
-                    for v in current_group[1:]:
-                        new_str = f'{new_str}&var={variable_info[v]["var_name"]}'
-                    submit_url = submit_url.replace(old_str, new_str)
-                    # file_name = f'HYCOM_{'-'.join([variable_info[v]["var_name"] for v in current_group])}_{download_time}.nc'
-                    file_name = f"HYCOM_{key}_{download_time}.nc"
+            if download_time < "2024081012":
+                varlist = [_ for _ in var]
+                for key, value in var_group.items():
+                    current_group = []
+                    for v in varlist:
+                        if v in value:
+                            current_group.append(v)
+                    if len(current_group) == 0:
+                        continue
+
+                    var = current_group[0]
+                    submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time, download_time_end)
+                    file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
+                    old_str = f'var={variable_info[var]["var_name"]}'
+                    new_str = f'var={variable_info[var]["var_name"]}'
+                    if len(current_group) > 1:
+                        for v in current_group[1:]:
+                            new_str = f'{new_str}&var={variable_info[v]["var_name"]}'
+                        submit_url = submit_url.replace(old_str, new_str)
+                        # file_name = f'HYCOM_{'-'.join([variable_info[v]["var_name"] for v in current_group])}_{download_time}.nc'
+                        file_name = f"HYCOM_{key}_{download_time}.nc"
+                        if download_time_end is not None:
+                            file_name = f"HYCOM_{key}_{download_time}_{download_time_end}.nc"
+                    download_file(submit_url, store_path, file_name, check)
+            else:
+                for v in var:
+                    submit_url = get_submit_url_var(v, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time, download_time_end)
+                    file_name = f"HYCOM_{variable_info[v]['var_name']}_{download_time}.nc"
                     if download_time_end is not None:
-                        file_name = f"HYCOM_{key}_{download_time}_{download_time_end}.nc"
-                download_file(submit_url, store_path, file_name, check)
+                        file_name = f"HYCOM_{variable_info[v]['var_name']}_{download_time}_{download_time_end}.nc"
+                    download_file(submit_url, store_path, file_name, check)
 
 
 def convert_full_name_to_short_name(full_name):
@@ -913,7 +1028,7 @@ def download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min
         parallel_counter = 0
         counter_lock = Lock()  # 创建一个锁，线程安全的计数器
     if ymdh_time_s == ymdh_time_e:
-        prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, None, depth, level, store_path, dataset_name, version_name)
+        prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, None, depth, level, store_path, dataset_name, version_name, check)
     elif int(ymdh_time_s) < int(ymdh_time_e):
         print("Downloading a series of files...")
         time_list = get_time_list(ymdh_time_s, ymdh_time_e, 3, "hour")
@@ -1111,9 +1226,9 @@ def how_to_use():
 
 if __name__ == "__main__":
     # help(hycom3h.download)
-    time_s, time_e = "2023010100", "2023123121"
+    time_s, time_e = "2024081012", "2024081115"
     merge_name = f"{time_s}_{time_e}"  # 合并后的文件名
-    root_path = r"G:\Data\HYCOM\3hourly"
+    root_path = r"G:\Data\HYCOM\3hourly_test"
     location_dict = {"west": 105, "east": 130, "south": 15, "north": 45}
     download_dict = {
         "water_u": {"simple_name": "u", "download": 1},
@@ -1141,6 +1256,8 @@ if __name__ == "__main__":
     download_switch, single_var = True, False
     combine_switch = False
     copy_switch, copy_dir = False, r"G:\Data\HYCOM\3hourly"
+    
+    # draw_time_range(pic_save_folder=r'I:\Delete')
 
     if download_switch:
         if single_var:

oafuncs/oa_down/user_agent.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# coding=utf-8
+"""
+Author: Liu Kun && 16031215@qq.com
+Date: 2024-12-26 08:06:34
+LastEditors: Liu Kun && 16031215@qq.com
+LastEditTime: 2024-12-26 08:06:34
+FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_down\\user_agent.py
+Description:
+EditPlatform: vscode
+ComputerInfo: XPS 15 9510
+SystemInfo: Windows 11
+Python Version: 3.12
+"""
+import os
+import random
+
+
+__all__ = ["get_ua"]
+
+
+def get_ua():
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    ua_file_txt = os.path.join(current_dir, "User_Agent-list.txt")
+
+    with open(ua_file_txt, "r") as f:
+        ua_list = f.readlines()
+        # 去掉换行符和空行
+        ua_list = [line.strip() for line in ua_list if line.strip()]
+
+    # if current_platform == 'Linux':
+    #     ua_list = [line for line in ua_list if 'Linux' in line]
+
+    return random.choice(ua_list)