oafuncs 0.0.98.10__py3-none-any.whl → 0.0.98.12__py3-none-any.whl

oafuncs/_script/netcdf_merge.py

@@ -4,7 +4,9 @@ from typing import List, Optional, Union
 import numpy as np
 import xarray as xr
 from dask.diagnostics import ProgressBar
+
 from oafuncs import pbar
+from oafuncs._script.netcdf_write import _calculate_scale_and_offset  # 新增导入
 
 
 def merge_nc(file_list: Union[str, List[str]], var_name: Optional[Union[str, List[str]]] = None, dim_name: Optional[str] = None, target_filename: Optional[str] = None) -> None:
@@ -83,24 +85,19 @@ def merge_nc(file_list: Union[str, List[str]], var_name: Optional[Union[str, Lis
         encoding = {}
         for var in merged_dataset.data_vars:
             data = merged_dataset[var].values
-            # print(f"Variable '{var}' ready for writing: min={data.min():.3f}, max={data.max():.3f}, mean={data.mean():.3f}")
             if data.dtype.kind in {"i", "u", "f"}:  # 仅对数值型数据进行压缩
-                data_range = data.max() - data.min()
-                if data_range > 0:  # 避免范围过小导致的精度问题
-                    scale_factor = data_range / (2**16 - 1)
-                    add_offset = data.min()
-                    encoding[var] = {
-                        "zlib": True,
-                        "complevel": 4,
-                        "dtype": "int16",
-                        "scale_factor": scale_factor,
-                        "add_offset": add_offset,
-                        "_FillValue": -32767,
-                    }
-                else:
-                    encoding[var] = {"zlib": True, "complevel": 4}  # 范围过小时禁用缩放
+                # 统一调用 netcdf_write 中的 scale/offset 计算
+                scale_factor, add_offset = _calculate_scale_and_offset(data, n=16)
+                encoding[var] = {
+                    "zlib": True,
+                    "complevel": 4,
+                    "dtype": "int16",
+                    "scale_factor": scale_factor,
+                    "add_offset": add_offset,
+                    "_FillValue": -32767,
+                }
             else:
-                encoding[var] = {"zlib": True, "complevel": 4}  # 非数值型数据不使用缩放
+                encoding[var] = {"zlib": True, "complevel": 4}
 
         # 确保写入时不会因编码问题导致数据丢失
         # merged_dataset.to_netcdf(target_filename, encoding=encoding)

oafuncs/oa_data.py

@@ -121,7 +121,26 @@ def _interp_single_worker(*args):
     """
     data_slice, origin_points, target_points, interpolation_method, target_shape = args
 
-    return griddata(origin_points, data_slice.ravel(), target_points, method=interpolation_method).reshape(target_shape)
+    # 过滤掉包含 NaN 的点
+    valid_mask = ~np.isnan(data_slice.ravel())
+    valid_data = data_slice.ravel()[valid_mask]
+    valid_points = origin_points[valid_mask]
+
+    if len(valid_data) < 10:  # 如果有效数据太少，用均值填充
+        return np.full(target_shape, np.nanmean(data_slice))
+
+    # 使用有效数据进行插值
+    result = griddata(valid_points, valid_data, target_points, method=interpolation_method)
+    result = result.reshape(target_shape)
+
+    # 检查插值结果中是否仍有 NaN，如果有，用最近邻插值填充
+    if np.any(np.isnan(result)):
+        # 使用最近邻方法填充剩余的 NaN 值
+        nan_mask = np.isnan(result)
+        result_nn = griddata(valid_points, valid_data, target_points[nan_mask.ravel()], method="nearest")
+        result.ravel()[nan_mask.ravel()] = result_nn
+
+    return result
 
 
 def interp_2d(
@@ -130,7 +149,7 @@ def interp_2d(
     source_x_coordinates: Union[np.ndarray, List[float]],
     source_y_coordinates: Union[np.ndarray, List[float]],
     source_data: np.ndarray,
-    interpolation_method: str = "linear",
+    interpolation_method: str = "cubic",
 ) -> np.ndarray:
     """
     Perform 2D interpolation on the last two dimensions of a multi-dimensional array.
@@ -141,7 +160,8 @@ def interp_2d(
         source_x_coordinates (Union[np.ndarray, List[float]]): Original grid's x-coordinates.
         source_y_coordinates (Union[np.ndarray, List[float]]): Original grid's y-coordinates.
         source_data (np.ndarray): Multi-dimensional array with the last two dimensions as spatial.
-        interpolation_method (str, optional): Interpolation method. Defaults to "linear".
+        interpolation_method (str, optional): Interpolation method. Defaults to "cubic".
+            >>> optional: 'linear', 'nearest', 'cubic', 'quintic', etc.
         use_parallel (bool, optional): Enable parallel processing. Defaults to True.
 
     Returns:
@@ -190,12 +210,12 @@ def interp_2d(
     for t_index in range(t):
         for z_index in range(z):
             paras.append((new_src_data[t_index, z_index], origin_points, target_points, interpolation_method, target_shape))
-    
+
     with PEx() as excutor:
         result = excutor.run(_interp_single_worker, paras)
-    excutor.shutdown()
 
-    return np.squeeze(np.array(result))
+    return np.squeeze(np.array(result).reshape(t, z, *target_shape))
+
 
 
 def mask_shapefile(

oafuncs/oa_down/hycom_3hourly.py

@@ -584,7 +584,7 @@ def _clear_existing_file(file_full_path):
 
 def _check_existing_file(file_full_path, avg_size):
     if os.path.exists(file_full_path):
-        print(f"[bold #FFA54F]{file_full_path} exists")
+        print(f"[bold #FFA54F]{file_full_path} exists ...")
         fsize = file_size(file_full_path)
         delta_size_ratio = (fsize - avg_size) / avg_size
         if abs(delta_size_ratio) > 0.025:
@@ -796,7 +796,7 @@ def _download_file(target_url, store_path, file_name, cover=False):
     get_mean_size = _get_mean_size_move(same_file, save_path)
 
     if _check_existing_file(save_path, get_mean_size):
-        print(f"[bold #FFA54F]{save_path} exists, skipping ...")
+        # print(f"[bold #FFA54F]{save_path} exists, skipping ...")
         count_dict["skip"] += 1
         return
 
@@ -901,7 +901,7 @@ def _prepare_url_to_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_ma
         else:
             if download_time < "2024081012":
                 varlist = [_ for _ in var]
-                for key, value in pbar(var_group.items(), description=f"Var Group {download_time} ->", total=len(var_group), color="#d7feb9", next_line=True):
+                for key, value in pbar(var_group.items(), description=f"Var Group {download_time}", total=len(var_group), next_line=True):
                     current_group = []
                     for v in varlist:
                         if v in value:
@@ -923,7 +923,7 @@ def _prepare_url_to_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_ma
                             file_name = f"HYCOM_{key}_{download_time}-{download_time_end}.nc"
                     _download_file(submit_url, store_path, file_name, cover)
             else:
-                for v in pbar(var, description=f"Var {download_time} ->", total=len(var), color="#d7feb9", next_line=True):
+                for v in pbar(var, description=f"Var {download_time}", total=len(var), next_line=True):
                     submit_url = _get_submit_url_var(v, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time, download_time_end)
                     file_name = f"HYCOM_{variable_info[v]['var_name']}_{download_time}.nc"
                     if download_time_end is not None:
@@ -968,7 +968,7 @@ def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_mi
         # with Progress() as progress:
             # task = progress.add_task(f"[cyan]{bar_desc}", total=len(time_list))
         if num_workers is None or num_workers <= 1:
-            for i, time_str in pbar(enumerate(time_list), description=f"{bar_desc}", total=len(time_list), cmap='colorful_1', next_line=True):
+            for i, time_str in pbar(enumerate(time_list), description=f"{bar_desc}", total=len(time_list), next_line=True):
                 _prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, time_str, None, depth, level, store_path, dataset_name, version_name, cover)
                 # progress.update(task, advance=1, description=f"[cyan]{bar_desc} {i + 1}/{len(time_list)}")
         else:
@@ -976,7 +976,7 @@ def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_mi
                 futures = [executor.submit(_download_task, var, time_str, None, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, cover) for time_str in time_list]
                 """ for feature in as_completed(futures):
                     _done_callback(feature, progress, task, len(time_list), counter_lock) """
-                for _ in pbar(as_completed(futures),description=f"{bar_desc}", total=len(futures),cmap='colorful_1',next_line=True):
+                for _ in pbar(as_completed(futures),description=f"{bar_desc}", total=len(futures),next_line=True):
                     pass
     else:
         print("[bold red]Please ensure the time_s is no more than time_e")
@@ -1121,10 +1121,10 @@ def download(
         workers = 1
         given_idm_engine = idm_path
         idm_download_list = []
-        bar_desc = "Submitting to IDM ->"
+        bar_desc = "Submitting to IDM"
     else:
         use_idm = False
-        bar_desc = "Downloading ->"
+        bar_desc = "Downloading"
 
     global match_time
     match_time = validate_time
@@ -1136,7 +1136,7 @@ def download(
         workers = 1
         print("*" * mark_len)
         print("[bold red]Only checking the time of existing files.")
-        bar_desc = "Checking time ->"
+        bar_desc = "Checking time"
 
     _download_hourly_func(
         variables,
@@ -1162,7 +1162,7 @@ def download(
         print("[bold #ecdbfe]*" * mark_len)
         if idm_download_list:
             remain_list = idm_download_list.copy()
-            for _ in pbar(range(len(idm_download_list)), cmap="diverging_1", description="Downloading ->"):
+            for _ in pbar(range(len(idm_download_list)), description="Downloading"):
                 success = False
                 while not success:
                     for f in remain_list:

oafuncs/oa_nc.py

@@ -6,7 +6,7 @@ import numpy as np
 import xarray as xr
 from rich import print
 
-__all__ = ["save", "merge", "modify", "rename", "check", "convert_longitude", "isel", "draw", "unpack_netcdf"]
+__all__ = ["save", "merge", "modify", "rename", "check", "convert_longitude", "isel", "draw", "compress_netcdf", "unpack_netcdf"]
 
 
 def save(
@@ -278,6 +278,26 @@ def draw(
         print("[red]No dataset or file provided.[/red]")
 
 
+def compress_netcdf(src_path, dst_path=None):
+    """
+    压缩 NetCDF 文件，使用 scale_factor/add_offset 压缩数据。
+    若 dst_path 省略，则自动生成新文件名，写出后删除原文件并将新文件改回原名。
+    """
+    # 判断是否要替换原文件
+    delete_orig = dst_path is None
+    if delete_orig:
+        dst_path = src_path.replace(".nc", "_compress.nc")
+
+    ds = xr.open_dataset(src_path)
+    save(dst_path, ds)
+    ds.close()
+
+    if delete_orig:
+        os.remove(src_path)
+        os.rename(dst_path, src_path)
+    pass
+
+
 def unpack_netcdf(src_path, dst_path=None):
     """解码 NetCDF 并移除 scale_factor/add_offset，写出真实值。
     若 dst_path 省略，则自动生成新文件名，写出后删除原文件并将新文件改回原名。

{oafuncs-0.0.98.10.dist-info → oafuncs-0.0.98.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oafuncs
-Version: 0.0.98.10
+Version: 0.0.98.12
 Summary: Oceanic and Atmospheric Functions
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu

{oafuncs-0.0.98.10.dist-info → oafuncs-0.0.98.12.dist-info}/RECORD

@@ -1,18 +1,18 @@
 oafuncs/__init__.py,sha256=T_-VtnWWllV3Q91twT5Yt2sUapeA051QbPNnBxmg9nw,1456
 oafuncs/oa_cmap.py,sha256=DimWT4Bg7uE5Lx8hSw1REp7whpsR2pFRStAwk1cowEM,11494
-oafuncs/oa_data.py,sha256=F0IR7T-BoWZho5aoDI_mWWUuuOvifohkkNLFm9Wlsqs,10312
+oafuncs/oa_data.py,sha256=PXn4EpSbLPHhYmNJXEStd8vIMwInl3a9O9sW2c9z-g0,11152
 oafuncs/oa_date.py,sha256=WhM6cyD4G3IeghjLTHhAMtlvJbA7kwQG2sHnxdTgyso,6303
 oafuncs/oa_draw.py,sha256=Wj2QBgyIPpV_dxaDrH10jqj_puK9ZM9rd-si-3VrsrE,17631
 oafuncs/oa_file.py,sha256=j9gXJgPOJsliu4IOUc4bc-luW4yBvQyNCEmMyDVjUwQ,16404
 oafuncs/oa_help.py,sha256=_4AZgRDq5Or0vauNvq5IDDHIBoBfdOQtzak-mG1wwAw,4537
-oafuncs/oa_nc.py,sha256=S23QL_GfIaENPr9p7oEeFT34nqJ2-7fiCcFQu72CmjI,11327
+oafuncs/oa_nc.py,sha256=lLEPjj4qgdEw1al0r1nKGZUnMP_ejT8A2NKD4lrs2kc,11936
 oafuncs/oa_python.py,sha256=NkopwkYFGSEuVljnTBvXCl6o2CeyRNBqRXSsUl3euEE,5192
 oafuncs/oa_tool.py,sha256=EqOlGPq3Rx2ohqVnGuCZhMvr2o9_XgglrETMbAdEifM,8471
 oafuncs/_data/hycom.png,sha256=MadKs6Gyj5n9-TOu7L4atQfTXtF9dvN9w-tdU9IfygI,10945710
 oafuncs/_data/oafuncs.png,sha256=o3VD7wm-kwDea5E98JqxXl04_78cBX7VcdUt7uQXGiU,3679898
 oafuncs/_script/cprogressbar.py,sha256=UIgGcLFs-6IgWlITuBLaQqrpt4OAK3Mst5RlCiNfZdQ,15772
 oafuncs/_script/email.py,sha256=lL4HGKrr524-g0xLlgs-4u7x4-u7DtgNoD9AL8XJKj4,3058
-oafuncs/_script/netcdf_merge.py,sha256=ktmTOgGfLHBNdS4HBc6xFDfO8B7E4DT7d1e6Dtare9Y,5596
+oafuncs/_script/netcdf_merge.py,sha256=ncNxstXJ77Ftyac5b1yRriVoBxbTVtaU111NU_7k_QA,5282
 oafuncs/_script/netcdf_modify.py,sha256=sGRUYNhfGgf9JV70rnBzw3bzuTRSXzBTL_RMDnDPeLQ,4552
 oafuncs/_script/netcdf_write.py,sha256=iO1Qv9bp6RLiw1D8Nrv7tX_8X-diUZaX3Nxhk6pJ5Nw,8556
 oafuncs/_script/parallel.py,sha256=T9Aie-e4LcbKlFTLZ0l4lhEN3SBVa84jRcrAsIm8s0I,8767
@@ -21,7 +21,7 @@ oafuncs/_script/plot_dataset.py,sha256=zkSEnO_-biyagorwWXPoihts_cwuvripzEt-l9bHJ
 oafuncs/_script/replace_file_content.py,sha256=eCFZjnZcwyRvy6b4mmIfBna-kylSZTyJRfgXd6DdCjk,5982
 oafuncs/oa_down/User_Agent-list.txt,sha256=pHaMlElMvZ8TG4vf4BqkZYKqe0JIGkr4kCN0lM1Y9FQ,514295
 oafuncs/oa_down/__init__.py,sha256=kRX5eTUCbAiz3zTaQM1501paOYS_3fizDN4Pa0mtNUA,585
-oafuncs/oa_down/hycom_3hourly.py,sha256=wWV14-OB9_LMmjUiZr3YXWBdKKwAyGXNa3Up7fSiWwk,55553
+oafuncs/oa_down/hycom_3hourly.py,sha256=VYxG9DPAnS1wk8gEXyqSIww4IvNRWOYGC4D6PTh5t3A,55450
 oafuncs/oa_down/hycom_3hourly_proxy.py,sha256=1eaoJGI_m-7w4ZZ3n7NGxkZaeFdsm0d3U-hyw8RFNbc,54563
 oafuncs/oa_down/idm.py,sha256=4z5IvgfTyIKEI1kOtqXZwN7Jnfjwp6qDBOIoVyOLp0I,1823
 oafuncs/oa_down/literature.py,sha256=2bF9gSKQbzcci9LcKE81j8JEjIJwON7jbwQB3gDDA3E,11331
@@ -37,8 +37,8 @@ oafuncs/oa_sign/__init__.py,sha256=QKqTFrJDFK40C5uvk48GlRRbGFzO40rgkYwu6dYxatM,5
 oafuncs/oa_sign/meteorological.py,sha256=8091SHo2L8kl4dCFmmSH5NGVHDku5i5lSiLEG5DLnOQ,6489
 oafuncs/oa_sign/ocean.py,sha256=xrW-rWD7xBWsB5PuCyEwQ1Q_RDKq2KCLz-LOONHgldU,5932
 oafuncs/oa_sign/scientific.py,sha256=a4JxOBgm9vzNZKpJ_GQIQf7cokkraV5nh23HGbmTYKw,5064
-oafuncs-0.0.98.10.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
-oafuncs-0.0.98.10.dist-info/METADATA,sha256=iyv12KtjFAqYtbJxBTq3RhgQ55iG5_yfCx4IzVWfJHw,4273
-oafuncs-0.0.98.10.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
-oafuncs-0.0.98.10.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
-oafuncs-0.0.98.10.dist-info/RECORD,,
+oafuncs-0.0.98.12.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
+oafuncs-0.0.98.12.dist-info/METADATA,sha256=TfDApyqtzs-wBr9sBG2sYBVHZFfJht6g1N3JtCQSHfU,4273
+oafuncs-0.0.98.12.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+oafuncs-0.0.98.12.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
+oafuncs-0.0.98.12.dist-info/RECORD,,

{oafuncs-0.0.98.10.dist-info → oafuncs-0.0.98.12.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.0)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any