oafuncs 0.0.98.9__py3-none-any.whl → 0.0.98.11__py3-none-any.whl

oafuncs/_script/cprogressbar.py

@@ -148,7 +148,7 @@ class ColorProgressBar:
         description: str = "Working ...",
         total: Optional[float] = None,
         completed: float = 0,
-        color: Any = "cyan",
+        color: Any = "green",
         cmap: Union[str, List[str]] = None,
         update_interval: float = 0.1,
         bar_length: int = None,

oafuncs/_script/parallel.py

@@ -1,4 +1,3 @@
-import atexit
 import logging
 import multiprocessing as mp
 import platform
@@ -30,6 +29,7 @@ class ParallelExecutor:
         self.running = True
         self.task_history = []
         self._executor = None
+        self._shutdown_called = False
 
         self.mode, default_workers = self._determine_optimal_settings()
         self.max_workers = max_workers or default_workers
@@ -37,7 +37,6 @@ class ParallelExecutor:
 
         self._init_platform_settings()
         self._start_resource_monitor()
-        atexit.register(self.shutdown)
 
         logging.info(f"Initialized {self.__class__.__name__} on {self.platform} (mode={self.mode}, workers={self.max_workers})")
 
@@ -99,16 +98,26 @@ class ParallelExecutor:
 
     def run(self, func: Callable, params: List[Tuple], chunk_size: Optional[int] = None) -> List[Any]:
         chunk_size = chunk_size or self.chunk_size
-        for retry in range(self.max_retries + 1):
-            try:
-                start_time = time.monotonic()
-                results = self._execute_batch(func, params, chunk_size)
-                self._update_settings(time.monotonic() - start_time, len(params))
-                return results
-            except Exception as e:
-                logging.error(f"Attempt {retry + 1} failed: {e}")
-                self._handle_failure()
-        raise RuntimeError(f"Failed after {self.max_retries} retries")
+        try:
+            for retry in range(self.max_retries + 1):
+                try:
+                    start_time = time.monotonic()
+                    results = self._execute_batch(func, params, chunk_size)
+                    self._update_settings(time.monotonic() - start_time, len(params))
+                    return results
+                except Exception as e:
+                    logging.error(f"Attempt {retry + 1} failed: {e}")
+                    self._handle_failure()
+            raise RuntimeError(f"Failed after {self.max_retries} retries")
+        finally:
+            # 仅关闭当前 executor，保留资源监控等运行状态
+            if self._executor:
+                try:
+                    self._executor.shutdown(wait=True)
+                except Exception as e:
+                    logging.error(f"Executor shutdown error: {e}")
+                finally:
+                    self._executor = None
 
     def _execute_batch(self, func: Callable, params: List[Tuple], chunk_size: int) -> List[Any]:
         if not params:
@@ -168,10 +177,14 @@ class ParallelExecutor:
             self._restart_executor()
 
     def shutdown(self):
+        if self._shutdown_called:
+            return
+        self._shutdown_called = True
         self.running = False
+        # 基类不再打印日志，由子类统一处理
         if self._executor:
             try:
-                self._executor.shutdown(wait=False)
+                self._executor.shutdown(wait=True)
             except Exception as e:
                 logging.error(f"Shutdown error: {e}")
             finally:

oafuncs/oa_data.py

@@ -121,7 +121,26 @@ def _interp_single_worker(*args):
     """
     data_slice, origin_points, target_points, interpolation_method, target_shape = args
 
-    return griddata(origin_points, data_slice.ravel(), target_points, method=interpolation_method).reshape(target_shape)
+    # 过滤掉包含 NaN 的点
+    valid_mask = ~np.isnan(data_slice.ravel())
+    valid_data = data_slice.ravel()[valid_mask]
+    valid_points = origin_points[valid_mask]
+
+    if len(valid_data) < 10:  # 如果有效数据太少，用均值填充
+        return np.full(target_shape, np.nanmean(data_slice))
+
+    # 使用有效数据进行插值
+    result = griddata(valid_points, valid_data, target_points, method=interpolation_method)
+    result = result.reshape(target_shape)
+
+    # 检查插值结果中是否仍有 NaN，如果有，用最近邻插值填充
+    if np.any(np.isnan(result)):
+        # 使用最近邻方法填充剩余的 NaN 值
+        nan_mask = np.isnan(result)
+        result_nn = griddata(valid_points, valid_data, target_points[nan_mask.ravel()], method="nearest")
+        result.ravel()[nan_mask.ravel()] = result_nn
+
+    return result
 
 
 def interp_2d(
@@ -130,7 +149,7 @@ def interp_2d(
     source_x_coordinates: Union[np.ndarray, List[float]],
     source_y_coordinates: Union[np.ndarray, List[float]],
     source_data: np.ndarray,
-    interpolation_method: str = "linear",
+    interpolation_method: str = "cubic",
 ) -> np.ndarray:
     """
     Perform 2D interpolation on the last two dimensions of a multi-dimensional array.
@@ -141,7 +160,8 @@ def interp_2d(
         source_x_coordinates (Union[np.ndarray, List[float]]): Original grid's x-coordinates.
         source_y_coordinates (Union[np.ndarray, List[float]]): Original grid's y-coordinates.
         source_data (np.ndarray): Multi-dimensional array with the last two dimensions as spatial.
-        interpolation_method (str, optional): Interpolation method. Defaults to "linear".
+        interpolation_method (str, optional): Interpolation method. Defaults to "cubic".
+            >>> optional: 'linear', 'nearest', 'cubic', 'quintic', etc.
         use_parallel (bool, optional): Enable parallel processing. Defaults to True.
 
     Returns:
@@ -190,11 +210,12 @@ def interp_2d(
     for t_index in range(t):
         for z_index in range(z):
             paras.append((new_src_data[t_index, z_index], origin_points, target_points, interpolation_method, target_shape))
-    
+
     with PEx() as excutor:
         result = excutor.run(_interp_single_worker, paras)
 
-    return np.squeeze(np.array(result))
+    return np.squeeze(np.array(result).reshape(t, z, *target_shape))
+
 
 
 def mask_shapefile(

oafuncs/oa_down/hycom_3hourly.py

@@ -584,7 +584,7 @@ def _clear_existing_file(file_full_path):
 
 def _check_existing_file(file_full_path, avg_size):
     if os.path.exists(file_full_path):
-        print(f"[bold #FFA54F]{file_full_path} exists")
+        print(f"[bold #FFA54F]{file_full_path} exists ...")
         fsize = file_size(file_full_path)
         delta_size_ratio = (fsize - avg_size) / avg_size
         if abs(delta_size_ratio) > 0.025:
@@ -796,7 +796,7 @@ def _download_file(target_url, store_path, file_name, cover=False):
     get_mean_size = _get_mean_size_move(same_file, save_path)
 
     if _check_existing_file(save_path, get_mean_size):
-        print(f"[bold #FFA54F]{save_path} exists, skipping ...")
+        # print(f"[bold #FFA54F]{save_path} exists, skipping ...")
         count_dict["skip"] += 1
         return
 
@@ -901,7 +901,7 @@ def _prepare_url_to_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_ma
         else:
             if download_time < "2024081012":
                 varlist = [_ for _ in var]
-                for key, value in pbar(var_group.items(), description=f"Var Group {download_time} ->", total=len(var_group), color="#d7feb9", next_line=True):
+                for key, value in pbar(var_group.items(), description=f"Var Group {download_time}", total=len(var_group), next_line=True):
                     current_group = []
                     for v in varlist:
                         if v in value:
@@ -923,7 +923,7 @@ def _prepare_url_to_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_ma
                             file_name = f"HYCOM_{key}_{download_time}-{download_time_end}.nc"
                     _download_file(submit_url, store_path, file_name, cover)
             else:
-                for v in pbar(var, description=f"Var {download_time} ->", total=len(var), color="#d7feb9", next_line=True):
+                for v in pbar(var, description=f"Var {download_time}", total=len(var), next_line=True):
                     submit_url = _get_submit_url_var(v, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time, download_time_end)
                     file_name = f"HYCOM_{variable_info[v]['var_name']}_{download_time}.nc"
                     if download_time_end is not None:
@@ -968,7 +968,7 @@ def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_mi
         # with Progress() as progress:
             # task = progress.add_task(f"[cyan]{bar_desc}", total=len(time_list))
         if num_workers is None or num_workers <= 1:
-            for i, time_str in pbar(enumerate(time_list), description=f"{bar_desc}", total=len(time_list), cmap='colorful_1', next_line=True):
+            for i, time_str in pbar(enumerate(time_list), description=f"{bar_desc}", total=len(time_list), next_line=True):
                 _prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, time_str, None, depth, level, store_path, dataset_name, version_name, cover)
                 # progress.update(task, advance=1, description=f"[cyan]{bar_desc} {i + 1}/{len(time_list)}")
         else:
@@ -976,7 +976,7 @@ def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_mi
                 futures = [executor.submit(_download_task, var, time_str, None, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, cover) for time_str in time_list]
                 """ for feature in as_completed(futures):
                     _done_callback(feature, progress, task, len(time_list), counter_lock) """
-                for _ in pbar(as_completed(futures),description=f"{bar_desc}", total=len(futures),cmap='colorful_1',next_line=True):
+                for _ in pbar(as_completed(futures),description=f"{bar_desc}", total=len(futures),next_line=True):
                     pass
     else:
         print("[bold red]Please ensure the time_s is no more than time_e")
@@ -1121,10 +1121,10 @@ def download(
         workers = 1
         given_idm_engine = idm_path
         idm_download_list = []
-        bar_desc = "Submitting to IDM ->"
+        bar_desc = "Submitting to IDM"
     else:
         use_idm = False
-        bar_desc = "Downloading ->"
+        bar_desc = "Downloading"
 
     global match_time
     match_time = validate_time
@@ -1136,7 +1136,7 @@ def download(
         workers = 1
         print("*" * mark_len)
         print("[bold red]Only checking the time of existing files.")
-        bar_desc = "Checking time ->"
+        bar_desc = "Checking time"
 
     _download_hourly_func(
         variables,
@@ -1162,7 +1162,7 @@ def download(
         print("[bold #ecdbfe]*" * mark_len)
         if idm_download_list:
             remain_list = idm_download_list.copy()
-            for _ in pbar(range(len(idm_download_list)), cmap="diverging_1", description="Downloading ->"):
+            for _ in pbar(range(len(idm_download_list)), description="Downloading"):
                 success = False
                 while not success:
                     for f in remain_list:

oafuncs/oa_file.py

@@ -287,7 +287,11 @@ def remove(target_pattern: str) -> None:
                 (shutil.rmtree if os.path.isdir(file_path) else os.remove)(file_path)
                 print(f"[green]Successfully deleted:[/green] [bold]{file_path}[/bold]")
             else:
-                print(f"[yellow]Skipping unknown file type:[/yellow] [bold]{file_path}[/bold]")
+                if not os.path.exists(file_path):
+                    # print(f"[yellow]File not found:[/yellow] [bold]{file_path}[/bold]")
+                    pass
+                else:
+                    print(f"[yellow]Skipping unknown file type:[/yellow] [bold]{file_path}[/bold]")
         except Exception as e:
             print(f"[red]Failed to delete:[/red] [bold]{file_path}[/bold]. Error: {e}")
 

oafuncs/oa_nc.py

@@ -6,7 +6,7 @@ import numpy as np
 import xarray as xr
 from rich import print
 
-__all__ = ["save", "merge", "modify", "rename", "check", "convert_longitude", "isel", "draw"]
+__all__ = ["save", "merge", "modify", "rename", "check", "convert_longitude", "isel", "draw", "compress_netcdf", "unpack_netcdf"]
 
 
 def save(
@@ -278,6 +278,48 @@ def draw(
         print("[red]No dataset or file provided.[/red]")
 
 
+def compress_netcdf(src_path, dst_path=None):
+    """
+    压缩 NetCDF 文件，使用 scale_factor/add_offset 压缩数据。
+    若 dst_path 省略，则自动生成新文件名，写出后删除原文件并将新文件改回原名。
+    """
+    # 判断是否要替换原文件
+    delete_orig = dst_path is None
+    if delete_orig:
+        dst_path = src_path.replace(".nc", "_compress.nc")
+
+    ds = xr.open_dataset(src_path)
+    save(dst_path, ds)
+    ds.close()
+
+    if delete_orig:
+        os.remove(src_path)
+        os.rename(dst_path, src_path)
+    pass
+
+
+def unpack_netcdf(src_path, dst_path=None):
+    """解码 NetCDF 并移除 scale_factor/add_offset，写出真实值。
+    若 dst_path 省略，则自动生成新文件名，写出后删除原文件并将新文件改回原名。
+    """
+    # 判断是否要替换原文件
+    delete_orig = dst_path is None
+    if delete_orig:
+        dst_path = src_path.replace(".nc", "_unpacked.nc")
+
+    ds = xr.open_dataset(src_path, decode_cf=True)
+    for var in ds.data_vars:
+        ds[var].attrs.pop("scale_factor", None)
+        ds[var].attrs.pop("add_offset", None)
+        ds[var].encoding.clear()
+    ds.to_netcdf(dst_path, mode="w", format="NETCDF4", engine="netcdf4")
+    ds.close()
+
+    if delete_orig:
+        os.remove(src_path)
+        os.rename(dst_path, src_path)
+
+
 if __name__ == "__main__":
     data = np.random.rand(100, 50)
     save(r"test.nc", data, "data", {"time": np.linspace(0, 120, 100), "lev": np.linspace(0, 120, 50)}, "a")

oafuncs/oa_tool.py

@@ -137,7 +137,7 @@ def pbar(
     description: str = "Working...",
     total: Optional[float] = None,
     completed: float = 0,
-    color: Any = "cyan",
+    color: Any = "None",
     cmap: Union[str, List[str], None] = None,
     update_interval: float = 0.1,
     bar_length: Optional[int] = None,
@@ -173,8 +173,23 @@ def pbar(
         ...     time.sleep(0.1)
     """
     from ._script.cprogressbar import ColorProgressBar
+    import random
 
-    print(f"[blue]{description}[/blue]")
+    def _generate_random_color_hex():
+        """Generate a random color in hexadecimal format."""
+        r = random.randint(0, 255)
+        g = random.randint(0, 255)
+        b = random.randint(0, 255)
+        return '#{r:02x}{g:02x}{b:02x}'.format(r=r, g=g, b=b)
+
+    if color == 'None' and cmap is None:
+        color = _generate_random_color_hex()
+
+    style = f"bold {color if color != 'None' else 'green'}"
+    print(f"[{style}]~*^* {description} *^*~[/{style}]")
+
+    description=''
+    
     return ColorProgressBar(
         iterable=iterable,
         description=description,

{oafuncs-0.0.98.9.dist-info → oafuncs-0.0.98.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oafuncs
-Version: 0.0.98.9
+Version: 0.0.98.11
 Summary: Oceanic and Atmospheric Functions
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu

{oafuncs-0.0.98.9.dist-info → oafuncs-0.0.98.11.dist-info}/RECORD

@@ -1,27 +1,27 @@
 oafuncs/__init__.py,sha256=T_-VtnWWllV3Q91twT5Yt2sUapeA051QbPNnBxmg9nw,1456
 oafuncs/oa_cmap.py,sha256=DimWT4Bg7uE5Lx8hSw1REp7whpsR2pFRStAwk1cowEM,11494
-oafuncs/oa_data.py,sha256=SqvG2mRVSTmr5j7H3Cn-ILgDo8iwSAl6W5NLpVpBx3c,10288
+oafuncs/oa_data.py,sha256=PXn4EpSbLPHhYmNJXEStd8vIMwInl3a9O9sW2c9z-g0,11152
 oafuncs/oa_date.py,sha256=WhM6cyD4G3IeghjLTHhAMtlvJbA7kwQG2sHnxdTgyso,6303
 oafuncs/oa_draw.py,sha256=Wj2QBgyIPpV_dxaDrH10jqj_puK9ZM9rd-si-3VrsrE,17631
-oafuncs/oa_file.py,sha256=goF5iRXJFFCIKhIjlkCnYYt0EYlJb_4r8AeYNZ0-SOk,16209
+oafuncs/oa_file.py,sha256=j9gXJgPOJsliu4IOUc4bc-luW4yBvQyNCEmMyDVjUwQ,16404
 oafuncs/oa_help.py,sha256=_4AZgRDq5Or0vauNvq5IDDHIBoBfdOQtzak-mG1wwAw,4537
-oafuncs/oa_nc.py,sha256=L1gqXxg93kIDsMOa87M0o-53KVmdqCipnXeF9XfzfY8,10513
+oafuncs/oa_nc.py,sha256=lLEPjj4qgdEw1al0r1nKGZUnMP_ejT8A2NKD4lrs2kc,11936
 oafuncs/oa_python.py,sha256=NkopwkYFGSEuVljnTBvXCl6o2CeyRNBqRXSsUl3euEE,5192
-oafuncs/oa_tool.py,sha256=UNdiXKWfk860eb01vSGH8XN_Bd3CP7ihaspRrudday0,7983
+oafuncs/oa_tool.py,sha256=EqOlGPq3Rx2ohqVnGuCZhMvr2o9_XgglrETMbAdEifM,8471
 oafuncs/_data/hycom.png,sha256=MadKs6Gyj5n9-TOu7L4atQfTXtF9dvN9w-tdU9IfygI,10945710
 oafuncs/_data/oafuncs.png,sha256=o3VD7wm-kwDea5E98JqxXl04_78cBX7VcdUt7uQXGiU,3679898
-oafuncs/_script/cprogressbar.py,sha256=wRU3SFPFtMI7ER26tTzg223kVKNo5RDWE9CzdIgUsuE,15771
+oafuncs/_script/cprogressbar.py,sha256=UIgGcLFs-6IgWlITuBLaQqrpt4OAK3Mst5RlCiNfZdQ,15772
 oafuncs/_script/email.py,sha256=lL4HGKrr524-g0xLlgs-4u7x4-u7DtgNoD9AL8XJKj4,3058
 oafuncs/_script/netcdf_merge.py,sha256=ktmTOgGfLHBNdS4HBc6xFDfO8B7E4DT7d1e6Dtare9Y,5596
 oafuncs/_script/netcdf_modify.py,sha256=sGRUYNhfGgf9JV70rnBzw3bzuTRSXzBTL_RMDnDPeLQ,4552
 oafuncs/_script/netcdf_write.py,sha256=iO1Qv9bp6RLiw1D8Nrv7tX_8X-diUZaX3Nxhk6pJ5Nw,8556
-oafuncs/_script/parallel.py,sha256=dRT7w_rBnR3mZkUlO6v6j05SwBTQpTccOna5CXI5Msg,8196
+oafuncs/_script/parallel.py,sha256=T9Aie-e4LcbKlFTLZ0l4lhEN3SBVa84jRcrAsIm8s0I,8767
 oafuncs/_script/parallel_test.py,sha256=0GBqZOX7IaCOKF2t1y8N8YYu53GJ33OkfsWgpvZNqM4,372
 oafuncs/_script/plot_dataset.py,sha256=zkSEnO_-biyagorwWXPoihts_cwuvripzEt-l9bHJ2E,13989
 oafuncs/_script/replace_file_content.py,sha256=eCFZjnZcwyRvy6b4mmIfBna-kylSZTyJRfgXd6DdCjk,5982
 oafuncs/oa_down/User_Agent-list.txt,sha256=pHaMlElMvZ8TG4vf4BqkZYKqe0JIGkr4kCN0lM1Y9FQ,514295
 oafuncs/oa_down/__init__.py,sha256=kRX5eTUCbAiz3zTaQM1501paOYS_3fizDN4Pa0mtNUA,585
-oafuncs/oa_down/hycom_3hourly.py,sha256=wWV14-OB9_LMmjUiZr3YXWBdKKwAyGXNa3Up7fSiWwk,55553
+oafuncs/oa_down/hycom_3hourly.py,sha256=VYxG9DPAnS1wk8gEXyqSIww4IvNRWOYGC4D6PTh5t3A,55450
 oafuncs/oa_down/hycom_3hourly_proxy.py,sha256=1eaoJGI_m-7w4ZZ3n7NGxkZaeFdsm0d3U-hyw8RFNbc,54563
 oafuncs/oa_down/idm.py,sha256=4z5IvgfTyIKEI1kOtqXZwN7Jnfjwp6qDBOIoVyOLp0I,1823
 oafuncs/oa_down/literature.py,sha256=2bF9gSKQbzcci9LcKE81j8JEjIJwON7jbwQB3gDDA3E,11331
@@ -37,8 +37,8 @@ oafuncs/oa_sign/__init__.py,sha256=QKqTFrJDFK40C5uvk48GlRRbGFzO40rgkYwu6dYxatM,5
 oafuncs/oa_sign/meteorological.py,sha256=8091SHo2L8kl4dCFmmSH5NGVHDku5i5lSiLEG5DLnOQ,6489
 oafuncs/oa_sign/ocean.py,sha256=xrW-rWD7xBWsB5PuCyEwQ1Q_RDKq2KCLz-LOONHgldU,5932
 oafuncs/oa_sign/scientific.py,sha256=a4JxOBgm9vzNZKpJ_GQIQf7cokkraV5nh23HGbmTYKw,5064
-oafuncs-0.0.98.9.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
-oafuncs-0.0.98.9.dist-info/METADATA,sha256=Ns5ahXYSJiK2OmggCSbzoFC5XHMnLsFdC_KMnVrAnGA,4272
-oafuncs-0.0.98.9.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
-oafuncs-0.0.98.9.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
-oafuncs-0.0.98.9.dist-info/RECORD,,
+oafuncs-0.0.98.11.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
+oafuncs-0.0.98.11.dist-info/METADATA,sha256=vbHI4zd7XEcXnbCI7VuGmyDEcnfv5hsVjR8XczfObTU,4273
+oafuncs-0.0.98.11.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+oafuncs-0.0.98.11.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
+oafuncs-0.0.98.11.dist-info/RECORD,,

{oafuncs-0.0.98.9.dist-info → oafuncs-0.0.98.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.0)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any