oafuncs 0.0.98.19__py3-none-any.whl → 0.0.98.21__py3-none-any.whl

oafuncs/oa_down/hycom_3hourly.py

@@ -1,18 +1,3 @@
-#!/usr/bin/env python
-# coding=utf-8
-"""
-Author: Liu Kun && 16031215@qq.com
-Date: 2025-04-07 10:51:09
-LastEditors: Liu Kun && 16031215@qq.com
-LastEditTime: 2025-04-07 10:51:09
-FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_down\\hycom_3hourly copy.py
-Description:
-EditPlatform: vscode
-ComputerInfo: XPS 15 9510
-SystemInfo: Windows 11
-Python Version: 3.12
-"""
-
 import asyncio
 import datetime
 import logging
@@ -23,7 +8,6 @@ import warnings
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from threading import Lock
-from oafuncs.oa_tool import pbar
 
 import httpx
 import matplotlib.pyplot as plt
@@ -38,6 +22,7 @@ from oafuncs.oa_down.user_agent import get_ua
 from oafuncs.oa_file import file_size
 from oafuncs.oa_nc import check as check_nc
 from oafuncs.oa_nc import modify as modify_nc
+from oafuncs.oa_tool import pbar
 
 logging.getLogger("httpx").setLevel(logging.WARNING)  # 关闭 httpx 的 INFO 日志，只显示 WARNING 及以上
 
@@ -724,7 +709,6 @@ class _HycomDownloader:
                                     logging.info(f"{file_name}: {percent}% ({downloaded / 1024:.1f} KB / {total / 1024:.1f} KB)")
                                     last_percent = percent
 
-
                     elapsed = datetime.datetime.now() - start
                     # logging.info(f"File {file_name} downloaded, Time: {elapsed}")
                     logging.info(f"Saving {file_name} ...")
@@ -966,7 +950,7 @@ def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_mi
             print("Downloading a series of files...")
         time_list = _get_time_list(ymdh_time_s, ymdh_time_e, interval_hour, "hour")
         # with Progress() as progress:
-            # task = progress.add_task(f"[cyan]{bar_desc}", total=len(time_list))
+        # task = progress.add_task(f"[cyan]{bar_desc}", total=len(time_list))
         if num_workers is None or num_workers <= 1:
             for i, time_str in pbar(enumerate(time_list), description=f"{bar_desc}", total=len(time_list), next_line=True):
                 _prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, time_str, None, depth, level, store_path, dataset_name, version_name, cover)
@@ -976,7 +960,7 @@ def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_mi
                 futures = [executor.submit(_download_task, var, time_str, None, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, cover) for time_str in time_list]
                 """ for feature in as_completed(futures):
                     _done_callback(feature, progress, task, len(time_list), counter_lock) """
-                for _ in pbar(as_completed(futures),description=f"{bar_desc}", total=len(futures),next_line=True):
+                for _ in pbar(as_completed(futures), description=f"{bar_desc}", total=len(futures), next_line=True):
                     pass
     else:
         print("[bold red]Please ensure the time_s is no more than time_e")
@@ -1174,7 +1158,7 @@ def download(
 
     count_dict["total"] = count_dict["success"] + count_dict["fail"] + count_dict["skip"] + count_dict["no_data"]
     print("[bold #ecdbfe]=" * mark_len)
-    print(f"[bold #ff80ab]Total: {count_dict['total']}\nSuccess: {count_dict['success']}\nFail: {count_dict['fail']}\nSkip: {count_dict['skip']}\nNo data: {count_dict['no_data']}")
+    print(f"[bold #ff80ab]Total  : {count_dict['total']}\nSuccess: {count_dict['success']}\nFail   : {count_dict['fail']}\nSkip   : {count_dict['skip']}\nNo data: {count_dict['no_data']}")
     print("[bold #ecdbfe]=" * mark_len)
     if count_dict["fail"] > 0:
         print("[bold #be5528]Please try again to download the failed data later.")

oafuncs/oa_draw.py

@@ -318,7 +318,7 @@ def add_gridlines(axes: plt.Axes, longitude_lines: list[float] = None, latitude_
     if latitude_lines is not None:
         gl.ylocator = mticker.FixedLocator(np.array(latitude_lines))
 
-    print("[green]Gridlines added successfully.[/green]")
+    # print("[green]Gridlines added successfully.[/green]")
     return axes, gl
 
 
@@ -365,7 +365,7 @@ def add_cartopy(axes: plt.Axes, longitude_data: np.ndarray = None, latitude_data
         lat_min, lat_max = np.nanmin(latitude_data), np.nanmax(latitude_data)
         axes.set_extent([lon_min, lon_max, lat_min, lat_max], crs=map_projection)
 
-    print("[green]Cartopy features added successfully.[/green]")
+    # print("[green]Cartopy features added successfully.[/green]")
     return axes
 
 

oafuncs/oa_nc.py

@@ -6,7 +6,8 @@ import numpy as np
 import xarray as xr
 from rich import print
 
-__all__ = ["save", "merge", "modify", "rename", "check", "convert_longitude", "isel", "draw", "compress_netcdf", "unpack_netcdf"]
+__all__ = ["save", "merge", "modify", "rename", "check", "convert_longitude", "isel", "draw", "compress", "unscale"]
+
 
 
 def save(
@@ -15,8 +16,10 @@ def save(
     variable_name: Optional[str] = None,
     coordinates: Optional[dict] = None,
     write_mode: str = "w",
+    convert_dtype: str = "int32",
     use_scale_offset: bool = True,
     use_compression: bool = True,
+    preserve_mask_values: bool = True,
 ) -> None:
     """
     Write data to a NetCDF file.
@@ -27,8 +30,10 @@ def save(
         variable_name (Optional[str]): Variable name for the data.
         coordinates (Optional[dict]): Coordinates, where keys are dimension names and values are coordinate data.
         write_mode (str): Write mode, 'w' for write, 'a' for append. Default is 'w'.
+        convert_dtype (str): Data type to convert to. Default is 'int32'.
         use_scale_offset (bool): Whether to use scale_factor and add_offset. Default is True.
         use_compression (bool): Whether to use compression parameters. Default is True.
+        preserve_mask_values (bool): Whether to preserve mask values. Default is True.
 
     Example:
         >>> save(r'test.nc', data, 'u', {'time': np.linspace(0, 120, 100), 'lev': np.linspace(0, 120, 50)}, 'a')
@@ -38,7 +43,7 @@ def save(
     """
     from ._script.netcdf_write import save_to_nc
 
-    save_to_nc(file_path, data, variable_name, coordinates, write_mode, use_scale_offset, use_compression)
+    save_to_nc(file_path, data, variable_name, coordinates, write_mode, convert_dtype,use_scale_offset, use_compression, preserve_mask_values)
     print(f"[green]Data successfully saved to {file_path}[/green]")
 
 
@@ -278,7 +283,7 @@ def draw(
         print("[red]No dataset or file provided.[/red]")
 
 
-def compress_netcdf(src_path, dst_path=None):
+def compress(src_path, dst_path=None,convert_dtype='int16'):
     """
     压缩 NetCDF 文件，使用 scale_factor/add_offset 压缩数据。
     若 dst_path 省略，则自动生成新文件名，写出后删除原文件并将新文件改回原名。
@@ -289,7 +294,7 @@ def compress_netcdf(src_path, dst_path=None):
         dst_path = src_path.replace(".nc", "_compress.nc")
 
     ds = xr.open_dataset(src_path)
-    save(dst_path, ds)
+    save(dst_path, ds, convert_dtype=convert_dtype, use_scale_offset=True, use_compression=True)
     ds.close()
 
     if delete_orig:
@@ -298,26 +303,88 @@ def compress_netcdf(src_path, dst_path=None):
     pass
 
 
-def unpack_netcdf(src_path, dst_path=None):
+def unscale(src_path, dst_path=None, compression_level=4):
     """解码 NetCDF 并移除 scale_factor/add_offset，写出真实值。
+    保留压缩功能，但不使用比例因子和偏移量，以控制文件大小。
     若 dst_path 省略，则自动生成新文件名，写出后删除原文件并将新文件改回原名。
+
+    Args:
+        src_path: 源文件路径
+        dst_path: 目标文件路径，None则替换原文件
+        compression_level: 压缩级别(1-9)，数值越大压缩比越高，速度越慢
     """
     # 判断是否要替换原文件
     delete_orig = dst_path is None
     if delete_orig:
         dst_path = src_path.replace(".nc", "_unpacked.nc")
 
+    # 打开原始文件，获取文件大小
+    orig_size = os.path.getsize(src_path) / (1024 * 1024)  # MB
+
+    # 先以原始模式打开，查看哪些变量使用了scale_factor/add_offset
+    with xr.open_dataset(src_path, decode_cf=False) as ds_raw:
+        has_scaling = []
+        for var in ds_raw.data_vars:
+            if "scale_factor" in ds_raw[var].attrs or "add_offset" in ds_raw[var].attrs:
+                has_scaling.append(var)
+
+    print(f"[yellow]文件: {src_path} (原始大小: {orig_size:.2f} MB)[/yellow]")
+    if has_scaling:
+        print(f"[yellow]发现 {len(has_scaling)} 个变量使用了比例因子: {', '.join(has_scaling)}[/yellow]")
+    else:
+        print("[yellow]未发现任何变量使用比例因子，解包可能不必要[/yellow]")
+
+    # 解码模式打开
     ds = xr.open_dataset(src_path, decode_cf=True)
+    encoding = {}
+
     for var in ds.data_vars:
+        # 保存原始的_FillValue
+        fill_value = None
+        if "_FillValue" in ds[var].attrs:
+            fill_value = ds[var].attrs["_FillValue"]
+        elif "_FillValue" in ds[var].encoding:
+            fill_value = ds[var].encoding["_FillValue"]
+
+        # 清除scale_factor和add_offset属性
         ds[var].attrs.pop("scale_factor", None)
         ds[var].attrs.pop("add_offset", None)
         ds[var].encoding.clear()
-    ds.to_netcdf(dst_path, mode="w", format="NETCDF4", engine="netcdf4")
+
+        # 仅对数值型变量处理
+        if np.issubdtype(ds[var].dtype, np.number):
+            # 强制转换为float32，避免float64导致文件暴涨
+            if np.issubdtype(ds[var].dtype, np.floating) and ds[var].dtype != np.float32:
+                ds[var] = ds[var].astype(np.float32)
+
+            # 设置压缩参数，但不使用scale_factor/add_offset
+            encoding[var] = {"zlib": True, "complevel": compression_level, "dtype": ds[var].dtype}
+            # 恢复_FillValue
+            if fill_value is not None:
+                encoding[var]["_FillValue"] = fill_value
+
+    # 使用save函数保存，传入encoding确保只压缩不使用scale_factor
+    ds.to_netcdf(dst_path, encoding=encoding)
     ds.close()
 
+    # 打印输出文件大小对比
+    if os.path.exists(dst_path):
+        new_size = os.path.getsize(dst_path) / (1024 * 1024)  # MB
+        size_change = new_size - orig_size
+        change_percent = (size_change / orig_size) * 100
+
+        color = "green" if size_change <= 0 else "red"
+        print(f"[{color}]解包后文件大小: {new_size:.2f} MB ({change_percent:+.1f}%)[/{color}]")
+
+        if size_change > orig_size * 0.5 and new_size > 100:  # 如果文件增长超过50%且大于100MB
+            print(f"[red]警告: 文件大小增长显著! 考虑增加压缩级别(当前:{compression_level})[/red]")
+
     if delete_orig:
         os.remove(src_path)
         os.rename(dst_path, src_path)
+        print(f"[green]已替换原文件: {src_path}[/green]")
+    else:
+        print(f"[green]已保存到: {dst_path}[/green]")
 
 
 if __name__ == "__main__":

oafuncs/oa_tool.py

@@ -135,7 +135,7 @@ def email(title: str = "Title", content: Optional[str] = None, send_to: str = "1
 
 def pbar(
     iterable: Iterable = range(100),
-    description: str = "Working...",
+    description: str = "Working",
     total: Optional[float] = None,
     completed: float = 0,
     color: Any = "None",

{oafuncs-0.0.98.19.dist-info → oafuncs-0.0.98.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oafuncs
-Version: 0.0.98.19
+Version: 0.0.98.21
 Summary: Oceanic and Atmospheric Functions
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu

{oafuncs-0.0.98.19.dist-info → oafuncs-0.0.98.21.dist-info}/RECORD

@@ -1,29 +1,29 @@
 oafuncs/__init__.py,sha256=T_-VtnWWllV3Q91twT5Yt2sUapeA051QbPNnBxmg9nw,1456
-oafuncs/oa_cmap.py,sha256=DimWT4Bg7uE5Lx8hSw1REp7whpsR2pFRStAwk1cowEM,11494
-oafuncs/oa_data.py,sha256=y11xxaVNZ6_eveVjSG4PisRXYpKr_FFsBBh0mj_ss2g,8436
+oafuncs/oa_cmap.py,sha256=NVKwEkmMUKXh9L1svx_WHqemLOx3evgX3_-UVDxc9ko,11498
+oafuncs/oa_data.py,sha256=Aat9ktxxRGevaqQya3IJWfXeoEs-FCXGUcNE2pKnzfU,10931
 oafuncs/oa_date.py,sha256=WhM6cyD4G3IeghjLTHhAMtlvJbA7kwQG2sHnxdTgyso,6303
-oafuncs/oa_draw.py,sha256=Wj2QBgyIPpV_dxaDrH10jqj_puK9ZM9rd-si-3VrsrE,17631
+oafuncs/oa_draw.py,sha256=IaBGDx-EOxyMM2IuJ4zLZt6ruHHV5qFStPItmUOXoWk,17635
 oafuncs/oa_file.py,sha256=j9gXJgPOJsliu4IOUc4bc-luW4yBvQyNCEmMyDVjUwQ,16404
 oafuncs/oa_help.py,sha256=_4AZgRDq5Or0vauNvq5IDDHIBoBfdOQtzak-mG1wwAw,4537
-oafuncs/oa_nc.py,sha256=lLEPjj4qgdEw1al0r1nKGZUnMP_ejT8A2NKD4lrs2kc,11936
+oafuncs/oa_nc.py,sha256=UUXnBg2cO5XiJ8w0jNqCZJg83FVKqxlEHxOJG5o08Z8,15201
 oafuncs/oa_python.py,sha256=NkopwkYFGSEuVljnTBvXCl6o2CeyRNBqRXSsUl3euEE,5192
-oafuncs/oa_tool.py,sha256=rpPkLqWhqMmqlCc5wjL8qMTg3gThCkSrYJckbX_0iJc,8631
+oafuncs/oa_tool.py,sha256=QBjJh3pf54yXVuOmu97rW6Tsr6uNMyZ5KqZbR4VQFTc,8628
 oafuncs/_data/hycom.png,sha256=MadKs6Gyj5n9-TOu7L4atQfTXtF9dvN9w-tdU9IfygI,10945710
 oafuncs/_data/oafuncs.png,sha256=o3VD7wm-kwDea5E98JqxXl04_78cBX7VcdUt7uQXGiU,3679898
 oafuncs/_script/cprogressbar.py,sha256=UIgGcLFs-6IgWlITuBLaQqrpt4OAK3Mst5RlCiNfZdQ,15772
-oafuncs/_script/data_interp.py,sha256=KJ-p-UN3Op1MmtCoN4KdjFVHFE3GNHrTD3vBjzaYSjQ,4688
-oafuncs/_script/data_interp_geo.py,sha256=X89KxLYhpltWi0Sf96gIhBL3r1M5aExd_JCmgBmmvUc,3742
+oafuncs/_script/data_interp.py,sha256=EiZbt6n5BEaRKcng88UgX7TFPhKE6TLVZniS01awXjg,5146
+oafuncs/_script/data_interp_geo.py,sha256=ZRFb3fKRiYQViZNHd19eW20C9i38BsiIU8w0fG5mbqM,7789
 oafuncs/_script/email.py,sha256=lL4HGKrr524-g0xLlgs-4u7x4-u7DtgNoD9AL8XJKj4,3058
-oafuncs/_script/netcdf_merge.py,sha256=9hCyxfeUHnBzs50_0v0jzVfxpMxTX4dNTo0pmsp_T6g,4226
+oafuncs/_script/netcdf_merge.py,sha256=4mZLMcxBL4Rehi_eW2EX6vqbMJgZBOL4_ceaMzcuzio,5565
 oafuncs/_script/netcdf_modify.py,sha256=sGRUYNhfGgf9JV70rnBzw3bzuTRSXzBTL_RMDnDPeLQ,4552
-oafuncs/_script/netcdf_write.py,sha256=iO1Qv9bp6RLiw1D8Nrv7tX_8X-diUZaX3Nxhk6pJ5Nw,8556
-oafuncs/_script/parallel.py,sha256=T9Aie-e4LcbKlFTLZ0l4lhEN3SBVa84jRcrAsIm8s0I,8767
+oafuncs/_script/netcdf_write.py,sha256=GvyUyUhzMonzSp3y4pT8ZAfbQrsh5J3dLnmINYJKhuE,21422
+oafuncs/_script/parallel.py,sha256=07-BJVHxXJNlrOrhrSGt7qCZiKWq6dBvNDBA1AANYnI,8861
 oafuncs/_script/parallel_test.py,sha256=0GBqZOX7IaCOKF2t1y8N8YYu53GJ33OkfsWgpvZNqM4,372
 oafuncs/_script/plot_dataset.py,sha256=zkSEnO_-biyagorwWXPoihts_cwuvripzEt-l9bHJ2E,13989
 oafuncs/_script/replace_file_content.py,sha256=eCFZjnZcwyRvy6b4mmIfBna-kylSZTyJRfgXd6DdCjk,5982
 oafuncs/oa_down/User_Agent-list.txt,sha256=pHaMlElMvZ8TG4vf4BqkZYKqe0JIGkr4kCN0lM1Y9FQ,514295
 oafuncs/oa_down/__init__.py,sha256=kRX5eTUCbAiz3zTaQM1501paOYS_3fizDN4Pa0mtNUA,585
-oafuncs/oa_down/hycom_3hourly.py,sha256=VYxG9DPAnS1wk8gEXyqSIww4IvNRWOYGC4D6PTh5t3A,55450
+oafuncs/oa_down/hycom_3hourly.py,sha256=R5fKfIcpNRuaQgPiov_hJRd8voWgAHVLWifAMzR6RQI,55075
 oafuncs/oa_down/hycom_3hourly_proxy.py,sha256=1eaoJGI_m-7w4ZZ3n7NGxkZaeFdsm0d3U-hyw8RFNbc,54563
 oafuncs/oa_down/idm.py,sha256=4z5IvgfTyIKEI1kOtqXZwN7Jnfjwp6qDBOIoVyOLp0I,1823
 oafuncs/oa_down/literature.py,sha256=2bF9gSKQbzcci9LcKE81j8JEjIJwON7jbwQB3gDDA3E,11331
@@ -39,8 +39,8 @@ oafuncs/oa_sign/__init__.py,sha256=QKqTFrJDFK40C5uvk48GlRRbGFzO40rgkYwu6dYxatM,5
 oafuncs/oa_sign/meteorological.py,sha256=8091SHo2L8kl4dCFmmSH5NGVHDku5i5lSiLEG5DLnOQ,6489
 oafuncs/oa_sign/ocean.py,sha256=xrW-rWD7xBWsB5PuCyEwQ1Q_RDKq2KCLz-LOONHgldU,5932
 oafuncs/oa_sign/scientific.py,sha256=a4JxOBgm9vzNZKpJ_GQIQf7cokkraV5nh23HGbmTYKw,5064
-oafuncs-0.0.98.19.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
-oafuncs-0.0.98.19.dist-info/METADATA,sha256=PJ_1BA6QOeA2QHSb61jwOFA2pDD2H4QT--m3tf_f44I,4273
-oafuncs-0.0.98.19.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-oafuncs-0.0.98.19.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
-oafuncs-0.0.98.19.dist-info/RECORD,,
+oafuncs-0.0.98.21.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
+oafuncs-0.0.98.21.dist-info/METADATA,sha256=Lk9y2XVdDKb9HB5PP_Pm-_aII0tnGmaladuGW1Y9otM,4273
+oafuncs-0.0.98.21.dist-info/WHEEL,sha256=ooBFpIzZCPdw3uqIQsOo4qqbA4ZRPxHnOH7peeONza0,91
+oafuncs-0.0.98.21.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
+oafuncs-0.0.98.21.dist-info/RECORD,,

{oafuncs-0.0.98.19.dist-info → oafuncs-0.0.98.21.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.1)
+Generator: setuptools (80.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any