oafuncs 0.0.97.8__tar.gz → 0.0.97.10__tar.gz

{oafuncs-0.0.97.8/oafuncs.egg-info → oafuncs-0.0.97.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oafuncs
-Version: 0.0.97.8
+Version: 0.0.97.10
 Summary: Oceanic and Atmospheric Functions
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu

{oafuncs-0.0.97.8 → oafuncs-0.0.97.10}/oafuncs/_script/netcdf_merge.py

@@ -21,25 +21,25 @@ import numpy as np
 import xarray as xr
 from dask.diagnostics import ProgressBar
 
-# 配置日志
+# Configure logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
 def merge(file_list: Union[str, List[str]], var_name: Union[str, List[str], None] = None, dim_name: str = "time", target_filename: str = "merged.nc", chunk_config: Dict = {"time": 1000}, compression: Union[bool, Dict] = True, sanity_check: bool = True, overwrite: bool = True, parallel: bool = True) -> None:
     """
-    终极版NetCDF合并函数
+    Ultimate NetCDF merge function
 
     Parameters:
-        file_list: 文件路径列表或单个文件路径
-        var_name: 需要合并的变量（单个变量名/变量列表/None表示全部）
-        dim_name: 合并维度，默认为'time'
-        target_filename: 输出文件路径
-        chunk_config: Dask分块配置，如{"time": 1000}
-        compression: 压缩配置（True启用默认压缩，或自定义编码字典）
-        sanity_check: 是否执行数据完整性校验
-        overwrite: 是否覆盖已存在文件
-        parallel: 是否启用并行处理
+        file_list: List of file paths or single file path
+        var_name: Variables to merge (single variable name/list of variables/None means all)
+        dim_name: Dimension to merge along, default is 'time'
+        target_filename: Output file path
+        chunk_config: Dask chunking configuration, e.g. {"time": 1000}
+        compression: Compression configuration (True enables default compression, or custom encoding dictionary)
+        sanity_check: Whether to perform data integrity validation
+        overwrite: Whether to overwrite existing files
+        parallel: Whether to enable parallel processing
 
     Example:
         merge(["data1.nc", "data2.nc"],
@@ -47,21 +47,21 @@ def merge(file_list: Union[str, List[str]], var_name: Union[str, List[str], None
              target_filename="result.nc",
              chunk_config={"time": 500})
     """
-    # ------------------------ 参数预处理 ------------------------#
+    # ------------------------ Parameter preprocessing ------------------------#
     file_list = _validate_and_preprocess_inputs(file_list, target_filename, overwrite)
     all_vars, var_names = _determine_variables(file_list, var_name)
     static_vars = _identify_static_vars(file_list[0], var_names, dim_name)
 
-    # 估计处理所需的内存
+    # Estimate required memory for processing
     _estimate_memory_usage(file_list, var_names, chunk_config)
 
-    # ------------------------ 数据校验阶段 ------------------------#
+    # ------------------------ Data validation phase ------------------------#
     if sanity_check:
         _perform_sanity_checks(file_list, var_names, dim_name, static_vars)
 
-    # ------------------------ 核心合并逻辑 ------------------------#
-    with xr.set_options(keep_attrs=True):  # 保留元数据属性
-        # 动态变量合并
+    # ------------------------ Core merging logic ------------------------#
+    with xr.set_options(keep_attrs=True):  # Preserve metadata attributes
+        # Merge dynamic variables
         merged_ds = xr.open_mfdataset(
             file_list,
             combine="nested",
@@ -69,49 +69,49 @@ def merge(file_list: Union[str, List[str]], var_name: Union[str, List[str], None
             data_vars=[var for var in var_names if var not in static_vars],
             chunks=chunk_config,
             parallel=parallel,
-            preprocess=lambda ds: ds[var_names],  # 仅加载目标变量
+            preprocess=lambda ds: ds[var_names],  # Only load target variables
         )
 
-        # 静态变量处理
+        # Process static variables
         if static_vars:
             with xr.open_dataset(file_list[0], chunks=chunk_config) as ref_ds:
                 merged_ds = merged_ds.assign({var: ref_ds[var] for var in static_vars})
 
-    # ------------------------ 时间维度处理 ------------------------#
+    # ------------------------ Time dimension processing ------------------------#
     if dim_name == "time":
         merged_ds = _process_time_dimension(merged_ds)
 
-    # ------------------------ 文件输出 ------------------------#
+    # ------------------------ File output ------------------------#
     encoding = _generate_encoding_config(merged_ds, compression)
     _write_to_netcdf(merged_ds, target_filename, encoding)
 
 
-# ------------------------ 辅助函数 ------------------------#
+# ------------------------ Helper functions ------------------------#
 def _validate_and_preprocess_inputs(file_list: Union[str, List[str]], target_filename: str, overwrite: bool) -> List[str]:
-    """输入参数校验与预处理"""
+    """Input parameter validation and preprocessing"""
     if not file_list:
-        raise ValueError("文件列表不能为空")
+        raise ValueError("File list cannot be empty")
 
     file_list = [file_list] if isinstance(file_list, str) else file_list
     for f in file_list:
         if not os.path.exists(f):
-            raise FileNotFoundError(f"输入文件不存在: {f}")
+            raise FileNotFoundError(f"Input file does not exist: {f}")
 
     target_dir = os.path.dirname(os.path.abspath(target_filename))
     os.makedirs(target_dir, exist_ok=True)
 
     if os.path.exists(target_filename):
         if overwrite:
-            logger.warning(f"覆盖已存在文件: {target_filename}")
+            logger.warning(f"Overwriting existing file: {target_filename}")
             os.remove(target_filename)
         else:
-            raise FileExistsError(f"目标文件已存在: {target_filename}")
+            raise FileExistsError(f"Target file already exists: {target_filename}")
 
     return file_list
 
 
 def _determine_variables(file_list: List[str], var_name: Union[str, List[str], None]) -> tuple:
-    """确定需要处理的变量列表"""
+    """Determine the list of variables to process"""
     with xr.open_dataset(file_list[0]) as ds:
         all_vars = list(ds.data_vars.keys())
 
@@ -119,40 +119,40 @@ def _determine_variables(file_list: List[str], var_name: Union[str, List[str], N
         return all_vars, all_vars
     elif isinstance(var_name, str):
         if var_name not in all_vars:
-            raise ValueError(f"无效变量名: {var_name}")
+            raise ValueError(f"Invalid variable name: {var_name}")
         return all_vars, [var_name]
     elif isinstance(var_name, list):
-        if not var_name:  # 处理空列表情况
-            logger.warning("提供了空的变量列表，将使用所有变量")
+        if not var_name:  # Handle empty list case
+            logger.warning("Empty variable list provided, will use all variables")
             return all_vars, all_vars
         invalid_vars = set(var_name) - set(all_vars)
         if invalid_vars:
-            raise ValueError(f"无效变量名: {invalid_vars}")
+            raise ValueError(f"Invalid variable names: {invalid_vars}")
         return all_vars, var_name
     else:
-        raise TypeError("var_name参数类型必须是str/list/None")
+        raise TypeError("var_name parameter must be of type str/list/None")
 
 
 def _identify_static_vars(sample_file: str, var_names: List[str], dim_name: str) -> List[str]:
-    """识别静态变量"""
+    """Identify static variables"""
     with xr.open_dataset(sample_file) as ds:
         return [var for var in var_names if dim_name not in ds[var].dims]
 
 
 def _perform_sanity_checks(file_list: List[str], var_names: List[str], dim_name: str, static_vars: List[str]) -> None:
-    """执行数据完整性校验"""
-    logger.info("正在执行数据完整性校验...")
+    """Perform data integrity validation"""
+    logger.info("Performing data integrity validation...")
 
-    # 静态变量一致性检查
+    # Check consistency of static variables
     with xr.open_dataset(file_list[0]) as ref_ds:
         for var in static_vars:
             ref = ref_ds[var]
             for f in file_list[1:]:
                 with xr.open_dataset(f) as ds:
                     if not ref.equals(ds[var]):
-                        raise ValueError(f"静态变量 {var} 不一致\n参考文件: {file_list[0]}\n问题文件: {f}")
+                        raise ValueError(f"Static variable {var} inconsistent\nReference file: {file_list[0]}\nProblem file: {f}")
 
-    # 动态变量维度检查
+    # Check dimensions of dynamic variables
     dim_sizes = {}
     for f in file_list:
         with xr.open_dataset(f) as ds:
@@ -160,53 +160,67 @@ def _perform_sanity_checks(file_list: List[str], var_names: List[str], dim_name:
                 if var not in static_vars:
                     dims = ds[var].dims
                     if dim_name not in dims:
-                        raise ValueError(f"变量 {var} 在文件 {f} 中缺少合并维度 {dim_name}")
+                        raise ValueError(f"Variable {var} in file {f} missing merge dimension {dim_name}")
                     dim_sizes.setdefault(var, []).append(ds[var].sizes[dim_name])
 
-    # 检查维度连续性
+    # Check dimension continuity
     for var, sizes in dim_sizes.items():
         if len(set(sizes[1:])) > 1:
-            raise ValueError(f"变量 {var} 的 {dim_name} 维度长度不一致: {sizes}")
+            raise ValueError(f"Variable {var} has inconsistent {dim_name} dimension lengths: {sizes}")
 
 
 def _process_time_dimension(ds: xr.Dataset) -> xr.Dataset:
-    """时间维度特殊处理"""
+    """Special processing for time dimension"""
     if "time" not in ds.dims:
         return ds
 
-    # 排序并去重
+    # Sort and deduplicate
     ds = ds.sortby("time")
-    # 找到唯一时间戳的索引
+    # Find indices of unique timestamps
     _, index = np.unique(ds["time"], return_index=True)
-    # 无需再次排序索引，因为我们需要保持时间的原始顺序
+    # No need to sort indices again as we want to keep original time order
     return ds.isel(time=index)
 
 
 def _generate_encoding_config(ds: xr.Dataset, compression: Union[bool, Dict]) -> Dict:
-    """生成压缩编码配置"""
+    """Generate compression encoding configuration"""
     if not compression:
         return {}
 
-    # 默认压缩设置基础
+    # Default compression settings base
     def _get_default_encoding(var):
         return {"zlib": True, "complevel": 3, "dtype": "float32" if ds[var].dtype == "float64" else ds[var].dtype}
 
-    # 处理自定义压缩配置
+    # Handle custom compression configuration
     encoding = {}
     if isinstance(compression, dict):
         for var in ds.data_vars:
             encoding[var] = _get_default_encoding(var)
-            encoding[var].update(compression.get(var, {}))  # 使用 dict.update() 合并字典
+            encoding[var].update(compression.get(var, {}))  # Use dict.update() to merge dictionaries
     else:
         for var in ds.data_vars:
             encoding[var] = _get_default_encoding(var)
 
     return encoding
 
+def _calculate_file_size(filepath: str) -> str:
+    """Calculate file size with adaptive unit conversion"""
+    if os.path.exists(filepath):
+        size_in_bytes = os.path.getsize(filepath)
+        if size_in_bytes < 1e3:
+            return f"{size_in_bytes:.2f} B"
+        elif size_in_bytes < 1e6:
+            return f"{size_in_bytes / 1e3:.2f} KB"
+        elif size_in_bytes < 1e9:
+            return f"{size_in_bytes / 1e6:.2f} MB"
+        else:
+            return f"{size_in_bytes / 1e9:.2f} GB"
+    else:
+        raise FileNotFoundError(f"File not found: {filepath}")
 
 def _write_to_netcdf(ds: xr.Dataset, filename: str, encoding: Dict) -> None:
-    """改进后的安全写入NetCDF文件"""
-    logger.info("开始写入文件...")
+    """Improved safe writing to NetCDF file"""
+    logger.info("Starting file write...")
     unlimited_dims = [dim for dim in ds.dims if ds[dim].encoding.get("unlimited", False)]
 
     delayed = ds.to_netcdf(filename, encoding=encoding, compute=False, unlimited_dims=unlimited_dims)
@@ -215,16 +229,17 @@ def _write_to_netcdf(ds: xr.Dataset, filename: str, encoding: Dict) -> None:
         with ProgressBar():
             delayed.compute()
 
-        logger.info(f"合并完成 → {filename}")
-        logger.info(f"文件大小: {os.path.getsize(filename) / 1e9:.2f}GB")
+        logger.info(f"Merge completed → {filename}")
+        # logger.info(f"File size: {os.path.getsize(filename) / 1e9:.2f}GB")
+        logger.info(f"File size: {_calculate_file_size(filename)}")
     except MemoryError as e:
-        _handle_write_error(filename, "内存不足，无法完成文件写入。请尝试调整chunk_config参数减少内存使用", e)
+        _handle_write_error(filename, "Insufficient memory to complete file write. Try adjusting chunk_config parameter to reduce memory usage", e)
     except Exception as e:
-        _handle_write_error(filename, f"写入文件失败: {str(e)}", e)
+        _handle_write_error(filename, f"Failed to write file: {str(e)}", e)
 
 
 def _handle_write_error(filename: str, message: str, exception: Exception) -> None:
-    """统一处理写入文件的异常"""
+    """Unified handling of file write exceptions"""
     logger.error(message)
     if os.path.exists(filename):
         os.remove(filename)
@@ -232,24 +247,24 @@ def _handle_write_error(filename: str, message: str, exception: Exception) -> No
 
 
 def _estimate_memory_usage(file_list: List[str], var_names: List[str], chunk_config: Dict) -> None:
-    """改进内存使用量估算"""
+    """Improved memory usage estimation"""
     try:
         total_size = 0
         sample_file = file_list[0]
         with xr.open_dataset(sample_file) as ds:
             for var in var_names:
                 if var in ds:
-                    # 考虑变量的维度大小
+                    # Consider variable dimension sizes
                     var_size = np.prod([ds[var].sizes[dim] for dim in ds[var].dims]) * ds[var].dtype.itemsize
                     total_size += var_size * len(file_list)
 
-        # 估算Dask处理时的内存使用量 (通常是原始数据的2-3倍)
+        # Estimate memory usage during Dask processing (typically 2-3x original data)
         estimated_memory = total_size * 3
 
         if estimated_memory > 8e9:
-            logger.warning(f"预计内存使用可能较高 (约 {estimated_memory / 1e9:.1f}GB)。如果遇到内存问题，请调整chunk_config参数: {chunk_config}")
+            logger.warning(f"Estimated memory usage may be high (approx. {estimated_memory / 1e9:.1f}GB). If memory issues occur, adjust chunk_config parameter: {chunk_config}")
     except Exception as e:
-        logger.debug(f"内存估计失败: {str(e)}")
+        logger.debug(f"Memory estimation failed: {str(e)}")
 
 
 if __name__ == "__main__":

{oafuncs-0.0.97.8 → oafuncs-0.0.97.10}/oafuncs/_script/plot_dataset.py

@@ -1,18 +1,17 @@
 import os
-from typing import Optional, Tuple
+from typing import Optional, Tuple, Union
 
 import matplotlib as mpl
 
 mpl.use("Agg")  # Use non-interactive backend
 
+import cartopy.crs as ccrs
 import cftime
 import matplotlib.pyplot as plt
 import numpy as np
-from rich import print
-import cartopy.crs as ccrs
-import xarray as xr
-
 import oafuncs
+import xarray as xr
+from rich import print
 
 
 def plot_1d(data: xr.DataArray, output_path: str, x_dim: str, y_dim: str, z_dim: str, t_dim: str) -> None:
@@ -264,33 +263,48 @@ def process_variable(var: str, data: xr.DataArray, dims: int, dims_name: Tuple[s
                             print(f"Error processing {var}_{dims_name[0]}-{i}_{dims_name[1]}-{j}: {e}")
 
 
-def func_plot_dataset(ds_in: xr.Dataset, output_dir: str, xyzt_dims: Tuple[str, str, str, str] = ("longitude", "latitude", "level", "time"), plot_type: str = "contourf", fixed_colorscale: bool = False) -> None:
+def func_plot_dataset(ds_in: Union[xr.Dataset, xr.DataArray], output_dir: str, xyzt_dims: Tuple[str, str, str, str] = ("longitude", "latitude", "level", "time"), plot_type: str = "contourf", fixed_colorscale: bool = False) -> None:
     """Plot variables from a NetCDF file and save the plots to the specified directory."""
     os.makedirs(output_dir, exist_ok=True)
     x_dim, y_dim, z_dim, t_dim = xyzt_dims
 
     # Main processing function
     try:
-        ds = ds_in
-        varlist = list(ds.data_vars)
-        print(f"Found {len(varlist)} variables in dataset")
-
-        for var in varlist:
+        # 检查输入是 DataArray 还是 Dataset
+        if isinstance(ds_in, xr.DataArray):
+            # 处理单个 DataArray
+            print("Processing a single DataArray")
+            var = ds_in.name if ds_in.name is not None else "unnamed_variable"
             print("=" * 120)
             print(f"Processing: {var}")
-            data = ds[var]
-            dims = len(data.shape)
-            dims_name = data.dims
             try:
-                process_variable(var, data, dims, dims_name, output_dir, x_dim, y_dim, z_dim, t_dim, fixed_colorscale, plot_type)
+                dims = len(ds_in.shape)
+                dims_name = ds_in.dims
+                process_variable(var, ds_in, dims, dims_name, output_dir, x_dim, y_dim, z_dim, t_dim, fixed_colorscale, plot_type)
             except Exception as e:
                 print(f"Error processing variable {var}: {e}")
+        else:
+            # 处理包含多个变量的 Dataset
+            ds = ds_in
+            varlist = list(ds.data_vars)
+            print(f"Found {len(varlist)} variables in dataset")
+
+            for var in varlist:
+                print("=" * 120)
+                print(f"Processing: {var}")
+                data = ds[var]
+                dims = len(data.shape)
+                dims_name = data.dims
+                try:
+                    process_variable(var, data, dims, dims_name, output_dir, x_dim, y_dim, z_dim, t_dim, fixed_colorscale, plot_type)
+                except Exception as e:
+                    print(f"Error processing variable {var}: {e}")
 
     except Exception as e:
         print(f"Error processing dataset: {e}")
     finally:
-        if "ds" in locals():
-            ds.close()
+        if isinstance(ds_in, xr.Dataset) and "ds_in" in locals():
+            ds_in.close()
             print("Dataset closed")
 
 

{oafuncs-0.0.97.8 → oafuncs-0.0.97.10}/oafuncs/oa_down/hycom_3hourly.py

@@ -1000,7 +1000,7 @@ def _done_callback(future, progress, task, total, counter_lock):
         progress.update(task, advance=1, description=f"[cyan]Downloading... {parallel_counter}/{total}")
 
 
-def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False, ftimes=1):
+def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False, ftimes=1, download_interval=3):
     """
     Description:
     Download the data of single time or a series of time
@@ -1032,7 +1032,7 @@ def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_mi
         _prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, None, depth, level, store_path, dataset_name, version_name, check)
     elif int(ymdh_time_s) < int(ymdh_time_e):
         print("Downloading a series of files...")
-        time_list = get_time_list(ymdh_time_s, ymdh_time_e, 3, "hour")
+        time_list = get_time_list(ymdh_time_s, ymdh_time_e, int(download_interval), "hour")
         with Progress() as progress:
             task = progress.add_task("[cyan]Downloading...", total=len(time_list))
             if ftimes == 1:
@@ -1071,7 +1071,7 @@ def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_mi
         print("[bold red]Please ensure the time_s is no more than time_e")
 
 
-def download(var, time_s, time_e=None, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False, ftimes=1, idm_engine=None, fill_time=None):
+def download(var, time_s, time_e=None, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False, ftimes=1, idm_engine=None, fill_time=None, download_interval_hour=3):
     """
     Description:
         Download the data of single time or a series of time
@@ -1094,6 +1094,7 @@ def download(var, time_s, time_e=None, lon_min=0, lon_max=359.92, lat_min=-80, l
         ftimes: int, the number of time in one file, default is 1, if set to 1, the data of single time will be downloaded; the maximum is 8, if set to 8, the data of 8 times will be downloaded in one file
         idm_engine: str, the IDM engine, default is None, if set, the IDM will be used to download the data; example: "D:\\Programs\\Internet Download Manager\\IDMan.exe"
         fill_time: bool or None, the mode to fill the time, default is None. None: only download the data; True: modify the real time of data to the time in the file name; False: check the time in the file name and the real time of data, if not match, delete the file
+        download_interval_hour: int, the interval time to download the data, default is 3, if set, the interval time will be used to download the data; example: 3, 6, ...
 
     Returns:
         None
@@ -1180,7 +1181,7 @@ def download(var, time_s, time_e=None, lon_min=0, lon_max=359.92, lat_min=-80, l
     global match_time
     match_time = fill_time
 
-    _download_hourly_func(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check, ftimes)
+    _download_hourly_func(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check, ftimes, download_interval_hour)
 
     if idm_engine is not None:
         if idm_download_list:

{oafuncs-0.0.97.8 → oafuncs-0.0.97.10}/oafuncs/oa_nc.py

@@ -203,7 +203,7 @@ def merge(file_list: Union[str, List[str]], var_name: Union[str, List[str], None
              target_filename="result.nc",
              chunk_config={"time": 500})
     """
-    from _script.netcdf_merge import merge as nc_merge
+    from ._script.netcdf_merge import merge as nc_merge
     nc_merge(file_list, var_name, dim_name, target_filename, chunk_config, compression, sanity_check, overwrite, parallel)
 
 

{oafuncs-0.0.97.8 → oafuncs-0.0.97.10/oafuncs.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oafuncs
-Version: 0.0.97.8
+Version: 0.0.97.10
 Summary: Oceanic and Atmospheric Functions
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu

{oafuncs-0.0.97.8 → oafuncs-0.0.97.10}/setup.py

@@ -18,7 +18,7 @@ URL = "https://github.com/Industry-Pays/OAFuncs"
 EMAIL = "liukun0312@stu.ouc.edu.cn"
 AUTHOR = "Kun Liu"
 REQUIRES_PYTHON = ">=3.9.0"  # 2025/03/13
-VERSION = "0.0.97.8"  # 下次用98.0，98.1已经被用过了
+VERSION = "0.0.97.10"  # 下次用98.0，98.1已经被用过了
 
 # What packages are required for this module to be executed?
 REQUIRED = [