climarraykit 0.2.0__py3-none-any.whl

climarraykit/__init__.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "conversions",
+    "data_manipulation",
+    "file_utils",
+    "patterns",
+    "xarray_obj_handler",
+]

climarraykit/conversions.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+#----------------#
+# Import modules #
+#----------------#
+
+import xarray as xr
+from pathlib import Path
+
+#------------------------#
+# Import project modules #
+#------------------------#
+
+from climarraykit.xarray_obj_handler import _save_ds_as_nc
+from paramlib.global_parameters import CLIMATE_FILE_EXTENSIONS
+from pygenutils.arrays_and_lists.conversions import flatten_to_string
+from pygenutils.arrays_and_lists.data_manipulation import flatten_list
+from pygenutils.operative_systems.os_operations import exit_info, run_system_command
+from pygenutils.strings.string_handler import (
+    find_substring_index,
+    get_obj_specs,
+    modify_obj_specs
+)
+
+#------------------#
+# Define functions #
+#------------------#
+
+# Xarray objects #
+#----------------#
+
+def grib2nc(
+        grib_file_list: str | list[str], 
+        on_shell: bool = False, 
+        option_str: str | None = None,
+        capture_output: bool = False,
+        return_output_name: bool = False,
+        encoding: str = "utf-8",
+        shell: bool = True) -> None:
+    """
+    Converts a GRIB file or list of GRIB files to netCDF format. The conversion
+    can be executed either via shell commands or programmatically using xarray.
+    
+    Parameters
+    ----------
+    grib_file_list : str | list[str]
+        The file path(s) of the GRIB file(s) to be converted.
+    on_shell : bool, optional
+        If True, the conversion will be handled through shell commands using 
+        the 'grib_to_netcdf' tool. If False, the conversion will be done 
+        programmatically using xarray.
+    option_str : str, optional
+        Additional options to pass to the shell command for 'grib_to_netcdf'. 
+        This parameter is only used if 'on_shell' is set to True.
+    capture_output : bool, optional
+        Whether to capture the command output. Default is False.
+    return_output_name : bool, optional
+        Whether to return file descriptor names. Default is False.
+    encoding : str, optional
+        Encoding to use when decoding command output. Default is "utf-8".
+    shell : bool, optional
+        Whether to execute the command through the shell. Default is True.
+        
+    Returns
+    -------
+    None
+        Converts the GRIB file(s) to netCDF format and saves the output 
+        netCDF file(s) in the same directory as the GRIB files.
+
+    Raises
+    ------
+    TypeError
+        If grib_file_list is not str or list of str.
+    ValueError
+        If any GRIB file path is invalid or empty.
+    FileNotFoundError
+        If any GRIB file doesn't exist.
+
+    Notes
+    -----
+    - When 'on_shell' is True, the function builds and runs a shell command 
+      that calls the 'grib_to_netcdf' tool, with optional flags.
+    - When 'on_shell' is False, xarray is used to directly open the GRIB file 
+      and convert it to netCDF format.
+    - The function will prompt for input in the case of multiple GRIB files if 
+      'on_shell' is True.
+    """
+    
+    # Parameter validation
+    if not isinstance(grib_file_list, (str, list)):
+        raise TypeError("grib_file_list must be a string or list of strings")
+    
+    # Flatten nested lists for defensive programming
+    if isinstance(grib_file_list, list):
+        grib_file_list = flatten_list(grib_file_list)
+        
+        # Validate all items are strings
+        if not all(isinstance(item, str) for item in grib_file_list):
+            raise TypeError("All items in grib_file_list must be strings")
+        
+        # Check for empty strings
+        if not all(item.strip() for item in grib_file_list):
+            raise ValueError("All GRIB file paths must be non-empty strings")
+    else:
+        # Single string validation
+        if not isinstance(grib_file_list, str) or not grib_file_list.strip():
+            raise ValueError("GRIB file path must be a non-empty string")
+    
+    # Check file existence
+    files_to_check = [grib_file_list] if isinstance(grib_file_list, str) else grib_file_list
+    for grib_file in files_to_check:
+        if not Path(grib_file).exists():
+            raise FileNotFoundError(f"GRIB file not found: {grib_file}")
+        
+        # Check if file has expected GRIB extension
+        if not any(grib_file.lower().endswith(ext.lower()) for ext in ['.grib', '.grb', '.grib2', '.grb2']):
+            print(f"Warning: File {grib_file} may not be a GRIB file based on extension")
+
+    # Shell-based conversion #
+    #-#-#-#-#-#-#-#-#-#-#-#-#
+    
+    if on_shell:
+        # Handle single GRIB file
+        if isinstance(grib_file_list, str):
+            nc_file_new = modify_obj_specs(grib_file_list, "ext", EXTENSIONS[0])
+        
+        # Handle list of GRIB files
+        else:
+            grib_allfile_info_str = flatten_to_string(grib_file_list)
+            
+            # Prompt user for the netCDF file name without extension
+            nc_file_new_noext = input("Please introduce a name "
+                                      "for the netCDF file, "
+                                      "WITHOUT THE EXTENSION: ")
+            
+            # Validate the file name using RegEx
+            allowed_minimum_char_idx = find_substring_index(nc_file_new_noext,
+                                                            REGEX_GRIB2NC,
+                                                            advanced_search=True)
+            
+            while allowed_minimum_char_idx == -1:
+                print("Invalid file name.\nIt can contain alphanumeric characters, "
+                      "as well as the following non-word characters: {. _ -}")
+                nc_file_new_noext = input("Please introduce a valid name: ")
+                allowed_minimum_char_idx = find_substring_index(nc_file_new_noext,
+                                                                REGEX_GRIB2NC,
+                                                                advanced_search=True)
+            
+            # Modify the file name to have the .nc extension
+            nc_file_new = modify_obj_specs(nc_file_new_noext,
+                                                 obj2modify="ext",
+                                                 new_obj=EXTENSIONS[0])
+        
+        # Construct the shell command for conversion
+        grib2nc_template = "grib_to_netcdf "
+        if option_str:
+            grib2nc_template += f"{option_str} "
+        grib2nc_template += f"-o {nc_file_new} {grib_allfile_info_str}"
+        
+        # Execute the shell command
+        try:
+            process_exit_info = run_system_command(
+                grib2nc_template,
+                capture_output=capture_output,
+                return_output_name=return_output_name,
+                encoding=encoding,
+                shell=shell
+            )
+            # Call exit_info with parameters based on capture_output
+            exit_info(
+                process_exit_info,
+                check_stdout=True,
+                check_stderr=True,
+                check_return_code=True
+            )
+        except Exception as e:
+            raise RuntimeError(f"Shell command execution failed: {e}")
+
+    # Programmatic conversion #
+    #-#-#-#-#-#-#-#-#-#-#-#-#-#
+    
+    else:
+        # Ensure grib_file_list is a list
+        if isinstance(grib_file_list, str):
+            grib_file_list = [grib_file_list]
+
+        # Convert each GRIB file in the list to netCDF
+        for grib_file in grib_file_list:
+            try:
+                grib_file_noext = get_obj_specs(grib_file, "name_noext", EXTENSIONS[0])
+                ds = xr.open_dataset(grib_file, engine="cfgrib")
+                _save_ds_as_nc(ds, grib_file_noext)
+                print(f"Successfully converted {grib_file} to netCDF format")
+            except Exception as e:
+                print(f"Error converting {grib_file}: {e}")
+                raise
+
+            
+#--------------------------#
+# Parameters and constants #
+#--------------------------#
+
+# Valid file extensions #
+EXTENSIONS = CLIMATE_FILE_EXTENSIONS[::3]
+  
+# RegEx control for GRIB-to-netCDF single file name #
+REGEX_GRIB2NC = r"^[a-zA-Z\d\._-]$"

climarraykit/data_manipulation.py

@@ -0,0 +1,386 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+#----------------#
+# Import modules #
+#----------------#
+
+import os
+from pathlib import Path
+
+#------------------------#
+# Import project modules #
+#------------------------#
+
+from filewise.file_operations.ops_handler import move_files
+from filewise.file_operations.path_utils import find_dirs_with_files, find_files
+from climarraykit.file_utils import ncfile_integrity_status
+from climarraykit.patterns import (
+    find_coordinate_variables,
+    get_latlon_bounds,
+    get_latlon_deltas,
+    get_times
+)
+from paramlib.global_parameters import CLIMATE_FILE_EXTENSIONS
+from pygenutils.strings.text_formatters import format_string, string_underliner
+from pygenutils.time_handling.date_and_time_utils import find_dt_key
+
+#-------------------------#
+# Define custom functions #
+#-------------------------#
+
+# Data extractors #
+#-----------------#
+
+def extract_latlon_bounds(delta_decimal_places: int, value_decimal_places: int) -> None:
+    """
+    Extract latitude and longitude bounds from netCDF files.
+
+    Parameters
+    ----------
+    delta_decimal_places : int
+        Number of decimal places to round off the delta between latitude and longitude points.
+    value_decimal_places : int
+        Number of decimal places to round off the latitude and longitude values.
+
+    Returns
+    -------
+    None
+
+    Raises
+    ------
+    TypeError
+        If decimal places parameters are not integers.
+    ValueError
+        If decimal places parameters are negative.
+
+    Notes
+    -----
+    - The extracted latitude and longitude arrays, their dimensions,
+      and deltas are saved in a report file.
+    - If any files are faulty or cannot be processed, relevant error information 
+      is recorded in the report.
+    """
+    # Parameter validation
+    if not isinstance(delta_decimal_places, int):
+        raise TypeError("delta_decimal_places must be an integer")
+    
+    if not isinstance(value_decimal_places, int):
+        raise TypeError("value_decimal_places must be an integer")
+    
+    if delta_decimal_places < 0:
+        raise ValueError("delta_decimal_places must be non-negative")
+    
+    if value_decimal_places < 0:
+        raise ValueError("value_decimal_places must be non-negative")
+    
+    nc_dirs = find_dirs_with_files(EXTENSIONS[0], search_path=CODE_CALL_DIR)
+    
+    for dir_num, dir_name in enumerate(nc_dirs, start=1):
+        nc_files = find_files(EXTENSIONS[0], dir_name, match_type="ext", top_path_only=True)
+        
+        with open(COORD_INFO_FNAME, "w") as report:
+            if nc_files:
+                for file_num, nc_file in enumerate(nc_files, start=1):
+                    print(f"Processing file {file_num} out of {len(nc_files)} "
+                          f"in directory {dir_num} out of {len(nc_dirs)}...")
+                    report.write(format_string(string_underliner(DIR_INFO_TEMPLATE, dir_name), "+"))
+                    
+                    try:
+                        ncfile_integrity_status(nc_file)
+                    except Exception as ncf_err:
+                        report.write(f"FAULTY FILE '{nc_file}': {ncf_err}\n")
+                    else:
+                        try:
+                            coord_vars = find_coordinate_variables(nc_file)
+                        except Exception as coord_err:
+                            report.write(f"ERROR IN FILE '{nc_file}': {coord_err}\n")
+                        else:
+                            try:
+                                lats, lons = get_latlon_bounds(nc_file, coord_vars[0], coord_vars[1], value_decimal_places)
+                                lat_delta, lon_delta = get_latlon_deltas(lats, lons, delta_decimal_places)
+                                
+                                format_args_latlon_bounds = (
+                                    nc_file, 
+                                    lats,
+                                    lons, 
+                                    len(lats), 
+                                    len(lons), 
+                                    lat_delta, 
+                                    lon_delta
+                                    )
+                                
+                                report.write(format_string(LATLON_INFO_TEMPLATE, format_args_latlon_bounds))
+                            except Exception as e:
+                                report.write(f"ERROR PROCESSING COORDINATES IN FILE '{nc_file}': {e}\n")
+                            
+                move_files(COORD_INFO_FNAME,
+                           input_directories=".", 
+                           destination_directories=dir_name, 
+                           match_type="glob")
+            else:
+                report.write(f"No netCDF files in directory {dir_name}\n")
+                move_files(COORD_INFO_FNAME,
+                           input_directories=".", 
+                           destination_directories=dir_name, 
+                           match_type="glob")
+
+
+def extract_time_bounds() -> None:
+    """
+    Extract the time bounds (start and end times) from netCDF files.
+
+    Parameters
+    ----------
+    None
+
+    Returns
+    -------
+    None
+
+    Notes
+    -----
+    - The time range (start and end times) and the total number of time records 
+      are saved in a report file.
+    - If any files are faulty or cannot be processed, relevant error information 
+      is recorded in the report.
+    """
+    nc_dirs = find_dirs_with_files(EXTENSIONS[0], search_path=CODE_CALL_DIR)
+    
+    for dir_num, dir_name in enumerate(nc_dirs, start=1):
+        nc_files = find_files(EXTENSIONS[0], dir_name, match_type="ext", top_path_only=True)
+        
+        with open(DATE_RANGE_INFO_FNAME, "w") as report:
+            if nc_files:
+                for file_num, nc_file in enumerate(nc_files, start=1):
+                    print(f"Processing file {file_num} out of {len(nc_files)} "
+                          f"in directory {dir_num} out of {len(nc_dirs)}...")
+                    report.write(format_string(string_underliner(DIR_INFO_TEMPLATE, dir_name), "+"))
+                    
+                    try:
+                        ncfile_integrity_status(nc_file)
+                    except Exception as ncf_err:
+                        report.write(f"FAULTY FILE '{nc_file}': {ncf_err}\n")
+                    else:
+                        try:
+                            time_var = find_dt_key(nc_file)
+                        except Exception as time_err:
+                            report.write(f"ERROR IN FILE '{nc_file}': {time_err}\n")
+                        else:
+                            try:
+                                times = get_times(nc_file, time_var)                            
+                                format_args_time_periods = (
+                                    nc_file, 
+                                    times[0].values,
+                                    times[-1].values, 
+                                    len(times)
+                                    )
+                                
+                                report.write(format_string(PERIOD_INFO_TEMPLATE, format_args_time_periods))
+                            except Exception as e:
+                                report.write(f"ERROR PROCESSING TIME DATA IN FILE '{nc_file}': {e}\n")
+                            
+                move_files(DATE_RANGE_INFO_FNAME,
+                           input_directories=".", 
+                           destination_directories=dir_name, 
+                           match_type="glob")
+            else:
+                report.write(f"No netCDF files in directory {dir_name}\n")
+                move_files(DATE_RANGE_INFO_FNAME,
+                           input_directories=".", 
+                           destination_directories=dir_name, 
+                           match_type="glob")
+
+
+def extract_time_formats() -> None:
+    """
+    Extract the time formats from netCDF files.
+
+    Parameters
+    ----------
+    None
+
+    Returns
+    -------
+    None
+
+    Notes
+    -----
+    - The extracted time formats and the total number of time records are saved 
+      in a report file.
+    - If any files are faulty or cannot be processed, relevant error information 
+      is recorded in the report.
+    """
+
+    nc_dirs = find_dirs_with_files(EXTENSIONS[0], search_path=CODE_CALL_DIR)
+    
+    for dir_num, dir_name in enumerate(nc_dirs, start=1):
+        nc_files = find_files(EXTENSIONS[0], dir_name, match_type="ext", top_path_only=True)
+        
+        with open(TIME_FORMATS_FILE_NAME, "w") as report:
+            if nc_files:
+                for file_num, nc_file in enumerate(nc_files, start=1):
+                    print(f"Processing file {file_num} out of {len(nc_files)} "
+                          f"in directory {dir_num} out of {len(nc_dirs)}...")
+                    report.write(format_string(string_underliner(DIR_INFO_TEMPLATE, dir_name), "+"))
+                    
+                    try:
+                        ncfile_integrity_status(nc_file)
+                    except Exception as ncf_err:
+                        report.write(f"FAULTY FILE '{nc_file}': {ncf_err}\n")
+                    else:
+                        try:
+                            time_var = find_dt_key(nc_file)
+                        except Exception as time_err:
+                            report.write(f"ERROR IN FILE '{nc_file}': {time_err}\n")
+                        else:
+                            try:
+                                times = get_times(nc_file, time_var)
+                                format_args_time_formats = (
+                                    nc_file, 
+                                    times.values, 
+                                    len(times)
+                                    )
+                                report.write(format_string(TIME_FORMAT_INFO_TEMPLATE, format_args_time_formats))
+                            except Exception as e:
+                                report.write(f"ERROR PROCESSING TIME FORMATS IN FILE '{nc_file}': {e}\n")
+                                
+                move_files(TIME_FORMATS_FILE_NAME,
+                           input_directories=".", 
+                           destination_directories=dir_name, 
+                           match_type="glob")
+            else:
+                report.write(f"No netCDF files in directory {dir_name}\n")
+                move_files(TIME_FORMATS_FILE_NAME,
+                           input_directories=".", 
+                           destination_directories=dir_name, 
+                           match_type="glob")
+            
+# File regridding #
+#-----------------#
+
+def netcdf_regridder(ds_in: 'xr.Dataset', ds_image: 'xr.Dataset', regrid_method: str = "bilinear") -> 'xr.Dataset':    
+    """
+    Function that regrids a xarray Dataset to that of the desired Dataset. 
+    It is similar to CDO but more intuitive and
+    easier to understand, supported by Python.
+    
+    Parameters
+    ----------
+    ds_in : xarray.Dataset
+        Input xarray data set
+    ds_image : xarray.Dataset
+        Xarray data set with grid specifications to which apply on ds_in.
+    regrid_method : {'bilinear', 'conservative', 'nearest_s2d', 'nearest_d2s', 'patch'}
+        Regridding method. Defaults 'bilinear'.
+    
+    Returns
+    -------
+    ds_out : xarray.Dataset
+        Output data set regridded according to the grid specs of ds_in.
+        
+    Raises
+    ------
+    TypeError
+        If input datasets are not xarray.Dataset objects.
+    ValueError
+        If regrid_method is not valid.
+    ImportError
+        If xesmf package is not available.
+    RuntimeError
+        If regridding operation fails.
+    """
+    # Parameter validation
+    try:
+        import xarray as xr
+        if not isinstance(ds_in, xr.Dataset):
+            raise TypeError("ds_in must be an xarray.Dataset")
+        
+        if not isinstance(ds_image, xr.Dataset):
+            raise TypeError("ds_image must be an xarray.Dataset")
+    except ImportError:
+        raise ImportError("xarray package is required but not available")
+    
+    if not isinstance(regrid_method, str):
+        raise TypeError("regrid_method must be a string")
+    
+    if regrid_method not in REGRID_METHOD_LIST:
+        raise ValueError("Invalid regridding method.\n"
+                         f"Choose one from {REGRID_METHOD_LIST}.")        
+    
+    try:
+        import xesmf as xe
+    except ImportError:
+        raise ImportError("xesmf package is required for regridding but not available")
+    
+    try:
+        regridder = xe.Regridder(ds_in, ds_image, regrid_method)
+        ds_out = regridder(ds_in)
+        return ds_out    
+    except Exception as e:
+        raise RuntimeError(f"Regridding operation failed: {e}")
+
+#--------------------------#
+# Parameters and constants #
+#--------------------------#
+
+# Directory from where this code is being called #
+CODE_CALL_DIR = os.getcwd()
+
+# File extensions #
+EXTENSIONS = CLIMATE_FILE_EXTENSIONS[::3]
+
+# Main file names #
+COORD_INFO_FNAME = "latlon_bounds.txt"
+DATE_RANGE_INFO_FNAME = "period_bounds.txt"
+TIME_FORMATS_FILE_NAME = "time_formats.txt"
+
+# Regridding method options #
+REGRID_METHOD_LIST = [
+    "bilinear",
+    "conservative",
+    "conservative_normed",
+    "nearest_s2d",
+    "nearest_d2s",
+    "patch"
+    ]
+
+# Template strings #
+#------------------#
+
+# Main parameter scanning info strings #
+LATLON_INFO_TEMPLATE = \
+"""=========================================================
+·File: {}
+
+·Latitudes:
+ {}
+
+·Longitudes:
+ {}
+
+-Latitude-longitude array dimensions = {} x {}
+-Latitude-longitude array delta = ({}, {})
+    
+"""
+
+PERIOD_INFO_TEMPLATE = \
+"""=========================================================
+·File: {}
+·Time range: {} -- {}
+-Range length = {}
+
+"""
+    
+TIME_FORMAT_INFO_TEMPLATE = \
+"""=========================================================
+·File: {}
+    
+·Time array:
+ {}
+
+-Array length = {}
+"""
+
+# File scanning progress information strings #
+DIR_INFO_TEMPLATE = """\nDirectory: {}"""