disdrodb 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

disdrodb/utils/encoding.py

@@ -17,20 +17,33 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """DISDRODB netCDF4 encoding utilities."""
+import os
+
 import xarray as xr
 
+from disdrodb.utils.yaml import read_yaml
+
 EPOCH = "seconds since 1970-01-01 00:00:00"
 
 
-def set_encodings(ds: xr.Dataset, encoding_dict: dict) -> xr.Dataset:
+def get_encodings_dict():
+    """Get encoding dictionary for DISDRODB product variables and coordinates."""
+    import disdrodb
+
+    configs_path = os.path.join(disdrodb.__root_path__, "disdrodb", "etc", "configs")
+    encodings_dict = read_yaml(os.path.join(configs_path, "encodings.yaml"))
+    return encodings_dict
+
+
+def set_encodings(ds: xr.Dataset, encodings_dict: dict) -> xr.Dataset:
     """Apply the encodings to the xarray Dataset.
 
     Parameters
     ----------
     ds  : xarray.Dataset
         Input xarray dataset.
-    encoding_dict : dict
-        Dictionary with encoding specifications.
+    encodings_dict : dict
+        Dictionary with encodings specifications.
 
     Returns
     -------
@@ -38,21 +51,22 @@ def set_encodings(ds: xr.Dataset, encoding_dict: dict) -> xr.Dataset:
         Output xarray dataset.
     """
     # Subset encoding dictionary
-    # - Here below encoding_dict contains only keys (variables) within the dataset
-    encoding_dict = {var: encoding_dict[var] for var in ds.data_vars if var in encoding_dict}
+    # - Here below encodings_dict contains only keys (variables) within the dataset
+    encodings_dict = {var: encodings_dict[var] for var in ds.data_vars if var in encodings_dict}
 
     # Ensure chunksize smaller than the array shape
-    encoding_dict = sanitize_encodings_dict(encoding_dict, ds)
+    encodings_dict = sanitize_encodings_dict(encodings_dict, ds)
 
     # Rechunk variables for fast writing !
     # - This pop the chunksize argument from the encoding dict !
-    ds = rechunk_dataset(ds, encoding_dict)
+    ds = rechunk_dataset(ds, encodings_dict)
 
     # Set time encoding
-    ds["time"].encoding.update(get_time_encoding())
+    if "time" in ds:
+        ds["time"].encoding.update(get_time_encoding())
 
     # Set the variable encodings
-    for var, encoding in encoding_dict.items():
+    for var, encoding in encodings_dict.items():
         ds[var].encoding.update(encoding)
 
     # Ensure no deprecated "missing_value" attribute
@@ -63,12 +77,12 @@ def set_encodings(ds: xr.Dataset, encoding_dict: dict) -> xr.Dataset:
     return ds
 
 
-def sanitize_encodings_dict(encoding_dict: dict, ds: xr.Dataset) -> dict:
+def sanitize_encodings_dict(encodings_dict: dict, ds: xr.Dataset) -> dict:
     """Ensure chunk size to be smaller than the array shape.
 
     Parameters
     ----------
-    encoding_dict : dict
+    encodings_dict : dict
         Dictionary containing the variable encodings.
     ds  : xarray.Dataset
         Input dataset.
@@ -79,23 +93,23 @@ def sanitize_encodings_dict(encoding_dict: dict, ds: xr.Dataset) -> dict:
         Encoding dictionary.
     """
     for var in ds.data_vars:
-        if var in encoding_dict:
+        if var in encodings_dict:
             shape = ds[var].shape
-            chunks = encoding_dict[var].get("chunksizes", None)
+            chunks = encodings_dict[var].get("chunksizes", None)
             if chunks is not None:
                 chunks = [shape[i] if chunks[i] > shape[i] else chunks[i] for i in range(len(chunks))]
-                encoding_dict[var]["chunksizes"] = chunks
-    return encoding_dict
+                encodings_dict[var]["chunksizes"] = chunks
+    return encodings_dict
 
 
-def rechunk_dataset(ds: xr.Dataset, encoding_dict: dict) -> xr.Dataset:
+def rechunk_dataset(ds: xr.Dataset, encodings_dict: dict) -> xr.Dataset:
     """Coerce the dataset arrays to have the chunk size specified in the encoding dictionary.
 
     Parameters
     ----------
     ds  : xarray.Dataset
         Input xarray dataset
-    encoding_dict : dict
+    encodings_dict : dict
         Dictionary containing the encoding to write the xarray dataset as a netCDF.
 
     Returns
@@ -104,8 +118,8 @@ def rechunk_dataset(ds: xr.Dataset, encoding_dict: dict) -> xr.Dataset:
         Output xarray dataset
     """
     for var in ds.data_vars:
-        if var in encoding_dict:
-            chunks = encoding_dict[var].pop("chunksizes", None)
+        if var in encodings_dict:
+            chunks = encodings_dict[var].pop("chunksizes", None)
             if chunks is not None:
                 dims = list(ds[var].dims)
                 chunks_dict = dict(zip(dims, chunks))

disdrodb/utils/logger.py

@@ -164,9 +164,16 @@ def _define_station_summary_log_file(list_logs, summary_filepath):
 
 
 def _define_station_problem_log_file(list_logs, problem_filepath):
-    # - Copy the log of files with warnings and error
-    list_keywords = ["ERROR"]  # "WARNING"
-    list_patterns = ["ValueError: Less than 5 timesteps available for day"]
+    # Copy the log of files with errors
+    list_keywords = ["ERROR"]
+    # Exclude lines with the following patterns
+    list_patterns = [
+        # Caused by no data with L2E and L2M filtering
+        "No timesteps with rain rate",
+        "No timesteps with N",
+        "No timesteps with Nbins",
+    ]
+    # Compile patterns to search, escaping any special regex characters
     re_keyword = re.compile("|".join(list_keywords))
     # Compile patterns to ignore, escaping any special regex characters
     re_patterns = re.compile("|".join(map(re.escape, list_patterns))) if list_patterns else None
@@ -221,7 +228,7 @@ def create_product_logs(
 
     The logs directory structure is the follow:
     /logs
-    - /files/<product_acronym>/<station> (same structure as data ... a log for each processed file)
+    - /files/<product_name>/<station> (same structure as data ... a log for each processed file)
     - /summary
       -->  SUMMARY.<PRODUCT_ACRONYM>.<CAMPAIGN_NAME>.<STATION_NAME>.log
     - /problems
@@ -269,7 +276,7 @@ def create_product_logs(
             # Product options
             **product_kwargs,
         )
-        list_logs = list_files(logs_dir, glob_pattern="*", recursive=True)
+        list_logs = list_files(logs_dir, recursive=True)
 
     # --------------------------------------------------------.
     # LogCaptureHandler of pytest does not have baseFilename attribute, so it returns None
@@ -332,5 +339,5 @@ def create_product_logs(
 
     # --------------------------------------------------------.
     # Remove /problem directory if empty !
-    if len(os.listdir(logs_problem_dir)) == 0:
+    if len(list_files(logs_problem_dir, glob_pattern="*.log")) == 0:
         os.rmdir(logs_problem_dir)

disdrodb/utils/manipulations.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Include functions helping for DISDRODB product manipulations."""
+
+import numpy as np
+
+from disdrodb.utils.xarray import unstack_datarray_dimension
+
+
+def get_diameter_bin_edges(ds):
+    """Retrieve diameter bin edges."""
+    bin_edges = np.append(ds["diameter_bin_lower"].compute().data, ds["diameter_bin_upper"].compute().data[-1])
+    return bin_edges
+
+
+def convert_from_decibel(x):
+    """Convert dB to unit."""
+    return np.power(10.0, 0.1 * x)  # x/10
+
+
+def convert_to_decibel(x):
+    """Convert unit to dB."""
+    return 10 * np.log10(x)
+
+
+def unstack_radar_variables(ds):
+    """Unstack radar variables."""
+    from disdrodb.scattering import RADAR_VARIABLES
+
+    for var in RADAR_VARIABLES:
+        if var in ds:
+            ds_unstack = unstack_datarray_dimension(ds[var], dim="frequency", prefix="", suffix="_")
+            ds.update(ds_unstack)
+            ds = ds.drop_vars(var)
+    if "frequency" in ds.dims:
+        ds = ds.drop_dims("frequency")
+    return ds
+
+
+def resample_drop_number_concentration(da, diameter_bin_edges, method="linear"):
+    """Resample drop number concentration N(D) DataArray to high resolution diameter bins."""
+    diameters_bin_center = diameter_bin_edges[:-1] + np.diff(diameter_bin_edges) / 2
+
+    da = da.interp(coords={"diameter_bin_center": diameters_bin_center}, method=method)
+    diameter_bin_width = np.diff(diameter_bin_edges)
+    diameter_bin_lower = diameter_bin_edges[:-1]
+    diameter_bin_upper = diameter_bin_edges[1:]
+    da = da.assign_coords(
+        {
+            "diameter_bin_width": ("diameter_bin_center", diameter_bin_width),
+            "diameter_bin_lower": ("diameter_bin_center", diameter_bin_lower),
+            "diameter_bin_upper": ("diameter_bin_center", diameter_bin_upper),
+        },
+    )
+    return da

disdrodb/utils/subsetting.py

@@ -0,0 +1,214 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# -----------------------------------------------------------------------------.
+"""This module contains functions for subsetting and aligning DISDRODB products."""
+
+import numpy as np
+from xarray.core.utils import either_dict_or_kwargs
+
+from disdrodb.constants import DIAMETER_DIMENSION, VELOCITY_DIMENSION
+
+
+def is_1d_non_dimensional_coord(xr_obj, coord):
+    """Checks if a coordinate is a 1d, non-dimensional coordinate."""
+    if coord not in xr_obj.coords:
+        return False
+    if xr_obj[coord].ndim != 1:
+        return False
+    is_1d_dim_coord = xr_obj[coord].dims[0] == coord
+    return not is_1d_dim_coord
+
+
+def _get_dim_of_1d_non_dimensional_coord(xr_obj, coord):
+    """Get the dimension of a 1D non-dimension coordinate."""
+    if not is_1d_non_dimensional_coord(xr_obj, coord):
+        raise ValueError(f"'{coord}' is not a dimension or a 1D non-dimensional coordinate.")
+    dim = xr_obj[coord].dims[0]
+    return dim
+
+
+def _get_dim_isel_on_non_dim_coord_from_isel(xr_obj, coord, isel_indices):
+    """Get dimension and isel_indices related to a 1D non-dimension coordinate.
+
+    Parameters
+    ----------
+    xr_obj : (xr.Dataset, xr.DataArray)
+        A xarray object.
+    coord : str
+        Name of the coordinate wishing to subset with .sel
+    isel_indices : (str, int, float, list, np.array)
+        Coordinate indices wishing to be selected.
+
+    Returns
+    -------
+    dim : str
+        Dimension related to the 1D non-dimension coordinate.
+    isel_indices : (int, list, slice)
+        Indices for index-based selection.
+    """
+    dim = _get_dim_of_1d_non_dimensional_coord(xr_obj, coord)
+    return dim, isel_indices
+
+
+def _get_dim_isel_indices_from_isel_indices(xr_obj, key, indices, method="dummy"):  # noqa
+    """Return the dimension and isel_indices related to the dimension position indices of a coordinate."""
+    # Non-dimensional coordinate case
+    if key not in xr_obj.dims:
+        key, indices = _get_dim_isel_on_non_dim_coord_from_isel(xr_obj, coord=key, isel_indices=indices)
+    return key, indices
+
+
+def _get_isel_indices_from_sel_indices(xr_obj, coord, sel_indices, method):
+    """Get isel_indices corresponding to sel_indices."""
+    da_coord = xr_obj[coord]
+    dim = da_coord.dims[0]
+    da_coord = da_coord.assign_coords({"isel_indices": (dim, np.arange(0, da_coord.size))})
+    da_subset = da_coord.swap_dims({dim: coord}).sel({coord: sel_indices}, method=method)
+    isel_indices = da_subset["isel_indices"].data
+    return isel_indices
+
+
+def _get_dim_isel_on_non_dim_coord_from_sel(xr_obj, coord, sel_indices, method):
+    """
+    Return the dimension and isel_indices related to a 1D non-dimension coordinate.
+
+    Parameters
+    ----------
+    xr_obj : (xr.Dataset, xr.DataArray)
+        A xarray object.
+    coord : str
+        Name of the coordinate wishing to subset with .sel
+    sel_indices : (str, int, float, list, np.array)
+        Coordinate values wishing to be selected.
+
+    Returns
+    -------
+    dim : str
+        Dimension related to the 1D non-dimension coordinate.
+    isel_indices : np.ndarray
+        Indices for index-based selection.
+    """
+    dim = _get_dim_of_1d_non_dimensional_coord(xr_obj, coord)
+    isel_indices = _get_isel_indices_from_sel_indices(xr_obj, coord=coord, sel_indices=sel_indices, method=method)
+    return dim, isel_indices
+
+
+def _get_dim_isel_indices_from_sel_indices(xr_obj, key, indices, method):
+    """Return the dimension and isel_indices related to values of a coordinate."""
+    # Dimension case
+    if key in xr_obj.dims:
+        if key not in xr_obj.coords:
+            raise ValueError(f"Can not subset with disdrodb.sel the dimension '{key}' if it is not also a coordinate.")
+        isel_indices = _get_isel_indices_from_sel_indices(xr_obj, coord=key, sel_indices=indices, method=method)
+    # Non-dimensional coordinate case
+    else:
+        key, isel_indices = _get_dim_isel_on_non_dim_coord_from_sel(
+            xr_obj,
+            coord=key,
+            sel_indices=indices,
+            method=method,
+        )
+    return key, isel_indices
+
+
+def _get_dim_isel_indices_function(func):
+    func_dict = {
+        "sel": _get_dim_isel_indices_from_sel_indices,
+        "isel": _get_dim_isel_indices_from_isel_indices,
+    }
+    return func_dict[func]
+
+
+def _subset(xr_obj, indexers=None, func="isel", drop=False, method=None, **indexers_kwargs):
+    """Perform selection with isel or isel."""
+    # Retrieve indexers
+    indexers = either_dict_or_kwargs(indexers, indexers_kwargs, func)
+    # Get function returning isel_indices
+    get_dim_isel_indices = _get_dim_isel_indices_function(func)
+    # Define isel_dict
+    isel_dict = {}
+    for key, indices in indexers.items():
+        key, isel_indices = get_dim_isel_indices(xr_obj, key=key, indices=indices, method=method)
+        if key in isel_dict:
+            raise ValueError(f"Multiple indexers point to the '{key}' dimension.")
+        isel_dict[key] = isel_indices
+
+    # Subset and update area
+    xr_obj = xr_obj.isel(isel_dict, drop=drop)
+    return xr_obj
+
+
+def isel(xr_obj, indexers=None, drop=False, **indexers_kwargs):
+    """Perform index-based dimension selection."""
+    return _subset(xr_obj, indexers=indexers, func="isel", drop=drop, **indexers_kwargs)
+
+
+def sel(xr_obj, indexers=None, drop=False, method=None, **indexers_kwargs):
+    """Perform value-based coordinate selection.
+
+    Slices are treated as inclusive of both the start and stop values, unlike normal Python indexing.
+    The disdrodb `sel` method is empowered to:
+
+    - slice by disdrodb-id strings !
+    - slice by any xarray coordinate value !
+
+    You can use string shortcuts for datetime coordinates (e.g., '2000-01' to select all values in January 2000).
+    """
+    return _subset(xr_obj, indexers=indexers, func="sel", drop=drop, method=method, **indexers_kwargs)
+
+
+def align(*args):
+    """Align DISDRODB products over time, velocity and diameter dimensions."""
+    list_xr_obj = args
+
+    # Check input
+    if len(list_xr_obj) <= 1:
+        raise ValueError("At least two xarray object are required for alignment.")
+
+    # Define dimensions used for alignment
+    dims_to_align = ["time", DIAMETER_DIMENSION, VELOCITY_DIMENSION]
+
+    # Check which dimensions and coordinates are available across all datasets
+    coords = [coord for coord in dims_to_align if all(coord in xr_obj.coords for xr_obj in list_xr_obj)]
+    if not coords:
+        raise ValueError("No common coordinates found among the input datasets for alignment.")
+
+    # Start with the input datasets
+    list_aligned = list(list_xr_obj)
+
+    # Loop over the dimensions which are available
+    for coord in coords:
+        # Retrieve list of coordinate values
+        list_coord_values = [xr_obj[coord].data for xr_obj in list_aligned]
+
+        # Retrieve intersection of coordinates values
+        # - np.atleast_1d ensure that the dimension is not dropped if only 1 value
+        # - np.intersect1d returns the sorted array of common unique elements
+        common_values = list_coord_values[0]
+        for coord_values in list_coord_values[1:]:
+            common_values = np.intersect1d(common_values, coord_values)
+        sel_indices = np.atleast_1d(common_values)
+
+        # Check there are common coordinate values
+        if len(sel_indices) == 0:
+            raise ValueError(f"No common {coord} values across input objects.")
+
+        # Subset dataset
+        new_list_aligned = [sel(xr_obj, {coord: sel_indices}) for xr_obj in list_aligned]
+        list_aligned = new_list_aligned
+
+    return list_aligned