disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl

disdrodb/l0/template_tools.py

@@ -16,14 +16,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
-"""Experimental code to help the implementation of DISDRODB L0 readers."""
+"""Useful tools helping in the implementation of the DISDRODB L0 readers."""
 
-from typing import Union
+from typing import Optional, Union
 
 import numpy as np
 import pandas as pd
 
 from disdrodb.l0.standards import (
+    allowed_l0_variables,
     get_field_nchar_dict,
     get_field_ndigits_decimals_dict,
     get_field_ndigits_dict,
@@ -68,24 +69,18 @@ def _check_columns_indices(column_indices, n_columns):
 
 
 def print_df_column_names(df: pd.DataFrame) -> None:
-    """Print dataframe columns names
+    """Print dataframe columns names.
 
     Parameters
     ----------
-    df : dataframe
-        The dataframe
-
-    Returns
-    -------
-    None
-        Nothing
+    df : pandas.DataFrame
+        The dataframe.
     """
     for i, column in enumerate(df.columns):
         print(" - Column", i, ":", column)
-    return None
 
 
-def print_valid_l0_column_names(sensor_name: str) -> None:
+def print_allowed_column_names(sensor_name: str) -> None:
     """Print valid columns names from the standard.
 
     Parameters
@@ -93,7 +88,9 @@ def print_valid_l0_column_names(sensor_name: str) -> None:
     sensor_name : str
         Name of the sensor.
     """
-    print(list(get_l0a_dtype(sensor_name)))
+    from pprint import pprint
+
+    pprint(allowed_l0_variables(sensor_name))
 
 
 def _print_column_index(i, column_name, print_column_names):
@@ -108,14 +105,14 @@ def _print_value(value):
 
 
 def print_df_with_any_nan_rows(df: pd.DataFrame) -> None:
-    """Print empty rows
+    """Print empty rows.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     """
-    df_bool_is_nan = df.isnull()
+    df_bool_is_nan = df.isna()
     idx_nan_rows = df_bool_is_nan.any(axis=1)
     df_nan_rows = df.loc[idx_nan_rows]
     if df_nan_rows.size != 0:
@@ -129,40 +126,35 @@ def print_df_first_n_rows(df: pd.DataFrame, n: int = 5, print_column_names: bool
 
     Parameters
     ----------
-    df : pd.DataFrame
-        Input dataframe
+    df : pandas.DataFrame
+        Input dataframe.
     n : int, optional
-        Number of row, by default 5
+        Number of row. The default is 5.
     column_names : bool , optional
-        If true columns name are printed, by default True
+        If true columns name are printed, by default ``True``.
     """
     columns = list(df.columns)
     for i in range(len(df.columns)):
         _print_column_index(i, column_name=columns[i], print_column_names=print_column_names)
-        _print_value(df.iloc[0 : (n + 1), i].values)
+        _print_value(df.iloc[0 : (n + 1), i].to_numpy())
 
 
 def print_df_random_n_rows(df: pd.DataFrame, n: int = 5, print_column_names: bool = True) -> None:
-    """Print the content of the dataframe by column, randomly chosen
+    """Print the content of the dataframe by column, randomly chosen.
 
     Parameters
     ----------
-    df : dataframe
-        The dataframe
+    df : pandas.DataFrame
+        The dataframe.
     n : int, optional
-        The number of row to print, by default 5
+        The number of row to print. The default is 5.
     print_column_names : bool, optional
-        If true, print the column names. The default is True.
-
-    Returns
-    -------
-    None
-        Nothing
+        If true, print the column names. The default value is ``True``.
     """
     columns = list(df.columns)
     df_sample = df.sample(n=n)
     for i in range(len(df_sample.columns)):
-        row_content = df_sample.iloc[0 : (n + 1), i].values
+        row_content = df_sample.iloc[0 : (n + 1), i].to_numpy()
         _print_column_index(i, column_name=columns[i], print_column_names=print_column_names)
         _print_value(row_content)
 
@@ -182,19 +174,19 @@ def _print_df_summary(df, indices, columns, print_column_names):
 
 def print_df_summary_stats(
     df: pd.DataFrame,
-    column_indices: Union[int, slice, list] = None,
+    column_indices: Optional[Union[int, slice, list]] = None,
     print_column_names: bool = True,
 ):
     """Create a columns statistics summary.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe
     column_indices : Union[int,slice,list], optional
-        Column indices. If None, select all columns.
+        Column indices. If ``None``, select all columns.
     print_column_names : bool, optional
-        If true, print the column names. The default is True.
+        If ``True``, print the column names. The default value is ``True``.
 
     Raises
     ------
@@ -205,7 +197,7 @@ def print_df_summary_stats(
     # Define columns of interest
     _, columns_of_interest = _get_selected_column_names(df, column_indices)
     # Remove columns of dtype object or string
-    indices_to_remove = np.where((df.dtypes == type(object)) | (df.dtypes == str))
+    indices_to_remove = np.where((df.dtypes == type(object)) | (df.dtypes == str))  # noqa
     indices = np.arange(0, len(df.columns))
     indices = indices[np.isin(indices, indices_to_remove, invert=True)]
     columns = df.columns[indices]
@@ -221,28 +213,39 @@ def print_df_summary_stats(
     _print_df_summary(df=df, indices=indices, columns=columns, print_column_names=print_column_names)
 
 
+def get_unique_sorted_values(array):
+    """Return unique sorted values.
+
+    It deals with np.nan within an array of string by converting object dtype to str.
+    """
+    arr = np.asanyarray(array)
+    if arr.dtype == object:
+        arr = arr.astype(str)
+    return np.unique(arr).tolist()
+
+
 def print_df_columns_unique_values(
     df: pd.DataFrame,
-    column_indices: Union[int, slice, list] = None,
+    column_indices: Optional[Union[int, slice, list]] = None,
     print_column_names: bool = True,
 ) -> None:
-    """Print columns' unique values
+    """Print columns' unique values.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe
     column_indices : Union[int,slice,list], optional
-        Column indices. If None, select all columns.
+        Column indices. If ``None``, select all columns.
     column_names : bool, optional
-        If true, print the column names. The default is True.
+        If ``True``, print the column names. The default value is ``True``.
 
     """
     column_indices, columns = _get_selected_column_names(df, column_indices)
     # Printing
     for i, column in zip(column_indices, columns):
         _print_column_index(i, column_name=column, print_column_names=print_column_names)
-        _print_value(sorted(df[column].unique().tolist()))
+        _print_value(get_unique_sorted_values(df[column]))
 
 
 ####--------------------------------------------------------------------------.
@@ -251,30 +254,27 @@ def print_df_columns_unique_values(
 
 def get_df_columns_unique_values_dict(
     df: pd.DataFrame,
-    column_indices: Union[int, slice, list] = None,
+    column_indices: Optional[Union[int, slice, list]] = None,
     column_names: bool = True,
 ):
-    """Create a dictionary {column: unique values}
+    """Create a dictionary {column: unique values}.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe
     column_indices : Union[int,slice,list], optional
-        Column indices. If None, select all columns.
+        Column indices. If ``None``, select all columns.
     column_names : bool, optional
-        If true, the dictionary key are the column names. The default is True.
+        If ``True``, the dictionary key are the column names. The default value is ``True``.
 
     """
     column_indices, columns = _get_selected_column_names(df, column_indices)
     # Create dictionary
     d = {}
     for i, column in zip(column_indices, columns):
-        if column_names:
-            key = column
-        else:
-            key = "Column " + str(i)
-        d[key] = sorted(df[column].unique().tolist())
+        key = column if column_names else "Column " + str(i)
+        d[key] = get_unique_sorted_values(df[column])
     # Return
     return d
 
@@ -288,15 +288,15 @@ def str_is_number(string: str) -> bool:
 
     Parameters
     ----------
-    string : Input string
+    string : str
+        Input string.
 
 
     Returns
     -------
     bool
-        True if float.
+        ``True`` if float.
     """
-
     try:
         float(string)
         return True
@@ -305,17 +305,18 @@ def str_is_number(string: str) -> bool:
 
 
 def str_is_integer(string: str) -> bool:
-    """Check if a string represent an integer
+    """Check if a string represent an integer.
 
     Parameters
     ----------
-    string : Input string
+    string : str
+        Input string.
 
 
     Returns
     -------
     bool
-        True if integer.
+        ``True`` if integer.
     """
     try:
         int(string)
@@ -325,12 +326,12 @@ def str_is_integer(string: str) -> bool:
 
 
 def str_has_decimal_digits(string: str) -> bool:
-    """Check if a string has decimals
+    """Check if a string has decimals.
 
     Parameters
     ----------
-    string :
-        Input string
+    string : str
+        Input string.
 
 
     Returns
@@ -338,54 +339,51 @@ def str_has_decimal_digits(string: str) -> bool:
     bool
         True if string has digits.
     """
-    if len(string.split(".")) == 2:
-        return True
-    else:
-        return False
+    return len(string.split(".")) == 2
 
 
 def get_decimal_ndigits(string: str) -> int:
-    """Get the decimal number of digit.
+    """Get the number of decimal digits.
 
     Parameters
     ----------
     string : str
-        Input string
+        Input string.
 
     Returns
     -------
     int
-        The number of digit.
+        The number of decimal digits.
     """
     if str_has_decimal_digits(string):
         return len(string.split(".")[1])
-    else:
-        return 0
+    return 0
 
 
 def get_natural_ndigits(string: str) -> int:
-    """Get the natural number of digit.
+    """Get the number of natural digits.
 
     Parameters
     ----------
     string : str
-        Input string
+        Input string.
 
     Returns
     -------
     int
-        The number of digit.
+        The number of natural digits.
     """
+    count_minus = int(string.startswith("-"))  # 0 if not start with -, else 1
+    string = string.replace("-", "")
     if str_is_integer(string):
-        return len(string.replace("-", ""))
+        return len(string) + count_minus
     if str_has_decimal_digits(string):
-        return len(string.split(".")[0].replace("-", ""))
-    else:
-        return 0
+        return len(string.split(".")[0]) + count_minus
+    return 0
 
 
 def get_ndigits(string: str) -> int:
-    """Get the number of digit.
+    """Get the number of total numeric digits.
 
     Parameters
     ----------
@@ -395,47 +393,46 @@ def get_ndigits(string: str) -> int:
     Returns
     -------
     int
-        Number of digit
+        The number of total digits.
     """
-
     if not str_is_number(string):
         return 0
+    count_minus = int(string.startswith("-"))  # 0 if not start with -, else 1
     string = string.replace("-", "")
     if str_has_decimal_digits(string):
-        return len(string) - 1  # remove .
-    else:
-        return len(string)
+        return len(string) - 1 + count_minus  # remove .
+    return len(string) + count_minus
 
 
 def get_nchar(string: str) -> int:
-    """Get the number of character.
+    """Get the number of characters.
 
     Parameters
     ----------
     string : str
-        Input string
+        Input string.
 
     Returns
     -------
     int
-        Number of character
+        The number of characters.
     """
     return len(string)
 
 
 def _has_constant_characters(arr: np.array) -> bool:
-    """Check if the content of an array has a constant number of characters
+    """Check if the content of an array has a constant number of characters.
 
     Parameters
     ----------
     arr : numpy.ndarray
         The array to analyse.
-        It convert numeric array to unicode before analyzing !
+        It converts numeric array to unicode before analyzing !
 
     Returns
     -------
     boolean
-        True if the number of character is constant.
+        ``True`` if the number of characters is constant.
         Empty array are considered constant !
 
     """
@@ -447,21 +444,20 @@ def _has_constant_characters(arr: np.array) -> bool:
 
 
 def _get_possible_keys(dict_options: dict, desired_value: str) -> set:
-    """Get the possible keys from the input values
+    """Get the possible keys from the input values.
 
     Parameters
     ----------
     dict_options : dict
-        Input dictionary
+        Input dictionary.
     desired_value : str
-        Input value
+        Input value.
 
     Returns
     -------
     set
         Keys that the value matches the desired input value.
     """
-
     list_key_match = []
     for k, v in dict_options.items():
         if v == desired_value:
@@ -471,19 +467,19 @@ def _get_possible_keys(dict_options: dict, desired_value: str) -> set:
 
 
 def _search_possible_columns(string: str, sensor_name: str) -> list:
-    """Define possible column
+    """Define possible columns.
 
     Parameters
     ----------
     string : str
-        Inpur string
+        Input string.
     sensor_name : str
-        Name of the sensor
+        Name of the sensor.
 
     Returns
     -------
     list
-        list of possible columns
+        List of possible columns.
     """
     dict_digits = get_field_ndigits_dict(sensor_name)
     dict_nchar_digits = get_field_nchar_dict(sensor_name)
@@ -504,17 +500,18 @@ def _search_possible_columns(string: str, sensor_name: str) -> list:
 #### Infer column names and checks validity
 
 
-def infer_column_names(df: pd.DataFrame, sensor_name: str, row_idx: int = 1):
+def infer_column_names(df: pd.DataFrame, sensor_name: str, row_idx: int = 0):
     """Try to guess the dataframe columns names based on string characteristics.
 
     Parameters
     ----------
-    df : numpy.ndarray
-        The array to analyse
+    df : pandas.DataFrame
+        The dataframe to analyse.
     sensor_name : str
-        name of the sensor
+        name of the sensor.
     row_idx : int, optional
-        The row ID of the array, by default 1
+        The row index of the dataframe to use to infer the column names.
+        The default row index is 0.
 
     Returns
     -------
@@ -526,12 +523,11 @@ def infer_column_names(df: pd.DataFrame, sensor_name: str, row_idx: int = 1):
         # Get string array
         arr = df.iloc[:, i]
         arr = np.asarray(arr).astype(str)
-
         # Check is the array contains a constant number of character
         if not _has_constant_characters(arr):
             print(
-                f"ATTENTION: Column {i} values have non-unique number of characters. "
-                f"Selecting row {row_idx} to infer possible columns."
+                f"WARNING: The number of characters of column {i} values is not constant. "
+                f"Column names are currently inferred using 'row_idx={row_idx}'.",
             )
 
         # Subset a single string
@@ -559,13 +555,12 @@ def check_column_names(column_names: list, sensor_name: str) -> None:
     TypeError
         Error if some columns do not meet the DISDRODB standards.
     """
-
     if not isinstance(column_names, list):
         raise TypeError("'column_names' must be a list of strings.")
     # Get valid columns
     dtype_dict = get_l0a_dtype(sensor_name)
     valid_columns = list(dtype_dict)
-    valid_columns = valid_columns + ["time"]
+    valid_columns = [*valid_columns, "time"]
     # --------------------------------------------
     # Create name sets
     column_names = set(column_names)
@@ -575,9 +570,9 @@ def check_column_names(column_names: list, sensor_name: str) -> None:
     invalid_columns = list(column_names.difference(valid_columns))
     if len(invalid_columns) > 0:
         print(f"The following columns do no met the DISDRODB standards: {invalid_columns}.")
-        print("Please remove such columns within the df_sanitizer_fun")
+        print("Please remove such columns in the reader function !")
     # --------------------------------------------
     # Check time column is present
     if "time" not in column_names:
-        print("Please be sure to create the 'time' column within the df_sanitizer_fun.")
+        print("Please be sure to create the 'time' column within the reader function !")
         print("The 'time' column must be datetime with resolution in seconds (dtype='M8[s]').")

disdrodb/l1/__init__.py

@@ -0,0 +1,17 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""DISDRODB L1 module."""