disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl

disdrodb/l0/__init__.py

@@ -1,26 +1,17 @@
-from disdrodb.l0.l0_processing import (
-    run_l0a,
-    run_l0b_from_nc,
-)
-from disdrodb.l0.l0_reader import available_readers
-from disdrodb.l0.routines import (
-    run_disdrodb_l0,
-    run_disdrodb_l0_station,
-    run_disdrodb_l0a,
-    run_disdrodb_l0a_station,
-    run_disdrodb_l0b,
-    run_disdrodb_l0b_station,
-)
-
-__all__ = [
-    "run_l0a",
-    "run_l0b_from_nc",
-    "available_readers",
-    # Functions invoking the disdrodb_run_* scripts in the terminal
-    "run_disdrodb_l0a_station",
-    "run_disdrodb_l0b_station",
-    "run_disdrodb_l0_station",
-    "run_disdrodb_l0",
-    "run_disdrodb_l0a",
-    "run_disdrodb_l0b",
-]
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""DISDRODB L0 software."""

disdrodb/l0/check_configs.py

@@ -19,7 +19,7 @@
 """Check configuration files."""
 
 import os
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 import numpy as np
 from pydantic import BaseModel, ValidationError, field_validator, model_validator
@@ -105,7 +105,7 @@ def _check_variable_consistency(sensor_name: str) -> None:
 
 
 class SchemaValidationException(Exception):
-    """Exception raised when schema validation fails"""
+    """Exception raised when schema validation fails."""
 
 
 def _schema_error(object_to_validate: Union[str, list], schema: BaseModel, message) -> bool:
@@ -114,12 +114,11 @@ def _schema_error(object_to_validate: Union[str, list], schema: BaseModel, messa
     Parameters
     ----------
     object_to_validate : Union[str,list]
-        Object to validate
+        Object to validate.
     schema : BaseModel
-        Base model
+        Base model.
 
     """
-
     try:
         schema(**object_to_validate)
     except ValidationError as e:
@@ -127,6 +126,8 @@ def _schema_error(object_to_validate: Union[str, list], schema: BaseModel, messa
 
 
 class L0BEncodingSchema(BaseModel):
+    """Pydantic model for DISDRODB L0B encodings."""
+
     contiguous: bool
     dtype: str
     zlib: bool
@@ -134,11 +135,12 @@ class L0BEncodingSchema(BaseModel):
     shuffle: bool
     fletcher32: bool
     _FillValue: Optional[Union[int, float]]
-    chunksizes: Optional[Union[int, List[int]]]
+    chunksizes: Optional[Union[int, list[int]]]
 
     # if contiguous=False, chunksizes specified, otherwise should be not !
     @model_validator(mode="before")
     def check_chunksizes_and_zlib(cls, values):
+        """Check the chunksizes validity."""
         contiguous = values.get("contiguous")
         chunksizes = values.get("chunksizes")
         if not contiguous and not chunksizes:
@@ -148,6 +150,7 @@ class L0BEncodingSchema(BaseModel):
     # if contiguous = True, then zlib must be set to False
     @model_validator(mode="before")
     def check_contiguous_and_zlib(cls, values):
+        """Check the the compression value validity."""
         contiguous = values.get("contiguous")
         zlib = values.get("zlib")
         if contiguous and zlib:
@@ -157,6 +160,7 @@ class L0BEncodingSchema(BaseModel):
     # if contiguous = True, then fletcher32 must be set to False
     @model_validator(mode="before")
     def check_contiguous_and_fletcher32(cls, values):
+        """Check the fletcher value validity."""
         contiguous = values.get("contiguous")
         fletcher32 = values.get("fletcher32")
         if contiguous and fletcher32:
@@ -165,14 +169,13 @@ class L0BEncodingSchema(BaseModel):
 
 
 def check_l0b_encoding(sensor_name: str) -> None:
-    """Check l0b_encodings.yml file based on the schema defined in the class L0BEncodingSchema.
+    """Check ``l0b_encodings.yml`` file based on the schema defined in the class ``L0BEncodingSchema``.
 
     Parameters
     ----------
     sensor_name : str
         Name of the sensor.
     """
-
     data = read_config_file(sensor_name, product="L0A", filename="l0b_encodings.yml")
 
     # check that the second level of the dictionary match the schema
@@ -185,7 +188,7 @@ def check_l0b_encoding(sensor_name: str) -> None:
 
 
 def check_l0a_encoding(sensor_name: str) -> None:
-    """Check l0a_encodings.yml file.
+    """Check ``l0a_encodings.yml`` file.
 
     Parameters
     ----------
@@ -208,27 +211,31 @@ def check_l0a_encoding(sensor_name: str) -> None:
 
 
 class RawDataFormatSchema(BaseModel):
+    """Pydantic model for the DISDRODB RAW Data Format YAML files."""
+
     n_digits: Optional[int]
     n_characters: Optional[int]
     n_decimals: Optional[int]
     n_naturals: Optional[int]
-    data_range: Optional[List[float]]
+    data_range: Optional[list[float]]
     nan_flags: Optional[Union[int, str]] = None
-    valid_values: Optional[List[float]] = None
-    dimension_order: Optional[List[str]] = None
+    valid_values: Optional[list[float]] = None
+    dimension_order: Optional[list[str]] = None
     n_values: Optional[int] = None
     field_number: Optional[str] = None
 
     @field_validator("data_range")
     def check_list_length(cls, value):
+        """Check the data_range validity."""
         if value:
             if len(value) != 2:
                 raise ValueError(f"data_range must have exactly 2 keys, {len(value)} element have been provided.")
             return value
+        return None
 
 
 def _check_raw_data_format(sensor_name: str) -> None:
-    """check raw_data_format.yml file based on the schema defined in the class RawDataFormatSchema.
+    """Check ``raw_data_format.yml`` file based on the schema defined in the class ``RawDataFormatSchema``.
 
     Parameters
     ----------
@@ -247,7 +254,7 @@ def _check_raw_data_format(sensor_name: str) -> None:
 
 
 def _check_cf_attributes(sensor_name: str) -> None:
-    """Check that the l0b_cf_attrs.yml description, long_name and units values are strings.
+    """Check that the ``l0b_cf_attrs.yml`` description, long_name and units values are strings.
 
     Parameters
     ----------
@@ -271,7 +278,6 @@ def _check_bin_consistency(sensor_name: str) -> None:
     sensor_name : str
         Name of the sensor.
     """
-
     diameter_bin_lower = np.array(get_diameter_bin_lower(sensor_name))
     diameter_bin_upper = np.array(get_diameter_bin_upper(sensor_name))
     diameter_bin_center = np.array(get_diameter_bin_center(sensor_name))
@@ -294,10 +300,16 @@ def _check_bin_consistency(sensor_name: str) -> None:
     if all(arr.size > 1 for arr in [velocity_bin_center, velocity_bin_lower, velocity_bin_upper, velocity_bin_width]):
         np.testing.assert_allclose(velocity_bin_upper - velocity_bin_lower, velocity_bin_width, atol=1e-3, rtol=1e-4)
         np.testing.assert_allclose(
-            velocity_bin_lower + velocity_bin_width / 2, velocity_bin_center, atol=1e-3, rtol=1e-4
+            velocity_bin_lower + velocity_bin_width / 2,
+            velocity_bin_center,
+            atol=1e-3,
+            rtol=1e-4,
         )
         np.testing.assert_allclose(
-            velocity_bin_upper - velocity_bin_width / 2, velocity_bin_center, atol=1e-3, rtol=1e-4
+            velocity_bin_upper - velocity_bin_width / 2,
+            velocity_bin_center,
+            atol=1e-3,
+            rtol=1e-4,
         )
 
 
@@ -318,7 +330,7 @@ def _check_raw_array(sensor_name: str) -> None:
 
     # Get keys in raw_data_format where the value is "dimension_order"
     dict_keys_with_dimension_order = {
-        key: value.get("dimension_order") for key, value in raw_data_format.items() if "dimension_order" in value.keys()
+        key: value.get("dimension_order") for key, value in raw_data_format.items() if "dimension_order" in value
     }
 
     l0b_encodings = read_config_file(sensor_name, product="L0A", filename="l0b_encodings.yml")
@@ -332,11 +344,11 @@ def _check_raw_array(sensor_name: str) -> None:
     # Get chunksizes in l0b_encoding.yml and check that if len > 1, has dimension_order key in raw_data_format
     list_attributes_l0b_encodings = [
         i
-        for i in l0b_encodings.keys()
+        for i in l0b_encodings
         if isinstance(l0b_encodings.get(i).get("chunksizes"), list) and len(l0b_encodings.get(i).get("chunksizes")) > 1
     ]
     list_attributes_from_raw_data_format = [
-        i for i in raw_data_format.keys() if raw_data_format.get(i).get("dimension_order") is not None
+        i for i in raw_data_format if raw_data_format.get(i).get("dimension_order") is not None
     ]
 
     if not sorted(list_attributes_l0b_encodings) == sorted(list_attributes_from_raw_data_format):
@@ -344,7 +356,7 @@ def _check_raw_array(sensor_name: str) -> None:
 
 
 def check_sensor_configs(sensor_name: str) -> None:
-    """check sensor configs.
+    """Check validity of sensor configuration YAML files.
 
     Parameters
     ----------
@@ -362,6 +374,6 @@ def check_sensor_configs(sensor_name: str) -> None:
 
 
 def check_all_sensors_configs() -> None:
-    """Check all sensors configs."""
-    for sensor_name in available_sensor_names(product="L0A"):
+    """Check all sensors configuration YAML files."""
+    for sensor_name in available_sensor_names():
         check_sensor_configs(sensor_name=sensor_name)

disdrodb/l0/check_standards.py

@@ -25,24 +25,21 @@ import numpy as np
 import pandas as pd
 
 from disdrodb.l0.standards import (
+    allowed_l0_variables,
     get_data_range_dict,
-    get_l0a_dtype,
     get_valid_values_dict,
 )
 
 # Logger
-from disdrodb.utils.logger import (
-    log_error,
-    log_info,
-)
+from disdrodb.utils.logger import log_info
 
 logger = logging.getLogger(__name__)
 
 
-def _check_valid_range(df, dict_data_range, verbose=False):
+def _check_valid_range(df, dict_data_range):
     """Check valid value range of dataframe columns.
 
-    It assumes the dict_data_range values are list [min_val, max_val]
+    It assumes the ``dict_data_range`` values are list ``[min_val, max_val]``.
     """
     list_wrong_columns = []
     for column in df.columns:
@@ -56,14 +53,13 @@ def _check_valid_range(df, dict_data_range, verbose=False):
 
     if len(list_wrong_columns) > 0:
         msg = f"Columns {list_wrong_columns} has values outside the expected data range."
-        log_error(logger=logger, msg=msg, verbose=False)
         raise ValueError(msg)
 
 
-def _check_valid_values(df, dict_valid_values, verbose=False):
+def _check_valid_values(df, dict_valid_values):
     """Check valid values of dataframe columns.
 
-    It assumes the dict_valid_values values are list [...].
+    It assumes the ``dict_valid_values`` values are lists.
     """
     list_msg = []
     list_wrong_columns = []
@@ -81,16 +77,20 @@ def _check_valid_values(df, dict_valid_values, verbose=False):
 
     if len(list_wrong_columns) > 0:
         msg = "\n".join(list_msg)
-        log_error(logger=logger, msg=msg, verbose=False)
         raise ValueError(f"Columns {list_wrong_columns} have invalid values.")
 
 
-def _check_raw_fields_available(df: pd.DataFrame, sensor_name: str, verbose: bool = False) -> None:
+def _check_raw_fields_available(
+    df: pd.DataFrame,
+    sensor_name: str,  # noqa: ARG001
+    logger=None,  # noqa: ARG001
+    verbose: bool = False,  # noqa: ARG001
+) -> None:
     """Check the presence of the raw spectrum data according to the type of sensor.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Dataframe
     sensor_name : str
         Name of the sensor.
@@ -98,26 +98,24 @@ def _check_raw_fields_available(df: pd.DataFrame, sensor_name: str, verbose: boo
     Raises
     ------
     ValueError
-        Error if the raw_drop_number field is missing.
+        Error if the ``raw_drop_number`` field is missing.
     """
-    from disdrodb.l0.standards import get_raw_array_nvalues
-
-    # Retrieve raw arrays that could be available (based on sensor_name)
-    n_bins_dict = get_raw_array_nvalues(sensor_name=sensor_name)
-    raw_vars = np.array(list(n_bins_dict.keys()))
+    # from disdrodb.l0.standards import get_raw_array_nvalues
 
     # Check that raw_drop_number is present
     if "raw_drop_number" not in df.columns:
         msg = "The 'raw_drop_number' column is not present in the dataframe."
-        log_error(logger=logger, msg=msg, verbose=False)
         raise ValueError(msg)
 
+    # Retrieve raw arrays that could be available (based on sensor_name)
+    #  n_bins_dict = get_raw_array_nvalues(sensor_name=sensor_name)
+
     # Report additional raw arrays that are missing
-    missing_vars = raw_vars[np.isin(raw_vars, list(df.columns), invert=True)]
-    if len(missing_vars) > 0:
-        msg = f"The following raw array variable are missing: {missing_vars}"
-        log_info(logger=logger, msg=msg, verbose=verbose)
-    return None
+    # raw_vars = np.array(list(n_bins_dict.keys()))
+    # missing_vars = raw_vars[np.isin(raw_vars, list(df.columns), invert=True)]
+    # if len(missing_vars) > 0:
+    #     msg = f"The following raw array variable are missing: {missing_vars}"
+    #     log_info(logger=logger, msg=msg, verbose=verbose)
 
 
 def check_l0a_column_names(df: pd.DataFrame, sensor_name: str) -> None:
@@ -125,7 +123,7 @@ def check_l0a_column_names(df: pd.DataFrame, sensor_name: str) -> None:
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     sensor_name : str
         Name of the sensor.
@@ -133,47 +131,42 @@ def check_l0a_column_names(df: pd.DataFrame, sensor_name: str) -> None:
     Raises
     ------
     ValueError
-        Error if some columns do not meet the DISDRODB standards or if the 'time' column is missing in the dataframe.
+        Error if some columns do not meet the DISDRODB standards or if the ``'time'``
+        column is missing in the dataframe.
 
     """
-
     # Get valid columns
-    dtype_dict = get_l0a_dtype(sensor_name)
-    valid_columns = list(dtype_dict)
-    valid_columns = valid_columns + ["time", "latitude", "longitude"]
-    valid_columns = set(valid_columns)
+    valid_columns = set(allowed_l0_variables(sensor_name))
+
     # Get dataframe column names
-    df_columns = list(df.columns)
-    df_columns = set(df_columns)
-    # --------------------------------------------
-    # Check there aren't valid columns
-    invalid_columns = list(df_columns.difference(valid_columns))
+    df_columns = set(df.columns)
+
+    # Find any columns in df that aren't in the valid list
+    invalid_columns = df_columns - valid_columns
+
+    # Raise error in case
     if len(invalid_columns) > 0:
         msg = f"The following columns do no met the DISDRODB standards: {invalid_columns}"
-        logger.error(msg)
         raise ValueError(msg)
-    # --------------------------------------------
+
     # Check time column is present
     if "time" not in df_columns:
         msg = "The 'time' column is missing in the dataframe."
-        log_error(logger=logger, msg=msg, verbose=False)
         raise ValueError(msg)
-    # --------------------------------------------
-    return None
 
 
-def check_l0a_standards(df: pd.DataFrame, sensor_name: str, verbose: bool = True) -> None:
+def check_l0a_standards(df: pd.DataFrame, sensor_name: str, logger=None, verbose: bool = True) -> None:
     """Checks that a file respects the DISDRODB L0A standards.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         L0A dataframe.
     sensor_name : str
         Name of the sensor.
     verbose : bool, optional
         Whether to verbose the processing.
-        The default is True.
+        The default value is ``True``.
 
     Raises
     ------
@@ -184,23 +177,24 @@ def check_l0a_standards(df: pd.DataFrame, sensor_name: str, verbose: bool = True
     # -------------------------------------
     # Check data range
     dict_data_range = get_data_range_dict(sensor_name)
-    _check_valid_range(df=df, dict_data_range=dict_data_range, verbose=verbose)
+    _check_valid_range(df=df, dict_data_range=dict_data_range)
 
     # -------------------------------------
     # Check categorical data values
     dict_valid_values = get_valid_values_dict(sensor_name)
-    _check_valid_values(df=df, dict_valid_values=dict_valid_values, verbose=verbose)
+    _check_valid_values(df=df, dict_valid_values=dict_valid_values)
 
     # -------------------------------------
     # Check if raw spectrum and 1D derivate exists
-    _check_raw_fields_available(df=df, sensor_name=sensor_name, verbose=verbose)
+    _check_raw_fields_available(df=df, sensor_name=sensor_name, logger=logger, verbose=verbose)
 
     # -------------------------------------
     # Check if latitude and longitude are columns of the dataframe
     # - They should be only provided if the instrument is moving !!!!
+    # - TODO: this should be removed and raise error if not platform_type: mobile
     if "latitude" in df.columns:
         msg = (
-            " - The L0A dataframe has column 'latitude'. "
+            "The L0A dataframe has column 'latitude'. "
             + "This should be included only if the sensor is moving. "
             + "Otherwise, specify the 'latitude' in the metadata !"
         )
@@ -208,7 +202,7 @@ def check_l0a_standards(df: pd.DataFrame, sensor_name: str, verbose: bool = True
 
     if "longitude" in df.columns:
         msg = (
-            " - The L0A dataframe has column 'longitude'. "
+            "The L0A dataframe has column 'longitude'. "
             + "This should be included only if the sensor is moving. "
             + "Otherwise, specify the 'longitude' in the metadata !"
         )
@@ -218,6 +212,7 @@ def check_l0a_standards(df: pd.DataFrame, sensor_name: str, verbose: bool = True
 
 
 def check_l0b_standards(x: str) -> None:
-    # TODO:
+    """Check L0B standards."""
     # - Check for realistic values after having removed the flags !!!!
+    x = "noqa"  # noqa: F841
     pass

disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml

@@ -23,71 +23,71 @@ center:
   21: 54
 bounds:
   0:
-  - 0.125
-  - 0.25
+    - 0.125
+    - 0.25
   1:
-  - 0.25
-  - 0.375
+    - 0.25
+    - 0.375
   2:
-  - 0.375
-  - 0.5
+    - 0.375
+    - 0.5
   3:
-  - 0.5
-  - 0.75
+    - 0.5
+    - 0.75
   4:
-  - 0.75
-  - 1.0
+    - 0.75
+    - 1.0
   5:
-  - 1.0
-  - 1.25
+    - 1.0
+    - 1.25
   6:
-  - 1.25
-  - 1.5
+    - 1.25
+    - 1.5
   7:
-  - 1.5
-  - 1.75
+    - 1.5
+    - 1.75
   8:
-  - 1.75
-  - 2.0
+    - 1.75
+    - 2.0
   9:
-  - 2.0
-  - 2.5
+    - 2.0
+    - 2.5
   10:
-  - 2.5
-  - 3.0
+    - 2.5
+    - 3.0
   11:
-  - 3.0
-  - 3.5
+    - 3.0
+    - 3.5
   12:
-  - 3.5
-  - 4.0
+    - 3.5
+    - 4.0
   13:
-  - 4.0
-  - 4.5
+    - 4.0
+    - 4.5
   14:
-  - 4.5
-  - 5.0
+    - 4.5
+    - 5.0
   15:
-  - 5.0
-  - 5.5
+    - 5.0
+    - 5.5
   16:
-  - 5.5
-  - 6.0
+    - 5.5
+    - 6.0
   17:
-  - 6.0
-  - 6.5
+    - 6.0
+    - 6.5
   18:
-  - 6.5
-  - 7.0
+    - 6.5
+    - 7.0
   19:
-  - 7.0
-  - 7.5
+    - 7.0
+    - 7.5
   20:
-   - 7.5
-   - 8.0
+    - 7.5
+    - 8.0
   21:
-  - 8.0
-  - 100
+    - 8.0
+    - 100
 width:
   0: 0.125
   1: 0.125