disdrodb 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

disdrodb/l0/check_configs.py

@@ -22,7 +22,7 @@ import os
 from typing import Optional, Union
 
 import numpy as np
-from pydantic import BaseModel, ValidationError, field_validator, model_validator
+from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
 
 from disdrodb.api.configs import available_sensor_names, get_sensor_configs_dir, read_config_file
 from disdrodb.l0.standards import (
@@ -47,7 +47,7 @@ CONFIG_FILES_LIST = [
 ]
 
 
-def _check_yaml_files_exists(sensor_name: str) -> None:
+def check_yaml_files_exists(sensor_name: str) -> None:
     """Check if all L0 config YAML files exist.
 
     Parameters
@@ -64,7 +64,7 @@ def _check_yaml_files_exists(sensor_name: str) -> None:
         raise FileNotFoundError(f"Missing YAML files {missing_keys_text} in {config_dir} for sensor {sensor_name}.")
 
 
-def _check_variable_consistency(sensor_name: str) -> None:
+def check_variable_consistency(sensor_name: str) -> None:
     """
     Check variable consistency across config files.
 
@@ -126,7 +126,7 @@ def _schema_error(object_to_validate: Union[str, list], schema: BaseModel, messa
 
 
 class L0BEncodingSchema(BaseModel):
-    """Pydantic model for DISDRODB L0B encodings."""
+    """Pydantic model for DISDRODB netCDF encodings."""
 
     contiguous: bool
     dtype: str
@@ -134,7 +134,7 @@ class L0BEncodingSchema(BaseModel):
     complevel: int
     shuffle: bool
     fletcher32: bool
-    _FillValue: Optional[Union[int, float]]
+    FillValue: Optional[Union[int, float]] = Field(default=None, alias="_FillValue")
     chunksizes: Optional[Union[int, list[int]]]
 
     # if contiguous=False, chunksizes specified, otherwise should be not !
@@ -167,6 +167,39 @@ class L0BEncodingSchema(BaseModel):
             raise ValueError("'fletcher32' must be set to False if 'contiguous' is True")
         return values
 
+    # if dtype is integer/unsigned integer, _FillValue must be specified and valid
+    @model_validator(mode="before")
+    def check_integer_fillvalue(cls, values):
+        """Check that integer dtypes have valid _FillValue."""
+        dtype = values.get("dtype")
+        fill_value = values.get("_FillValue", None)
+        integer_types = ["int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64"]
+        # Check if dtype is an integer type
+        if dtype in integer_types:
+            # _FillValue must be specified for integer types
+            if fill_value is None:
+                raise ValueError(f"'_FillValue' must be specified for integer dtype '{dtype}'")
+
+            # Check that _FillValue is within valid range for the dtype
+            dtype_info = np.iinfo(dtype)
+            max_value = dtype_info.max
+            # min_value = dtype_info.min
+
+            # if not (min_value <= fill_value <= max_value):
+            #     raise ValueError(
+            #         f"'_FillValue' ({fill_value}) is out of range for dtype '{dtype}'. "
+            #         f"Valid range is [{min_value}, {max_value}]",
+            #     )
+
+            # Check that _FillValue corresponds to the maximum allowed value
+            if fill_value != max_value:
+                raise ValueError(
+                    f"'_FillValue' ({fill_value}) should be set to the maximum allowed value "
+                    f"({max_value}) for integer dtype '{dtype}'",
+                )
+
+        return values
+
 
 def check_l0b_encoding(sensor_name: str) -> None:
     """Check ``l0b_encodings.yml`` file based on the schema defined in the class ``L0BEncodingSchema``.
@@ -176,7 +209,7 @@ def check_l0b_encoding(sensor_name: str) -> None:
     sensor_name : str
         Name of the sensor.
     """
-    data = read_config_file(sensor_name, product="L0A", filename="l0b_encodings.yml")
+    data = read_config_file(sensor_name, product="L0B", filename="l0b_encodings.yml")
 
     # check that the second level of the dictionary match the schema
     for key, value in data.items():
@@ -234,7 +267,7 @@ class RawDataFormatSchema(BaseModel):
         return None
 
 
-def _check_raw_data_format(sensor_name: str) -> None:
+def check_raw_data_format(sensor_name: str) -> None:
     """Check ``raw_data_format.yml`` file based on the schema defined in the class ``RawDataFormatSchema``.
 
     Parameters
@@ -253,7 +286,7 @@ def _check_raw_data_format(sensor_name: str) -> None:
         )
 
 
-def _check_cf_attributes(sensor_name: str) -> None:
+def check_cf_attributes(sensor_name: str) -> None:
     """Check that the ``l0b_cf_attrs.yml`` description, long_name and units values are strings.
 
     Parameters
@@ -268,7 +301,7 @@ def _check_cf_attributes(sensor_name: str) -> None:
                 raise ValueError(f"Wrong value for {key} in {var} for sensor {sensor_name}.")
 
 
-def _check_bin_consistency(sensor_name: str) -> None:
+def check_bin_consistency(sensor_name: str) -> None:
     """Check bin consistency from config file.
 
     Do not check the first and last bin !
@@ -313,7 +346,7 @@ def _check_bin_consistency(sensor_name: str) -> None:
         )
 
 
-def _check_raw_array(sensor_name: str) -> None:
+def check_raw_array(sensor_name: str) -> None:
     """Check raw array consistency from config file.
 
     Parameters
@@ -363,14 +396,14 @@ def check_sensor_configs(sensor_name: str) -> None:
     sensor_name : str
         Name of the sensor.
     """
-    _check_yaml_files_exists(sensor_name)
-    _check_variable_consistency(sensor_name)
+    check_yaml_files_exists(sensor_name)
+    check_variable_consistency(sensor_name)
     check_l0b_encoding(sensor_name=sensor_name)
     check_l0a_encoding(sensor_name=sensor_name)
-    _check_raw_data_format(sensor_name=sensor_name)
-    _check_cf_attributes(sensor_name=sensor_name)
-    _check_bin_consistency(sensor_name=sensor_name)
-    _check_raw_array(sensor_name=sensor_name)
+    check_raw_data_format(sensor_name=sensor_name)
+    check_cf_attributes(sensor_name=sensor_name)
+    check_bin_consistency(sensor_name=sensor_name)
+    check_raw_array(sensor_name=sensor_name)
 
 
 def check_all_sensors_configs() -> None:

disdrodb/l0/configs/LPM/l0a_encodings.yml

@@ -15,7 +15,7 @@ reflectivity: "float32"
 quality_index: "float32" # 'uint8'
 max_hail_diameter: "float32"
 laser_status: "float32" # 'uint8'
-static_signal: "float32" # 'uint8'
+static_signal_status: "float32" # 'uint8'
 laser_temperature_analog_status: "float32" # 'uint8'
 laser_temperature_digital_status: "float32" # 'uint8'
 laser_current_analog_status: "float32" # 'uint8'
@@ -29,7 +29,7 @@ current_heating_house_status: "float32" # 'uint8'
 current_heating_heads_status: "float32" # 'uint8'
 current_heating_carriers_status: "float32" # 'uint8'
 control_output_laser_power_status: "float32" # 'uint8'
-reserve_status: "float32" # 'uint8'
+reserved_status: "float32" # 'uint8'
 temperature_interior: "float32" # 'uint16'
 laser_temperature: "float32" # 'uint16'
 laser_current_average: "float32" # 'uint16'

disdrodb/l0/configs/LPM/l0b_cf_attrs.yml

@@ -66,7 +66,7 @@ laser_status:
   description: Status Laser (OK/on:0, off:1)
   long_name: Status laser
   units: ""
-static_signal:
+static_signal_status:
   description: Static signal (OK:0, Error:1)
   long_name: Static signal
   units: ""
@@ -122,7 +122,7 @@ control_output_laser_power_status:
   description: Status Control output laser power (OK:0, warning:1)
   long_name: Status control output laser
   units: ""
-reserve_status:
+reserved_status:
   description: Reserve Status (0)
   long_name: Reserve status
   units: ""

disdrodb/l0/configs/LPM/l0b_encodings.yml

@@ -184,7 +184,7 @@ laser_status:
   contiguous: false
   chunksizes: 5000
   _FillValue: 255
-static_signal:
+static_signal_status:
   dtype: uint8
   zlib: true
   complevel: 3
@@ -310,7 +310,7 @@ control_output_laser_power_status:
   contiguous: false
   chunksizes: 5000
   _FillValue: 255
-reserve_status:
+reserved_status:
   dtype: uint8
   zlib: true
   complevel: 3

disdrodb/l0/configs/LPM/raw_data_format.yml

@@ -213,7 +213,7 @@ laser_status:
     - 0
     - 1
   field_number: "22"
-static_signal:
+static_signal_status:
   n_digits: 1
   n_characters: 1
   n_decimals: 1
@@ -389,7 +389,7 @@ control_output_laser_power_status:
     - 0
     - 1
   field_number: "36"
-reserve_status:
+reserved_status:
   n_digits: 1
   n_characters: 1
   n_decimals: 1

disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml

@@ -216,7 +216,7 @@ rainfall_rate_16bit:
   contiguous: false
   chunksizes: 5000
 rainfall_rate_12bit:
-  dtype: object
+  dtype: float32
   zlib: true
   complevel: 3
   shuffle: true

disdrodb/l0/configs/PWS100/l0b_encodings.yml

@@ -68,6 +68,7 @@ relative_humidity:
   shuffle: true
   fletcher32: false
   contiguous: false
+  _FillValue: 65535
   chunksizes: 5000
 wetbulb_temperature:
   dtype: uint16

disdrodb/l0/configs/SWS250/bins_diameter.yml

@@ -0,0 +1,108 @@
+center:
+  0: 0.2
+  1: 0.45
+  2: 0.55
+  3: 0.65
+  4: 0.75
+  5: 0.85
+  6: 0.95
+  7: 1.05
+  8: 1.15
+  9: 1.25
+  10: 1.35
+  11: 1.5
+  12: 1.75
+  13: 2.08
+  14: 2.475
+  15: 2.945
+  16: 3.5
+  17: 4.16
+  18: 4.95
+  19: 5.89
+  20: 6.7
+bounds:
+  0:
+    - 0.0
+    - 0.4
+  1:
+    - 0.4
+    - 0.5
+  2:
+    - 0.5
+    - 0.6
+  3:
+    - 0.6
+    - 0.7
+  4:
+    - 0.7
+    - 0.8
+  5:
+    - 0.8
+    - 0.9
+  6:
+    - 0.9
+    - 1.0
+  7:
+    - 1.0
+    - 1.1
+  8:
+    - 1.1
+    - 1.2
+  9:
+    - 1.2
+    - 1.3
+  10:
+    - 1.3
+    - 1.4
+  11:
+    - 1.4
+    - 1.6
+  12:
+    - 1.6
+    - 1.9
+  13:
+    - 1.9
+    - 2.26
+  14:
+    - 2.26
+    - 2.69
+  15:
+    - 2.69
+    - 3.2
+  16:
+    - 3.2
+    - 3.8
+  17:
+    - 3.8
+    - 4.52
+  18:
+    - 4.52
+    - 5.38
+  19:
+    - 5.38
+    - 6.4
+  20:
+    - 6.4
+    - 7.0
+width:
+  0: 0.4
+  1: 0.1
+  2: 0.1
+  3: 0.1
+  4: 0.1
+  5: 0.1
+  6: 0.1
+  7: 0.1
+  8: 0.1
+  9: 0.1
+  10: 0.1
+  11: 0.2
+  12: 0.3
+  13: 0.36
+  14: 0.43
+  15: 0.51
+  16: 0.6
+  17: 0.72
+  18: 0.86
+  19: 1.02
+  20: 0.6

disdrodb/l0/configs/SWS250/bins_velocity.yml

@@ -0,0 +1,83 @@
+center:
+  0: 0.55
+  1: 0.95
+  2: 1.41
+  3: 1.84
+  4: 2.465
+  5: 3.27
+  6: 4.01
+  7: 4.625
+  8: 5.275
+  9: 6.325
+  10: 7.63
+  11: 9.51
+  12: 11.745
+  13: 14.14
+  14: 17.775
+  15: 22.5
+bounds:
+  0:
+    - 0.40
+    - 0.70
+  1:
+    - 0.70
+    - 1.20
+  2:
+    - 1.20
+    - 1.62
+  3:
+    - 1.62
+    - 2.06
+  4:
+    - 2.06
+    - 2.87
+  5:
+    - 2.87
+    - 3.67
+  6:
+    - 3.67
+    - 4.35
+  7:
+    - 4.35
+    - 4.90
+  8:
+    - 4.90
+    - 5.65
+  9:
+    - 5.65
+    - 7.00
+  10:
+    - 7.00
+    - 8.26
+  11:
+    - 8.26
+    - 10.76
+  12:
+    - 10.76
+    - 12.73
+  13:
+    - 12.73
+    - 15.55
+  14:
+    - 15.55
+    - 20.00
+  15:
+    - 20.00
+    - 25.00
+width:
+  0: 0.3
+  1: 0.5
+  2: 0.42
+  3: 0.44
+  4: 0.81
+  5: 0.8
+  6: 0.68
+  7: 0.55
+  8: 0.75
+  9: 1.35
+  10: 1.26
+  11: 2.5
+  12: 1.97
+  13: 2.82
+  14: 4.45
+  15: 5.0

disdrodb/l0/configs/SWS250/l0a_encodings.yml

@@ -0,0 +1,18 @@
+precipitation_rate: "float32"
+precipitation_accumulated: "float32"
+weather_code_synop_4680: "float32" # 'uint16'
+weather_code_metar_4678: "str"
+past_weather1: "str"
+past_weather2: "str"
+mor_visibility_5min: "float32"
+mor_visibility: "float32" # 'uint16'
+number_particles: "float32" #  'uint32'
+sensor_temperature: "float32" #  'uint16'
+obstruction_status: "str"
+total_extinction_coefficient: "float32"
+transmissometer_extinction_coefficient: "float32"
+back_scatter_extinction_coefficient: "float32"
+ambient_light_sensor_signal: "float32"
+sensor_status: "str" #
+ambient_light_sensor_signal_status: "str"
+raw_drop_number: "str"

disdrodb/l0/configs/SWS250/l0b_cf_attrs.yml

@@ -0,0 +1,72 @@
+precipitation_rate:
+  description: Rainfall rate
+  long_name: Precipitation rate
+  units: mm/h
+precipitation_accumulated:
+  description: Accumulated rain amount over the measurement interval
+  long_name: Precipitation accumulated
+  units: mm
+weather_code_synop_4680:
+  description: SYNOP weather code according to table 4680 of Parsivel documentation
+  long_name: Weather code SYNOP 4680
+  units: ""
+weather_code_metar_4678:
+  description: METAR/SPECI weather code according to table 4678 of Parsivel documentation
+  long_name: Weather code METAR 4678
+  units: ""
+past_weather1:
+  description: Past weather code 1
+  long_name: Past weather 1
+  units: ""
+past_weather2:
+  description: Past weather code 2
+  long_name: Past weather 2
+  units: ""
+mor_visibility_5min:
+  description: Meteorological Optical Range in precipitation (5 minute average)
+  long_name: MOR visibility 5 minute
+  units: m
+mor_visibility:
+  description: Meteorological Optical Range in precipitation
+  long_name: MOR visibility
+  units: m
+number_particles:
+  description: Number of particles detected and validated
+  long_name: Number of detected particles
+  units: ""
+sensor_temperature:
+  description: Temperature in sensor housing
+  long_name: Temperature of the sensor
+  units: "degC"
+obstruction_status:
+  description: Obstruction status of the sensor
+  long_name: Obstruction status
+  units: ""
+total_extinction_coefficient:
+  description: Total extinction coefficient
+  long_name: Total extinction coefficient
+  units: "1/km"
+transmissometer_extinction_coefficient:
+  description: Transmissometer extinction coefficient
+  long_name: Transmissometer extinction coefficient
+  units: "1/km"
+back_scatter_extinction_coefficient:
+  description: Back scatter extinction coefficient
+  long_name: Back scatter extinction coefficient
+  units: "1/km"
+ambient_light_sensor_signal:
+  description: Ambient light sensor signal
+  long_name: Ambient light sensor signal
+  units: "cd/m2"
+sensor_status:
+  description: Sensor status
+  long_name: Sensor status
+  units: ""
+ambient_light_sensor_signal_status:
+  description: Ambient light sensor signal status
+  long_name: Ambient light sensor signal status
+  units: ""
+raw_drop_number:
+  description: Drop counts per diameter and velocity class
+  long_name: Raw drop number
+  units: ""

disdrodb/l0/configs/SWS250/l0b_encodings.yml

@@ -0,0 +1,155 @@
+precipitation_rate:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+precipitation_accumulated:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+weather_code_synop_4680:
+  dtype: uint8
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 255
+weather_code_metar_4678:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+past_weather1:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+past_weather2:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+mor_visibility_5min:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 65535
+mor_visibility:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 65535
+number_particles:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 65535
+sensor_temperature:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -60.0
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 65535
+obstruction_status:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+sensor_status:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+ambient_light_sensor_signal_status:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+total_extinction_coefficient:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+transmissometer_extinction_coefficient:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+back_scatter_extinction_coefficient:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+ambient_light_sensor_signal:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+raw_drop_number:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes:
+    - 5000
+    - 16
+    - 21