disdrodb 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

disdrodb/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.4'
-__version_tuple__ = version_tuple = (0, 1, 4)
+__version__ = version = '0.1.5'
+__version_tuple__ = version_tuple = (0, 1, 5)
 
 __commit_id__ = commit_id = None

disdrodb/api/create_directories.py

@@ -85,8 +85,6 @@ def create_l0_directory_structure(
     ``product = "L0A"`` will call ``run_l0a``.
     ``product = "L0B"`` will call ``run_l0b_nc``.
     """
-    from disdrodb.configs import get_data_archive_dir, get_metadata_archive_dir
-
     # Retrieve the DISDRODB Metadata Archive directory
     data_archive_dir = get_data_archive_dir(data_archive_dir)
     metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)

disdrodb/cli/disdrodb_create_summary.py

@@ -39,6 +39,14 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
 @click_data_archive_dir_option
 @click_metadata_archive_dir_option
 @click.option("-p", "--parallel", type=bool, show_default=True, default=False, help="Read files in parallel")
+@click.option(
+    "-t",
+    "--temporal_resolution",
+    type=str,
+    show_default=True,
+    default="1MIN",
+    help="Temporal resolution of the L2E product to be used for the summary.",
+)
 def disdrodb_create_summary(
     # Stations options
     data_sources: Optional[str] = None,
@@ -46,6 +54,7 @@ def disdrodb_create_summary(
     station_names: Optional[str] = None,
     # Processing options:
     parallel=False,
+    temporal_resolution="1MIN",
     # DISDRODB root directories
     data_archive_dir: Optional[str] = None,
     metadata_archive_dir: Optional[str] = None,
@@ -92,6 +101,7 @@ def disdrodb_create_summary(
         station_names=station_names,
         # Options
         parallel=parallel,
+        temporal_resolution=temporal_resolution,
         # DISDRODB root directory
         data_archive_dir=data_archive_dir,
         metadata_archive_dir=metadata_archive_dir,

disdrodb/cli/disdrodb_create_summary_station.py

@@ -36,6 +36,14 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
 @click_station_arguments
 @click_data_archive_dir_option
 @click.option("-p", "--parallel", type=bool, show_default=True, default=False, help="Read files in parallel")
+@click.option(
+    "-t",
+    "--temporal_resolution",
+    type=str,
+    show_default=True,
+    default="1MIN",
+    help="Temporal resolution of the L2E product to be used for the summary.",
+)
 def disdrodb_create_summary_station(
     # Station arguments
     data_source: str,
@@ -43,6 +51,7 @@ def disdrodb_create_summary_station(
     station_name: str,
     # Processing options:
     parallel=False,
+    temporal_resolution="1MIN",
     # DISDRODB root directories
     data_archive_dir: Optional[str] = None,
 ):
@@ -81,6 +90,7 @@ def disdrodb_create_summary_station(
         station_name=station_name,
         # Options
         parallel=parallel,
+        temporal_resolution=temporal_resolution,
         # DISDRODB root directory
         data_archive_dir=data_archive_dir,
     )

disdrodb/constants.py

@@ -41,7 +41,7 @@ COORDINATES = [
     "time",
     "sample_interval",
 ]
-OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "PWS100"]
+OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "PWS100", "SWS250"]
 IMPACT_SENSORS = ["RD80"]
 
 PRODUCTS = ["RAW", "L0A", "L0B", "L0C", "L1", "L2E", "L2M"]

disdrodb/etc/products/L1/global.yaml

@@ -1,5 +1,5 @@
 product_options:
-  fall_velocity_method: "Beard1976"
+  fall_velocity_model: "Beard1976"
   minimum_diameter: 0
   maximum_diameter: 10
   minimum_velocity: 0

disdrodb/etc/products/L2E/5MIN.yaml

@@ -0,0 +1 @@
+radar_enabled: True

disdrodb/etc/products/L2E/global.yaml

@@ -1,4 +1,4 @@
-temporal_resolutions: ["1MIN", "5MIN", "10MIN", "ROLL1MIN"]
+temporal_resolutions: [1MIN"] # "5MIN", "10MIN", "ROLL1MIN"]
 archive_options:
   strategy: time_block
   strategy_options:

disdrodb/etc/products/L2M/GAMMA_GS_ND_MAE.yaml

@@ -0,0 +1,6 @@
+psd_model: "GammaPSD"
+optimization: "GS"
+optimization_kwargs:
+  target: "ND"
+  transformation: "identity"
+  error_order: 1

disdrodb/etc/products/L2M/GAMMA_ML.yaml

@@ -1,7 +1,7 @@
 psd_model: "GammaPSD"
 optimization: "ML"
 optimization_kwargs:
-  init_method: "M346"
+  init_method: "None"
   probability_method: "cdf"
   likelihood: "multinomial"
   truncated_likelihood: True

disdrodb/etc/products/L2M/LOGNORMAL_GS_LOG_ND_MAE.yaml

@@ -0,0 +1,6 @@
+psd_model: "LognormalPSD"
+optimization: "GS"
+optimization_kwargs:
+  target: "ND"
+  transformation: "log"
+  error_order: 1

disdrodb/etc/products/L2M/LOGNORMAL_GS_ND_MAE.yaml

@@ -0,0 +1,6 @@
+psd_model: "LognormalPSD"
+optimization: "GS"
+optimization_kwargs:
+  target: "ND"
+  transformation: "identity"
+  error_order: 1

disdrodb/etc/products/L2M/LOGNORMAL_ML.yaml

@@ -0,0 +1,8 @@
+psd_model: "LognormalPSD"
+optimization: "ML"
+optimization_kwargs:
+  init_method: "None"
+  probability_method: "cdf"
+  likelihood: "multinomial"
+  truncated_likelihood: True
+  optimizer: "Nelder-Mead"

disdrodb/etc/products/L2M/global.yaml

@@ -1,12 +1,20 @@
-temporal_resolutions: ["1MIN", "5MIN", "10MIN"]
-models: ["GAMMA_ML", "NGAMMA_GS_LOG_ND_MAE"]
+temporal_resolutions: ["1MIN"] #, "5MIN", "10MIN"]
+models:
+  [
+    "GAMMA_ML",
+    "GAMMA_GS_ND_MAE",
+    "NGAMMA_GS_LOG_ND_MAE",
+    "NGAMMA_GS_ND_MAE",
+    "LOGNORMAL_ML",
+    "LOGNORMAL_GS_ND_MAE",
+  ]
 archive_options:
   strategy: time_block
   strategy_options:
     freq: month
   folder_partitioning: ""
 product_options:
-  fall_velocity_method: "Beard1976"
+  fall_velocity_model: "Beard1976"
   diameter_min: 0
   diameter_max: 10
   diameter_spacing: 0.05

disdrodb/l0/check_configs.py

@@ -22,7 +22,7 @@ import os
 from typing import Optional, Union
 
 import numpy as np
-from pydantic import BaseModel, ValidationError, field_validator, model_validator
+from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
 
 from disdrodb.api.configs import available_sensor_names, get_sensor_configs_dir, read_config_file
 from disdrodb.l0.standards import (
@@ -47,7 +47,7 @@ CONFIG_FILES_LIST = [
 ]
 
 
-def _check_yaml_files_exists(sensor_name: str) -> None:
+def check_yaml_files_exists(sensor_name: str) -> None:
     """Check if all L0 config YAML files exist.
 
     Parameters
@@ -64,7 +64,7 @@ def _check_yaml_files_exists(sensor_name: str) -> None:
         raise FileNotFoundError(f"Missing YAML files {missing_keys_text} in {config_dir} for sensor {sensor_name}.")
 
 
-def _check_variable_consistency(sensor_name: str) -> None:
+def check_variable_consistency(sensor_name: str) -> None:
     """
     Check variable consistency across config files.
 
@@ -126,7 +126,7 @@ def _schema_error(object_to_validate: Union[str, list], schema: BaseModel, messa
 
 
 class L0BEncodingSchema(BaseModel):
-    """Pydantic model for DISDRODB L0B encodings."""
+    """Pydantic model for DISDRODB netCDF encodings."""
 
     contiguous: bool
     dtype: str
@@ -134,7 +134,7 @@ class L0BEncodingSchema(BaseModel):
     complevel: int
     shuffle: bool
     fletcher32: bool
-    _FillValue: Optional[Union[int, float]]
+    FillValue: Optional[Union[int, float]] = Field(default=None, alias="_FillValue")
     chunksizes: Optional[Union[int, list[int]]]
 
     # if contiguous=False, chunksizes specified, otherwise should be not !
@@ -167,6 +167,39 @@ class L0BEncodingSchema(BaseModel):
             raise ValueError("'fletcher32' must be set to False if 'contiguous' is True")
         return values
 
+    # if dtype is integer/unsigned integer, _FillValue must be specified and valid
+    @model_validator(mode="before")
+    def check_integer_fillvalue(cls, values):
+        """Check that integer dtypes have valid _FillValue."""
+        dtype = values.get("dtype")
+        fill_value = values.get("_FillValue", None)
+        integer_types = ["int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64"]
+        # Check if dtype is an integer type
+        if dtype in integer_types:
+            # _FillValue must be specified for integer types
+            if fill_value is None:
+                raise ValueError(f"'_FillValue' must be specified for integer dtype '{dtype}'")
+
+            # Check that _FillValue is within valid range for the dtype
+            dtype_info = np.iinfo(dtype)
+            max_value = dtype_info.max
+            # min_value = dtype_info.min
+
+            # if not (min_value <= fill_value <= max_value):
+            #     raise ValueError(
+            #         f"'_FillValue' ({fill_value}) is out of range for dtype '{dtype}'. "
+            #         f"Valid range is [{min_value}, {max_value}]",
+            #     )
+
+            # Check that _FillValue corresponds to the maximum allowed value
+            if fill_value != max_value:
+                raise ValueError(
+                    f"'_FillValue' ({fill_value}) should be set to the maximum allowed value "
+                    f"({max_value}) for integer dtype '{dtype}'",
+                )
+
+        return values
+
 
 def check_l0b_encoding(sensor_name: str) -> None:
     """Check ``l0b_encodings.yml`` file based on the schema defined in the class ``L0BEncodingSchema``.
@@ -176,7 +209,7 @@ def check_l0b_encoding(sensor_name: str) -> None:
     sensor_name : str
         Name of the sensor.
     """
-    data = read_config_file(sensor_name, product="L0A", filename="l0b_encodings.yml")
+    data = read_config_file(sensor_name, product="L0B", filename="l0b_encodings.yml")
 
     # check that the second level of the dictionary match the schema
     for key, value in data.items():
@@ -234,7 +267,7 @@ class RawDataFormatSchema(BaseModel):
         return None
 
 
-def _check_raw_data_format(sensor_name: str) -> None:
+def check_raw_data_format(sensor_name: str) -> None:
     """Check ``raw_data_format.yml`` file based on the schema defined in the class ``RawDataFormatSchema``.
 
     Parameters
@@ -253,7 +286,7 @@ def _check_raw_data_format(sensor_name: str) -> None:
         )
 
 
-def _check_cf_attributes(sensor_name: str) -> None:
+def check_cf_attributes(sensor_name: str) -> None:
     """Check that the ``l0b_cf_attrs.yml`` description, long_name and units values are strings.
 
     Parameters
@@ -268,7 +301,7 @@ def _check_cf_attributes(sensor_name: str) -> None:
                 raise ValueError(f"Wrong value for {key} in {var} for sensor {sensor_name}.")
 
 
-def _check_bin_consistency(sensor_name: str) -> None:
+def check_bin_consistency(sensor_name: str) -> None:
     """Check bin consistency from config file.
 
     Do not check the first and last bin !
@@ -313,7 +346,7 @@ def _check_bin_consistency(sensor_name: str) -> None:
         )
 
 
-def _check_raw_array(sensor_name: str) -> None:
+def check_raw_array(sensor_name: str) -> None:
     """Check raw array consistency from config file.
 
     Parameters
@@ -363,14 +396,14 @@ def check_sensor_configs(sensor_name: str) -> None:
     sensor_name : str
         Name of the sensor.
     """
-    _check_yaml_files_exists(sensor_name)
-    _check_variable_consistency(sensor_name)
+    check_yaml_files_exists(sensor_name)
+    check_variable_consistency(sensor_name)
     check_l0b_encoding(sensor_name=sensor_name)
     check_l0a_encoding(sensor_name=sensor_name)
-    _check_raw_data_format(sensor_name=sensor_name)
-    _check_cf_attributes(sensor_name=sensor_name)
-    _check_bin_consistency(sensor_name=sensor_name)
-    _check_raw_array(sensor_name=sensor_name)
+    check_raw_data_format(sensor_name=sensor_name)
+    check_cf_attributes(sensor_name=sensor_name)
+    check_bin_consistency(sensor_name=sensor_name)
+    check_raw_array(sensor_name=sensor_name)
 
 
 def check_all_sensors_configs() -> None:

disdrodb/l0/configs/LPM/l0a_encodings.yml

@@ -15,7 +15,7 @@ reflectivity: "float32"
 quality_index: "float32" # 'uint8'
 max_hail_diameter: "float32"
 laser_status: "float32" # 'uint8'
-static_signal: "float32" # 'uint8'
+static_signal_status: "float32" # 'uint8'
 laser_temperature_analog_status: "float32" # 'uint8'
 laser_temperature_digital_status: "float32" # 'uint8'
 laser_current_analog_status: "float32" # 'uint8'
@@ -29,7 +29,7 @@ current_heating_house_status: "float32" # 'uint8'
 current_heating_heads_status: "float32" # 'uint8'
 current_heating_carriers_status: "float32" # 'uint8'
 control_output_laser_power_status: "float32" # 'uint8'
-reserve_status: "float32" # 'uint8'
+reserved_status: "float32" # 'uint8'
 temperature_interior: "float32" # 'uint16'
 laser_temperature: "float32" # 'uint16'
 laser_current_average: "float32" # 'uint16'

disdrodb/l0/configs/LPM/l0b_cf_attrs.yml

@@ -66,7 +66,7 @@ laser_status:
   description: Status Laser (OK/on:0, off:1)
   long_name: Status laser
   units: ""
-static_signal:
+static_signal_status:
   description: Static signal (OK:0, Error:1)
   long_name: Static signal
   units: ""
@@ -122,7 +122,7 @@ control_output_laser_power_status:
   description: Status Control output laser power (OK:0, warning:1)
   long_name: Status control output laser
   units: ""
-reserve_status:
+reserved_status:
   description: Reserve Status (0)
   long_name: Reserve status
   units: ""

disdrodb/l0/configs/LPM/l0b_encodings.yml

@@ -184,7 +184,7 @@ laser_status:
   contiguous: false
   chunksizes: 5000
   _FillValue: 255
-static_signal:
+static_signal_status:
   dtype: uint8
   zlib: true
   complevel: 3
@@ -310,7 +310,7 @@ control_output_laser_power_status:
   contiguous: false
   chunksizes: 5000
   _FillValue: 255
-reserve_status:
+reserved_status:
   dtype: uint8
   zlib: true
   complevel: 3

disdrodb/l0/configs/LPM/raw_data_format.yml

@@ -213,7 +213,7 @@ laser_status:
     - 0
     - 1
   field_number: "22"
-static_signal:
+static_signal_status:
   n_digits: 1
   n_characters: 1
   n_decimals: 1
@@ -389,7 +389,7 @@ control_output_laser_power_status:
     - 0
     - 1
   field_number: "36"
-reserve_status:
+reserved_status:
   n_digits: 1
   n_characters: 1
   n_decimals: 1

disdrodb/l0/configs/PWS100/l0b_encodings.yml

@@ -68,6 +68,7 @@ relative_humidity:
   shuffle: true
   fletcher32: false
   contiguous: false
+  _FillValue: 65535
   chunksizes: 5000
 wetbulb_temperature:
   dtype: uint16

disdrodb/l0/configs/SWS250/bins_diameter.yml

@@ -0,0 +1,108 @@
+center:
+  0: 0.2
+  1: 0.45
+  2: 0.55
+  3: 0.65
+  4: 0.75
+  5: 0.85
+  6: 0.95
+  7: 1.05
+  8: 1.15
+  9: 1.25
+  10: 1.35
+  11: 1.5
+  12: 1.75
+  13: 2.08
+  14: 2.475
+  15: 2.945
+  16: 3.5
+  17: 4.16
+  18: 4.95
+  19: 5.89
+  20: 6.7
+bounds:
+  0:
+    - 0.0
+    - 0.4
+  1:
+    - 0.4
+    - 0.5
+  2:
+    - 0.5
+    - 0.6
+  3:
+    - 0.6
+    - 0.7
+  4:
+    - 0.7
+    - 0.8
+  5:
+    - 0.8
+    - 0.9
+  6:
+    - 0.9
+    - 1.0
+  7:
+    - 1.0
+    - 1.1
+  8:
+    - 1.1
+    - 1.2
+  9:
+    - 1.2
+    - 1.3
+  10:
+    - 1.3
+    - 1.4
+  11:
+    - 1.4
+    - 1.6
+  12:
+    - 1.6
+    - 1.9
+  13:
+    - 1.9
+    - 2.26
+  14:
+    - 2.26
+    - 2.69
+  15:
+    - 2.69
+    - 3.2
+  16:
+    - 3.2
+    - 3.8
+  17:
+    - 3.8
+    - 4.52
+  18:
+    - 4.52
+    - 5.38
+  19:
+    - 5.38
+    - 6.4
+  20:
+    - 6.4
+    - 7.0
+width:
+  0: 0.4
+  1: 0.1
+  2: 0.1
+  3: 0.1
+  4: 0.1
+  5: 0.1
+  6: 0.1
+  7: 0.1
+  8: 0.1
+  9: 0.1
+  10: 0.1
+  11: 0.2
+  12: 0.3
+  13: 0.36
+  14: 0.43
+  15: 0.51
+  16: 0.6
+  17: 0.72
+  18: 0.86
+  19: 1.02
+  20: 0.6

disdrodb/l0/configs/SWS250/bins_velocity.yml

@@ -0,0 +1,83 @@
+center:
+  0: 0.55
+  1: 0.95
+  2: 1.41
+  3: 1.84
+  4: 2.465
+  5: 3.27
+  6: 4.01
+  7: 4.625
+  8: 5.275
+  9: 6.325
+  10: 7.63
+  11: 9.51
+  12: 11.745
+  13: 14.14
+  14: 17.775
+  15: 22.5
+bounds:
+  0:
+    - 0.40
+    - 0.70
+  1:
+    - 0.70
+    - 1.20
+  2:
+    - 1.20
+    - 1.62
+  3:
+    - 1.62
+    - 2.06
+  4:
+    - 2.06
+    - 2.87
+  5:
+    - 2.87
+    - 3.67
+  6:
+    - 3.67
+    - 4.35
+  7:
+    - 4.35
+    - 4.90
+  8:
+    - 4.90
+    - 5.65
+  9:
+    - 5.65
+    - 7.00
+  10:
+    - 7.00
+    - 8.26
+  11:
+    - 8.26
+    - 10.76
+  12:
+    - 10.76
+    - 12.73
+  13:
+    - 12.73
+    - 15.55
+  14:
+    - 15.55
+    - 20.00
+  15:
+    - 20.00
+    - 25.00
+width:
+  0: 0.3
+  1: 0.5
+  2: 0.42
+  3: 0.44
+  4: 0.81
+  5: 0.8
+  6: 0.68
+  7: 0.55
+  8: 0.75
+  9: 1.35
+  10: 1.26
+  11: 2.5
+  12: 1.97
+  13: 2.82
+  14: 4.45
+  15: 5.0

disdrodb/l0/configs/SWS250/l0a_encodings.yml

@@ -0,0 +1,18 @@
+precipitation_rate: "float32"
+precipitation_accumulated: "float32"
+weather_code_synop_4680: "float32" # 'uint16'
+weather_code_metar_4678: "str"
+past_weather1: "str"
+past_weather2: "str"
+mor_visibility_5min: "float32"
+mor_visibility: "float32" # 'uint16'
+number_particles: "float32" #  'uint32'
+sensor_temperature: "float32" #  'uint16'
+obstruction_status: "str"
+total_extinction_coefficient: "float32"
+transmissometer_extinction_coefficient: "float32"
+back_scatter_extinction_coefficient: "float32"
+ambient_light_sensor_signal: "float32"
+sensor_status: "str" #
+ambient_light_sensor_signal_status: "str"
+raw_drop_number: "str"