disdrodb 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

disdrodb/l0/configs/LPM_V0/raw_data_format.yml

@@ -0,0 +1,474 @@
+start_identifier:
+  n_digits: 1
+  n_characters: 1
+  n_decimals: 0
+  n_naturals: 1
+  data_range:
+    - 0
+    - 9
+  nan_flags: null
+  field_number: "01"
+sensor_serial_number:
+  n_digits: 6
+  n_characters: 6
+  n_decimals: 0
+  n_naturals: 6
+  data_range:
+    - 0
+    - 999999
+  nan_flags: null
+  field_number: "02"
+weather_code_synop_4680_5min:
+  n_digits: 2
+  n_characters: 2
+  n_decimals: 0
+  n_naturals: 2
+  data_range:
+    - -1
+    - 89
+  nan_flags: -1
+  field_number: "03"
+weather_code_metar_4678_5min:
+  n_digits: 0
+  n_characters: 5
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: "?????"
+  field_number: "04"
+precipitation_rate_5min:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 3
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 999.999
+  nan_flags: 999.999
+  field_number: "05"
+weather_code_synop_4680:
+  n_digits: 0
+  n_characters: 5
+  n_decimals: 0
+  n_naturals: 0
+  data_range:
+    - -1
+    - 89
+  nan_flags: -1
+  field_number: "06"
+weather_code_metar_4678:
+  n_digits: 0
+  n_characters: 5
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: "?????"
+  field_number: "07"
+precipitation_rate:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 3
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 999.999
+  nan_flags: 999.999
+  field_number: "08"
+precipitation_accumulated:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 4
+  n_naturals: 2
+  data_range:
+    - 0.0
+    - 9999.99
+  nan_flags: 9999.99
+  field_number: "09"
+sensor_time:
+  n_digits: 6
+  n_characters: 8
+  n_decimals: 6
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  field_number: "10"
+temperature_interior:
+  n_digits: 3
+  n_characters: 3
+  n_decimals: 3
+  n_naturals: 0
+  data_range:
+    - -30
+    - 999
+  nan_flags: null
+  field_number: "11"
+laser_temperature:
+  n_digits: 2
+  n_characters: 2
+  n_decimals: 2
+  n_naturals: 0
+  data_range:
+    - 0
+    - 80
+  nan_flags: null
+  field_number: "12"
+laser_current_average:
+  n_digits: 4
+  n_characters: 4
+  n_decimals: 4
+  n_naturals: 0
+  data_range:
+    - 0
+    - 9999
+  nan_flags: null
+  field_number: "13"
+control_voltage:
+  n_digits: 4
+  n_characters: 4
+  n_decimals: 4
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  field_number: "14"
+optical_control_voltage_output:
+  n_digits: 4
+  n_characters: 4
+  n_decimals: 4
+  n_naturals: 0
+  data_range:
+    - 0
+    - 9999
+  nan_flags: null
+  field_number: "15"
+number_particles:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "16"
+number_particles_internal_data:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "17"
+number_particles_min_speed:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "18"
+number_particles_min_speed_internal_data:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "19"
+number_particles_max_speed:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "20"
+number_particles_max_speed_internal_data:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "21"
+number_particles_min_diameter:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "22"
+number_particles_min_diameter_internal_data:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "23"
+number_particles_no_hydrometeor:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "24"
+number_particles_no_hydrometeor_internal_data:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "25"
+number_particles_unknown_classification:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "26"
+total_gross_volume_unknown_classification:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "27"
+number_particles_hail:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "28"
+total_gross_volume_hail:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "29"
+number_particles_solid_precipitation:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "30"
+total_gross_volume_solid_precipitation:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "31"
+number_particles_great_pellet:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "32"
+total_gross_volume_great_pellet:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "33"
+number_particles_small_pellet:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "34"
+total_gross_volume_small_pellet:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "35"
+number_particles_snowgrain:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "36"
+total_gross_volume_snowgrain:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "37"
+number_particles_rain:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "38"
+total_gross_volume_rain:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "39"
+number_particles_small_rain:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "40"
+total_gross_volume_small_rain:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "41"
+number_particles_drizzle:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 5
+  n_naturals: 0
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "42"
+total_gross_volume_drizzle:
+  n_digits: 8
+  n_characters: 9
+  n_decimals: 5
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 99999.999
+  nan_flags: null
+  field_number: "43"
+raw_drop_number:
+  n_digits: 1317
+  n_characters: 1756
+  n_decimals: 1317
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  dimension_order:
+    - diameter_bin_center
+    - velocity_bin_center
+  n_values: 400
+  field_number: "81"
+air_temperature:
+  n_digits: 4
+  n_characters: 5
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - -40
+    - 70
+  nan_flags: 99999
+  field_number: "444"
+relative_humidity:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 0
+  n_naturals: 5
+  data_range:
+    - 0
+    - 99999
+  nan_flags: 99999
+  field_number: "445"
+wind_speed:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - 0
+    - 60
+  nan_flags: null
+  field_number: "446"
+wind_direction:
+  n_digits: 3
+  n_characters: 3
+  n_decimals: 0
+  n_naturals: 3
+  data_range:
+    - 0
+    - 360
+  nan_flags: 999
+  field_number: "447"

disdrodb/l0/configs/PARSIVEL/raw_data_format.yml

@@ -6,7 +6,7 @@ rainfall_rate_32bit:
   data_range:
     - 0
     - 9999.999
-  nan_flags: null
+  nan_flags: 9999.999
   field_number: "01"
 rainfall_accumulated_32bit:
   n_digits: 6
@@ -16,7 +16,7 @@ rainfall_accumulated_32bit:
   data_range:
     - 0
     - 9999.0
-  nan_flags: null
+  nan_flags: 9999.0
   field_number: "02"
 weather_code_synop_4680:
   n_digits: 2
@@ -62,7 +62,7 @@ reflectivity_32bit:
   data_range:
     - -9.999
     - 99.999
-  nan_flags: null
+  nan_flags: 99.999
   field_number: "07"
 mor_visibility:
   n_digits: 4
@@ -92,7 +92,7 @@ laser_amplitude:
   data_range:
     - 0
     - 99999
-  nan_flags: null
+  nan_flags: 99999
   field_number: "10"
 number_particles:
   n_digits: 5
@@ -102,7 +102,7 @@ number_particles:
   data_range:
     - 0
     - 99999
-  nan_flags: null
+  nan_flags: 99999
   field_number: "11"
 sensor_temperature:
   n_digits: 3
@@ -221,7 +221,7 @@ rainfall_amount_absolute_32bit:
   data_range:
     - 0
     - 999.999
-  nan_flags: null
+  nan_flags: 999.999
   field_number: "24"
 error_code:
   n_digits: 3
@@ -245,7 +245,7 @@ rainfall_rate_16bit:
   data_range:
     - 0
     - 9999.999
-  nan_flags: null
+  nan_flags: 9999.999
   field_number: "30"
 rainfall_rate_12bit:
   n_digits: 5
@@ -273,7 +273,7 @@ reflectivity_16bit:
   data_range:
     - -9.99
     - 99.99
-  nan_flags: null
+  nan_flags: 99.99
   field_number: "33"
 raw_drop_concentration:
   n_digits: 0

disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml

@@ -6,7 +6,7 @@ rainfall_rate_32bit:
   data_range:
     - 0
     - 9999.999
-  nan_flags: null
+  nan_flags: 9999.999
   field_number: "01"
 rainfall_accumulated_32bit:
   n_digits: 6
@@ -16,7 +16,7 @@ rainfall_accumulated_32bit:
   data_range:
     - 0
     - 9999.0
-  nan_flags: null
+  nan_flags: 9999.0
   field_number: "02"
 weather_code_synop_4680:
   n_digits: 2
@@ -62,7 +62,7 @@ reflectivity_32bit:
   data_range:
     - -9.999
     - 99.999
-  nan_flags: null
+  nan_flags: 99.999
   field_number: "07"
 mor_visibility:
   n_digits: 5
@@ -92,7 +92,7 @@ laser_amplitude:
   data_range:
     - 0
     - 99999
-  nan_flags: null
+  nan_flags: 99999
   field_number: "10"
 number_particles:
   n_digits: 5
@@ -102,7 +102,7 @@ number_particles:
   data_range:
     - 0
     - 99999
-  nan_flags: null
+  nan_flags: 99999
   field_number: "11"
 sensor_temperature:
   n_digits: 3
@@ -221,7 +221,7 @@ rainfall_amount_absolute_32bit:
   data_range:
     - 0
     - 999.999
-  nan_flags: null
+  nan_flags: 999.999
   field_number: "24"
 error_code:
   n_digits: 3
@@ -305,7 +305,7 @@ reflectivity_16bit:
   data_range:
     - -9.99
     - 99.99
-  nan_flags: null
+  nan_flags: 99.99
   field_number: "33"
 rain_kinetic_energy:
   n_digits: 6
@@ -315,7 +315,7 @@ rain_kinetic_energy:
   data_range:
     - 0
     - 9999.99
-  nan_flags: null
+  nan_flags: 9999.99
   field_number: "34"
 snowfall_rate:
   n_digits: 6
@@ -325,7 +325,7 @@ snowfall_rate:
   data_range:
     - 0
     - 999.999
-  nan_flags: null
+  nan_flags: 999.999
   field_number: "35"
 number_particles_all:
   n_digits: 8

disdrodb/l0/l0a_processing.py

@@ -267,8 +267,8 @@ def remove_issue_timesteps(df, issue_dict, logger=None, verbose=False):
     n_initial_rows = len(df)
 
     # Retrieve timesteps and time_periods
-    timesteps = issue_dict.get("timesteps", None)
-    time_periods = issue_dict.get("time_periods", None)
+    timesteps = issue_dict.get("timesteps")
+    time_periods = issue_dict.get("time_periods")
     timesteps = [] if timesteps is None else timesteps
     time_periods = [] if time_periods is None else time_periods
 
@@ -615,6 +615,10 @@ def sanitize_df(
     # - Strip first and last delimiter from the raw arrays
     df = strip_delimiter_from_raw_arrays(df)
 
+    # - Ensure raw drop number variable max value is < 999
+    #   - Raise error otherwise
+    # TODO:
+
     # - Remove corrupted rows
     df = remove_corrupted_rows(df)
 

disdrodb/l0/l0b_processing.py

@@ -91,7 +91,7 @@ def format_string_array(string: str, n_values: int) -> np.array:
 
         e.g. : format_string_array("2,44,22,33", 4) will return [ 2. 44. 22. 33.]
 
-    If empty string ("") --> Return an arrays of zeros
+    If empty string ("") or "" --> Return an arrays of zeros
     If the list length is not n_values -> Return an arrays of np.nan
 
     The function strip potential delimiters at start and end before splitting.
@@ -108,31 +108,38 @@ def format_string_array(string: str, n_values: int) -> np.array:
     np.array
         array of float
     """
-    split_str = infer_split_str(string)
-    values = np.array(string.strip(split_str).split(split_str))
-
-    # -------------------------------------------------------------------------.
-    ## Assumptions !!!
-    # If empty list --> Assume no precipitation recorded. Return an arrays of zeros
-    if len(values) == 0:
+    # Check for empty string or "0" case
+    # - Assume no precipitation recorded. Return an arrays of zeros
+    if string in {"", "0"}:
         values = np.zeros(n_values)
         return values
 
-    # -------------------------------------------------------------------------.
+    # Check for NaN case
+    # - Assume no data available. Return an arrays of NaN
+    if string == "NaN":
+        values = np.zeros(n_values) * np.nan
+        return values
+
+    # Retrieve list of values
+    split_str = infer_split_str(string)
+    values = np.array(string.strip(split_str).split(split_str))
+
     # If the length is not as expected --> Assume data corruption
     # --> Return an array with nan
     if len(values) != n_values:
         values = np.zeros(n_values) * np.nan
-    else:
-        # Ensure string type
-        values = values.astype("str")
-        # Replace '' with 0
-        values = replace_empty_strings_with_zeros(values)
-        # Replace "-9.999" with 0
-        values = np.char.replace(values, "-9.999", "0")
-        # Cast values to float type
-        # --> Note: the disk encoding is specified in the l0b_encodings.yml
-        values = values.astype(float)
+        return values
+
+    # Otherwise sanitize the list of value
+    # Ensure string type
+    values = values.astype("str")
+    # Replace '' with 0
+    values = replace_empty_strings_with_zeros(values)
+    # Replace "-9.999" with 0
+    values = np.char.replace(values, "-9.999", "0")
+    # Cast values to float type
+    # --> Note: the disk encoding is specified in the l0b_encodings.yml
+    values = values.astype(float)
     return values
 
 

disdrodb/l0/l0c_processing.py

@@ -678,6 +678,16 @@ def create_l0c_datasets(
         log_info(logger=logger, msg=f"No data between {start_time} and {end_time}.", verbose=verbose)
         return {}
 
+    # If 1 or 2 timesteps per time block, return empty dictionary
+    n_timesteps = len(ds["time"])
+    if n_timesteps < 3:
+        log_info(
+            logger=logger,
+            msg=f"Only {n_timesteps} timesteps between {start_time} and {end_time}.",
+            verbose=verbose,
+        )
+        return {}
+
     # ---------------------------------------------------------------------------------------.
     # If sample interval is a dataset variable, drop timesteps with unexpected measurement intervals !
     if "sample_interval" in ds: