disdrodb 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

disdrodb/l0/configs/ODM470/l0b_encodings.yml

@@ -0,0 +1,106 @@
+precipitation_rate:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 43200
+precip_flag:
+  dtype: int8
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 43200
+  _FillValue: 127
+reference_voltage:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 43200
+weather_code_synop_4677:
+  dtype: uint8
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 255
+relative_wind_speed:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200
+relative_wind_direction:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200
+raw_drop_number:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes:
+    - 43200
+    - 128
+air_temperature:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200
+relative_humidity:
+  dtype: uint16
+  scale_factor: 0.01
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200
+wind_speed:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200
+wind_direction:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200

disdrodb/l0/configs/ODM470/raw_data_format.yml

@@ -0,0 +1,111 @@
+precipitation_rate:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 3
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 999.999
+  nan_flags: null
+  field_number: "1"
+reference_voltage:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 3
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 5.5
+  nan_flags: null
+  field_number: "2"
+relative_wind_speed:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 3
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 100
+  nan_flags: null
+  field_number: "3"
+relative_wind_direction:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 3
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 360
+  nan_flags: null
+  field_number: "4"
+precip_flag:
+  n_digits: 1
+  n_characters: 1
+  n_decimals: 0
+  n_naturals: 0
+  data_range:
+    - -1
+    - 5
+  nan_flags: null
+  field_number: "5"
+weather_code_synop_4677:
+  n_digits: 2
+  n_characters: 2
+  n_decimals: 0
+  n_naturals: 2
+  data_range:
+    - 0
+    - 90
+  nan_flags: null
+  field_number: "6"
+raw_drop_number:
+  n_digits: 1317
+  n_characters: 1756
+  n_decimals: 1317
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  dimension_order:
+    - diameter_bin_center
+  n_values: 128
+  field_number: "81"
+air_temperature:
+  n_digits: 4
+  n_characters: 5
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - -40
+    - 70
+  nan_flags: 99999
+  field_number: "521"
+relative_humidity:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 0
+  n_naturals: 5
+  data_range:
+    - 0
+    - 99999
+  nan_flags: 99999
+  field_number: "522"
+wind_speed:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - 0
+    - 60
+  nan_flags: null
+  field_number: "523"
+wind_direction:
+  n_digits: 3
+  n_characters: 3
+  n_decimals: 0
+  n_naturals: 3
+  data_range:
+    - 0
+    - 360
+  nan_flags: 999
+  field_number: "524"

disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml

@@ -121,7 +121,7 @@ reflectivity_16bit:
 raw_drop_concentration:
   description: Particle number concentrations per diameter class
   long_name: Raw drop concentration
-  units: 1/(m3*mm)
+  units: log10 1/(m3*mm)
 raw_drop_average_velocity:
   description: Average particle velocities for each diameter class
   long_name: Raw drop average velocity

disdrodb/l0/configs/PARSIVEL/raw_data_format.yml

@@ -6,7 +6,7 @@ rainfall_rate_32bit:
   data_range:
     - 0
     - 9999.999
-  nan_flags: null
+  nan_flags: 9999.999
   field_number: "01"
 rainfall_accumulated_32bit:
   n_digits: 6
@@ -16,7 +16,7 @@ rainfall_accumulated_32bit:
   data_range:
     - 0
     - 9999.0
-  nan_flags: null
+  nan_flags: 9999.0
   field_number: "02"
 weather_code_synop_4680:
   n_digits: 2
@@ -62,7 +62,7 @@ reflectivity_32bit:
   data_range:
     - -9.999
     - 99.999
-  nan_flags: null
+  nan_flags: 99.999
   field_number: "07"
 mor_visibility:
   n_digits: 4
@@ -92,7 +92,7 @@ laser_amplitude:
   data_range:
     - 0
     - 99999
-  nan_flags: null
+  nan_flags: 99999
   field_number: "10"
 number_particles:
   n_digits: 5
@@ -102,7 +102,7 @@ number_particles:
   data_range:
     - 0
     - 99999
-  nan_flags: null
+  nan_flags: 99999
   field_number: "11"
 sensor_temperature:
   n_digits: 3
@@ -221,7 +221,7 @@ rainfall_amount_absolute_32bit:
   data_range:
     - 0
     - 999.999
-  nan_flags: null
+  nan_flags: 999.999
   field_number: "24"
 error_code:
   n_digits: 3
@@ -245,7 +245,7 @@ rainfall_rate_16bit:
   data_range:
     - 0
     - 9999.999
-  nan_flags: null
+  nan_flags: 9999.999
   field_number: "30"
 rainfall_rate_12bit:
   n_digits: 5
@@ -273,7 +273,7 @@ reflectivity_16bit:
   data_range:
     - -9.99
     - 99.99
-  nan_flags: null
+  nan_flags: 99.99
   field_number: "33"
 raw_drop_concentration:
   n_digits: 0

disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml

@@ -6,7 +6,7 @@ rainfall_rate_32bit:
   data_range:
     - 0
     - 9999.999
-  nan_flags: null
+  nan_flags: 9999.999
   field_number: "01"
 rainfall_accumulated_32bit:
   n_digits: 6
@@ -16,7 +16,7 @@ rainfall_accumulated_32bit:
   data_range:
     - 0
     - 9999.0
-  nan_flags: null
+  nan_flags: 9999.0
   field_number: "02"
 weather_code_synop_4680:
   n_digits: 2
@@ -62,7 +62,7 @@ reflectivity_32bit:
   data_range:
     - -9.999
     - 99.999
-  nan_flags: null
+  nan_flags: 99.999
   field_number: "07"
 mor_visibility:
   n_digits: 5
@@ -92,7 +92,7 @@ laser_amplitude:
   data_range:
     - 0
     - 99999
-  nan_flags: null
+  nan_flags: 99999
   field_number: "10"
 number_particles:
   n_digits: 5
@@ -102,7 +102,7 @@ number_particles:
   data_range:
     - 0
     - 99999
-  nan_flags: null
+  nan_flags: 99999
   field_number: "11"
 sensor_temperature:
   n_digits: 3
@@ -221,7 +221,7 @@ rainfall_amount_absolute_32bit:
   data_range:
     - 0
     - 999.999
-  nan_flags: null
+  nan_flags: 999.999
   field_number: "24"
 error_code:
   n_digits: 3
@@ -305,7 +305,7 @@ reflectivity_16bit:
   data_range:
     - -9.99
     - 99.99
-  nan_flags: null
+  nan_flags: 99.99
   field_number: "33"
 rain_kinetic_energy:
   n_digits: 6
@@ -315,7 +315,7 @@ rain_kinetic_energy:
   data_range:
     - 0
     - 9999.99
-  nan_flags: null
+  nan_flags: 9999.99
   field_number: "34"
 snowfall_rate:
   n_digits: 6
@@ -325,7 +325,7 @@ snowfall_rate:
   data_range:
     - 0
     - 999.999
-  nan_flags: null
+  nan_flags: 999.999
   field_number: "35"
 number_particles_all:
   n_digits: 8

disdrodb/l0/l0_reader.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

disdrodb/l0/l0a_processing.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -267,8 +265,8 @@ def remove_issue_timesteps(df, issue_dict, logger=None, verbose=False):
     n_initial_rows = len(df)
 
     # Retrieve timesteps and time_periods
-    timesteps = issue_dict.get("timesteps", None)
-    time_periods = issue_dict.get("time_periods", None)
+    timesteps = issue_dict.get("timesteps")
+    time_periods = issue_dict.get("time_periods")
     timesteps = [] if timesteps is None else timesteps
     time_periods = [] if time_periods is None else time_periods
 
@@ -615,6 +613,10 @@ def sanitize_df(
     # - Strip first and last delimiter from the raw arrays
     df = strip_delimiter_from_raw_arrays(df)
 
+    # - Ensure raw drop number variable max value is < 999
+    #   - Raise error otherwise
+    # TODO:
+
     # - Remove corrupted rows
     df = remove_corrupted_rows(df)
 

disdrodb/l0/l0b_nc_processing.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -50,7 +48,7 @@ def _check_dict_names_validity(dict_names, sensor_name):
     keys = np.array(list(dict_names.keys()))
     values = np.array(list(dict_names.values()))
     # Get invalid keys
-    invalid_keys = keys[np.isin(values, valid_names, invert=True)]
+    invalid_keys = keys[np.isin(values, valid_names, invert=True)].tolist()
     if len(invalid_keys) > 0:
         # Report invalid keys and raise error
         invalid_dict = {k: dict_names[k] for k in invalid_keys}

disdrodb/l0/l0b_processing.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -91,7 +89,7 @@ def format_string_array(string: str, n_values: int) -> np.array:
 
         e.g. : format_string_array("2,44,22,33", 4) will return [ 2. 44. 22. 33.]
 
-    If empty string ("") --> Return an arrays of zeros
+    If empty string ("") or "" --> Return an arrays of zeros
     If the list length is not n_values -> Return an arrays of np.nan
 
     The function strip potential delimiters at start and end before splitting.
@@ -108,31 +106,38 @@ def format_string_array(string: str, n_values: int) -> np.array:
     np.array
         array of float
     """
-    split_str = infer_split_str(string)
-    values = np.array(string.strip(split_str).split(split_str))
-
-    # -------------------------------------------------------------------------.
-    ## Assumptions !!!
-    # If empty list --> Assume no precipitation recorded. Return an arrays of zeros
-    if len(values) == 0:
+    # Check for empty string or "0" case
+    # - Assume no precipitation recorded. Return an arrays of zeros
+    if string in {"", "0"}:
         values = np.zeros(n_values)
         return values
 
-    # -------------------------------------------------------------------------.
+    # Check for NaN case
+    # - Assume no data available. Return an arrays of NaN
+    if string == "NaN":
+        values = np.zeros(n_values) * np.nan
+        return values
+
+    # Retrieve list of values
+    split_str = infer_split_str(string)
+    values = np.array(string.strip(split_str).split(split_str))
+
     # If the length is not as expected --> Assume data corruption
     # --> Return an array with nan
     if len(values) != n_values:
         values = np.zeros(n_values) * np.nan
-    else:
-        # Ensure string type
-        values = values.astype("str")
-        # Replace '' with 0
-        values = replace_empty_strings_with_zeros(values)
-        # Replace "-9.999" with 0
-        values = np.char.replace(values, "-9.999", "0")
-        # Cast values to float type
-        # --> Note: the disk encoding is specified in the l0b_encodings.yml
-        values = values.astype(float)
+        return values
+
+    # Otherwise sanitize the list of value
+    # Ensure string type
+    values = values.astype("str")
+    # Replace '' with 0
+    values = replace_empty_strings_with_zeros(values)
+    # Replace "-9.999" with 0
+    values = np.char.replace(values, "-9.999", "0")
+    # Cast values to float type
+    # --> Note: the disk encoding is specified in the l0b_encodings.yml
+    values = values.astype(float)
     return values
 
 

disdrodb/l0/l0c_processing.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -340,6 +338,20 @@ def get_problematic_timestep_indices(timesteps, sample_interval):
     return idx_previous_missing, idx_next_missing, idx_isolated_missing
 
 
+def nearest_expected_times(times, expected_times):
+    """Return index of nearest expected time."""
+    # both must be sorted ascending
+    idx = np.searchsorted(expected_times, times)
+    idx = np.clip(idx, 1, len(expected_times) - 1)
+
+    # Compare distance to the previous vs next expected time
+    prev = expected_times[idx - 1]
+    next_ = expected_times[idx]
+    choose_next = (times - prev) > (next_ - times)
+    nearest = np.where(choose_next, next_, prev)
+    return nearest
+
+
 def regularize_timesteps(ds, sample_interval, robust=False, add_quality_flag=True, logger=None, verbose=True):
     """Ensure timesteps match with the sample_interval.
 
@@ -364,13 +376,16 @@ def regularize_timesteps(ds, sample_interval, robust=False, add_quality_flag=Tru
     times = times.to_numpy(dtype="M8[s]")
     expected_times = expected_times.to_numpy(dtype="M8[s]")
 
+    # Vectorized mapping of observed times → nearest expected times
+    adjusted_times = nearest_expected_times(times, expected_times)
+
     # Map original times to the nearest expected times
-    # Calculate the difference between original times and expected times
-    time_deltas = np.abs(times - expected_times[:, None]).astype(int)
+    # # Calculate the difference between original times and expected times
+    # time_deltas = np.abs(times - expected_times[:, None]).astype(int)
 
-    # Find the index of the closest expected time for each original time
-    nearest_indices = np.argmin(time_deltas, axis=0)
-    adjusted_times = expected_times[nearest_indices]
+    # # Find the index of the closest expected time for each original time
+    # nearest_indices = np.argmin(time_deltas, axis=0)
+    # adjusted_times = expected_times[nearest_indices]
 
     # Check for duplicates in adjusted times
     unique_times, counts = np.unique(adjusted_times, return_counts=True)
@@ -466,10 +481,10 @@ def regularize_timesteps(ds, sample_interval, robust=False, add_quality_flag=Tru
         #     qc_flag[-1] = 0
 
         # Add time quality flag variable
-        ds["time_qc"] = xr.DataArray(qc_flag, dims="time")
+        ds["qc_time"] = xr.DataArray(qc_flag, dims="time")
 
-        # Add CF attributes for time_qc
-        ds["time_qc"].attrs = {
+        # Add CF attributes for qc_time
+        ds["qc_time"].attrs = {
             "long_name": "time quality flag",
             "standard_name": "status_flag",
             "units": "1",
@@ -587,6 +602,7 @@ def _finalize_l0c_dataset(ds, sample_interval, sensor_name, verbose=True, logger
     ds = add_sample_interval(ds, sample_interval=sample_interval)
 
     # Regularize timesteps (for trailing seconds)
+    # --> This remove time encoding
     ds = regularize_timesteps(
         ds,
         sample_interval=sample_interval,
@@ -678,6 +694,16 @@ def create_l0c_datasets(
         log_info(logger=logger, msg=f"No data between {start_time} and {end_time}.", verbose=verbose)
         return {}
 
+    # If 1 or 2 timesteps per time block, return empty dictionary
+    n_timesteps = len(ds["time"])
+    if n_timesteps < 3:
+        log_info(
+            logger=logger,
+            msg=f"Only {n_timesteps} timesteps between {start_time} and {end_time}.",
+            verbose=verbose,
+        )
+        return {}
+
     # ---------------------------------------------------------------------------------------.
     # If sample interval is a dataset variable, drop timesteps with unexpected measurement intervals !
     if "sample_interval" in ds:

disdrodb/l0/readers/LPM/ARM/ARM_LPM.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by