disdrodb 0.2.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

disdrodb/l0/configs/ODM470/l0b_encodings.yml

@@ -0,0 +1,106 @@
+precipitation_rate:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 43200
+precip_flag:
+  dtype: int8
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 43200
+  _FillValue: 127
+reference_voltage:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 43200
+weather_code_synop_4677:
+  dtype: uint8
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 255
+relative_wind_speed:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200
+relative_wind_direction:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200
+raw_drop_number:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes:
+    - 43200
+    - 128
+air_temperature:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200
+relative_humidity:
+  dtype: uint16
+  scale_factor: 0.01
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200
+wind_speed:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200
+wind_direction:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 43200

disdrodb/l0/configs/ODM470/raw_data_format.yml

@@ -0,0 +1,111 @@
+precipitation_rate:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 3
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 999.999
+  nan_flags: null
+  field_number: "1"
+reference_voltage:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 3
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 5.5
+  nan_flags: null
+  field_number: "2"
+relative_wind_speed:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 3
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 100
+  nan_flags: null
+  field_number: "3"
+relative_wind_direction:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 3
+  n_naturals: 3
+  data_range:
+    - 0.0
+    - 360
+  nan_flags: null
+  field_number: "4"
+precip_flag:
+  n_digits: 1
+  n_characters: 1
+  n_decimals: 0
+  n_naturals: 0
+  data_range:
+    - -1
+    - 5
+  nan_flags: null
+  field_number: "5"
+weather_code_synop_4677:
+  n_digits: 2
+  n_characters: 2
+  n_decimals: 0
+  n_naturals: 2
+  data_range:
+    - 0
+    - 90
+  nan_flags: null
+  field_number: "6"
+raw_drop_number:
+  n_digits: 1317
+  n_characters: 1756
+  n_decimals: 1317
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  dimension_order:
+    - diameter_bin_center
+  n_values: 128
+  field_number: "81"
+air_temperature:
+  n_digits: 4
+  n_characters: 5
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - -40
+    - 70
+  nan_flags: 99999
+  field_number: "521"
+relative_humidity:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 0
+  n_naturals: 5
+  data_range:
+    - 0
+    - 99999
+  nan_flags: 99999
+  field_number: "522"
+wind_speed:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - 0
+    - 60
+  nan_flags: null
+  field_number: "523"
+wind_direction:
+  n_digits: 3
+  n_characters: 3
+  n_decimals: 0
+  n_naturals: 3
+  data_range:
+    - 0
+    - 360
+  nan_flags: 999
+  field_number: "524"

disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml

@@ -121,7 +121,7 @@ reflectivity_16bit:
 raw_drop_concentration:
   description: Particle number concentrations per diameter class
   long_name: Raw drop concentration
-  units: 1/(m3*mm)
+  units: log10 1/(m3*mm)
 raw_drop_average_velocity:
   description: Average particle velocities for each diameter class
   long_name: Raw drop average velocity

disdrodb/l0/l0_reader.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Define DISDRODB L0 readers routines."""
+
 import inspect
 import logging
 import os

disdrodb/l0/l0a_processing.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,11 +18,9 @@
 
 import logging
 import os
-from typing import Union
 
 import numpy as np
 import pandas as pd
-import pyarrow.parquet as pq
 
 from disdrodb.l0.check_standards import check_l0a_column_names, check_l0a_standards
 from disdrodb.l0.l0b_processing import infer_split_str
@@ -771,7 +767,7 @@ def concatenate_dataframe(list_df: list, logger=None, verbose: bool = False) ->
 
 
 def read_l0a_dataframe(
-    filepaths: Union[str, list],
+    filepaths: str | list,
     debugging_mode: bool = False,
 ) -> pd.DataFrame:
     """Read DISDRODB L0A Apache Parquet file(s).
@@ -792,6 +788,8 @@ def read_l0a_dataframe(
         L0A Dataframe.
 
     """
+    from disdrodb.api.io import open_parquet_files
+
     # ----------------------------------------
     # Check filepaths validity
     if not isinstance(filepaths, (list, str)):
@@ -808,7 +806,7 @@ def read_l0a_dataframe(
 
     # ---------------------------------------------------
     # Define the list of dataframe
-    df = pq.ParquetDataset(filepaths).read().to_pandas()
+    df = open_parquet_files(filepaths, use_threads=False)
 
     # Reduce rows
     if debugging_mode:
@@ -834,7 +832,7 @@ def read_l0a_dataframe(
 
 
 def generate_l0a(
-    filepaths: Union[list, str],
+    filepaths: list | str,
     reader,
     sensor_name,
     issue_dict=None,

disdrodb/l0/l0b_nc_processing.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,7 +17,6 @@
 """Functions to process DISDRODB raw netCDF files into DISDRODB L0B netCDF files."""
 
 import logging
-from typing import Union
 
 import numpy as np
 
@@ -50,7 +47,7 @@ def _check_dict_names_validity(dict_names, sensor_name):
     keys = np.array(list(dict_names.keys()))
     values = np.array(list(dict_names.values()))
     # Get invalid keys
-    invalid_keys = keys[np.isin(values, valid_names, invert=True)]
+    invalid_keys = keys[np.isin(values, valid_names, invert=True)].tolist()
     if len(invalid_keys) > 0:
         # Report invalid keys and raise error
         invalid_dict = {k: dict_names[k] for k in invalid_keys}
@@ -536,7 +533,7 @@ def open_raw_netcdf_file(
 
 
 def generate_l0b_from_nc(
-    filepaths: Union[list, str],
+    filepaths: list | str,
     reader,
     sensor_name,
     metadata,

disdrodb/l0/l0b_processing.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -42,6 +40,7 @@ from disdrodb.utils.attrs import (
     set_coordinate_attributes,
     set_disdrodb_attrs,
 )
+from disdrodb.utils.coords import add_dataset_crs_coords
 from disdrodb.utils.encoding import set_encodings
 from disdrodb.utils.logger import log_info
 from disdrodb.utils.time import ensure_sorted_by_time
@@ -393,19 +392,6 @@ def set_variable_attributes(ds: xr.Dataset, sensor_name: str) -> xr.Dataset:
     return ds
 
 
-def add_dataset_crs_coords(ds):
-    """Add the CRS coordinate to the xr.Dataset."""
-    # TODO: define CF-compliant CRS !
-    # - CF compliant
-    # - wkt
-    # - add grid_mapping name
-    # -->
-    # attrs["EPSG"] = 4326
-    # attrs["proj4_string"] = "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
-    ds = ds.assign_coords({"crs": ["WGS84"]})
-    return ds
-
-
 ####--------------------------------------------------------------------------.
 #### L0B Raw DataFrame Preprocessing
 

disdrodb/l0/l0c_processing.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Functions to process DISDRODB L0B files into DISDRODB L0C netCDF files."""
+
 import logging
 
 import numpy as np
@@ -52,7 +51,7 @@ def drop_timesteps_with_invalid_sample_interval(ds, measurement_intervals, verbo
         # Log information for each invalid timestep
         invalid_timesteps = pd.to_datetime(timesteps[indices_invalid_sample_interval]).strftime("%Y-%m-%d %H:%M:%S")
         invalid_sample_intervals = sample_interval[indices_invalid_sample_interval]
-        for tt, ss in zip(invalid_timesteps, invalid_sample_intervals):
+        for tt, ss in zip(invalid_timesteps, invalid_sample_intervals, strict=True):
             msg = f"Unexpected sampling interval ({ss} s) at {tt}. The measurement has been dropped."
             log_warning(logger=logger, msg=msg, verbose=verbose)
         # Remove timesteps with invalid sample intervals
@@ -340,6 +339,20 @@ def get_problematic_timestep_indices(timesteps, sample_interval):
     return idx_previous_missing, idx_next_missing, idx_isolated_missing
 
 
+def nearest_expected_times(times, expected_times):
+    """Return index of nearest expected time."""
+    # both must be sorted ascending
+    idx = np.searchsorted(expected_times, times)
+    idx = np.clip(idx, 1, len(expected_times) - 1)
+
+    # Compare distance to the previous vs next expected time
+    prev = expected_times[idx - 1]
+    next_ = expected_times[idx]
+    choose_next = (times - prev) > (next_ - times)
+    nearest = np.where(choose_next, next_, prev)
+    return nearest
+
+
 def regularize_timesteps(ds, sample_interval, robust=False, add_quality_flag=True, logger=None, verbose=True):
     """Ensure timesteps match with the sample_interval.
 
@@ -364,13 +377,16 @@ def regularize_timesteps(ds, sample_interval, robust=False, add_quality_flag=Tru
     times = times.to_numpy(dtype="M8[s]")
     expected_times = expected_times.to_numpy(dtype="M8[s]")
 
+    # Vectorized mapping of observed times → nearest expected times
+    adjusted_times = nearest_expected_times(times, expected_times)
+
     # Map original times to the nearest expected times
-    # Calculate the difference between original times and expected times
-    time_deltas = np.abs(times - expected_times[:, None]).astype(int)
+    # # Calculate the difference between original times and expected times
+    # time_deltas = np.abs(times - expected_times[:, None]).astype(int)
 
-    # Find the index of the closest expected time for each original time
-    nearest_indices = np.argmin(time_deltas, axis=0)
-    adjusted_times = expected_times[nearest_indices]
+    # # Find the index of the closest expected time for each original time
+    # nearest_indices = np.argmin(time_deltas, axis=0)
+    # adjusted_times = expected_times[nearest_indices]
 
     # Check for duplicates in adjusted times
     unique_times, counts = np.unique(adjusted_times, return_counts=True)
@@ -466,10 +482,10 @@ def regularize_timesteps(ds, sample_interval, robust=False, add_quality_flag=Tru
         #     qc_flag[-1] = 0
 
         # Add time quality flag variable
-        ds["time_qc"] = xr.DataArray(qc_flag, dims="time")
+        ds["qc_time"] = xr.DataArray(qc_flag, dims="time")
 
-        # Add CF attributes for time_qc
-        ds["time_qc"].attrs = {
+        # Add CF attributes for qc_time
+        ds["qc_time"].attrs = {
             "long_name": "time quality flag",
             "standard_name": "status_flag",
             "units": "1",
@@ -587,6 +603,7 @@ def _finalize_l0c_dataset(ds, sample_interval, sensor_name, verbose=True, logger
     ds = add_sample_interval(ds, sample_interval=sample_interval)
 
     # Regularize timesteps (for trailing seconds)
+    # --> This remove time encoding
     ds = regularize_timesteps(
         ds,
         sample_interval=sample_interval,

disdrodb/l0/readers/LPM/ARM/ARM_LPM.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Reader for ARM Thies LPM sensor."""
+
 from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
 from disdrodb.l0.l0b_nc_processing import open_raw_netcdf_file, replace_custom_nan_flags, standardize_raw_dataset
 

disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -176,23 +176,23 @@ def reader(
         "number_particles_no_hydrometeor",
         "number_particles_no_hydrometeor_internal_data",
         "number_particles_unknown_classification",
-        "number_particles_unknown_classification_internal_data",
-        "number_particles_class_1",
-        "number_particles_class_1_internal_data",
-        "number_particles_class_2",
-        "number_particles_class_2_internal_data",
-        "number_particles_class_3",
-        "number_particles_class_3_internal_data",
-        "number_particles_class_4",
-        "number_particles_class_4_internal_data",
-        "number_particles_class_5",
-        "number_particles_class_5_internal_data",
-        "number_particles_class_6",
-        "number_particles_class_6_internal_data",
-        "number_particles_class_7",
-        "number_particles_class_7_internal_data",
-        "number_particles_class_8",
-        "number_particles_class_8_internal_data",
+        "total_gross_volume_unknown_classification",
+        "number_particles_hail",
+        "total_gross_volume_hail",
+        "number_particles_solid_precipitation",
+        "total_gross_volume_solid_precipitation",
+        "number_particles_large_pellet",
+        "total_gross_volume_large_pellet",
+        "number_particles_small_pellet",
+        "total_gross_volume_small_pellet",
+        "number_particles_snowgrain",
+        "total_gross_volume_snowgrain",
+        "number_particles_rain",
+        "total_gross_volume_rain",
+        "number_particles_small_rain",
+        "total_gross_volume_small_rain",
+        "number_particles_drizzle",
+        "total_gross_volume_drizzle",
         "number_particles_class_9",
         "number_particles_class_9_internal_data",
         "TO_BE_FURTHER_PROCESSED",

disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -135,23 +135,23 @@ def reader(
         "number_particles_no_hydrometeor",
         "number_particles_no_hydrometeor_internal_data",
         "number_particles_unknown_classification",
-        "number_particles_unknown_classification_internal_data",
-        "number_particles_class_1",
-        "number_particles_class_1_internal_data",
-        "number_particles_class_2",
-        "number_particles_class_2_internal_data",
-        "number_particles_class_3",
-        "number_particles_class_3_internal_data",
-        "number_particles_class_4",
-        "number_particles_class_4_internal_data",
-        "number_particles_class_5",
-        "number_particles_class_5_internal_data",
-        "number_particles_class_6",
-        "number_particles_class_6_internal_data",
-        "number_particles_class_7",
-        "number_particles_class_7_internal_data",
-        "number_particles_class_8",
-        "number_particles_class_8_internal_data",
+        "total_gross_volume_unknown_classification",
+        "number_particles_hail",
+        "total_gross_volume_hail",
+        "number_particles_solid_precipitation",
+        "total_gross_volume_solid_precipitation",
+        "number_particles_large_pellet",
+        "total_gross_volume_large_pellet",
+        "number_particles_small_pellet",
+        "total_gross_volume_small_pellet",
+        "number_particles_snowgrain",
+        "total_gross_volume_snowgrain",
+        "number_particles_rain",
+        "total_gross_volume_rain",
+        "number_particles_small_rain",
+        "total_gross_volume_small_rain",
+        "number_particles_drizzle",
+        "total_gross_volume_drizzle",
         "number_particles_class_9",
         "number_particles_class_9_internal_data",
         "raw_drop_number",

disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -135,23 +135,23 @@ def reader(
         "number_particles_no_hydrometeor",
         "number_particles_no_hydrometeor_internal_data",
         "number_particles_unknown_classification",
-        "number_particles_unknown_classification_internal_data",
-        "number_particles_class_1",
-        "number_particles_class_1_internal_data",
-        "number_particles_class_2",
-        "number_particles_class_2_internal_data",
-        "number_particles_class_3",
-        "number_particles_class_3_internal_data",
-        "number_particles_class_4",
-        "number_particles_class_4_internal_data",
-        "number_particles_class_5",
-        "number_particles_class_5_internal_data",
-        "number_particles_class_6",
-        "number_particles_class_6_internal_data",
-        "number_particles_class_7",
-        "number_particles_class_7_internal_data",
-        "number_particles_class_8",
-        "number_particles_class_8_internal_data",
+        "total_gross_volume_unknown_classification",
+        "number_particles_hail",
+        "total_gross_volume_hail",
+        "number_particles_solid_precipitation",
+        "total_gross_volume_solid_precipitation",
+        "number_particles_large_pellet",
+        "total_gross_volume_large_pellet",
+        "number_particles_small_pellet",
+        "total_gross_volume_small_pellet",
+        "number_particles_snowgrain",
+        "total_gross_volume_snowgrain",
+        "number_particles_rain",
+        "total_gross_volume_rain",
+        "number_particles_small_rain",
+        "total_gross_volume_small_rain",
+        "number_particles_drizzle",
+        "total_gross_volume_drizzle",
         "number_particles_class_9",
         "number_particles_class_9_internal_data",
         "raw_drop_number",