disdrodb 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

disdrodb/l0/configs/LPM_V0/l0b_cf_attrs.yml

@@ -110,11 +110,11 @@ total_gross_volume_solid_precipitation:
   description: Total (gross) volume of particles classified as solid precipitation
   long_name: Total (gross) volume of particles classified as solid precipitation
   units: ""
-number_particles_great_pellet:
+number_particles_large_pellet:
   description: Number of particles classified as great pellet
   long_name: Number of particles classified as great pellet
   units: ""
-total_gross_volume_great_pellet:
+total_gross_volume_large_pellet:
   description: Total (gross) volume of particles classified as great pellet
   long_name: Total (gross) volume of particles classified as great pellet
   units: ""

disdrodb/l0/configs/LPM_V0/l0b_encodings.yml

@@ -253,7 +253,7 @@ total_gross_volume_solid_precipitation:
   fletcher32: false
   contiguous: false
   chunksizes: 5000
-number_particles_great_pellet:
+number_particles_large_pellet:
   dtype: uint16
   zlib: true
   complevel: 3
@@ -262,7 +262,7 @@ number_particles_great_pellet:
   contiguous: false
   chunksizes: 5000
   _FillValue: 65535
-total_gross_volume_great_pellet:
+total_gross_volume_large_pellet:
   dtype: float32
   zlib: true
   complevel: 3

disdrodb/l0/configs/LPM_V0/raw_data_format.yml

@@ -300,7 +300,7 @@ total_gross_volume_solid_precipitation:
     - 99999.999
   nan_flags: null
   field_number: "31"
-number_particles_great_pellet:
+number_particles_large_pellet:
   n_digits: 5
   n_characters: 5
   n_decimals: 5
@@ -310,7 +310,7 @@ number_particles_great_pellet:
     - 99999
   nan_flags: null
   field_number: "32"
-total_gross_volume_great_pellet:
+total_gross_volume_large_pellet:
   n_digits: 8
   n_characters: 9
   n_decimals: 5

disdrodb/l0/l0_reader.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Define DISDRODB L0 readers routines."""
+
 import inspect
 import logging
 import os

disdrodb/l0/l0a_processing.py

@@ -18,11 +18,9 @@
 
 import logging
 import os
-from typing import Union
 
 import numpy as np
 import pandas as pd
-import pyarrow.parquet as pq
 
 from disdrodb.l0.check_standards import check_l0a_column_names, check_l0a_standards
 from disdrodb.l0.l0b_processing import infer_split_str
@@ -769,7 +767,7 @@ def concatenate_dataframe(list_df: list, logger=None, verbose: bool = False) ->
 
 
 def read_l0a_dataframe(
-    filepaths: Union[str, list],
+    filepaths: str | list,
     debugging_mode: bool = False,
 ) -> pd.DataFrame:
     """Read DISDRODB L0A Apache Parquet file(s).
@@ -790,6 +788,8 @@ def read_l0a_dataframe(
         L0A Dataframe.
 
     """
+    from disdrodb.api.io import open_parquet_files
+
     # ----------------------------------------
     # Check filepaths validity
     if not isinstance(filepaths, (list, str)):
@@ -806,7 +806,7 @@ def read_l0a_dataframe(
 
     # ---------------------------------------------------
     # Define the list of dataframe
-    df = pq.ParquetDataset(filepaths).read().to_pandas()
+    df = open_parquet_files(filepaths, use_threads=False)
 
     # Reduce rows
     if debugging_mode:
@@ -832,7 +832,7 @@ def read_l0a_dataframe(
 
 
 def generate_l0a(
-    filepaths: Union[list, str],
+    filepaths: list | str,
     reader,
     sensor_name,
     issue_dict=None,

disdrodb/l0/l0b_nc_processing.py

@@ -17,7 +17,6 @@
 """Functions to process DISDRODB raw netCDF files into DISDRODB L0B netCDF files."""
 
 import logging
-from typing import Union
 
 import numpy as np
 
@@ -534,7 +533,7 @@ def open_raw_netcdf_file(
 
 
 def generate_l0b_from_nc(
-    filepaths: Union[list, str],
+    filepaths: list | str,
     reader,
     sensor_name,
     metadata,

disdrodb/l0/l0b_processing.py

@@ -40,6 +40,7 @@ from disdrodb.utils.attrs import (
     set_coordinate_attributes,
     set_disdrodb_attrs,
 )
+from disdrodb.utils.coords import add_dataset_crs_coords
 from disdrodb.utils.encoding import set_encodings
 from disdrodb.utils.logger import log_info
 from disdrodb.utils.time import ensure_sorted_by_time
@@ -391,19 +392,6 @@ def set_variable_attributes(ds: xr.Dataset, sensor_name: str) -> xr.Dataset:
     return ds
 
 
-def add_dataset_crs_coords(ds):
-    """Add the CRS coordinate to the xr.Dataset."""
-    # TODO: define CF-compliant CRS !
-    # - CF compliant
-    # - wkt
-    # - add grid_mapping name
-    # -->
-    # attrs["EPSG"] = 4326
-    # attrs["proj4_string"] = "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
-    ds = ds.assign_coords({"crs": ["WGS84"]})
-    return ds
-
-
 ####--------------------------------------------------------------------------.
 #### L0B Raw DataFrame Preprocessing
 

disdrodb/l0/l0c_processing.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Functions to process DISDRODB L0B files into DISDRODB L0C netCDF files."""
+
 import logging
 
 import numpy as np
@@ -50,7 +51,7 @@ def drop_timesteps_with_invalid_sample_interval(ds, measurement_intervals, verbo
         # Log information for each invalid timestep
         invalid_timesteps = pd.to_datetime(timesteps[indices_invalid_sample_interval]).strftime("%Y-%m-%d %H:%M:%S")
         invalid_sample_intervals = sample_interval[indices_invalid_sample_interval]
-        for tt, ss in zip(invalid_timesteps, invalid_sample_intervals):
+        for tt, ss in zip(invalid_timesteps, invalid_sample_intervals, strict=True):
             msg = f"Unexpected sampling interval ({ss} s) at {tt}. The measurement has been dropped."
             log_warning(logger=logger, msg=msg, verbose=verbose)
         # Remove timesteps with invalid sample intervals

disdrodb/l0/readers/LPM/ARM/ARM_LPM.py

@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Reader for ARM Thies LPM sensor."""
+
 from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
 from disdrodb.l0.l0b_nc_processing import open_raw_netcdf_file, replace_custom_nan_flags, standardize_raw_dataset
 

disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py

@@ -176,23 +176,23 @@ def reader(
         "number_particles_no_hydrometeor",
         "number_particles_no_hydrometeor_internal_data",
         "number_particles_unknown_classification",
-        "number_particles_unknown_classification_internal_data",
-        "number_particles_class_1",
-        "number_particles_class_1_internal_data",
-        "number_particles_class_2",
-        "number_particles_class_2_internal_data",
-        "number_particles_class_3",
-        "number_particles_class_3_internal_data",
-        "number_particles_class_4",
-        "number_particles_class_4_internal_data",
-        "number_particles_class_5",
-        "number_particles_class_5_internal_data",
-        "number_particles_class_6",
-        "number_particles_class_6_internal_data",
-        "number_particles_class_7",
-        "number_particles_class_7_internal_data",
-        "number_particles_class_8",
-        "number_particles_class_8_internal_data",
+        "total_gross_volume_unknown_classification",
+        "number_particles_hail",
+        "total_gross_volume_hail",
+        "number_particles_solid_precipitation",
+        "total_gross_volume_solid_precipitation",
+        "number_particles_large_pellet",
+        "total_gross_volume_large_pellet",
+        "number_particles_small_pellet",
+        "total_gross_volume_small_pellet",
+        "number_particles_snowgrain",
+        "total_gross_volume_snowgrain",
+        "number_particles_rain",
+        "total_gross_volume_rain",
+        "number_particles_small_rain",
+        "total_gross_volume_small_rain",
+        "number_particles_drizzle",
+        "total_gross_volume_drizzle",
         "number_particles_class_9",
         "number_particles_class_9_internal_data",
         "TO_BE_FURTHER_PROCESSED",

disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py

@@ -135,23 +135,23 @@ def reader(
         "number_particles_no_hydrometeor",
         "number_particles_no_hydrometeor_internal_data",
         "number_particles_unknown_classification",
-        "number_particles_unknown_classification_internal_data",
-        "number_particles_class_1",
-        "number_particles_class_1_internal_data",
-        "number_particles_class_2",
-        "number_particles_class_2_internal_data",
-        "number_particles_class_3",
-        "number_particles_class_3_internal_data",
-        "number_particles_class_4",
-        "number_particles_class_4_internal_data",
-        "number_particles_class_5",
-        "number_particles_class_5_internal_data",
-        "number_particles_class_6",
-        "number_particles_class_6_internal_data",
-        "number_particles_class_7",
-        "number_particles_class_7_internal_data",
-        "number_particles_class_8",
-        "number_particles_class_8_internal_data",
+        "total_gross_volume_unknown_classification",
+        "number_particles_hail",
+        "total_gross_volume_hail",
+        "number_particles_solid_precipitation",
+        "total_gross_volume_solid_precipitation",
+        "number_particles_large_pellet",
+        "total_gross_volume_large_pellet",
+        "number_particles_small_pellet",
+        "total_gross_volume_small_pellet",
+        "number_particles_snowgrain",
+        "total_gross_volume_snowgrain",
+        "number_particles_rain",
+        "total_gross_volume_rain",
+        "number_particles_small_rain",
+        "total_gross_volume_small_rain",
+        "number_particles_drizzle",
+        "total_gross_volume_drizzle",
         "number_particles_class_9",
         "number_particles_class_9_internal_data",
         "raw_drop_number",

disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py

@@ -135,23 +135,23 @@ def reader(
         "number_particles_no_hydrometeor",
         "number_particles_no_hydrometeor_internal_data",
         "number_particles_unknown_classification",
-        "number_particles_unknown_classification_internal_data",
-        "number_particles_class_1",
-        "number_particles_class_1_internal_data",
-        "number_particles_class_2",
-        "number_particles_class_2_internal_data",
-        "number_particles_class_3",
-        "number_particles_class_3_internal_data",
-        "number_particles_class_4",
-        "number_particles_class_4_internal_data",
-        "number_particles_class_5",
-        "number_particles_class_5_internal_data",
-        "number_particles_class_6",
-        "number_particles_class_6_internal_data",
-        "number_particles_class_7",
-        "number_particles_class_7_internal_data",
-        "number_particles_class_8",
-        "number_particles_class_8_internal_data",
+        "total_gross_volume_unknown_classification",
+        "number_particles_hail",
+        "total_gross_volume_hail",
+        "number_particles_solid_precipitation",
+        "total_gross_volume_solid_precipitation",
+        "number_particles_large_pellet",
+        "total_gross_volume_large_pellet",
+        "number_particles_small_pellet",
+        "total_gross_volume_small_pellet",
+        "number_particles_snowgrain",
+        "total_gross_volume_snowgrain",
+        "number_particles_rain",
+        "total_gross_volume_rain",
+        "number_particles_small_rain",
+        "total_gross_volume_small_rain",
+        "number_particles_drizzle",
+        "total_gross_volume_drizzle",
         "number_particles_class_9",
         "number_particles_class_9_internal_data",
         "raw_drop_number",

disdrodb/l0/readers/LPM/GERMANY/DWD.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """DISDRODB reader for DWD stations."""
+
 import glob
 import os
 from pathlib import Path
@@ -83,23 +84,23 @@ COLUMNS = [
     "number_particles_no_hydrometeor",
     "number_particles_no_hydrometeor_internal_data",
     "number_particles_unknown_classification",
-    "number_particles_unknown_classification_internal_data",
-    "number_particles_class_1",
-    "number_particles_class_1_internal_data",
-    "number_particles_class_2",
-    "number_particles_class_2_internal_data",
-    "number_particles_class_3",
-    "number_particles_class_3_internal_data",
-    "number_particles_class_4",
-    "number_particles_class_4_internal_data",
-    "number_particles_class_5",
-    "number_particles_class_5_internal_data",
-    "number_particles_class_6",
-    "number_particles_class_6_internal_data",
-    "number_particles_class_7",
-    "number_particles_class_7_internal_data",
-    "number_particles_class_8",
-    "number_particles_class_8_internal_data",
+    "total_gross_volume_unknown_classification",
+    "number_particles_hail",
+    "total_gross_volume_hail",
+    "number_particles_solid_precipitation",
+    "total_gross_volume_solid_precipitation",
+    "number_particles_large_pellet",
+    "total_gross_volume_large_pellet",
+    "number_particles_small_pellet",
+    "total_gross_volume_small_pellet",
+    "number_particles_snowgrain",
+    "total_gross_volume_snowgrain",
+    "number_particles_rain",
+    "total_gross_volume_rain",
+    "number_particles_small_rain",
+    "total_gross_volume_small_rain",
+    "number_particles_drizzle",
+    "total_gross_volume_drizzle",
     "number_particles_class_9",
     "number_particles_class_9_internal_data",
     "raw_drop_number",
@@ -178,7 +179,9 @@ def retrieve_synop_filepaths(df, filepath):
         ymd = d.strftime("%Y%m%d")
         fname_pattern = f"synop10min_{station_id}_{ymd}*1.0days.dat"
         glob_pattern = os.path.join(synop_base_dir, y, m, fname_pattern)
-        synop_filepaths.append(*glob.glob(glob_pattern))
+        files = glob.glob(glob_pattern)
+        if len(files) >= 1:
+            synop_filepaths.append(*files)
     return synop_filepaths
 
 
@@ -416,23 +419,23 @@ def parse_format_v2(df):
         "number_particles_no_hydrometeor",
         "number_particles_no_hydrometeor_internal_data",
         "number_particles_unknown_classification",
-        "number_particles_unknown_classification_internal_data",
-        "number_particles_class_1",
-        "number_particles_class_1_internal_data",
-        "number_particles_class_2",
-        "number_particles_class_2_internal_data",
-        "number_particles_class_3",
-        "number_particles_class_3_internal_data",
-        "number_particles_class_4",
-        "number_particles_class_4_internal_data",
-        "number_particles_class_5",
-        "number_particles_class_5_internal_data",
-        "number_particles_class_6",
-        "number_particles_class_6_internal_data",
-        "number_particles_class_7",
-        "number_particles_class_7_internal_data",
-        "number_particles_class_8",
-        "number_particles_class_8_internal_data",
+        "total_gross_volume_unknown_classification",
+        "number_particles_hail",
+        "total_gross_volume_hail",
+        "number_particles_solid_precipitation",
+        "total_gross_volume_solid_precipitation",
+        "number_particles_large_pellet",
+        "total_gross_volume_large_pellet",
+        "number_particles_small_pellet",
+        "total_gross_volume_small_pellet",
+        "number_particles_snowgrain",
+        "total_gross_volume_snowgrain",
+        "number_particles_rain",
+        "total_gross_volume_rain",
+        "number_particles_small_rain",
+        "total_gross_volume_small_rain",
+        "number_particles_drizzle",
+        "total_gross_volume_drizzle",
         "number_particles_class_9",
         "number_particles_class_9_internal_data",
         "raw_drop_number",
@@ -536,23 +539,23 @@ def parse_format_v3(df):
         # "number_particles_no_hydrometeor",
         # "number_particles_no_hydrometeor_internal_data",
         # "number_particles_unknown_classification",
-        # "number_particles_unknown_classification_internal_data",
-        # "number_particles_class_1",
-        # "number_particles_class_1_internal_data",
-        # "number_particles_class_2",
-        # "number_particles_class_2_internal_data",
-        # "number_particles_class_3",
-        # "number_particles_class_3_internal_data",
-        # "number_particles_class_4",
-        # "number_particles_class_4_internal_data",
-        # "number_particles_class_5",
-        # "number_particles_class_5_internal_data",
-        # "number_particles_class_6",
-        # "number_particles_class_6_internal_data",
-        # "number_particles_class_7",
-        # "number_particles_class_7_internal_data",
-        # "number_particles_class_8",
-        # "number_particles_class_8_internal_data",
+        # "total_gross_volume_unknown_classification",
+        # "number_particles_hail",
+        # "total_gross_volume_hail",
+        # "number_particles_solid_precipitation",
+        # "total_gross_volume_solid_precipitation",
+        # "number_particles_large_pellet",
+        # "total_gross_volume_large_pellet",
+        # "number_particles_small_pellet",
+        # "total_gross_volume_small_pellet",
+        # "number_particles_snowgrain",
+        # "total_gross_volume_snowgrain",
+        # "number_particles_rain",
+        # "total_gross_volume_rain",
+        # "number_particles_small_rain",
+        # "total_gross_volume_small_rain",
+        # "number_particles_drizzle",
+        # "total_gross_volume_drizzle",
         # "number_particles_class_9",
         # "number_particles_class_9_internal_data",
         "raw_drop_number",

disdrodb/l0/readers/LPM/ITALY/GID_LPM.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """DISDRODB reader for GID LPM sensors not measuring wind."""
+
 import os
 
 import numpy as np
@@ -160,23 +161,23 @@ def reader(
         "number_particles_no_hydrometeor",
         "number_particles_no_hydrometeor_internal_data",
         "number_particles_unknown_classification",
-        "number_particles_unknown_classification_internal_data",
-        "number_particles_class_1",
-        "number_particles_class_1_internal_data",
-        "number_particles_class_2",
-        "number_particles_class_2_internal_data",
-        "number_particles_class_3",
-        "number_particles_class_3_internal_data",
-        "number_particles_class_4",
-        "number_particles_class_4_internal_data",
-        "number_particles_class_5",
-        "number_particles_class_5_internal_data",
-        "number_particles_class_6",
-        "number_particles_class_6_internal_data",
-        "number_particles_class_7",
-        "number_particles_class_7_internal_data",
-        "number_particles_class_8",
-        "number_particles_class_8_internal_data",
+        "total_gross_volume_unknown_classification",
+        "number_particles_hail",
+        "total_gross_volume_hail",
+        "number_particles_solid_precipitation",
+        "total_gross_volume_solid_precipitation",
+        "number_particles_large_pellet",
+        "total_gross_volume_large_pellet",
+        "number_particles_small_pellet",
+        "total_gross_volume_small_pellet",
+        "number_particles_snowgrain",
+        "total_gross_volume_snowgrain",
+        "number_particles_rain",
+        "total_gross_volume_rain",
+        "number_particles_small_rain",
+        "total_gross_volume_small_rain",
+        "number_particles_drizzle",
+        "total_gross_volume_drizzle",
         "number_particles_class_9",
         "number_particles_class_9_internal_data",
         "raw_drop_number",