disdrodb 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

disdrodb/l0/configs/PWS100/raw_data_format.yml

@@ -0,0 +1,182 @@
+mor_visibility:
+  n_digits: 5
+  n_characters: 6
+  n_decimals: 1
+  n_naturals: 4
+  data_range:
+    - 0
+    - 9999.9
+  nan_flags: null
+  field_number: "20"
+weather_code_synop_4680:
+  n_digits: 2
+  n_characters: 2
+  n_decimals: 0
+  n_naturals: 2
+  data_range:
+    - 0
+    - 89
+  nan_flags: null
+  field_number: "21"
+weather_code_metar_4678:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+  field_number: "22"
+weather_code_nws:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+  field_number: "23"
+alarms:
+  n_digits: 0
+  n_characters: 31
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  field_number: "24"
+sensor_status:
+  n_digits: 1
+  n_characters: 1
+  n_decimals: 0
+  n_naturals: 1
+  data_range:
+    - 0
+    - 4
+  nan_flags: null
+  valid_values:
+    - 0
+    - 1
+    - 2
+    - 3
+    - 4
+  field_number: "25"
+air_temperature:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 2
+  n_naturals: 1
+  data_range:
+    - -99.9
+    - 99.9
+  nan_flags: null
+  field_number: "30"
+relative_humidity:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 2
+  n_naturals: 2
+  data_range:
+    - 0
+    - 100
+  nan_flags: null
+  field_number: "30"
+wetbulb_temperature:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 2
+  n_naturals: 1
+  data_range:
+    - -99.9
+    - 99.9
+  nan_flags: null
+  field_number: "30"
+air_temperature_max:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 2
+  n_naturals: 1
+  data_range:
+    - -99.9
+    - 99.9
+  nan_flags: null
+  field_number: "31"
+air_temperature_min:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 2
+  n_naturals: 1
+  data_range:
+    - -99.9
+    - 99.9
+  nan_flags: null
+  field_number: "31"
+rainfall_rate:
+  n_digits: 5
+  n_characters: 6
+  n_decimals: 4
+  n_naturals: 1
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "40"
+rainfall_accumulated:
+  n_digits: 5
+  n_characters: 6
+  n_decimals: 4
+  n_naturals: 1
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "41"
+drop_size_distribution:
+  n_digits: 0
+  n_characters: 300
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  field_number: "42"
+average_drop_velocity:
+  n_digits: 4
+  n_characters: 5
+  n_decimals: 3
+  n_naturals: 1
+  data_range: null
+  nan_flags: null
+  field_number: "43"
+average_drop_size:
+  n_digits: 4
+  n_characters: 5
+  n_decimals: 3
+  n_naturals: 1
+  data_range: null
+  nan_flags: null
+  field_number: "43"
+type_distribution:
+  n_digits: 0
+  n_characters: 11
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  field_number: "44"
+raw_drop_number:
+  n_digits: 0
+  n_characters: 4624
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  dimension_order:
+    - diameter_bin_center
+    - velocity_bin_center
+  n_values: 1156
+  field_number: "47"
+peak_to_pedestal_hist:
+  n_digits: 0
+  n_characters: 50
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  field_number: "48"

disdrodb/l0/configs/RD80/raw_data_format.yml

@@ -130,9 +130,7 @@ N0:
   n_characters: 9
   n_decimals: 4
   n_naturals: 4
-  data_range:
-    - 0
-    - 9999.9999
+  data_range: null
   nan_flags: null
   field_number: "34"
 slope:
@@ -140,9 +138,7 @@ slope:
   n_characters: 9
   n_decimals: 4
   n_naturals: 1
-  data_range:
-    - 0
-    - 9.9999
+  data_range: null
   nan_flags: null
   field_number: "35"
 Dmax:

disdrodb/l0/l0b_nc_processing.py

@@ -480,7 +480,7 @@ def sanitize_ds(
     ds = set_nan_invalid_values(ds, sensor_name=sensor_name, logger=logger, verbose=verbose)
 
     # Finalize dataset
-    ds = finalize_dataset(ds, sensor_name=sensor_name, attrs=metadata)
+    ds = finalize_dataset(ds, sensor_name=sensor_name, metadata=metadata)
 
     # Return dataset
     return ds

disdrodb/l0/l0b_processing.py

@@ -170,6 +170,8 @@ def _reshape_raw_spectrum(
         {"diameter_bin_center": 32, "velocity_bin_center": 32}
         For LPM
         {"diameter_bin_center": 22, "velocity_bin_center": 20}
+        For PWS100
+        {"diameter_bin_center": 34, "velocity_bin_center": 34}
     n_timesteps : int
         Number of timesteps.
 
@@ -256,7 +258,7 @@ def retrieve_l0b_arrays(
 
         # For key='raw_drop_number', if 2D spectrum, reshape to 2D matrix
         # Example:
-        # - This applies i.e for PARSIVEL* and LPM
+        # - This applies i.e for PARSIVEL*, LPM, PWS100
         # - This does not apply to RD80
         if key == "raw_drop_number" and len(dims_order) == 2:
             arr, dims = _reshape_raw_spectrum(
@@ -416,15 +418,15 @@ def create_l0b_from_l0a(
         Error if the DISDRODB L0B xarray dataset can not be created.
     """
     # Retrieve sensor name
-    attrs = metadata.copy()
-    sensor_name = attrs["sensor_name"]
+    metadata = metadata.copy()
+    sensor_name = metadata["sensor_name"]
 
     # Define Dataset variables and coordinates
     data_vars = _define_dataset_variables(df, sensor_name=sensor_name, logger=logger, verbose=verbose)
 
     # Create xarray Dataset
     ds = xr.Dataset(data_vars=data_vars)
-    ds = finalize_dataset(ds, sensor_name=sensor_name, attrs=attrs)
+    ds = finalize_dataset(ds, sensor_name=sensor_name, metadata=metadata)
     return ds
 
 
@@ -432,7 +434,7 @@ def create_l0b_from_l0a(
 #### L0B netCDF4 Writer
 
 
-def set_geolocation_coordinates(ds, attrs):
+def set_geolocation_coordinates(ds, metadata):
     """Add geolocation coordinates to dataset."""
     # Assumption
     # - If coordinate is present in L0A, overrides the one specified in the attributes
@@ -443,22 +445,22 @@ def set_geolocation_coordinates(ds, attrs):
     for coord in coords:
         # If coordinate not present, add it from dictionary
         if coord not in ds:
-            ds = ds.assign_coords({coord: attrs.pop(coord, np.nan)})
+            ds = ds.assign_coords({coord: metadata.pop(coord, np.nan)})
         # Else if set coordinates the variable in the dataset (present in the raw data)
         else:
             ds = ds.set_coords(coord)
-            _ = attrs.pop(coord, None)
+            _ = metadata.pop(coord, None)
 
     # Set -9999 flag value to np.nan
     for coord in coords:
         ds[coord] = xr.where(ds[coord] == -9999, np.nan, ds[coord])
 
     # Set attributes without geolocation coordinates
-    ds.attrs = attrs
+    ds.attrs = metadata
     return ds
 
 
-def finalize_dataset(ds, sensor_name, attrs):
+def finalize_dataset(ds, sensor_name, metadata):
     """Finalize DISDRODB L0B Dataset."""
     # Ensure sorted by time
     ds = ensure_sorted_by_time(ds)
@@ -467,7 +469,7 @@ def finalize_dataset(ds, sensor_name, attrs):
     ds = ds.assign_coords(get_bin_coords_dict(sensor_name=sensor_name))
 
     # Set geolocation coordinates and attributes
-    ds = set_geolocation_coordinates(ds, attrs=attrs)
+    ds = set_geolocation_coordinates(ds, metadata=metadata)
 
     # Add dataset CRS coordinate
     ds = add_dataset_crs_coords(ds)

disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py

@@ -89,9 +89,11 @@ def reader(
     if len(df) == 0 or len(df) == 1:
         raise ValueError("No data to process.")
 
-    # Retrieve time
+    # Retrieve time column and format in datetime64
     df_time = df[::2]
     df_time = df_time.reset_index(drop=True)
+    df_time = df_time["TO_BE_PARSED"].str.replace("-", "", n=1)
+    df_time = pd.to_datetime(df_time, format="%Y-%m-%d %H:%M:%S", errors="coerce")
 
     # Retrieve data
     df_data = df[1::2]
@@ -100,12 +102,6 @@ def reader(
     if len(df_time) != len(df_data):
         raise ValueError("Likely corrupted data. Not same number of timesteps and data.")
 
-    # Remove starting - from timestep
-    df_time = df_time["TO_BE_PARSED"].str.replace("-", "", n=1)
-
-    # Format time in datetime64
-    df_time = pd.to_datetime(df_time, format="%Y-%m-%d %H:%M:%S", errors="coerce")
-
     # Create dataframe
     df_data["time"] = df_time.to_numpy()
 
@@ -199,19 +195,31 @@ def reader(
         "number_particles_class_8_internal_data",
         "number_particles_class_9",
         "number_particles_class_9_internal_data",
-        "raw_drop_number",
+        "TO_BE_FURTHER_PROCESSED",
     ]
     df.columns = column_names
 
+    # Extract the last variables remained in raw_drop_number
+    df_parsed = df["TO_BE_FURTHER_PROCESSED"].str.rsplit(";", n=6, expand=True)
+    df_parsed.columns = [
+        "raw_drop_number",
+        "air_temperature",
+        "relative_humidity",
+        "wind_speed",
+        "wind_direction",
+        "checksum",
+        "dummy",
+    ]
+
+    # Assign columns to the original dataframe
+    df[df_parsed.columns] = df_parsed
+
     # Drop row if start_identifier different than 00
     df["time"] = df_data["time"]
     df = df[df["start_identifier"].astype(str) == "00"]
 
-    # Clean raw_drop_number (ignore last 5 column)
-    df["raw_drop_number"] = df["raw_drop_number"].str[:1760]
-
     # Drop rows with invalid raw_drop_number
-    df = df[df["raw_drop_number"].astype(str).str.len() == 1760]
+    df = df[df["raw_drop_number"].astype(str).str.len() == 1759]
 
     # Drop columns not agreeing with DISDRODB L0 standards
     columns_to_drop = [
@@ -220,7 +228,9 @@ def reader(
         "sensor_serial_number",
         "sensor_date",
         "sensor_time",
+        "TO_BE_FURTHER_PROCESSED",
+        "checksum",
+        "dummy",
     ]
     df = df.drop(columns=columns_to_drop)
-
     return df

disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py

@@ -167,11 +167,11 @@ def reader(
     # Drop row if start_identifier different than 00
     df = df[df["start_identifier"].astype(str) == "00"]
 
-    # Clean raw_drop_number (ignore last "AC" character)
-    df["raw_drop_number"] = df["raw_drop_number"].str[:1760]
+    # Remove checksum from raw_drop_number
+    df["raw_drop_number"] = df["raw_drop_number"].str.rsplit(";", n=2, expand=True)[0]
 
     # Drop rows with invalid raw_drop_number
-    df = df[df["raw_drop_number"].astype(str).str.len() == 1760]
+    df = df[df["raw_drop_number"].astype(str).str.len() == 1759]
 
     # Drop columns not agreeing with DISDRODB L0 standards
     columns_to_drop = [

disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py

@@ -162,14 +162,16 @@ def reader(
     df["time"] = df["sensor_date"] + "-" + df["sensor_time"]
     df["time"] = pd.to_datetime(df["time"], format="%d.%m.%y-%H:%M:%S", errors="coerce")
 
+    # TODO: correct time is unavailable yet !
+
     # Drop row if start_identifier different than 00
     df = df[df["start_identifier"].astype(str) == "00"]
 
-    # Clean raw_drop_number (ignore last "AC" character)
-    df["raw_drop_number"] = df["raw_drop_number"].str[:1760]
+    # Remove checksum from raw_drop_number
+    df["raw_drop_number"] = df["raw_drop_number"].str.rsplit(";", n=2, expand=True)[0]
 
     # Drop rows with invalid raw_drop_number
-    df = df[df["raw_drop_number"].astype(str).str.len() == 1760]
+    df = df[df["raw_drop_number"].astype(str).str.len() == 1759]
 
     # Drop columns not agreeing with DISDRODB L0 standards
     columns_to_drop = [

disdrodb/l0/readers/LPM/ITALY/GID_LPM.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+
 # -----------------------------------------------------------------------------.
 # Copyright (c) 2021-2023 DISDRODB developers
 #
@@ -15,7 +16,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
-"""Reader for the GID LPM network."""
+"""DISDRODB reader for GID LPM sensors not measuring wind."""
 import pandas as pd
 
 from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
@@ -29,32 +30,43 @@ def reader(
 ):
     """Reader."""
     ##------------------------------------------------------------------------.
-    #### Define column names
+    #### - Define raw data headers
     column_names = ["TO_BE_SPLITTED"]
 
     ##------------------------------------------------------------------------.
     #### Define reader options
+    # - For more info: https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
     reader_kwargs = {}
+
     # - Define delimiter
-    reader_kwargs["delimiter"] = "\n"
-    # Skip first row as columns names
-    reader_kwargs["header"] = None
-    # - Avoid first column to become df index
+    reader_kwargs["delimiter"] = "\\n"
+
+    # - Avoid first column to become df index !!!
     reader_kwargs["index_col"] = False
+
+    # Since column names are expected to be passed explicitly, header is set to None
+    reader_kwargs["header"] = None
+
+    # - Number of rows to be skipped at the beginning of the file
+    reader_kwargs["skiprows"] = None
+
     # - Define behaviour when encountering bad lines
     reader_kwargs["on_bad_lines"] = "skip"
+
     # - Define reader engine
     #   - C engine is faster
     #   - Python engine is more feature-complete
     reader_kwargs["engine"] = "python"
+
     # - Define on-the-fly decompression of on-disk data
     #   - Available: gzip, bz2, zip
     reader_kwargs["compression"] = "infer"
+
     # - Strings to recognize as NA/NaN and replace with standard NA flags
     #   - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
     #                       '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
     #                       'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
-    reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
+    reader_kwargs["na_values"] = ["na", "", "error"]
 
     ##------------------------------------------------------------------------.
     #### Read the data
@@ -67,14 +79,17 @@ def reader(
 
     ##------------------------------------------------------------------------.
     #### Adapt the dataframe to adhere to DISDRODB L0 standards
-    # Split columns
-    df = df["TO_BE_SPLITTED"].str.split(";", n=79, expand=True)
+    # Count number of delimiters to identify valid rows
+    df = df[df["TO_BE_SPLITTED"].str.count(";") == 519]
+
+    # Split by ; delimiter (before raw drop number)
+    df = df["TO_BE_SPLITTED"].str.split(";", expand=True, n=79)
 
     # Assign column names
     column_names = [
         "start_identifier",
+        "device_address",
         "sensor_serial_number",
-        "software_version",
         "sensor_date",
         "sensor_time",
         "weather_code_synop_4677_5min",
@@ -155,25 +170,26 @@ def reader(
     ]
     df.columns = column_names
 
-    # Remove checksum at end of raw_drop_number
-    df["raw_drop_number"] = df["raw_drop_number"].str.slice(stop=1760)
+    # Remove checksum from raw_drop_number
+    df["raw_drop_number"] = df["raw_drop_number"].str.rsplit(";", n=1, expand=True)[0]
+
+    # Define datetime "time" column
+    df["time"] = df["sensor_date"] + "-" + df["sensor_time"]
+    df["time"] = pd.to_datetime(df["time"], format="%d.%m.%y-%H:%M:%S", errors="coerce")
 
-    # Define 'time column
-    df["time"] = df["sensor_date"].astype(str) + " " + df["sensor_time"].astype(str)
+    # Drop row if start_identifier different than 00
+    df = df[df["start_identifier"].astype(str) == "00"]
 
-    # Convert time column to datetime
-    df["time"] = pd.to_datetime(df["time"], format="%d.%m.%y %H:%M:%S", errors="coerce")
+    # Drop rows with invalid raw_drop_number
+    df = df[df["raw_drop_number"].astype(str).str.len() == 1759]
 
     # Drop columns not agreeing with DISDRODB L0 standards
     columns_to_drop = [
         "start_identifier",
-        "software_version",
+        "device_address",
         "sensor_serial_number",
         "sensor_date",
         "sensor_time",
     ]
     df = df.drop(columns=columns_to_drop)
-    df = df.drop(columns=["sensor_date", "sensor_time"])
-
-    # Return the dataframe adhering to DISDRODB L0 standards
     return df