disdrodb 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml

@@ -0,0 +1,76 @@
+mor_visibility:
+  description: Meteorological Optical Range in precipitation
+  long_name: MOR visibility
+  units: m
+weather_code_synop_4680:
+  description: SYNOP weather code according to table 4680
+  long_name: Weather code SYNOP 4680
+  units: ""
+weather_code_metar_4678:
+  description: METAR/SPECI weather code according to table 4678
+  long_name: Weather code METAR 4678
+  units: ""
+weather_code_nws:
+  description: NWS weather code according to Parsivel documentation
+  long_name: Weather code NWS
+  units: ""
+alarms:
+  description: PWS alarms flags
+  long_name: PWS alarms flags
+  units: ""
+sensor_status:
+  description: Sensor status
+  long_name: Sensor status
+  units: ""
+air_temperature:
+  description: "Air temperature in degrees Celsius"
+  long_name: Air temperature
+  units: "C"
+relative_humidity:
+  description: "Relative humidity in percent (%)"
+  long_name: Relative humidity
+  units: "%"
+wetbulb_temperature:
+  description: "Wet bulb temperature in degrees Celsius"
+  long_name: Wet bulb temperature
+  units: "C"
+air_temperature_max:
+  description: "Maximum air temperature in degrees Celsius"
+  long_name: Maximum air temperature
+  units: "C"
+air_temperature_min:
+  description: "Minimum air temperature in degrees Celsius"
+  long_name: Minimum air temperature
+  units: "C"
+rainfall_rate:
+  description: Rainfall rate
+  long_name: Rainfall rate
+  units: mm/h
+rainfall_accumulated:
+  description: Accumulated rain amount over the measurement interval
+  long_name: Rainfall accumulated
+  units: mm
+average_drop_velocity:
+  description: Average drop velocity
+  long_name: Average drop velocity
+  units: "m/s"
+average_drop_size:
+  description: Average drop size
+  long_name: Average drop size
+  units: "mm"
+raw_drop_number:
+  description: Drop counts per diameter and velocity class
+  long_name: Raw drop number
+  units: ""
+type_distribution:
+  description: Particle type distribution
+  long_name: Particle type distribution
+  units: ""
+drop_size_distribution:
+  description: Drop size distribution
+  long_name: Drop size distribution
+  units: ""
+peak_to_pedestal_hist:
+  description: Peak to pedestal histogram
+  long_name: Peak to pedestal histogram
+  units: ""

disdrodb/l0/configs/PWS100/l0b_encodings.yml

@@ -0,0 +1,176 @@
+mor_visibility:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 65535
+weather_code_synop_4680:
+  dtype: uint8
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 255
+weather_code_metar_4678:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+weather_code_nws:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+alarms:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+sensor_status:
+  dtype: uint8
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 255
+air_temperature:
+  dtype: uint16
+  scale_factor: 0.01
+  add_offset: -99.99
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+relative_humidity:
+  dtype: uint16
+  scale_factor: 0.01
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+wetbulb_temperature:
+  dtype: uint16
+  scale_factor: 0.01
+  add_offset: -99.99
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+air_temperature_max:
+  dtype: uint16
+  scale_factor: 0.01
+  add_offset: -99.99
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+air_temperature_min:
+  dtype: uint16
+  scale_factor: 0.01
+  add_offset: -99.99
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+rainfall_rate:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+rainfall_accumulated:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+raw_drop_number:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes:
+    - 5000
+    - 34
+    - 34
+drop_size_distribution:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+peak_to_pedestal_hist:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+type_distribution:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+average_drop_velocity:
+  dtype: uint16
+  scale_factor: 0.01
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 65535
+average_drop_size:
+  dtype: uint16
+  scale_factor: 0.01
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 65535

disdrodb/l0/configs/PWS100/raw_data_format.yml

@@ -0,0 +1,182 @@
+mor_visibility:
+  n_digits: 5
+  n_characters: 6
+  n_decimals: 1
+  n_naturals: 4
+  data_range:
+    - 0
+    - 9999.9
+  nan_flags: null
+  field_number: "20"
+weather_code_synop_4680:
+  n_digits: 2
+  n_characters: 2
+  n_decimals: 0
+  n_naturals: 2
+  data_range:
+    - 0
+    - 89
+  nan_flags: null
+  field_number: "21"
+weather_code_metar_4678:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+  field_number: "22"
+weather_code_nws:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+  field_number: "23"
+alarms:
+  n_digits: 0
+  n_characters: 31
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  field_number: "24"
+sensor_status:
+  n_digits: 1
+  n_characters: 1
+  n_decimals: 0
+  n_naturals: 1
+  data_range:
+    - 0
+    - 4
+  nan_flags: null
+  valid_values:
+    - 0
+    - 1
+    - 2
+    - 3
+    - 4
+  field_number: "25"
+air_temperature:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 2
+  n_naturals: 1
+  data_range:
+    - -99.9
+    - 99.9
+  nan_flags: null
+  field_number: "30"
+relative_humidity:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 2
+  n_naturals: 2
+  data_range:
+    - 0
+    - 100
+  nan_flags: null
+  field_number: "30"
+wetbulb_temperature:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 2
+  n_naturals: 1
+  data_range:
+    - -99.9
+    - 99.9
+  nan_flags: null
+  field_number: "30"
+air_temperature_max:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 2
+  n_naturals: 1
+  data_range:
+    - -99.9
+    - 99.9
+  nan_flags: null
+  field_number: "31"
+air_temperature_min:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 2
+  n_naturals: 1
+  data_range:
+    - -99.9
+    - 99.9
+  nan_flags: null
+  field_number: "31"
+rainfall_rate:
+  n_digits: 5
+  n_characters: 6
+  n_decimals: 4
+  n_naturals: 1
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "40"
+rainfall_accumulated:
+  n_digits: 5
+  n_characters: 6
+  n_decimals: 4
+  n_naturals: 1
+  data_range:
+    - 0
+    - 99999
+  nan_flags: null
+  field_number: "41"
+drop_size_distribution:
+  n_digits: 0
+  n_characters: 300
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  field_number: "42"
+average_drop_velocity:
+  n_digits: 4
+  n_characters: 5
+  n_decimals: 3
+  n_naturals: 1
+  data_range: null
+  nan_flags: null
+  field_number: "43"
+average_drop_size:
+  n_digits: 4
+  n_characters: 5
+  n_decimals: 3
+  n_naturals: 1
+  data_range: null
+  nan_flags: null
+  field_number: "43"
+type_distribution:
+  n_digits: 0
+  n_characters: 11
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  field_number: "44"
+raw_drop_number:
+  n_digits: 0
+  n_characters: 4624
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  dimension_order:
+    - diameter_bin_center
+    - velocity_bin_center
+  n_values: 1156
+  field_number: "47"
+peak_to_pedestal_hist:
+  n_digits: 0
+  n_characters: 50
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  field_number: "48"

disdrodb/l0/configs/RD80/raw_data_format.yml

@@ -130,9 +130,7 @@ N0:
   n_characters: 9
   n_decimals: 4
   n_naturals: 4
-  data_range:
-    - 0
-    - 9999.9999
+  data_range: null
   nan_flags: null
   field_number: "34"
 slope:
@@ -140,9 +138,7 @@ slope:
   n_characters: 9
   n_decimals: 4
   n_naturals: 1
-  data_range:
-    - 0
-    - 9.9999
+  data_range: null
   nan_flags: null
   field_number: "35"
 Dmax:

disdrodb/l0/l0b_nc_processing.py

@@ -480,7 +480,7 @@ def sanitize_ds(
     ds = set_nan_invalid_values(ds, sensor_name=sensor_name, logger=logger, verbose=verbose)
 
     # Finalize dataset
-    ds = finalize_dataset(ds, sensor_name=sensor_name, attrs=metadata)
+    ds = finalize_dataset(ds, sensor_name=sensor_name, metadata=metadata)
 
     # Return dataset
     return ds

disdrodb/l0/l0b_processing.py

@@ -170,6 +170,8 @@ def _reshape_raw_spectrum(
         {"diameter_bin_center": 32, "velocity_bin_center": 32}
         For LPM
         {"diameter_bin_center": 22, "velocity_bin_center": 20}
+        For PWS100
+        {"diameter_bin_center": 34, "velocity_bin_center": 34}
     n_timesteps : int
         Number of timesteps.
 
@@ -256,7 +258,7 @@ def retrieve_l0b_arrays(
 
         # For key='raw_drop_number', if 2D spectrum, reshape to 2D matrix
         # Example:
-        # - This applies i.e for PARSIVEL* and LPM
+        # - This applies i.e for PARSIVEL*, LPM, PWS100
         # - This does not apply to RD80
         if key == "raw_drop_number" and len(dims_order) == 2:
             arr, dims = _reshape_raw_spectrum(
@@ -416,15 +418,15 @@ def create_l0b_from_l0a(
         Error if the DISDRODB L0B xarray dataset can not be created.
     """
     # Retrieve sensor name
-    attrs = metadata.copy()
-    sensor_name = attrs["sensor_name"]
+    metadata = metadata.copy()
+    sensor_name = metadata["sensor_name"]
 
     # Define Dataset variables and coordinates
     data_vars = _define_dataset_variables(df, sensor_name=sensor_name, logger=logger, verbose=verbose)
 
     # Create xarray Dataset
     ds = xr.Dataset(data_vars=data_vars)
-    ds = finalize_dataset(ds, sensor_name=sensor_name, attrs=attrs)
+    ds = finalize_dataset(ds, sensor_name=sensor_name, metadata=metadata)
     return ds
 
 
@@ -432,7 +434,7 @@ def create_l0b_from_l0a(
 #### L0B netCDF4 Writer
 
 
-def set_geolocation_coordinates(ds, attrs):
+def set_geolocation_coordinates(ds, metadata):
     """Add geolocation coordinates to dataset."""
     # Assumption
     # - If coordinate is present in L0A, overrides the one specified in the attributes
@@ -443,22 +445,22 @@ def set_geolocation_coordinates(ds, attrs):
     for coord in coords:
         # If coordinate not present, add it from dictionary
         if coord not in ds:
-            ds = ds.assign_coords({coord: attrs.pop(coord, np.nan)})
+            ds = ds.assign_coords({coord: metadata.pop(coord, np.nan)})
         # Else if set coordinates the variable in the dataset (present in the raw data)
         else:
             ds = ds.set_coords(coord)
-            _ = attrs.pop(coord, None)
+            _ = metadata.pop(coord, None)
 
     # Set -9999 flag value to np.nan
     for coord in coords:
         ds[coord] = xr.where(ds[coord] == -9999, np.nan, ds[coord])
 
     # Set attributes without geolocation coordinates
-    ds.attrs = attrs
+    ds.attrs = metadata
     return ds
 
 
-def finalize_dataset(ds, sensor_name, attrs):
+def finalize_dataset(ds, sensor_name, metadata):
     """Finalize DISDRODB L0B Dataset."""
     # Ensure sorted by time
     ds = ensure_sorted_by_time(ds)
@@ -467,7 +469,7 @@ def finalize_dataset(ds, sensor_name, attrs):
     ds = ds.assign_coords(get_bin_coords_dict(sensor_name=sensor_name))
 
     # Set geolocation coordinates and attributes
-    ds = set_geolocation_coordinates(ds, attrs=attrs)
+    ds = set_geolocation_coordinates(ds, metadata=metadata)
 
     # Add dataset CRS coordinate
     ds = add_dataset_crs_coords(ds)

disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py

@@ -89,9 +89,11 @@ def reader(
     if len(df) == 0 or len(df) == 1:
         raise ValueError("No data to process.")
 
-    # Retrieve time
+    # Retrieve time column and format in datetime64
     df_time = df[::2]
     df_time = df_time.reset_index(drop=True)
+    df_time = df_time["TO_BE_PARSED"].str.replace("-", "", n=1)
+    df_time = pd.to_datetime(df_time, format="%Y-%m-%d %H:%M:%S", errors="coerce")
 
     # Retrieve data
     df_data = df[1::2]
@@ -100,12 +102,6 @@ def reader(
     if len(df_time) != len(df_data):
         raise ValueError("Likely corrupted data. Not same number of timesteps and data.")
 
-    # Remove starting - from timestep
-    df_time = df_time["TO_BE_PARSED"].str.replace("-", "", n=1)
-
-    # Format time in datetime64
-    df_time = pd.to_datetime(df_time, format="%Y-%m-%d %H:%M:%S", errors="coerce")
-
     # Create dataframe
     df_data["time"] = df_time.to_numpy()
 
@@ -199,19 +195,31 @@ def reader(
         "number_particles_class_8_internal_data",
         "number_particles_class_9",
         "number_particles_class_9_internal_data",
-        "raw_drop_number",
+        "TO_BE_FURTHER_PROCESSED",
     ]
     df.columns = column_names
 
+    # Extract the last variables remained in raw_drop_number
+    df_parsed = df["TO_BE_FURTHER_PROCESSED"].str.rsplit(";", n=6, expand=True)
+    df_parsed.columns = [
+        "raw_drop_number",
+        "air_temperature",
+        "relative_humidity",
+        "wind_speed",
+        "wind_direction",
+        "checksum",
+        "dummy",
+    ]
+
+    # Assign columns to the original dataframe
+    df[df_parsed.columns] = df_parsed
+
     # Drop row if start_identifier different than 00
     df["time"] = df_data["time"]
     df = df[df["start_identifier"].astype(str) == "00"]
 
-    # Clean raw_drop_number (ignore last 5 column)
-    df["raw_drop_number"] = df["raw_drop_number"].str[:1760]
-
     # Drop rows with invalid raw_drop_number
-    df = df[df["raw_drop_number"].astype(str).str.len() == 1760]
+    df = df[df["raw_drop_number"].astype(str).str.len() == 1759]
 
     # Drop columns not agreeing with DISDRODB L0 standards
     columns_to_drop = [
@@ -220,7 +228,9 @@ def reader(
         "sensor_serial_number",
         "sensor_date",
         "sensor_time",
+        "TO_BE_FURTHER_PROCESSED",
+        "checksum",
+        "dummy",
     ]
     df = df.drop(columns=columns_to_drop)
-
     return df

disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py

@@ -167,11 +167,11 @@ def reader(
     # Drop row if start_identifier different than 00
     df = df[df["start_identifier"].astype(str) == "00"]
 
-    # Clean raw_drop_number (ignore last "AC" character)
-    df["raw_drop_number"] = df["raw_drop_number"].str[:1760]
+    # Remove checksum from raw_drop_number
+    df["raw_drop_number"] = df["raw_drop_number"].str.rsplit(";", n=2, expand=True)[0]
 
     # Drop rows with invalid raw_drop_number
-    df = df[df["raw_drop_number"].astype(str).str.len() == 1760]
+    df = df[df["raw_drop_number"].astype(str).str.len() == 1759]
 
     # Drop columns not agreeing with DISDRODB L0 standards
     columns_to_drop = [

disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py

@@ -162,14 +162,16 @@ def reader(
     df["time"] = df["sensor_date"] + "-" + df["sensor_time"]
     df["time"] = pd.to_datetime(df["time"], format="%d.%m.%y-%H:%M:%S", errors="coerce")
 
+    # TODO: correct time is unavailable yet !
+
     # Drop row if start_identifier different than 00
     df = df[df["start_identifier"].astype(str) == "00"]
 
-    # Clean raw_drop_number (ignore last "AC" character)
-    df["raw_drop_number"] = df["raw_drop_number"].str[:1760]
+    # Remove checksum from raw_drop_number
+    df["raw_drop_number"] = df["raw_drop_number"].str.rsplit(";", n=2, expand=True)[0]
 
     # Drop rows with invalid raw_drop_number
-    df = df[df["raw_drop_number"].astype(str).str.len() == 1760]
+    df = df[df["raw_drop_number"].astype(str).str.len() == 1759]
 
     # Drop columns not agreeing with DISDRODB L0 standards
     columns_to_drop = [