disdrodb 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

disdrodb/l0/configs/SWS250/l0b_cf_attrs.yml

@@ -0,0 +1,72 @@
+precipitation_rate:
+  description: Rainfall rate
+  long_name: Precipitation rate
+  units: mm/h
+precipitation_accumulated:
+  description: Accumulated rain amount over the measurement interval
+  long_name: Precipitation accumulated
+  units: mm
+weather_code_synop_4680:
+  description: SYNOP weather code according to table 4680 of Parsivel documentation
+  long_name: Weather code SYNOP 4680
+  units: ""
+weather_code_metar_4678:
+  description: METAR/SPECI weather code according to table 4678 of Parsivel documentation
+  long_name: Weather code METAR 4678
+  units: ""
+past_weather1:
+  description: Past weather code 1
+  long_name: Past weather 1
+  units: ""
+past_weather2:
+  description: Past weather code 2
+  long_name: Past weather 2
+  units: ""
+mor_visibility_5min:
+  description: Meteorological Optical Range in precipitation (5 minute average)
+  long_name: MOR visibility 5 minute
+  units: m
+mor_visibility:
+  description: Meteorological Optical Range in precipitation
+  long_name: MOR visibility
+  units: m
+number_particles:
+  description: Number of particles detected and validated
+  long_name: Number of detected particles
+  units: ""
+sensor_temperature:
+  description: Temperature in sensor housing
+  long_name: Temperature of the sensor
+  units: "degC"
+obstruction_status:
+  description: Obstruction status of the sensor
+  long_name: Obstruction status
+  units: ""
+total_extinction_coefficient:
+  description: Total extinction coefficient
+  long_name: Total extinction coefficient
+  units: "1/km"
+transmissometer_extinction_coefficient:
+  description: Transmissometer extinction coefficient
+  long_name: Transmissometer extinction coefficient
+  units: "1/km"
+back_scatter_extinction_coefficient:
+  description: Back scatter extinction coefficient
+  long_name: Back scatter extinction coefficient
+  units: "1/km"
+ambient_light_sensor_signal:
+  description: Ambient light sensor signal
+  long_name: Ambient light sensor signal
+  units: "cd/m2"
+sensor_status:
+  description: Sensor status
+  long_name: Sensor status
+  units: ""
+ambient_light_sensor_signal_status:
+  description: Ambient light sensor signal status
+  long_name: Ambient light sensor signal status
+  units: ""
+raw_drop_number:
+  description: Drop counts per diameter and velocity class
+  long_name: Raw drop number
+  units: ""

disdrodb/l0/configs/SWS250/l0b_encodings.yml

@@ -0,0 +1,155 @@
+precipitation_rate:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+precipitation_accumulated:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+weather_code_synop_4680:
+  dtype: uint8
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 255
+weather_code_metar_4678:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+past_weather1:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+past_weather2:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+mor_visibility_5min:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 65535
+mor_visibility:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 65535
+number_particles:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 65535
+sensor_temperature:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -60.0
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+  _FillValue: 65535
+obstruction_status:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+sensor_status:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+ambient_light_sensor_signal_status:
+  dtype: str
+  zlib: false
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+total_extinction_coefficient:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+transmissometer_extinction_coefficient:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+back_scatter_extinction_coefficient:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+ambient_light_sensor_signal:
+  dtype: float32
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  chunksizes: 5000
+raw_drop_number:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes:
+    - 5000
+    - 16
+    - 21

disdrodb/l0/configs/SWS250/raw_data_format.yml

@@ -0,0 +1,148 @@
+precipitation_rate:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 3
+  n_naturals: 3
+  data_range:
+    - 0
+    - 9999.999
+  nan_flags: null
+precipitation_accumulated:
+  n_digits: 6
+  n_characters: 7
+  n_decimals: 2
+  n_naturals: 4
+  data_range:
+    - 0
+    - 9999.0
+  nan_flags: null
+weather_code_synop_4680:
+  n_digits: 2
+  n_characters: 2
+  n_decimals: 0
+  n_naturals: 2
+  data_range:
+    - 0
+    - 89
+  nan_flags: null
+weather_code_metar_4678:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+past_weather1:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+past_weather2:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+mor_visibility_5min:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+mor_visibility:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+number_particles:
+  n_digits: 4
+  n_characters: 4
+  n_decimals: 0
+  n_naturals: 4
+  data_range:
+    - 0
+    - 9999
+  nan_flags: null
+sensor_temperature:
+  n_digits: 4
+  n_characters: 6
+  n_decimals: 1
+  n_naturals: 3
+  data_range:
+    - -99
+    - 100
+  nan_flags: null
+obstruction_status:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+total_extinction_coefficient:
+  n_digits: 5
+  n_characters: 6
+  n_decimals: 2
+  n_naturals: 3
+  data_range:
+    - 0
+    - 999.99
+  nan_flags: null
+transmissometer_extinction_coefficient:
+  n_digits: 5
+  n_characters: 6
+  n_decimals: 2
+  n_naturals: 3
+  data_range:
+    - 0
+    - 999.99
+  nan_flags: null
+back_scatter_extinction_coefficient:
+  n_digits: 5
+  n_characters: 7
+  n_decimals: 2
+  n_naturals: 3
+  data_range:
+    - -999.99
+    - 999.99
+  nan_flags: null
+ambient_light_sensor_signal:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 0
+  n_naturals: 5
+  data_range:
+    - 0
+    - 99998
+  nan_flags: 99999
+sensor_status:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+ambient_light_sensor_signal_status:
+  n_digits: null
+  n_characters: null
+  n_decimals: null
+  n_naturals: null
+  data_range: null
+  nan_flags: null
+raw_drop_number:
+  n_digits: 0
+  n_characters: 4096
+  n_decimals: 0
+  n_naturals: 0
+  data_range: null
+  nan_flags: null
+  dimension_order:
+    - velocity_bin_center
+    - diameter_bin_center
+  n_values: 336

disdrodb/l0/l0b_processing.py

@@ -80,15 +80,16 @@ def infer_split_str(string: str) -> str:
     return split_str
 
 
-def _replace_empty_strings_with_zeros(values):
+def replace_empty_strings_with_zeros(values):
+    """Replace empty comma separated strings with '0'."""
     values[np.char.str_len(values) == 0] = "0"
     return values
 
 
-def _format_string_array(string: str, n_values: int) -> np.array:
+def format_string_array(string: str, n_values: int) -> np.array:
     """Split a string with multiple numbers separated by a delimiter into an 1D array.
 
-        e.g. : _format_string_array("2,44,22,33", 4) will return [ 2. 44. 22. 33.]
+        e.g. : format_string_array("2,44,22,33", 4) will return [ 2. 44. 22. 33.]
 
     If empty string ("") --> Return an arrays of zeros
     If the list length is not n_values -> Return an arrays of np.nan
@@ -126,7 +127,7 @@ def _format_string_array(string: str, n_values: int) -> np.array:
         # Ensure string type
         values = values.astype("str")
         # Replace '' with 0
-        values = _replace_empty_strings_with_zeros(values)
+        values = replace_empty_strings_with_zeros(values)
         # Replace "-9.999" with 0
         values = np.char.replace(values, "-9.999", "0")
         # Cast values to float type
@@ -135,7 +136,7 @@ def _format_string_array(string: str, n_values: int) -> np.array:
     return values
 
 
-def _reshape_raw_spectrum(
+def reshape_raw_spectrum(
     arr: np.array,
     dims_order: list,
     dims_size_dict: dict,
@@ -243,17 +244,17 @@ def retrieve_l0b_arrays(
         # Ensure is a string, get a numpy array for each row and then stack
         # - Option 1: Clear but lot of copies
         # df_series = df[key].astype(str)
-        # list_arr = df_series.apply(_format_string_array, n_values=n_values)
+        # list_arr = df_series.apply(format_string_array, n_values=n_values)
         # arr = np.stack(list_arr, axis=0)
 
         # - Option 2: still copies
-        # arr = np.vstack(_format_string_array(s, n_values=n_values) for s in df_series.astype(str))
+        # arr = np.vstack(format_string_array(s, n_values=n_values) for s in df_series.astype(str))
 
         # - Option 3: more memory efficient
         n_timesteps = len(df[key])
         arr = np.empty((n_timesteps, n_values), dtype=float)  # preallocates
         for i, s in enumerate(df[key].astype(str)):
-            arr[i, :] = _format_string_array(s, n_values=n_values)
+            arr[i, :] = format_string_array(s, n_values=n_values)
 
         # Retrieve dimensions
         dims_order = dims_order_dict[key]
@@ -263,7 +264,7 @@ def retrieve_l0b_arrays(
         # - This applies i.e for PARSIVEL*, LPM, PWS100
         # - This does not apply to RD80
         if key == "raw_drop_number" and len(dims_order) == 2:
-            arr, dims = _reshape_raw_spectrum(
+            arr, dims = reshape_raw_spectrum(
                 arr=arr,
                 dims_order=dims_order,
                 dims_size_dict=dims_size_dict,
@@ -288,7 +289,57 @@ def retrieve_l0b_arrays(
 #### L0B Coords and attributes
 
 
-def _convert_object_variables_to_string(ds: xr.Dataset) -> xr.Dataset:
+def ensure_valid_geolocation(ds: xr.Dataset, coord: str, errors: str = "ignore") -> xr.Dataset:
+    """Ensure valid geolocation coordinates.
+
+    'altitude' must be >= 0, 'latitude' must be within [-90, 90] and
+    'longitude' within [-180, 180].
+
+    It can deal with coordinates varying with time.
+
+    Parameters
+    ----------
+    ds : xarray.Dataset
+        Dataset containing the coordinate.
+    coord : str
+        Name of the coordinate variable to validate.
+    errors : {"ignore", "raise", "coerce"}, default "ignore"
+        - "ignore": nothing is done.
+        - "raise" : raise ValueError if invalid values are found.
+        - "coerce": out-of-range values are replaced with NaN.
+
+    Returns
+    -------
+    xr.Dataset
+        Dataset with validated coordinate values.
+    """
+    # Define coordinates ranges
+    ranges = {
+        "altitude": (0, np.inf),
+        "latitude": (-90, 90),
+        "longitude": (-180, 180),  # used only for "raise"/"coerce"
+    }
+
+    # Check coordinate is available and correctly defined.
+    if coord not in ds:
+        raise ValueError(f"Coordinate '{coord}' not found in dataset.")
+    if coord not in list(ranges):
+        raise ValueError(f"Valid geolocation coordinates are: {list(ranges)}.")
+
+    # Validate coordinate
+    vmin, vmax = ranges[coord]
+    invalid = (ds[coord] < vmin) | (ds[coord] > vmax)
+    invalid = invalid.compute()
+
+    # Deal within invalid errors
+    if errors == "raise" and invalid.any():
+        raise ValueError(f"{coord} out of range {vmin}-{vmax}.")
+    if errors == "coerce":
+        ds[coord] = ds[coord].where(~invalid)
+    return ds
+
+
+def convert_object_variables_to_string(ds: xr.Dataset) -> xr.Dataset:
     """Convert variables with ``object`` dtype to ``string``.
 
     Parameters
@@ -307,7 +358,7 @@ def _convert_object_variables_to_string(ds: xr.Dataset) -> xr.Dataset:
     return ds
 
 
-def _set_variable_attributes(ds: xr.Dataset, sensor_name: str) -> xr.Dataset:
+def set_variable_attributes(ds: xr.Dataset, sensor_name: str) -> xr.Dataset:
     """Set attributes to each ``xr.Dataset`` variable.
 
     Parameters
@@ -353,7 +404,7 @@ def add_dataset_crs_coords(ds):
 
 
 def _define_dataset_variables(df, sensor_name, logger=None, verbose=False):
-    """Define DISDRODB L0B netCDF variables."""
+    """Define DISDRODB L0B netCDF array variables."""
     # Preprocess raw_spectrum, diameter and velocity arrays if available
     raw_fields = ["raw_drop_concentration", "raw_drop_average_velocity", "raw_drop_number"]
     if np.any(np.isin(raw_fields, df.columns)):
@@ -436,7 +487,7 @@ def set_geolocation_coordinates(ds, metadata):
         # If coordinate not present, add it from dictionary
         if coord not in ds:
             ds = ds.assign_coords({coord: metadata.pop(coord, np.nan)})
-        # Else if set coordinates the variable in the dataset (present in the raw data)
+        # Else ensure coord is a dataset coordinates
         else:
             ds = ds.set_coords(coord)
             _ = metadata.pop(coord, None)
@@ -445,6 +496,10 @@ def set_geolocation_coordinates(ds, metadata):
     for coord in coords:
         ds[coord] = xr.where(ds[coord] == -9999, np.nan, ds[coord])
 
+    # Ensure valid geolocation coordinates
+    for coord in coords:
+        ds = ensure_valid_geolocation(ds=ds, coord=coord, errors="coerce")
+
     # Set attributes without geolocation coordinates
     ds.attrs = metadata
     return ds
@@ -469,11 +524,11 @@ def finalize_dataset(ds, sensor_name, metadata):
     ds = ds.transpose("time", "diameter_bin_center", ...)
 
     # Ensure variables with dtype object are converted to string
-    ds = _convert_object_variables_to_string(ds)
+    ds = convert_object_variables_to_string(ds)
 
     # Add netCDF variable and coordinate attributes
     # - Add variable attributes: long_name, units, descriptions, valid_min, valid_max
-    ds = _set_variable_attributes(ds=ds, sensor_name=sensor_name)
+    ds = set_variable_attributes(ds=ds, sensor_name=sensor_name)
     # - Add netCDF coordinate attributes
     ds = set_coordinate_attributes(ds=ds)
     #  - Set DISDRODB global attributes

disdrodb/l0/readers/LPM/ARM/ARM_LPM.py

@@ -69,7 +69,7 @@ def reader(
         "quality_measurement": "quality_index",
         "max_diameter_hail": "max_hail_diameter",
         "laser_status": "laser_status",
-        "static_signal": "static_signal",
+        "static_signal_status": "static_signal_status",
         "interior_temperature": "temperature_interior",
         "laser_temperature": "laser_temperature",
         "laser_temperature_analog_status": "laser_temperature_analog_status",

disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py

@@ -137,7 +137,7 @@ def reader(
         "quality_index",
         "max_hail_diameter",
         "laser_status",
-        "static_signal",
+        "static_signal_status",
         "laser_temperature_analog_status",
         "laser_temperature_digital_status",
         "laser_current_analog_status",
@@ -151,7 +151,7 @@ def reader(
         "current_heating_heads_status",
         "current_heating_carriers_status",
         "control_output_laser_power_status",
-        "reserve_status",
+        "reserved_status",
         "temperature_interior",
         "laser_temperature",
         "laser_current_average",