disdrodb 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

disdrodb/l2/empirical_dsd.py

@@ -101,6 +101,101 @@ def count_bins_with_drops(ds):
     return da
 
 
+def _compute_qc_bins_metrics(arr):
+    # Find indices of non-zero elements
+    arr = arr.copy()
+    arr[np.isnan(arr)] = 0
+    non_zero_indices = np.nonzero(arr)[0]
+    if non_zero_indices.size == 0:
+        return np.array([0, len(arr), 1, len(arr)])
+
+    # Define bins interval with drops
+    start_idx, end_idx = non_zero_indices[0], non_zero_indices[-1]
+    segment = arr[start_idx : end_idx + 1]
+
+    # Compute number of bins with drops
+    total_bins = segment.size
+
+    # Compute number of missing bins (zeros)
+    n_missing_bins = int(np.sum(segment == 0))
+
+    # Compute fraction of bins with missing drops
+    fraction_missing = n_missing_bins / total_bins
+
+    # Identify longest with with consecutive zeros
+    zero_mask = (segment == 0).astype(int)
+    # - Pad with zeros at both ends to detect edges
+    padded = np.pad(zero_mask, (1, 1), "constant", constant_values=0)
+    diffs = np.diff(padded)
+    # - Start and end indices of runs
+    run_starts = np.where(diffs == 1)[0]
+    run_ends = np.where(diffs == -1)[0]
+    run_lengths = run_ends - run_starts
+    max_consecutive_missing = run_lengths.max() if run_lengths.size > 0 else 0
+
+    # Define output
+    output = np.array([total_bins, n_missing_bins, fraction_missing, max_consecutive_missing])
+    return output
+
+
+def compute_qc_bins_metrics(ds):
+    """
+    Compute quality-control metrics for drop-count bins along the diameter dimension.
+
+    This function selects the first available drop-related variable from the dataset,
+    optionally collapses over velocity methods and the velocity dimension, then
+    computes four metrics per time step:
+
+      1. Nbins: total number of diameter bins between the first and last non-zero count
+      2. Nbins_missing: number of bins with zero or NaN counts in that interval
+      3. Nbins_missing_fraction: fraction of missing bins (zeros) in the interval
+      4. Nbins_missing_consecutive: maximum length of consecutive missing bins
+
+    Parameters
+    ----------
+    ds : xr.Dataset
+        Input dataset containing one of the following variables:
+        'drop_counts', 'drop_number_concentration', or 'drop_number'.
+        If a 'velocity_method' dimension exists, only the first method is used.
+        If a velocity dimension (specified by VELOCITY_DIMENSION) exists, it is summed over.
+
+    Returns
+    -------
+    xr.Dataset
+        Dataset with a new 'metric' dimension of size 4 and coordinates:
+        ['Nbins', 'Nbins_missing', 'Nbins_missing_fraction', 'Nbins_missing_consecutive'],
+        indexed by 'time'.
+    """
+    # Select useful variable
+    candidate_variables = ["drop_counts", "drop_number_concentration", "drop_number"]
+    available_variables = [var for var in candidate_variables if var in ds]
+    if len(available_variables) == 0:
+        raise ValueError(f"One of these variables is required: {candidate_variables}")
+    da = ds[available_variables[0]]
+    if "velocity_method" in da.dims:
+        da = da.isel(velocity_method=0)
+        da = da.drop_vars("velocity_method")
+    if VELOCITY_DIMENSION in da.dims:
+        da = da.sum(dim=VELOCITY_DIMENSION)
+
+    # Compute QC metrics
+    da_qc_bins = xr.apply_ufunc(
+        _compute_qc_bins_metrics,
+        da,
+        input_core_dims=[[DIAMETER_DIMENSION]],
+        output_core_dims=[["metric"]],
+        vectorize=True,
+        dask="parallelized",
+        output_dtypes=[float],
+        dask_gufunc_kwargs={"output_sizes": {"metric": 4}},
+    )
+
+    # Assign meaningful labels to the qc 'metric' dimension
+    variables = ["Nbins", "Nbins_missing", "Nbins_missing_fraction", "Nbins_missing_consecutive"]
+    ds_qc_bins = da_qc_bins.assign_coords(metric=variables).to_dataset(dim="metric")
+    return ds_qc_bins
+
+
 ####-------------------------------------------------------------------------------------------------------------------.
 #### DSD Spectrum, Concentration, Moments
 
@@ -117,13 +212,16 @@ def get_effective_sampling_area(sensor_name, diameter):
         B = 30 / 1000  # Width of the Parsivel beam in m (30mm)
         sampling_area = L * (B - diameter / 2)
         return sampling_area
-    if sensor_name in "LPM":
+    if sensor_name == "LPM":
         # Calculate sampling area for each diameter bin (S_i)
         L = 228 / 1000  # Length of the Parsivel beam in m (228 mm)
         B = 20 / 1000  # Width of the Parsivel beam in m (20 mm)
         sampling_area = L * (B - diameter / 2)
         return sampling_area
-    if sensor_name in "RD80":
+    if sensor_name == "PWS100":
+        sampling_area = 0.004  # m2  # TODO: L * (B - diameter / 2) ?
+        return sampling_area
+    if sensor_name == "RD80":
         sampling_area = 0.005  # m2
         return sampling_area
     raise NotImplementedError(f"Effective sampling area for {sensor_name} must yet to be specified in the software.")

disdrodb/l2/event.py

@@ -43,7 +43,7 @@ def identify_events(
 ):
     """Return a list of rainy events.
 
-    Rainy timesteps are defined when n_drops_selected > min_n_drops.
+    Rainy timesteps are defined when N > min_n_drops.
     Any rainy isolated timesteps (based on neighborhood criteria) is removed.
     Then, consecutive rainy timesteps are grouped into the same event if the time gap between them does not
     exceed `intra_event_max_time_gap`. Finally, events that do not meet minimum size or duration
@@ -90,7 +90,7 @@ def identify_events(
     else:
         list_ds = [xr.open_dataset(filepath, chunks={}, cache=False, decode_timedelta=False) for filepath in filepaths]
     # Filter dataset for requested variables
-    variables = ["time", "n_drops_selected"]
+    variables = ["time", "N"]
     list_ds = [ds[variables] for ds in list_ds]
     # Concat datasets
     ds = xr.concat(list_ds, dim="time", compat="no_conflicts", combine_attrs="override")
@@ -102,7 +102,7 @@ def identify_events(
     # Sort dataset by time
     ds = ensure_sorted_by_time(ds)
     # Define candidate timesteps to group into events
-    idx_valid = ds["n_drops_selected"].data > min_n_drops
+    idx_valid = ds["N"].data > min_n_drops
     timesteps = ds["time"].data[idx_valid]
     # Define event list
     event_list = group_timesteps_into_event(

disdrodb/l2/processing.py

@@ -24,8 +24,8 @@ from disdrodb.l1.fall_velocity import get_raindrop_fall_velocity
 from disdrodb.l1_env.routines import load_env_dataset
 from disdrodb.l2.empirical_dsd import (
     compute_integral_parameters,
+    compute_qc_bins_metrics,
     compute_spectrum_parameters,
-    count_bins_with_drops,
     get_drop_average_velocity,
     get_drop_number_concentration,
     get_effective_sampling_area,
@@ -140,11 +140,12 @@ def generate_l2_empirical(ds, ds_env=None, compute_spectra=False):
     # Discard all timesteps without measured drops
     # - This allow to speed up processing
     # - Regularization can be done at the end
-    ds = ds.isel(time=ds["n_drops_selected"] > 0)
+    ds = ds.isel(time=ds["N"] > 0)
 
     # Count number of diameter bins with data
-    if "n_bins_with_drops" not in ds:
-        ds["n_bins_with_drops"] = count_bins_with_drops(ds)
+    if "Nbins" not in ds:
+        # Add bins statistics
+        ds.update(compute_qc_bins_metrics(ds))
 
     # Retrieve ENV dataset or take defaults
     # --> Used for fall velocity and water density estimates
@@ -174,8 +175,8 @@ def generate_l2_empirical(ds, ds_env=None, compute_spectra=False):
         "drop_number",  # 2D V x D
         "drop_counts",  # 1D D
         "sample_interval",
-        "n_drops_selected",
-        "n_drops_discarded",
+        "N",
+        "Nremoved",
         "Dmin",
         "Dmax",
         "fall_velocity",
@@ -291,14 +292,14 @@ def generate_l2_model(
     fall_velocity_method="Beard1976",
     # PSD discretization
     diameter_min=0,
-    diameter_max=8,
+    diameter_max=10,
     diameter_spacing=0.05,
     # Fitting options
     psd_model=None,
     optimization=None,
     optimization_kwargs=None,
     # Filtering options
-    min_bins_with_drops=4,
+    min_nbins=4,
     remove_timesteps_with_few_bins=False,
     mask_timesteps_with_few_bins=False,
     # GOF metrics options
@@ -357,11 +358,12 @@ def generate_l2_model(
     ####------------------------------------------------------.
     #### Preprocessing
     # Count number of diameter bins with data
-    if "n_bins_with_drops" not in ds:
-        ds["n_bins_with_drops"] = count_bins_with_drops(ds)
+    if "Nbins" not in ds:
+        # Add bins statistics
+        ds.update(compute_qc_bins_metrics(ds))
 
     # Identify timesteps with enough diameter bins with counted trops
-    valid_timesteps = ds["n_bins_with_drops"] >= min_bins_with_drops
+    valid_timesteps = ds["Nbins"] >= min_nbins
 
     # Drop such timesteps if asked
     if remove_timesteps_with_few_bins:
@@ -466,7 +468,14 @@ def generate_l2_model(
 
 
 @check_pytmatrix_availability
-def generate_l2_radar(ds, radar_band=None, canting_angle_std=7, diameter_max=8, axis_ratio="Thurai2007", parallel=True):
+def generate_l2_radar(
+    ds,
+    radar_band=None,
+    canting_angle_std=7,
+    diameter_max=10,
+    axis_ratio="Thurai2007",
+    parallel=True,
+):
     """Simulate polarimetric radar variables from empirical drop number concentration or the estimated PSD.
 
     Parameters

disdrodb/l2/processing_options.py

@@ -7,16 +7,16 @@ DEFAULT_CONFIG = {
     "global_settings": {
         "time_integration": [
             "1MIN",
+            "5MIN",
             "10MIN",
             "ROLL1MIN",
-            "ROLL10MIN",
         ],  # ["10S", "30S", "1MIN",  "5MIN", "10MIN", "15MIN", "30MIN", "1H", "ROLL5MIN", "ROLL10MIN"],
         # Radar options
         "radar_simulation_enabled": False,
         "radar_simulation_options": {
             "radar_band": ["S", "C", "X", "Ku", "Ka", "W"],
             "canting_angle_std": 7,
-            "diameter_max": 8,
+            "diameter_max": 10,
             "axis_ratio": "Thurai2007",
         },
         # L2E options
@@ -25,10 +25,10 @@ DEFAULT_CONFIG = {
         "l2m_options": {
             "fall_velocity_method": "Beard1976",
             "diameter_min": 0,
-            "diameter_max": 8,
+            "diameter_max": 10,
             "diameter_spacing": 0.05,
             "gof_metrics": True,
-            "min_bins_with_drops": 4,
+            "min_nbins": 4,
             "remove_timesteps_with_few_bins": False,
             "mask_timesteps_with_few_bins": False,
             "models": {
@@ -112,7 +112,7 @@ TEST_CONFIG = {
         "radar_simulation_options": {
             "radar_band": ["S", "C", "X", "Ku", "Ka", "W"],
             "canting_angle_std": 7,
-            "diameter_max": 8,
+            "diameter_max": 10,
             "axis_ratio": "Thurai2007",
         },
         # L2E options
@@ -121,10 +121,10 @@ TEST_CONFIG = {
         "l2m_options": {
             "fall_velocity_method": "Beard1976",
             "diameter_min": 0,
-            "diameter_max": 8,
+            "diameter_max": 10,
             "diameter_spacing": 0.05,
             "gof_metrics": True,
-            "min_bins_with_drops": 4,
+            "min_nbins": 4,
             "remove_timesteps_with_few_bins": False,
             "mask_timesteps_with_few_bins": False,
             "models": {

disdrodb/l2/routines.py

@@ -156,11 +156,11 @@ def _generate_l2e(
 
         ##------------------------------------------------------------------------.
         # Remove timesteps with no drops or NaN (from L2E computations)
-        # timestep_zero_drops = ds["time"].data[ds["n_drops_selected"].data == 0]
-        # timestep_nan = ds["time"].data[np.isnan(ds["n_drops_selected"].data)]
+        # timestep_zero_drops = ds["time"].data[ds["N"].data == 0]
+        # timestep_nan = ds["time"].data[np.isnan(ds["N"].data)]
         # TODO: Make it a choice !
         indices_valid_timesteps = np.where(
-            ~np.logical_or(ds["n_drops_selected"].data == 0, np.isnan(ds["n_drops_selected"].data)),
+            ~np.logical_or(ds["N"].data == 0, np.isnan(ds["N"].data)),
         )[0]
         ds = ds.isel(time=indices_valid_timesteps)
 

disdrodb/metadata/checks.py

@@ -30,7 +30,7 @@ from disdrodb.api.info import (
 from disdrodb.configs import get_metadata_archive_dir
 from disdrodb.metadata.reader import read_station_metadata
 from disdrodb.metadata.search import get_list_metadata
-from disdrodb.metadata.standards import get_valid_metadata_keys
+from disdrodb.metadata.standards import METADATA_KEYS, METADATA_VALUES
 from disdrodb.utils.yaml import read_yaml
 
 #### --------------------------------------------------------------------------.
@@ -40,19 +40,17 @@ from disdrodb.utils.yaml import read_yaml
 def get_metadata_missing_keys(metadata):
     """Return the DISDRODB metadata keys which are missing."""
     keys = list(metadata.keys())
-    valid_keys = get_valid_metadata_keys()
     # Identify missing keys
-    idx_missing_keys = np.where(np.isin(valid_keys, keys, invert=True))[0]
-    missing_keys = np.array(valid_keys)[idx_missing_keys].tolist()
+    idx_missing_keys = np.where(np.isin(METADATA_KEYS, keys, invert=True))[0]
+    missing_keys = np.array(METADATA_KEYS)[idx_missing_keys].tolist()
     return missing_keys
 
 
 def get_metadata_invalid_keys(metadata):
     """Return the DISDRODB metadata keys which are not valid."""
     keys = list(metadata.keys())
-    valid_keys = get_valid_metadata_keys()
     # Identify invalid keys
-    idx_invalid_keys = np.where(np.isin(keys, valid_keys, invert=True))[0]
+    idx_invalid_keys = np.where(np.isin(keys, METADATA_KEYS, invert=True))[0]
     invalid_keys = np.array(keys)[idx_invalid_keys].tolist()
     return invalid_keys
 
@@ -73,11 +71,22 @@ def _check_metadata_values(metadata):
     """Check validity of metadata values.
 
     If null is specified in the YAML files (or None in the dict) raise error.
+    For specific keys, check that values match one of the allowed options in METADATA_VALUES.
     """
     for key, value in metadata.items():
+        # Check for None/null values
         if isinstance(value, type(None)):
             raise ValueError(f"The metadata key {key} has None or null value. Use '' instead.")
 
+        # Check that values match allowed options for specific keys
+        if key in METADATA_VALUES:
+            allowed_values = METADATA_VALUES[key]
+            if value not in allowed_values:
+                allowed_str = ", ".join([f"'{v}'" for v in allowed_values])
+                raise ValueError(
+                    f"Invalid value '{value}' for metadata key '{key}'. " f"Allowed values are: {allowed_str}.",
+                )
+
 
 def _check_metadata_campaign_name(metadata, expected_name):
     """Check metadata ``campaign_name``."""

disdrodb/metadata/manipulation.py

@@ -41,8 +41,8 @@ def add_missing_metadata_keys(metadata):
 
 def sort_metadata_dictionary(metadata):
     """Sort the keys of the metadata dictionary by ``valid_metadata_keys`` list order."""
-    from disdrodb.metadata.standards import get_valid_metadata_keys
+    from disdrodb.metadata.standards import METADATA_KEYS
 
-    list_metadata_keys = get_valid_metadata_keys()
+    list_metadata_keys = METADATA_KEYS
     metadata = {k: metadata[k] for k in list_metadata_keys}
     return metadata

disdrodb/metadata/standards.py

@@ -16,85 +16,89 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
-"""Define DISDRODB Metadata Standards."""
+"""Define the DISDRODB Metadata standards.
 
+When editing one of these variables, one need to update all YAML files in
+the DISDRODB-METADATA repository
+"""
 
-def get_valid_metadata_keys() -> list:
-    """Get DISDRODB valid metadata list.
+# Define valid values for specific metadata keys
+METADATA_VALUES = {
+    "deployment_status": ["ongoing", "terminated"],
+    "platform_type": ["fixed", "mobile"],
+    "deployment_mode": ["land", "ship", "truck", "cable"],
+    "platform_protection": ["shielded", "unshielded", ""],
+}
 
-    Returns
-    -------
-    list
-        List of valid metadata keys
-    """
-    # NOTE: When updating one of these keys, one need to update the yaml files in/at:
-    # - the DISDRODB-METADATA repository
-    # - disdrodb/data/DISDRODB/RAW/DATA_SOURCE/CAMPAIGN_NAME/metadata/*.yml
-    list_attrs = [
-        ## Mandatory fields
-        "data_source",
-        "campaign_name",
-        "station_name",
-        "sensor_name",
-        "measurement_interval",  # sampling_interval ? [in seconds]
-        # DISDRODB reader info
-        "reader",
-        "raw_data_glob_pattern",
-        "raw_data_format",  # 'txt', 'netcdf'
-        "platform_type",  # 'fixed', 'mobile'
-        ## DISDRODB keys
-        "disdrodb_data_url",
-        ## Source
-        "source",
-        "source_convention",
-        "source_processing_date",
-        ## Description
-        "title",
-        "description",
-        "project_name",
-        "keywords",
-        "summary",
-        "history",
-        "comment",
-        "station_id",
-        "location",
-        "country",
-        "continent",
-        ## Deployment Info
-        "latitude",  # in degrees North
-        "longitude",  # in degrees East
-        "altitude",  # in meter above sea level
-        "deployment_status",  # 'ended', 'ongoing'
-        "deployment mode",  # 'land', 'ship', 'truck', 'cable'
-        "platform_protection",  # 'shielded', 'unshielded'
-        "platform_orientation",  # [0-360] from N (clockwise)
-        ## Sensor info
-        "sensor_long_name",
-        "sensor_manufacturer",
-        "sensor_wavelength",
-        "sensor_serial_number",
-        "firmware_iop",
-        "firmware_dsp",
-        "firmware_version",
-        "sensor_beam_length",
-        "sensor_beam_width",
-        "sensor_nominal_width",
-        "calibration_sensitivity",
-        "calibration_certification_date",
-        "calibration_certification_url",
-        ## Attribution
-        "contributors",
-        "authors",
-        "authors_url",
-        "contact",
-        "contact_information",
-        "acknowledgement",  # acknowledgements?
-        "references",
-        "documentation",
-        "website",
-        "institution",
-        "source_repository",
-        "license",
-        "doi",
-    ]
-    return list_attrs
+
+METADATA_KEYS = [
+    ## Mandatory fields
+    "data_source",
+    "campaign_name",
+    "station_name",
+    "sensor_name",
+    # DISDRODB reader info
+    "reader",
+    "raw_data_glob_pattern",
+    "raw_data_format",  # 'txt', 'netcdf'
+    "measurement_interval",  # sampling_interval ? [in seconds]
+    ## Deployment Info
+    "deployment_status",  # 'terminated', 'ongoing'
+    "deployment_mode",  # 'land', 'ship', 'truck', 'cable'
+    "platform_type",  # 'fixed', 'mobile'
+    "latitude",  # in degrees North
+    "longitude",  # in degrees East
+    "altitude",  # in meter above sea level
+    # Platform info
+    "platform_protection",  # 'shielded', 'unshielded'
+    "platform_orientation",  # [0-360] from N (clockwise)
+    ## Time info
+    "time_coverage_start",  # YYYY-MM-DDTHH:MM:SS
+    "time_coverage_end",  # YYYY-MM-DDTHH:MM:SS
+    ## DISDRODB data url
+    "disdrodb_data_url",
+    ## Source
+    "source",
+    "source_convention",
+    "source_processing_date",
+    ## Description
+    "title",
+    "description",
+    "project_name",
+    "keywords",
+    "summary",
+    "history",
+    "comment",
+    "station_id",
+    "location",
+    "country",
+    "continent",
+    ## Sensor info
+    "sensor_long_name",
+    "sensor_manufacturer",
+    "sensor_wavelength",
+    "sensor_serial_number",
+    "firmware_iop",
+    "firmware_dsp",
+    "firmware_version",
+    "sensor_beam_length",
+    "sensor_beam_width",
+    "sensor_nominal_width",
+    "calibration_sensitivity",
+    "calibration_certification_date",
+    "calibration_certification_url",
+    ## Attribution
+    "contributors",
+    "authors",
+    "authors_url",
+    "contact",
+    "contact_information",
+    "acknowledgement",  # acknowledgements?
+    "references",
+    "documentation",
+    "website",
+    "institution",
+    "source_repository",
+    "license",
+    "doi",
+]

disdrodb/metadata/writer.py

@@ -22,7 +22,7 @@ import os
 
 from disdrodb.api.path import define_metadata_filepath
 from disdrodb.metadata.manipulation import sort_metadata_dictionary
-from disdrodb.metadata.standards import get_valid_metadata_keys
+from disdrodb.metadata.standards import METADATA_KEYS
 from disdrodb.utils.yaml import write_yaml
 
 
@@ -35,7 +35,7 @@ def get_default_metadata_dict() -> dict:
         Dictionary of attributes standard
     """
     # Get valid metadata keys
-    list_attrs = get_valid_metadata_keys()
+    list_attrs = METADATA_KEYS
     attrs = dict.fromkeys(list_attrs, "")
 
     # Add default values for certain keys