disdrodb 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

disdrodb/l0/readers/SW250/BELGIUM/KMI.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""DISDRODB reader for KMI Biral SW250 sensors."""
+import pandas as pd
+
+from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
+from disdrodb.l0.l0a_processing import read_raw_text_file
+
+
+def parse_spectrum_line_to_string(line):
+    """Parse one M... spectrum line into a zero-padded string with 21 values."""
+    # Define number of velocity bins values expected
+    n_cols = 21
+
+    # Split spectrum line
+    parts = line.split(",")
+
+    # Check line validity
+    n_values = len(parts)
+    if n_values > n_cols:
+        raise ValueError(f"Unexpected number of velocity bins: {n_values}.")
+
+    # Strip  'M' from first bin
+    parts[0] = parts[0].replace(":00M", "")
+
+    # Strip last two letter from last value
+    parts[-1] = parts[-1][:3]
+
+    # Define list of values
+    values = [int(x) for x in parts]
+    if len(values) < n_cols:
+        values.extend([0] * (n_cols - len(values)))
+    values = values[:n_cols]
+
+    # Define comma-separated string
+    string = ",".join(str(v) for v in values)
+    return string
+
+
+def parse_spectrum_block(lines):
+    """Parse an M-block into a fixed (16 x 21) matrix."""
+    n_values = len(lines)
+    if n_values != 16:
+        raise ValueError(f"Unexpected number of diameter bins: {n_values}.")
+    raw_drop_number_string = ",".join([parse_spectrum_line_to_string(line) for line in lines])
+    return raw_drop_number_string
+
+
+def build_spectrum_block(group):
+    """Create SWS250 raw spectrum string."""
+    try:
+        return pd.Series(
+            {
+                "raw_drop_number": parse_spectrum_block(group["spectrum_line"].tolist()),
+            },
+        )
+    except Exception:
+        return pd.Series({"raw_drop_number": "NaN"})
+
+
+@is_documented_by(reader_generic_docstring)
+def reader(
+    filepath,
+    logger=None,
+):
+    """Reader."""
+    ##------------------------------------------------------------------------.
+    #### Define raw data headers
+    column_names = ["TO_PARSE"]
+
+    ##------------------------------------------------------------------------.
+    #### Define reader options
+    # - For more info: https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
+    reader_kwargs = {}
+
+    # - Define delimiter
+    reader_kwargs["delimiter"] = "\\n"
+
+    # - Avoid first column to become df index !!!
+    reader_kwargs["index_col"] = False
+
+    # Since column names are expected to be passed explicitly, header is set to None
+    reader_kwargs["header"] = None
+
+    # - Number of rows to be skipped at the beginning of the file
+    reader_kwargs["skiprows"] = None
+
+    # - Define behaviour when encountering bad lines
+    reader_kwargs["on_bad_lines"] = "skip"
+
+    # - Define reader engine
+    #   - C engine is faster
+    #   - Python engine is more feature-complete
+    reader_kwargs["engine"] = "python"
+
+    # - Define on-the-fly decompression of on-disk data
+    #   - Available: gzip, bz2, zip
+    reader_kwargs["compression"] = "infer"
+
+    # - Skip rows with badly encoded data
+    reader_kwargs["encoding_errors"] = "replace"
+
+    # - Strings to recognize as NA/NaN and replace with standard NA flags
+    #   - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
+    #                       '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
+    #                       'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
+    reader_kwargs["na_values"] = ["na", "", "error"]
+
+    ##------------------------------------------------------------------------.
+    #### Read the data
+    df = read_raw_text_file(
+        filepath=filepath,
+        column_names=column_names,
+        reader_kwargs=reader_kwargs,
+        logger=logger,
+    )
+
+    ##------------------------------------------------------------------------.
+    #### Adapt the dataframe to adhere to DISDRODB L0 standards
+    # Identify rows with data
+    df_params = df[df["TO_PARSE"].str.count(",") == 23]
+
+    # Identify rows with spectrum matrix
+    df_spectrum = df[df["TO_PARSE"].str.startswith(":00M")]
+    if len(df_spectrum) == 0:
+        raise ValueError("No spectrum available.")
+
+    df_spectrum = df_spectrum["TO_PARSE"].str.rsplit(",", expand=True, n=2)
+    df_spectrum.columns = ["spectrum_line", "date", "time"]
+    df_spectrum["datetime"] = pd.to_datetime(
+        df_spectrum["date"] + " " + df_spectrum["time"],
+        format="%d/%m/%Y %H:%M:%S",
+    )
+
+    # Define groups
+    # - Mark new group when time gap > 10 s
+    is_new_group = (df_spectrum["datetime"].diff().dt.total_seconds() > 10).fillna(True)
+    group_id = is_new_group.cumsum()
+    # - Assign the first datetime of each group
+    df_spectrum["group_time"] = df_spectrum.groupby(group_id)["datetime"].transform("first")
+
+    # Group spectrum by timesteps
+    df_raw_drop_number = (
+        df_spectrum.groupby("group_time", as_index=False)
+        .apply(build_spectrum_block, include_groups=False)
+        .reset_index(drop=True)
+    )
+
+    # Retrieve 1-min data
+    # - Split by ; delimiter (before raw drop number)
+    df_data = df_params["TO_PARSE"].str.split(",", expand=True)
+
+    # - Assign column names
+    names = [
+        "date",
+        "time",
+        "sws250",
+        "sensor_id",
+        "sample_interval",
+        "mor_visibility_5min",  # remove unit and ensure in meters  !
+        "weather_code_synop_4680",
+        "past_weather1",
+        "past_weather2",
+        "obstruction_status",
+        "weather_code_metar_4678",
+        "precipitation_rate",
+        "mor_visibility",  # remove unit and ensure in meters  !
+        "total_extinction_coefficient",  # [km-1]
+        "transmissometer_extinction_coefficient",  # [km-1]
+        "back_scatter_extinction_coefficient",  # [km-1]
+        "sensor_temperature",  # [degrees]  or air_temperature ?
+        "ambient_light_sensor_signal",  # [cd/m2] # ALS
+        "sensor_status",
+        "number_particles",
+        "precipitation_accumulated",  # [mm] over sample_interval
+        "ambient_light_sensor_signal_status",
+        "date1",
+        "time1",
+    ]
+    df_data.columns = names
+
+    # Clean out variables
+    df_data["mor_visibility_5min"] = df_data["mor_visibility_5min"].str.replace("M", "")
+    df_data["mor_visibility"] = df_data["mor_visibility"].str.replace("M", "")
+    df_data["sensor_temperature"] = df_data["sensor_temperature"].str.replace("C", "")
+    df_data["ambient_light_sensor_signal"] = df_data["ambient_light_sensor_signal"].str.replace("+99999", "NaN")
+
+    # Define datetime
+    df_data["datetime"] = pd.to_datetime(df_data["date1"] + " " + df_data["time1"], format="%d/%m/%Y %H:%M:%S")
+
+    # Merge df_data on df_raw_drop_number
+    # TODO list
+    # - should we aggregate variables to 5 min temporal resolution
+    #   to match raw_drop_number
+    # - should we infill df_raw_drop_number with 0 when no time every 5 min?
+    df = pd.merge_asof(
+        df_raw_drop_number,
+        df_data,
+        left_on="group_time",
+        right_on="datetime",
+        direction="nearest",
+        tolerance=pd.Timedelta("10s"),  # max difference allowed
+    )
+
+    # Define final time
+    # TODO list
+    # - which time should we take as final time?
+    # - raw_drop_number time is end of measurement interval right?
+    df["time"] = df["group_time"]
+
+    # Drop columns not agreeing with DISDRODB L0 standards
+    columns_to_drop = [
+        "group_time",
+        "date",
+        "sws250",
+        "sample_interval",
+        "sensor_id",
+        "date1",
+        "time1",
+        "datetime",
+    ]
+    df = df.drop(columns=columns_to_drop)
+    return df

disdrodb/l1/beard_model.py

@@ -456,67 +456,67 @@ def get_raindrop_reynolds_number(diameter, temperature, air_density, water_densi
     return reynolds_number
 
 
-def get_fall_velocity_beard_1976(diameter, temperature, air_density, water_density, g):
+def get_drag_coefficient(diameter, air_density, water_density, fall_velocity, g=9.81):
     """
-    Computes the terminal fall velocity of a raindrop in still air.
-
-    Reference: Beard 1976; Pruppacher & Klett 1978
+    Computes the drag coefficient for a raindrop.
 
     Parameters
     ----------
     diameter : float
         Diameter of the raindrop in meters.
-    temperature : float
-        Temperature in Kelvin.
     air_density : float
         Density of air in kg/m^3.
     water_density : float
         Density of water in kg/m^3.
+    fall_velocity : float
+        Terminal fall velocity of the raindrop in m/s.
     g : float
         Gravitational acceleration in m/s^2.
 
     Returns
     -------
     float
-        Terminal fall velocity of the raindrop in m/s.
+        Drag coefficient of the raindrop.
     """
-    air_viscosity = get_air_dynamic_viscosity(temperature)
-    reynolds_number = get_raindrop_reynolds_number(
-        diameter=diameter,
-        temperature=temperature,
-        air_density=air_density,
-        water_density=water_density,
-        g=g,
-    )
-    fall_velocity = air_viscosity * reynolds_number / (air_density * diameter)
-    return fall_velocity
+    delta_density = water_density - air_density
+    drag_coefficient = 4 * delta_density * g * diameter / (3 * air_density * fall_velocity**2)
+    return drag_coefficient
 
 
-def get_drag_coefficient(diameter, air_density, water_density, fall_velocity, g=9.81):
+def get_fall_velocity_beard_1976(diameter, temperature, air_density, water_density, g):
     """
-    Computes the drag coefficient for a raindrop.
+    Computes the terminal fall velocity of a raindrop in still air.
+
+    Reference: Beard 1976; Pruppacher & Klett 1978
 
     Parameters
     ----------
     diameter : float
         Diameter of the raindrop in meters.
+    temperature : float
+        Temperature in Kelvin.
     air_density : float
         Density of air in kg/m^3.
     water_density : float
         Density of water in kg/m^3.
-    fall_velocity : float
-        Terminal fall velocity of the raindrop in m/s.
     g : float
         Gravitational acceleration in m/s^2.
 
     Returns
     -------
     float
-        Drag coefficient of the raindrop.
+        Terminal fall velocity of the raindrop in m/s.
     """
-    delta_density = water_density - air_density
-    drag_coefficient = 4 * delta_density * g * diameter / (3 * air_density * fall_velocity**2)
-    return drag_coefficient
+    air_viscosity = get_air_dynamic_viscosity(temperature)
+    reynolds_number = get_raindrop_reynolds_number(
+        diameter=diameter,
+        temperature=temperature,
+        air_density=air_density,
+        water_density=water_density,
+        g=g,
+    )
+    fall_velocity = air_viscosity * reynolds_number / (air_density * diameter)
+    return fall_velocity
 
 
 def retrieve_fall_velocity(
@@ -573,19 +573,24 @@ def retrieve_fall_velocity(
             gas_constant_dry_air=gas_constant_dry_air,
         )
 
+    # else
+    # --> Estimate sea_level_air_pressure from air_pressure ?
+
     # Retrieve vapour pressure (from relative humidity)
     vapor_pressure = get_vapor_actual_pressure(
         relative_humidity=relative_humidity,
         temperature=temperature,
     )
 
-    # Retrieve air density and water density
+    # Retrieve air density
     air_density = get_air_density(
         temperature=temperature,
         air_pressure=air_pressure,
         vapor_pressure=vapor_pressure,
         gas_constant_dry_air=gas_constant_dry_air,
     )
+
+    # Retrieve water density
     water_density = get_water_density(
         temperature=temperature,
         air_pressure=air_pressure,
@@ -611,106 +616,3 @@ def retrieve_fall_velocity(
     #                                         fall_velocity=fall_velocity)
 
     return fall_velocity
-
-
-####-----------------------------------------------------------------------------------------
-#### OLD CODE
-
-
-# def get_fall_velocity_beard_1977(diameter):
-#     """
-#     Compute the fall velocity of raindrops using the Beard (1977) relationship.
-
-#     Parameters
-#     ----------
-#     diameter : array-like
-#         Diameter of the raindrops in millimeters.
-#         Valid up to 7 mm (0.7 cm).
-
-#     Returns
-#     -------
-#     fall_velocity : array-like
-#         Fall velocities in meters per second.
-
-#     Notes
-#     -----
-#     This method uses an exponential function based on the work of Beard (1977),
-#     valid at sea level conditions (pressure = 1 atm, temperature = 20°C,
-#     air density = 1.194 kg/m³).
-
-#     References
-#     ----------
-#     Beard, K. V. (1977).
-#     Terminal velocity adjustment for cloud and precipitation drops aloft.
-#     Journal of the Atmospheric Sciences, 34(8), 1293-1298.
-#     https://doi.org/10.1175/1520-0469(1977)034<1293:TVAFCA>2.0.CO;2
-
-#     """
-#     diameter_cm = diameter/1000
-#     c = [7.06037, 1.74951, 4.86324, 6.60631, 4.84606, 2.14922, 0.58714, 0.096348, 0.00869209, 0.00033089]
-#     log_diameter = np.log(diameter_cm)
-#     y = c[0] + sum(c * log_diameter**i for i, c in enumerate(c[1:], start=1))
-#     fall_velocity = np.exp(y)
-#     return fall_velocity
-
-
-# def get_fall_velocity_beard_1977(diameter, temperature, air_pressure, gas_constant_dry_air=287.04):
-#     """
-#     Computes the terminal fall velocity of a raindrop in still air.
-
-#     This function is based on the Table 4 coefficients of Kenneth V. Beard (1977),
-#     "Terminal Velocity and Shape of Cloud and Precipitation Drops Aloft",
-#     Journal of the Atmospheric Sciences, Vol. 34, pp. 1293-1298.
-
-#     Note: This approximation is valid at sea level with conditions:
-#           Pressure = 1 atm, Temperature = 20°C, (saturated) air density = 1.194 kg/m³.
-
-#     Parameters
-#     ----------
-#     diameter : array-like
-#         Array of equivolume drop diameters in meters.
-
-#     Returns
-#     -------
-#     fall_velocity : array-like
-#         Array of terminal fall velocity in meters per second (m/s).
-#         For diameters greater than 7 mm, the function returns NaN.
-
-#     """
-#     # PROBLEMATIC
-#     # Compute sea level velocity
-#     c = [7.06037, 1.74951, 4.86324, 6.60631, 4.84606, 2.14922, 0.58714, 0.096348, 0.00869209, 0.00033089]
-#     log_diameter = np.log(diameter / 1000 * 10)
-#     y = c[0] + sum(c * log_diameter**i for i, c in enumerate(c[1:], start=1))
-#     v0 = np.exp(y)
-
-#     # Compute fall velocity
-#     t_20 = 273.15 + 20
-#     eps_s = get_air_dynamic_viscosity(t_20) / get_air_dynamic_viscosity(temperature) - 1
-#     eps_c = -1 + (
-#         np.sqrt(
-#             get_air_density(
-#                 temperature=t_20,
-#                 air_pressure=101325,
-#                 vapor_pressure=0,
-#                 gas_constant_dry_air=gas_constant_dry_air,
-#             )
-#             / get_air_density(
-#                 temperature=temperature,
-#                 air_pressure=air_pressure,
-#                 vapor_pressure=0,
-#                 gas_constant_dry_air=gas_constant_dry_air,
-#             ),
-#         )
-#     )
-#     a = 1.104 * eps_s
-#     b = (1.058 * eps_c - 1.104 * eps_s) / 5.01
-#     x = np.log(diameter) + 5.52
-#     f = (a + b * x) + 1
-#     fall_velocity = v0 * f
-#     # fall_velocity.plot()
-
-#     eps = 1.104 * eps_s + (1.058 * eps_c - 1.104 * eps_s) * np.log(diameter / 1e-3) / 5.01
-#     # eps = 1.104 * eps_s + (1.058 * eps_c - 1.104 * eps_s) * np.log(diameter / 4e-5) / 5.01
-#     fall_velocity = 0.01 * v0 * (1 + eps)
-#     return fall_velocity