disdrodb 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

disdrodb/l0/readers/PWS100/AUSTRIA/HOAL.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""DISDRODB reader for TU Wien PWS100 raw text data."""
+import os
+
+import pandas as pd
+
+from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
+from disdrodb.l0.l0a_processing import read_raw_text_file
+
+
+def reader_spectrum(
+    filepath,
+    logger=None,
+):
+    """Reader spectrum file."""
+    ##------------------------------------------------------------------------.
+    #### Define column names
+    column_names = ["TO_SPLIT"]
+
+    ##------------------------------------------------------------------------.
+    #### Define reader options
+    reader_kwargs = {}
+
+    # - Define delimiter
+    reader_kwargs["delimiter"] = "\\n"
+
+    # - Skip first row as columns names
+    reader_kwargs["header"] = None
+
+    # - Skip header
+    reader_kwargs["skiprows"] = 4
+
+    # - Define encoding
+    reader_kwargs["encoding"] = "ISO-8859-1"
+
+    # - Avoid first column to become df index !!!
+    reader_kwargs["index_col"] = False
+
+    # - Define behaviour when encountering bad lines
+    reader_kwargs["on_bad_lines"] = "skip"
+
+    # - Define reader engine
+    #   - C engine is faster
+    #   - Python engine is more feature-complete
+    reader_kwargs["engine"] = "python"
+
+    # - Define on-the-fly decompression of on-disk data
+    #   - Available: gzip, bz2, zip
+    # reader_kwargs['compression'] = 'xz'
+
+    # - Strings to recognize as NA/NaN and replace with standard NA flags
+    #   - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
+    #                       '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
+    #                       'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
+    reader_kwargs["na_values"] = ["na", "error", "-.-", " NA", "NAN"]
+
+    ##------------------------------------------------------------------------.
+    #### Read the data
+    df = read_raw_text_file(
+        filepath=filepath,
+        column_names=column_names,
+        reader_kwargs=reader_kwargs,
+        logger=logger,
+    )
+
+    ##------------------------------------------------------------------------.
+    #### Adapt the dataframe to adhere to DISDRODB L0 standards
+    # Remove corrupted rows (and header)
+    df = df[df["TO_SPLIT"].str.count(",") == 1157]
+
+    # Split into columns
+    df = df["TO_SPLIT"].str.split(",", expand=True, n=2)
+
+    # Assign columns names
+    names = [
+        "time",
+        "record",
+        "raw_drop_number",  # "Size_0.00_Vel_0.00","Size_0.00_Vel_0.10", ...
+    ]
+    df.columns = names
+
+    # Add datetime time column
+    df["time"] = df["time"].str.replace('"', "")
+    df["time"] = pd.to_datetime(df["time"], format="%Y-%m-%d %H:%M:%S", errors="coerce")
+
+    # Clean raw_drop_number '"NAN"' --> 'NaN'
+    df["raw_drop_number"] = df["raw_drop_number"].str.replace('"NAN"', "NaN")
+
+    # Drop columns not needed
+    df = df.drop(columns=["record"])
+    return df
+
+
+def reader_met_file(filepath, logger):
+    """Reader MET file."""
+    ##------------------------------------------------------------------------.
+    #### Define column names
+    column_names = ["TO_SPLIT"]
+
+    ##------------------------------------------------------------------------.
+    #### Define reader options
+    reader_kwargs = {}
+
+    # - Define delimiter
+    reader_kwargs["delimiter"] = "\\n"
+
+    # - Skip first row as columns names
+    reader_kwargs["header"] = None
+
+    # - Skip header
+    reader_kwargs["skiprows"] = 4
+
+    # - Define encoding
+    reader_kwargs["encoding"] = "ISO-8859-1"
+
+    # - Avoid first column to become df index !!!
+    reader_kwargs["index_col"] = False
+
+    # - Define behaviour when encountering bad lines
+    reader_kwargs["on_bad_lines"] = "skip"
+
+    # - Define reader engine
+    #   - C engine is faster
+    #   - Python engine is more feature-complete
+    reader_kwargs["engine"] = "python"
+
+    # - Define on-the-fly decompression of on-disk data
+    #   - Available: gzip, bz2, zip
+    # reader_kwargs['compression'] = 'xz'
+
+    # - Strings to recognize as NA/NaN and replace with standard NA flags
+    #   - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
+    #                       '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
+    #                       'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
+    reader_kwargs["na_values"] = ["na", "error", "-.-", " NA", "NAN"]
+
+    ##------------------------------------------------------------------------.
+    #### Read the data
+    df = read_raw_text_file(
+        filepath=filepath,
+        column_names=column_names,
+        reader_kwargs=reader_kwargs,
+        logger=logger,
+    )
+
+    ##------------------------------------------------------------------------.
+    #### Adapt the dataframe to adhere to DISDRODB L0 standards
+    # Remove corrupted rows
+    df = df[df["TO_SPLIT"].str.count(",") == 40]
+
+    # Split into columns
+    df = df["TO_SPLIT"].str.split(",", expand=True)
+
+    # Assign columns names
+    names = [
+        "time",
+        "RECORD",
+        "PWS100_Year",
+        "PWS100_Month",
+        "PWS100_Day",
+        "PWS100_Hours",
+        "PWS100_Minutes",
+        "PWS100_Seconds",
+        "mor_visibility",
+        "weather_code_synop_4680",
+        "weather_code_metar_4678",
+        "weather_code_nws",
+        "PWS100_PWCode_NWS_String",
+        "air_temperature",
+        "relative_humidity",
+        "air_temperature_min",
+        "air_temperature_max",
+        "rainfall_rate",
+        "rainfall_accumulated",
+        "average_drop_velocity",
+        "average_drop_size",
+        "PWS100_PartType_Drizzle",
+        "PWS100_PartType_FreezingDrizzle",
+        "PWS100_PartType_Rain",
+        "PWS100_PartType_FreezingRain",
+        "PWS100_PartType_SnowGrains",
+        "PWS100_PartType_SnowFlakes",
+        "PWS100_PartType_IcePellets",
+        "PWS100_PartType_Hail",
+        "PWS100_PartType_Graupel",
+        "PWS100_PartType_Error",
+        "PWS100_PartType_Unknown",
+        "PWS100_VISAlarm1",
+        "PWS100_VISAlarm2",
+        "PWS100_VISAlarm3",
+        "PWS100_CleanLaserWindow",
+        "PWS100_CleanUpperWindow",
+        "PWS100_CleanLowerWindow",
+        "sensor_status",
+        "PWS100_FaultStatus_EN",
+        "PWS100_PowerStatus",
+    ]
+    df.columns = names
+
+    # Remove rows with only NaN
+    df = df[df["PWS100_Year"] != '"NAN"']
+
+    # Define type distribution variable
+    type_distribution_columns = [
+        "PWS100_PartType_Drizzle",
+        "PWS100_PartType_FreezingDrizzle",
+        "PWS100_PartType_Rain",
+        "PWS100_PartType_FreezingRain",
+        "PWS100_PartType_SnowGrains",
+        "PWS100_PartType_SnowFlakes",
+        "PWS100_PartType_IcePellets",
+        "PWS100_PartType_Hail",
+        "PWS100_PartType_Graupel",
+        "PWS100_PartType_Error",
+        "PWS100_PartType_Unknown",
+    ]
+    df["type_distribution"] = df[type_distribution_columns].agg(",".join, axis=1)
+
+    # Define alarms
+    # - should be 16 values
+    # alarms_columns = [
+    #     "PWS100_VISAlarm1",
+    #     "PWS100_VISAlarm2",
+    #     "PWS100_VISAlarm3",
+    #     "PWS100_CleanLaserWindow",
+    #     "PWS100_CleanUpperWindow",
+    #     "PWS100_CleanLowerWindow",
+    #     "PWS100_FaultStatus",
+    #     "PWS100_FaultStatus_EN",
+    #     "PWS100_PowerStatus",
+    # ]
+    # df["alarms"] = df[alarms_columns].agg(",".join, axis=1)
+
+    # Define datetime "time" column from filename
+    df["time"] = df["time"].str.replace('"', "")
+    df["time"] = pd.to_datetime(df["time"], format="%Y-%m-%d %H:%M:%S")
+
+    # # Drop columns not agreeing with DISDRODB L0 standards
+    columns_to_drop = [
+        "RECORD",
+        "PWS100_Year",
+        "PWS100_Month",
+        "PWS100_Day",
+        "PWS100_Hours",
+        "PWS100_Minutes",
+        "PWS100_Seconds",
+        "PWS100_PartType_Drizzle",
+        "PWS100_PartType_FreezingDrizzle",
+        "PWS100_PartType_Rain",
+        "PWS100_PartType_FreezingRain",
+        "PWS100_PartType_SnowGrains",
+        "PWS100_PartType_SnowFlakes",
+        "PWS100_PartType_IcePellets",
+        "PWS100_PartType_Hail",
+        "PWS100_PartType_Graupel",
+        "PWS100_PartType_Error",
+        "PWS100_PartType_Unknown",
+        "PWS100_VISAlarm1",
+        "PWS100_VISAlarm2",
+        "PWS100_VISAlarm3",
+        "PWS100_CleanLaserWindow",
+        "PWS100_CleanUpperWindow",
+        "PWS100_CleanLowerWindow",
+        "PWS100_FaultStatus_EN",
+        "PWS100_PowerStatus",
+        "PWS100_PWCode_NWS_String",
+    ]
+    df = df.drop(columns=columns_to_drop)
+    return df
+
+
+@is_documented_by(reader_generic_docstring)
+def reader(
+    filepath,
+    logger=None,
+):
+    """Reader."""
+    # Retrieve spectrum filepath
+    spectrum_filepath = filepath.replace("WS_MET_PWS100_Data", "WS_MET_Size_Vel_distr")
+
+    # Read integral variables
+    df = reader_met_file(filepath, logger=logger)
+
+    # Drop duplicates timesteps
+    df = df.drop_duplicates(subset="time", keep="first")
+
+    # Initialize raw_drop_number array
+    # --> 0 values array produced in L0B
+    df["raw_drop_number"] = ""
+
+    # Add raw spectrum if available
+    if os.path.exists(spectrum_filepath):
+        # Read raw spectrum for corresponding timesteps
+        df_raw_spectrum = reader_spectrum(spectrum_filepath, logger=logger)
+        df_raw_spectrum = df_raw_spectrum.drop_duplicates(subset="time", keep="first")
+        # Add raw array to df
+        df = df.set_index("time")
+        df_raw_spectrum = df_raw_spectrum.set_index("time")
+        df.update(df_raw_spectrum)
+        # Set back time as column
+        df = df.reset_index()
+
+    # Return the dataframe adhering to DISDRODB L0 standards
+    return df

disdrodb/l0/readers/SW250/BELGIUM/KMI.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""DISDRODB reader for KMI Biral SW250 sensors."""
+import pandas as pd
+
+from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
+from disdrodb.l0.l0a_processing import read_raw_text_file
+
+
+def parse_spectrum_line_to_string(line):
+    """Parse one M... spectrum line into a zero-padded string with 21 values."""
+    # Define number of velocity bins values expected
+    n_cols = 21
+
+    # Split spectrum line
+    parts = line.split(",")
+
+    # Check line validity
+    n_values = len(parts)
+    if n_values > n_cols:
+        raise ValueError(f"Unexpected number of velocity bins: {n_values}.")
+
+    # Strip  'M' from first bin
+    parts[0] = parts[0].replace(":00M", "")
+
+    # Strip last two letter from last value
+    parts[-1] = parts[-1][:3]
+
+    # Define list of values
+    values = [int(x) for x in parts]
+    if len(values) < n_cols:
+        values.extend([0] * (n_cols - len(values)))
+    values = values[:n_cols]
+
+    # Define comma-separated string
+    string = ",".join(str(v) for v in values)
+    return string
+
+
+def parse_spectrum_block(lines):
+    """Parse an M-block into a fixed (16 x 21) matrix."""
+    n_values = len(lines)
+    if n_values != 16:
+        raise ValueError(f"Unexpected number of diameter bins: {n_values}.")
+    raw_drop_number_string = ",".join([parse_spectrum_line_to_string(line) for line in lines])
+    return raw_drop_number_string
+
+
+def build_spectrum_block(group):
+    """Create SWS250 raw spectrum string."""
+    try:
+        return pd.Series(
+            {
+                "raw_drop_number": parse_spectrum_block(group["spectrum_line"].tolist()),
+            },
+        )
+    except Exception:
+        return pd.Series({"raw_drop_number": "NaN"})
+
+
+@is_documented_by(reader_generic_docstring)
+def reader(
+    filepath,
+    logger=None,
+):
+    """Reader."""
+    ##------------------------------------------------------------------------.
+    #### Define raw data headers
+    column_names = ["TO_PARSE"]
+
+    ##------------------------------------------------------------------------.
+    #### Define reader options
+    # - For more info: https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
+    reader_kwargs = {}
+
+    # - Define delimiter
+    reader_kwargs["delimiter"] = "\\n"
+
+    # - Avoid first column to become df index !!!
+    reader_kwargs["index_col"] = False
+
+    # Since column names are expected to be passed explicitly, header is set to None
+    reader_kwargs["header"] = None
+
+    # - Number of rows to be skipped at the beginning of the file
+    reader_kwargs["skiprows"] = None
+
+    # - Define behaviour when encountering bad lines
+    reader_kwargs["on_bad_lines"] = "skip"
+
+    # - Define reader engine
+    #   - C engine is faster
+    #   - Python engine is more feature-complete
+    reader_kwargs["engine"] = "python"
+
+    # - Define on-the-fly decompression of on-disk data
+    #   - Available: gzip, bz2, zip
+    reader_kwargs["compression"] = "infer"
+
+    # - Skip rows with badly encoded data
+    reader_kwargs["encoding_errors"] = "replace"
+
+    # - Strings to recognize as NA/NaN and replace with standard NA flags
+    #   - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
+    #                       '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
+    #                       'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
+    reader_kwargs["na_values"] = ["na", "", "error"]
+
+    ##------------------------------------------------------------------------.
+    #### Read the data
+    df = read_raw_text_file(
+        filepath=filepath,
+        column_names=column_names,
+        reader_kwargs=reader_kwargs,
+        logger=logger,
+    )
+
+    ##------------------------------------------------------------------------.
+    #### Adapt the dataframe to adhere to DISDRODB L0 standards
+    # Identify rows with data
+    df_params = df[df["TO_PARSE"].str.count(",") == 23]
+
+    # Identify rows with spectrum matrix
+    df_spectrum = df[df["TO_PARSE"].str.startswith(":00M")]
+    if len(df_spectrum) == 0:
+        raise ValueError("No spectrum available.")
+
+    df_spectrum = df_spectrum["TO_PARSE"].str.rsplit(",", expand=True, n=2)
+    df_spectrum.columns = ["spectrum_line", "date", "time"]
+    df_spectrum["datetime"] = pd.to_datetime(
+        df_spectrum["date"] + " " + df_spectrum["time"],
+        format="%d/%m/%Y %H:%M:%S",
+    )
+
+    # Define groups
+    # - Mark new group when time gap > 10 s
+    is_new_group = (df_spectrum["datetime"].diff().dt.total_seconds() > 10).fillna(True)
+    group_id = is_new_group.cumsum()
+    # - Assign the first datetime of each group
+    df_spectrum["group_time"] = df_spectrum.groupby(group_id)["datetime"].transform("first")
+
+    # Group spectrum by timesteps
+    df_raw_drop_number = (
+        df_spectrum.groupby("group_time", as_index=False)
+        .apply(build_spectrum_block, include_groups=False)
+        .reset_index(drop=True)
+    )
+
+    # Retrieve 1-min data
+    # - Split by ; delimiter (before raw drop number)
+    df_data = df_params["TO_PARSE"].str.split(",", expand=True)
+
+    # - Assign column names
+    names = [
+        "date",
+        "time",
+        "sws250",
+        "sensor_id",
+        "sample_interval",
+        "mor_visibility_5min",  # remove unit and ensure in meters  !
+        "weather_code_synop_4680",
+        "past_weather1",
+        "past_weather2",
+        "obstruction_status",
+        "weather_code_metar_4678",
+        "precipitation_rate",
+        "mor_visibility",  # remove unit and ensure in meters  !
+        "total_extinction_coefficient",  # [km-1]
+        "transmissometer_extinction_coefficient",  # [km-1]
+        "back_scatter_extinction_coefficient",  # [km-1]
+        "sensor_temperature",  # [degrees]  or air_temperature ?
+        "ambient_light_sensor_signal",  # [cd/m2] # ALS
+        "sensor_status",
+        "number_particles",
+        "precipitation_accumulated",  # [mm] over sample_interval
+        "ambient_light_sensor_signal_status",
+        "date1",
+        "time1",
+    ]
+    df_data.columns = names
+
+    # Clean out variables
+    df_data["mor_visibility_5min"] = df_data["mor_visibility_5min"].str.replace("M", "")
+    df_data["mor_visibility"] = df_data["mor_visibility"].str.replace("M", "")
+    df_data["sensor_temperature"] = df_data["sensor_temperature"].str.replace("C", "")
+    df_data["ambient_light_sensor_signal"] = df_data["ambient_light_sensor_signal"].str.replace("+99999", "NaN")
+
+    # Define datetime
+    df_data["datetime"] = pd.to_datetime(df_data["date1"] + " " + df_data["time1"], format="%d/%m/%Y %H:%M:%S")
+
+    # Merge df_data on df_raw_drop_number
+    # TODO list
+    # - should we aggregate variables to 5 min temporal resolution
+    #   to match raw_drop_number
+    # - should we infill df_raw_drop_number with 0 when no time every 5 min?
+    df = pd.merge_asof(
+        df_raw_drop_number,
+        df_data,
+        left_on="group_time",
+        right_on="datetime",
+        direction="nearest",
+        tolerance=pd.Timedelta("10s"),  # max difference allowed
+    )
+
+    # Define final time
+    # TODO list
+    # - which time should we take as final time?
+    # - raw_drop_number time is end of measurement interval right?
+    df["time"] = df["group_time"]
+
+    # Drop columns not agreeing with DISDRODB L0 standards
+    columns_to_drop = [
+        "group_time",
+        "date",
+        "sws250",
+        "sample_interval",
+        "sensor_id",
+        "date1",
+        "time1",
+        "datetime",
+    ]
+    df = df.drop(columns=columns_to_drop)
+    return df