rainfallqc 0.0.9__py3-none-any.whl

rainfallqc/__init__.py

@@ -0,0 +1,22 @@
+"""Top-level package for RainfallQC."""
+
+__author__ = """Tom Keel"""
+__email__ = "tomkee@ceh.ac.uk"
+__version__ = "0.0.9"
+
+from rainfallqc.checks import (
+    comparison_checks,
+    gauge_checks,
+    neighbourhood_checks,
+    timeseries_checks,
+)
+from rainfallqc.qc_frameworks import apply_qc_framework
+
+__all__ = [
+    "apply_qc_framework",
+    "comparison_checks",
+    "gauge_checks",
+    "neighbourhood_checks",
+    "timeseries_checks",
+    "apply_qc_framework",
+]

rainfallqc/checks/__init__.py

@@ -0,0 +1 @@
+"""Quality checks."""

rainfallqc/checks/comparison_checks.py

@@ -0,0 +1,342 @@
+# -*- coding: utf-8 -*-
+"""
+Quality control checks relying on comparison with a benchmark dataset.
+
+Comparison checks are defined as QC checks that: "detect abnormalities in rainfall record based on benchmarks."
+
+Classes and functions ordered by appearance in IntenseQC framework.
+"""
+
+import numpy as np
+import polars as pl
+import xarray as xr
+
+from rainfallqc.utils import data_readers, neighbourhood_utils, stats
+
+
+def check_annual_exceedance_etccdi_r99p(
+    data: pl.DataFrame, rain_col: str, gauge_lat: int | float, gauge_lon: int | float
+) -> list:
+    """
+    Check annual exceedance of maximum R99p from ETCCDI dataset.
+
+    This is QC8 from the IntenseQC framework.
+
+    Parameters
+    ----------
+    data :
+        Rainfall data
+    rain_col :
+        Column with rainfall data
+    gauge_lat :
+        latitude of the rain gauge
+    gauge_lon :
+        longitude of the rain gauge
+
+    Returns
+    -------
+    flag_list :
+        List of flags
+
+    """
+    # 1. Load R99p data
+    etccdi_r99p = data_readers.load_etccdi_data(etccdi_var="R99p")
+
+    # 2. Get nearest local R99p value to the gauge coordinates
+    nearby_etccdi_r99p = neighbourhood_utils.get_nearest_etccdi_val_to_gauge(etccdi_r99p, gauge_lat, gauge_lon)
+
+    # 3. Get sum of rainfall above the 99th percentile per year
+    sum_rainfall_above_99percentile_per_year = get_sum_rainfall_above_percentile_per_year(
+        data, rain_col, percentile=0.99
+    )
+
+    # 4. Get flags of exceedance for R99p variable where the 0.99 percentile sum is more than ETCCDI max
+    exceedance_flags = flag_exceedance_of_max_etccdi_variable(
+        sum_rainfall_above_99percentile_per_year, rain_col, nearby_etccdi_r99p, etccdi_var="R99p"
+    )
+
+    return exceedance_flags
+
+
+def check_annual_exceedance_etccdi_prcptot(
+    data: pl.DataFrame, rain_col: str, gauge_lat: int | float, gauge_lon: int | float
+) -> list:
+    """
+    Check annual exceedance of maximum PRCPTOT from ETCCDI dataset.
+
+    This is QC9 from the IntenseQC framework.
+
+    Parameters
+    ----------
+    data :
+        Rainfall data
+    rain_col :
+        Column with rainfall data
+    gauge_lat :
+        latitude of the rain gauge
+    gauge_lon :
+        longitude of the rain gauge
+
+    Returns
+    -------
+    exceedance_flags :
+        List of flags (see `exceedance_flagger` function)
+
+    """
+    # 1. Load PRCPTOT data
+    etccdi_prcptot = data_readers.load_etccdi_data(etccdi_var="PRCPTOT")
+
+    # 2. Get nearest local PRCPTOT value to the gauge coordinates
+    nearby_etccdi_prcptot = neighbourhood_utils.get_nearest_etccdi_val_to_gauge(etccdi_prcptot, gauge_lat, gauge_lon)
+
+    # 3. Get sum of rainfall above the 99th percentile per year
+    sum_rainfall_above_99percentile_per_year = get_sum_rainfall_above_percentile_per_year(
+        data, rain_col, percentile=0.99
+    )
+
+    # 4. Get flags of exceedance for PRCPTOT variable where the 0.99 percentile sum is more than ETCCDI max
+    exceedance_flags = flag_exceedance_of_max_etccdi_variable(
+        sum_rainfall_above_99percentile_per_year, rain_col, nearby_etccdi_prcptot, etccdi_var="PRCPTOT"
+    )
+
+    return exceedance_flags
+
+
+def check_exceedance_of_rainfall_world_record(data: pl.DataFrame, rain_col: str, time_res: str) -> pl.DataFrame:
+    """
+    Check exceedance of rainfall world record.
+
+    See Also `utils/stats.py` from world record sources.
+
+    This is QC10 from the IntenseQC framework.
+
+    Parameters
+    ----------
+    data :
+        Rainfall data
+    rain_col :
+        Column with rainfall data
+    time_res :
+        Time resolution
+
+    Returns
+    -------
+    data_w_flags:
+        Rainfall data with exceedance of World Record (see `flag_exceedance_of_ref_val_as_col` function)
+
+    """
+    rainfall_world_records = stats.get_rainfall_world_records()
+    return flag_exceedance_of_ref_val_as_col(
+        data, rain_col, ref_val=rainfall_world_records[time_res], new_col_name="world_record_check"
+    )
+
+
+def check_annual_exceedance_etccdi_rx1day(
+    data: pl.DataFrame, rain_col: str, gauge_lat: int | float, gauge_lon: int | float
+) -> pl.DataFrame:
+    """
+    Check exceedance of rainfall world record.
+
+    See Also `utils/stats.py` from world record sources.
+
+    This is QC11 from the IntenseQC framework.
+
+    Parameters
+    ----------
+    data :
+        Rainfall data
+    rain_col :
+        Column with rainfall data
+    gauge_lat :
+        latitude of the rain gauge
+    gauge_lon :
+        longitude of the rain gauge
+
+    Returns
+    -------
+    data_w_flags:
+        Rainfall data with exceedance of Rx1day Record (see `flag_exceedance_of_ref_val_as_col` function)
+
+    """
+    # 1. Load Rx1day data
+    etccdi_rx1day = data_readers.load_etccdi_data(etccdi_var="Rx1day")
+
+    # 2. Get nearest local Rx1day value to the gauge coordinates
+    nearby_etccdi_rx1day = neighbourhood_utils.get_nearest_etccdi_val_to_gauge(etccdi_rx1day, gauge_lat, gauge_lon)
+
+    # 3. Get local maximum ETCCDI value
+    max_nearby_etccdi_rx1day = np.max(nearby_etccdi_rx1day["Rx1day"])
+
+    # 4. Flag exceedance of max ETCCDI value
+    return flag_exceedance_of_ref_val_as_col(
+        data, rain_col, ref_val=max_nearby_etccdi_rx1day, new_col_name="rx1day_check"
+    )
+
+
+def get_sum_rainfall_above_percentile_per_year(
+    data: pl.DataFrame,
+    rain_col: str,
+    percentile: float,
+) -> pl.DataFrame:
+    """
+    Check annual exceedance of maximum PRCPTOT from ETCCDI dataset.
+
+    Parameters
+    ----------
+    data :
+        Rainfall data
+    rain_col :
+        Column with rainfall data
+    percentile :
+        nth percentile to check for values above
+
+    Returns
+    -------
+    exceedance_flags :
+        List of flags (see `exceedance_flagger` function)
+
+    """
+    # 1. Add a daily year column to the data
+    data = add_daily_year_col(data)
+
+    # 2. Calculate percentiles
+    data_percentiles = data.group_by("year").agg(
+        pl.col(rain_col).fill_nan(0.0).quantile(percentile).alias("percentile")
+    )
+
+    # 3. Join percentiles back to the main DataFrame
+    data_yearly_percentiles = data.join(data_percentiles, on="year").fill_nan(0.0)
+
+    # 4. Filter values above the nth percentile
+    data_above_annual_percentile = data_yearly_percentiles.filter(pl.col(rain_col) > pl.col("percentile"))
+
+    # 5. Get number of values per year above nth percentile
+    sum_rainfall_above_percentile = data_above_annual_percentile.group_by_dynamic("time", every="1y").agg(
+        pl.col(rain_col).sum()
+    )
+
+    return sum_rainfall_above_percentile
+
+
+def flag_exceedance_of_max_etccdi_variable(
+    annual_sum_rainfall: pl.DataFrame, rain_col: str, nearby_etccdi_data: xr.Dataset, etccdi_var: str
+) -> list:
+    """
+    Flag exceedance of maximum ETCCDI variable, comparing the maximum sums of each year.
+
+    Parameters
+    ----------
+    annual_sum_rainfall :
+        Rainfall data as by year sums
+    rain_col :
+        Column with rainfall data
+    nearby_etccdi_data :
+        ETCCDI data with given variable to check
+    etccdi_var :
+        variable to load from ETCCDI
+
+    Returns
+    -------
+    exceedance_flags :
+        Flags of exceedances of max ETCCDI value
+
+    """
+    # 1. Get local maximum ETCCDI value
+    etccdi_var_max = np.max(nearby_etccdi_data[etccdi_var])
+
+    # 2. Get flags.
+    exceedance_flags = [
+        flag_exceedance_of_ref_val(val=yr, ref_val=etccdi_var_max) for yr in annual_sum_rainfall[rain_col]
+    ]
+    return exceedance_flags
+
+
+def add_daily_year_col(data: pl.DataFrame) -> pl.DataFrame:
+    """
+    Make a year column for the data. This method will first upsample data so that it is every day.
+
+    Parameters
+    ----------
+    data :
+        Rainfall data
+
+    Returns
+    -------
+    data_w_year_col :
+        Rainfall data with year column
+
+    """
+    data_daily_upsample = data.upsample("time", every="1d")
+    return data_daily_upsample.with_columns(pl.col("time").dt.year().alias("year"))
+
+
+def flag_exceedance_of_ref_val(val: int | float, ref_val: int | float) -> int:
+    """
+    Exceedance flagger from intenseqc.
+
+    Parameters
+    ----------
+    val :
+        Value to check
+    ref_val :
+        Reference value to compare against
+
+    Returns
+    -------
+    Flag :
+        Exceedance flag
+
+    """
+    if val is None or np.isnan(val):
+        return np.nan
+    elif val >= ref_val * 1.5:
+        return 4
+    elif val >= ref_val * 1.33:
+        return 3
+    elif val >= ref_val * 1.2:
+        return 2
+    elif val >= ref_val:
+        return 1
+    else:
+        return 0
+
+
+def flag_exceedance_of_ref_val_as_col(
+    data: pl.DataFrame, rain_col: str, ref_val: int | float, new_col_name: str
+) -> pl.DataFrame:
+    """
+    Flag exceedance of maximum reference value and return as column.
+
+    Used in QC11 of the IntenseQC framework. TODO: could this be used in QC8+9?
+
+    Parameters
+    ----------
+    data :
+        Rainfall data.
+    rain_col :
+        Column with rainfall data
+    ref_val :
+        Reference value.
+    new_col_name :
+        New column name.
+
+    Returns
+    -------
+    data :
+        Data with exceedance flags between 0-4.
+
+    """
+    return data.with_columns(
+        pl.when(pl.col(rain_col).is_null() | pl.col(rain_col).is_nan())
+        .then(np.nan)
+        .when(pl.col(rain_col) >= ref_val * 1.5)
+        .then(4)
+        .when(pl.col(rain_col) >= ref_val * 1.33)
+        .then(3)
+        .when(pl.col(rain_col) >= ref_val * 1.2)
+        .then(2)
+        .when(pl.col(rain_col) >= ref_val)
+        .then(1)
+        .otherwise(0)
+        .alias(new_col_name)
+    )

rainfallqc/checks/gauge_checks.py

@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+"""
+Quality control checks examining suspicious rain gauges.
+
+Gauge checks are defined as QC checks that: "detect abnormalities in summary and descriptive statistics of rain gauges."
+
+Classes and functions ordered by appearance in IntenseQC framework.
+"""
+
+import polars as pl
+import scipy.stats
+
+from rainfallqc.utils import data_utils, stats
+
+
+def check_years_where_nth_percentile_is_zero(data: pl.DataFrame, rain_col: str, quantile: float) -> list:
+    """
+    Return years where the n-th percentiles is zero.
+
+    This is QC1 from the IntenseQC framework
+
+    Parameters
+    ----------
+    data :
+        Rainfall data
+    rain_col :
+        Column with rainfall data
+    quantile :
+        Between 0 & 1
+
+    Returns
+    -------
+    year_list :
+        List of years where n-th percentile is zero.
+
+    """
+    nth_perc = data.group_by_dynamic("time", every="1y").agg(pl.quantile(rain_col, quantile))
+    return nth_perc.filter(pl.col(rain_col) == 0)["time"].dt.year().to_list()
+
+
+def check_years_where_annual_mean_k_top_rows_are_zero(data: pl.DataFrame, rain_col: str, k: int) -> list:
+    """
+    Return year list where the annual mean top-K rows are zero.
+
+    This is QC2 from the IntenseQC framework
+
+    Parameters
+    ----------
+    data :
+        Rainfall data
+    rain_col :
+        Column with rainfall data
+    k :
+        Number of top values check i.e. k==5 is top 5
+
+    Returns
+    -------
+    year_list :
+        List of years where k-largest are zero.
+
+    """
+    data_top_k = data.group_by_dynamic("time", every="1y").agg(pl.col(rain_col).top_k(k).min())
+    return data_top_k.filter(pl.col(rain_col) == 0)["time"].dt.year().to_list()
+
+
+def check_temporal_bias(
+    data: pl.DataFrame,
+    rain_col: str,
+    time_granularity: str,
+    p_threshold: float = 0.01,
+) -> int:
+    """
+    Perform a two-sided t-test on the distribution of mean rainfall over time slices.
+
+    This is QC3 (day of week bias) and QC4 (hour-of-day bias) from the IntenseQC framework.
+
+    Parameters
+    ----------
+    data :
+        Rainfall data
+    rain_col :
+        Column with rainfall data
+    time_granularity :
+        Temporal grouping, either 'weekday' or 'hour'
+    p_threshold :
+        Significance level for the test
+
+    Returns
+    -------
+    flag : int
+        1 if bias is detected (p < threshold), 0 otherwise
+
+    """
+    if time_granularity == "weekday":
+        time_group = pl.col("time").dt.weekday()
+    elif time_granularity == "hour":
+        time_group = pl.col("time").dt.hour()
+    else:
+        raise ValueError("time_granularity must be either 'weekday' or 'hour'")
+
+    # 1. Get time-average mean
+    grouped_means = data.group_by(time_group).agg(pl.col(rain_col).drop_nans().mean())[rain_col]
+
+    # 2. Get data mean
+    overall_mean = data[rain_col].drop_nans().mean()
+
+    # 3. Compute 1-sample t-test
+    _, p_val = scipy.stats.ttest_1samp(grouped_means, overall_mean)
+    return int(p_val < p_threshold)
+
+
+def check_intermittency(
+    data: pl.DataFrame, rain_col: str, no_data_threshold: int = 2, annual_count_threshold: int = 5
+) -> list:
+    """
+    Return years where more than five periods of missing data are bounded by zeros.
+
+    TODO: split into multiple sub-functions and write more tests!
+    This is QC5 from the IntenseQC framework.
+
+    Parameters
+    ----------
+    data :
+        Rainfall data
+    rain_col :
+        Column with rainfall data
+    no_data_threshold :
+        Number of missing values needed to be counted as a no data period (default: 2 (days))
+    annual_count_threshold :
+        Number of missing data periods above no_data_threshold per year (default: 5)
+
+    Returns
+    -------
+    years_w_intermittency :
+        List of years with intermittency issues.
+
+    """
+    # 1. Identify missing values
+    data = data_utils.replace_missing_vals_with_nan(data, rain_col)  # drops None by default
+    missing_vals_mask = data[rain_col].is_nan()
+    data = data.with_columns(missing_vals_mask.alias("is_missing"))
+
+    # 2. Identify group numbers for consecutive nulls
+    gauge_data_missing_groups = data.with_columns(
+        (pl.when(data["is_missing"]).then((~data["is_missing"]).cum_sum()).otherwise(None)).alias("group")
+    )
+
+    # 3. Get length of groups of missing data
+    gauge_data_missing_group_counts = gauge_data_missing_groups.group_by("group").agg(
+        pl.col("is_missing").sum().alias("count")
+    )
+
+    # 4. Get groups with missing values above or at the `no_data_threshold`
+    no_data_period_groups = gauge_data_missing_group_counts.filter(pl.col("count") >= no_data_threshold)["group"]
+
+    # 5. Select rows belonging to 'no data periods'
+    gauge_data_no_data_periods = gauge_data_missing_groups.filter(
+        pl.col("group").is_in(no_data_period_groups.to_list())
+    )
+
+    # 6. Get annual counts of no data periods
+    gauge_data_year_counts = gauge_data_no_data_periods.select(pl.col("time").dt.year()).to_series().value_counts()
+
+    # 7. Filter out years above or at the threshold of `annual_count_threshold`
+    years_w_intermittency = gauge_data_year_counts.filter(pl.col("count") >= annual_count_threshold)["time"].to_list()
+    return years_w_intermittency
+
+
+def check_breakpoints(
+    data: pl.DataFrame,
+    rain_col: str,
+    p_threshold: float = 0.01,
+) -> int:
+    """
+    Use a Pettitt test rainfall data to check for breakpoints.
+
+    This is QC6 from the IntenseQC framework.
+
+    Parameters
+    ----------
+    data :
+        Rainfall data.
+    rain_col :
+        Column with rainfall data.
+    p_threshold :
+        Significance level for the test.
+
+    Returns
+    -------
+    flag : int
+        1 if breakpoint is detected (p < p_threshold), 0 otherwise
+
+    """
+    # 1. Upsample data to daily
+    data_upsampled = data.upsample("time", every="1d")
+
+    # 2. Compute Pettitt test for breakpoints
+    _, p_val = stats.pettitt_test(data_upsampled[rain_col].fill_nan(0.0))
+    if p_val < p_threshold:
+        return 1
+    else:
+        return 0
+
+
+def check_min_val_change(data: pl.DataFrame, rain_col: str, expected_min_val: float) -> list:
+    """
+    Return years when the minimum recorded value changes.
+
+    Used to determine whether there are possible changes to the measuring equipment.
+    This is QC7 from the IntenseQC framework.
+
+    Parameters
+    ----------
+    data :
+        Rainfall data
+    rain_col :
+        Column with rainfall data.
+    expected_min_val :
+        Expected value of rainfall i.e. basically the resolution of data.
+
+    Returns
+    -------
+    yr_list :
+        List of years with minimum value changes.
+
+    """
+    # 1. Filter out non-zero years
+    data_non_zero = data.filter(pl.col(rain_col) > 0)
+
+    # 2. Get minimum value each year
+    data_min_by_year = data_non_zero.group_by_dynamic(pl.col("time"), every="1y").agg(pl.col(rain_col).min())
+
+    non_res_years = data_min_by_year.filter(pl.col(rain_col) != expected_min_val)
+    return non_res_years["time"].dt.year().to_list()