pypromice 1.5.3__py3-none-any.whl → 1.6.0__py3-none-any.whl

pypromice/__init__.py

@@ -0,0 +1,2 @@
+from importlib.metadata import version
+__version__ = version("pypromice")

pypromice/{qc → core/qc}/percentiles/compute_thresholds.py

@@ -3,10 +3,10 @@ from datetime import datetime
 
 import pandas as pd
 
-from pypromice.process import AWS
+from pypromice.pipeline.aws import AWS
 from pathlib import Path
 import logging
-from pypromice.qc.github_data_issues import adjustTime, flagNAN, adjustData
+from pypromice.core.qc.github_data_issues import adjustTime, flagNAN, adjustData
 
 
 # %%

pypromice/{qc → core/qc}/persistence.py

@@ -19,27 +19,22 @@ DEFAULT_VARIABLE_THRESHOLDS = {
     "t_i": {"max_diff": 0.0001, "period": 2},
     "t_u": {"max_diff": 0.0001, "period": 2},
     "t_l": {"max_diff": 0.0001, "period": 2},
-    "p_i": {"max_diff": 0.0001, "period": 2},
-    # "p_u": {"max_diff": 0.0001, "period": 2},
-    # "p_l": {"max_diff": 0.0001, "period": 2},
-    "gps_lat_lon": {
-        "max_diff": 0.000001,
-        "period": 6,
-    },  # gets special handling to remove simultaneously constant gps_lat and gps_lon
+
+    "p_i": {"max_diff": 0.0001, "period": 3},
+    "p_u": {"max_diff": 0.0001, "period": 150},
+    "p_l": {"max_diff": 0.0001, "period": 150},
+
+    # gets special handling to remove simultaneously constant gps_lat and gps_lon
+    "gps_lat_lon": {"max_diff": 0.000001, "period": 6},
+
     "gps_alt": {"max_diff": 0.0001, "period": 6},
     "t_rad": {"max_diff": 0.0001, "period": 2},
-    "rh_i": {
-        "max_diff": 0.0001,
-        "period": 2,
-    },  # gets special handling to allow constant 100%
-    "rh_u": {
-        "max_diff": 0.0001,
-        "period": 2,
-    },  # gets special handling to allow constant 100%
-    "rh_l": {
-        "max_diff": 0.0001,
-        "period": 2,
-    },  # gets special handling to allow constant 100%
+
+    # gets special handling to allow constant 100%
+    "rh_i": {"max_diff": 0.0001, "period": 2},
+    "rh_u": {"max_diff": 0.0001, "period": 2},
+    "rh_l": {"max_diff": 0.0001, "period": 2},
+
     "wspd_i": {"max_diff": 0.0001, "period": 6},
     "wspd_u": {"max_diff": 0.0001, "period": 6},
     "wspd_l": {"max_diff": 0.0001, "period": 6},
@@ -83,15 +78,11 @@ def persistence_qc(
         variable_thresholds = DEFAULT_VARIABLE_THRESHOLDS
         logger.debug(f"Running persistence_qc using {variable_thresholds}")
     else:
-        logger.info(f"Running persistence_qc using custom thresholds:\n {variable_thresholds}")    
+        logger.info(f"Running persistence_qc using custom thresholds:\n {variable_thresholds}")
 
     for k in variable_thresholds.keys():
         if k in ["t", "p", "rh", "wspd", "wdir", "z_boom"]:
-            var_all = [
-                k + "_u",
-                k + "_l",
-                k + "_i",
-            ]  # apply to upper, lower boom, and instant
+            var_all = [k + l for l in ["_u", "_l", "_i"]]  # apply to upper, lower boom, and instant
         else:
             var_all = [k]
         max_diff = variable_thresholds[k]["max_diff"]  # loading persistent limit
@@ -140,10 +131,12 @@ def find_persistent_regions(
     """
     Algorithm that ensures values can stay the same within the outliers_mask
     """
-    consecutive_true_df = count_consecutive_persistent_values(data, max_diff)
-    persistent_regions = consecutive_true_df >= min_repeats
-    # Ignore entries which already nan in the input data
-    persistent_regions[data.isna()] = False
+    consecutive_true_df  = count_consecutive_persistent_values(data, max_diff)
+    persistent_regions = consecutive_true_df  >= min_repeats
+    for i in range(1, min_repeats):
+        persistent_regions |= persistent_regions.shift(-1, fill_value=False)
+        # Ignore entries which already nan in the input data
+        persistent_regions[data.isna()] = False
     return persistent_regions
 
 

pypromice/{process → core/qc}/value_clipping.py

@@ -2,7 +2,7 @@ import numpy as np
 import pandas
 import xarray
 
-from pypromice.utilities.dependency_graph import DependencyGraph
+from pypromice.core.dependency_graph import DependencyGraph
 
 
 def clip_values(
@@ -24,11 +24,11 @@ def clip_values(
     ds : `xarray.Dataset`
         Dataset with clipped data
     """
-    cols = ["lo", "hi", "OOL"]
+    cols = ["lo", "hi", "dependent_variables"]
     assert set(cols) <= set(var_configurations.columns)
 
     variable_limits = var_configurations[cols].assign(
-        dependents=lambda df: df.OOL.fillna("").str.split(),
+        dependents=lambda df: df.dependent_variables.fillna("").str.split(),
         # Find the closure of dependents using the DependencyGraph class
         dependents_closure=lambda df: DependencyGraph.from_child_mapping(
             df.dependents

pypromice/core/variables/__init__.py

@@ -0,0 +1 @@
+

pypromice/core/variables/air_temperature.py

@@ -0,0 +1,64 @@
+__all__=["clip_and_interpolate", "get_cloud_coefficients"]
+
+import pandas as pd
+import xarray as xr
+
+T_0=273.15                  # degrees Celsius to Kelvin conversion
+eps_overcast = 1.0          # Clouds overcast default coefficient
+eps_clear = 9.36508e-6      # Clouds clear default coefficient
+
+def clip_and_interpolate(temp : xr.DataArray,
+                         lo : float,
+                         hi : float,
+                         max_interp : pd.Timedelta = pd.Timedelta(12,'h')
+) -> xr.DataArray:
+    """Clip and interpolate temperature dataset for use in
+    corrections
+
+    Parameters
+    ----------
+    temp : `xr.DataArray`
+        Array of temperature data
+    lo : float
+        Minimum threshold value for clipping
+    hi : float
+        Maximum threshold value for clipping
+    max_interp : `pd.Timedelta`
+        Maximum time steps to interpolate across.
+        The default is 12 hours.
+
+    Returns
+    -------
+    temp_interp : `xr.DataArray`
+        Array of interpolated temperature data
+    """
+    # Clip values to high and low threshold values
+    temp = temp.where((temp >= lo) & (temp <= hi))
+
+    # Drop duplicates and interpolate across NaN values
+    temp_interp = temp.interpolate_na(dim='time',
+                                      max_gap=max_interp)
+
+    return temp_interp
+
+
+def get_cloud_coefficients(temp: xr.DataArray
+) -> tuple[xr.DataArray, xr.DataArray]:
+    """Get overcast and clear cloud longwave coefficients using
+    air temperature, based on assumptions from Swinbank (1963)
+
+    Parameters
+    ----------
+    temp : xr.DataArray
+        Air temperature
+
+    Returns
+    -------
+    LR_overcast : xr.DataArray
+        Overcast cloud coefficients, using overcast cloud assumption from Swinbank (1963)
+    LR_clear : xr.DataArray
+        Clear cloud coefficients, using clear cloud assumption, from Swinbank (1963)
+    """
+    LR_overcast = eps_overcast * 5.67e-8 * (temp + T_0) ** 4
+    LR_clear = eps_clear * 5.67e-8 * (temp + T_0) ** 6
+    return LR_overcast, LR_clear

pypromice/core/variables/gps.py

@@ -0,0 +1,221 @@
+__all__ = ["decode_and_convert",  "filter",
+           "decode", "convert_from_degrees_and_decimal_minutes",
+           "convert_from_decimal_minutes"]
+import re
+import xarray as xr
+import numpy as np
+import pandas as pd
+from sklearn.linear_model import LinearRegression
+
+import logging
+logger = logging.getLogger(__name__)
+
+def decode_and_convert(gps_lat: xr.DataArray,
+                       gps_lon: xr.DataArray,
+                       gps_time: xr.DataArray,
+                       latitude: float,
+                       longitude: float
+) -> tuple[xr.DataArray,xr.DataArray,xr.DataArray]:
+    """Decode and convert GPS latitude, longtitude and time values."flag_decimal_minutes",
+           "flag_for_decoding"
+    Decoding is performed if values are detected as string types.
+    Conversion consists of transforming to decimal degrees (DD),
+    from either decimal minutes (mm.mmmmm) or degrees and
+    decimal minutes (ddmm.mmmm)
+
+    Parameters
+    ----------
+    gps_lat : `xr.DataArray`
+        GPS latitude
+    gps_lon : `xr.DataArray`
+        GPS longitude
+    gps_time : `xr.DataArray`
+        GPS time
+
+    Returns
+    -------
+    gps_lat : `xr.DataArray`
+        Decoded and converted GPS latitude
+    gps_lon : `xr.DataArray`
+        Decoded and converted GPS longitude
+    gps_time : `xr.DataArray`
+        Decoded and converted GPS time
+    """
+    # Retain GPS array attributes
+    lat_attrs = gps_lat.attrs
+    lon_attrs = gps_lon.attrs
+    time_attrs = gps_time.attrs
+
+    # Decode GPS information if array is an object array
+    if gps_lat.dtype.kind == "O":
+        lat, lon, time = decode(gps_lat, gps_lon, gps_time)
+        if lat is None:
+            logger.warning("GPS decoding failed, skipping this routine.")
+        else:
+            gps_lat, gps_lon, gps_time = lat, lon, time
+
+    # Reformat values to numeric
+    gps_lat.values = pd.to_numeric(gps_lat, errors='coerce')
+    gps_lon.values = pd.to_numeric(gps_lon, errors='coerce')
+    gps_time.values = pd.to_numeric(gps_time, errors='coerce')
+
+    # Convert GPS positions to decimal degrees
+    if np.any((gps_lat <= 90) & (gps_lat > 0)):
+        gps_lat = convert_from_decimal_minutes(gps_lat, latitude)
+        gps_lon = convert_from_decimal_minutes(gps_lon, longitude)
+    else:
+        gps_lat = convert_from_degrees_and_decimal_minutes(gps_lat)
+        gps_lon = convert_from_degrees_and_decimal_minutes(gps_lon)
+
+    # Reassign GPS array attributes
+    gps_lat.attrs = lat_attrs
+    gps_lon.attrs = lon_attrs
+    gps_time.attrs = time_attrs
+
+    return gps_lat, gps_lon, gps_time
+
+
+def filter(gps_lat: xr.DataArray,
+           gps_lon: xr.DataArray,
+           gps_alt: xr.DataArray
+) -> tuple[xr.DataArray, xr.DataArray, xr.DataArray]:
+    """ Filter GPS latitude, longitude and altitude based on the difference
+    to a baseline elevation. The baseline elevation is a gap-filled monthly
+    median elevation based on the inputted GPS altitude.
+
+    Parameters
+    ----------
+    gps_lat : xr.DataArray
+        GPS latitude
+    gps_lon : xr.DataArray
+        GPS longitude
+    gps_alt : xr.DataArray
+        GPS altitude values with a time dimension
+
+    Returns
+    ----------
+    gps_lat_filtered : xr.DataArray
+        Filtered latitude values
+    gps_lon_filtered : xr.DataArray
+        Filtered longitude values
+    gps_alt_filtered : xr.DataArray
+        Filtered altitude values
+    """
+    # Get altitude monthly median (at month start)
+    # This will serve as baseline elevations for filtering
+    ser = gps_alt.to_series()
+    monthly_median = ser.resample("MS").median()
+    baseline_elevation = (
+        monthly_median
+        .reindex(ser.index, method="nearest")
+        .ffill()
+        .bfill()
+    )
+
+    # Produce conditional mask
+    mask = (np.abs(gps_alt - baseline_elevation) < 100) | gps_alt.isnull()
+
+    # Apply mask
+    gps_lat_filtered = gps_lat.where(mask)
+    gps_lon_filtered = gps_lon.where(mask)
+    gps_alt_filtered = gps_alt.where(mask)
+
+    return gps_lat_filtered, gps_lon_filtered, gps_alt_filtered
+
+
+def convert_from_degrees_and_decimal_minutes(gps):
+    """Convert positions (i.e. latitude, longitude) from degrees
+    and decimal minutes (ddmm.mmmm) to decimal degree values (DD)"""
+    return np.floor(gps / 100) + (gps / 100 - np.floor(gps / 100)) * 100 / 60
+
+
+def convert_from_decimal_minutes(gps: xr.DataArray, pos: float
+) -> xr.DataArray:
+    """Convert decimal minutes (mm.mmmmm) to decimal degree
+    values (DD), using a predefined position to append values to.
+    Needed in the case of PROMICE v1 stations, where logger
+    programs saved positions only in decimal minutes."""
+    new_gps = np.sign(pos) * (gps + 100 * np.floor(np.abs(pos)))
+    return convert_from_degrees_and_decimal_minutes(new_gps)
+
+
+def decode(gps_lat: xr.DataArray,
+           gps_lon: xr.DataArray,
+           gps_time: xr.DataArray
+) -> tuple[xr.DataArray,xr.DataArray,xr.DataArray]:
+    """Decode GPS information. This should be applied if gps information
+    consists of strings and not float values. GPS information is returned in
+    decimal degrees (ddmm.mmmm) format.
+
+    Parameters
+    ----------
+    gps_lat : `xr.DataArray`
+        GPS latitude
+    gps_lon : `xr.DataArray`
+        GPS longitude
+    gps_time : `xr.DataArray`
+        GPS time
+
+    Returns
+    -------
+    new_lat : `xr.DataArray`
+        Decoded GPS latitude
+    new_lon : `xr.DataArray`
+        Decoded GPS longitude
+    new_time : `xr.DataArray`
+        Decoded GPS time
+    """
+    # Pick the first non-null sample safely and detect decoding format
+    non_null = gps_lat.dropna(dim='time').values
+    sample_value = str(non_null[0])
+
+    try:
+        # Object decoding
+        if "NH" in sample_value:
+            new_lat = gps_object_decoder(gps_lat)
+            new_lon = gps_object_decoder(gps_lon)
+            new_time = gps_object_decoder(gps_time)
+            return new_lat, new_lon, new_time
+
+        # L-string decoding
+        elif "L" in sample_value:
+            logger.info("Found 'L' in GPS string; applying decode + scaling.")
+            new_lat = gps_l_string_decoder(gps_lat)
+            new_lon = gps_l_string_decoder(gps_lon)
+            new_time = gps_object_decoder(gps_time)
+            return new_lat, new_lon, new_time
+
+        # Unknown format, attempt to decode
+        else:
+            logger.info("Unknown GPS string format; attempting generic decode.")
+            new_lat = gps_object_decoder(gps_lat)
+            new_lon = gps_object_decoder(gps_lon)
+            new_time = gps_object_decoder(gps_time)
+            return new_lat, new_lon, new_time
+
+    except Exception as e:
+        logger.error(f"Failed to decode GPS data: {e!r} "
+                     f"(dtype={gps_lat.dtype})")
+        return None, None, None
+
+
+def gps_object_decoder(gps : xr.DataArray) -> xr.DataArray:
+    """GPS decoder for object array formatting. For example, PROMICE v2
+    stations should send information as 'NH6429.01544,WH04932.86061'
+    original formatting (NUK_L 2022); PROMICE v3 stations should send
+    coordinates as '6430,4916' (NUK_Uv3); and GC-Net stations should
+    send coordinates as '6628.93936',04617.59187' (DY2)"""
+    str2nums = [re.findall(r"[-+]?\d*\.\d+|\d+", _) if isinstance(_, str) else [np.nan] for _ in gps.values]
+    gps[:] = pd.DataFrame(str2nums).astype(float).T.values[0]
+    gps = gps.astype(float)
+    return gps
+
+
+def gps_l_string_decoder(gps : xr.DataArray) -> xr.DataArray:
+    """GPS L-string decoder"""
+    # Convert from object array
+    gps = gps_object_decoder(gps)
+
+    # Convert from integer-like values to degrees
+    gps = gps/100000
+    return gps

pypromice/core/variables/humidity.py

@@ -0,0 +1,111 @@
+__all__ = ["adjust", "convert", "calculate_specific_humidity"]
+
+import xarray as xr
+import numpy as np
+
+# Define constants
+T_0 = 273.15                # Ice point temperature (Kelvins)
+T_100 = T_0+100             # Steam point temperature (Kelvins)
+ews = 1013.246              # Saturation vapour pressure at steam point temperature (normal atmosphere) (hPa)
+ei0 = 6.1071                # Saturation vapour pressure at ice melting point temperature (normal atmosphere) (hPa)
+eps=0.622                   # Ratio of molar masses of vapor and dry air
+
+def adjust(rh: xr.DataArray,
+           t: xr.DataArray
+) -> xr.DataArray:
+    """Correct relative humidity so that values are given with respect to
+    saturation over ice in subfreezing conditions, and with respect to
+    saturation over water (as given by the instrument) above the melting
+    point temperature. Saturation water vapors are calculated after
+    Groff & Gratch method.
+
+    Parameters
+    ----------
+    rh : xr.DataArray
+        Relative humidity
+    t : xr.DataArray
+        Air temperature
+
+    Returns
+    -------
+    rh_wrt_ice_or_water : xr.DataArray
+        CAdjusted relative humidity
+    """
+    # Convert to hPa (Groff & Gratch)
+    e_s_wtr = 10**(-7.90298 * (T_100 / (t + T_0) - 1)
+                   + 5.02808 * np.log10(T_100 / (t + T_0))
+                   - 1.3816E-7 * (10**(11.344 * (1 - (t + T_0) / T_100)) - 1)
+                   + 8.1328E-3 * (10**(-3.49149 * (T_100/(t + T_0) - 1)) -1)
+                   + np.log10(ews))
+    e_s_ice = 10**(-9.09718 * (T_0 / (t + T_0) - 1)
+                   - 3.56654 * np.log10(T_0 / (t + T_0))
+                   + 0.876793 * (1 - (t + T_0) / T_0)
+                   + np.log10(ei0))
+
+    # Define freezing point. Why > -100?
+    nan_mask = t.notnull()
+    freezing = (t < 0) & (t > -100) & nan_mask
+
+    # Set to Groff & Gratch values when freezing, otherwise just rh
+    rh_wrt_ice_or_water = rh.where(~freezing & nan_mask,
+                                   other=rh*(e_s_wtr/e_s_ice))
+    return rh_wrt_ice_or_water
+
+
+def calculate_specific_humidity(t, p, rh_wrt_ice_or_water):
+    """Calculate specific humidity.
+
+    Parameters
+    ----------
+    t : xr.DataArray
+        Air temperature
+    p : xr.DataArray
+        Air pressure
+    rh_wrt_ice_or_water : xr.DataArray
+        Adjusted relative humidity
+
+    Returns
+    -------
+    xr.DataArray
+        Specific humidity (kg/kg)
+    """
+    # Saturation vapour pressure above 0 C (hPa)
+    es_wtr = 10**(-7.90298 * (T_100 / (t + T_0) - 1) + 5.02808 * np.log10(T_100 / (t + T_0))
+                  - 1.3816E-7 * (10**(11.344 * (1 - (t + T_0) / T_100)) - 1)
+                  + 8.1328E-3 * (10**(-3.49149 * (T_100 / (t + T_0) -1)) - 1) + np.log10(ews))
+
+    # Saturation vapour pressure below 0 C (hPa)
+    es_ice = 10**(-9.09718 * (T_0 / (t + T_0) - 1) - 3.56654
+                  * np.log10(T_0 / (t + T_0)) + 0.876793
+                  * (1 - (t + T_0) / T_0)
+                  + np.log10(ei0))
+
+    # Specific humidity at saturation (incorrect below melting point)
+    q_sat = eps * es_wtr / (p - (1 - eps) * es_wtr)
+
+    # Replace saturation specific humidity values below melting point
+    freezing = t < 0
+    q_sat[freezing] = eps * es_ice[freezing] / (p[freezing] - (1 - eps) * es_ice[freezing])
+
+    # Mask where temperature or pressure are null values
+    q_nan = np.isnan(t) | np.isnan(p)
+    q_sat[q_nan] = np.nan
+
+    # Convert to kg/kg
+    return rh_wrt_ice_or_water * q_sat / 100
+
+def convert(qh: xr.DataArray
+) -> xr.DataArray:
+    """Convert specific humidity from kg/kg to g/kg units
+
+    Parameters
+    ----------
+    qh : xr.DataArray
+        Specific humidity (kg/kg)
+
+    Returns
+    -------
+    xr.DataArray
+        Specific humidity (g/kg)
+    """
+    return 1000 * qh

pypromice/core/variables/precipitation.py

@@ -0,0 +1,108 @@
+__all__ = ["correct_rainfall_undercatch", "get_rainfall_per_timestep", "filter_lufft_errors"]
+
+import numpy as np
+import xarray as xr
+
+
+def filter_lufft_errors(
+    precip: xr.DataArray, t: xr.DataArray, p: xr.DataArray, rh: xr.DataArray
+) -> xr.DataArray:
+    """Filter precipitation measurements where air temperature, pressure, or
+    relative humidity measurements are null values. This assumes that
+    air temperature, air pressure, relative humidity and precipitation
+    measurements are measured using the same instrument, e.g. a
+    lufft instrument.
+
+    Parameters
+    ----------
+    precip : xr.DataArray
+        Cumulative precipitation measurements
+    t : xr.DataArray
+        Air temperature measurements
+    p : xr.DataArray
+        Air pressure measurements
+    rh : xr.DataArray
+        Relative humidity measurements
+
+    Returns
+    -------
+    xr.DataArray
+        Filtered precipitation values
+    """
+    mask = (t.isnull() | p.isnull() | rh.isnull()) & (precip == 0)
+    return precip.where(~mask)
+
+
+def correct_rainfall_undercatch(
+    rainfall_per_timestep: xr.DataArray, wspd: xr.DataArray
+) -> xr.DataArray:
+    """Corrects rainfall amount per timestep for undercatch as in
+    Yang et al. (1999) and Box et al. (2022), based on Goodison et al. (1998).
+
+    Yang, D., Ishida, S., Goodison, B. E., and Gunther, T.: Bias correction of
+    daily precipitation measurements for Greenland,
+    https://doi.org/10.1029/1998jd200110, 1999.
+
+    Box, J., Wehrle, A., van As, D., Fausto, R., Kjeldsen, K., Dachauer, A.,
+    Ahlstrom, A. P., and Picard, G.: Greenland Ice Sheet rainfall, heat and
+    albedo feedback imapacts from the Mid-August 2021 atmospheric river,
+    Geophys. Res. Lett. 49 (11), e2021GL097356,
+    https://doi.org/10.1029/2021GL097356, 2022.
+
+    Goodison, B. E., Louie, P. Y. T., and Yang, D.: Solid Precipitation
+    Measurement Intercomparison, WMO, 1998
+
+    Parameters
+    ----------
+    rainfall : xr.DataArray
+        Uncorrected rainfall per timestep
+    wspd : xr.DataArray
+        Wind speed measurements
+
+    Returns
+    -------
+    rainfall_cor : xr.DataArray
+        Corrected rainfall per timestep
+    """
+
+    # Calculate undercatch correction factor
+    corr = 100 / (100.00 - 4.37 * wspd + 0.35 * wspd * wspd)
+
+    # Fix all values below 1.02 to 1.02
+    corr = corr.where(corr > 1.02, other=1.02)
+
+    # Apply correction to rate
+    rainfall_per_timestep_cor = rainfall_per_timestep * corr
+
+    return rainfall_per_timestep_cor
+
+def get_rainfall_per_timestep(
+    precip: xr.DataArray,
+    t: xr.DataArray
+) -> xr.DataArray:
+    """
+    Derive rainfall per timestep from cumulative precipitation data.
+
+    Parameters
+    ----------
+    precip : xr.DataArray
+        Cumulative precipitation measurements.
+    t : xr.DataArray
+        Air temperature measurements.
+
+    Returns
+    -------
+    xr.DataArray
+        Rainfall per timestep with negative values removed and
+        cold-season precipitation (T < -2 °C) filtered out.
+    """
+    rainfall_per_timestep = precip.diff("time").reindex_like(precip)
+
+    # Removing all negative precipitation, both corrected and uncorrected
+    rainfall_per_timestep = rainfall_per_timestep.where(rainfall_per_timestep >= 0)
+
+    # Filtering cold season precipitation, both corrected and uncorrected
+    rain_in_cold = (rainfall_per_timestep > 0) & (t < -2)
+    rainfall_per_timestep = rainfall_per_timestep.where(~rain_in_cold)
+
+    return rainfall_per_timestep