pypromice 1.5.3__py3-none-any.whl → 1.7.0__py3-none-any.whl

pypromice/__init__.py

@@ -0,0 +1,2 @@
+from importlib.metadata import version
+__version__ = version("pypromice")

pypromice/{qc → core/qc}/github_data_issues.py

@@ -159,7 +159,7 @@ def adjustData(ds, adj_dir, var_list=[], skip_var=[]):
         adj_info.loc[adj_info.t0.isnull()|(adj_info.t0==''), "t0"] = None
 		
         # if "*" is in the variable name then we interpret it as regex
-        selec =  adj_info['variable'].str.contains('\*') & (adj_info['variable'] != "*")
+        selec = adj_info['variable'].str.contains(r'\*') & (adj_info['variable'] != "*")
         for ind in adj_info.loc[selec, :].index:
             line_template = adj_info.loc[ind, :].copy()
             regex = adj_info.loc[ind, 'variable']
@@ -209,23 +209,11 @@ def adjustData(ds, adj_dir, var_list=[], skip_var=[]):
 
                 if func == "add":
                     ds_out[var].loc[index_slice] = ds_out[var].loc[index_slice].values + val
-                    # flagging adjusted values
-                    # if var + "_adj_flag" not in ds_out.columns:
-                    #     ds_out[var + "_adj_flag"] = 0
-                    # msk = ds_out[var].loc[index_slice])].notnull()
-                    # ind = ds_out[var].loc[index_slice])].loc[msk].time
-                    # ds_out.loc[ind, var + "_adj_flag"] = 1
 
                 if func == "multiply":
                     ds_out[var].loc[index_slice] = ds_out[var].loc[index_slice].values * val
                     if "DW" in var:
                         ds_out[var].loc[index_slice] = ds_out[var].loc[index_slice] % 360
-                    # flagging adjusted values
-                    # if var + "_adj_flag" not in ds_out.columns:
-                    #     ds_out[var + "_adj_flag"] = 0
-                    # msk = ds_out[var].loc[index_slice].notnull()
-                    # ind = ds_out[var].loc[index_slice].loc[msk].time
-                    # ds_out.loc[ind, var + "_adj_flag"] = 1
 
                 if func == "min_filter":
                     tmp = ds_out[var].loc[index_slice].values
@@ -277,6 +265,27 @@ def adjustData(ds, adj_dir, var_list=[], skip_var=[]):
                     ds_out[var2].loc[index_slice] = val_var
                     ds_out[var].loc[index_slice] = val_var2
 
+                if "delete_when_same_as_" in func:
+                    var2 = func.replace('delete_when_same_as_','')
+                    tmp = ds_out[var].loc[index_slice]
+                    msk = np.abs(tmp - ds_out[var2].loc[index_slice]) < val
+                    tmp = tmp.where(~msk)
+                    # remove isolated singletons and pairs surrounded by NaNs
+                    m1 = tmp.notnull() & tmp.shift(time=1).isnull() & tmp.shift(time=-1).isnull()
+                    
+                    m2_first  = (tmp.notnull()
+                                 & tmp.shift(time=1).isnull()     # left is NaN
+                                 & tmp.shift(time=-1).notnull()   # right is value
+                                 & tmp.shift(time=-2).isnull())   # right+1 is NaN
+                    
+                    m2_second = (tmp.notnull()
+                                 & tmp.shift(time=-1).isnull()    # right is NaN
+                                 & tmp.shift(time=1).notnull()    # left is value
+                                 & tmp.shift(time=2).isnull())    # left-1 is NaN
+                    
+                    tmp = tmp.where(~(m1 | m2_first | m2_second))
+                    ds_out[var].loc[index_slice] = tmp.values
+
                 if func == "rotate":
                     ds_out[var].loc[index_slice] = (ds_out[var].loc[index_slice].values + val) % 360
 

pypromice/{qc → core/qc}/percentiles/compute_thresholds.py

@@ -3,10 +3,10 @@ from datetime import datetime
 
 import pandas as pd
 
-from pypromice.process import AWS
+from pypromice.pipeline.aws import AWS
 from pathlib import Path
 import logging
-from pypromice.qc.github_data_issues import adjustTime, flagNAN, adjustData
+from pypromice.core.qc.github_data_issues import adjustTime, flagNAN, adjustData
 
 
 # %%

pypromice/{qc → core/qc}/persistence.py

@@ -19,27 +19,22 @@ DEFAULT_VARIABLE_THRESHOLDS = {
     "t_i": {"max_diff": 0.0001, "period": 2},
     "t_u": {"max_diff": 0.0001, "period": 2},
     "t_l": {"max_diff": 0.0001, "period": 2},
-    "p_i": {"max_diff": 0.0001, "period": 2},
-    # "p_u": {"max_diff": 0.0001, "period": 2},
-    # "p_l": {"max_diff": 0.0001, "period": 2},
-    "gps_lat_lon": {
-        "max_diff": 0.000001,
-        "period": 6,
-    },  # gets special handling to remove simultaneously constant gps_lat and gps_lon
+
+    "p_i": {"max_diff": 0.0001, "period": 3},
+    "p_u": {"max_diff": 0.0001, "period": 150},
+    "p_l": {"max_diff": 0.0001, "period": 150},
+
+    # gets special handling to remove simultaneously constant gps_lat and gps_lon
+    "gps_lat_lon": {"max_diff": 0.000001, "period": 6},
+
     "gps_alt": {"max_diff": 0.0001, "period": 6},
     "t_rad": {"max_diff": 0.0001, "period": 2},
-    "rh_i": {
-        "max_diff": 0.0001,
-        "period": 2,
-    },  # gets special handling to allow constant 100%
-    "rh_u": {
-        "max_diff": 0.0001,
-        "period": 2,
-    },  # gets special handling to allow constant 100%
-    "rh_l": {
-        "max_diff": 0.0001,
-        "period": 2,
-    },  # gets special handling to allow constant 100%
+
+    # gets special handling to allow constant 100%
+    "rh_i": {"max_diff": 0.0001, "period": 2},
+    "rh_u": {"max_diff": 0.0001, "period": 2},
+    "rh_l": {"max_diff": 0.0001, "period": 2},
+
     "wspd_i": {"max_diff": 0.0001, "period": 6},
     "wspd_u": {"max_diff": 0.0001, "period": 6},
     "wspd_l": {"max_diff": 0.0001, "period": 6},
@@ -83,15 +78,11 @@ def persistence_qc(
         variable_thresholds = DEFAULT_VARIABLE_THRESHOLDS
         logger.debug(f"Running persistence_qc using {variable_thresholds}")
     else:
-        logger.info(f"Running persistence_qc using custom thresholds:\n {variable_thresholds}")    
+        logger.info(f"Running persistence_qc using custom thresholds:\n {variable_thresholds}")
 
     for k in variable_thresholds.keys():
         if k in ["t", "p", "rh", "wspd", "wdir", "z_boom"]:
-            var_all = [
-                k + "_u",
-                k + "_l",
-                k + "_i",
-            ]  # apply to upper, lower boom, and instant
+            var_all = [k + l for l in ["_u", "_l", "_i"]]  # apply to upper, lower boom, and instant
         else:
             var_all = [k]
         max_diff = variable_thresholds[k]["max_diff"]  # loading persistent limit
@@ -140,10 +131,12 @@ def find_persistent_regions(
     """
     Algorithm that ensures values can stay the same within the outliers_mask
     """
-    consecutive_true_df = count_consecutive_persistent_values(data, max_diff)
-    persistent_regions = consecutive_true_df >= min_repeats
-    # Ignore entries which already nan in the input data
-    persistent_regions[data.isna()] = False
+    consecutive_true_df  = count_consecutive_persistent_values(data, max_diff)
+    persistent_regions = consecutive_true_df  >= min_repeats
+    for i in range(1, min_repeats):
+        persistent_regions |= persistent_regions.shift(-1, fill_value=False)
+        # Ignore entries which already nan in the input data
+        persistent_regions[data.isna()] = False
     return persistent_regions
 
 

pypromice/{process → core/qc}/value_clipping.py

@@ -2,7 +2,7 @@ import numpy as np
 import pandas
 import xarray
 
-from pypromice.utilities.dependency_graph import DependencyGraph
+from pypromice.core.dependency_graph import DependencyGraph
 
 
 def clip_values(
@@ -24,11 +24,11 @@ def clip_values(
     ds : `xarray.Dataset`
         Dataset with clipped data
     """
-    cols = ["lo", "hi", "OOL"]
+    cols = ["lo", "hi", "dependent_variables"]
     assert set(cols) <= set(var_configurations.columns)
 
     variable_limits = var_configurations[cols].assign(
-        dependents=lambda df: df.OOL.fillna("").str.split(),
+        dependents=lambda df: df.dependent_variables.fillna("").str.split(),
         # Find the closure of dependents using the DependencyGraph class
         dependents_closure=lambda df: DependencyGraph.from_child_mapping(
             df.dependents

pypromice/core/resampling.py

@@ -0,0 +1,142 @@
+import datetime
+import numpy as np
+import pandas as pd
+
+DEFAULT_COMPLETENESS_THRESHOLDS = {
+    "default": 0.8,
+    "albedo": 0.2,
+    "p_u": 0.5,
+    "p_l": 0.5,
+    "z_boom_u": 0.1,
+    "z_boom_l": 0.1,
+    "z_boom_cor_u": 0.1,
+    "z_boom_cor_l": 0.1,
+    "z_stake": 0.1,
+    "z_stake_cor": 0.1,
+    "z_surf_combined": 0.1,
+    "t_i_1": 0.1,
+    "t_i_2": 0.1,
+    "t_i_3": 0.1,
+    "t_i_4": 0.1,
+    "t_i_5": 0.1,
+    "t_i_6": 0.1,
+    "t_i_7": 0.1,
+    "t_i_8": 0.1,
+    "t_i_9": 0.1,
+    "t_i_10": 0.1,
+    "t_i_11": 0.1,
+    "gps_lat": 0.1,
+    "gps_lon": 0.1,
+    "gps_alt": 0.1,
+    "batt_v": 0.1,
+}
+
+ALLOWED_TIME_STAMP_DURATIONS = (
+    datetime.timedelta(minutes=10),
+    datetime.timedelta(minutes=30),
+    datetime.timedelta(hours=1),
+    datetime.timedelta(hours=6),
+    datetime.timedelta(days=1),
+)
+
+
+def classify_timestamp_durations(
+        index: pd.DatetimeIndex,
+) -> pd.TimedeltaIndex:
+    """
+    Classifies the durations between consecutive timestamps in a given DatetimeIndex.
+
+    The function computes the time differences between consecutive timestamps and
+    checks if these differences belong to a predefined set of allowed durations.
+    It performs backward filling to handle missing values
+
+    Parameters
+    ----------
+    index : pd.DatetimeIndex
+        A pandas DatetimeIndex containing the timestamps to classify.
+
+    Returns
+    -------
+    pd.TimedeltaIndex
+        A TimedeltaIndex containing the classified durations for the corresponding
+        timestamps in the input index.
+    """
+    return pd.TimedeltaIndex(
+        index.to_series()
+        .diff()
+        .where(lambda d: d.isin(ALLOWED_TIME_STAMP_DURATIONS))
+        .bfill()
+    )
+
+
+def get_completeness_mask(
+    data_frame: pd.DataFrame,
+    resample_offset: str,
+    completeness_thresholds: dict[str, float] = DEFAULT_COMPLETENESS_THRESHOLDS,
+    *,
+    atol: float = 1e-9,
+) -> pd.DataFrame:
+    """
+    Returns a completeness mask for the given DataFrame based on the specified
+    resampling offset, completeness threshold, and tolerance for over-completeness.
+
+    This function evaluates the completeness of timestamped data, ensuring that
+    records match the expected durations defined by the `resample_offset`. It
+    computes whether each resampled group of data satisfies the completeness
+    constraints defined by the `completeness_threshold` and `atol`.
+
+    Parameters
+    ----------
+    data_frame : pd.DataFrame
+        Input data containing a DatetimeIndex and associated values. The index must
+        be a DatetimeIndex as the function relies on timestamp durations for
+        computations.
+    resample_offset : str
+        Offset string defining resampling frequency. Examples include 'MS' (month
+        start) or other Pandas-compatible offset strings.
+    completeness_threshold : float, optional
+        Dictionary containing the variable-specific minimum completeness ratio
+        required to consider a time period as valid. Must contain a key 'default'
+        used for variables not explicitly listed.
+        Defaults to the dictionary `DEFAULT_COMPLETENESS_THRESHOLD`.
+    atol : float, optional
+        Absolute tolerance for over-completeness. Specifies an allowable margin by
+        which completeness can exceed 1. Defaults to 1e-9.
+
+    Returns
+    -------
+    pd.DataFrame
+        A DataFrame containing Boolean values, where True indicates that the data
+        for the corresponding time period satisfies the completeness constraints,
+        while False indicates the data is either under-complete or over-complete.
+    """
+    if resample_offset in ['MS', 'ME']:
+        offset_timedelta = datetime.timedelta(days=30)
+        # Increase tolerance for overcomplete values in monthly resampling
+        # to handle months with 31 days.
+        atol = 1/30 + atol
+    else:
+        offset_timedelta = pd.to_timedelta(resample_offset)
+
+    index = data_frame.index
+    assert isinstance(index, pd.DatetimeIndex)
+
+    timestamp_durations = classify_timestamp_durations(index)
+    timestamp_coverage = timestamp_durations / np.array(offset_timedelta)
+    data_frame_is_valid = data_frame.notna()
+
+    completeness = (
+        data_frame_is_valid
+        .mul(timestamp_coverage, axis=0)
+        .resample(resample_offset).sum()
+    )
+
+    thresholds = pd.Series(
+        {col: completeness_thresholds.get(col, completeness_thresholds["default"])
+         for col in data_frame.columns}
+    )
+
+    is_under_complete = completeness.lt(thresholds, axis=1)
+    is_over_complete = completeness.gt(1 + atol)
+    completeness_mask = ~(is_under_complete | is_over_complete)
+    return completeness_mask

pypromice/core/variables/__init__.py

@@ -0,0 +1 @@
+

pypromice/core/variables/air_temperature.py

@@ -0,0 +1,64 @@
+__all__=["clip_and_interpolate", "get_cloud_coefficients"]
+
+import pandas as pd
+import xarray as xr
+
+T_0=273.15                  # degrees Celsius to Kelvin conversion
+eps_overcast = 1.0          # Clouds overcast default coefficient
+eps_clear = 9.36508e-6      # Clouds clear default coefficient
+
+def clip_and_interpolate(temp : xr.DataArray,
+                         lo : float,
+                         hi : float,
+                         max_interp : pd.Timedelta = pd.Timedelta(12,'h')
+) -> xr.DataArray:
+    """Clip and interpolate temperature dataset for use in
+    corrections
+
+    Parameters
+    ----------
+    temp : `xr.DataArray`
+        Array of temperature data
+    lo : float
+        Minimum threshold value for clipping
+    hi : float
+        Maximum threshold value for clipping
+    max_interp : `pd.Timedelta`
+        Maximum time steps to interpolate across.
+        The default is 12 hours.
+
+    Returns
+    -------
+    temp_interp : `xr.DataArray`
+        Array of interpolated temperature data
+    """
+    # Clip values to high and low threshold values
+    temp = temp.where((temp >= lo) & (temp <= hi))
+
+    # Drop duplicates and interpolate across NaN values
+    temp_interp = temp.interpolate_na(dim='time',
+                                      max_gap=max_interp)
+
+    return temp_interp
+
+
+def get_cloud_coefficients(temp: xr.DataArray
+) -> tuple[xr.DataArray, xr.DataArray]:
+    """Get overcast and clear cloud longwave coefficients using
+    air temperature, based on assumptions from Swinbank (1963)
+
+    Parameters
+    ----------
+    temp : xr.DataArray
+        Air temperature
+
+    Returns
+    -------
+    LR_overcast : xr.DataArray
+        Overcast cloud coefficients, using overcast cloud assumption from Swinbank (1963)
+    LR_clear : xr.DataArray
+        Clear cloud coefficients, using clear cloud assumption, from Swinbank (1963)
+    """
+    LR_overcast = eps_overcast * 5.67e-8 * (temp + T_0) ** 4
+    LR_clear = eps_clear * 5.67e-8 * (temp + T_0) ** 6
+    return LR_overcast, LR_clear

pypromice/core/variables/gps.py

@@ -0,0 +1,221 @@
+__all__ = ["decode_and_convert",  "filter",
+           "decode", "convert_from_degrees_and_decimal_minutes",
+           "convert_from_decimal_minutes"]
+import re
+import xarray as xr
+import numpy as np
+import pandas as pd
+from sklearn.linear_model import LinearRegression
+
+import logging
+logger = logging.getLogger(__name__)
+
+def decode_and_convert(gps_lat: xr.DataArray,
+                       gps_lon: xr.DataArray,
+                       gps_time: xr.DataArray,
+                       latitude: float,
+                       longitude: float
+) -> tuple[xr.DataArray,xr.DataArray,xr.DataArray]:
+    """Decode and convert GPS latitude, longtitude and time values."flag_decimal_minutes",
+           "flag_for_decoding"
+    Decoding is performed if values are detected as string types.
+    Conversion consists of transforming to decimal degrees (DD),
+    from either decimal minutes (mm.mmmmm) or degrees and
+    decimal minutes (ddmm.mmmm)
+
+    Parameters
+    ----------
+    gps_lat : `xr.DataArray`
+        GPS latitude
+    gps_lon : `xr.DataArray`
+        GPS longitude
+    gps_time : `xr.DataArray`
+        GPS time
+
+    Returns
+    -------
+    gps_lat : `xr.DataArray`
+        Decoded and converted GPS latitude
+    gps_lon : `xr.DataArray`
+        Decoded and converted GPS longitude
+    gps_time : `xr.DataArray`
+        Decoded and converted GPS time
+    """
+    # Retain GPS array attributes
+    lat_attrs = gps_lat.attrs
+    lon_attrs = gps_lon.attrs
+    time_attrs = gps_time.attrs
+
+    # Decode GPS information if array is an object array
+    if gps_lat.dtype.kind == "O":
+        lat, lon, time = decode(gps_lat, gps_lon, gps_time)
+        if lat is None:
+            logger.warning("GPS decoding failed, skipping this routine.")
+        else:
+            gps_lat, gps_lon, gps_time = lat, lon, time
+
+    # Reformat values to numeric
+    gps_lat.values = pd.to_numeric(gps_lat, errors='coerce')
+    gps_lon.values = pd.to_numeric(gps_lon, errors='coerce')
+    gps_time.values = pd.to_numeric(gps_time, errors='coerce')
+
+    # Convert GPS positions to decimal degrees
+    if np.any((gps_lat <= 90) & (gps_lat > 0)):
+        gps_lat = convert_from_decimal_minutes(gps_lat, latitude)
+        gps_lon = convert_from_decimal_minutes(gps_lon, longitude)
+    else:
+        gps_lat = convert_from_degrees_and_decimal_minutes(gps_lat)
+        gps_lon = convert_from_degrees_and_decimal_minutes(gps_lon)
+
+    # Reassign GPS array attributes
+    gps_lat.attrs = lat_attrs
+    gps_lon.attrs = lon_attrs
+    gps_time.attrs = time_attrs
+
+    return gps_lat, gps_lon, gps_time
+
+
+def filter(gps_lat: xr.DataArray,
+           gps_lon: xr.DataArray,
+           gps_alt: xr.DataArray
+) -> tuple[xr.DataArray, xr.DataArray, xr.DataArray]:
+    """ Filter GPS latitude, longitude and altitude based on the difference
+    to a baseline elevation. The baseline elevation is a gap-filled monthly
+    median elevation based on the inputted GPS altitude.
+
+    Parameters
+    ----------
+    gps_lat : xr.DataArray
+        GPS latitude
+    gps_lon : xr.DataArray
+        GPS longitude
+    gps_alt : xr.DataArray
+        GPS altitude values with a time dimension
+
+    Returns
+    ----------
+    gps_lat_filtered : xr.DataArray
+        Filtered latitude values
+    gps_lon_filtered : xr.DataArray
+        Filtered longitude values
+    gps_alt_filtered : xr.DataArray
+        Filtered altitude values
+    """
+    # Get altitude monthly median (at month start)
+    # This will serve as baseline elevations for filtering
+    ser = gps_alt.to_series()
+    monthly_median = ser.resample("MS").median()
+    baseline_elevation = (
+        monthly_median
+        .reindex(ser.index, method="nearest")
+        .ffill()
+        .bfill()
+    )
+
+    # Produce conditional mask
+    mask = (np.abs(gps_alt - baseline_elevation) < 100) | gps_alt.isnull()
+
+    # Apply mask
+    gps_lat_filtered = gps_lat.where(mask)
+    gps_lon_filtered = gps_lon.where(mask)
+    gps_alt_filtered = gps_alt.where(mask)
+
+    return gps_lat_filtered, gps_lon_filtered, gps_alt_filtered
+
+
+def convert_from_degrees_and_decimal_minutes(gps):
+    """Convert positions (i.e. latitude, longitude) from degrees
+    and decimal minutes (ddmm.mmmm) to decimal degree values (DD)"""
+    return np.floor(gps / 100) + (gps / 100 - np.floor(gps / 100)) * 100 / 60
+
+
+def convert_from_decimal_minutes(gps: xr.DataArray, pos: float
+) -> xr.DataArray:
+    """Convert decimal minutes (mm.mmmmm) to decimal degree
+    values (DD), using a predefined position to append values to.
+    Needed in the case of PROMICE v1 stations, where logger
+    programs saved positions only in decimal minutes."""
+    new_gps = np.sign(pos) * (gps + 100 * np.floor(np.abs(pos)))
+    return convert_from_degrees_and_decimal_minutes(new_gps)
+
+
+def decode(gps_lat: xr.DataArray,
+           gps_lon: xr.DataArray,
+           gps_time: xr.DataArray
+) -> tuple[xr.DataArray,xr.DataArray,xr.DataArray]:
+    """Decode GPS information. This should be applied if gps information
+    consists of strings and not float values. GPS information is returned in
+    decimal degrees (ddmm.mmmm) format.
+
+    Parameters
+    ----------
+    gps_lat : `xr.DataArray`
+        GPS latitude
+    gps_lon : `xr.DataArray`
+        GPS longitude
+    gps_time : `xr.DataArray`
+        GPS time
+
+    Returns
+    -------
+    new_lat : `xr.DataArray`
+        Decoded GPS latitude
+    new_lon : `xr.DataArray`
+        Decoded GPS longitude
+    new_time : `xr.DataArray`
+        Decoded GPS time
+    """
+    # Pick the first non-null sample safely and detect decoding format
+    non_null = gps_lat.dropna(dim='time').values
+    sample_value = str(non_null[0])
+
+    try:
+        # Object decoding
+        if "NH" in sample_value:
+            new_lat = gps_object_decoder(gps_lat)
+            new_lon = gps_object_decoder(gps_lon)
+            new_time = gps_object_decoder(gps_time)
+            return new_lat, new_lon, new_time
+
+        # L-string decoding
+        elif "L" in sample_value:
+            logger.info("Found 'L' in GPS string; applying decode + scaling.")
+            new_lat = gps_l_string_decoder(gps_lat)
+            new_lon = gps_l_string_decoder(gps_lon)
+            new_time = gps_object_decoder(gps_time)
+            return new_lat, new_lon, new_time
+
+        # Unknown format, attempt to decode
+        else:
+            logger.info("Unknown GPS string format; attempting generic decode.")
+            new_lat = gps_object_decoder(gps_lat)
+            new_lon = gps_object_decoder(gps_lon)
+            new_time = gps_object_decoder(gps_time)
+            return new_lat, new_lon, new_time
+
+    except Exception as e:
+        logger.error(f"Failed to decode GPS data: {e!r} "
+                     f"(dtype={gps_lat.dtype})")
+        return None, None, None
+
+
+def gps_object_decoder(gps : xr.DataArray) -> xr.DataArray:
+    """GPS decoder for object array formatting. For example, PROMICE v2
+    stations should send information as 'NH6429.01544,WH04932.86061'
+    original formatting (NUK_L 2022); PROMICE v3 stations should send
+    coordinates as '6430,4916' (NUK_Uv3); and GC-Net stations should
+    send coordinates as '6628.93936',04617.59187' (DY2)"""
+    str2nums = [re.findall(r"[-+]?\d*\.\d+|\d+", _) if isinstance(_, str) else [np.nan] for _ in gps.values]
+    gps[:] = pd.DataFrame(str2nums).astype(float).T.values[0]
+    gps = gps.astype(float)
+    return gps
+
+
+def gps_l_string_decoder(gps : xr.DataArray) -> xr.DataArray:
+    """GPS L-string decoder"""
+    # Convert from object array
+    gps = gps_object_decoder(gps)
+
+    # Convert from integer-like values to degrees
+    gps = gps/100000
+    return gps