pypromice 1.3.5__py3-none-any.whl → 1.4.0__py3-none-any.whl

pypromice/qc/persistence.py

@@ -9,16 +9,40 @@ __all__ = [
     "persistence_qc",
     "find_persistent_regions",
     "count_consecutive_persistent_values",
-    "count_consecutive_true",
+    "get_duration_consecutive_true",
 ]
 
 logger = logging.getLogger(__name__)
 
+# period is given in hours, 2 persistent 10 min values will be flagged if period < 0.333
 DEFAULT_VARIABLE_THRESHOLDS = {
-    "t": {"max_diff": 0.0001, "period": 2},
-    "p": {"max_diff": 0.0001, "period": 2},
-    # Relative humidity can be very stable around 100%.
-    #"rh": {"max_diff": 0.0001, "period": 2},
+    "t_i": {"max_diff": 0.0001, "period": 2},
+    "t_u": {"max_diff": 0.0001, "period": 2},
+    "t_l": {"max_diff": 0.0001, "period": 2},
+    "p_i": {"max_diff": 0.0001, "period": 2},
+    # "p_u": {"max_diff": 0.0001, "period": 2},
+    # "p_l": {"max_diff": 0.0001, "period": 2},
+    "gps_lat_lon": {
+        "max_diff": 0.000001,
+        "period": 6,
+    },  # gets special handling to remove simultaneously constant gps_lat and gps_lon
+    "gps_alt": {"max_diff": 0.0001, "period": 6},
+    "t_rad": {"max_diff": 0.0001, "period": 2},
+    "rh_i": {
+        "max_diff": 0.0001,
+        "period": 2,
+    },  # gets special handling to allow constant 100%
+    "rh_u": {
+        "max_diff": 0.0001,
+        "period": 2,
+    },  # gets special handling to allow constant 100%
+    "rh_l": {
+        "max_diff": 0.0001,
+        "period": 2,
+    },  # gets special handling to allow constant 100%
+    "wspd_i": {"max_diff": 0.0001, "period": 6},
+    "wspd_u": {"max_diff": 0.0001, "period": 6},
+    "wspd_l": {"max_diff": 0.0001, "period": 6},
 }
 
 
@@ -57,21 +81,27 @@ def persistence_qc(
 
     if variable_thresholds is None:
         variable_thresholds = DEFAULT_VARIABLE_THRESHOLDS
-
-    logger.debug(f"Running persistence_qc using {variable_thresholds}")
+        logger.debug(f"Running persistence_qc using {variable_thresholds}")
+    else:
+        logger.info(f"Running persistence_qc using custom thresholds:\n {variable_thresholds}")    
 
     for k in variable_thresholds.keys():
-        var_all = [
-            k + "_u",
-            k + "_l",
-            k + "_i",
-        ]  # apply to upper, lower boom, and instant
+        if k in ["t", "p", "rh", "wspd", "wdir", "z_boom"]:
+            var_all = [
+                k + "_u",
+                k + "_l",
+                k + "_i",
+            ]  # apply to upper, lower boom, and instant
+        else:
+            var_all = [k]
         max_diff = variable_thresholds[k]["max_diff"]  # loading persistent limit
         period = variable_thresholds[k]["period"]  # loading diff period
 
         for v in var_all:
             if v in df:
                 mask = find_persistent_regions(df[v], period, max_diff)
+                if "rh" in v:
+                    mask = mask & (df[v] < 99)
                 n_masked = mask.sum()
                 n_samples = len(mask)
                 logger.debug(
@@ -79,6 +109,19 @@ def persistence_qc(
                 )
                 # setting outliers to NaN
                 df.loc[mask, v] = np.nan
+            elif v == "gps_lat_lon":
+                mask = find_persistent_regions(
+                    df["gps_lon"], period, max_diff
+                ) & find_persistent_regions(df["gps_lat"], period, max_diff)
+
+                n_masked = mask.sum()
+                n_samples = len(mask)
+                logger.debug(
+                    f"Applying persistent QC in {v}. Filtering {n_masked}/{n_samples} samples"
+                )
+                # setting outliers to NaN
+                df.loc[mask, "gps_lon"] = np.nan
+                df.loc[mask, "gps_lat"] = np.nan
 
     # Back to xarray, and re-assign the original attrs
     ds_out = df.to_xarray()
@@ -110,33 +153,36 @@ def count_consecutive_persistent_values(
 ) -> pd.Series:
     diff = data.ffill().diff().abs()  # forward filling all NaNs!
     mask: pd.Series = diff < max_diff
-    return count_consecutive_true(mask)
+    return get_duration_consecutive_true(mask)
 
 
-def count_consecutive_true(
-    series: Union[pd.Series, pd.DataFrame]
-) -> Union[pd.Series, pd.DataFrame]:
+def get_duration_consecutive_true(
+    series: pd.Series,
+) -> pd.Series:
     """
-    Convert boolean series to integer series where the values represent the number of connective true values.
+    From a boolean series, calculates the duration, in hours, of the periods with concecutive true values.
+
+    The first value will be set to NaN, as it is not possible to calculate the duration of a single value.
 
     Examples
     --------
-    >>> count_consecutive_true(pd.Series([False, True, False, False, True, True, True, False, True]))
-    pd.Series([0, 1, 0, 0, 1, 2, 3, 0, 1])
+    >>> get_duration_consecutive_true(pd.Series([False, True, False, False, True, True, True, False, True]))
+    pd.Series([np.nan, 1, 0, 0, 1, 2, 3, 0, 1])
 
     Parameters
     ----------
-    series
+    pd.Series
         Boolean pandas Series or DataFrame
 
     Returns
     -------
-    consecutive_true_count
+    pd.Series
         Integer pandas Series or DataFrame with values representing the number of connective true values.
 
     """
-    # assert series.dtype == bool
-    cumsum = series.cumsum()
     is_first = series.astype("int").diff() == 1
-    offset = (is_first * cumsum).replace(0, np.nan).fillna(method="ffill").fillna(0)
-    return ((cumsum - offset + 1) * series).astype("int")
+    delta_time = (series.index.to_series().diff().dt.total_seconds() / 3600)
+    cumsum = delta_time.cumsum()
+    offset = (is_first * (cumsum - delta_time)).replace(0, np.nan).ffill().fillna(0)
+
+    return (cumsum - offset) * series

pypromice/resources/__init__.py

@@ -0,0 +1,28 @@
+import csv
+from pathlib import Path
+from typing import Dict, Union
+
+import pandas as pd
+
+DEFAULT_METADATA_PATH = (Path(__file__).parent / "file_attributes.csv").absolute()
+DEFAULT_VARIABLES_PATH = (Path(__file__).parent / "variables.csv").absolute()
+DEFAULT_VARIABLES_ALIASES_GCNET_PATH = (Path(__file__).parent / "variable_aliases_GC-Net.csv").absolute()
+
+def load_metadata(path: Union[None, str, Path] = None) -> Dict[str, str]:
+    """
+    Load metadata table from csv file
+    """
+    if path is None:
+        path = DEFAULT_METADATA_PATH
+    with open(path, "r") as f:
+        csv_reader = csv.reader(f)
+        return {row[0]: row[1] for row in csv_reader}
+
+
+def load_variables(path: Union[None, str, Path] = None) -> pd.DataFrame:
+    """
+    Load variables table from csv file
+    """
+    if path is None:
+        path = DEFAULT_VARIABLES_PATH
+    return pd.read_csv(path, index_col=0, comment="#")

pypromice/{process/metadata.csv → resources/file_attributes.csv}

@@ -40,11 +40,9 @@ metadata_link,
 naming_authority,dk.geus.promice
 platform,
 platform_vocabulary,GCMD:GCMD Keywords
-processing_level,Level 3
 product_status,beta
 product_version,4
 program,PROMICE
-project,PROMICE
 publisher_email,info@promice.dk
 publisher_institution,GEUS
 publisher_name,GEUS

pypromice/resources/variable_aliases_GC-Net.csv

@@ -0,0 +1,78 @@
+GEUS_name,old_name
+time,timestamp
+p_u,P
+p_l,
+t_u,TA2
+t_l,TA1
+rh_u,RH2
+rh_u_cor,RH2_cor
+qh_u,Q2
+rh_l,RH1
+rh_l_cor,RH1_cor
+qh_l,Q1
+wspd_u,VW2
+wspd_l,VW1
+wdir_u,DW2
+wdir_l,DW1
+dsr,
+dsr_cor,ISWR
+usr,
+usr_cor,OSWR
+albedo,Alb
+dlr,
+ulr,
+cc,
+t_surf,
+dlhf_u,LHF
+dlhf_l,
+dshf_u,SHF
+dshf_l,
+z_boom_u,HW2
+z_boom_l,HW1
+precip_u,
+precip_u_cor,
+precip_l,
+precip_l_cor,
+t_i_1,TS1
+t_i_2,TS2
+t_i_3,TS3
+t_i_4,TS4
+t_i_5,TS5
+t_i_6,TS6
+t_i_7,TS7
+t_i_8,TS8
+t_i_9,TS9
+t_i_10,TS10
+t_i_11,
+tilt_x,
+tilt_y,
+rot,
+lat,latitude
+lon,longitude
+alt,elevation
+gps_time,
+gps_geounit,
+gps_hdop,
+batt_v,V
+fan_dc_u,
+fan_dc_l,
+t_rad,
+msg_lat,
+msg_lon,
+z_surf_1,HS1
+z_surf_2,HS2
+z_surf_1_adj_flag,HS1_adj_flag
+z_surf_2_adj_flag,HS2_adj_flag
+z_surf_combined,HS_combined
+d_t_i_1,DTS1
+d_t_i_2,DTS2
+d_t_i_3,DTS3
+d_t_i_4,DTS4
+d_t_i_5,DTS5
+d_t_i_6,DTS6
+d_t_i_7,DTS7
+d_t_i_8,DTS8
+d_t_i_9,DTS9
+d_t_i_10,DTS10
+d_t_i_11,
+t_i_10m,TS_10m

pypromice/resources/variables.csv

@@ -0,0 +1,106 @@
+field,standard_name,long_name,units,coverage_content_type,coordinates,instantaneous_hourly,where_to_find,lo,hi,OOL,station_type,L0,L2,L3,max_decimals
+time,time,Time,yyyy-mm-dd HH:MM:SS,physicalMeasurement,time,,,,,,all,1,1,1,
+rec,record,Record,-,referenceInformation,time,,L0 or L2,,,,all,1,1,0,0
+p_u,air_pressure,Air pressure (upper boom),hPa,physicalMeasurement,time,FALSE,,650,1100,z_pt z_pt_cor dshf_u dlhf_u qh_u,all,1,1,1,4
+p_l,air_pressure,Air pressure (lower boom),hPa,physicalMeasurement,time,FALSE,,650,1100,dshf_l dlhf_l qh_l,two-boom,1,1,1,4
+t_u,air_temperature,Air temperature (upper boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,rh_u_cor cc dsr_cor usr_cor z_boom z_stake dshf_u dlhf_u qh_u,all,1,1,1,4
+t_l,air_temperature,Air temperature (lower boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,rh_l_cor z_boom_l dshf_l dlhf_l qh_l,two-boom,1,1,1,4
+rh_u,relative_humidity,Relative humidity (upper boom),%,physicalMeasurement,time,FALSE,,0,100,rh_u_cor,all,1,1,1,4
+rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,modelResult,time,FALSE,L2 or later,0,150,dshf_u dlhf_u qh_u,all,0,1,1,4
+qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,modelResult,time,FALSE,L2 or later,0,100,,all,0,1,1,4
+rh_l,relative_humidity,Relative humidity (lower boom),%,physicalMeasurement,time,FALSE,,0,100,rh_l_cor,two-boom,1,1,1,4
+rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,modelResult,time,FALSE,L2 or later,0,150,dshf_l dlhf_l qh_l,two-boom,0,1,1,4
+qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,modelResult,time,FALSE,L2 or later,0,100,,two-boom,0,1,1,4
+wspd_u,wind_speed,Wind speed (upper boom),m s-1,physicalMeasurement,time,FALSE,,0,100,"wdir_u wspd_x_u wspd_y_u dshf_u dlhf_u qh_u, precip_u",all,1,1,1,4
+wspd_l,wind_speed,Wind speed (lower boom),m s-1,physicalMeasurement,time,FALSE,,0,100,"wdir_l wspd_x_l wspd_y_l dshf_l dlhf_l qh_l , precip_l",two-boom,1,1,1,4
+wdir_u,wind_from_direction,Wind from direction (upper boom),degrees,physicalMeasurement,time,FALSE,,1,360,wspd_x_u wspd_y_u,all,1,1,1,4
+wdir_std_u,wind_from_direction_standard_deviation,Wind from direction (standard deviation),degrees,qualityInformation,time,FALSE,L0 or L2,,,,one-boom,1,1,0,4
+wdir_l,wind_from_direction,Wind from direction (lower boom),degrees,physicalMeasurement,time,FALSE,,1,360,wspd_x_l wspd_y_l,two-boom,1,1,1,4
+wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_u wspd_u,all,0,1,1,4
+wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_u wspd_u,all,0,1,1,4
+wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_l wspd_l,two-boom,0,1,1,4
+wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_l wspd_l,two-boom,0,1,1,4
+dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,physicalMeasurement,time,FALSE,,-10,1500,albedo dsr_cor usr_cor,all,1,1,1,4
+dsr_cor,surface_downwelling_shortwave_flux_corrected,Downwelling shortwave radiation - corrected,W m-2,modelResult,time,FALSE,L2 or later,,,,all,0,1,1,4
+usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,physicalMeasurement,time,FALSE,,-10,1000,albedo dsr_cor usr_cor,all,1,1,1,4
+usr_cor,surface_upwelling_shortwave_flux_corrected,Upwelling shortwave radiation - corrected,W m-2,modelResult,time,FALSE,L2 or later,0,1000,,all,0,1,1,4
+albedo,surface_albedo,Albedo,-,modelResult,time,FALSE,L2 or later,,,,all,0,1,1,4
+dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,physicalMeasurement,time,FALSE,,50,500,albedo dsr_cor usr_cor cc t_surf,all,1,1,1,4
+ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,physicalMeasurement,time,FALSE,,50,500,t_surf,all,1,1,1,4
+cc,cloud_area_fraction,Cloud cover,%,modelResult,time,FALSE,L2 or later,,,,all,0,1,1,4
+t_surf,surface_temperature,Surface temperature,C,modelResult,time,FALSE,L2 or later,-80,40,dshf_u dlhf_u qh_u,all,0,1,1,4
+dlhf_u,surface_downward_latent_heat_flux,Latent heat flux (upper boom),W m-2,modelResult,time,FALSE,L3 or later,,,,all,0,0,1,4
+dlhf_l,surface_downward_latent_heat_flux,Latent heat flux (lower boom),W m-2,modelResult,time,FALSE,L3 or later,,,,two-boom,0,0,1,4
+dshf_u,surface_downward_sensible_heat_flux,Sensible heat flux (upper boom),W m-2,modelResult,time,FALSE,L3 or later,,,,all,0,0,1,4
+dshf_l,surface_downward_sensible_heat_flux,Sensible heat flux (lower boom),W m-2,modelResult,time,FALSE,L3 or later,,,,two-boom,0,0,1,4
+z_boom_u,distance_to_surface_from_boom,Upper boom height,m,physicalMeasurement,time,TRUE,,0.3,10,dshf_u dlhf_u qh_u,all,1,1,1,4
+z_boom_q_u,distance_to_surface_from_boom_quality,Upper boom height (quality),-,qualityInformation,time,TRUE,L0 or L2,,,,all,1,1,0,4
+z_boom_l,distance_to_surface_from_boom,Lower boom height,m,physicalMeasurement,time,TRUE,,0.3,5,dshf_l dlhf_l qh_l,two-boom,1,1,1,4
+z_boom_q_l,distance_to_surface_from_boom_quality,Lower boom height (quality),-,qualityInformation,time,TRUE,L0 or L2,,,,two-boom,1,1,0,4
+z_stake,distance_to_surface_from_stake_assembly,Stake height,m,physicalMeasurement,time,TRUE,,0.3,8,,one-boom,1,1,1,4
+z_stake_q,distance_to_surface_from_stake_assembly_quality,Stake height (quality),-,qualityInformation,time,TRUE,L0 or L2,,,,one-boom,1,1,0,4
+z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,physicalMeasurement,time,FALSE,,0,30,z_pt_cor,one-boom,1,1,1,4
+z_pt_cor,depth_of_pressure_transducer_in_ice_corrected,Depth of pressure transducer in ice - corrected,m,modelResult,time,FALSE,L2 or later,0,30,,one-boom,0,1,1,4
+z_surf_combined,height_of_surface_combined,"Surface height combined from multiple sensors, relative to ice surface height at installation",m,modelResult,time,FALSE,L3,,,,all,0,0,1,4
+z_ice_surf,height_of_ice_surface,"Ice surface height, relative to ice surface height at installation and calculated from pt_cor and z_stake",m,modelResult,time,FALSE,L3,,,,one-boom,0,0,1,4
+snow_height,height_of_snow,"Snow surface height, relative to ice surface",m,modelResult,time,FALSE,L3,0,,,one-boom,0,0,1,4
+precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,physicalMeasurement,time,TRUE,,0,,precip_u_cor precip_u_rate,all,1,1,1,4
+precip_u_cor,precipitation_corrected,Precipitation (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,all,0,1,1,4
+precip_u_rate,precipitation_rate,Precipitation rate (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,all,0,1,1,4
+precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,physicalMeasurement,time,TRUE,,0,,precip_l_cor precip_l_rate,two-boom,1,1,1,4
+precip_l_cor,precipitation_corrected,Precipitation (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,two-boom,0,1,1,4
+precip_l_rate,precipitation_rate,Precipitation rate (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,two-boom,0,1,1,4
+t_i_1,ice_temperature_at_t1,Ice temperature at sensor 1,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,all,1,1,1,4
+t_i_2,ice_temperature_at_t2,Ice temperature at sensor 2,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,all,1,1,1,4
+t_i_3,ice_temperature_at_t3,Ice temperature at sensor 3,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,all,1,1,1,4
+t_i_4,ice_temperature_at_t4,Ice temperature at sensor 4,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,all,1,1,1,4
+t_i_5,ice_temperature_at_t5,Ice temperature at sensor 5,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,all,1,1,1,4
+t_i_6,ice_temperature_at_t6,Ice temperature at sensor 6,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,all,1,1,1,4
+t_i_7,ice_temperature_at_t7,Ice temperature at sensor 7,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,all,1,1,1,4
+t_i_8,ice_temperature_at_t8,Ice temperature at sensor 8,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,all,1,1,1,4
+t_i_9,ice_temperature_at_t9,Ice temperature at sensor 9,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,two-boom,1,1,1,4
+t_i_10,ice_temperature_at_t10,Ice temperature at sensor 10,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,two-boom,1,1,1,4
+t_i_11,ice_temperature_at_t11,Ice temperature at sensor 11,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,two-boom,1,1,1,4
+d_t_i_1,depth_of_thermistor_1,Depth of thermistor 1,m,modelResult,time,FALSE,L3,-10,100,,all,0,0,1,4
+d_t_i_2,depth_of_thermistor_2,Depth of thermistor 2,m,modelResult,time,FALSE,L3,-10,100,,all,0,0,1,4
+d_t_i_3,depth_of_thermistor_3,Depth of thermistor 3,m,modelResult,time,FALSE,L3,-10,100,,all,0,0,1,4
+d_t_i_4,depth_of_thermistor_4,Depth of thermistor 4,m,modelResult,time,FALSE,L3,-10,100,,all,0,0,1,4
+d_t_i_5,depth_of_thermistor_5,Depth of thermistor 5,m,modelResult,time,FALSE,L3,-10,100,,all,0,0,1,4
+d_t_i_6,depth_of_thermistor_6,Depth of thermistor 6,m,modelResult,time,FALSE,L3,-10,100,,all,0,0,1,4
+d_t_i_7,depth_of_thermistor_7,Depth of thermistor 7,m,modelResult,time,FALSE,L3,-10,100,,all,0,0,1,4
+d_t_i_8,depth_of_thermistor_8,Depth of thermistor 8,m,modelResult,time,FALSE,L3,-10,100,,all,0,0,1,4
+d_t_i_9,depth_of_thermistor_9,Depth of thermistor 9,m,modelResult,time,FALSE,L3,-10,100,,two-boom,0,0,1,4
+d_t_i_10,depth_of_thermistor_10,Depth of thermistor 10,m,modelResult,time,FALSE,L3,-10,100,,two-boom,0,0,1,4
+d_t_i_11,depth_of_thermistor_11,Depth of thermistor 11,m,modelResult,time,FALSE,L3,-10,100,,two-boom,0,0,1,4
+t_i_10m,10m_subsurface_temperature,10 m subsurface temperature,degrees_C,modelResult,time,FALSE,L3,-70,0,,all,0,0,1,4
+tilt_x,platform_view_angle_x,Tilt to east,degrees,physicalMeasurement,time,FALSE,,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4
+tilt_y,platform_view_angle_y,Tilt to north,degrees,physicalMeasurement,time,FALSE,,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4
+rot,platform_azimuth_angle,Station rotation from true North,degrees,physicalMeasurement,time,FALSE,,0,360,,all,1,1,1,2
+gps_lat,gps_latitude,Latitude,degrees_north,physicalMeasurement,time,TRUE,,50,83,,all,1,1,1,6
+gps_lon,gps_longitude,Longitude,degrees_east,physicalMeasurement,time,TRUE,,5,70,,all,1,1,1,6
+gps_alt,gps_altitude,Altitude above mean sea level (orthometric height),m,physicalMeasurement,time,TRUE,,0,3000,,all,1,1,1,2
+gps_time,gps_time,GPS time,s,physicalMeasurement,time,TRUE,L0 or L2,0,240000,,all,1,1,0,
+gps_geoid,gps_geoid_separation,Height of EGM96 geoid over WGS84 ellipsoid,m,physicalMeasurement,time,TRUE,L0 or L2,,,,one-boom,1,1,0,
+gps_geounit,gps_geounit,GeoUnit,-,qualityInformation,time,TRUE,L0 or L2,,,,all,1,1,0,
+gps_hdop,gps_hdop,GPS horizontal dillution of precision (HDOP),m,qualityInformation,time,TRUE,L0 or L2,,,,all,1,1,0,2
+gps_numsat,gps_numsat,GPS number of satellites,-,qualityInformation,time,TRUE,L0 or L2,,,,,1,1,0,0
+gps_q,gps_q,Quality,-,qualityInformation,time,TRUE,L0 or L2,,,,,1,1,0,
+lat,latitude_postprocessed,smoothed and interpolated latitude of station,degrees_north,modelResult,time,TRUE,L3,,,,all,0,0,1,6
+lon,longitude_postprocessed,smoothed and interpolated longitude of station,degrees_east,modelResult,time,TRUE,L3,,,,all,0,0,1,6
+alt,altitude_postprocessed,smoothed and interpolated altitude of station above mean sea level (orthometric height),m,modelResult,time,TRUE,L3,,,,all,0,0,1,2
+batt_v,battery_voltage,Battery voltage,V,physicalMeasurement,time,TRUE,,0,30,,all,1,1,1,2
+batt_v_ini,,,-,physicalMeasurement,time,TRUE,L0 or L2,0,30,,,1,1,0,2
+batt_v_ss,battery_voltage_at_sample_start,Battery voltage (sample start),V,physicalMeasurement,time,TRUE,L0 or L2,0,30,,,1,1,0,2
+fan_dc_u,fan_current,Fan current (upper boom),mA,physicalMeasurement,time,TRUE,L0 or L2,0,200,,all,1,1,0,2
+fan_dc_l,fan_current,Fan current (lower boom),mA,physicalMeasurement,time,TRUE,,0,200,,two-boom,1,1,0,2
+freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,physicalMeasurement,time,TRUE,L0 or L2,0,10000,precip_u,,1,1,0,
+t_log,temperature_of_logger,Logger temperature,degrees_C,physicalMeasurement,time,TRUE,,-80,40,,one-boom,1,1,0,4
+t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,physicalMeasurement,time,FALSE,,-80,40,t_surf dlr ulr,all,1,1,1,4
+p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,physicalMeasurement,time,TRUE,,-350,100,,all,1,1,1,4
+t_i,air_temperature,Air temperature (instantaneous),degrees_C,physicalMeasurement,time,TRUE,,-80,40,,all,1,1,1,4
+rh_i,relative_humidity,Relative humidity (instantaneous),%,physicalMeasurement,time,TRUE,,0,150,rh_i_cor,all,1,1,1,4
+rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,modelResult,time,TRUE,L2 or later,0,100,,all,0,1,1,4
+wspd_i,wind_speed,Wind speed (instantaneous),m s-1,physicalMeasurement,time,TRUE,,0,100,wdir_i wspd_x_i wspd_y_i,all,1,1,1,4
+wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,physicalMeasurement,time,TRUE,,1,360,wspd_x_i wspd_y_i,all,1,1,1,4
+wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,modelResult,time,TRUE,L2 or later,-100,100,wdir_i wspd_i,all,0,1,1,4
+wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,modelResult,time,TRUE,L2 or later,-100,100,wdir_i wspd_i,all,0,1,1,4

pypromice/station_configuration.py

@@ -0,0 +1,118 @@
+import logging
+from pathlib import Path
+from typing import Optional, Dict, Mapping, Sequence
+
+import attrs
+import toml
+
+
+@attrs.define
+class StationConfiguration:
+    """
+    Helper class for storing station specific configurations with respect to
+
+    * Installation specific distance measurements such as height differences between instruments
+    * Reference strings such as stid, station_site and wmo_id
+    * BUFR export specific parameters
+
+    # TODO: The station related meta data should be fetched from a station specific configuration files in the future or
+    # from header data in data source.
+    """
+
+    stid: str
+    station_site: str = None
+    project: Optional[str] = None
+    station_type: Optional[str] = None
+    wmo_id: Optional[str] = None
+    barometer_from_gps: Optional[float] = None
+    anemometer_from_sonic_ranger: Optional[float] = None
+    temperature_from_sonic_ranger: Optional[float] = None
+    height_of_gps_from_station_ground: Optional[float] = None
+    sonic_ranger_from_gps: Optional[float] = None
+    static_height_of_gps_from_mean_sea_level: Optional[float] = None
+    station_relocation: Sequence[str] = attrs.field(factory=list)
+    location_type: Optional[str] = None
+
+    # The station data will be exported to BUFR if True. Otherwise, it will only export latest position
+    export_bufr: bool = False
+    comment: Optional[str] = None
+
+    # skip specific variables for stations
+    # If a variable has known bad data, use this collection to skip the variable
+    # Note that if a station is not reporting both air temp and pressure it will be skipped,
+    # as currently implemented in csv2bufr.min_data_check().
+    # ['p_i'], # EXAMPLE
+    skipped_variables: Sequence[str] = attrs.field(factory=list)
+
+    positions_update_timestamp_only: bool = False
+
+    @classmethod
+    def load_toml(cls, path, skip_unexpected_fields=False):
+        config_fields = {field.name for field in attrs.fields(cls)}
+        input_dict = toml.load(path)
+        unexpected_fields = set(input_dict.keys()) - config_fields
+        if unexpected_fields:
+            if skip_unexpected_fields:
+                logging.info(
+                    f"Skipping unexpected fields in toml file {path}: "
+                    + ", ".join(unexpected_fields)
+                )
+                for field in unexpected_fields:
+                    input_dict.pop(field)
+            else:
+                raise ValueError(f"Unexpected fields: {unexpected_fields}")
+
+        return cls(**input_dict)
+
+    def dump_toml(self, path: Path):
+        with path.open("w") as fp:
+            toml.dump(self.as_dict(), fp)
+
+    def as_dict(self) -> Dict:
+        return attrs.asdict(self)
+
+
+def load_station_configuration_mapping(
+    configurations_root_dir: Path,
+    **kwargs,
+) -> Mapping[str, StationConfiguration]:
+    """
+    Load station configurations from toml files in configurations_root_dir
+
+    Parameters
+    ----------
+    configurations_root_dir
+        Root directory containing toml files
+    kwargs
+        Additional arguments to pass to StationConfiguration.load_toml
+
+    Returns
+    -------
+    Mapping from stid to StationConfiguration
+
+    """
+    return {
+        config_file.stem: StationConfiguration.load_toml(config_file, **kwargs)
+        for config_file in configurations_root_dir.glob("*.toml")
+    }
+
+
+def write_station_configuration_mapping(
+    station_configurations: Mapping[str, StationConfiguration],
+    configurations_root_dir: Path,
+) -> None:
+    """
+    Write station configurations to toml files in configurations_root_dir
+
+    Parameters
+    ----------
+    station_configurations
+        Mapping from stid to StationConfiguration
+    configurations_root_dir
+        Output directory
+
+    """
+    configurations_root_dir.mkdir(parents=True, exist_ok=True)
+    for stid, station_configuration in station_configurations.items():
+        with (configurations_root_dir / f"{stid}.toml").open("w") as fp:
+            toml.dump(station_configuration.as_dict(), fp)

pypromice/tx/get_l0tx.py

@@ -6,7 +6,7 @@ import os, imaplib, email, toml, re, unittest
 from glob import glob
 from datetime import datetime, timedelta
 
-from pypromice.tx import getMail, L0tx, sortLines
+from pypromice.tx import getMail, L0tx, sortLines, isModified
 
 
 def parse_arguments_l0tx():
@@ -134,9 +134,12 @@ def get_l0tx():
  	#----------------------------------
  	
     if out_dir is not None:
-        # Sort L0tx files and add tails    
-        print(out_dir+'/'+args.name+'*.txt')
-        for f in glob(out_dir+'/'+args.name+'*.txt'):
+
+        # Find modified files (within the last hour)         
+        mfiles = [mfile for mfile in glob(out_dir+'/'+args.name+'*.txt') if isModified(mfile, 1)]          
+        
+        # Sort L0tx files and add tails 
+        for f in mfiles:
         
         	# Sort lines in L0tx file and remove duplicates
             in_dirn, in_fn = os.path.split(f)    

pypromice/tx/payload_formats.csv

@@ -66,6 +66,7 @@ type,expected_values,format,description,flags,notes
 90,47,tfffffffffffffffffffffffffffffffffgneffffffffff,THE GC-NET VERSION!,don’t use,GC-NET stations
 90,48, tfffffffffffffffffffffffffffffffffgnefffffffffff,THE GC-NET VERSION!,don’t use,GC-NET stations
 90,48,tfffffffffffffffffffffffffffffffffgnefffffffffff,THE GC-NET VERSION!,,GC-NET stations
+95,46,tfffffffffffffffffffffffffffffffgnefffffffffff,THE GC-NET VERSION!,,GC-NET stations
 220,29,tfffffffffffffffffffffffnefff,ZAMG Freya 2015 summer message,,ZAMG Freya aws
 221,0,,,,ZAMG Freya aws
 222,29,tfffffffffffffffffffffffnefff,ZAMG Freya 2015 winter message,,ZAMG Freya aws

pypromice/tx/tx.py

@@ -886,13 +886,13 @@ def sortLines(in_file, out_file, replace_unsorted=True):                       #
         lines = in_f.readlines()
     
     # Remove duplicate lines and sort
-    unique_lines = findDuplicates(lines)
+    unique_lines = findDuplicates(lines.copy())
     unique_lines.sort()
-    
-    # Write sorted file
-    with open(out_file, 'w') as out_f:
-        # out_f.write(headers)
-        out_f.writelines(unique_lines)
+    if lines != unique_lines:
+        # Write sorted file
+        with open(out_file, 'w') as out_f:
+            # out_f.write(headers)
+            out_f.writelines(unique_lines)
     
     # Replace input file with new sorted file
     if replace_unsorted:
@@ -930,6 +930,27 @@ def addTail(in_file, out_dir, aws_name, header_names='', lines_limit=100):
         out_f.writelines(tail) 
         print(f'Tails file written to {out_pn}')
 
+def isModified(filename, time_threshold=1):
+    '''Return flag denoting if file is modified within a certain timeframe
+    
+    Parameters
+    ----------
+    filename : str
+        File path
+    time_threshold : int
+        Time threshold (provided in hours)
+    
+    Returns
+    -------
+    bool
+        Flag denoting if modified (True) or not (False)
+    '''
+    delta = time.time() - os.path.getmtime(filename)
+    delta = delta / (60*60)
+    if delta < time_threshold:
+        return True
+    return False
+
 def _loadTestMsg():
     '''Load test .msg email file'''
     with pkg_resources.resource_stream('pypromice', 'test/test_email') as stream:

pypromice/utilities/git.py

@@ -0,0 +1,61 @@
+import subprocess
+import os
+from pathlib import Path
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def get_commit_hash_and_check_dirty(file_path) -> str:
+    repo_path = Path(file_path).parent
+
+    try:
+        # Ensure the file path is relative to the repository
+        relative_file_path = os.path.relpath(file_path, repo_path)
+
+        # Get the latest commit hash for the file
+        commit_hash = (
+            subprocess.check_output(
+                [
+                    "git",
+                    "-C",
+                    repo_path,
+                    "log",
+                    "-n",
+                    "1",
+                    "--pretty=format:%H",
+                    #"--",
+                    #relative_file_path,
+                ],
+                stderr=subprocess.STDOUT,
+            )
+            .strip()
+            .decode("utf-8")
+        )
+
+        # Check if the file is dirty (has uncommitted changes)
+        diff_output = (
+            subprocess.check_output(
+                ["git", "-C", repo_path, "diff"],
+                stderr=subprocess.STDOUT,
+            )
+            .strip()
+            .decode("utf-8")
+        )
+
+        # If diff_output is not empty, the file has uncommitted changes
+        is_dirty = len(diff_output) > 0
+
+        if is_dirty:
+            logger.warning(f"Warning: The file {file_path} is dirty compared to the last commit. {commit_hash}")
+            return 'unknown'
+        if commit_hash == "":
+            logger.warning(f"Warning: The file {file_path} is not under version control.")
+            return 'unknown'
+
+        print(f"Commit hash: {commit_hash}")
+        return commit_hash
+    except subprocess.CalledProcessError as e:
+        logger.warning(f"Error: {e.output.decode('utf-8')}")
+        return 'unknown'