pypromice 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

pypromice/process/L0toL1.py

@@ -57,7 +57,9 @@ def toL1(L0, vars_df, T_0=273.15, tilt_threshold=-100):
         ds['ulr'] = ((ds['ulr'] * 10) / ds.attrs['ulr_eng_coef']) + 5.67E-8*(ds['t_rad'] + T_0)**4
 
     ds['z_boom_u'] = _reformatArray(ds['z_boom_u'])                            # Reformat boom height
-    ds['z_boom_u'] = ds['z_boom_u'] * ((ds['t_u'] + T_0)/T_0)**0.5             # Adjust sonic ranger readings for sensitivity to air temperature       
+    
+    ds['t_u_interp'] = interpTemp(ds['t_u'], vars_df)
+    ds['z_boom_u'] = ds['z_boom_u'] * ((ds['t_u_interp'] + T_0)/T_0)**0.5      # Adjust sonic ranger readings for sensitivity to air temperature       
     
     if ds['gps_lat'].dtype.kind == 'O':                                        # Decode and reformat GPS information
         if 'NH' in ds['gps_lat'].dropna(dim='time').values[1]:
@@ -113,7 +115,8 @@ def toL1(L0, vars_df, T_0=273.15, tilt_threshold=-100):
         
     elif ds.attrs['number_of_booms']==2:                                       # 2-boom processing
         ds['z_boom_l'] = _reformatArray(ds['z_boom_l'])                        # Reformat boom height    
-        ds['z_boom_l'] = ds['z_boom_l'] * ((ds['t_l'] + T_0)/T_0)**0.5         # Adjust sonic ranger readings for sensitivity to air temperature    
+        ds['t_l_interp'] = interpTemp(ds['t_l'], vars_df)
+        ds['z_boom_l'] = ds['z_boom_l'] * ((ds['t_l_interp']+ T_0)/T_0)**0.5   # Adjust sonic ranger readings for sensitivity to air temperature    
 
     ds = clip_values(ds, vars_df)
     for key in ['hygroclip_t_offset', 'dsr_eng_coef', 'usr_eng_coef',
@@ -169,14 +172,16 @@ def addTimeShift(ds, vars_df):
     elif ds.attrs['format'] == 'STM':
         # hourly-averaged, non-transmitted
         # shift everything except instantaneous, any logger type
-        df_a = df_a.shift(periods=-1, freq="H")
+        df_a = df_a.shift(periods=-1, freq="h")
         df_out = pd.concat([df_a, df_i], axis=1) # different columns, same datetime indices
+        df_out = df_out.sort_index()
     elif ds.attrs['format'] == 'TX':
         if ds.attrs['logger_type'] == 'CR1000X':
             # v3, data is hourly all year long
             # shift everything except instantaneous
             df_a = df_a.shift(periods=-1, freq="H")
             df_out = pd.concat([df_a, df_i], axis=1) # different columns, same datetime indices
+            df_out = df_out.sort_index()
         elif ds.attrs['logger_type'] == 'CR1000':
             # v2, data is hourly (6-hr for instantaneous) for DOY 100-300, otherwise daily at 00 UTC
             # shift non-instantaneous hourly for DOY 100-300, else do not shift daily
@@ -186,7 +191,7 @@ def addTimeShift(ds, vars_df):
             df_a_daily_2 = df_a.loc[(df_a['doy'] > 300)]
 
             # shift the hourly ave data
-            df_a_hourly = df_a_hourly.shift(periods=-1, freq="H")
+            df_a_hourly = df_a_hourly.shift(periods=-1, freq="h")
 
             # stitch everything back together
             df_concat_u = pd.concat([df_a_daily_1, df_a_daily_2, df_a_hourly], axis=0) # same columns, different datetime indices
@@ -254,6 +259,41 @@ def getPressDepth(z_pt, p, pt_antifreeze, pt_z_factor, pt_z_coef, pt_z_p_coef):
     
     return z_pt_cor, z_pt
 
+
+def interpTemp(temp, var_configurations, max_interp=pd.Timedelta(12,'h')):
+    '''Clip and interpolate temperature dataset for use in corrections
+    
+    Parameters
+    ----------
+    temp : `xarray.DataArray`
+        Array of temperature data
+    vars_df : `pandas.DataFrame`
+        Dataframe to retrieve attribute hi-lo values from for temperature clipping
+    max_interp : `pandas.Timedelta`
+        Maximum time steps to interpolate across. The default is 12 hours.
+    
+    Returns
+    -------
+    temp_interp : `xarray.DataArray`
+        Array of interpolatedtemperature data
+    '''
+    # Determine if upper or lower temperature array
+    var = temp.name.lower()
+    
+    # Find range threshold and use it to clip measurements
+    cols = ["lo", "hi", "OOL"]
+    assert set(cols) <= set(var_configurations.columns)
+    variable_limits = var_configurations[cols].dropna(how="all")
+    temp = temp.where(temp >= variable_limits.loc[var,'lo'])
+    temp = temp.where(temp <= variable_limits.loc[var, 'hi'])
+    
+    # Drop duplicates and interpolate across NaN values
+#    temp_interp = temp.drop_duplicates(dim='time', keep='first')
+    temp_interp = temp.interpolate_na(dim='time', max_gap=max_interp)
+    
+    return temp_interp
+
+
 def smoothTilt(tilt, win_size):
     '''Smooth tilt values using a rolling window. This is translated from the
     previous IDL/GDL smoothing algorithm:

pypromice/process/value_clipping.py

@@ -29,22 +29,14 @@ def clip_values(
 
     variable_limits = var_configurations[cols].dropna(how="all")
     for var, row in variable_limits.iterrows():
+
         if var not in list(ds.variables):
             continue
 
-        if var in ["rh_u_cor", "rh_l_cor"]:
-            ds[var] = ds[var].where(ds[var] >= row.lo, other=0)
-            ds[var] = ds[var].where(ds[var] <= row.hi, other=100)
-
-            # Mask out invalid corrections based on uncorrected var
-            var_uncor = var.split("_cor")[0]
-            ds[var] = ds[var].where(~np.isnan(ds[var_uncor]), other=np.nan)
-
-        else:
-            if ~np.isnan(row.lo):
-                ds[var] = ds[var].where(ds[var] >= row.lo)
-            if ~np.isnan(row.hi):
-                ds[var] = ds[var].where(ds[var] <= row.hi)
+        if ~np.isnan(row.lo):
+            ds[var] = ds[var].where(ds[var] >= row.lo)
+        if ~np.isnan(row.hi):
+            ds[var] = ds[var].where(ds[var] <= row.hi)
 
         other_vars = row.OOL
         if isinstance(other_vars, str) and ~ds[var].isnull().all():
@@ -56,4 +48,5 @@ def clip_values(
                         ds[var] = ds[var].where(ds[var] >= row.lo)
                     if ~np.isnan(row.hi):
                         ds[var] = ds[var].where(ds[var] <= row.hi)
+
     return ds

pypromice/process/variables.csv

@@ -5,12 +5,12 @@ p_u,air_pressure,Air pressure (upper boom),hPa,650,1100,z_pt z_pt_cor dshf_u dlh
 p_l,air_pressure,Air pressure (lower boom),hPa,650,1100,dshf_l dlhf_l qh_l,two-boom,all,4,physicalMeasurement,time lat lon alt,False,
 t_u,air_temperature,Air temperature (upper boom),degrees_C,-80,40,rh_u_cor cc dsr_cor usr_cor z_boom z_stake dshf_u dlhf_u qh_u,all,all,4,physicalMeasurement,time lat lon alt,False,
 t_l,air_temperature,Air temperature (lower boom),degrees_C,-80,40,rh_l_cor z_boom_l dshf_l dlhf_l qh_l ,two-boom,all,4,physicalMeasurement,time lat lon alt,False,PT100 temperature at boom
-rh_u,relative_humidity,Relative humidity (upper boom),%,0,150,rh_u_cor,all,all,4,physicalMeasurement,time lat lon alt,False,
+rh_u,relative_humidity,Relative humidity (upper boom),%,0,100,rh_u_cor,all,all,4,physicalMeasurement,time lat lon alt,False,
 rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,0,150,dshf_u dlhf_u qh_u,all,all,4,modelResult,time lat lon alt,False,
-qh_u,specific_humidity,Specific humidity (upper boom),%,0,100,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
-rh_l,relative_humidity,Relative humidity (lower boom),%,0,150,rh_l_cor,two-boom,all,4,physicalMeasurement,time lat lon alt,False,
+qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,0,100,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
+rh_l,relative_humidity,Relative humidity (lower boom),%,0,100,rh_l_cor,two-boom,all,4,physicalMeasurement,time lat lon alt,False,
 rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,0,150,dshf_l dlhf_l qh_l,two-boom,all,4,modelResult,time lat lon alt,False,
-qh_l,specific_humidity,Specific humidity (lower boom),%,0,100,,two-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
+qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,0,100,,two-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
 wspd_u,wind_speed,Wind speed (upper boom),m s-1,0,100,"wdir_u wspd_x_u wspd_y_u dshf_u dlhf_u qh_u, precip_u",all,all,4,physicalMeasurement,time lat lon alt,False,
 wspd_l,wind_speed,Wind speed (lower boom),m s-1,0,100,"wdir_l wspd_x_l wspd_y_l dshf_l dlhf_l qh_l , precip_l",two-boom,all,4,physicalMeasurement,time lat lon alt,False,
 wdir_u,wind_from_direction,Wind from direction (upper boom),degrees,1,360,wspd_x_u wspd_y_u,all,all,4,physicalMeasurement,time lat lon alt,False,
@@ -81,14 +81,12 @@ fan_dc_l,fan_current,Fan current (lower boom),mA,0,200,,two-boom,all,2,physicalM
 freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,0,10000,precip_u,,all,,physicalMeasurement,time lat lon alt,True,L0 only
 t_log,temperature_of_logger,Logger temperature,degrees_C,-80,40,,one-boom,all,4,physicalMeasurement,time lat lon alt,True,LoggerTemperature(C)
 t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,-80,40,t_surf dlr ulr,all,all,4,physicalMeasurement,time lat lon alt,False,
-p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,-350,100,,all,TX,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
-t_i,air_temperature,Air temperature (instantaneous),degrees_C,-80,40,,all,TX,4,physicalMeasurement,time lat lon alt,True,"PT100 temperature at boom, for meteorological observations"
-rh_i,relative_humidity,Relative humidity (instantaneous),%,0,150,rh_i_cor,all,TX,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
-rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,0,100,,all,TX,4,modelResult,time lat lon alt,True,For meteorological observations
-wspd_i,wind_speed,Wind speed (instantaneous),m s-1,0,100,wdir_i wspd_x_i wspd_y_i,all,TX,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
-wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,1,360,wspd_x_i wspd_y_i,all,TX,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
-wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,-100,100,wdir_i wspd_i,all,TX,4,modelResult,time lat lon alt,True,For meteorological observations
-wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,-100,100,wdir_i wspd_i,all,TX,4,modelResult,time lat lon alt,True,For meteorological observations
-msg_i,message,Message string (instantaneous),-,,,,all,TX,,qualityInformation,time lat lon alt,True,For meteorological observations
-msg_lat,message_latitude,latitude from modem (email text),degrees_north,50,83,,all,all,6,coordinate,time lat lon alt,True,
-msg_lon,message_longitude,longitude from modem (email text),degrees_east,-72,13,,all,all,6,coordinate,time lat lon alt,True,
+p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,-350,100,,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
+t_i,air_temperature,Air temperature (instantaneous),degrees_C,-80,40,,all,all,4,physicalMeasurement,time lat lon alt,True,"PT100 temperature at boom, for meteorological observations"
+rh_i,relative_humidity,Relative humidity (instantaneous),%,0,150,rh_i_cor,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
+rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,0,100,,all,all,4,modelResult,time lat lon alt,True,For meteorological observations
+wspd_i,wind_speed,Wind speed (instantaneous),m s-1,0,100,wdir_i wspd_x_i wspd_y_i,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
+wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,1,360,wspd_x_i wspd_y_i,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
+wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,-100,100,wdir_i wspd_i,all,all,4,modelResult,time lat lon alt,True,For meteorological observations
+wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,-100,100,wdir_i wspd_i,all,all,4,modelResult,time lat lon alt,True,For meteorological observations
+msg_i,message,Message string (instantaneous),-,,,,all,all,,qualityInformation,time lat lon alt,True,For meteorological observations

pypromice/qc/github_data_issues.py

@@ -17,8 +17,7 @@ logger = logging.getLogger(__name__)
 
 
 def flagNAN(ds_in,
-            flag_url='https://raw.githubusercontent.com/GEUS-Glaciology-and-Climate/PROMICE-AWS-data-issues/master/flags/',
-            flag_dir='local/flags/'):
+            flag_dir='../PROMICE-AWS-data-issues/flags'):
     '''Read flagged data from .csv file. For each variable, and downstream
     dependents, flag as invalid (or other) if set in the flag .csv
 
@@ -26,8 +25,6 @@ def flagNAN(ds_in,
     ----------
     ds_in : xr.Dataset
         Level 0 dataset
-    flag_url : str
-        URL to directory where .csv flag files can be found
     flag_dir : str
         File directory where .csv flag files can be found
 
@@ -36,13 +33,10 @@ def flagNAN(ds_in,
     ds : xr.Dataset
         Level 0 data with flagged data
     '''
-    ds = ds_in.copy()
+    ds = ds_in.copy(deep=True)
     df = None
 
-    df = _getDF(flag_url + ds.attrs["station_id"] + ".csv",
-                os.path.join(flag_dir, ds.attrs["station_id"] + ".csv"),
-                # download = False,  # only for working on draft local flag'n'fix files
-                )
+    df = _getDF(os.path.join(flag_dir, ds.attrs["station_id"] + ".csv"))
 
     if isinstance(df, pd.DataFrame):
         df.t0 = pd.to_datetime(df.t0).dt.tz_localize(None)
@@ -80,8 +74,7 @@ def flagNAN(ds_in,
 
 
 def adjustTime(ds,
-               adj_url="https://raw.githubusercontent.com/GEUS-Glaciology-and-Climate/PROMICE-AWS-data-issues/master/adjustments/",
-               adj_dir='local/adjustments/',
+               adj_dir='../PROMICE-AWS-data-issues/adjustments/',
                var_list=[], skip_var=[]):
     '''Read adjustment data from .csv file. Only applies the "time_shift" adjustment
 
@@ -89,8 +82,6 @@ def adjustTime(ds,
     ----------
     ds : xr.Dataset
         Level 0 dataset
-    adj_url : str
-        URL to directory where .csv adjustment files can be found
     adj_dir : str
         File directory where .csv adjustment files can be found
 
@@ -99,11 +90,10 @@ def adjustTime(ds,
     ds : xr.Dataset
         Level 0 data with flagged data
     '''
-    ds_out = ds.copy()
+    ds_out = ds.copy(deep=True)
     adj_info=None
 
-    adj_info = _getDF(adj_url + ds.attrs["station_id"] + ".csv",
-                      os.path.join(adj_dir, ds.attrs["station_id"] + ".csv"),)
+    adj_info = _getDF(os.path.join(adj_dir, ds.attrs["station_id"] + ".csv"))
 
     if isinstance(adj_info, pd.DataFrame):
 
@@ -145,8 +135,7 @@ def adjustTime(ds,
 
 
 def adjustData(ds,
-               adj_url="https://raw.githubusercontent.com/GEUS-Glaciology-and-Climate/PROMICE-AWS-data-issues/master/adjustments/",
-               adj_dir='local/adjustments/',
+               adj_dir='../PROMICE-AWS-data-issues/adjustments/',
                var_list=[], skip_var=[]):
     '''Read adjustment data from .csv file. For each variable, and downstream
     dependents, adjust data accordingly if set in the adjustment .csv
@@ -155,8 +144,6 @@ def adjustData(ds,
     ----------
     ds : xr.Dataset
         Level 0 dataset
-    adj_url : str
-        URL to directory where .csv adjustment files can be found
     adj_dir : str
         File directory where .csv adjustment files can be found
 
@@ -165,24 +152,19 @@ def adjustData(ds,
     ds : xr.Dataset
         Level 0 data with flagged data
     '''
-    ds_out = ds.copy()
+    ds_out = ds.copy(deep=True)
     adj_info=None
-    adj_info = _getDF(adj_url + ds.attrs["station_id"] + ".csv",
-                      os.path.join(adj_dir, ds.attrs["station_id"] + ".csv"),
-                      # download = False,  # only for working on draft local flag'n'fix files
-                      )
+    adj_info = _getDF(os.path.join(adj_dir, ds.attrs["station_id"] + ".csv"))
 
     if isinstance(adj_info, pd.DataFrame):
         # removing potential time shifts from the adjustment list
         adj_info = adj_info.loc[adj_info.adjust_function != "time_shift", :]
 
-        # if t1 is left empty, then adjustment is applied until the end of the file
-        adj_info.loc[adj_info.t0.isnull(), "t0"] = ds_out.time.values[0]
-        adj_info.loc[adj_info.t1.isnull(), "t1"] = ds_out.time.values[-1]
-        # making all timestamps timezone naive (compatibility with xarray)
-        adj_info.t0 = pd.to_datetime(adj_info.t0).dt.tz_localize(None)
-        adj_info.t1 = pd.to_datetime(adj_info.t1).dt.tz_localize(None)
-
+        # making sure that t0 and t1 columns are object dtype then replaceing nan with None
+        adj_info[['t0','t1']] = adj_info[['t0','t1']].astype(object)
+        adj_info.loc[adj_info.t1.isnull()|(adj_info.t1==''), "t1"] = None      
+        adj_info.loc[adj_info.t0.isnull()|(adj_info.t0==''), "t0"] = None
+		
         # if "*" is in the variable name then we interpret it as regex
         selec =  adj_info['variable'].str.contains('\*') & (adj_info['variable'] != "*")
         for ind in adj_info.loc[selec, :].index:
@@ -217,104 +199,104 @@ def adjustData(ds,
                 adj_info.loc[var].adjust_function,
                 adj_info.loc[var].adjust_value,
             ):
-                if (t0 > pd.to_datetime(ds_out.time.values[-1])) | (t1 < pd.to_datetime(ds_out.time.values[0])):
+                # making all timestamps timezone naive (compatibility with xarray)
+                if isinstance(t0, str):
+                    t0 = pd.to_datetime(t0, utc=True).tz_localize(None)
+                if isinstance(t1, str):
+                    t1 = pd.to_datetime(t1, utc=True).tz_localize(None)
+
+                index_slice = dict(time=slice(t0, t1))
+
+                if len(ds_out[var].loc[index_slice].time.time) == 0:
+                    logger.info("Time range does not intersect with dataset")
                     continue
+
                 logger.info(f'---> {t0} {t1} {var} {func} {val}')
+				
                 if func == "add":
-                    ds_out[var].loc[dict(time=slice(t0, t1))] = ds_out[var].loc[dict(time=slice(t0, t1))].values + val
+                    ds_out[var].loc[index_slice] = ds_out[var].loc[index_slice].values + val
                     # flagging adjusted values
                     # if var + "_adj_flag" not in ds_out.columns:
                     #     ds_out[var + "_adj_flag"] = 0
-                    # msk = ds_out[var].loc[dict(time=slice(t0, t1))])].notnull()
-                    # ind = ds_out[var].loc[dict(time=slice(t0, t1))])].loc[msk].time
+                    # msk = ds_out[var].loc[index_slice])].notnull()
+                    # ind = ds_out[var].loc[index_slice])].loc[msk].time
                     # ds_out.loc[ind, var + "_adj_flag"] = 1
 
                 if func == "multiply":
-                    ds_out[var].loc[dict(time=slice(t0, t1))] = ds_out[var].loc[dict(time=slice(t0, t1))].values * val
+                    ds_out[var].loc[index_slice] = ds_out[var].loc[index_slice].values * val
                     if "DW" in var:
-                        ds_out[var].loc[dict(time=slice(t0, t1))] = ds_out[var].loc[dict(time=slice(t0, t1))] % 360
+                        ds_out[var].loc[index_slice] = ds_out[var].loc[index_slice] % 360
                     # flagging adjusted values
                     # if var + "_adj_flag" not in ds_out.columns:
                     #     ds_out[var + "_adj_flag"] = 0
-                    # msk = ds_out[var].loc[dict(time=slice(t0, t1))].notnull()
-                    # ind = ds_out[var].loc[dict(time=slice(t0, t1))].loc[msk].time
+                    # msk = ds_out[var].loc[index_slice].notnull()
+                    # ind = ds_out[var].loc[index_slice].loc[msk].time
                     # ds_out.loc[ind, var + "_adj_flag"] = 1
 
                 if func == "min_filter":
-                    tmp = ds_out[var].loc[dict(time=slice(t0, t1))].values
+                    tmp = ds_out[var].loc[index_slice].values
                     tmp[tmp < val] = np.nan
 
                 if func == "max_filter":
-                    tmp = ds_out[var].loc[dict(time=slice(t0, t1))].values
+                    tmp = ds_out[var].loc[index_slice].values
                     tmp[tmp > val] = np.nan
-                    ds_out[var].loc[dict(time=slice(t0, t1))] = tmp
+                    ds_out[var].loc[index_slice] = tmp
 
                 if func == "upper_perc_filter":
-                    tmp = ds_out[var].loc[dict(time=slice(t0, t1))].copy()
-                    df_w = ds_out[var].loc[dict(time=slice(t0, t1))].resample("14D").quantile(1 - val / 100)
-                    df_w = ds_out[var].loc[dict(time=slice(t0, t1))].resample("14D").var()
+                    tmp = ds_out[var].loc[index_slice].copy()
+                    df_w = ds_out[var].loc[index_slice].resample(time="14D").quantile(1 - val / 100)
+                    df_w = ds_out[var].loc[index_slice].resample(time="14D").var()
                     for m_start, m_end in zip(df_w.time[:-2], df_w.time[1:]):
                         msk = (tmp.time >= m_start) & (tmp.time < m_end)
                         values_month = tmp.loc[msk].values
                         values_month[values_month < df_w.loc[m_start]] = np.nan
                         tmp.loc[msk] = values_month
 
-                    ds_out[var].loc[dict(time=slice(t0, t1))] = tmp.values
+                    ds_out[var].loc[index_slice] = tmp.values
 
                 if func == "biweekly_upper_range_filter":
-                    tmp = ds_out[var].loc[dict(time=slice(t0, t1))].copy()
-                    df_max = ds_out[var].loc[dict(time=slice(t0, t1))].resample("14D").max()
-                    for m_start, m_end in zip(df_max.time[:-2], df_max.time[1:]):
-                        msk = (tmp.time >= m_start) & (tmp.time < m_end)
-                        lim = df_max.loc[m_start] - val
-                        values_month = tmp.loc[msk].values
-                        values_month[values_month < lim] = np.nan
-                        tmp.loc[msk] = values_month
-                    # remaining samples following outside of the last 2 weeks window
-                    msk = tmp.time >= m_end
-                    lim = df_max.loc[m_start] - val
-                    values_month = tmp.loc[msk].values
-                    values_month[values_month < lim] = np.nan
-                    tmp.loc[msk] = values_month
-                    # updating original pandas
-                    ds_out[var].loc[dict(time=slice(t0, t1))] = tmp.values
+                    df_max = (
+                        ds_out[var].loc[index_slice].copy(deep=True)
+                        .resample(time="14D",offset='7D').max()
+                        .sel(time=ds_out[var].loc[index_slice].time.values, method='ffill')
+                        )
+                    df_max['time'] = ds_out[var].loc[index_slice].time
+                    # updating original pandas                   
+                    ds_out[var].loc[index_slice] = ds_out[var].loc[index_slice].where(ds_out[var].loc[index_slice] > df_max-val)
+                                        
 
                 if func == "hampel_filter":
-                    tmp = ds_out[var].loc[dict(time=slice(t0, t1))]
+                    tmp = ds_out[var].loc[index_slice]
                     tmp = _hampel(tmp, k=7 * 24, t0=val)
-                    ds_out[var].loc[dict(time=slice(t0, t1))] = tmp.values
+                    ds_out[var].loc[index_slice] = tmp.values
 
                 if func == "grad_filter":
-                    tmp = ds_out[var].loc[dict(time=slice(t0, t1))].copy()
-                    msk = ds_out[var].loc[dict(time=slice(t0, t1))].copy().diff()
+                    tmp = ds_out[var].loc[index_slice].copy()
+                    msk = ds_out[var].loc[index_slice].copy().diff()
                     tmp[np.roll(msk.abs() > val, -1)] = np.nan
-                    ds_out[var].loc[dict(time=slice(t0, t1))] = tmp
+                    ds_out[var].loc[index_slice] = tmp
 
                 if "swap_with_" in func:
                     var2 = func[10:]
-                    val_var = ds_out[var].loc[dict(time=slice(t0, t1))].values.copy()
-                    val_var2 = ds_out[var2].loc[dict(time=slice(t0, t1))].values.copy()
-                    ds_out[var2].loc[dict(time=slice(t0, t1))] = val_var
-                    ds_out[var].loc[dict(time=slice(t0, t1))] = val_var2
+                    val_var = ds_out[var].loc[index_slice].values.copy()
+                    val_var2 = ds_out[var2].loc[index_slice].values.copy()
+                    ds_out[var2].loc[index_slice] = val_var
+                    ds_out[var].loc[index_slice] = val_var2
 
                 if func == "rotate":
-                    ds_out[var].loc[dict(time=slice(t0, t1))] = (ds_out[var].loc[dict(time=slice(t0, t1))].values + val) % 360
+                    ds_out[var].loc[index_slice] = (ds_out[var].loc[index_slice].values + val) % 360
 
     return ds_out
 
 
-def _getDF(flag_url, flag_file, download=True):
+def _getDF(flag_file):
     '''Get dataframe from flag or adjust file. First attempt to retrieve from
     URL. If this fails then attempt to retrieve from local file
 
     Parameters
     ----------
-    flag_url : str
-        URL address to file
     flag_file : str
         Local path to file
-    download : bool
-        Flag to download file from URL
 
     Returns
     -------
@@ -322,17 +304,7 @@ def _getDF(flag_url, flag_file, download=True):
         Flag or adjustment dataframe
     '''
 
-    # Download local copy as csv
-    if download==True:
-        os.makedirs(os.path.dirname(flag_file), exist_ok = True)
-
-        try:
-            urllib.request.urlretrieve(flag_url, flag_file)
-            logger.info(f"Downloaded a {flag_file.split('/')[-2][:-1],} file to {flag_file}")
-        except (HTTPError, URLError) as e:
-            logger.info(f"Unable to download {flag_file.split('/')[-2][:-1],} file, using local file: {flag_file}")
-    else:
-        logger.info(f"Using local {flag_file.split('/')[-2][:-1],} file: {flag_file}")
+    logger.info(f"Using file: {flag_file}")
 
     if os.path.isfile(flag_file):
         df = pd.read_csv(

{pypromice-1.3.2.dist-info → pypromice-1.3.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pypromice
-Version: 1.3.2
+Version: 1.3.4
 Summary: PROMICE/GC-Net data processing toolbox
 Home-page: https://github.com/GEUS-Glaciology-and-Climate/pypromice
 Author: GEUS Glaciology and Climate
@@ -27,6 +27,7 @@ Requires-Dist: scikit-learn >=1.1.0
 Requires-Dist: Bottleneck
 Requires-Dist: netcdf4
 Requires-Dist: pyDataverse
+Requires-Dist: eccodes
 
 # pypromice
 [![PyPI version](https://badge.fury.io/py/pypromice.svg)](https://badge.fury.io/py/pypromice) [![Anaconda-Server Badge](https://anaconda.org/conda-forge/pypromice/badges/version.svg)](https://anaconda.org/conda-forge/pypromice) [![Anaconda-Server Badge](https://anaconda.org/conda-forge/pypromice/badges/platforms.svg)](https://anaconda.org/conda-forge/pypromice) [![](<https://img.shields.io/badge/Dataverse DOI-10.22008/FK2/3TSBF0-orange>)](https://www.doi.org/10.22008/FK2/3TSBF0) [![DOI](https://joss.theoj.org/papers/10.21105/joss.05298/status.svg)](https://doi.org/10.21105/joss.05298) [![Documentation Status](https://readthedocs.org/projects/pypromice/badge/?version=latest)](https://pypromice.readthedocs.io/en/latest/?badge=latest)

{pypromice-1.3.2.dist-info → pypromice-1.3.4.dist-info}/RECORD

@@ -3,10 +3,13 @@ pypromice/get/__init__.py,sha256=n2L6P9EeUsdjsHaeU7BEanBjlkCBX9csGseT8z-laew,32
 pypromice/get/get.py,sha256=tPrHyllrukGotOR8PjBYJb5tcuL4S_t6WP7wvhbWkg8,7688
 pypromice/get/get_promice_data.py,sha256=bluNCaP50iRlWBzdEOXLrSPepOQdGB7SeQLkTWiqK4c,1806
 pypromice/postprocess/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypromice/postprocess/csv2bufr.py,sha256=9IyhGyk5VRmlIvkNDsj2yuOkCaaeNT3vJ0umtc8aeNw,20046
-pypromice/postprocess/get_bufr.py,sha256=tM3LruCTYYxNG6IBKwf6mx76eCXEHJf5Y2fjRw6HGow,10855
-pypromice/postprocess/wmo_config.py,sha256=5ln5vaenG6dz8SppESzE8rrHXkJib-qM1z8SbCNh7n8,8413
-pypromice/process/L0toL1.py,sha256=lxL6HRqnrQhUKsEoteUaV4oD8YeMUvzRBOqi8oOd_kA,21384
+pypromice/postprocess/bufr_to_csv.py,sha256=RQZSJ3rC5v-R76rLQf60HNhgH7JQL3WhzXGgySHUyRg,317
+pypromice/postprocess/bufr_utilities.py,sha256=P9hAdYvW-G04cZalV5hm65S8JjZ4ppEKlHkdbzSthtI,17032
+pypromice/postprocess/get_bufr.py,sha256=H8n-LE9KbU70FT1vPdKvd__L2runQSeZpRDCXm9RlBs,20790
+pypromice/postprocess/positions_seed.csv,sha256=0kVCQ8UfEALdeXNYCddmwxpseRqLRudbFStqp_bZRBw,224
+pypromice/postprocess/real_time_utilities.py,sha256=hmJJ0t_dq6nU-VjFOgnWC_hAksECy6gOUw3jblgeekQ,8710
+pypromice/postprocess/station_configurations.toml,sha256=o_NOXTjf8KBNPSa2vQBtSZZPCymIjixTNniVvwUtltk,17025
+pypromice/process/L0toL1.py,sha256=eQw_cJjEEOj4zOuONp1SxTSDXPG5S0SIVlac7tuSlsc,22818
 pypromice/process/L1toL2.py,sha256=IYrZ9okxGzuJK3dfthk3C75RjsU9yNCgvELK7H41iQg,25538
 pypromice/process/L2toL3.py,sha256=LnpxGgk1auTHLhef4JUx6iWEhraKoXCRkVsyWI2oYR4,18238
 pypromice/process/__init__.py,sha256=xvd0I-9nIyVw4M4qjgkQ5vXYpNuKcVSkIVIROQsZDo0,147
@@ -14,10 +17,10 @@ pypromice/process/aws.py,sha256=fIQqOP3WrCBs17a4kMxXzVD-8OD3Mlagy3OxJ-ys_mw,2984
 pypromice/process/get_l3.py,sha256=jwb5nO22d9Pk5I3SrijkXbTwgqnmqVdxQn0_ACMabH0,1656
 pypromice/process/join_l3.py,sha256=aAu_7vCSVhigcU2siMOdkz7DL1iA9smImWKNQOYgS3M,4979
 pypromice/process/metadata.csv,sha256=Zp8Z40m23ohQfzUdLT4z43uHGFvx39rODXZHpUuYn6M,7055
-pypromice/process/value_clipping.py,sha256=KTSIENWQz8OcIeoZharXJO-lNrW3yCjmXcclmN-HjzE,1966
-pypromice/process/variables.csv,sha256=kXCBlDVjbBgxwow86Xnl-fY48Dh07ZUFasdIehINJzY,13571
+pypromice/process/value_clipping.py,sha256=FkBiDT_HK_BDFiVjB7NdWH-_nab7vONG9LOd2PpEBI8,1573
+pypromice/process/variables.csv,sha256=l6yqVNWlUntnTrXm6u1nrK8ZcrwpMmb_60Hhj-xfli0,13340
 pypromice/qc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypromice/qc/github_data_issues.py,sha256=GvqLEb5_bE_9K5706KwHqb9XzlEA-MLinpcJf495IKY,15646
+pypromice/qc/github_data_issues.py,sha256=DAu2F2HuPSYO_JICKI8-N8zGlbkVOsW6vr1qtMPfe5k,13537
 pypromice/qc/persistence.py,sha256=TuXxbedsekR1LSBdc0V9xypvlj8mkSE3xqI-U9xqcnY,4647
 pypromice/qc/persistence_test.py,sha256=DbltqVnnKmxV5T-UiF3AGVMXSVsbzqjYRqhVV1pAL9c,5771
 pypromice/qc/percentiles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -42,9 +45,9 @@ pypromice/tx/get_watsontx.py,sha256=vFSuDs_vvkATe_6WCF8OLVsx7Wa-MxLATZRfP9qUZqI,
 pypromice/tx/payload_formats.csv,sha256=4pClTt5qjdB4XViDY2QzuLeJY5EBkEBia0E7Y-aMLVo,7726
 pypromice/tx/payload_types.csv,sha256=C1-xCmHytAqqAzgzPwBLWqabzWu6s6tKAd8AjVd935s,457
 pypromice/tx/tx.py,sha256=s1ADVUwX0hoykFMSo_BDgopsIUWTJh2l7KJ05nMNmQg,33692
-pypromice-1.3.2.dist-info/LICENSE.txt,sha256=gXf5dRMhNSbfLPYYTY_5hsZ1r7UU1OaKQEAQUhuIBkM,18092
-pypromice-1.3.2.dist-info/METADATA,sha256=oEA0M7Yo-xcAAUw_AAx4qs4fK0OMG0I8IMsbDOrUb10,5073
-pypromice-1.3.2.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-pypromice-1.3.2.dist-info/entry_points.txt,sha256=XQW0vu3Jj7lkA0C2_wuOUU6NaWMlbTkZBxK2GN7oDGg,356
-pypromice-1.3.2.dist-info/top_level.txt,sha256=cBdfwgSbWDQq3a07nKRjrfmLC7jdaYXs98GG58HpTks,10
-pypromice-1.3.2.dist-info/RECORD,,
+pypromice-1.3.4.dist-info/LICENSE.txt,sha256=gXf5dRMhNSbfLPYYTY_5hsZ1r7UU1OaKQEAQUhuIBkM,18092
+pypromice-1.3.4.dist-info/METADATA,sha256=uCKUJ1oPZZCZqfy8X9W_QhfQIfg27pS8IMEOHHkdZnY,5096
+pypromice-1.3.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+pypromice-1.3.4.dist-info/entry_points.txt,sha256=8CGN4oJByZuvVFt9rL8HG41F11XL2L6aSe-EPjHTkkE,352
+pypromice-1.3.4.dist-info/top_level.txt,sha256=cBdfwgSbWDQq3a07nKRjrfmLC7jdaYXs98GG58HpTks,10
+pypromice-1.3.4.dist-info/RECORD,,

{pypromice-1.3.2.dist-info → pypromice-1.3.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.3)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{pypromice-1.3.2.dist-info → pypromice-1.3.4.dist-info}/entry_points.txt

@@ -1,5 +1,5 @@
 [console_scripts]
-get_bufr = pypromice.postprocess.get_bufr:get_bufr
+get_bufr = pypromice.postprocess.get_bufr:main
 get_l0tx = pypromice.tx.get_l0tx:get_l0tx
 get_l3 = pypromice.process.get_l3:get_l3
 get_msg = pypromice.tx.get_msg:get_msg