pypromice 1.4.2__tar.gz → 1.4.4__tar.gz

{pypromice-1.4.2/src/pypromice.egg-info → pypromice-1.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pypromice
-Version: 1.4.2
+Version: 1.4.4
 Summary: PROMICE/GC-Net data processing toolbox
 Home-page: https://github.com/GEUS-Glaciology-and-Climate/pypromice
 Author: GEUS Glaciology and Climate

{pypromice-1.4.2 → pypromice-1.4.4}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setuptools.setup(
     name="pypromice",
-    version="1.4.2",
+    version="1.4.4",
     author="GEUS Glaciology and Climate",
     description="PROMICE/GC-Net data processing toolbox",
     long_description=long_description,

{pypromice-1.4.2 → pypromice-1.4.4}/src/pypromice/postprocess/bufr_utilities.py

@@ -120,7 +120,8 @@ class BUFRVariables:
     )
     # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/012101
     # Scale: 2, unit: K
-    airTemperature: float = attrs.field(converter=round_converter(2))
+    # NOTE: The expected scale is 2, but our instantanous data is rounded to 1 decimal.
+    airTemperature: float = attrs.field(converter=round_converter(1))
     # There is also a Dewpoint temperature in this template: 012103 which is currently unused.
     # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/012103
     # Scale: 0, unit: %

{pypromice-1.4.2 → pypromice-1.4.4}/src/pypromice/process/L1toL2.py

@@ -94,7 +94,7 @@ def toL2(
                           .ffill().bfill())
     mask = (np.abs(ds.gps_alt - baseline_elevation) < 100) & ds.gps_alt.notnull()
     ds[['gps_alt','gps_lon', 'gps_lat']] = ds[['gps_alt','gps_lon', 'gps_lat']].where(mask)
-    
+
     # removing dlr and ulr that are missing t_rad
     # this is done now becasue t_rad can be filtered either manually or with persistence
     ds['dlr'] = ds.dlr.where(ds.t_rad.notnull())
@@ -113,12 +113,12 @@ def toL2(
         if ~ds['t_i'].isnull().all():
             ds['rh_i_cor'] = correctHumidity(ds['rh_i'], ds['t_i'],
                                              T_0, T_100, ews, ei0)
-    
+
     # Determiune cloud cover for on-ice stations
     cc = calcCloudCoverage(ds['t_u'], T_0, eps_overcast, eps_clear,        # Calculate cloud coverage
                            ds['dlr'], ds.attrs['station_id'])
     ds['cc'] = (('time'), cc.data)
-    
+
     # Determine surface temperature
     ds['t_surf'] = calcSurfaceTemperature(T_0, ds['ulr'], ds['dlr'],           # Calculate surface temperature
                                           emissivity)
@@ -136,7 +136,7 @@ def toL2(
     else:
         lat = ds['gps_lat'].mean()
         lon = ds['gps_lon'].mean()
-    
+
     # smoothing tilt and rot
     ds['tilt_x'] = smoothTilt(ds['tilt_x'])
     ds['tilt_y'] = smoothTilt(ds['tilt_y'])
@@ -151,8 +151,8 @@ def toL2(
     ZenithAngle_rad, ZenithAngle_deg = calcZenith(lat, Declination_rad,        # Calculate zenith
                                                   HourAngle_rad, deg2rad,
                                                   rad2deg)
-    
-    
+
+
     # Correct Downwelling shortwave radiation
     DifFrac = 0.2 + 0.8 * cc
     CorFac_all = calcCorrectionFactor(Declination_rad, phi_sensor_rad,         # Calculate correction
@@ -186,9 +186,9 @@ def toL2(
     TOA_crit_nopass = (ds['dsr_cor'] > (0.9 * isr_toa + 10))                   # Determine filter
     ds['dsr_cor'][TOA_crit_nopass] = np.nan                                    # Apply filter and interpolate
     ds['usr_cor'][TOA_crit_nopass] = np.nan
-    
-    ds['dsr_cor'] = ds.dsr_cor.where(ds.dsr.notnull())  
-    ds['usr_cor'] = ds.usr_cor.where(ds.usr.notnull())  
+
+    ds['dsr_cor'] = ds.dsr_cor.where(ds.dsr.notnull())
+    ds['usr_cor'] = ds.usr_cor.where(ds.usr.notnull())
     # # Check sun position
     # sundown = ZenithAngle_deg >= 90
     # _checkSunPos(ds, OKalbedos, sundown, sunonlowerdome, TOA_crit_nopass)
@@ -205,27 +205,33 @@ def toL2(
             ds['precip_l_cor'], ds['precip_l_rate']= correctPrecip(ds['precip_l'],
                                                                    ds['wspd_l'])
 
-    # Get directional wind speed 
-    ds['wdir_u'] = ds['wdir_u'].where(ds['wspd_u'] != 0)                   
-    ds['wspd_x_u'], ds['wspd_y_u'] = calcDirWindSpeeds(ds['wspd_u'], ds['wdir_u']) 
+    get_directional_wind_speed(ds)                                            # Get directional wind speed
+
+    ds = clip_values(ds, vars_df)
+    return ds
+
+def get_directional_wind_speed(ds: xr.Dataset) -> xr.Dataset:
+    """
+    Calculate directional wind speed from wind speed and direction and mutates the dataset
+    """
+
+    ds['wdir_u'] = ds['wdir_u'].where(ds['wspd_u'] != 0)
+    ds['wspd_x_u'], ds['wspd_y_u'] = calcDirWindSpeeds(ds['wspd_u'], ds['wdir_u'])
 
-    if ds.attrs['number_of_booms']==2:                                         
-        ds['wdir_l'] = ds['wdir_l'].where(ds['wspd_l'] != 0) 
+    if ds.attrs['number_of_booms']==2:
+        ds['wdir_l'] = ds['wdir_l'].where(ds['wspd_l'] != 0)
         ds['wspd_x_l'], ds['wspd_y_l'] = calcDirWindSpeeds(ds['wspd_l'], ds['wdir_l'])
 
     if hasattr(ds, 'wdir_i'):
         if ~ds['wdir_i'].isnull().all() and ~ds['wspd_i'].isnull().all():
-            ds['wdir_i'] = ds['wdir_i'].where(ds['wspd_i'] != 0)                   
-            ds['wspd_x_i'], ds['wspd_y_i'] = calcDirWindSpeeds(ds['wspd_i'], ds['wdir_i'])   
-    
-
-    ds = clip_values(ds, vars_df)
+            ds['wdir_i'] = ds['wdir_i'].where(ds['wspd_i'] != 0)
+            ds['wspd_x_i'], ds['wspd_y_i'] = calcDirWindSpeeds(ds['wspd_i'], ds['wdir_i'])
     return ds
 
 
 def calcDirWindSpeeds(wspd, wdir, deg2rad=np.pi/180):
     '''Calculate directional wind speed from wind speed and direction
-    
+
     Parameters
     ----------
     wspd : xr.Dataarray
@@ -234,16 +240,16 @@ def calcDirWindSpeeds(wspd, wdir, deg2rad=np.pi/180):
         Wind direction data array
     deg2rad : float
         Degree to radians coefficient. The default is np.pi/180
-    
+
     Returns
     -------
     wspd_x : xr.Dataarray
         Wind speed in X direction
     wspd_y : xr.Datarray
         Wind speed in Y direction
-    '''        
+    '''
     wspd_x = wspd * np.sin(wdir * deg2rad)
-    wspd_y = wspd * np.cos(wdir * deg2rad) 
+    wspd_y = wspd * np.cos(wdir * deg2rad)
     return wspd_x, wspd_y
 
 
@@ -328,7 +334,7 @@ def smoothTilt(da: xr.DataArray, threshold=0.2):
         either X or Y smoothed tilt inclinometer measurements
     '''
     # we calculate the moving standard deviation over a 3-day sliding window
-    # hourly resampling is necessary to make sure the same threshold can be used 
+    # hourly resampling is necessary to make sure the same threshold can be used
     # for 10 min and hourly data
     moving_std_gap_filled = da.to_series().resample('h').median().rolling(
                     3*24, center=True, min_periods=2

{pypromice-1.4.2 → pypromice-1.4.4}/src/pypromice/process/value_clipping.py

@@ -2,6 +2,8 @@ import numpy as np
 import pandas
 import xarray
 
+from pypromice.utilities.dependency_graph import DependencyGraph
+
 
 def clip_values(
     ds: xarray.Dataset,
@@ -27,9 +29,15 @@ def clip_values(
     cols = ["lo", "hi", "OOL"]
     assert set(cols) <= set(var_configurations.columns)
 
-    variable_limits = var_configurations[cols].dropna(how="all")
-    for var, row in variable_limits.iterrows():
+    variable_limits = var_configurations[cols].assign(
+        dependents=lambda df: df.OOL.fillna("").str.split(),
+        # Find the closure of dependents using the DependencyGraph class
+        dependents_closure=lambda df: DependencyGraph.from_child_mapping(
+            df.dependents
+        ).child_closure_mapping(),
+    )
 
+    for var, row in variable_limits.iterrows():
         if var not in list(ds.variables):
             continue
 
@@ -38,15 +46,10 @@ def clip_values(
         if ~np.isnan(row.hi):
             ds[var] = ds[var].where(ds[var] <= row.hi)
 
-        other_vars = row.OOL
-        if isinstance(other_vars, str) and ~ds[var].isnull().all():
-            for o in other_vars.split():
-                if o not in list(ds.variables):
-                    continue
-                else:
-                    if ~np.isnan(row.lo):
-                        ds[var] = ds[var].where(ds[var] >= row.lo)
-                    if ~np.isnan(row.hi):
-                        ds[var] = ds[var].where(ds[var] <= row.hi)
+        # Flag dependents as NaN if parent is NaN
+        for o in row.dependents_closure:
+            if o not in list(ds.variables):
+                continue
+            ds[o] = ds[o].where(ds[var].notnull())
 
     return ds

{pypromice-1.4.2 → pypromice-1.4.4}/src/pypromice/resources/variables.csv

@@ -1,53 +1,53 @@
 field,standard_name,long_name,units,coverage_content_type,coordinates,instantaneous_hourly,where_to_find,lo,hi,OOL,station_type,L0,L2,L3,max_decimals
 time,time,Time,yyyy-mm-dd HH:MM:SS,physicalMeasurement,time,,,,,,all,1,1,1,
 rec,record,Record,-,referenceInformation,time,,L0 or L2,,,,all,1,1,0,0
-p_u,air_pressure,Air pressure (upper boom),hPa,physicalMeasurement,time,FALSE,,650,1100,z_pt z_pt_cor dshf_u dlhf_u qh_u,all,1,1,1,4
-p_l,air_pressure,Air pressure (lower boom),hPa,physicalMeasurement,time,FALSE,,650,1100,dshf_l dlhf_l qh_l,two-boom,1,1,1,4
-t_u,air_temperature,Air temperature (upper boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,rh_u_cor cc dsr_cor usr_cor z_boom z_stake dshf_u dlhf_u qh_u,all,1,1,1,4
-t_l,air_temperature,Air temperature (lower boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,rh_l_cor z_boom_l dshf_l dlhf_l qh_l,two-boom,1,1,1,4
-rh_u,relative_humidity,Relative humidity (upper boom),%,physicalMeasurement,time,FALSE,,0,100,rh_u_cor,all,1,1,1,4
-rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,modelResult,time,FALSE,L2 or later,0,150,dshf_u dlhf_u qh_u,all,0,1,1,4
-qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,modelResult,time,FALSE,L2 or later,0,100,,all,0,1,1,4
-rh_l,relative_humidity,Relative humidity (lower boom),%,physicalMeasurement,time,FALSE,,0,100,rh_l_cor,two-boom,1,1,1,4
-rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,modelResult,time,FALSE,L2 or later,0,150,dshf_l dlhf_l qh_l,two-boom,0,1,1,4
+p_u,air_pressure,Air pressure (upper boom),hPa,physicalMeasurement,time,FALSE,,650,1100,"",all,1,1,1,4
+p_l,air_pressure,Air pressure (lower boom),hPa,physicalMeasurement,time,FALSE,,650,1100,"",two-boom,1,1,1,4
+t_u,air_temperature,Air temperature (upper boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,"",all,1,1,1,4
+t_l,air_temperature,Air temperature (lower boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,"",two-boom,1,1,1,4
+rh_u,relative_humidity,Relative humidity (upper boom),%,physicalMeasurement,time,FALSE,,0,100,"",all,1,1,1,4
+rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,modelResult,time,FALSE,L2 or later,0,150,"",all,0,1,1,4
+qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,modelResult,time,FALSE,L2 or later,0,100,"",all,0,1,1,4
+rh_l,relative_humidity,Relative humidity (lower boom),%,physicalMeasurement,time,FALSE,,0,100,"",two-boom,1,1,1,4
+rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,modelResult,time,FALSE,L2 or later,0,150,"",two-boom,0,1,1,4
 qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,modelResult,time,FALSE,L2 or later,0,100,,two-boom,0,1,1,4
-wspd_u,wind_speed,Wind speed (upper boom),m s-1,physicalMeasurement,time,FALSE,,0,100,"wdir_u wspd_x_u wspd_y_u dshf_u dlhf_u qh_u, precip_u",all,1,1,1,4
-wspd_l,wind_speed,Wind speed (lower boom),m s-1,physicalMeasurement,time,FALSE,,0,100,"wdir_l wspd_x_l wspd_y_l dshf_l dlhf_l qh_l , precip_l",two-boom,1,1,1,4
+wspd_u,wind_speed,Wind speed (upper boom),m s-1,physicalMeasurement,time,FALSE,,0,100,wdir_u wspd_x_u wspd_y_u,all,1,1,1,4
+wspd_l,wind_speed,Wind speed (lower boom),m s-1,physicalMeasurement,time,FALSE,,0,100,wdir_l wspd_x_l wspd_y_l,two-boom,1,1,1,4
 wdir_u,wind_from_direction,Wind from direction (upper boom),degrees,physicalMeasurement,time,FALSE,,1,360,wspd_x_u wspd_y_u,all,1,1,1,4
 wdir_std_u,wind_from_direction_standard_deviation,Wind from direction (standard deviation),degrees,qualityInformation,time,FALSE,L0 or L2,,,,one-boom,1,1,0,4
 wdir_l,wind_from_direction,Wind from direction (lower boom),degrees,physicalMeasurement,time,FALSE,,1,360,wspd_x_l wspd_y_l,two-boom,1,1,1,4
-wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_u wspd_u,all,0,1,1,4
-wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_u wspd_u,all,0,1,1,4
-wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_l wspd_l,two-boom,0,1,1,4
-wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_l wspd_l,two-boom,0,1,1,4
-dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,physicalMeasurement,time,FALSE,,-10,1500,albedo dsr_cor usr_cor,all,1,1,1,4
+wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,"",all,0,1,1,4
+wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,"",all,0,1,1,4
+wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,"",two-boom,0,1,1,4
+wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,"",two-boom,0,1,1,4
+dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,physicalMeasurement,time,FALSE,,-10,1500,"",all,1,1,1,4
 dsr_cor,surface_downwelling_shortwave_flux_corrected,Downwelling shortwave radiation - corrected,W m-2,modelResult,time,FALSE,L2 or later,,,,all,0,1,1,4
-usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,physicalMeasurement,time,FALSE,,-10,1000,albedo dsr_cor usr_cor,all,1,1,1,4
+usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,physicalMeasurement,time,FALSE,,-10,1000,"",all,1,1,1,4
 usr_cor,surface_upwelling_shortwave_flux_corrected,Upwelling shortwave radiation - corrected,W m-2,modelResult,time,FALSE,L2 or later,0,1000,,all,0,1,1,4
 albedo,surface_albedo,Albedo,-,modelResult,time,FALSE,L2 or later,,,,all,0,1,1,4
-dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,physicalMeasurement,time,FALSE,,50,500,albedo dsr_cor usr_cor cc t_surf,all,1,1,1,4
-ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,physicalMeasurement,time,FALSE,,50,500,t_surf,all,1,1,1,4
+dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,physicalMeasurement,time,FALSE,,50,500,"",all,1,1,1,4
+ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,physicalMeasurement,time,FALSE,,50,500,"",all,1,1,1,4
 cc,cloud_area_fraction,Cloud cover,%,modelResult,time,FALSE,L2 or later,,,,all,0,1,1,4
-t_surf,surface_temperature,Surface temperature,C,modelResult,time,FALSE,L2 or later,-80,40,dshf_u dlhf_u qh_u,all,0,1,1,4
+t_surf,surface_temperature,Surface temperature,C,modelResult,time,FALSE,L2 or later,-80,40,"",all,0,1,1,4
 dlhf_u,surface_downward_latent_heat_flux,Latent heat flux (upper boom),W m-2,modelResult,time,FALSE,L3 or later,,,,all,0,0,1,4
 dlhf_l,surface_downward_latent_heat_flux,Latent heat flux (lower boom),W m-2,modelResult,time,FALSE,L3 or later,,,,two-boom,0,0,1,4
 dshf_u,surface_downward_sensible_heat_flux,Sensible heat flux (upper boom),W m-2,modelResult,time,FALSE,L3 or later,,,,all,0,0,1,4
 dshf_l,surface_downward_sensible_heat_flux,Sensible heat flux (lower boom),W m-2,modelResult,time,FALSE,L3 or later,,,,two-boom,0,0,1,4
-z_boom_u,distance_to_surface_from_boom,Upper boom height,m,physicalMeasurement,time,TRUE,,0.3,10,dshf_u dlhf_u qh_u,all,1,1,1,4
+z_boom_u,distance_to_surface_from_boom,Upper boom height,m,physicalMeasurement,time,TRUE,,0.3,10,"",all,1,1,1,4
 z_boom_q_u,distance_to_surface_from_boom_quality,Upper boom height (quality),-,qualityInformation,time,TRUE,L0 or L2,,,,all,1,1,0,4
-z_boom_l,distance_to_surface_from_boom,Lower boom height,m,physicalMeasurement,time,TRUE,,0.3,5,dshf_l dlhf_l qh_l,two-boom,1,1,1,4
+z_boom_l,distance_to_surface_from_boom,Lower boom height,m,physicalMeasurement,time,TRUE,,0.3,5,"",two-boom,1,1,1,4
 z_boom_q_l,distance_to_surface_from_boom_quality,Lower boom height (quality),-,qualityInformation,time,TRUE,L0 or L2,,,,two-boom,1,1,0,4
 z_stake,distance_to_surface_from_stake_assembly,Stake height,m,physicalMeasurement,time,TRUE,,0.3,8,,one-boom,1,1,1,4
 z_stake_q,distance_to_surface_from_stake_assembly_quality,Stake height (quality),-,qualityInformation,time,TRUE,L0 or L2,,,,one-boom,1,1,0,4
-z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,physicalMeasurement,time,FALSE,,0,30,z_pt_cor,one-boom,1,1,1,4
+z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,physicalMeasurement,time,FALSE,,0,30,"",one-boom,1,1,1,4
 z_pt_cor,depth_of_pressure_transducer_in_ice_corrected,Depth of pressure transducer in ice - corrected,m,modelResult,time,FALSE,L2 or later,0,30,,one-boom,0,1,1,4
 z_surf_combined,height_of_surface_combined,"Surface height combined from multiple sensors, relative to ice surface height at installation",m,modelResult,time,FALSE,L3,,,,all,0,0,1,4
 z_ice_surf,height_of_ice_surface,"Ice surface height, relative to ice surface height at installation and calculated from pt_cor and z_stake",m,modelResult,time,FALSE,L3,,,,one-boom,0,0,1,4
 snow_height,height_of_snow,"Snow surface height, relative to ice surface",m,modelResult,time,FALSE,L3,0,,,one-boom,0,0,1,4
-precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,physicalMeasurement,time,TRUE,,0,,precip_u_cor precip_u_rate,all,1,1,1,4
+precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,physicalMeasurement,time,TRUE,,0,,"",all,1,1,1,4
 precip_u_cor,precipitation_corrected,Precipitation (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,all,0,1,1,4
 precip_u_rate,precipitation_rate,Precipitation rate (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,all,0,1,1,4
-precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,physicalMeasurement,time,TRUE,,0,,precip_l_cor precip_l_rate,two-boom,1,1,1,4
+precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,physicalMeasurement,time,TRUE,,0,,"",two-boom,1,1,1,4
 precip_l_cor,precipitation_corrected,Precipitation (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,two-boom,0,1,1,4
 precip_l_rate,precipitation_rate,Precipitation rate (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,two-boom,0,1,1,4
 t_i_1,ice_temperature_at_t1,Ice temperature at sensor 1,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,all,1,1,1,4
@@ -73,8 +73,8 @@ d_t_i_9,depth_of_thermistor_9,Depth of thermistor 9,m,modelResult,time,FALSE,L3,
 d_t_i_10,depth_of_thermistor_10,Depth of thermistor 10,m,modelResult,time,FALSE,L3,-10,100,,two-boom,0,0,1,4
 d_t_i_11,depth_of_thermistor_11,Depth of thermistor 11,m,modelResult,time,FALSE,L3,-10,100,,two-boom,0,0,1,4
 t_i_10m,10m_subsurface_temperature,10 m subsurface temperature,degrees_C,modelResult,time,FALSE,L3,-70,0,,all,0,0,1,4
-tilt_x,platform_view_angle_x,Tilt to east,degrees,physicalMeasurement,time,FALSE,,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4
-tilt_y,platform_view_angle_y,Tilt to north,degrees,physicalMeasurement,time,FALSE,,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4
+tilt_x,platform_view_angle_x,Tilt to east,degrees,physicalMeasurement,time,FALSE,,-30,30,"",all,1,1,1,4
+tilt_y,platform_view_angle_y,Tilt to north,degrees,physicalMeasurement,time,FALSE,,-30,30,"",all,1,1,1,4
 rot,platform_azimuth_angle,Station rotation from true North,degrees,physicalMeasurement,time,FALSE,,0,360,,all,1,1,1,2
 gps_lat,gps_latitude,Latitude,degrees_north,physicalMeasurement,time,TRUE,,50,83,,all,1,1,1,6
 gps_lon,gps_longitude,Longitude,degrees_east,physicalMeasurement,time,TRUE,,5,70,,all,1,1,1,6
@@ -93,14 +93,14 @@ batt_v_ini,,,-,physicalMeasurement,time,TRUE,L0 or L2,0,30,,,1,1,0,2
 batt_v_ss,battery_voltage_at_sample_start,Battery voltage (sample start),V,physicalMeasurement,time,TRUE,L0 or L2,0,30,,,1,1,0,2
 fan_dc_u,fan_current,Fan current (upper boom),mA,physicalMeasurement,time,TRUE,L0 or L2,0,200,,all,1,1,0,2
 fan_dc_l,fan_current,Fan current (lower boom),mA,physicalMeasurement,time,TRUE,,0,200,,two-boom,1,1,0,2
-freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,physicalMeasurement,time,TRUE,L0 or L2,0,10000,precip_u,,1,1,0,
+freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,physicalMeasurement,time,TRUE,L0 or L2,0,10000,"",,1,1,0,
 t_log,temperature_of_logger,Logger temperature,degrees_C,physicalMeasurement,time,TRUE,,-80,40,,one-boom,1,1,0,4
-t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,physicalMeasurement,time,FALSE,,-80,40,t_surf dlr ulr,all,1,1,1,4
+t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,physicalMeasurement,time,FALSE,,-80,40,"",all,1,1,1,4
 p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,physicalMeasurement,time,TRUE,,-350,100,,all,1,1,1,4
 t_i,air_temperature,Air temperature (instantaneous),degrees_C,physicalMeasurement,time,TRUE,,-80,40,,all,1,1,1,4
-rh_i,relative_humidity,Relative humidity (instantaneous),%,physicalMeasurement,time,TRUE,,0,150,rh_i_cor,all,1,1,1,4
+rh_i,relative_humidity,Relative humidity (instantaneous),%,physicalMeasurement,time,TRUE,,0,150,"",all,1,1,1,4
 rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,modelResult,time,TRUE,L2 or later,0,100,,all,0,1,1,4
 wspd_i,wind_speed,Wind speed (instantaneous),m s-1,physicalMeasurement,time,TRUE,,0,100,wdir_i wspd_x_i wspd_y_i,all,1,1,1,4
 wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,physicalMeasurement,time,TRUE,,1,360,wspd_x_i wspd_y_i,all,1,1,1,4
-wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,modelResult,time,TRUE,L2 or later,-100,100,wdir_i wspd_i,all,0,1,1,4
-wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,modelResult,time,TRUE,L2 or later,-100,100,wdir_i wspd_i,all,0,1,1,4
+wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,modelResult,time,TRUE,L2 or later,-100,100,"",all,0,1,1,4
+wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,modelResult,time,TRUE,L2 or later,-100,100,"",all,0,1,1,4

{pypromice-1.4.2 → pypromice-1.4.4}/src/pypromice/tx/tx.py

@@ -894,10 +894,10 @@ def sortLines(in_file, out_file, replace_unsorted=True):                       #
             # out_f.write(headers)
             out_f.writelines(unique_lines)
     
-    # Replace input file with new sorted file
-    if replace_unsorted:
-        os.remove(in_file)
-        os.rename(out_file, in_file)
+        # Replace input file with new sorted file
+        if replace_unsorted:
+            os.remove(in_file)
+            os.rename(out_file, in_file)
 
 def addTail(in_file, out_dir, aws_name, header_names='', lines_limit=100):    
     '''Generate tails file from L0tx file

pypromice-1.4.4/src/pypromice/utilities/dependency_graph.py

@@ -0,0 +1,101 @@
+from typing import Mapping, Set, MutableMapping, Optional
+
+import attr
+
+__all__ = [
+    "DependencyNode",
+    "DependencyGraph",
+]
+
+
+@attr.define
+class DependencyNode:
+    name: str = attr.field()
+    parents: Set["DependencyNode"] = attr.field(factory=set)
+    children: Set["DependencyNode"] = attr.field(factory=set)
+
+    def add_child(self, child: "DependencyNode"):
+        self.children.add(child)
+        child.parents.add(self)
+
+    def get_children_closure(self, closure: Optional[Set[str]] = None) -> Set[str]:
+        is_root = closure is None
+        if closure is None:
+            closure = set()
+        if self.name in closure:
+            return closure
+        closure.add(self.name)
+        for child in self.children:
+            closure |= child.get_children_closure(closure)
+
+        if is_root:
+            closure.remove(self.name)
+        return closure
+
+    def get_parents_closure(self, closure: Optional[Set[str]] = None) -> Set[str]:
+        is_root = closure is None
+        if closure is None:
+            closure = set()
+        if self.name in closure:
+            return closure
+        closure.add(self.name)
+        for parent in self.parents:
+            closure |= parent.get_parents_closure(closure)
+
+        if is_root:
+            closure.remove(self.name)
+        return closure
+
+    def __hash__(self):
+        return hash(self.name)
+
+
+@attr.define
+class DependencyGraph:
+    nodes: MutableMapping[str, DependencyNode] = attr.field(factory=dict)
+
+    def add_node(self, name: str) -> DependencyNode:
+        if name not in self.nodes:
+            self.nodes[name] = DependencyNode(name)
+        return self.nodes[name]
+
+    def add_edge(self, parent: str, child: str):
+        parent_node = self.add_node(parent)
+        child_node = self.add_node(child)
+        parent_node.add_child(child_node)
+
+    @classmethod
+    def from_child_mapping(cls, mapping: Mapping[str, Set[str]]) -> "DependencyGraph":
+        graph = cls()
+        for var, children in mapping.items():
+            graph.add_node(var)
+            for child in children:
+                graph.add_edge(var, child)
+        return graph
+
+    @classmethod
+    def from_parent_mapping(cls, mapping: Mapping[str, Set[str]]) -> "DependencyGraph":
+        graph = cls()
+        for var, parents in mapping.items():
+            graph.add_node(var)
+            for parent in parents:
+                graph.add_edge(parent, var)
+        return graph
+
+    def child_mapping(self) -> Mapping[str, Set[str]]:
+        return {
+            node.name: {child.name for child in node.children}
+            for node in self.nodes.values()
+        }
+
+    def child_closure_mapping(self) -> Mapping[str, Set[str]]:
+        return {node.name: node.get_children_closure() for node in self.nodes.values()}
+
+    def parent_mapping(self) -> Mapping[str, Set[str]]:
+        return {
+            node.name: {parent.name for parent in node.parents}
+            for node in self.nodes.values()
+        }
+
+    def parent_closure_mapping(self) -> Mapping[str, Set[str]]:
+        return {node.name: node.get_parents_closure() for node in self.nodes.values()}

{pypromice-1.4.2 → pypromice-1.4.4/src/pypromice.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pypromice
-Version: 1.4.2
+Version: 1.4.4
 Summary: PROMICE/GC-Net data processing toolbox
 Home-page: https://github.com/GEUS-Glaciology-and-Climate/pypromice
 Author: GEUS Glaciology and Climate

{pypromice-1.4.2 → pypromice-1.4.4}/src/pypromice.egg-info/SOURCES.txt

@@ -49,9 +49,9 @@ src/pypromice/resources/variables.csv
 src/pypromice/tx/__init__.py
 src/pypromice/tx/get_l0tx.py
 src/pypromice/tx/get_msg.py
-src/pypromice/tx/get_watsontx.py
 src/pypromice/tx/payload_formats.csv
 src/pypromice/tx/payload_types.csv
 src/pypromice/tx/tx.py
 src/pypromice/utilities/__init__.py
+src/pypromice/utilities/dependency_graph.py
 src/pypromice/utilities/git.py

pypromice-1.4.2/src/pypromice/tx/get_watsontx.py

@@ -1,147 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Jul 22 16:20:09 2022
-
-Script to get L0tx transmission messages from the Watson River station using 
-the tx module
-
-@author: Penelope How, pho@geus.dk
-"""
-from argparse import ArgumentParser
-
-from configparser import ConfigParser
-import os, imaplib, email, re
-from glob import glob
-from datetime import datetime
-
-from pypromice.tx import getMail, L0tx, sortLines
-
-
-def parse_arguments_watson():
-    parser = ArgumentParser(description="AWS L0 transmission fetcher for Watson River measurements")       
-    parser.add_argument('-a', '--account', default=None, type=str, required=True, help='Email account .ini file')
-    parser.add_argument('-p', '--password', default=None, type=str, required=True, help='Email credentials .ini file')                      
-    parser.add_argument('-o', '--outpath', default=None, type=str, required=False, help='Path where to write output (if given)')           
-    parser.add_argument('-f', '--formats', default=None, type=str, required=False, help='Path to Payload format .csv file')
-    parser.add_argument('-t', '--types', default=None, type=str, required=False, help='Path to Payload type .csv file')  	
-    parser.add_argument('-u', '--uid', default=None, type=str, required=True, help='Last AWS uid .ini file')	        
-    args = parser.parse_args()
-    return args
-
-#------------------------------------------------------------------------------
-def get_watsontx():
-    """Executed from the command line"""
-    args = parse_arguments_watson()
-    
-     # Set payload formatter paths
-    formatter_file = args.formats
-    type_file = args.types
-
- 	# Set credential paths
-    accounts_file = args.account
-    credentials_file = args.password 
-
- 	# Set last aws uid path
- 	# last_uid = 1000000
-    uid_file = args.uid
-    
-    # Set last aws uid path
-    with open(uid_file, 'r') as last_uid_f:
-        last_uid = int(last_uid_f.readline())
-    
-    # Set output file directory
-    out_dir = args.outpath
-    if not os.path.exists(out_dir):
-        os.mkdir(out_dir)
-    
-#------------------------------------------------------------------------------
-    
-    # Define accounts and credentials ini file paths
-    accounts_ini = ConfigParser()
-    accounts_ini.read_file(open(accounts_file))
-    accounts_ini.read(credentials_file)
-                 
-    # Get credentials
-    account = accounts_ini.get('aws', 'account')
-    server = accounts_ini.get('aws', 'server')
-    port = accounts_ini.getint('aws', 'port')    
-    password = accounts_ini.get('aws', 'password')
-    if not password:
-        password = input('password for AWS email account: ')
-    print('AWS data from server %s, account %s' %(server, account))
-    
-#------------------------------------------------------------------------------
-    
-    # Log in to email server
-    mail_server = imaplib.IMAP4_SSL(server, port)
-    typ, accountDetails = mail_server.login(account, password)
-    if typ != 'OK':
-        print('Not able to sign in!')
-        raise
-        
-    # Grab new emails
-    result, data = mail_server.select(mailbox='"[Gmail]/All Mail"', 
-                                      readonly=True)
-    print('mailbox contains %s messages' %data[0])
-    
-#------------------------------------------------------------------------------
-    
-    # Get L0tx datalines from email transmissions
-    for uid, mail in getMail(mail_server, last_uid=last_uid):
-        message = email.message_from_string(mail)
-        try:
-            name = str(message.get_all('subject')[0])
-            d = datetime.strptime(message.get_all('date')[0], 
-                                  '%a, %d %b %Y %H:%M:%S %z')
-        except:
-            name=None
-            d=None
-        
-        if name and ('Watson' in name or 'GIOS' in name):
-            print(f'Watson/GIOS station message, {d.strftime("%Y-%m-%d %H:%M:%S")}')
-
-            l0 = L0tx(message, formatter_file, type_file, 
-                      sender_name=['emailrelay@konectgds.com','sbdservice'])
-            
-            if l0.msg: 
-                content, attachment = l0.getEmailBody()
-                attachment_name = str(attachment.get_filename())               
-                out_fn = re.sub(r'\d*\.dat$', '', attachment_name) + '.txt'
-                out_path = os.sep.join((out_dir, out_fn))
-        
-                print(f'Writing to {out_fn}')
-                print(l0.msg)
-            
-                with open(out_path, mode='a') as out_f:
-                    out_f.write(l0.msg + '\n')    
-    
-#------------------------------------------------------------------------------
-
-    # Sort L0tx files and add tails    
-    for f in glob(out_dir+'/*.txt'):
-        
-        # Sort lines in L0tx file and remove duplicates
-        in_dirn, in_fn = os.path.split(f)    
-        out_fn = 'sorted_' + in_fn
-        out_pn = os.sep.join((in_dirn, out_fn))
-        sortLines(f, out_pn)
-        
-    # Close mail server if open
-    if 'mail_server' in locals():
-        print(f'\nClosing {account}')
-        mail_server.close()
-        resp = mail_server.logout()
-        assert resp[0].upper() == 'BYE'
-    
-    # Write last aws uid to ini file
-    try:
-        with open(uid_file, 'w') as last_uid_f:
-            last_uid_f.write(uid)
-    except:
-        print(f'Could not write last uid {uid} to {uid_file}')
-         
-    print('Finished')
-        
-if __name__ == "__main__":  
-    get_watsontx()