pypromice 1.4.3__py3-none-any.whl → 1.5.0__py3-none-any.whl

pypromice/process/aws.py

@@ -94,7 +94,7 @@ class AWS(object):
         formats = {dataset.attrs["format"].lower() for dataset in self.L0}
         if "raw" in formats:
             self.format = "raw"
-        elif "STM" in formats:
+        elif "stm" in formats:
             self.format = "STM"
         elif "tx" in formats:
             self.format = "tx"
@@ -123,22 +123,6 @@ class AWS(object):
         self.getL2()
         self.getL3()
 
-    def writeL2(self, outpath):
-        """Write L2 data to .csv and .nc file"""
-        if os.path.isdir(outpath):
-            self.writeArr(self.L2, outpath)
-        else:
-            logger.info(f"Outpath f{outpath} does not exist. Unable to save to file")
-            pass
-
-    def writeL3(self, outpath):
-        """Write L3 data to .csv and .nc file"""
-        if os.path.isdir(outpath):
-            self.writeArr(self.L3, outpath)
-        else:
-            logger.info(f"Outpath f{outpath} does not exist. Unable to save to file")
-            pass
-
     def getL1(self):
         """Perform L0 to L1 data processing"""
         logger.info("Level 1 processing...")
@@ -164,28 +148,6 @@ class AWS(object):
         logger.info("Level 3 processing...")
         self.L3 = toL3(self.L2, data_adjustments_dir=self.data_issues_repository / "adjustments")
 
-    def writeArr(self, dataset, outpath, t=None):
-        """Write L3 data to .nc and .csv hourly and daily files
-
-        Parameters
-        ----------
-        dataset : xarray.Dataset
-            Dataset to write to file
-        outpath : str
-            Output directory
-        t : str
-            Resampling string. This is automatically defined based
-            on the data type if not given. The default is None.
-        """
-        if t is not None:
-            write.prepare_and_write(dataset, outpath, self.vars, self.meta, t)
-        else:
-            f = [l.attrs["format"] for l in self.L0]
-            if "raw" in f or "STM" in f:
-                write.prepare_and_write(dataset, outpath, self.vars, self.meta, "10min")
-            else:
-                write.prepare_and_write(dataset, outpath, self.vars, self.meta, "60min")
-
     def loadConfig(self, config_file, inpath):
         """Load configuration from .toml file
 

pypromice/process/join_l3.py

@@ -538,9 +538,9 @@ def join_l3(config_folder, site, folder_l3, folder_gcnet, outpath, variables, me
     v = pypromice.resources.load_variables(variables)
     m = pypromice.resources.load_metadata(metadata)
     if outpath is not None:
-        prepare_and_write(l3_merged, outpath, v, m, "60min")
-        prepare_and_write(l3_merged, outpath, v, m, "1D")
-        prepare_and_write(l3_merged, outpath, v, m, "M")
+        prepare_and_write(l3_merged, outpath, v, m, "60min", nc_compression=True)
+        prepare_and_write(l3_merged, outpath, v, m, "1D", nc_compression=True)
+        prepare_and_write(l3_merged, outpath, v, m, "M", nc_compression=True)
     return l3_merged, sorted_list_station_data
 
 

pypromice/process/resample.py

@@ -51,12 +51,13 @@ def resample_dataset(ds_h, t):
     # taking the 10 min data and using it as instantaneous values:
     is_10_minutes_timestamp = (ds_h.time.diff(dim='time') / np.timedelta64(1, 's') == 600)
     if (t == '60min') and is_10_minutes_timestamp.any():
-        cols_to_update = ['p_i', 't_i', 'rh_i', 'rh_i_cor', 'wspd_i', 'wdir_i','wspd_x_i','wspd_y_i']
+        cols_to_update = ['p_i', 't_i', 'rh_i', 'rh_i_wrt_ice_or_water', 'wspd_i', 'wdir_i','wspd_x_i','wspd_y_i']
+        cols_origin = ['p_u', 't_u', 'rh_u', 'rh_u_wrt_ice_or_water', 'wspd_u', 'wdir_u','wspd_x_u','wspd_y_u']
         timestamp_10min = ds_h.time.where(is_10_minutes_timestamp, drop=True).to_index()
         timestamp_round_hour = df_d.index
         timestamp_to_update = timestamp_round_hour.intersection(timestamp_10min)
         
-        for col in cols_to_update:
+        for col, col_org in zip(cols_to_update, cols_origin):
             if col not in df_d.columns:
                 df_d[col] = np.nan
             else:
@@ -67,7 +68,7 @@ def resample_dataset(ds_h, t):
                 timestamp_to_update = timestamp_to_update[missing_instantaneous]
             df_d.loc[timestamp_to_update, col] = ds_h.reindex(
                 time= timestamp_to_update
-                )[col.replace('_i','_u')].values
+                )[col_org].values
             if col == 'p_i':
                 df_d.loc[timestamp_to_update, col] = df_d.loc[timestamp_to_update, col].values-1000
             
@@ -95,8 +96,8 @@ def resample_dataset(ds_h, t):
                 
                 df_d[var] = (p_vap.to_series().resample(t).mean() \
                            / es_wtr.to_series().resample(t).mean())*100
-                if var+'_cor' in df_d.keys():
-                    df_d[var+'_cor'] = (p_vap.to_series().resample(t).mean() \
+                if var+'_wrt_ice_or_water' in df_d.keys():
+                    df_d[var+'_wrt_ice_or_water'] = (p_vap.to_series().resample(t).mean() \
                                / es_cor.to_series().resample(t).mean())*100
     
     # passing each variable attribute to the ressample dataset

pypromice/process/value_clipping.py

@@ -2,6 +2,8 @@ import numpy as np
 import pandas
 import xarray
 
+from pypromice.utilities.dependency_graph import DependencyGraph
+
 
 def clip_values(
     ds: xarray.Dataset,
@@ -9,8 +11,6 @@ def clip_values(
 ):
     """
     Clip values in dataset to defined "hi" and "lo" variables from dataframe.
-    There is a special treatment here for rh_u and rh_l variables, where values
-    are clipped and not assigned to NaN. This is for replication purposes
 
     Parameters
     ----------
@@ -27,9 +27,15 @@ def clip_values(
     cols = ["lo", "hi", "OOL"]
     assert set(cols) <= set(var_configurations.columns)
 
-    variable_limits = var_configurations[cols].dropna(how="all")
-    for var, row in variable_limits.iterrows():
+    variable_limits = var_configurations[cols].assign(
+        dependents=lambda df: df.OOL.fillna("").str.split(),
+        # Find the closure of dependents using the DependencyGraph class
+        dependents_closure=lambda df: DependencyGraph.from_child_mapping(
+            df.dependents
+        ).child_closure_mapping(),
+    )
 
+    for var, row in variable_limits.iterrows():
         if var not in list(ds.variables):
             continue
 
@@ -38,15 +44,10 @@ def clip_values(
         if ~np.isnan(row.hi):
             ds[var] = ds[var].where(ds[var] <= row.hi)
 
-        other_vars = row.OOL
-        if isinstance(other_vars, str) and ~ds[var].isnull().all():
-            for o in other_vars.split():
-                if o not in list(ds.variables):
-                    continue
-                else:
-                    if ~np.isnan(row.lo):
-                        ds[var] = ds[var].where(ds[var] >= row.lo)
-                    if ~np.isnan(row.hi):
-                        ds[var] = ds[var].where(ds[var] <= row.hi)
+        # Flag dependents as NaN if parent is NaN
+        for o in row.dependents_closure:
+            if o not in list(ds.variables):
+                continue
+            ds[o] = ds[o].where(ds[var].notnull())
 
     return ds

pypromice/process/write.py

@@ -17,7 +17,13 @@ logger = logging.getLogger(__name__)
 
 
 def prepare_and_write(
-    dataset, output_path: Path | str, vars_df=None, meta_dict=None, time="60min", resample=True
+    dataset,
+        output_path: Path | str,
+        vars_df=None,
+        meta_dict=None,
+        time="60min",
+        resample=True,
+        nc_compression:bool=False,
 ):
     """Prepare data with resampling, formating and metadata population; then
     write data to .nc and .csv hourly and daily files
@@ -100,14 +106,14 @@ def prepare_and_write(
     elif t == 86400:
         # removing instantaneous values from daily and monthly files
         for v in col_names:
-            if ("_i" in v) and ("_i_" not in v):
+            if v in ['p_i', 't_i', 'rh_i', 'wspd_i', 'wdir_i', 'wspd_x_i', 'wspd_y_i']:
                 col_names.remove(v)
         out_csv = output_dir / f"{name}_day.csv"
         out_nc = output_dir / f"{name}_day.nc"
     else:
         # removing instantaneous values from daily and monthly files
         for v in col_names:
-            if ("_i" in v) and ("_i_" not in v):
+            if v in ['p_i', 't_i', 'rh_i', 'wspd_i', 'wdir_i', 'wspd_x_i', 'wspd_y_i']:
                 col_names.remove(v)
         out_csv = output_dir / f"{name}_month.csv"
         out_nc = output_dir / f"{name}_month.nc"
@@ -117,40 +123,11 @@ def prepare_and_write(
     writeCSV(out_csv, d2, col_names)
 
     # Write to netcdf file
-    writeNC(out_nc, d2, col_names)
+    writeNC(out_nc, d2, col_names, compression=nc_compression)
     logger.info(f"Written to {out_csv}")
     logger.info(f"Written to {out_nc}")
 
 
-def writeAll(outpath, station_id, l3_h, l3_d, l3_m, csv_order=None):
-    """Write L3 hourly, daily and monthly datasets to .nc and .csv
-    files
-
-    Parameters
-    ----------
-    outpath : str
-        Output file path
-    station_id : str
-        Station name
-    l3_h : xr.Dataset
-        L3 hourly data
-    l3_d : xr.Dataset
-        L3 daily data
-    l3_m : xr.Dataset
-        L3 monthly data
-    csv_order : list, optional
-        List order of variables
-    """
-    if not os.path.isdir(outpath):
-        os.mkdir(outpath)
-    outfile_h = os.path.join(outpath, station_id + "_hour")
-    outfile_d = os.path.join(outpath, station_id + "_day")
-    outfile_m = os.path.join(outpath, station_id + "_month")
-    for o, l in zip([outfile_h, outfile_d, outfile_m], [l3_h, l3_d, l3_m]):
-        writeCSV(o + ".csv", l, csv_order)
-        writeNC(o + ".nc", l)
-
-
 def writeCSV(outfile, Lx, csv_order):
     """Write data product to CSV file
 
@@ -170,8 +147,8 @@ def writeCSV(outfile, Lx, csv_order):
     Lcsv.to_csv(outfile)
 
 
-def writeNC(outfile, Lx, col_names=None):
-    """Write data product to NetCDF file
+def writeNC(outfile, Lx, col_names=None, compression=False):
+    """Write data product to NetCDF file with compression
 
     Parameters
     ----------
@@ -187,7 +164,14 @@ def writeNC(outfile, Lx, col_names=None):
     else:
         names = list(Lx.keys())
 
-    Lx[names].to_netcdf(outfile, mode="w", format="NETCDF4", compute=True)
+    encoding = {var: dict() for var in names}
+
+    if compression:
+        comp = dict(zlib=True, complevel=4)
+        for var in names:
+            encoding[var].update(comp)
+
+    Lx[names].to_netcdf(outfile, mode="w", format="NETCDF4", compute=True, encoding=encoding)
 
 
 def getColNames(vars_df, ds, remove_nan_fields=False):
@@ -484,7 +468,7 @@ def reformat_time(dataset):
     return dataset
 
 
-def reformat_lon(dataset, exempt=["UWN", "Roof_GEUS", "Roof_PROMICE"]):
+def reformat_lon(dataset, exempt=["UWN", "Roof_GEUS", "Roof_PROMICE", "ORO"]):
     """Switch gps_lon to negative values (degrees_east). We do this here, and
     NOT in addMeta, otherwise we switch back to positive when calling getMeta
     in joinL2"""

pypromice/resources/variable_aliases_GC-Net.csv

@@ -5,10 +5,10 @@ p_l,
 t_u,TA2
 t_l,TA1
 rh_u,RH2
-rh_u_cor,RH2_cor
+rh_u_wrt_ice_or_water,RH2_cor
 qh_u,Q2
 rh_l,RH1
-rh_l_cor,RH1_cor
+rh_l_wrt_ice_or_water,RH1_cor
 qh_l,Q1
 wspd_u,VW2
 wspd_l,VW1

pypromice/resources/variables.csv

@@ -1,53 +1,53 @@
 field,standard_name,long_name,units,coverage_content_type,coordinates,instantaneous_hourly,where_to_find,lo,hi,OOL,station_type,L0,L2,L3,max_decimals
 time,time,Time,yyyy-mm-dd HH:MM:SS,physicalMeasurement,time,,,,,,all,1,1,1,
 rec,record,Record,-,referenceInformation,time,,L0 or L2,,,,all,1,1,0,0
-p_u,air_pressure,Air pressure (upper boom),hPa,physicalMeasurement,time,FALSE,,650,1100,z_pt z_pt_cor dshf_u dlhf_u qh_u,all,1,1,1,4
-p_l,air_pressure,Air pressure (lower boom),hPa,physicalMeasurement,time,FALSE,,650,1100,dshf_l dlhf_l qh_l,two-boom,1,1,1,4
-t_u,air_temperature,Air temperature (upper boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,rh_u_cor cc dsr_cor usr_cor z_boom z_stake dshf_u dlhf_u qh_u,all,1,1,1,4
-t_l,air_temperature,Air temperature (lower boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,rh_l_cor z_boom_l dshf_l dlhf_l qh_l,two-boom,1,1,1,4
-rh_u,relative_humidity,Relative humidity (upper boom),%,physicalMeasurement,time,FALSE,,0,100,rh_u_cor,all,1,1,1,4
-rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,modelResult,time,FALSE,L2 or later,0,150,dshf_u dlhf_u qh_u,all,0,1,1,4
-qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,modelResult,time,FALSE,L2 or later,0,100,,all,0,1,1,4
-rh_l,relative_humidity,Relative humidity (lower boom),%,physicalMeasurement,time,FALSE,,0,100,rh_l_cor,two-boom,1,1,1,4
-rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,modelResult,time,FALSE,L2 or later,0,150,dshf_l dlhf_l qh_l,two-boom,0,1,1,4
+p_u,air_pressure,Air pressure (upper boom),hPa,physicalMeasurement,time,FALSE,,650,1100,"",all,1,1,1,4
+p_l,air_pressure,Air pressure (lower boom),hPa,physicalMeasurement,time,FALSE,,650,1100,"",two-boom,1,1,1,4
+t_u,air_temperature,Air temperature (upper boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,"",all,1,1,1,4
+t_l,air_temperature,Air temperature (lower boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,"",two-boom,1,1,1,4
+rh_u,relative_humidity,Relative humidity (upper boom),%,physicalMeasurement,time,FALSE,,0,100,"",all,1,1,1,4
+rh_u_wrt_ice_or_water,relative_humidity_with_respect_to_ice_or_water,Relative humidity (upper boom) with respect to saturation over ice in subfreezing conditions and over water otherwise,%,modelResult,time,FALSE,L2 or later,0,150,"",all,0,1,1,4
+qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,modelResult,time,FALSE,L2 or later,0,100,"",all,0,1,1,4
+rh_l,relative_humidity,Relative humidity (lower boom),%,physicalMeasurement,time,FALSE,,0,100,"",two-boom,1,1,1,4
+rh_l_wrt_ice_or_water,relative_humidity_with_respect_to_ice_or_water,Relative humidity (lower boom) with respect to saturation over ice in subfreezing conditions and over water otherwise,%,modelResult,time,FALSE,L2 or later,0,150,"",two-boom,0,1,1,4
 qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,modelResult,time,FALSE,L2 or later,0,100,,two-boom,0,1,1,4
-wspd_u,wind_speed,Wind speed (upper boom),m s-1,physicalMeasurement,time,FALSE,,0,100,"wdir_u wspd_x_u wspd_y_u dshf_u dlhf_u qh_u, precip_u",all,1,1,1,4
-wspd_l,wind_speed,Wind speed (lower boom),m s-1,physicalMeasurement,time,FALSE,,0,100,"wdir_l wspd_x_l wspd_y_l dshf_l dlhf_l qh_l , precip_l",two-boom,1,1,1,4
+wspd_u,wind_speed,Wind speed (upper boom),m s-1,physicalMeasurement,time,FALSE,,0,100,wdir_u wspd_x_u wspd_y_u,all,1,1,1,4
+wspd_l,wind_speed,Wind speed (lower boom),m s-1,physicalMeasurement,time,FALSE,,0,100,wdir_l wspd_x_l wspd_y_l,two-boom,1,1,1,4
 wdir_u,wind_from_direction,Wind from direction (upper boom),degrees,physicalMeasurement,time,FALSE,,1,360,wspd_x_u wspd_y_u,all,1,1,1,4
 wdir_std_u,wind_from_direction_standard_deviation,Wind from direction (standard deviation),degrees,qualityInformation,time,FALSE,L0 or L2,,,,one-boom,1,1,0,4
 wdir_l,wind_from_direction,Wind from direction (lower boom),degrees,physicalMeasurement,time,FALSE,,1,360,wspd_x_l wspd_y_l,two-boom,1,1,1,4
-wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_u wspd_u,all,0,1,1,4
-wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_u wspd_u,all,0,1,1,4
-wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_l wspd_l,two-boom,0,1,1,4
-wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,wdir_l wspd_l,two-boom,0,1,1,4
-dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,physicalMeasurement,time,FALSE,,-10,1500,albedo dsr_cor usr_cor,all,1,1,1,4
+wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,"",all,0,1,1,4
+wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,"",all,0,1,1,4
+wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,"",two-boom,0,1,1,4
+wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,modelResult,time,FALSE,L0 or L2,-100,100,"",two-boom,0,1,1,4
+dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,physicalMeasurement,time,FALSE,,-10,1500,"",all,1,1,1,4
 dsr_cor,surface_downwelling_shortwave_flux_corrected,Downwelling shortwave radiation - corrected,W m-2,modelResult,time,FALSE,L2 or later,,,,all,0,1,1,4
-usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,physicalMeasurement,time,FALSE,,-10,1000,albedo dsr_cor usr_cor,all,1,1,1,4
+usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,physicalMeasurement,time,FALSE,,-10,1000,"",all,1,1,1,4
 usr_cor,surface_upwelling_shortwave_flux_corrected,Upwelling shortwave radiation - corrected,W m-2,modelResult,time,FALSE,L2 or later,0,1000,,all,0,1,1,4
 albedo,surface_albedo,Albedo,-,modelResult,time,FALSE,L2 or later,,,,all,0,1,1,4
-dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,physicalMeasurement,time,FALSE,,50,500,albedo dsr_cor usr_cor cc t_surf,all,1,1,1,4
-ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,physicalMeasurement,time,FALSE,,50,500,t_surf,all,1,1,1,4
+dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,physicalMeasurement,time,FALSE,,50,500,"",all,1,1,1,4
+ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,physicalMeasurement,time,FALSE,,50,500,"",all,1,1,1,4
 cc,cloud_area_fraction,Cloud cover,%,modelResult,time,FALSE,L2 or later,,,,all,0,1,1,4
-t_surf,surface_temperature,Surface temperature,C,modelResult,time,FALSE,L2 or later,-80,40,dshf_u dlhf_u qh_u,all,0,1,1,4
+t_surf,surface_temperature,Surface temperature,C,modelResult,time,FALSE,L2 or later,-80,40,"",all,0,1,1,4
 dlhf_u,surface_downward_latent_heat_flux,Latent heat flux (upper boom),W m-2,modelResult,time,FALSE,L3 or later,,,,all,0,0,1,4
 dlhf_l,surface_downward_latent_heat_flux,Latent heat flux (lower boom),W m-2,modelResult,time,FALSE,L3 or later,,,,two-boom,0,0,1,4
 dshf_u,surface_downward_sensible_heat_flux,Sensible heat flux (upper boom),W m-2,modelResult,time,FALSE,L3 or later,,,,all,0,0,1,4
 dshf_l,surface_downward_sensible_heat_flux,Sensible heat flux (lower boom),W m-2,modelResult,time,FALSE,L3 or later,,,,two-boom,0,0,1,4
-z_boom_u,distance_to_surface_from_boom,Upper boom height,m,physicalMeasurement,time,TRUE,,0.3,10,dshf_u dlhf_u qh_u,all,1,1,1,4
+z_boom_u,distance_to_surface_from_boom,Upper boom height,m,physicalMeasurement,time,TRUE,,0.3,10,"",all,1,1,1,4
 z_boom_q_u,distance_to_surface_from_boom_quality,Upper boom height (quality),-,qualityInformation,time,TRUE,L0 or L2,,,,all,1,1,0,4
-z_boom_l,distance_to_surface_from_boom,Lower boom height,m,physicalMeasurement,time,TRUE,,0.3,5,dshf_l dlhf_l qh_l,two-boom,1,1,1,4
+z_boom_l,distance_to_surface_from_boom,Lower boom height,m,physicalMeasurement,time,TRUE,,0.3,5,"",two-boom,1,1,1,4
 z_boom_q_l,distance_to_surface_from_boom_quality,Lower boom height (quality),-,qualityInformation,time,TRUE,L0 or L2,,,,two-boom,1,1,0,4
 z_stake,distance_to_surface_from_stake_assembly,Stake height,m,physicalMeasurement,time,TRUE,,0.3,8,,one-boom,1,1,1,4
 z_stake_q,distance_to_surface_from_stake_assembly_quality,Stake height (quality),-,qualityInformation,time,TRUE,L0 or L2,,,,one-boom,1,1,0,4
-z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,physicalMeasurement,time,FALSE,,0,30,z_pt_cor,one-boom,1,1,1,4
+z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,physicalMeasurement,time,FALSE,,0,30,"",one-boom,1,1,1,4
 z_pt_cor,depth_of_pressure_transducer_in_ice_corrected,Depth of pressure transducer in ice - corrected,m,modelResult,time,FALSE,L2 or later,0,30,,one-boom,0,1,1,4
 z_surf_combined,height_of_surface_combined,"Surface height combined from multiple sensors, relative to ice surface height at installation",m,modelResult,time,FALSE,L3,,,,all,0,0,1,4
 z_ice_surf,height_of_ice_surface,"Ice surface height, relative to ice surface height at installation and calculated from pt_cor and z_stake",m,modelResult,time,FALSE,L3,,,,one-boom,0,0,1,4
 snow_height,height_of_snow,"Snow surface height, relative to ice surface",m,modelResult,time,FALSE,L3,0,,,one-boom,0,0,1,4
-precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,physicalMeasurement,time,TRUE,,0,,precip_u_cor precip_u_rate,all,1,1,1,4
+precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,physicalMeasurement,time,TRUE,,0,,"",all,1,1,1,4
 precip_u_cor,precipitation_corrected,Precipitation (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,all,0,1,1,4
 precip_u_rate,precipitation_rate,Precipitation rate (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,all,0,1,1,4
-precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,physicalMeasurement,time,TRUE,,0,,precip_l_cor precip_l_rate,two-boom,1,1,1,4
+precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,physicalMeasurement,time,TRUE,,0,,"",two-boom,1,1,1,4
 precip_l_cor,precipitation_corrected,Precipitation (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,two-boom,0,1,1,4
 precip_l_rate,precipitation_rate,Precipitation rate (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time,TRUE,L2 or later,0,,,two-boom,0,1,1,4
 t_i_1,ice_temperature_at_t1,Ice temperature at sensor 1,degrees_C,physicalMeasurement,time,FALSE,,-80,1,,all,1,1,1,4
@@ -73,8 +73,8 @@ d_t_i_9,depth_of_thermistor_9,Depth of thermistor 9,m,modelResult,time,FALSE,L3,
 d_t_i_10,depth_of_thermistor_10,Depth of thermistor 10,m,modelResult,time,FALSE,L3,-10,100,,two-boom,0,0,1,4
 d_t_i_11,depth_of_thermistor_11,Depth of thermistor 11,m,modelResult,time,FALSE,L3,-10,100,,two-boom,0,0,1,4
 t_i_10m,10m_subsurface_temperature,10 m subsurface temperature,degrees_C,modelResult,time,FALSE,L3,-70,0,,all,0,0,1,4
-tilt_x,platform_view_angle_x,Tilt to east,degrees,physicalMeasurement,time,FALSE,,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4
-tilt_y,platform_view_angle_y,Tilt to north,degrees,physicalMeasurement,time,FALSE,,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4
+tilt_x,platform_view_angle_x,Tilt to east,degrees,physicalMeasurement,time,FALSE,,-30,30,"",all,1,1,1,4
+tilt_y,platform_view_angle_y,Tilt to north,degrees,physicalMeasurement,time,FALSE,,-30,30,"",all,1,1,1,4
 rot,platform_azimuth_angle,Station rotation from true North,degrees,physicalMeasurement,time,FALSE,,0,360,,all,1,1,1,2
 gps_lat,gps_latitude,Latitude,degrees_north,physicalMeasurement,time,TRUE,,50,83,,all,1,1,1,6
 gps_lon,gps_longitude,Longitude,degrees_east,physicalMeasurement,time,TRUE,,5,70,,all,1,1,1,6
@@ -93,14 +93,14 @@ batt_v_ini,,,-,physicalMeasurement,time,TRUE,L0 or L2,0,30,,,1,1,0,2
 batt_v_ss,battery_voltage_at_sample_start,Battery voltage (sample start),V,physicalMeasurement,time,TRUE,L0 or L2,0,30,,,1,1,0,2
 fan_dc_u,fan_current,Fan current (upper boom),mA,physicalMeasurement,time,TRUE,L0 or L2,0,200,,all,1,1,0,2
 fan_dc_l,fan_current,Fan current (lower boom),mA,physicalMeasurement,time,TRUE,,0,200,,two-boom,1,1,0,2
-freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,physicalMeasurement,time,TRUE,L0 or L2,0,10000,precip_u,,1,1,0,
+freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,physicalMeasurement,time,TRUE,L0 or L2,0,10000,"",,1,1,0,
 t_log,temperature_of_logger,Logger temperature,degrees_C,physicalMeasurement,time,TRUE,,-80,40,,one-boom,1,1,0,4
-t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,physicalMeasurement,time,FALSE,,-80,40,t_surf dlr ulr,all,1,1,1,4
+t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,physicalMeasurement,time,FALSE,,-80,40,"",all,1,1,1,4
 p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,physicalMeasurement,time,TRUE,,-350,100,,all,1,1,1,4
 t_i,air_temperature,Air temperature (instantaneous),degrees_C,physicalMeasurement,time,TRUE,,-80,40,,all,1,1,1,4
-rh_i,relative_humidity,Relative humidity (instantaneous),%,physicalMeasurement,time,TRUE,,0,150,rh_i_cor,all,1,1,1,4
-rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,modelResult,time,TRUE,L2 or later,0,100,,all,0,1,1,4
+rh_i,relative_humidity,Relative humidity (instantaneous),%,physicalMeasurement,time,TRUE,,0,150,"",all,1,1,1,4
+rh_i_wrt_ice_or_water,relative_humidity_with_respect_to_ice_or_water,Relative humidity (instantaneous) with respect to saturation over ice in subfreezing conditions and over water otherwise,%,modelResult,time,TRUE,L2 or later,0,100,,all,0,1,1,4
 wspd_i,wind_speed,Wind speed (instantaneous),m s-1,physicalMeasurement,time,TRUE,,0,100,wdir_i wspd_x_i wspd_y_i,all,1,1,1,4
 wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,physicalMeasurement,time,TRUE,,1,360,wspd_x_i wspd_y_i,all,1,1,1,4
-wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,modelResult,time,TRUE,L2 or later,-100,100,wdir_i wspd_i,all,0,1,1,4
-wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,modelResult,time,TRUE,L2 or later,-100,100,wdir_i wspd_i,all,0,1,1,4
+wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,modelResult,time,TRUE,L2 or later,-100,100,"",all,0,1,1,4
+wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,modelResult,time,TRUE,L2 or later,-100,100,"",all,0,1,1,4

pypromice/tx/payload_formats.csv

@@ -67,6 +67,7 @@ type,expected_values,format,description,flags,notes
 90,48, tfffffffffffffffffffffffffffffffffgnefffffffffff,THE GC-NET VERSION!,don’t use,GC-NET stations
 90,48,tfffffffffffffffffffffffffffffffffgnefffffffffff,THE GC-NET VERSION!,,GC-NET stations
 95,46,tfffffffffffffffffffffffffffffffgnefffffffffff,THE GC-NET VERSION!,,GC-NET stations
+115,47,tfffffffffffffffffffffffffffffffffgneffffffffff,PREMAS version,,PREMAS stations
 220,29,tfffffffffffffffffffffffnefff,ZAMG Freya 2015 summer message,,ZAMG Freya aws
 221,0,,,,ZAMG Freya aws
 222,29,tfffffffffffffffffffffffnefff,ZAMG Freya 2015 winter message,,ZAMG Freya aws

pypromice/utilities/dependency_graph.py

@@ -0,0 +1,101 @@
+from typing import Mapping, Set, MutableMapping, Optional
+
+import attr
+
+__all__ = [
+    "DependencyNode",
+    "DependencyGraph",
+]
+
+
+@attr.define
+class DependencyNode:
+    name: str = attr.field()
+    parents: Set["DependencyNode"] = attr.field(factory=set)
+    children: Set["DependencyNode"] = attr.field(factory=set)
+
+    def add_child(self, child: "DependencyNode"):
+        self.children.add(child)
+        child.parents.add(self)
+
+    def get_children_closure(self, closure: Optional[Set[str]] = None) -> Set[str]:
+        is_root = closure is None
+        if closure is None:
+            closure = set()
+        if self.name in closure:
+            return closure
+        closure.add(self.name)
+        for child in self.children:
+            closure |= child.get_children_closure(closure)
+
+        if is_root:
+            closure.remove(self.name)
+        return closure
+
+    def get_parents_closure(self, closure: Optional[Set[str]] = None) -> Set[str]:
+        is_root = closure is None
+        if closure is None:
+            closure = set()
+        if self.name in closure:
+            return closure
+        closure.add(self.name)
+        for parent in self.parents:
+            closure |= parent.get_parents_closure(closure)
+
+        if is_root:
+            closure.remove(self.name)
+        return closure
+
+    def __hash__(self):
+        return hash(self.name)
+
+
+@attr.define
+class DependencyGraph:
+    nodes: MutableMapping[str, DependencyNode] = attr.field(factory=dict)
+
+    def add_node(self, name: str) -> DependencyNode:
+        if name not in self.nodes:
+            self.nodes[name] = DependencyNode(name)
+        return self.nodes[name]
+
+    def add_edge(self, parent: str, child: str):
+        parent_node = self.add_node(parent)
+        child_node = self.add_node(child)
+        parent_node.add_child(child_node)
+
+    @classmethod
+    def from_child_mapping(cls, mapping: Mapping[str, Set[str]]) -> "DependencyGraph":
+        graph = cls()
+        for var, children in mapping.items():
+            graph.add_node(var)
+            for child in children:
+                graph.add_edge(var, child)
+        return graph
+
+    @classmethod
+    def from_parent_mapping(cls, mapping: Mapping[str, Set[str]]) -> "DependencyGraph":
+        graph = cls()
+        for var, parents in mapping.items():
+            graph.add_node(var)
+            for parent in parents:
+                graph.add_edge(parent, var)
+        return graph
+
+    def child_mapping(self) -> Mapping[str, Set[str]]:
+        return {
+            node.name: {child.name for child in node.children}
+            for node in self.nodes.values()
+        }
+
+    def child_closure_mapping(self) -> Mapping[str, Set[str]]:
+        return {node.name: node.get_children_closure() for node in self.nodes.values()}
+
+    def parent_mapping(self) -> Mapping[str, Set[str]]:
+        return {
+            node.name: {parent.name for parent in node.parents}
+            for node in self.nodes.values()
+        }
+
+    def parent_closure_mapping(self) -> Mapping[str, Set[str]]:
+        return {node.name: node.get_parents_closure() for node in self.nodes.values()}

{pypromice-1.4.3.dist-info → pypromice-1.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pypromice
-Version: 1.4.3
+Version: 1.5.0
 Summary: PROMICE/GC-Net data processing toolbox
 Home-page: https://github.com/GEUS-Glaciology-and-Climate/pypromice
 Author: GEUS Glaciology and Climate
@@ -18,16 +18,16 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: numpy ~=1.23
-Requires-Dist: pandas >=1.5.0
-Requires-Dist: xarray >=2022.6.0
+Requires-Dist: numpy~=1.23
+Requires-Dist: pandas>=1.5.0
+Requires-Dist: xarray>=2022.6.0
 Requires-Dist: toml
-Requires-Dist: scipy >=1.9.0
+Requires-Dist: scipy>=1.9.0
 Requires-Dist: Bottleneck
 Requires-Dist: netcdf4
-Requires-Dist: pyDataverse ==0.3.1
+Requires-Dist: pyDataverse==0.3.1
 Requires-Dist: eccodes
-Requires-Dist: scikit-learn >=1.1.0
+Requires-Dist: scikit-learn>=1.1.0
 
 # pypromice
 [![PyPI version](https://badge.fury.io/py/pypromice.svg)](https://badge.fury.io/py/pypromice) [![Anaconda-Server Badge](https://anaconda.org/conda-forge/pypromice/badges/version.svg)](https://anaconda.org/conda-forge/pypromice) [![Anaconda-Server Badge](https://anaconda.org/conda-forge/pypromice/badges/platforms.svg)](https://anaconda.org/conda-forge/pypromice) [![](<https://img.shields.io/badge/Dataverse DOI-10.22008/FK2/3TSBF0-orange>)](https://www.doi.org/10.22008/FK2/3TSBF0) [![DOI](https://joss.theoj.org/papers/10.21105/joss.05298/status.svg)](https://doi.org/10.21105/joss.05298) [![Documentation Status](https://readthedocs.org/projects/pypromice/badge/?version=latest)](https://pypromice.readthedocs.io/en/latest/?badge=latest)

{pypromice-1.4.3.dist-info → pypromice-1.5.0.dist-info}/RECORD

@@ -5,26 +5,26 @@ pypromice/get/get.py,sha256=8tdIbvdeXCpRWU7KmcKGIP9ZPdqIry3MjtJp9krumvo,7705
 pypromice/get/get_promice_data.py,sha256=bluNCaP50iRlWBzdEOXLrSPepOQdGB7SeQLkTWiqK4c,1806
 pypromice/postprocess/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypromice/postprocess/bufr_to_csv.py,sha256=ntTJZhfBRYPbLn2u9GzO5PFCkg9DIypUZONYEs2XrqA,507
-pypromice/postprocess/bufr_utilities.py,sha256=bc9U7ZC_7jtPbzaLYNnsz-nq3xn_PHGkgk1bVWa3ltc,21176
+pypromice/postprocess/bufr_utilities.py,sha256=O1vnpKipUC6nMPmSrPvWECJocjdsB9FAb2ILXD8b8fs,21264
 pypromice/postprocess/create_bufr_files.py,sha256=GizmNAW_v54MkvelPVzqIklBNHWH6nTRQC2qOo5amys,5657
 pypromice/postprocess/get_bufr.py,sha256=98BZ7tTbmvvrVAzhxJ8LgHib3w7Q3S_x757pCubHToU,16693
 pypromice/postprocess/make_metadata_csv.py,sha256=lUw8fZC6dzG10IePZ4d7eZIrTbquHYCDuppYP4r_dF0,9776
 pypromice/postprocess/positions_seed.csv,sha256=0kVCQ8UfEALdeXNYCddmwxpseRqLRudbFStqp_bZRBw,224
 pypromice/postprocess/real_time_utilities.py,sha256=2GZUua5R8ocp-TnIjEA4ig8lC4TlI3SkiimzLxoHXFE,9476
 pypromice/process/L0toL1.py,sha256=aQmVEXhtRisHNrJuPGIg7NA4hjjmV9jUBu1DClRIGzs,23175
-pypromice/process/L1toL2.py,sha256=Hcelx8kYDq5ERtzdAO1YUuBu8_aRwgWJ9FEDHuAVZu8,30478
-pypromice/process/L2toL3.py,sha256=xJXEHqfgWtKj5DxmuFq2F_Ynj0gMEpBxw3-RKpcnfUs,62311
+pypromice/process/L1toL2.py,sha256=PM6lToL-WXbZk5p2m90XCxY9S_RNNV1QUB3klZlQDzc,30832
+pypromice/process/L2toL3.py,sha256=XqnUGKO17ADJkTOn0p8TESeCAKQGtPXpFadNfF52g6E,61378
 pypromice/process/__init__.py,sha256=xvd0I-9nIyVw4M4qjgkQ5vXYpNuKcVSkIVIROQsZDo0,147
-pypromice/process/aws.py,sha256=SLVfJ4VMY6F3DGzlAQXvgHqdQnqppYZofSU2VEUeRkQ,8960
+pypromice/process/aws.py,sha256=5N4rLptO23Ic9mgyR5K8NhtNmGPa9xb9Cu7SgzAgoi8,7561
 pypromice/process/get_l2.py,sha256=ALXJCMJ8qgg0_dEKx-dV5TQ9IAJnLLLGPUxlr5QVfpk,3076
 pypromice/process/get_l2tol3.py,sha256=4Qu2d5rT25H2dObyCc70ivtJg3vw6WA-hzI-kRD6ybQ,4544
 pypromice/process/join_l2.py,sha256=ifjuhFR9scVvZt3xuy-ELp-iRchxV1dEK9qJ4UNh5bE,4567
-pypromice/process/join_l3.py,sha256=nLLQbX0vuuvHET8r33ZAt5g1dtIk3foUJ9RZkwkEwE4,20158
+pypromice/process/join_l3.py,sha256=F90uvcy85iSl6GWxyj6C3K4j2eAoQvhQB8XqOXpShvo,20221
 pypromice/process/load.py,sha256=iaFvJeaDanAA60caVj4BWupZpqgQNj3CiNU4csz9FdU,4585
-pypromice/process/resample.py,sha256=x0t9CE4YnwYMrn6wDRArhrQHqHOIJpYpTrgEcARUppo,7138
+pypromice/process/resample.py,sha256=VhcQ0UD5LkqFPHlwEFmwYIz3kLkUfvxTenxYadmCsSE,7301
 pypromice/process/utilities.py,sha256=1pqSaF3bIbvRNtOjb25mbegHfuW9MY4KpCBDVXWyML8,1773
-pypromice/process/value_clipping.py,sha256=FkBiDT_HK_BDFiVjB7NdWH-_nab7vONG9LOd2PpEBI8,1573
-pypromice/process/write.py,sha256=fRCCK4g_W07M4EEsJErdTSN2Pldr9SLgqM2w_rsp2ZQ,16257
+pypromice/process/value_clipping.py,sha256=_pTAxogsqfkjwBQslGyyE3jZzhYY6_Wp-g5ZPHcudMc,1506
+pypromice/process/write.py,sha256=ZpA9NAgtnzJIbnjEiJ0sOtsE0J1haHASbjHqzMvBDJE,15818
 pypromice/qc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypromice/qc/github_data_issues.py,sha256=gbbF62oMMWbXiLnsrs60vXbwfAqSUP113plhidVeqCU,13353
 pypromice/qc/persistence.py,sha256=2ruS5w_QXt8730jwh6dwN1F3h7MGKFYStfO3T0YmmCY,6582
@@ -34,19 +34,20 @@ pypromice/qc/percentiles/outlier_detector.py,sha256=5_458aMm9MAubfYv7oIz1Lp9ME6S
 pypromice/qc/percentiles/thresholds.csv,sha256=KTQcYsg2VCZmR-Rf1Zzx1Jn-ZmR5yPPWWXYZ0Z03PDQ,9516
 pypromice/resources/__init__.py,sha256=MpKmvV11R4tcqmyvJpXQt-_It3oRI0WEIQNbMST--4w,907
 pypromice/resources/file_attributes.csv,sha256=NFqfg6u0PmPzY2bqE2bp646E4LUiPLqvzhyxDauXoPU,7014
-pypromice/resources/variable_aliases_GC-Net.csv,sha256=Z4-S9wThgWxwZTgeIWjjAoAP2Rnuu5kdY4m3aAqnMIM,967
-pypromice/resources/variables.csv,sha256=YyYng6ZL2eA0EIDdNAT8ACp6kg49R_38mTD5dWz_Vps,13513
+pypromice/resources/variable_aliases_GC-Net.csv,sha256=v881RUTozleq-f7q7sVT3fqidcBH7t-kVUqDnjtbDm8,993
+pypromice/resources/variables.csv,sha256=cW8JbQMb5rlge9I3EjbWuKACg3nvbVmbd84W3ouFJdI,13282
 pypromice/tx/__init__.py,sha256=-62bhHWJGfzFh5JwHcLqRj2jcGzmqzYOLWByhO706YY,30
 pypromice/tx/get_l0tx.py,sha256=b34-96KGshTyTN2tBFaAIBl7oZZzbRB_JR7sXtDNfXA,6957
 pypromice/tx/get_msg.py,sha256=OGS60OHjy4Wf8JExTfOdK-9xhjFdjhuChxoTSPe_MjI,3417
-pypromice/tx/payload_formats.csv,sha256=tzTTNuvmVwlwd7z3aF8A2dhjKNQ4lVumpnNBs3e3YeQ,7816
+pypromice/tx/payload_formats.csv,sha256=A46-XcYdpe9-gzmADylP2UVizLi_UphF-BPT5u3Lyn8,7903
 pypromice/tx/payload_types.csv,sha256=C1-xCmHytAqqAzgzPwBLWqabzWu6s6tKAd8AjVd935s,457
 pypromice/tx/tx.py,sha256=mghUjwGqUKe_4JWuAEgWMyH4ME2QRufeMPPHoL72R08,34267
 pypromice/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypromice/utilities/dependency_graph.py,sha256=bqoXasC8pg5ipjBd6rqDhfHwIq11t2_cFlNT72ncw4w,3135
 pypromice/utilities/git.py,sha256=7EUGjDs_VZucrckakXKyZEclDAZ_mKIxhTWzhopCIxM,1785
-pypromice-1.4.3.dist-info/LICENSE.txt,sha256=gXf5dRMhNSbfLPYYTY_5hsZ1r7UU1OaKQEAQUhuIBkM,18092
-pypromice-1.4.3.dist-info/METADATA,sha256=mMnZyZwB5jdkzlnxlj-2vxptCrrM1vUHk85tfObm90U,4762
-pypromice-1.4.3.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-pypromice-1.4.3.dist-info/entry_points.txt,sha256=ufX1npmY3nqMPtSVRKVxn3MhG9IyFHD5FjPZQcELVXo,618
-pypromice-1.4.3.dist-info/top_level.txt,sha256=cBdfwgSbWDQq3a07nKRjrfmLC7jdaYXs98GG58HpTks,10
-pypromice-1.4.3.dist-info/RECORD,,
+pypromice-1.5.0.dist-info/LICENSE.txt,sha256=gXf5dRMhNSbfLPYYTY_5hsZ1r7UU1OaKQEAQUhuIBkM,18092
+pypromice-1.5.0.dist-info/METADATA,sha256=EJQZp4E1oGs0OyC39bwzG2wQdrfx5xH3RK8twy_b3iQ,4756
+pypromice-1.5.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+pypromice-1.5.0.dist-info/entry_points.txt,sha256=ufX1npmY3nqMPtSVRKVxn3MhG9IyFHD5FjPZQcELVXo,618
+pypromice-1.5.0.dist-info/top_level.txt,sha256=cBdfwgSbWDQq3a07nKRjrfmLC7jdaYXs98GG58HpTks,10
+pypromice-1.5.0.dist-info/RECORD,,

{pypromice-1.4.3.dist-info → pypromice-1.5.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any