pypromice 1.3.5__py3-none-any.whl → 1.4.0__py3-none-any.whl

pypromice/process/get_l2.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+import logging
+import os
+import sys
+from argparse import ArgumentParser
+from pathlib import Path
+
+from pypromice.process.aws import AWS
+from pypromice.process.write import prepare_and_write
+
+
+def parse_arguments_l2():
+    parser = ArgumentParser(description="AWS L2 processor")
+
+    parser.add_argument('-c', '--config_file', type=str, required=True,
+                        help='Path to config (TOML) file')
+    parser.add_argument('-i', '--inpath', type=str, required=True, 
+                        help='Path to input data')
+    parser.add_argument('-o', '--outpath', default=None, type=str, required=False, 
+                        help='Path where to write output')
+    parser.add_argument('-v', '--variables', default=None, type=str, 
+                        required=False, help='File path to variables look-up table')
+    parser.add_argument('-m', '--metadata', default=None, type=str, 
+                        required=False, help='File path to metadata')
+    parser.add_argument('--data_issues_path', '--issues', default=None, help="Path to data issues repository")
+    args = parser.parse_args()
+    return args
+
+
+def get_l2(config_file, inpath, outpath, variables, metadata, data_issues_path: Path) -> AWS:
+    # Define input path
+    station_name = config_file.split('/')[-1].split('.')[0] 
+    station_path = os.path.join(inpath, station_name)
+    
+    # checking that data_issues_path is valid
+    if data_issues_path is None:
+        data_issues_path = Path("../PROMICE-AWS-data-issues")
+        if data_issues_path.exists():
+            logging.warning(f"data_issues_path is missing. Using default data issues path: {data_issues_path}")
+        else:
+            raise ValueError("data_issues_path is missing. Please provide a valid path to the data issues repository")
+
+    if os.path.exists(station_path):
+        aws = AWS(config_file, 
+                  station_path,
+                  data_issues_repository=data_issues_path, 
+                  var_file=variables, 
+                  meta_file=metadata)
+    else:
+        aws = AWS(config_file, 
+                  inpath, 
+                  data_issues_repository=data_issues_path, 
+                  var_file=variables, 
+                  meta_file=metadata)
+
+    # Perform level 1 and 2 processing
+    aws.getL1()
+    aws.getL2()
+    # Write out level 2
+    if outpath is not None:
+        if not os.path.isdir(outpath):
+            os.mkdir(outpath)
+        if aws.L2.attrs['format'] == 'raw':
+            prepare_and_write(aws.L2, outpath, aws.vars, aws.meta, '10min')
+        prepare_and_write(aws.L2, outpath, aws.vars, aws.meta, '60min')
+    return aws
+
+
+def main():
+    args = parse_arguments_l2()
+
+    logging.basicConfig(
+        format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
+        level=logging.INFO,
+        stream=sys.stdout,
+    )
+
+    _ = get_l2(
+        args.config_file,
+        args.inpath,
+        args.outpath,
+        args.variables,
+        args.metadata,
+        args.data_issues_path,
+    )
+
+
+if __name__ == "__main__":  
+    main()
+        

pypromice/process/get_l2tol3.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+import logging, sys, toml
+from pathlib import Path
+
+import xarray as xr
+from argparse import ArgumentParser
+import pypromice
+from pypromice.process.L2toL3 import toL3
+import pypromice.resources
+from pypromice.process.write import prepare_and_write
+logger = logging.getLogger(__name__)
+
+def parse_arguments_l2tol3(debug_args=None):
+    parser = ArgumentParser(description="AWS L3 script for the processing L3 "+
+                            "data from L2. An hourly, daily and monthly L3 "+
+                            "data product is outputted to the defined output path")
+    parser.add_argument('-c', '--config_folder', type=str, required=True,
+                        default='../aws-l0/metadata/station_configurations/',
+                        help='Path to folder with sites configuration (TOML) files')
+    parser.add_argument('-i', '--inpath', type=str, required=True, 
+                        help='Path to Level 2 .nc data file')
+    parser.add_argument('-o', '--outpath', default=None, type=str, required=False, 
+                        help='Path where to write output')
+    parser.add_argument('-v', '--variables', default=None, type=str, 
+                        required=False, help='File path to variables look-up table')
+    parser.add_argument('-m', '--metadata', default=None, type=str, 
+                        required=False, help='File path to metadata')
+    parser.add_argument('--data_issues_path', '--issues', default=None, help="Path to data issues repository")
+
+
+    args = parser.parse_args(args=debug_args)
+    return args
+
+def get_l2tol3(config_folder: Path|str, inpath, outpath, variables, metadata, data_issues_path: Path|str):
+    if isinstance(config_folder, str):
+        config_folder = Path(config_folder)
+
+    logging.basicConfig(
+        format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
+        level=logging.INFO,
+        stream=sys.stdout,
+    )
+  
+    # Define Level 2 dataset from file
+    with xr.open_dataset(inpath) as l2:
+        l2.load()
+    
+    # Remove encoding attributes from NetCDF
+    for varname in l2.variables:
+        if l2[varname].encoding!={}:
+            l2[varname].encoding = {}
+
+    if 'bedrock' in l2.attrs.keys():
+        l2.attrs['bedrock'] = l2.attrs['bedrock'] == 'True'
+    if 'number_of_booms' in l2.attrs.keys():
+        l2.attrs['number_of_booms'] = int(l2.attrs['number_of_booms'])
+    
+    # importing station_config (dict) from config_folder (str path)
+    config_file = config_folder / (l2.attrs['station_id']+'.toml')
+
+    if config_file.exists():
+        # File exists, load the configuration
+        station_config = toml.load(config_file)
+    else:
+        # File does not exist, initialize with standard info 
+        # this was prefered by RSF over exiting with error
+        logger.error("\n***\nNo station_configuration file for %s.\nPlease create one on AWS-L0/metadata/station_configurations.\n***"%l2.attrs['station_id'])
+        station_config = {"stid":l2.attrs['station_id'],
+                        "station_site":l2.attrs['station_id'],
+                        "project": "PROMICE",
+                        "location_type": "ice sheet",
+                        }
+        
+    # checking that the adjustement directory is properly given
+    if data_issues_path is None:
+        data_issues_path = Path("../PROMICE-AWS-data-issues")
+        if data_issues_path.exists():
+            logging.warning(f"data_issues_path is missing. Using default data issues path: {data_issues_path}")
+        else:
+            raise ValueError("data_issues_path is missing. Please provide a valid path to the data issues repository")
+    else:
+        data_issues_path = Path(data_issues_path)
+
+    data_adjustments_dir = data_issues_path / "adjustments"
+    
+    # Perform Level 3 processing
+    l3 = toL3(l2, data_adjustments_dir, station_config)
+
+    # Write Level 3 dataset to file if output directory given
+    v = pypromice.resources.load_variables(variables)
+    m = pypromice.resources.load_metadata(metadata)
+    if outpath is not None:
+        prepare_and_write(l3, outpath, v, m, '60min')
+        prepare_and_write(l3, outpath, v, m, '1D')
+        prepare_and_write(l3, outpath, v, m, 'M')
+    return l3
+
+def main():
+    args = parse_arguments_l2tol3()
+    
+
+
+    _ = get_l2tol3(args.config_folder, 
+                   args.inpath, 
+                   args.outpath,
+                   args.variables, 
+                   args.metadata, 
+                   args.data_issues_path)
+    
+if __name__ == "__main__":  
+    main()

pypromice/process/join_l2.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+import logging, sys, os, unittest
+import pandas as pd
+import xarray as xr
+from argparse import ArgumentParser
+from pypromice.process.L1toL2 import correctPrecip
+from pypromice.process.write import prepare_and_write
+logger = logging.getLogger(__name__)
+
+def parse_arguments_join():
+    parser = ArgumentParser(description="AWS L2 joiner for merging together two L2 products, for example an L2 RAW and L2 TX data product. An hourly, daily and monthly L2 data product is outputted to the defined output path")
+    parser.add_argument('-s', '--file1', type=str, required=True,
+                        help='Path to source L2 file, which will be preferenced in merge process')
+    parser.add_argument('-t', '--file2', type=str, required=True, 
+                        help='Path to target L2 file, which will be used to fill gaps in merge process')
+    parser.add_argument('-o', '--outpath', default=os.getcwd(), type=str, required=True, 
+                        help='Path where to write output')
+    parser.add_argument('-v', '--variables', default=None, type=str, required=False, 
+    			 help='Path to variables look-up table .csv file for variable name retained'''),
+    parser.add_argument('-m', '--metadata', default=None, type=str, required=False, 
+    			 help='Path to metadata table .csv file for metadata information'''),
+    args = parser.parse_args()
+    return args
+
+def loadArr(infile):
+    if infile.split('.')[-1].lower() == 'csv':
+        df = pd.read_csv(infile, index_col=0, parse_dates=True)
+        ds = xr.Dataset.from_dataframe(df)  
+    elif infile.split('.')[-1].lower() == 'nc':
+        with xr.open_dataset(infile) as ds:
+            ds.load()
+        # Remove encoding attributes from NetCDF
+        for varname in ds.variables:
+            if ds[varname].encoding!={}:
+                ds[varname].encoding = {}
+
+    try:
+        name = ds.attrs['station_id'] 
+    except:
+        name = infile.split('/')[-1].split('.')[0].split('_hour')[0].split('_10min')[0]
+        ds.attrs['station_id'] = name
+    if 'bedrock' in ds.attrs.keys():
+        ds.attrs['bedrock'] = ds.attrs['bedrock'] == 'True'
+    if 'number_of_booms' in ds.attrs.keys():
+        ds.attrs['number_of_booms'] = int(ds.attrs['number_of_booms'])
+
+    logger.info(f'{name} array loaded from {infile}')
+    return ds, name
+    
+
+def join_l2(file1,file2,outpath,variables,metadata) -> xr.Dataset:
+    logging.basicConfig(
+        format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
+        level=logging.INFO,
+        stream=sys.stdout,
+    )
+    # Check files
+    if os.path.isfile(file1) and os.path.isfile(file2): 
+
+        # Load data arrays
+        ds1, n1 = loadArr(file1)
+        ds2, n2 = loadArr(file2)    	
+        
+        # Check stations match
+        if n1.lower() == n2.lower():
+            
+        	# Merge arrays
+            logger.info(f'Combining {file1} with {file2}...')
+            name = n1
+            all_ds = ds1.combine_first(ds2)
+            
+            # Re-calculate corrected precipitation
+            if hasattr(all_ds, 'precip_u_cor'):
+                if ~all_ds['precip_u_cor'].isnull().all():
+                    all_ds['precip_u_cor'],  _ = correctPrecip(all_ds['precip_u'], 
+                                                                all_ds['wspd_u'])
+            if hasattr(all_ds, 'precip_l_cor'):
+                if ~all_ds['precip_l_cor'].isnull().all():
+                    all_ds['precip_l_cor'],  _ = correctPrecip(all_ds['precip_l'], 
+                                                                all_ds['wspd_l'])                    
+        else:
+            logger.info(f'Mismatched station names {n1}, {n2}')
+            exit()            
+    
+    elif os.path.isfile(file1):  
+        ds1, name = loadArr(file1)
+        logger.info(f'Only one file found {file1}...')
+        all_ds = ds1  
+
+    elif os.path.isfile(file2):
+        ds2, name = loadArr(file2)
+        logger.info(f'Only one file found {file2}...')
+        all_ds = ds2  
+    
+    else:
+        logger.info(f'Invalid files {file1}, {file2}')
+        exit()
+
+    all_ds.attrs['format'] = 'merged RAW and TX'
+
+    # Resample to hourly, daily and monthly datasets and write to file
+    prepare_and_write(all_ds, outpath, variables, metadata, resample = False)
+    
+    logger.info(f'Files saved to {os.path.join(outpath, name)}...')
+    return all_ds
+
+def main():
+    args = parse_arguments_join()
+    _ = join_l2(args.file1, args.file2, args.outpath, args.variables, args.metadata)
+    
+if __name__ == "__main__":  
+    main()