hspf 2.0.0__py3-none-any.whl

hspf/uci.py

@@ -0,0 +1,643 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Jul 11 08:39:57 2022
+
+@author: mfratki
+"""    
+
+
+#lines = reader('C:/Users/mfratki/Documents/Projects/LacQuiParle/ucis/LacQuiParle_0.uci')
+import subprocess
+import numpy as np
+import pandas as pd
+from .parser.parsers import Table
+from .parser.graph import reachNetwork
+
+#from hspf_tools.parser import setup
+
+from pathlib import Path
+
+
+parseTable = pd.read_csv(Path(__file__).parent/'data/ParseTable.csv',
+                          dtype = {'width': 'Int64',
+                                  'start': 'Int64',
+                                  'stop': 'Int64',
+                                  'space': 'Int64'})
+
+#timeseriesCatalog = pd.read_csv(Path(__file__).parent/'TimeseriesCatalog.csv')
+
+#timeseriesCatalog = pd.read_csv('C:/Users/mfratki/Documents/GitHub/hspf_tools/parser/TimeseriesCatalog.csv')
+#                             dtype = {'width': 'Int64',
+#                                       'start': 'Int64',
+#                                       'stop': 'Int64',
+#                                       'space': 'Int64'})
+#uci interface
+class UCI():
+    def __init__(self, filepath,infer_metzones = True):
+        self.filepath = Path(filepath)
+        self.name = self.filepath.name.split('.')[0]
+        self.lines = reader(filepath)
+        self.run_comments = RUN_comments(self.lines)
+        self.uci = build_uci(self.lines) # UCI converted into a nested dictionary. # Could convert into a class with only tables? 
+        self.wdm_paths = self.get_filepaths('.wdm')
+        self.hbn_paths = self.get_filepaths('.hbn')
+
+        # Require to get valid opnids - Business rule
+        opnseq = self.table('OPN SEQUENCE')
+        self.valid_opnids=  {'PERLND': opnseq['SEGMENT'][opnseq['OPERATION'] == 'PERLND'].astype(int).to_list(),
+                             'RCHRES': opnseq['SEGMENT'][opnseq['OPERATION'] == 'RCHRES'].astype(int).to_list(),
+                             'IMPLND': opnseq['SEGMENT'][opnseq['OPERATION'] == 'IMPLND'].astype(int).to_list(),
+                             'GENER' : opnseq['SEGMENT'][opnseq['OPERATION'] == 'GENER'].astype(int).to_list(),
+                             'COPY'  : opnseq['SEGMENT'][opnseq['OPERATION'] == 'COPY'].astype(int).to_list()}
+        self.network = reachNetwork(self)
+
+        if infer_metzones:
+            self.opnid_dict = self.get_metzones()
+        self._LSID_flag = 0
+
+        #compositions or totally separate classes?
+        # self.network = network class
+        # tableParser - Responsible for converting uci text to and from a pandas dataframe
+        # tableUpdater - Responsible for updating individual tables
+    
+    
+    # def supplemental(self):
+    #     for block in ['RCHRES','PERLND','IMPLND']:
+    #         keys = list([key for key in list(self.uci.keys()) if key[0] == block])
+    #         for key in keys:
+    #             lines = self.uci[key]
+    #             for line in lines:
+    #                 if '***' in line:
+    #                     pass
+    #                 elif '~' in line:
+    #                     line.split('~') # assuming there will only ever be 2 ~ in a line
+    
+    def get_parameter(self,parameter):
+        raise NotImplementedError()
+    
+                 
+    def table(self,block,table_name = 'na',table_id = 0,drop_comments = True):
+        # Dynamic parsing of tables when called by user
+        assert block in ['FILES','PERLND','IMPLND','RCHRES','SCHEMATIC','OPN SEQUENCE','MASS-LINK','EXT SOURCES','NETWORK','GENER','MONTH-DATA','EXT TARGETS','COPY']
+        
+        table = self.uci[(block,table_name,table_id)] #[block][table_name][table_id]
+        #TODO move the format_opnids into the Table class?
+        if table.data is None:
+            table.parse()
+            if block in ['PERLND','RCHRES','IMPLND','GENER','COPY']     :
+                table.replace(format_opnids(table.data,self.valid_opnids[block]))
+            elif block in ['EXT SOURCES']:
+                table.replace(expand_extsources(table.data,self.valid_opnids))
+                
+        table_data = table.data.copy()
+        if drop_comments:
+            table_data =table_data[table_data['comments'] == '']
+            table_data = table_data.drop('comments',axis = 1)       
+        
+        return table_data
+    
+    def _table(self,block,table_name,table_id):
+        return self.uci[(block,table_name,table_id)]
+    
+    def replace_table(self,table,block,table_name = 'na',table_id = 0): #replace an entire table 
+        self.uci[(block,table_name,table_id)].replace(table)
+
+    def table_lines(self,block,table_name = 'na',table_id = 0):
+        return self.uci[(block,table_name,table_id)].lines
+        
+    def comments(block,table_name = None,table_id = 0): # comments of a table
+        raise NotImplementedError()
+        
+    def table_names(self,block):
+        return list(set([key[1] for key in list(self.uci.keys()) if key[0] == block]))
+        
+    def block_names(self): #blocks present in a particular uci file
+        return set([key[0] for key in list(self.uci.keys())])
+    
+    def add_comment(self,comment):
+        raise NotImplementedError()
+                
+    def update_table(self,value,operation,table_name,table_id,opnids = None,columns = None,operator = '*',axis = 0):
+        # This should be moved up one layer as this is a user/business requirement. I would pass a Table object from this layer (data lyaer?) to the business layer, make changes, then pass it back down to this layer.
+        # ensures data has been parsed and allows for determining opnids and column values
+        table = self.table(operation,table_name,table_id,True)
+        
+        if opnids is None:
+            opnids = table.index
+        if columns is None:
+            columns = table.columns
+        
+        # Cases where some tables don't have an opnid specified but the timeseries we are comparing might
+        # opnids = table.index.intersection(opnids)
+        
+        # simple methods for changing all values by the same value/operator combination
+        if operator == 'set':
+            self.uci[(operation,table_name,table_id)].set_value(opnids,columns,value, axis)
+        elif operator == '*':
+            self.uci[(operation,table_name,table_id)].mul(opnids,columns,value, axis)
+        elif operator == '/':
+            self.uci[(operation,table_name,table_id)].div(opnids,columns,value, axis)
+        elif operator == '-':
+            self.uci[(operation,table_name,table_id)].sub(opnids,columns,value, axis)
+        elif operator == '+':
+            self.uci[(operation,table_name,table_id)].add(opnids,columns,value, axis)
+        elif operator == 'chuck':
+            assert(table_name in ['MON-IFLW-CONC','MON-GRND-CONC'])
+            values = chuck(value,table).loc[opnids,columns]
+            self.uci[(operation,table_name,table_id)].set_value(opnids,columns,values)
+        else:
+            print('Select valid operator (set,*,/,-,+')
+    
+    def merge_lines(self): # write uci to a txt file
+        lines = ['RUN']
+        lines += self.run_comments
+        
+        # properly ordered blocks
+        blocks = {}
+        for key in self.uci.keys():
+            if key[0] in blocks.keys():
+                blocks[key[0]].append(key)
+            else:
+                blocks[key[0]] = [key]
+                
+        for block,keys in blocks.items():
+            lines += [block]
+            for key in keys:
+                table = self.uci[key]
+                if key[1] == 'na':
+                    lines += table.lines
+                else:
+                    lines += [table.header]
+                    lines += table.lines
+                    lines += [table.footer]
+                    lines += ['']
+                    
+            lines += ['END ' + block]
+            lines += ['']
+        lines += ['END RUN']
+        self.lines = lines       
+        
+    
+    def write(self,new_uci_path):
+        self.merge_lines()
+        with open(new_uci_path, 'w') as the_file:
+            for line in self.lines:    
+                the_file.write(line+'\n')   
+
+    def update_bino(self,name):
+        #TODO: Move up to busniess/presentation layer
+        table = self.table('FILES',drop_comments = False) # initialize the table
+        indexs = table[table['FTYPE'] == 'BINO'].index
+        for index in indexs: 
+            table.loc[index,'FILENAME'] = name + '-' + table.loc[index,'FILENAME'].split('-')[-1]          
+        self.replace_table(table,'FILES')
+        #self.uci[('FILES','na',0)].set_value(index,'FILENAME',filename)
+    
+    def get_metzones(self):
+        '''
+        Only keeps reaches that are recieving meteorlogical inputs.
+        
+        '''
+        operations = ['PERLND','IMPLND','RCHRES']
+        dic = {}
+        
+        extsrc = self.table('EXT SOURCES')
+        # GROUP = 'EXTNL'
+        # DOMAIN = 'MET'
+        # tmemns = timeseriesCatalog.loc[(timeseriesCatalog['Domain'] == 'MET') & (timeseriesCatalog['Group'] == 'EXTNL'),'Member'].str.strip().to_list()
+        
+        # All metzones assuming every implnd,perlnd, and rchres recives precip input
+        metzones = extsrc.loc[(extsrc['TMEMN'] == 'PREC') & (extsrc['TVOL'].isin(operations)),'SVOLNO'].sort_values().unique()
+        metzone_map = {metzone:num for num,metzone in zip(range(len(metzones)),metzones)}
+        
+        
+        
+        for operation in operations:
+            opnids = extsrc.loc[(extsrc['TMEMN'].isin(['PREC'])) & (extsrc['TVOL'] == operation),['TOPFST','SVOLNO']]
+            opnids = opnids.drop_duplicates(subset = 'TOPFST')
+            opnids['metzone'] = opnids['SVOLNO'].map(metzone_map).values
+            opnids.set_index(['TOPFST'],inplace = True)
+           
+            # Only keep opnids that are recieving preciptiation inputs.
+            geninfo = self.table(operation,'GEN-INFO')
+            geninfo = geninfo.loc[ list(set(geninfo.index).intersection(set(opnids.index)))] .reset_index()
+            geninfo = geninfo.drop_duplicates(subset = 'OPNID').sort_values(by = 'OPNID')
+            if operation == 'RCHRES':
+                opnids.loc[geninfo['OPNID'],['RCHID','LKFG']] = pd.NA
+                opnids['RCHID'] = geninfo['RCHID'].to_list()
+                opnids['LKFG'] = geninfo['LKFG'].to_list()
+            else:     
+                landcovers = geninfo['LSID'].unique()
+                landcover_map =  {landcover:num for num,landcover in zip(range(len(landcovers)),landcovers)}
+                opnids['LSID'] = pd.NA
+                opnids.loc[geninfo['OPNID'],'LSID'] = geninfo['LSID'].to_list() # index of opnid is the OPNID
+                opnids['landcover'] = opnids['LSID'].map(landcover_map).values
+                
+               
+                
+            dic[operation] = opnids
+        return dic
+    
+    
+# Convience methods. TODO: put in separate module that takes uci object as input. Should not be instance method
+    def get_filepaths(self,file_extension):
+        files = self.table('FILES')
+        filepaths = files.loc[(files['FILENAME'].str.endswith(file_extension.lower())) |  (files['FILENAME'].str.endswith(file_extension.upper())),'FILENAME'].to_list()
+        filepaths = [self.filepath.parent.joinpath(filepath) for filepath in filepaths]
+        return filepaths
+    
+    def get_dsns(self,operation,opnid,smemn):
+        dsns = self.table('EXT SOURCES')
+        assert (smemn in dsns['SMEMN'].unique())
+        dsns = dsns.loc[(dsns['TVOL'] == operation) & (dsns['TOPFST'] == opnid) & (dsns['SMEMN'] == smemn)]
+        files = self.table('FILES').set_index('FTYPE')
+        dsns.loc[:,'FILENAME'] = files.loc[dsns['SVOL'],'FILENAME'].values
+        dsns = dsns[['FILENAME','SVOLNO','SMEMN','TOPFST','TVOL']]
+        return dsns
+    
+        
+    def initialize(self,name = None, default_output = 4,n=5,reach_ids = None):
+        if name is None:
+            name = self.name
+            
+        setup_files(self,name,n)
+        setup_geninfo(self)
+        setup_binaryinfo(self,default_output = default_output,reach_ids = reach_ids)
+        setup_qualid(self)
+
+    def initialize_binary_info(self,default_output = 4,reach_ids = None):
+        setup_binaryinfo(self,default_output = default_output,reach_ids = reach_ids)
+        
+    
+    def build_targets(self):
+        geninfo = self.table('PERLND','GEN-INFO')  
+        targets = self.opnid_dict['PERLND'].loc[:,['LSID','landcover']] #.drop_duplicates(subset = 'landcover').loc[:,['LSID','landcover']].reset_index(drop = True)
+        targets.columns = ['LSID','lc_number']
+        schematic = self.table('SCHEMATIC')
+        schematic = schematic.astype({'TVOLNO': int, "SVOLNO": int, 'AFACTR':float})
+        schematic = schematic[(schematic['SVOL'] == 'PERLND')]
+        schematic = schematic[(schematic['TVOL'] == 'PERLND') | (schematic['TVOL'] == 'IMPLND') | (schematic['TVOL'] == 'RCHRES')]
+        areas = []
+        for lc_number in targets['lc_number'].unique():
+            areas.append(np.sum([schematic['AFACTR'][schematic['SVOLNO'] == perland].sum() for perland in targets.index[targets['lc_number'] == lc_number]]))
+        areas = np.array(areas)
+        
+        
+        lc_number = targets['lc_number'].drop_duplicates()
+        uci_names = geninfo.loc[targets['lc_number'].drop_duplicates().index]['LSID']
+        targets = pd.DataFrame([uci_names.values,lc_number.values,areas]).transpose()
+        targets.columns = ['uci_name','lc_number','area']
+        targets['npsl_name'] = ''
+        
+        targets[['TSS','N','TKN','OP','BOD']] = ''
+        
+        targets['dom_lc'] = ''
+        targets.loc[targets['area'].astype('float').argmax(),'dom_lc'] = 1
+        return targets        
+
+
+#TODO: More conveince methods that should probably be in a separate module
+
+def run_model(uci_file):
+    winHSPF = str(Path(__file__).resolve().parent.parent) + '\\bin\\WinHSPFLt\\WinHspfLt.exe'     
+    subprocess.run([winHSPF,uci_file.as_posix()]) #, stdout=subprocess.PIPE, creationflags=0x08000000)
+
+def get_filepaths(uci,file_extension):
+    files = uci.table('FILES')
+    filepaths = files.loc[(files['FILENAME'].str.endswith(file_extension.lower())) |  (files['FILENAME'].str.endswith(file_extension.upper())),'FILENAME'].to_list()
+    filepaths = [uci.filepath.parent.joinpath(filepath) for filepath in filepaths]
+    return filepaths
+
+
+
+def setup_files(uci,name,n = 5):
+    table = uci.table('FILES',drop_comments = False)
+    for index, row in table.iterrows():
+        filename = Path(row['FILENAME'])
+        if filename.suffix in ['.wdm','.ech','.out']:
+            table.loc[index,'FILENAME'] = filename.name
+        if filename.suffix in ['.hbn']:
+            table.loc[index,'FILENAME'] = filename.name
+        if filename.suffix in ['.plt']:
+            table.drop(index,inplace = True)
+            
+    # Get new binary number and create new BINO rows
+    bino_nums = []
+    invalid = table['UNIT'].values
+    for num in range(15,100):
+        if num not in invalid:
+            bino_nums.append(num)
+        if len(bino_nums) == n:
+            break
+        
+    binary_names = [name + '-' + str(num) + '.hbn' for num in range(len( bino_nums))]
+    rows = [['BINO',bino_num,binary_name,''] for bino_num,binary_name in zip(bino_nums,binary_names)]
+    rows = pd.DataFrame(rows, columns = table.columns).astype({'FTYPE':'string','UNIT':'Int64','FILENAME':'string','comments':'string'} )
+    # Drop old BINO rows and insert new BINO rows
+    table = table.loc[table['FTYPE'] != 'BINO'].reset_index(drop=True)
+    rows = pd.DataFrame(rows, columns = table.columns).astype(table.dtypes) #{'FTYPE':'string','UNIT':'Int64','FILENAME':'string','comments':'string'} )
+    table = pd.concat([table,rows])
+    table.reset_index(drop=True,inplace=True)
+    
+    # Update table in the uci
+    uci.replace_table(table,'FILES')
+    
+
+
+def setup_geninfo(uci):
+    # Initialize Gen-Info
+    bino_nums = uci.table('FILES').set_index('FTYPE').loc['BINO','UNIT'].tolist()
+    if isinstance(bino_nums,int): #Pands is poorly designed. Why would tolist not return a goddamn list...?
+        bino_nums = [bino_nums]
+  
+    #opnids = uci.table(operation,'GEN-INFO').index
+    for operation in ['RCHRES','PERLND','IMPLND']:
+        opnids = np.array_split(uci.table(operation,'GEN-INFO').index.to_list(),len(bino_nums))
+        
+        for opnid,bino_num in zip(opnids,bino_nums):
+            if operation == 'RCHRES': #TODO convert BUNITE to BUNIT1 to get rid of this if statement
+                uci.update_table(bino_num,'RCHRES','GEN-INFO',0,opnids = opnid,columns = 'BUNITE',operator = 'set')
+            else:
+                uci.update_table(bino_num,operation,'GEN-INFO',0,opnids = opnid,columns = 'BUNIT1',operator = 'set')
+
+def setup_binaryinfo(uci,default_output = 4,reach_ids = None):
+    # Initialize Binary-Info
+    uci.update_table(default_output,'PERLND','BINARY-INFO',0,
+                     columns = ['AIRTPR', 'SNOWPR', 'PWATPR', 'SEDPR', 'PSTPR', 'PWGPR', 'PQALPR','MSTLPR', 'PESTPR', 'NITRPR', 'PHOSPR', 'TRACPR'],
+                     operator = 'set')
+    uci.update_table(default_output,'IMPLND','BINARY-INFO',0,
+                     columns = ['ATMPPR', 'SNOWPR', 'IWATPR', 'SLDPR', 'IWGPR', 'IQALPR'],
+                     operator = 'set')
+    uci.update_table(default_output,'RCHRES','BINARY-INFO',0, 
+                     columns = ['HYDRPR', 'ADCAPR', 'CONSPR', 'HEATPR', 'SEDPR', 'GQLPR', 'OXRXPR', 'NUTRPR', 'PLNKPR', 'PHCBPR'],
+                     operator = 'set')
+        
+    uci.update_table(default_output,'PERLND','BINARY-INFO',0,columns = ['SNOWPR','SEDPR','PWATPR','PQALPR'],operator = 'set')
+    uci.update_table(default_output,'IMPLND','BINARY-INFO',0,columns = ['SNOWPR','IWATPR','SLDPR','IQALPR'],operator = 'set')
+    uci.update_table(default_output,'RCHRES','BINARY-INFO',0,columns = ['HYDRPR','SEDPR','HEATPR','OXRXPR','NUTRPR','PLNKPR'],operator = 'set')
+    if reach_ids is not None:
+        uci.update_table(2,'RCHRES','BINARY-INFO',0,columns = ['SEDPR','OXRXPR','NUTRPR','PLNKPR','HEATPR','HYDRPR'],opnids = reach_ids,operator = 'set')
+
+
+def setup_qualid(uci):
+    #### Standardize QUAL-ID Names
+    # Perlands
+    uci.update_table('NH3+NH4','PERLND','QUAL-PROPS',0,columns = 'QUALID',operator = 'set')
+    uci.update_table('NO3','PERLND','QUAL-PROPS',1,columns = 'QUALID',operator = 'set')
+    uci.update_table('ORTHO P','PERLND','QUAL-PROPS',2,columns = 'QUALID',operator = 'set')
+    uci.update_table('BOD','PERLND','QUAL-PROPS',3,columns = 'QUALID',operator = 'set')
+    
+    # Implands
+    uci.update_table('NH3+NH4','IMPLND','QUAL-PROPS',0,columns = 'QUALID',operator = 'set')
+    uci.update_table('NO3','IMPLND','QUAL-PROPS',1,columns = 'QUALID',operator = 'set')
+    uci.update_table('ORTHO P','IMPLND','QUAL-PROPS',2,columns = 'QUALID',operator = 'set')
+    uci.update_table('BOD','IMPLND','QUAL-PROPS',3,columns = 'QUALID',operator = 'set')
+
+
+
+
+def chuck(adjustment,table):
+    # If increasing monthly concentration increase the minimum concnetration value of Mi and Mi+1
+    # If decreasing monthly concentration decrease the maximum concnetration value of Mi and Mi+1
+    # If concnetration values are equal increase both equally
+    table['dummy'] = table.iloc[:,0]
+    zero_table = table.copy()*0
+    count_table = zero_table.copy()
+    for index, value in enumerate(adjustment):
+            next_index = index+1             
+            if value > 1:
+                for row,(a,b) in enumerate(zip(table.iloc[:,index].values, table.iloc[:,next_index].values)):
+                    zero_table.iloc[row,index+np.nanargmin([a,b])] += np.nanmin([a,b])*value
+                    count_table.iloc[row,index+np.nanargmin([a,b])] += 1
+            elif value < 1:
+                for row,(a,b) in enumerate(zip(table.iloc[:,index].values, table.iloc[:,next_index].values)):
+                    zero_table.iloc[row,index+np.nanargmax([a,b])] += np.nanmax([a,b])*value
+                    count_table.iloc[row,index+np.nanargmax([a,b])] += 1
+    
+    
+    zero_table.drop('dummy',axis=1,inplace=True)
+    count_table.drop('dummy',axis=1,inplace=True)
+    
+    zero_table[count_table == 0] = table[count_table==0]
+    count_table[count_table == 0] = 1
+    zero_table = zero_table/count_table
+    return zero_table       
+
+
+
+
+# Expanding opnid-opnid in tables
+def format_opnids(table,valid_opnids):
+    table = table.reset_index()
+    indexes = table.loc[table[~(table['OPNID'] == '')].index,'OPNID']
+    for index, value in indexes.items():
+        try:
+            #table.loc[index,'OPNID'] = int(value[0])
+            int(value)
+        except ValueError:
+            value = value.split()
+            opnids = np.arange(int(value[0]),int(value[1])+1)
+            opnids = [opnid for opnid in opnids if opnid in valid_opnids]
+            if len(opnids) == 0: # incase the x-x mapping covers no valid opnids
+                table.drop(index,inplace = True)
+            else:
+                df = pd.DataFrame([table.loc[index]]*len(opnids))
+                df['OPNID'] = opnids
+                # The insertion method takes advantage of the fact
+                # that Pandas does not automatically reset indexes.
+                table = insert_rows(index,table,df,reset_index = False)
+    
+    
+    #table.loc[table.index[table['OPNID'] == ''],'OPNID'] = pd.NA
+    table['OPNID'] = pd.to_numeric(table['OPNID']).astype('Int64')
+    
+    
+    # Only keep rows that are being simulated    
+    table = table.loc[(table['OPNID'].isin(valid_opnids)) | (table['OPNID'].isna())]
+    table = table.set_index('OPNID',drop = True)
+    return table
+
+def expand_extsources(data,valid_opnids):
+    start_column = 'TOPFST'
+    end_column = 'TOPLST'
+    indexes = data.loc[~data[end_column].isna()]#[[start_column,end_column,'']]
+
+    for index, row in indexes.iterrows():
+        opnids = np.arange(int(row[start_column]),int(row[end_column])+1)
+        opnids = [opnid for opnid in opnids if opnid in valid_opnids[row['TVOL']]]
+
+        if len(opnids) == 0: # incase the x-x mapping covers no valid opnids
+            data.drop(index,inplace = True)
+        else:
+            df = pd.DataFrame([data.loc[index]]*len(opnids))
+            df[start_column] = opnids
+            df[end_column] = pd.NA
+            df = df.astype(data.dtypes.to_dict())
+            # The insertion method takes advantage of the fact
+            # that Pandas does not automatically reset indexes.
+            data = insert_rows(index,data,df,reset_index = False)
+    
+    
+    #table.loc[table.index[table['OPNID'] == ''],'OPNID'] = pd.NA
+    data[start_column] = pd.to_numeric(data[start_column]).astype('Int64')
+    data[end_column] = pd.to_numeric(data[end_column]).astype('Int64')
+    data = data.reset_index(drop = True)
+
+    opnids = sum(list(valid_opnids.values()), []) #Note slow method for collapsing lists but fine for this case
+    data = data.loc[(data['TOPFST'].isin(opnids) )| (data['TOPFST'].isna())]
+    
+    # Only keep rows that are being simulated    
+    for operation in valid_opnids.keys():
+        data = data.drop(data.loc[(data['TVOL'] == operation) & ~(data['TOPFST'].isin(valid_opnids[operation]))].index)
+    
+    return data
+
+
+def insert_rows(insertion_point,a,b,drop = True,reset_index = True):    
+    if drop: a = a.drop(insertion_point)
+    df = pd.concat([a.loc[:insertion_point], b, a.loc[insertion_point:]])
+    if reset_index: df = df.reset_index(drop=True)
+    return df
+    
+
+
+
+def keep_valid_opnids(table,opnid_column,valid_opnids):
+    table = table.reset_index(drop = True)
+    valid_indexes = [table.index[(table[opnid_column].isin(valid_opnids[operation])) & (table['TVOL'] == operation)] for operation in valid_opnids.keys()]
+    valid_indexes.append(table.index[table['comments'] != ''])
+    table = pd.concat([table.loc[valid_index] for valid_index in valid_indexes])
+    table = table.sort_index().reset_index(drop=True)
+    return table
+
+
+def  RUN_comments(lines):
+    # assuems no blank lines (ie lines have been read in using the reader function)
+    comments = []
+    
+    RUN_start = lines.index('RUN')
+    if RUN_start > 0:
+        comment_lines = lines[:RUN_start]
+    else:
+        comment_lines = lines[1:]
+    
+    for line in comment_lines:
+        if '***' in line:
+            comments.append(line)
+        else:
+            if any(c.isalpha() for c in line):
+                break
+    return comments
+
+# Functions for converting the uci text file into a dictionary structure made up of my custom Table class
+def reader(filepath):
+    # simple reader to return non blank, non comment and proper length lines
+    
+    #TODO: Address this encoding issue that seems pretty common across our text files.
+    # It's not a huge deal since we are using ASCII and no information will be lost.
+    with open(filepath, encoding="utf-8",errors="ignore") as fp:
+        
+           lines = []
+           content = fp.readlines()
+           for line in content:
+               if line.strip():
+                   if '***' in line:
+                       lines.append(line.rstrip())
+                   else:
+                       lines.append(line[:80].rstrip())
+    return lines
+                                              
+def decompose_perlands(metzones,landcovers):
+    perlands = {}
+    for metzone in metzones:
+        metzone = int(metzone)
+        for landcover in landcovers:
+            landcover = int(landcover)
+            perlands[metzone+landcover] = (metzone,landcover)
+    return perlands
+
+def split_number(s):
+     head = s.rstrip('0123456789')
+     tail = s[len(head):]
+     return head.strip(), tail
+
+#TODO merge the get_blocks and build_uci into a single function to reduce number of for loops
+def get_blocks(lines):
+    dic = {}
+    shift = len(lines)-1
+    for index,line in enumerate(reversed(lines)):
+        if '***' in line:
+            pass
+        else:
+            line,number = split_number(line.strip()) # Sensitive method to separate numbers
+            line_strip = line.strip() + number
+            if line_strip.startswith('END'):
+                if (line_strip[4:] in parseTable['block'].values): # | (line_strip[4:] in structure['block'].values):
+                    current_name = line_strip[4:]                
+                    dic[current_name] = {}
+                    dic[current_name]['indcs'] = [shift-index]
+                    #names.append(current_name)
+                    #start_indcs.append(shift - index)
+                    #table_id.append(number)
+            elif line_strip == current_name: #line_strip.startswith(current_name):
+                    dic[current_name]['indcs'].append(shift-index)
+                    #end_indcs.append(shift - index)
+    
+    # df = pd.DataFrame([names,table_id,start_indcs,end_indcs]).transpose()
+    # df.columns = ['name','id','start','stop']
+    return dic
+
+def build_uci(lines):
+    blocks = get_blocks(lines)
+    current_name = None
+    keys = []
+    tables = []
+    for k,v in blocks.items():
+        if 'na' in parseTable[parseTable['block']==k]['table'].unique():
+            table = Table(k,'na')
+            table.lines = lines[v['indcs'][1]:v['indcs'][0]+1][1:-1]
+            table.footer = lines[v['indcs'][1]:v['indcs'][0]+1][1]
+            table.header = lines[v['indcs'][1]:v['indcs'][0]+1][-1]
+            table.data = None
+            table.indcs = v['indcs'][1]+1
+            keys.append([k,'na'])
+            tables.append(table)
+        else:
+            #block_lines = lines[v['indcs'][1]+1:v['indcs'][0]]
+            for index,line in enumerate(reversed(lines[v['indcs'][1]+1:v['indcs'][0]])):   
+                if '***' in line:
+                    pass
+                else:
+                    split_line,number = split_number(line.strip()) # Sensitive method to separate numbers
+                    line_strip = split_line.strip()
+                    if line_strip.startswith('END'):
+                        if (line_strip[4:] in parseTable['table'].values) | (line_strip[4:]+number in parseTable['table'].values):
+                            current_name = (line_strip[4:] + number).strip()  
+                            current_name_len = len(current_name)
+                            start = v['indcs'][0]-index
+                        #else: print(line)
+                    elif (line_strip + number).strip()[0:current_name_len] == current_name: #line_strip.startswith(current_name):
+                            end = v['indcs'][0]-index-1
+                            table = Table(k,current_name)
+                            table.lines = lines[end+1:start-1]
+                            table.header = lines[end]
+                            table.footer = lines[start-1]
+                            table.data = None
+                            table.indcs = end+1
+                            
+                            keys.append([k,current_name])
+                            tables.append(table)
+                            current_name = None  
+                            current_name_len = None
+                            
+    # Cumulative count of duplicate key names as some tables appear multiple times within a block
+    #   Since I am looping through the uci file backwards I have to ensure the order of the duplicate
+    #   tables are properly labeled in the correct order they appear from top to bottom in the uci file.          
+    keys.reverse()
+    tables.reverse()
+    # Can't find a base python method for cumulative counting elements. collections.Counter only sums the duplicates
+    table_ids = list(pd.DataFrame(keys).groupby(by=[0,1]).cumcount())
+    ordered_keys = [(key[0],key[1],table_id) for key,table_id in zip(keys,table_ids)]
+    dic = dict(zip(ordered_keys,tables))
+    return dic
+