hspf 2.0.1__tar.gz → 2.0.3__tar.gz

hspf-2.0.3/.gitignore

@@ -0,0 +1,3 @@
+*.pyc
+*.whl
+*.gz

{hspf-2.0.1 → hspf-2.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hspf
-Version: 2.0.1
+Version: 2.0.3
 Summary: Python package for downloading and running HSPF models
 Project-URL: Homepage, https://github.com/mfratkin1/pyHSPF
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>

{hspf-2.0.1 → hspf-2.0.3}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "hspf"
 urls = { "Homepage" = "https://github.com/mfratkin1/pyHSPF" }  # ? Add this!
-version = "2.0.1"
+version = "2.0.3"
 dependencies = [
   "pandas",
   "requests",

{hspf-2.0.1 → hspf-2.0.3}/src/hspf/hbn.py

@@ -12,9 +12,8 @@ import math
 from struct import unpack
 from numpy import fromfile
 from pandas import DataFrame
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta #, timezone
 from collections import defaultdict
-import numpy as np
 #from pathlib import Path
 
 
@@ -350,11 +349,13 @@ class hbnClass:
         rows = []
         times = []
         nvals = len(self.mapn[operation, id, activity]) # number constituent timeseries
-        utc_offset = timezone(timedelta(hours=-6)) #UTC is 6hours ahead of CST
+        #utc_offset = timezone(timedelta(hours=-6)) #UTC is 6hours ahead of CST
         for (index, reclen) in self.mapd[operation, id, activity, tcode]:
             yr, mo, dy, hr, mn = unpack('5I', self.data[index + 36: index + 56])
             hr = hr-1
-            dt = datetime(yr, mo, dy, 0, mn ,tzinfo=utc_offset) + timedelta(hours=hr)
+            #dt = datetime(yr, mo, dy, 0, mn ,tzinfo=utc_offset) + timedelta(hours=hr)
+            dt = datetime(yr, mo, dy, 0, mn ) + timedelta(hours=hr)
+
             times.append(dt)
 
             index += 56

{hspf-2.0.1 → hspf-2.0.3}/src/hspf/parser/graph.py

@@ -17,43 +17,7 @@ class Node(object):
 
     def __str__(self):
         return self._label
-    
-# class PerlndNode(Node):
-#     raise NotImplementedError
-
-# class ReachNode(Node):
-#     raise NotImplementedError
-
-# class ImplndNode(Node):
-#     raise NotImplementedError
-
-# class SourceNode(Node):
-#     raise NotImplementedError
-
-# class TargetNode(Node):
-#     raise NotImplementedError
-
-# class MetNode(Node):
-#     raise NotImplementedError
-
-
-
-# class wdmNode(Node):
-#     raise NotImplementedError
-
-
-# # Add Parameter Nodes Add edges at same time since it's expensive to determine associated plern/implnd/reach node
-# keys = [key for key in uci.uci.keys() if key[0] in ['IMPLND','RCHRES','PERLND']]
-# for operation,table_name,table_id in keys:  
-#     parms = uci.table(operation,table_name,table_id)
-#     for opnid, row in parms.iterrows():
-#         target_node = graph.get_node(G,operation,opnid)
-#         for parameter in row.index:
-#             G.add_node(max(G.nodes) + 1, type = 'Parameter', value = row[parameter], name = parameter, operation = operation, table_name = table_name, table_id = table_id) 
-#             #labels[(operation,parameter,table_id)] = [max(G.nodes)]  
-#             G.add_edge(max(G.nodes), target_node)
-
-    
+   
     
 def create_graph(uci):
     
@@ -101,27 +65,6 @@ def create_graph(uci):
                              tmemsb1 = row['TMEMSB1'],
                              tmemsb2 = row['TMEMSB2'])
         
-    # _ = [G.add_edge(row['snode'],row['tnode'],
-    #                  mlno = row['MLNO'],
-    #                  area = row['AFACTR'],
-    #                  tmemsb1 = row['TMEMSB1'],
-    #                  tmemsb2 = row['TMEMSB2']) for index, row in schematic.iterrows()]
-    
-    
-    
-    #Define edges from Ext Sources
-    # ext_sources['snode'] = ext_sources.index.map(labels)
-    # ext_sources.set_index(['TVOL','TOPFST'],inplace=True)
-    # ext_sources['tnode'] = ext_sources.index.map(labels)
-    # _ = [G.add_edge(row['snode'],row['tnode'],
-    #                  smemn = row['SMEMN'],
-    #                  smemsb = row['SMEMSB'],
-    #                  mfactor = row['MFACTOR'],
-    #                  tran = row['TRAN'],
-    #                  tmemn = row['TMEMN'],
-    #                  tmemsb1 = row['TMEMSB1'],
-    #                  tmemsb2 = row['TMEMSB2']) for index, row in ext_sources.iterrows()]
-    
  
 
     # Add property information
@@ -139,200 +82,10 @@ def create_graph(uci):
         G.nodes[labels[('RCHRES',index)]]['name'] = row['RCHID']
         G.nodes[labels[('RCHRES',index)]]['lkfg'] = row['LKFG']
 
-    
-    # # Add property information
-    # bininfo = uci.table('PERLND','BINARY-INFO')
-    # for index,row in geninfo.iterrows():
-    #     G.nodes[labels[('PERLND',index)]]['name'] = row['LSID']
-        
-    # bininfo = uci.table('IMPLND','BINARY-INFO')
-    # for index,row in geninfo.iterrows():
-    #     G.nodes[labels[('IMPLND',index)]]['name'] = row['LSID']
-        
-    # bininfo = uci.table('RCHRES','BINARY-INFO')
-    # for index,row in geninfo.iterrows():
-    #     G.nodes[labels[('RCHRES',index)]]['name'] = row['RCHID']
-    #     G.nodes[labels[('RCHRES',index)]]['lkfg'] = row['LKFG']
-
-
-    # labels = {}
-    # for n, d in G.nodes(data=True):
-    #     l = (d['operation'],d['opnid'])
-    #     labels[l] = labels.get(l, [])
-    #     labels[l].append(n)
-
-
-
     G.labels = labels
     return G
 
 
-# def create_subgraph(G,start_node):
-#     sub_G = nx.MultDiGraph()
-#     for n in G.successors_iter(start_node):
-#         sub_G.add_path([start_node,n])
-#         create_subgraph(G,sub_G,n)
-
-# Binary info
-
-"""
-CREATE TABLE Operation (
-    opn VARCHAR,
-    opnid INTEGER,
-    PRIMARY KEY (opn, opnid)
-
- 
-    
- )
-
-
-
-
-"""
-
-
-
-
-"""
-CREATE TABLE Files (
-    ftype VARCHAR,
-    unit INTEGER NOT NULL PRIMARY KEY,,
-    filename VARCHAR
-);
-
-"""
-
-
-"""
-CREATE TABLE GenInfo (
-    pk INTEGER NOT NULL,
-    opn VARCHAR,
-    opnid INTEGER,
-    PRIMARY KEY (opn, opnid)
-    iunits INTEGER,
-    ounits INTEGER,
-    punit1 INTEGER,
-    punit2 INTEGER,
-    BUNIT1 INTEGER,
-    BUNIT2 INTEGER
-);
-
-"""
-
-# # Files
-# files = uci.table('FILES')
-# files['FTYPE'] = files['FTYPE'].replace({'WDM': 'WDM1'})
-# dfs = []
-
-# # PerlndInfo
-# operation = 'PERLND'
-# geninfo = uci.table(operation,'GEN-INFO')
-# binaryinfo = uci.table(operation,'BINARY-INFO')
-# if operation == 'RCHRES':
-#     geninfo = geninfo.rename(columns = {'RCHID':'LSID',
-#                                         'BUNITE':'BUNIT1',
-#                                         'BUNITM': 'BUNIT2',
-#                                         'PUNITE': 'PUNIT1',
-#                                         'PUNITM': 'PUNIT2'})
-# df = pd.merge(geninfo,binaryinfo, left_index = True, right_index = True, how = 'outer').reset_index()
-# df.insert(0,'OPN',operation)
-# df = pd.merge(df,files, left_on = 'BUNIT1', right_on = 'UNIT')
-
-# # Schematic Table
-# schematic = uci.table('SCHEMATIC')
-
-# # Masslink Table
-# masslinks = []
-# for table_name in uci.table_names('MASS-LINK'):
-#     mlno = table_name.split('MASS-LINK')[1]
-#     masslink = uci.table('MASS-LINK',table_name)
-#     masslink.insert(0,'MLNO',mlno)
-#     masslinks.append(masslink)
-# masslinks = pd.concat(masslinks)
-
-# #masslinks['QUALID'] = (masslinks['SMEMSB1'].str.strip().replace('','0').astype(int)-1).replace(-1,pd.NA)
-
-
-
-# hbn_name = uci.table('PERLND','QUAL-PROPS', int(row['SMEMSB1']) - 1).iloc[0]['QUALID']
-
-# operation = row['SVOL']
-# activity = row['SGRPN']
-# ts_name = row['SMEMN']
-
-# hbn_name = row['SMEMN'] + hbn_name
-
-
-# schematic = uci.table('SCHEMATIC')
-# schematic = pd.merge(schematic,masslinks,left_on = 'MLNO',right_on = 'MLNO')
-
-# all(schematic['SVOL_x'] == schematic['SVOL_y'])
-# all(schematic['TVOL_x'] == schematic['TVOL_y'])
-# schematic.loc[schematic['TMEMSB1_x'] == '', 'TMEMSB1_y'] = schematic['TMEMSB1_x']
-# schematic.loc[schematic['TMEMSB2_x'] == '', 'TMEMSB2_y'] = schematic['TMEMSB2_x']
-
-# schematic = schematic.drop(columns=['TMEMSB1_y','TMEMSB2_y','TVOL_y','SVOL_y'])
-# schematic = schematic.rename(columns = {'SVOL_x':'SVOL',
-#                               'TVOL_x':'TVOL',
-#                               'TMEMSB2_x':'TMEMSB2',
-#                               'TMEMSB1_x':'TMEMSB1'})
-
-
-
-# # Watershed Weighted Mean 
-# subwatersheds = uci.network.subwatersheds()
-# subwatersheds = subwatersheds.loc[subwatersheds['SVOL'] == 'PERLND'].reset_index()
-# df = cal.model.hbns.get_multiple_timeseries('PERLND',5,'PERO',test['SVOLNO'].values).mean().reset_index()
-# df.columns = ['OPNID','value']
-# weighted_mean = df[['value','AFACTR']].groupby(df['LSID']).apply(lambda x: (x['value'] * x['AFACTR']).sum() / x['AFACTR'].sum())
-# weighted_mean.loc['combined'] =  (df['value'] * df['AFACTR']).sum() / df['AFACTR'].sum()
-
-
-# # annual weighted timeseries watershed
-# reach_ids = [103,119,104,118]
-# subwatersheds = uci.network.subwatersheds().loc[reach_ids]
-# subwatersheds = subwatersheds.loc[subwatersheds['SVOL'] == 'PERLND'].reset_index()
-# df = cal.model.hbns.get_multiple_timeseries('PERLND',5,'PERO',test['SVOLNO'].values).mean().reset_index()
-# df.columns = ['OPNID','value']
-# df = pd.merge(subwatersheds,df,left_on = 'SVOLNO', right_on='OPNID')
-# weighted_mean = (df['value'] * df['AFACTR']).sum() / df['AFACTR'].sum()
-# df[f'weighted_{ts_name}'] = df.groupby('LSID')[parameter].transform(lambda x: (x * df.loc[x.index, 'AFACTR']).sum() / df.loc[x.index, 'AFACTR'].sum())
-# weighted_mean.loc['combined'] =  (df['value'] * df['AFACTR']).sum() / df['AFACTR'].sum()
-
-
-
-# # parameter average weighted by landcover area
-# table_name = 'PWAT-PARM2'
-# parameter = 'LZSN'
-# table_id = 0
-# operation = 'PERLND'
-
-# subwatersheds = uci.network.subwatersheds()
-# subwatersheds = subwatersheds.loc[subwatersheds['SVOL'] == 'PERLND'].reset_index()
-# df = uci.table(operation,table_name,table_id)[parameter].reset_index()
-# df.columns = ['OPNID',parameter]
-# df = pd.merge(subwatersheds,df,left_on = 'SVOLNO', right_on='OPNID')
-# df[f'weighted_{parameter}'] = df.groupby('TVOLNO')[parameter].transform(lambda x: (x * df.loc[x.index, 'AFACTR']).sum() / df.loc[x.index, 'AFACTR'].sum())
-
-
-# #df[f'weighted_{parameter}'] = df.groupby('LSID')[parameter].transform(lambda x: (x * df.loc[x.index, 'AFACTR']).sum() / df.loc[x.index, 'AFACTR'].sum())
-
-
-
-
-# extsources = uci.table('EXT SOURCES')
-# extsources['SVOL'] = extsources['SVOL'].replace({'WDM': 'WDM1'})
-
-# df = pd.merge(extsources,df,left_on = 'SVOL',right_on = 'FTYPE',how = 'right')
-
-
-# exttargets = uci.table('EXT TARGETS')
-# schematic = uci.table('SCHEMATIC')
-
-
-#%% Methods using universal node id
-
-
 def _add_subgraph_labels(G,G_sub):
     G_sub.labels = {label:node for label, node in G.labels.items() if node in G_sub.nodes}
     return G_sub
@@ -417,17 +170,19 @@ def nodes(G,node_type,node_type_id,adjacent_node_type):
 #%% Methods using node_type, node_type_id interface
 
 def upstream_network(G,reach_id):
-    return G.subgraph(nx.ancestors(G,get_node_id(G,'RCHRES',reach_id))).copy()
+    node_id = get_node_id(G,'RCHRES',reach_id)
+    return G.subgraph([node_id] + list(nx.ancestors(G,node_id))).copy()
 
 def downstream_network(G,reach_id):
-    return G.subgraph(nx.descendants(G,get_node_id(G,'RCHRES',reach_id))).copy()
+    node_id = get_node_id(G,'RCHRES',reach_id)
+    return G.subgraph([node_id] + list(nx.descendants(G,node_id))).copy()
 
 def subset_network(G,reach_id,upstream_reach_ids = None):
     G = upstream_network(G,reach_id)
     if upstream_reach_ids is not None:
-        [G.remove_nodes_from(nx.ancestors(G,upstream_reach_id)) for upstream_reach_id in upstream_reach_ids if upstream_reach_id in G.nodes]
-        [G.remove_nodes_from([upstream_reach_id]) for upstream_reach_id in upstream_reach_ids if upstream_reach_id in G.nodes]
-    #assert([len(sinks(G)) == 0,sinks(G)[0] == reach_id])
+        for upstream_reach_id in upstream_reach_ids:
+            G.remove_nodes_from(get_node_ids(upstream_network(G,upstream_reach_id),'RCHRES'))
+        #assert([len(sinks(G)) == 0,sinks(G)[0] == reach_id])
     return G
 
 def upstream_nodes(G,reach_id,upstream_node_type):
@@ -519,7 +274,24 @@ def paths(G,reach_id,source_type = 'RCHRES'):
 def count_ancestors(G,node_type,ancestor_node_type):
     return {node['type_id']:len(ancestors(G,node['id'],ancestor_node_type)) for node in get_nodes(G,node_type)}
 
-
+# def catchment_ids(G):
+#     result = []
+#     for node in get_node_ids(G,'RCHRES'):
+#         upstream_nodes = G.predecessors(node)
+#         if any([G.nodes[up]['type'] in ['PERLND','IMPLND'] for up in upstream_nodes]):
+#             result.append(G.nodes[node]['type_id'])
+#     return result
+
+# Very expensive. Should probably standardize it so routing reaches have no implnds/perlnds
+def catchment_ids(G):
+    result = []
+    for node in get_node_ids(G,'RCHRES'):
+        upstream_nodes = G.predecessors(node)
+        if any([G.nodes[up]['type'] in ['PERLND','IMPLND'] for up in upstream_nodes]):
+            cat = make_catchment(G,G.nodes[node]['type_id'])
+            if area(cat) > 0:
+                result.append(G.nodes[node]['type_id'])
+    return result
 
 # Catchment constructor
 def make_catchment(G,reach_id):
@@ -550,8 +322,28 @@ def make_watershed(G,reach_ids):
     return watershed
 
 
-def catcments(G):
-    return None
+# def catcments(G):
+#     cats = [Catchment(graph.make_catchment(G,reach_id) for reach_id in graph.get_node_type_ids(G,'RCHRES'))]
+
+#     return 
+        
+#         for u, v, edge_data in graph.make_catchment(G,reach_id).edges(data=True):
+#             source_node_attributes = G.nodes[u]
+#             # Add or update edge attributes with source node attributes
+#             edge_data["source_type"] = source_node_attributes.get("type")
+#             edge_data["source_name"] = source_node_attributes.get("name")
+#             edge_data["source_type_id"] = source_node_attributes.get("type_id")
+#             cats.append(edge_data)
+        
+#     return pd.DataFrame(cats)
+                     
+                     
+#                      for node in G.nodes:
+#         upstream_nodes = G.predecessors(node)
+#         if any(G.nodes[up]['type'] in ['PELND','IMPLND'] for up in upstream_nodes):
+#             result.append(node)
+
+#     return None
 # Catchment selectors
 
 '''
@@ -637,9 +429,12 @@ class Catchment():
 #%% Legacy Methods for Backwards compatability
 class reachNetwork():
     def __init__(self,uci,reach_id = None):
+        self.uci = uci
         self.G = create_graph(uci)
+        self.catchment_ids = catchment_ids(self.G)
+        self.routing_reaches = self._routing_reaches()
+        self.lakes = self._lakes()
         self.schematic = uci.table('SCHEMATIC').astype({'TVOLNO': int, "SVOLNO": int, 'AFACTR':float})
-        self.uci = uci
     
     def get_node_type_ids(self,node_type):
         return get_node_type_ids(self.G, node_type)
@@ -709,15 +504,18 @@ class reachNetwork():
     
     def subwatersheds(self,reach_ids = None):
         df = subwatersheds(self.uci)
-        if reach_ids is not None:
-            df = df.loc[df.index.intersection(reach_ids)]
-        return df
+        if reach_ids is None:
+            reach_ids = get_node_type_ids(self.G,'RCHRES')
+        return df.loc[df.index.intersection(reach_ids)]
     
     def subwatershed(self,reach_id):
         return subwatershed(self.uci,reach_id) #.loc[reach_id]
     
     def subwatershed_area(self,reach_id):
-        return self.drainage(reach_id).query("source_type in ['PERLND','IMPLND']")['area'].sum()
+        area = self.drainage(reach_id).query("source_type in ['PERLND','IMPLND']")['area'].sum()
+        # if (reach_id in self.lakes()) & (f'FTABLE{reach_id}' in self.uci.table_names('FTABLES')):
+        #     area = area + self.lake_area(reach_id)
+        return area
     
     def reach_contributions(self,operation,opnids):
         return reach_contributions(self.uci,operation,opnids)
@@ -736,6 +534,15 @@ class reachNetwork():
     def outlets(self):
         return [self.G.nodes[node]['type_id'] for node, out_degree in self.G.out_degree() if (out_degree == 0) & (self.G.nodes[node]['type'] == 'RCHRES')]
 
+    def _lakes(self):
+        return list(self.uci.table('RCHRES','GEN-INFO').query('LKFG == 1',engine = 'python').index.astype(int))        
+    
+    def lake_area(self,reach_id):
+        return self.uci.table('FTABLES',f'FTABLE{reach_id}')['Area'].max()
+    
+    def _routing_reaches(self):
+        return [reach_id for reach_id in self.get_node_type_ids('RCHRES') if reach_id not in self.catchment_ids]
+
     def paths(self,reach_id):
         return paths(self.G,reach_id)
     
@@ -761,6 +568,7 @@ def calibration_order(G,reach_id,upstream_reach_ids = None):
 
 def get_opnids(G,operation,reach_id = None, upstream_reach_ids = None):
     G = subset_network(G,reach_id,upstream_reach_ids)
+    return ancestors(G,get_node_id(G,'RCHRES',reach_id),operation)
     perlnds = [node['type_id'] for node in get_nodes(G,'PERLND')]
     implnds = [node['type_id'] for node in get_nodes(G,'IMPLND')]
     reachs = [node['type_id'] for node in get_nodes(G,'RCHRES')]
@@ -799,6 +607,7 @@ def subwatersheds(uci):
     schematic = schematic[schematic['TVOL'] == 'RCHRES'][['SVOLNO','TVOLNO','AFACTR','MLNO']].astype({'SVOLNO':int,'TVOLNO':int,'AFACTR':float,'MLNO':int})
     schematic.reset_index(inplace=True,drop=False)
     schematic.set_index('TVOLNO',inplace=True)
+    schematic = schematic.loc[catchment_ids(uci.network.G)]
     
     dfs = []
     for operation in ['PERLND','IMPLND']:

{hspf-2.0.1 → hspf-2.0.3}/src/hspf/parser/parsers.py

@@ -52,7 +52,10 @@ class Table():
     
         self.parser = parserSelector[self.block]
         #self.updater = Updater
-        
+    
+    def _delimiters(self):
+        return delimiters(self.block,self.name)
+    
     def parse(self):
         self.data = self.parser.parse(self.block,self.name,self.lines)
         

{hspf-2.0.1 → hspf-2.0.3}/src/hspf/uci.py

@@ -78,7 +78,7 @@ class UCI():
                  
     def table(self,block,table_name = 'na',table_id = 0,drop_comments = True):
         # Dynamic parsing of tables when called by user
-        assert block in ['FILES','PERLND','IMPLND','RCHRES','SCHEMATIC','OPN SEQUENCE','MASS-LINK','EXT SOURCES','NETWORK','GENER','MONTH-DATA','EXT TARGETS','COPY']
+        assert block in ['FILES','PERLND','IMPLND','RCHRES','SCHEMATIC','OPN SEQUENCE','MASS-LINK','EXT SOURCES','NETWORK','GENER','MONTH-DATA','EXT TARGETS','COPY','FTABLES']
         
         table = self.uci[(block,table_name,table_id)] #[block][table_name][table_id]
         #TODO move the format_opnids into the Table class?
@@ -176,13 +176,40 @@ class UCI():
             lines += ['']
         lines += ['END RUN']
         self.lines = lines       
+
+
+    def _write(self,filepath):
+        with open(filepath, 'w') as the_file:
+            for line in self.lines:    
+                the_file.write(line+'\n')
+
+    def add_parameter_template(self,block,table_name,table_id,parameter,tpl_char = '~'):
         
-    
+        table = self.table(block,table_name,0,False).reset_index()
+        column_names,dtypes,starts,stops = self.uci[(block,table_name,table_id)]._delimiters()
+        
+        width = stops[column_names.index(parameter)] - starts[column_names.index(parameter)]
+
+        ids = ~table[parameter].isna() # Handle comment lines in uci
+
+        # Replace paramter name with PEST/PEST++ specification. Note this does not use the HSPF supplemental file so parameters are limited to width of uci file column
+        pest_param = tpl_char + parameter.lower() +  table.loc[ids,'OPNID'].astype(str)
+        pest_param = pest_param.apply(lambda name: name + ' '*(width-len(name)-1)+ tpl_char)
+
+        table.loc[ids,parameter] = pest_param
+        table = table.set_index('OPNID')
+        self.replace_table(table,block,table_name,table_id)
+
+    def write_tpl(self,tpl_char = '~',new_tpl_path = None):    
+        if new_tpl_path is None:
+            new_tpl_path = self.filepath.parent.joinpath(self.filepath.stem + '.tpl')
+        self.merge_lines()
+        self.lines.insert(0,tpl_char)
+        self._write(new_tpl_path)
+
     def write(self,new_uci_path):
         self.merge_lines()
-        with open(new_uci_path, 'w') as the_file:
-            for line in self.lines:    
-                the_file.write(line+'\n')   
+        self._write(new_uci_path) 
 
     def update_bino(self,name):
         #TODO: Move up to busniess/presentation layer