pyhcal 1.1.1__tar.gz → 1.1.3__tar.gz

{pyhcal-1.1.1 → pyhcal-1.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyhcal
-Version: 1.1.1
+Version: 1.1.3
 Summary: Python package for calibrating MPCA HSPF models
 Project-URL: Homepage, https://github.com/mfratkin1/pyhcal
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>

{pyhcal-1.1.1 → pyhcal-1.1.3}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "pyhcal"
 urls = { "Homepage" = "https://github.com/mfratkin1/pyhcal" }  # ? Add this!
-version = "1.1.1"
+version = "1.1.3"
 dependencies = [
   "hspf",
   "mpcaHydro",

{pyhcal-1.1.1 → pyhcal-1.1.3}/src/pyhcal/calibrators.py

@@ -38,6 +38,7 @@ def validate_project_folder(project_folder):
     assert project_path.joinpath('data').exists(), 'Data folder does not exist'
     assert project_path.joinpath('output').exists(), 'Output folder does not exist'
     assert project_path.joinpath('targets.csv').exists(), 'targets.csv file does not exist in project folder'
+    assert project_path.joinpath('gis').exists(), 'GIS folder does not exist'
     return True
  
 class calibrator:
@@ -56,10 +57,13 @@ class calibrator:
      
         self.targets = None
         if self.project_path.joinpath('targets.csv').exists():
-            self.targets = pd.read_csv(self.project_path.joinpath('targets.csv'))  
-              
+            self._load_targets()
+
         self.MODL_DB = pd.read_csv(self.project_path.joinpath('_'.join([self.project_name ,'MODL_DB.csv'])))
         
+        if 'repo_name' in self.MODL_DB.columns:
+            self.model_name =self.MODL_DB['repo_name'].to_list()[0]
+            
         # Alot of effort to try and include the subwatershed gdf if it exists. TODO: refactor
         self.subwatershed_gdf_filepath = self.gis_path.joinpath('_'.join([self.project_name ,'Subwatersheds.shp']))
         if self.subwatershed_gdf_filepath.exists():
@@ -98,6 +102,9 @@ class calibrator:
         else:
             self.subwatershed_gdf = None
 
+    def _load_targets(self):
+        self.targets = pd.read_csv(self.project_path.joinpath('targets.csv'))
+
     def _update_mapper(self):
         if self.subwatershed_gdf is not None:
             self.mapper = Mapper(self.project_name,self.uci,self.subwatershed_gdf,hbn = self.model.hbns)
@@ -145,14 +152,24 @@ class calibrator:
         #winHSPF = str(Path(__file__).resolve().parent.parent) + '\\bin\\WinHSPFLt\\WinHspfLt.exe'     
         subprocess.run([self.model.winHSPF,uci_file]) #, stdout=subprocess.PIPE, creationflags=0x08000000)
     
-
+    def get_outlets(self):
+        df = self.dm.get_outlets(self.model_name)
+        outlets = {}
+        for outlet_id in df['outlet_id'].unique():
+            outlets[int(outlet_id)] = {}
+            df_outlet = df.loc[df['outlet_id'] == outlet_id]
+            outlets[int(outlet_id)]['station_ids'] = list(set(df_outlet['station_id']))
+            outlets[int(outlet_id)]['reach_ids'] = list(set(df_outlet['reach_id']))
+            outlets[int(outlet_id)]['model_name'] = df_outlet['repository_name'].iloc[0]
+        return outlets
+    
     def get_simulated_output(self,reach_ids,constituent,time_step = 'YE'):
         sim = self.model.hbns.get_reach_constituent(constituent,reach_ids,time_step)
         sim.name = 'simulated'
         return sim
     
     def get_observed_data(self,station_ids,constituent,time_step = 'YE',baseflow_percentage = None):
-        obs = self.dm.get_station_data(station_ids,constituent,agg_period = time_step)['observed'].sort_index(level = 'index')
+        obs = self.dm.get_observation_data(station_ids,constituent,agg_period = time_step)['observed'].sort_index(level = 'index')
         obs.name = 'observed'
         return obs
      
@@ -560,110 +577,3 @@ def threshold(adjustment,threshold,max_change):
     #Note that in uci.update_table() there is further screening to account for adjustments below the model precision
     return adjustment  
 
-
-#class hydrologyCalibrator(calibrator):
-    
-#class nutrientCalibrator(calibrator):
-        
-class sedimentCalibrator(calibrator):
-    
-    def update_kser(self,method,opnid = None):   
-        #TODO account for the additional comment column
-        assert method in ['load','landcover','sftl']
-        
-        table = self.uci.table('PERLND','SED-PARM3',0,False)
-        
-        
-        if method == 'load':
-            adjustment = self.compare(0,aggregate = True).loc['Mean']['ratio']      
-        elif method == 'landcover':
-            adjustment = self.landcover(0)['target']
-            table = self.uci.table('PERLND','SED-PARM3',0)
-            if opnid == None:
-                opnid = table.index
-            adjustment = np.array(adjustment.loc[opnid])[:,None]
-        elif method == 'sftl':
-            adjustment = self.sftl()
-        
-        self.uci.replace_table('PERLND','SED-PARM3',0)
-    
-    def update_erosivity(self,param = 'M',opnid = None,update_alg = '*'):
-        adjustment = self.scour()  
-        table = self.uci.table('RCHRES','SILT-CLAY-PM',0)
-        if opnid == None:
-            opnid = table.index
-        adjustment = np.array(adjustment.loc[opnid])[:,None]
-        self.uci.update_table(adjustment,'RCHRES','SILT-CLAY-PM',table_id = 0,opnid = opnid,columns = [param],update_alg = update_alg)
-        
-        adjustment = self.scour()
-        adjustment = np.array(adjustment.loc[opnid])[:,None]
-        self.uci.update_table(adjustment,'RCHRES','SILT-CLAY-PM',table_id = 1,opnid = opnid,columns = [param],update_alg = update_alg) 
-    
-        
-    def fit_param(self,param,m_factor,N = 2,opnid = None,run = None):
-        bounds = {'M':[.000000001,.01,2,5], #maxlow,low,high,maxhigh
-                  'TAUCD':[.001,.01,.3,1],
-                  'TAUCS':[.01,.05,.5,3]}
-        if run == None:
-            run = self.run
-        
-        data = self.load_data('scour',N=10000)
-        data = data.loc[:,range(run-N+1,run+1),:]
-    
-        if opnid == None:
-            opnid = data.reset_index(level=[1]).index.unique() # assumes multiindex
-        
-        for index in opnid:
-            if any(data.loc[index]['LKFG'] == 0):
-                x = data.loc[index]['depscour']
-                y = data.loc[index][param]
-                linear_model=np.polyfit(x,y,1)
-                linear_model_fn=np.poly1d(linear_model)
-                m = linear_model_fn(-data.loc[index]['nonpoint'].iloc[1]*.25)
-                if m < bounds[param][0]:
-                    m = bounds[param][0]
-                if m > bounds[param][3]:
-                    m = bounds[param][3]
-                self.update_table('RCHRES','SILT-CLAY-PM',0,m,'set',opnid = index,columns = [param]) #mod.update_table(operation,table_name,table_id,adjustment,operator,opnids,columns)
-                self.update_table('RCHRES','SILT-CLAY-PM',1,m*m_factor,'set',opnid = index,columns = [param]) #mod.update_table(operation,table_name,table_id,adjustment,operator,opnids,columns)
-    
-    def erosivity(self,m_factor,param = 'M',opnid = None,run = None,iterations = 1):
-        
-        if run == None:
-            run = self.run
-        
-        # run model updating erosivity for N iterations
-        for iteration in range(iterations):
-            self.update_erosivity(param = param,opnid = opnid)
-            self.run_model() # creates the run+1 uci file and runs it using WinHspfLT
-            run = run + 1
-            self.load_model(run)
-            self.save_data()
-     
-    
-        self.fit_param(param,m_factor,iterations+1,opnid,run)
-        self.run_model() # creates the run+1 uci file and runs it using WinHspfLT
-        
-        run = run + 1
-        self.load_model(run)
-        self.save_data()
-
-    def scour(hbn,uci):     
-        # Erosivity adjustment only
-        scour  = reports.scour_report(hbn,uci)
-        #TODO: add check for this
-        # Assume all nonpoint values are greater than 0...
-        # if depscour is greater than 0
-        target = scour['nonpoint']*.25 # Assuming nonpoint load is set
-        adjustment = np.abs(scour['depscour'])/target
-        adjustment[(adjustment < 1.05) & (adjustment > .95)] = 1 # Don't change reaches where the depscour is close to the target
-        adjustment[adjustment > 1.05] = .95 # Since depscour is negative we have to swap this. I think if I do target/depscour this line would be less confusing
-        adjustment[adjustment < .95] = 1.05
-        adjustment[scour['depscour'] > 0] = 2 # Double any values where the depscour is positive
-        adjustment[scour['LKFG'] == 1] = 1  # Ignore lake flags
-        adjustment[np.isnan(adjustment)] = 1
-        
-        return adjustment
-
-
-    

{pyhcal-1.1.1 → pyhcal-1.1.3}/src/pyhcal/mappers.py

@@ -101,5 +101,13 @@ class Mapper():
         self.subwatershed_gdf.join(subwatersheds).plot(column = output_name,ax = ax,cmap='viridis',legend=True)
         plt.title(output_name)
     
+    def map_table(self,df, mapping_col):
+        '''Maps a dataframe column to the subwatershed geodataframe based on subwatershed IDs.
+        Assumes the dataframe index contains the subwatershed IDs.'''
+        fig, ax = plt.subplots()
+        #[table.plot(column = parameter,ax = ax) for table in tables]
+        self.subwatershed_gdf.join(df).plot(column = mapping_col,ax = ax,cmap='viridis',legend=True)
+        plt.title(mapping_col)
         #return self.subwatershed_gdf.join(subwatersheds)
 
+ 

{pyhcal-1.1.1 → pyhcal-1.1.3}/src/pyhcal/repository.py

@@ -6,7 +6,7 @@ Created on Wed Nov 27 09:16:30 2024
 """
 
 import pandas as pd
-from pyhcal import modl_db
+from mpcaHydro import outlets
 from pathlib import Path
 import shutil
 
@@ -38,7 +38,7 @@ class Repository():
         
         self.REPOSITORY_PATH = repository_path
         huc_directory = self.HUC_DIRECTORY.loc[self.HUC_DIRECTORY['Repository_HUC8 Name'] == model_name]
-        self.modl_db = modl_db.get_model_db(model_name) #self.MODL_DB.loc[self.MODL_DB['repository_name'] == model_name]
+        self.modl_db = outlets.get_model_db(model_name) #self.MODL_DB.loc[self.MODL_DB['repository_name'] == model_name]
         #self.modl_db  = pd.concat([self.MODL_DB.loc[self.MODL_DB['repository_name'].str.startswith(huc8_id,na=False)] for huc8_id in huc8_ids])        
         self.model_name = model_name
         self.huc8_ids = list(huc_directory['USGS HUC-8'])
@@ -48,8 +48,8 @@ class Repository():
         self.uci_file = self.repo_folder.joinpath('HSPF','.'.join([self.model_name,'uci']))
         self.wdm_files = [item for item in self.repo_folder.joinpath('HSPF').iterdir() if (item.name.endswith('.wdm')) | (item.name.endswith('.WDM'))]
         self.shapefiles = {item.name.split('.')[0].split('_')[-1]:item for item in self.repo_folder.joinpath('GIS').iterdir() if (item.name.endswith('.shp')) | (item.name.endswith('.SHP'))}
-        self.wiski_stations = modl_db.wiski_stations(model_name)
-        self.equis_stations = modl_db.equis_stations(model_name)
+        self.wiski_stations = outlets.wiski_stations(model_name)
+        self.equis_stations = outlets.equis_stations(model_name)
 
 
     def copy(self,copy_path):

{pyhcal-1.1.1 → pyhcal-1.1.3}/src/pyhcal/setup_utils.py

@@ -7,8 +7,9 @@ Created on Wed Jun 15 15:21:35 2022
 from mpcaHydro.data_manager import dataManager
 from hspf.wdmReader import readWDM
 from hspf.uci import UCI
+from hpsf.hspfModel import hspfModel
 from pyhcal.repository import Repository
-from pyhcal import modl_db
+from mpcaHydro import outlets
 
 import numpy as np
 import pandas as pd
@@ -16,8 +17,8 @@ from pathlib import Path
 import subprocess
 
 
-def create_calibration_project(model_name,project_location,download_station_data = True,run_model = True,convert_wdms = True):
-    project = Builder(model_name)
+def create_calibration_project(model_name,project_location, download_station_data = True,run_model = True,convert_wdms = True,oracle_username = None, oracle_password = None):
+    project = Builder(model_name,oracle_username = oracle_username, oracle_password = oracle_password)
     project.copy(project_location,model_name)
     project.load_uci()
     project.format_uci()
@@ -25,6 +26,7 @@ def create_calibration_project(model_name,project_location,download_station_data
     if convert_wdms: project.convert_wdms()
     if download_station_data: project.download_station_data()
     if run_model: project.run_model()
+    return project
    
     
     
@@ -32,32 +34,34 @@ def create_calibration_project(model_name,project_location,download_station_data
 
 class Builder():
     
-    def __init__(self,model_name):
+    def __init__(self,model_name,oracle_username = None, oracle_password = None):
         self.repository = Repository(model_name)
+        self.model_name = model_name
         self.project_path = None
         self.project_name = None
         self.new_uci = None
         self.uci = None
         self.dm = None
-        self.calibration_reaches = modl_db.wplmn_station_opnids(model_name)
-
+        self.calibration_reaches = outlets.wplmn_station_opnids(model_name)
+        self.oracle_username = oracle_username
+        self.oracle_password = oracle_password
     
     def valid_models():
         return Repository.valid_models()
     
-    def set_project_path(self,project_path):
-        self.project_path = Path(project_path)
-        self.project_name = Path(project_path).name
+    def set_project_path(self,project_location,project_name):
+        self.project_path = Path(project_location).joinpath(project_name)
+        self.project_name = project_name
+        self.dm = dataManager(self.project_path.joinpath('data'),oracle_username = self.oracle_username, oracle_password = self.oracle_password)
+        self.dm._build_warehouse()
         #self.new_uci = self.project_path.joinpath('model','_'.join([self.project_name,'0.uci']))
         #self.uci = UCI(self.project_path.joinpath('model','.'.join([self.project_name,'uci'])))
         
     def copy(self,project_location,project_name):
-        self.project_path = Path(project_location).joinpath(project_name)
-        self.project_name = project_name
+        self.set_project_path(project_location,project_name)
         self.repository.copy(self.project_path)
-        self.dm = dataManager(self.project_path.joinpath('data'))
+        
 
-    
     def load_uci(self):
         self.new_uci = self.project_path.joinpath('model','_'.join([self.project_name,'0.uci']))
         self.uci = UCI(self.project_path.joinpath('model','.'.join([self.project_name,'uci'])))
@@ -80,21 +84,27 @@ class Builder():
         setup_qualid(self.uci)
         self.uci.write(self.new_uci)
         
-    
-    def download_station_data(self,start_year,end_year):#:,station_id,source_id):
-        #%% Old approach. Store as indvidual processed station files then load to warehouse
-        if len(equis_stations) > 0:
-            if self.dm.credentials_exist():
-                station_origin = 'equis'
-            else: station_origin = 'swd'
-        
-            for station_id in equis_stations:
-                self.dm.download_station_data(station_id,station_origin, True)
+    def download_wiski_data(self,station_ids):
+        if len(station_ids) > 0:
+            self.dm._download_wiski_data(station_ids)
+        else:
+            print("No Wiski stations have been manually matched to modeled reaches.")
         
-        if len(wiski_stations) > 0:
-            for station_id in wiski_stations:
-                self.dm.download_station_data(station_id,'wiski', True)
-                 
+    def download_equis_data(self,station_ids):
+        if len(station_ids) > 0:
+            if self.dm.credentials_exist():
+                self.dm.connect_to_oracle()
+                self.dm._download_equis_data(station_ids)
+            else:
+                print("Oracle credentials not provided. Cannot download Equis data.")
+        else:
+            print("No Equis stations have been manually matched to modeled reaches.")
+    
+    def download_station_data(self):
+        equis_stations = self.dm.outlets.mapped_equis_stations(self.model_name)
+        wiski_stations = self.dm.outlets.mapped_wiski_stations(self.model_name)
+        self.download_equis_data(equis_stations)
+        self.download_wiski_data(wiski_stations)
     
     def convert_wdms(self):
         copy_path = Path(self.project_path.joinpath('model'))
@@ -104,32 +114,10 @@ class Builder():
                     copy_path.joinpath(wdm_file.name.replace('.wdm','.hdf5').replace('.WDM','hdf5')))
         
     def run_model(self, wait_for_completion=True):
-        #run_model(self.new_uci, wait_for_completion)
         # Run the uci file
-        winHSPF = str(Path(__file__).resolve().parent.parent) + '\\bin\\WinHSPFLt\\WinHspfLt.exe'     
+        winHSPF = hspfModel.winHSPF #TODO: fix this hardcoding
         subprocess.run([winHSPF,self.new_uci]) #, stdout=subprocess.PIPE, creationflags=0x08000000)    
         
-def run_model(uci_file, wait_for_completion=True):
-    winHSPF = str(Path(__file__).resolve().parent.parent) + '\\bin\\WinHSPFlt\\WinHspfLt.exe'
-    
-    # Arguments for the subprocess
-    args = [winHSPF, uci_file.as_posix()]
-
-    if wait_for_completion:
-        # Use subprocess.run to wait for the process to complete (original behavior)
-        subprocess.run(args)
-    else:
-        # Use subprocess.Popen to run the process in the background without waiting
-        # On Windows, you can use creationflags to prevent a console window from appearing
-        if sys.platform.startswith('win'):
-            # Use a variable for the flag to ensure it's only used on Windows
-            creationflags = subprocess.CREATE_NO_WINDOW
-            subprocess.Popen(args, creationflags=creationflags)
-        else:
-            # For other platforms (like Linux/macOS), Popen without special flags works fine
-            subprocess.Popen(args)
-
-
 
 ### functions for setting up the UCI file properly 
 def setup(uci,name,run = 0,reach_ids = None,n = 1,time_step = 3):
@@ -584,3 +572,5 @@ def setup_qualid(uci):
 
 
 
+
+# %%