hspf 2.1.2__tar.gz → 2.1.3__tar.gz

{hspf-2.1.2 → hspf-2.1.3}/.gitignore

@@ -1,3 +1,3 @@
 *.pyc
-*.whl
-*.gz
+*.whl
+*.gz

{hspf-2.1.2 → hspf-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hspf
-Version: 2.1.2
+Version: 2.1.3
 Summary: Python package for downloading and running HSPF models
 Project-URL: Homepage, https://github.com/mfratkin1/pyHSPF
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>

{hspf-2.1.2 → hspf-2.1.3}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "hspf"
 urls = { "Homepage" = "https://github.com/mfratkin1/pyHSPF" }  # ? Add this!
-version = "2.1.2"
+version = "2.1.3"
 dependencies = [
   "pandas",
   "requests",

{hspf-2.1.2 → hspf-2.1.3}/src/hspf/hbn.py

@@ -6,7 +6,7 @@ nutrients relevant for our current calibration methods. (See calibration_helpers
 
 @author: mfratki
 """
-from . import helpers
+from hspf import helpers
 import pandas as pd
 import math
 from struct import unpack
@@ -14,6 +14,7 @@ from numpy import fromfile
 from pandas import DataFrame
 from datetime import datetime, timedelta #, timezone
 from collections import defaultdict
+from collections.abc import MutableMapping
 #from pathlib import Path
 
 
@@ -189,11 +190,30 @@ class hbnInterface:
     def _clear_cache(self):
         [hbn._clear_cache() for hbn in self.hbns]
         
+
+        
     def get_time_series(self, t_opn, t_cons, t_code, opnid, activity = None):
-        return pd.concat([hbn.get_time_series(t_opn, t_cons, t_code, opnid, activity) for hbn in self.hbns],axis = 1)
+        df = pd.concat([hbn._get_time_series(t_opn, t_cons, t_code, opnid, activity) for hbn in self.hbns],axis = 1)
+        if df.empty:
+            raise ValueError(f"No data found for {t_opn} {t_cons} {t_code} {opnid} {activity}")
+        
+        if long_format:
+            df = df.reset_index().melt(id_vars = ['index'],var_name = 'OPNID',value_name = t_con)
+            df.rename(columns = {'index':'datetime'},inplace = True)
+            df['OPERATION'] = t_opn
+        return df
+        
+    def get_multiple_timeseries(self,t_opn,t_code,t_con,opnids = None,activity = None,axis = 1,long_format = False):
+        df = pd.concat([hbn._get_multiple_timeseries(t_opn,t_code,t_con,opnids,activity) for hbn in self.hbns],axis = 1)
+        if df.empty:
+            raise ValueError(f"No data found for {t_opn} {t_con} {t_code} {opnids} {activity}")
         
-    def get_multiple_timeseries(self,t_opn,t_code,t_con,opnids = None,activity = None,axis = 1):
-        return pd.concat([hbn.get_multiple_timeseries(t_opn,t_code,t_con,opnids,activity) for hbn in self.hbns],axis = 1)
+        if long_format:
+            df = df.reset_index().melt(id_vars = ['index'],var_name = 'OPNID',value_name = 'value')
+            df.rename(columns = {'index':'datetime'},inplace = True)
+            df['TIMESERIES'] = t_con
+            df['OPERATION'] = t_opn
+        return df
 
     def get_perlnd_constituent(self,constituent,perlnd_ids = None,time_step = 5):
         return get_simulated_perlnd_constituent(self,constituent,time_step)
@@ -217,13 +237,33 @@ class hbnInterface:
         # for dic in dics:
         #     for key, vals in dic.items():
         #         [dd[key].append(val) for val in vals]
-        dd = defaultdict(set)    
+        # dd = defaultdict(set)    
         dics =  [hbn.output_names() for hbn in self.hbns]
+        return merge_dicts(dics)
+        # for dic in dics:
+        #     for operation, vals in dic.items():
+        #         for activity,v in vals.items():
+        #             [dd[operation][activity].add(t) for t in v]
+        # return dd
+
+    def _timeseries(self):
+        mapn = self._mapn()
+        timeseries = []
+        for key, vals in mapn.items():
+            _key = list(key)
+            for val in vals:
+                timeseries.append(_key + [val])
+        return timeseries      
+            
+
+    def _mapn(self):
+        dd = defaultdict(set)    
+        dics =  [hbn.mapn for hbn in self.hbns]
         for dic in dics:
             for key, vals in dic.items():
                 [dd[key].add(val) for val in vals]
-        return dd
-
+        return dd 
+    
     def get_perlnd_data(self,constituent,t_code = 'yearly'):
         t_cons = helpers.get_tcons(constituent,'PERLND')
         
@@ -236,14 +276,13 @@ class hbnInterface:
         return df
          
           
-    def get_rchres_data(self,constituent,reach_ids,units = 'mg/l',t_code = 'daily'):
+    def get_rchres_output(self,constituent,units = 'mg/l',t_code = 5):
         '''
         Convience function for accessing the hbn time series associated with our current
         calibration method. Assumes you are summing across all dataframes.
        '''
-        
-        df = pd.concat([self.get_reach_constituent(constituent,[reach_id],t_code,units) for reach_id in reach_ids], axis = 1)
-        df.columns = reach_ids
+        t_cons = helpers.get_tcons(constituent,'RCHRES',units)
+        df = sum([self.get_multiple_timeseries('RCHRES',t_code,t_con) for t_con in t_cons])
         df.attrs['unit'] = units
         df.attrs['constituent'] = constituent
         return df
@@ -399,19 +438,27 @@ class hbnClass:
     def infer_opnids(self,t_opn, t_cons,activity):
         result = [k[-2] for k,v in self.mapn.items() if (t_cons in v) & (k[0] == t_opn) & (k[-1] == activity)]
         if len(result) == 0:
-            return print('No Constituent-OPNID relationship found')
+            result = [-1]
+        #     return print('No Constituent-OPNID relationship found')
         return result
     
     
     def infer_activity(self,t_opn, t_cons):  
         result = [k[-1] for k,v in self.mapn.items() if (t_cons in v) & (k[0] == t_opn)]
         if len(result) == 0:
-            return print('No Constituent-Activity relationship found')
-        assert(len(set(result)) == 1)
-        return result[0]
-    
+            result = ''
+        else:#     return print('No Constituent-Activity relationship found')
+            assert(len(set(result)) == 1)
+            result = result[0]
+        return result
     
     def get_time_series(self, t_opn, t_cons, t_code, opnid, activity = None):
+        df = self._get_time_series(t_opn, t_cons, t_code, opnid, activity)
+        if df.empty:
+            raise ValueError(f"No data found for {t_opn} {t_cons} {t_code} {opnid} {activity}")
+        return df
+
+    def _get_time_series(self, t_opn, t_cons, t_code, opnid, activity = None):
         """
         get a single time series based on:
         1.      t_opn: RCHRES, IMPLND, PERLND
@@ -420,13 +467,15 @@ class hbnClass:
         4. t_activity: HYDR, IQUAL, etc
         5.  time_unit: yearly, monthly, full (default is 'full' simulation duration)
         """
+
+
         if isinstance(t_code,str):
             t_code = self.tcodes[t_code]
         
         if activity is None:
             activity = self.infer_activity(t_opn,t_cons)        
-            if activity is None:
-                return None
+
+            
         summaryindx = f'{t_opn}_{activity}_{opnid:03d}_{t_code}'
         if summaryindx in self.summaryindx:
             df = self.data_frames[summaryindx][t_cons].copy()
@@ -438,25 +487,31 @@ class hbnClass:
             #df.index = df.index.shift(-1,TCODES2FREQ[t_code])
             df = df[df.index >= '1996-01-01']
         else:
-            df = None
+            df = pd.DataFrame()
             
         return df
+    
     def get_multiple_timeseries(self,t_opn,t_code,t_con,opnids = None,activity = None):
+        df = self._get_multiple_timeseries(t_opn,t_code,t_con,opnids,activity)
+        if df.empty:
+            raise ValueError(f"No data found for {t_opn} {t_con} {t_code} {opnids} {activity}")
+        return df
+    
+    def _get_multiple_timeseries(self,t_opn,t_code,t_con,opnids = None,activity = None):
         # a single constituent but multiple opnids
+
+        
         if isinstance(t_code,str):
             t_code = self.tcodes[t_code]
             
         if activity is None:
-            activity = self.infer_activity(t_opn,t_con)
-            if activity is None:
-                return None
-           
+            activity = self.infer_activity(t_opn,t_con)   
+
         if opnids is None:
             opnids = self.infer_opnids(t_opn,t_con,activity)
-            if opnids is None:
-                return None 
+
            
-        df = None
+        df = pd.DataFrame()
         frames = []
         mapd_list = list(self.mapd.keys())
         for opnid in opnids:
@@ -475,9 +530,76 @@ class hbnClass:
             dic[activity] = set([item for sublist in t_cons for item in sublist])
         return dic
     
+
+    def output_names(self):
+
+        activities = []
+        operations = []
+        for k, v in self.mapn.items():
+            operations.append(k[0])
+            activities.append(k[-1])
+
+        operations = set(operations)
+        activities = set(activities)
+        #activities = set([k[-1] for k,v in self.mapn.items()])
+
+        dic = {}
+        for operation in operations:
+            acitivities = set([k[-1] for k,v in self.mapn.items() if k[0] == operation])
+            dic[operation] = {}
+            for activity in acitivities:
+                t_cons = [v for k,v in self.mapn.items() if (k[0] == operation) & (k[-1] == activity)]   
+                dic[operation][activity] = set([item for sublist in t_cons for item in sublist])
+        # for activity in activities:
+        #     t_cons = [v for k,v in self.mapn.items() if k[-1] == activity]   
+        #     dic[activity] = set([item for sublist in t_cons for item in sublist])
+        return dic
+    
+    def get_timeseries(self):
+        mapn = self.mapn
+        timeseries = []
+        for key, vals in mapn.items():
+            _key = list(key)
+            for val in vals:
+                timeseries.append(_key + [val])
+        return timeseries      
+
     @staticmethod          
     def get_perlands(summary_indxs):
          perlands =  [int(summary_indx.split('_')[-2]) for summary_indx in summary_indxs]
          return perlands
      
- 
+
+def merge_dicts(dicts):
+    """
+    Merge a list of dictionaries into a single dictionary, combining sets
+    at the leaf level and properly merging nested dictionaries.
+    
+    Args:
+        dicts (list): A list of dictionaries to merge.
+    
+    Returns:
+        dict: The merged dictionary.
+    """
+    def recursive_merge(d1, d2):
+        for key, value in d2.items():
+            if key in d1:
+                # If the value is a dictionary, recurse
+                if isinstance(d1[key], MutableMapping) and isinstance(value, MutableMapping):
+                    recursive_merge(d1[key], value)
+                # If the value is a set, merge the sets
+                elif isinstance(d1[key], set) and isinstance(value, set):
+                    d1[key].update(value)
+                else:
+                    raise ValueError(f"Incompatible types for key '{key}': {type(d1[key])} vs {type(value)}")
+            else:
+                # If the key does not exist in d1, copy it
+                d1[key] = value
+    
+    # Start with an empty dictionary
+    merged_dict = {}
+    
+    for d in dicts:
+        recursive_merge(merged_dict, d)
+    
+    return merged_dict

{hspf-2.1.2 → hspf-2.1.3}/src/hspf/hspfModel.py

@@ -7,17 +7,19 @@ Created on Thu Oct 13 09:26:05 2022
 from pathlib import Path
 import os.path
 import subprocess
+import concurrent.futures
 
-from .uci import UCI
-from . import hbn
-from .reports import Reports
-from .wdm import wdmInterface
-from . import wdmReader
+from hspf.uci import UCI
+from hspf import hbn
+from hspf.reports import Reports
+from hspf.wdm import wdmInterface
+from hspf import wdmReader
 
 
 
 
 
+winHSPF = str(Path(__file__).resolve().parent) + '\\bin\\WinHSPFLt\\WinHspfLt.exe'
 
 
 # Only for accessing information regarding a specific uci_file
@@ -51,6 +53,15 @@ class hspfModel():
         # Compositions
         self.reports = Reports(self.uci,self.hbns,self.wdms)
         
+    def _reinitialize(self,uci_file:str,run_model:bool = False):
+        self.uci = UCI(uci_file)
+        self.validate_uci(run_model = run_model)
+        self.hbns = hbn.hbnInterface(self.hbn_paths)
+        try:
+            self.wdms = wdmInterface(self.wdm_paths)
+        except:
+            self.wdms = None
+        self.reports = Reports(self.uci,self.hbns,self.wdms)
 
     def validate_wdms(self):
         # Ensure wdm files exist and the folders for the other file types exist relative
@@ -92,15 +103,16 @@ class hspfModel():
         else:
             self.run_model()
 
-    def run_model(self,new_uci_file = None):
+    def run_model(self,new_uci_file = None,):
         
         if new_uci_file is None:
             new_uci_file = self.uci_file
         
         # new_uci_file = self.model_path.joinpath(uci_name)
         # self.uci.write(new_uci_file)
-        subprocess.run([self.winHSPF,self.uci_file.as_posix()]) #, stdout=subprocess.PIPE, creationflags=0x08000000)
-        self.load_uci(new_uci_file,run_model = False)
+
+        subprocess.run([winHSPF,self.uci_file.as_posix()]) #, stdout=subprocess.PIPE, creationflags=0x08000000)
+        self._reinitialize(new_uci_file,run_model = False)
 
     def load_hbn(self,hbn_name):
         self.hbns[hbn_name] = hbn.hbnClass(self.uci_file.parent.joinpath(hbn_name).as_posix())
@@ -177,8 +189,35 @@ class hspfModel():
 
 
 
+def run_uci(uci_file:str, ):
+    """
+    convenience function to run a single model uci file.
+    """
+    print(f"Starting model: {uci_file}")
+    subprocess.run([winHSPF, uci_file]) 
+    print(f"Completed model: {uci_file}")
 
 
+def run_batch_files(file_list, max_concurrent=4):
+    """
+    Takes a list of .uci file paths and runs them N at a time.
+    """
+    # Create a pool of workers (threads)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_concurrent) as executor:
+        # Submit all jobs to the pool
+        future_to_file = {
+            executor.submit(run_uci, uci_file): uci_file 
+            for uci_file in file_list
+        }
+        
+        # Monitor completion (optional, but good for error catching)
+        for future in concurrent.futures.as_completed(future_to_file):
+            uci_file = future_to_file[future]
+            try:
+                future.result() # This will raise exceptions if run_uci failed
+            except Exception as exc:
+                print(f"File {uci_file} generated an exception: {exc}")
+                
 
 # class runManager():
 #     def __init__()

{hspf-2.1.2 → hspf-2.1.3}/src/hspf/parser/graph.py

@@ -635,7 +635,8 @@ class reachNetwork():
         areas = areas.groupby(['source_type','source_type_id','source_name'])['area'].sum()[['PERLND','IMPLND']]
 
         if group:  
-            areas = pd.concat([areas[operation].groupby('source_name').sum()  for operation in ['PERLND','IMPLND']])
+            areas = areas.groupby(['source_type','source_name']).sum()
+            #areas = pd.concat([areas[operation].groupby('source_name').sum()  for operation in ['PERLND','IMPLND']])
             #areas = pd.concat([areas[operation].groupby(self.uci.opnid_dict[operation].loc[areas[operation].index,'LSID'].values).sum() for operation in ['PERLND','IMPLND']])
         return areas