pyhcal 1.0.2__py3-none-any.whl → 1.1.0__py3-none-any.whl

pyhcal/calibrators.py

@@ -9,10 +9,6 @@ from copy import deepcopy
 import subprocess
 #non-standard imports
 import pandas as pd
-pd.set_option('display.max_columns', None)
-pd.set_option('display.max_rows', None)
-# to reset this
-pd.reset_option('display.max_columns')
 import numpy as np
 from pathlib import Path
 
@@ -23,48 +19,26 @@ from hspf import helpers
 from mpcaHydro import data_manager as dm
 from pyhcal import metrics
 from pyhcal import figures
-from pyhcal.setup_utils import Builder
-#from hspf_tools.orm.monitoring_db import MonitoringDatabase
+from pyhcal import setup_utils
 
 
-class calProject():
-    #valid_models = Builder.valid_models()
-    def __init__(self,project_location):
-        self.project_location = Path(project_location)
-        
-    
-    def new_project(self,model_name):
-        return Builder(model_name) #self._builder.new_project(project_location,model_name)
-    
-    def load_project(self,model_name):
-        if model_name in [f.name for f in self.project_location.iterdir() if f.is_dir()]:
-            return calibrator(self.project_location.joinpath(model_name))
-        else:
-            answer = input("No calibration project for that model. Would you like to set on up? (yes or no")
-            if answer.lower() in ["y","yes"]:
-                self.new_project(model_name)
-                return calibrator(self.project_location.joinpath(model_name))
-            elif answer.lower() in ["n","no"]:
-                return
-                # Do other stuff
-            else:
-                print('please enter yes or no')
-            
-    
-def config_info(project_folder):
-    project_path = Path(project_folder)
-    info = {'project_path'  : project_path,
-             'project_name' : project_path.name,
-             'model_path'   : project_path.joinpath('model'),
-             'output_path'  : project_path.joinpath('output'),
-             'start_date'   : '1996-01-01',
-             'end_date'     : '2100-01-01',
-            }     
-    return info
+def new_calibration(project_folder,model_name,download_station_data = True,run_model = True,convert_wdms = True):
+    return setup_utils.create_calibration_project(model_name,project_folder,download_station_data,run_model,convert_wdms)
 
 
+def load(project_folder):
+    validate_project_folder(project_folder)
+    return calibrator(project_folder)
 
-        
+def validate_project_folder(project_folder):
+    project_path = Path(project_folder)
+    assert project_path.exists(), 'Project folder does not exist'
+    assert project_path.joinpath('model').exists(), 'Model folder does not exist'
+    assert project_path.joinpath('data').exists(), 'Data folder does not exist'
+    assert project_path.joinpath('output').exists(), 'Output folder does not exist'
+    assert project_path.joinpath('targets.csv').exists(), 'targets.csv file does not exist in project folder'
+    return True
+ 
 class calibrator:
     def __init__(self,project_folder):
         self.project_path = Path(project_folder)
@@ -72,13 +46,11 @@ class calibrator:
         self.model_path = self.project_path.joinpath('model')
         self.output_path = self.project_path.joinpath('output')
         self.run = None
-        #self.winHSPF = str(Path(__file__).resolve().parent.parent) + '\\bin\\WinHSPFLt\\WinHspfLt.exe'     
         self.start_date = '1996-01-01'
         self.end_date = '2100-01-01'
         
         # Load observational data into memory TODO: Convert to database?
         self.dm = dm.dataManager(self.project_path.joinpath('data'))
-        #self.odm = MonitoringDatabase(cal.project_path.joinpath(cal.project_name))
      
         self.targets = None
         if self.project_path.joinpath('targets.csv').exists():
@@ -128,10 +100,7 @@ class calibrator:
         self.model.reports.wdms = self._wdms
         self.uci = deepcopy(self.model.uci) #uci to be manipulated 
 
-        
-    # def setup_run(self, reach_ids = None, time_Step = 3,n = 1):
-    #     setup_utils.setup(self.uci,self.project_name,run = self.run,reach_ids = reach_ids,n = 1,time_step = 3)
-    
+
     def run_model(self,name = None,overwrite_hbn = False): # NO STATE CHANGE 
         
         if name is None:
@@ -146,8 +115,8 @@ class calibrator:
         
         uci_file = self.model_path.joinpath(name + '.uci').as_posix()        
         self.uci.write(uci_file)
-        winHSPF = str(Path(__file__).resolve().parent.parent) + '\\bin\\WinHSPFLt\\WinHspfLt.exe'     
-        subprocess.run([winHSPF,uci_file]) #, stdout=subprocess.PIPE, creationflags=0x08000000)
+        #winHSPF = str(Path(__file__).resolve().parent.parent) + '\\bin\\WinHSPFLt\\WinHspfLt.exe'     
+        subprocess.run([self.model.winHSPF,uci_file]) #, stdout=subprocess.PIPE, creationflags=0x08000000)
     
 
     def get_simulated_output(self,reach_ids,constituent,time_step = 'YE'):
@@ -155,13 +124,13 @@ class calibrator:
         sim.name = 'simulated'
         return sim
     
-    def get_observed_data(self,station_ids,constituent,time_step = 'YE'):
+    def get_observed_data(self,station_ids,constituent,time_step = 'YE',baseflow_percentage = None):
         obs = self.dm._get_data(station_ids,constituent,agg_period = time_step).sort_index(level = 'index')
         obs.name = 'observed'
         return obs
      
     
-    def compare_simulated_observed(self,station_ids,reach_ids,constituent,time_step,flow_station_ids = None, dropna = False):
+    def compare_simulated_observed(self,station_ids,reach_ids,constituent,time_step,flow_station_ids = None, dropna = False, baseflow_percentage = None):
         obs = self.get_observed_data(station_ids,constituent,time_step)
         sim = self.get_simulated_output(reach_ids,constituent,time_step = time_step)
         
@@ -177,11 +146,21 @@ class calibrator:
         # matching flow data
         sim_flow = self.get_simulated_output(reach_ids,'Q',time_step)
         sim_flow.name = 'simulated_flow'
+        sim_flow.columns = ['simulated_flow']
         df = df.join(sim_flow,how = 'inner')
         obs_flow = self.get_observed_data(flow_station_ids,'Q',time_step)
         obs_flow.name = 'observed_flow'
+        obs_flow.columns = ['observed_flow']
         df = df.join(obs_flow,how='left')
-        df.columns = ['simulated','observed','simulated_flow','observed_flow']
+
+        obs_baseflow = self.get_observed_data(flow_station_ids,'QB',time_step)
+        obs_baseflow.name = 'observed_baseflow'
+        obs_baseflow.columns = ['observed_baseflow']
+        df = df.join(obs_baseflow,how='left')
+        df.columns = ['simulated','observed','simulated_flow','observed_flow','observed_baseflow']
+        
+        if baseflow_percentage is not None:
+            df = df.loc[(df['observed_baseflow']/df['observed_flow'] >= baseflow_percentage/100)]
         
         # Add metadata
         df.attrs['station_ids'] = station_ids
@@ -225,14 +204,7 @@ class calibrator:
         obs_flow.name = 'observed_flow'
         df = df.join(obs_flow,how='left')
         df.columns = ['simulated','observed','simulated_flow','observed_flow']
-        # sim_flow = self.model.hbns.get_rchres_data('Q',reach_ids, 'cfs','D')
-        # sim_flow.name = 'simulated_flow'
-        # df = df.join(sim_flow,how = 'inner')
-        # obs_flow = pd.concat([self.dm.get_data(station_id,'Q','cfs','D') for station_id in station_ids])
-        # obs_flow.name = 'observed_flow'
-        # df = df.join(obs_flow,how='left')
-        # df.columns = ['simulated','observed','simulated_flow','observed_flow']
-        
+        # Add metadata
         df.attrs['station_ids'] = station_ids
         df.attrs['reach_ids'] = reach_ids
         df.attrs['constituent'] = constituent
@@ -271,7 +243,7 @@ class calibrator:
 
         return df_agg
 
-    def landcover(cal,constituent,ref_landcover = None):
+    def landcover(self,constituent,ref_landcover = None):
         def normalize_group(group, ref_landcover):
             if ref_landcover in group['LSID'].to_list():
                 ref_value = group.loc[group['LSID'] == ref_landcover, 'mean'].values[0]
@@ -281,19 +253,20 @@ class calibrator:
             return group
 
         if ref_landcover is None:
-            ref_landcover = cal.targets['uci_name'][cal.targets['dom_lc']==1].values[0]
+            ref_landcover = self.targets['uci_name'][self.targets['dom_lc']==1].values[0]
 
-        df = cal.model.hbns.get_perlnd_data(constituent)*2000 #tons/ac/yr to lbs/acr/year
-        df = df[(df.index >= cal.start_date) & (df.index <= cal.end_date)]
+        df = self.model.hbns.get_perlnd_data(constituent)*2000 #tons/ac/yr to lbs/acr/year
+        df = df[(df.index >= self.start_date) & (df.index <= self.end_date)]
         df = df.mean().rename('mean').to_frame()
-        df = pd.merge(df, cal.uci.opnid_dict['PERLND'], left_index = True,right_on='TOPFST', how='inner')
+        df = pd.merge(df, self.uci.opnid_dict['PERLND'], left_index = True,right_on='TOPFST', how='inner')
         df_normalized = pd.concat([normalize_group(group, ref_landcover) for _, group in list(df.groupby('metzone'))])
         
-        targets = cal.targets.set_index('uci_name')[constituent]
+        targets = self.targets.set_index('uci_name')[constituent]
         targets = targets/targets.loc[ref_landcover]
         
         df_normalized['target'] = targets.loc[df_normalized['LSID']].values/df_normalized['mean_norm']
         df_normalized = df_normalized.fillna(1) #don't make any changes to 0 concentration perlands
+        #df_normalized = df_normalized.join(self.uci.table('PERLND','SED-PARM3'))
         return df_normalized
 
 

pyhcal/modl_db.py

@@ -8,74 +8,296 @@ Created on Thu May  1 09:51:51 2025
 from pathlib import Path
 import geopandas as gpd
 import pandas as pd
+import duckdb
 #from hspf_tools.calibrator import etlWISKI, etlSWD
 
 
 #stations_wiski = gpd.read_file('C:/Users/mfratki/Documents/GitHub/pyhcal/src/pyhcal/data/stations_wiski.gpkg')
 
 
-stations_wiski = gpd.read_file(str(Path(__file__).resolve().parent/'data\\stations_wiski.gpkg')).dropna(subset='opnids')[['station_id','true_opnid','opnids','comments','modeled','repository_name','wplmn_flag']]
+_stations_wiski = gpd.read_file(str(Path(__file__).resolve().parent/'data\\stations_wiski.gpkg'))
+stations_wiski = _stations_wiski.dropna(subset='opnids')[['station_id','true_opnid','opnids','comments','modeled','repository_name','wplmn_flag']]
 stations_wiski['source'] = 'wiski'
-stations_equis = gpd.read_file(str(Path(__file__).resolve().parent/'data\\stations_EQUIS.gpkg')).dropna(subset='opnids')[['id_code','true_opnid','opnids','comments','modeled','repository_name']]
+_stations_equis = gpd.read_file(str(Path(__file__).resolve().parent/'data\\stations_EQUIS.gpkg'))
+stations_equis = _stations_equis.dropna(subset='opnids')[['station_id','true_opnid','opnids','comments','modeled','repository_name']]
 stations_equis['source'] = 'equis'
 stations_equis['wplmn_flag'] = 0
-stations_equis = stations_equis.rename(columns = {'id_code':'station_id'})
 
 
+DB_PATH = str(Path(__file__).resolve().parent/'data\\outlets.duckdb')
+
 MODL_DB = pd.concat([stations_wiski,stations_equis])
+MODL_DB['opnids'] = MODL_DB['opnids'].str.strip().replace('',pd.NA)
+MODL_DB = MODL_DB.dropna(subset='opnids')
+MODL_DB = MODL_DB.drop_duplicates(['station_id','source']).reset_index(drop=True)
 
-database  = """
-    -- Stations/Locations table
-    CREATE TABLE IF NOT EXISTS Station (
-        stationPK INTEGER PRIMARY KEY AUTOINCREMENT,
-        reachPK INTEGER REFERENCES Reach(reachPK),
-        stationID TEXT NOT NULL,
-        stationName TEXT,
-        stationOrigin TEXT NOT NULL,
-        latitude REAL,
-        longitude REAL,
-        stationType TEXT,
-        UNIQUE(stationID, stationOrigin)
-    );
-    
-    -- Station Associations table
-    CREATE TABLE IF NOT EXISTS StationAssociations (
-        stationPK INTEGER REFERENCES Station(stationPK),
-        associationPK INTEGER REFERENCES Station(stationPK)
-    );
-
-    -- Station Aliases table
-    CREATE TABLE IF NOT EXISTS StationAliases (
-        stationPK INTEGER NOT NULL,
-        aliasPK INTEGER NOT NULL,
-        FOREIGN KEY (stationPK) REFERENCES Station(stationPK),
-        FOREIGN KEY (aliasPK) REFERENCES Station(stationPK)
-    );
-    
-    CREATE TABLE Reach (
-        reachPK INTEGER PRIMARY KEY,
-        modelName TEXT NOT NULL,
-        reachID INTEGER NOT NULL,
-        drainageArea FLOAT 
-    );
-    
-    CREATE TABLE Outlet (
-        outletPK INTEGER PRIMARY KEY,
-        outletName TEXT
-    );
-    
-    -- Outlet-Station Associations table
-    CREATE TABLE IF NOT EXISTS StationAssociations (
-        outletPK INTEGER NOT NULL REFERENCES Outlet(outletPK),
-        stationPK  INTEGER NOT NULL REFERENCES Station(reachPK)
-    );
-    
-    -- Outlet-Reach Associations table
-    CREATE TABLE IF NOT EXISTS StationAssociations (
-        outletPK INTEGER NOT NULL REFERENCES Outlet(outletPK),
-        reachPK  INTEGER NOT NULL REFERENCES Station(reachPK)
-        exclude INTEGER NOT NULL
-    );"""
+def _reload():
+    global _stations_wiski, stations_wiski, _stations_equis, stations_equis, MODL_DB
+    _stations_wiski = gpd.read_file(str(Path(__file__).resolve().parent/'data\\stations_wiski.gpkg'))
+    stations_wiski = _stations_wiski.dropna(subset='opnids')[['station_id','true_opnid','opnids','comments','modeled','repository_name','wplmn_flag']]
+    stations_wiski['source'] = 'wiski'
+    _stations_equis = gpd.read_file(str(Path(__file__).resolve().parent/'data\\stations_EQUIS.gpkg'))
+    stations_equis = _stations_equis.dropna(subset='opnids')[['station_id','true_opnid','opnids','comments','modeled','repository_name']]
+    stations_equis['source'] = 'equis'
+    stations_equis['wplmn_flag'] = 0
+
+    MODL_DB = pd.concat([stations_wiski,stations_equis])
+    MODL_DB['opnids'] = MODL_DB['opnids'].str.strip().replace('',pd.NA)
+    MODL_DB = MODL_DB.dropna(subset='opnids')
+    MODL_DB = MODL_DB.drop_duplicates(['station_id','source']).reset_index(drop=True)
+
+
+def get_model_db(model_name: str):
+    return MODL_DB.query('repository_name == @model_name')
+
+def split_opnids(opnids: list):
+    return [abs(int(float(j))) for i in opnids for j in i]
+
+def valid_models():
+    return MODL_DB['repository_name'].unique().tolist()
+
+def wplmn_station_opnids(model_name):
+    opnids = MODL_DB.query('repository_name == @model_name and wplmn_flag == 1 and source == "wiski"')['opnids'].str.split(',').to_list()
+    return split_opnids(opnids)
+
+def wiski_station_opnids(model_name):
+    opnids = MODL_DB.query('repository_name == @model_name and source == "wiski"')['opnids'].str.split(',').to_list()
+    return split_opnids(opnids)
+
+def equis_station_opnids(model_name):
+    opnids = MODL_DB.query('repository_name == @model_name and source == "equis"')['opnids'].str.split(',').to_list()
+    return split_opnids(opnids)
+
+def station_opnids(model_name):
+    opnids = MODL_DB.query('repository_name == @model_name')['opnids'].str.split(',').to_list()
+    return split_opnids(opnids)
+
+def equis_stations(model_name):
+    return MODL_DB.query('repository_name == @model_name and source == "equis"')['station_id'].tolist()
+
+def wiski_stations(model_name):
+    return MODL_DB.query('repository_name == @model_name and source == "wiski"')['station_id'].tolist()
+
+def wplmn_stations(model_name):
+    return MODL_DB.query('repository_name == @model_name and wplmn_flag == 1 and source == "wiski"')['station_id'].tolist()
+
+def outlets(model_name):
+    return [group for _, group in MODL_DB.query('repository_name == @model_name').groupby(by = ['opnids','repository_name'])]
+
+def outlet_stations(model_name):
+    return [group['station_id'].to_list() for _, group in MODL_DB.query('repository_name == @model_name').groupby(by = ['opnids','repository_name'])]
+
+def _split_opnids(opnids: list):
+    return [int(float(j)) for i in opnids for j in i]
+
+def connect(db_path):
+    Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+    return duckdb.connect(db_path)
+
+
+def init_db(db_path: str,reset: bool = False):
+    """
+    Initialize the DuckDB database: create staging and analytics schemas
+    """
+    db_path = Path(db_path)
+    if reset and db_path.exists():
+        db_path.unlink()
+
+    with connect(db_path.as_posix()) as con:
+        con.execute(OUTLETS_SCHEMA)
+
+
+
+# Accessors:
+def get_outlets_by_model(model_name: str):
+    with connect(DB_PATH) as con:
+        df = con.execute(
+            """
+            SELECT r.*
+            FROM station_reach_pairs r
+            WHERE r.repository_name = ?
+            """,
+            [model_name]
+        ).fetchdf()
+    return df
+
+def get_outlets_by_reach(reach_id: int, model_name: str):
+    """
+    Return all outlet rows for outlets that include the given reach_id in the given model_name.
+    """
+    with connect(DB_PATH) as con:
+        df = con.execute(
+            """
+            SELECT r.*
+            FROM station_reach_pairs r
+            WHERE r.reach_id = ? AND r.repository_name = ?
+            """,
+        [reach_id, model_name]).fetchdf()
+    return df
+ 
+def get_outlets_by_station(station_id: str, station_origin: str):
+    """
+    Return all outlet rows for outlets that include the given reach_id in the given model_name.
+    """
+    with connect(DB_PATH) as con:
+
+        df = con.execute(
+        """
+        SELECT r.*
+        FROM station_reach_pairs r
+        WHERE r.station_id = ? AND r.station_origin = ?
+        """,
+        [station_id, station_origin]).fetchdf()
+    return df
+
+# constructors:
+def build_outlet_db(db_path: str = None):
+    if db_path is None:
+        db_path = DB_PATH
+    init_db(db_path,reset=True)
+    with connect(db_path) as con:
+        for index, (_, group) in enumerate(MODL_DB.drop_duplicates(['station_id','source']).groupby(by = ['opnids','repository_name'])):
+            repo_name = group['repository_name'].iloc[0]    
+            add_outlet(con, outlet_id = index, outlet_name = None, repository_name = repo_name, notes = None)
+            
+            opnids = set(_split_opnids(group['opnids'].str.split(',').to_list()))
+
+            for opnid in opnids:
+                if opnid < 0:
+                    exclude = 1
+                else:
+                    exclude = 0
+                add_reach(con, outlet_id = index, reach_id = abs(opnid),exclude = exclude, repository_name = repo_name)
+
+            for _, row in group.drop_duplicates(subset=['station_id', 'source']).iterrows():
+                add_station(con, outlet_id = index, station_id = row['station_id'], station_origin = row['source'], true_opnid = row['true_opnid'], repository_name= repo_name, comments = row['comments'])
+
+ 
+def create_outlet_schema(con, model_name : str):
+    for index, (_, group) in enumerate(modl_db.outlets(model_name)):
+        repo_name = group['repository_name'].iloc[0]    
+        add_outlet(con, outlet_id = index, outlet_name = None, repository_name = repo_name, notes = None)
+        
+        opnids = set(_split_opnids(group['opnids'].str.split(',').to_list()))
+
+        for opnid in opnids:
+            if opnid < 0:
+                exclude = 1
+            else:
+                exclude = 0
+            add_reach(con, outlet_id = index, reach_id = abs(opnid),exclude = exclude, repository_name = repo_name)
+
+        for _, row in group.drop_duplicates(subset=['station_id', 'source']).iterrows():
+            add_station(con, outlet_id = index, station_id = row['station_id'], station_origin = row['source'], true_opnid = row['true_opnid'], repository_name= repo_name, comments = row['comments'])
+
+
+def add_outlet(con,
+               outlet_id: str,
+               repository_name: str,
+               outlet_name = None,
+               notes = None):
+    """
+    Insert an outlet. repository_name is required.
+    """
+    con.execute(
+        "INSERT INTO outlets (outlet_id, repository_name, outlet_name, notes) VALUES (?, ?, ?, ?)",
+        [outlet_id, repository_name, outlet_name, notes]
+    )
+
+def add_station(con,
+                outlet_id: str,
+                station_id: str,
+                station_origin: str,
+                true_opnid: str,
+                repository_name: str,
+                comments = None):
+    """
+    Insert a station membership for an outlet.
+    Constraints:
+    - PRIMARY KEY (station_id, station_origin): unique per origin across all outlets.
+    - true_opnid and true_opnid_repository_name are required per schema.
+    """
+    con.execute(
+        """INSERT INTO outlet_stations
+           (outlet_id, station_id, station_origin, true_opnid, repository_name, comments)
+           VALUES (?, ?, ?, ?, ?, ?)""",
+        [outlet_id, station_id, station_origin, true_opnid, repository_name, comments]
+    )
+
+def add_reach(con,
+              outlet_id: str,
+              reach_id: str,
+              repository_name: str,
+              exclude: int = 0):
+    """
+    Insert a reach membership for an outlet.
+    - repository_name is required and participates in the PK (reach_id, repository_name).
+    - exclude = 1 to mark a reach as excluded from association views.
+    """
+    con.execute(
+        """INSERT INTO outlet_reaches (outlet_id, reach_id, repository_name, exclude)
+           VALUES (?, ?, ?, ?)""",
+        [outlet_id, reach_id, repository_name, int(exclude)]
+    )
+
+
+OUTLETS_SCHEMA  = """-- schema.sql
+-- Simple 3-table design to manage associations between model reaches and observation stations via outlets.
+-- Compatible with DuckDB and SQLite.
+
+-- Table 1: outlets
+-- Represents a logical grouping that ties stations and reaches together.
+CREATE TABLE IF NOT EXISTS outlets (
+  outlet_id TEXT PRIMARY KEY,
+  repository_name TEXT NOT NULL,
+  outlet_name TEXT,
+  notes TEXT             -- optional: general notes about the outlet grouping
+);
+
+-- Table 2: outlet_stations
+-- One-to-many: outlet -> stations
+CREATE TABLE IF NOT EXISTS outlet_stations (
+  outlet_id TEXT NOT NULL,
+  station_id TEXT NOT NULL,
+  station_origin TEXT NOT NULL,       -- e.g., 'wiski', 'equis'
+  repository_name TEXT NOT NULL,  -- repository model the station is physically located in
+  true_opnid TEXT NOT NULL,           -- The specific reach the station physically sits on (optional)
+  comments TEXT,             -- Per-station comments, issues, etc.
+  CONSTRAINT uq_station_origin UNIQUE (station_id, station_origin),
+  FOREIGN KEY (outlet_id) REFERENCES outlets(outlet_id)
+);
+
+-- Table 3: outlet_reaches
+-- One-to-many: outlet -> reaches
+-- A reach can appear in multiple outlets, enabling many-to-many overall.
+CREATE TABLE IF NOT EXISTS outlet_reaches (
+  outlet_id TEXT NOT NULL,
+  reach_id TEXT NOT NULL,    -- model reach identifier (aka opind)
+  repository_name TEXT NOT NULL,  -- optional: where the mapping comes from
+  exclude INTEGER DEFAULT 0, -- flag to indicate if this reach should be excluded (1) or included (0)
+  FOREIGN KEY (outlet_id) REFERENCES outlets(outlet_id)
+);
+
+-- Useful views:
+
+-- View: station_reach_pairs
+-- Derives the implicit many-to-many station <-> reach relationship via shared outlet_id
+CREATE VIEW IF NOT EXISTS station_reach_pairs AS
+SELECT
+  s.outlet_id,
+  s.station_id,
+  s.station_origin,
+  r.reach_id,
+  r.exclude,
+  r.repository_name,
+FROM outlet_stations s
+JOIN outlet_reaches r
+  ON s.outlet_id = r.outlet_id;
+
+-- Example indexes (SQLite will accept CREATE INDEX; DuckDB treats them as metadata but it’s okay to define):
+CREATE INDEX IF NOT EXISTS idx_outlet_stations_outlet ON outlet_stations(outlet_id);
+CREATE INDEX IF NOT EXISTS idx_outlet_reaches_outlet ON outlet_reaches(outlet_id);
+CREATE INDEX IF NOT EXISTS idx_station_reach_pairs_station ON outlet_stations(station_id);"""
     
     
 #row = modl_db.MODL_DB.iloc[0]

pyhcal/repository.py

@@ -6,7 +6,7 @@ Created on Wed Nov 27 09:16:30 2024
 """
 
 import pandas as pd
-from pyhcal.modl_db import MODL_DB
+from pyhcal import modl_db
 from pathlib import Path
 import shutil
 
@@ -20,7 +20,7 @@ class Repository():
                                                                                                                   'USGS HUC-4':'string',
                                                                                                                   'USGS HUC-2':'string'})
     
-    MODL_DB = MODL_DB
+    
 
     
     
@@ -38,7 +38,7 @@ class Repository():
         
         self.REPOSITORY_PATH = repository_path
         huc_directory = self.HUC_DIRECTORY.loc[self.HUC_DIRECTORY['Repository_HUC8 Name'] == model_name]
-        self.modl_db = self.MODL_DB.loc[self.MODL_DB['repository_name'] == model_name]
+        self.modl_db = modl_db.get_model_db(model_name) #self.MODL_DB.loc[self.MODL_DB['repository_name'] == model_name]
         #self.modl_db  = pd.concat([self.MODL_DB.loc[self.MODL_DB['repository_name'].str.startswith(huc8_id,na=False)] for huc8_id in huc8_ids])        
         self.model_name = model_name
         self.huc8_ids = list(huc_directory['USGS HUC-8'])
@@ -48,6 +48,9 @@ class Repository():
         self.uci_file = self.repo_folder.joinpath('HSPF','.'.join([self.model_name,'uci']))
         self.wdm_files = [item for item in self.repo_folder.joinpath('HSPF').iterdir() if (item.name.endswith('.wdm')) | (item.name.endswith('.WDM'))]
         self.shapefiles = {item.name.split('.')[0].split('_')[-1]:item for item in self.repo_folder.joinpath('GIS').iterdir() if (item.name.endswith('.shp')) | (item.name.endswith('.SHP'))}
+        self.wiski_stations = modl_db.wiski_stations(model_name)
+        self.equis_stations = modl_db.equis_stations(model_name)
+
 
     def copy(self,copy_path):
         copy_path = Path(copy_path)
@@ -73,7 +76,7 @@ class Repository():
             files = [file for file in shapefile.parent.iterdir() if file.stem == shapefile.stem]
             [shutil.copyfile(file,Path(copy_path).joinpath(Path(file).name)) for file in files]
             
-
+    
     
 def build_folders(trg_path):
     

pyhcal/setup_utils.py

@@ -8,6 +8,7 @@ from mpcaHydro.data_manager import dataManager
 from hspf.wdmReader import readWDM
 from hspf.uci import UCI
 from pyhcal.repository import Repository
+from pyhcal import modl_db
 
 import numpy as np
 import pandas as pd
@@ -38,6 +39,8 @@ class Builder():
         self.new_uci = None
         self.uci = None
         self.dm = None
+        self.calibration_reaches = modl_db.wplmn_station_opnids(model_name)
+
     
     def valid_models():
         return Repository.valid_models()
@@ -65,44 +68,33 @@ class Builder():
         self.download_station_data()
         self.uci.build_targets().to_csv(self.project_path.joinpath('targets.csv'))
 
-        # if not self.project_path.joinpath('model',self.project_name + '_0-0.hbn').exists():
-        #     self.run_model()
-        
-    #TODO move to UCI class
     
     def format_uci(self,calibration_reaches = None):
-        
-        
+        if calibration_reaches is None:
+            calibration_reaches = self.calibration_reaches
+
         setup_files(self.uci,self.project_name,run = 0)
         setup_geninfo(self.uci)
-        
-        if calibration_reaches is None:
-            calibration_reaches = self.repository.modl_db.loc[self.repository.modl_db['wplmn_flag'] == 1]['opnids'].str.split(',').to_list()
-            calibration_reaches = [abs(int(j)) for i in calibration_reaches for j in i]        
-        #calibration_reaches = self.uci.network.station_order(calibration_reaches)[-1][0]
-        
-        
-        self.uci.initialize(name = self.project_name + '_0')
-        # for reaches in self._MODL_DB['opnids'].str.split('+').to_list():
-        #     [calibration_reaches.append(int(reach)) for reach in reaches if ~pd.isna(reach)]
-            
+        self.uci.initialize(name = self.project_name + '_0')  
         setup_binaryinfo(self.uci,reach_ids = calibration_reaches)
         setup_qualid(self.uci)
         self.uci.write(self.new_uci)
         
-        # Download observation data
-        # Sources/Databases WISKI and EQUIS (DELTA databases?)
-    #TODO: use a single WISKI etl script for csg and wplmn data
-    
-    
-    def download_station_data(self):#:,station_id,source_id):
-        stations = self.repository.modl_db.drop_duplicates(subset = ['source','station_id'])
-        if not stations.empty:
-            for index, row in stations.iterrows():
-                assert(row['source'].lower() in ['wiski','equis','swd'])
-                station_id = row['station_id']
-                self.dm.download_station_data(station_id,row['source'].lower())
-            
+    
+    def download_station_data(self,start_year,end_year):#:,station_id,source_id):
+        #%% Old approach. Store as indvidual processed station files then load to warehouse
+        if len(equis_stations) > 0:
+            if self.dm.credentials_exist():
+                station_origin = 'equis'
+            else: station_origin = 'swd'
+        
+            for station_id in equis_stations:
+                self.dm.download_station_data(station_id,station_origin, True)
+        
+        if len(wiski_stations) > 0:
+            for station_id in wiski_stations:
+                self.dm.download_station_data(station_id,'wiski', True)
+                 
     
     def convert_wdms(self):
         copy_path = Path(self.project_path.joinpath('model'))
@@ -111,11 +103,31 @@ class Builder():
             readWDM(wdm_file,
                     copy_path.joinpath(wdm_file.name.replace('.wdm','.hdf5').replace('.WDM','hdf5')))
         
-    def run_model(self):
+    def run_model(self, wait_for_completion=True):
+        #run_model(self.new_uci, wait_for_completion)
         # Run the uci file
         winHSPF = str(Path(__file__).resolve().parent.parent) + '\\bin\\WinHSPFLt\\WinHspfLt.exe'     
         subprocess.run([winHSPF,self.new_uci]) #, stdout=subprocess.PIPE, creationflags=0x08000000)    
         
+def run_model(uci_file, wait_for_completion=True):
+    winHSPF = str(Path(__file__).resolve().parent.parent) + '\\bin\\WinHSPFlt\\WinHspfLt.exe'
+    
+    # Arguments for the subprocess
+    args = [winHSPF, uci_file.as_posix()]
+
+    if wait_for_completion:
+        # Use subprocess.run to wait for the process to complete (original behavior)
+        subprocess.run(args)
+    else:
+        # Use subprocess.Popen to run the process in the background without waiting
+        # On Windows, you can use creationflags to prevent a console window from appearing
+        if sys.platform.startswith('win'):
+            # Use a variable for the flag to ensure it's only used on Windows
+            creationflags = subprocess.CREATE_NO_WINDOW
+            subprocess.Popen(args, creationflags=creationflags)
+        else:
+            # For other platforms (like Linux/macOS), Popen without special flags works fine
+            subprocess.Popen(args)
 
 
 
@@ -134,11 +146,12 @@ def setup_files(uci,name,run,n = 5):
     table = uci.table('FILES',drop_comments = False)
     for index, row in table.iterrows():
         filename = Path(row['FILENAME'])
-        if filename.suffix in ['.wdm','.ech','.out']:
+        if filename.suffix in ['.wdm','.ech','.out','.mut']:
             table.loc[index,'FILENAME'] = filename.name
-        if filename.suffix in ['.hbn']:
+        elif filename.suffix in ['.hbn']:
             table.loc[index,'FILENAME'] = filename.name
-        if filename.suffix in ['.plt']:
+        #if filename.suffix in ['.plt']:
+        else:
             table.drop(index,inplace = True)
             
     # Get new binary number and create new BINO rows

{pyhcal-1.0.2.dist-info → pyhcal-1.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyhcal
-Version: 1.0.2
+Version: 1.1.0
 Summary: Python package for calibrating MPCA HSPF models
 Project-URL: Homepage, https://github.com/mfratkin1/pyhcal
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>

pyhcal-1.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+pyhcal/__init__.py,sha256=4TEpGD-PfEY8yK-od8DpEMA4_iQ-q9y0PBvROXSPdB0,94
+pyhcal/calibrators.py,sha256=lDxvXpjLj1Xhp-NPCVUyyWeaMTZdP5FA9Bh-PuwrOHs,30383
+pyhcal/figures.py,sha256=Iu7LaN_i2IuDA_nfxj-a8AkG-FTLZVicJ3-efIs5OiE,45534
+pyhcal/metrics.py,sha256=GUGHd-op-g1Foj8wnS_JVURSms4ifcC0a5h8ketQ29I,17911
+pyhcal/modl_db.py,sha256=z8trT387Gcbg15xAqAcW62c1dbgSSjymv3qOyH3Dyx8,12569
+pyhcal/repository.py,sha256=VPHeSbrnFAG1F6tFXdnI2C72xm3dTPb7Z3rkPr--srI,4603
+pyhcal/setup_utils.py,sha256=sIINj8_h-MgQhcCH_u95nFTJVw0QEgnR0fZgAitX1iQ,30398
+pyhcal/data/HUC_Names.csv,sha256=UGmd3Q5E8DyFWggXzaXWpsRze7sFyrlpYqaYpMWAiGM,18946
+pyhcal/data/WISKI_EQUIS_XREF.csv,sha256=bPYq-f4-Qc6jsvUgl81lwXBeFamfDe5TjohqUV1XJlg,1244704
+pyhcal/data/outlets.duckdb,sha256=zB1t2NsnrUed-3HOT61DYmtOy89PZ9uWZAwmAEUhleY,2895872
+pyhcal/data/stations_EQUIS.gpkg,sha256=SPyxGK5H3bbqMikv45n_ul-KULPNWyad9XcDq_9mXwM,2019328
+pyhcal/data/stations_wiski.gpkg,sha256=vlh03SihjlQMIbn25rfPqOKQtJsSYS2FMR65zAznTQQ,905216
+pyhcal-1.1.0.dist-info/METADATA,sha256=xRX7vV5LA1UxSsAZze6VMRzTJI6ePZ749Pdf8h-VQgw,560
+pyhcal-1.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+pyhcal-1.1.0.dist-info/RECORD,,

{pyhcal-1.0.2.dist-info → pyhcal-1.1.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

pyhcal-1.0.2.dist-info/RECORD

@@ -1,14 +0,0 @@
-pyhcal/__init__.py,sha256=4TEpGD-PfEY8yK-od8DpEMA4_iQ-q9y0PBvROXSPdB0,94
-pyhcal/calibrators.py,sha256=O2m29Qat74g0AVPFABPoQXDCqosWqh1YBJfFdiWPdxU,31328
-pyhcal/figures.py,sha256=Iu7LaN_i2IuDA_nfxj-a8AkG-FTLZVicJ3-efIs5OiE,45534
-pyhcal/metrics.py,sha256=GUGHd-op-g1Foj8wnS_JVURSms4ifcC0a5h8ketQ29I,17911
-pyhcal/modl_db.py,sha256=0NoWNNxfbF1fy43geOO4EJt-Uev6Og4Hw-KNn9xvAys,3163
-pyhcal/repository.py,sha256=4Ep6htWIAWEnLDs-z3gjCKpl7WEEl3t6Kt98rTS9QB8,4456
-pyhcal/setup_utils.py,sha256=Hl244HimWFVJYLXtplmWlXleiTQ6vN-PhY7WXyvZciY,29861
-pyhcal/data/HUC_Names.csv,sha256=UGmd3Q5E8DyFWggXzaXWpsRze7sFyrlpYqaYpMWAiGM,18946
-pyhcal/data/WISKI_EQUIS_XREF.csv,sha256=bPYq-f4-Qc6jsvUgl81lwXBeFamfDe5TjohqUV1XJlg,1244704
-pyhcal/data/stations_EQUIS.gpkg,sha256=KyWMRbNoSSMDB1IweoLhyFeKu1P-YoYpQbXna6VYp8I,3284992
-pyhcal/data/stations_wiski.gpkg,sha256=HlH5EwUo9qEEoxtbEpL0cJZHEW57XEUA-ROy4sN7eE4,892928
-pyhcal-1.0.2.dist-info/METADATA,sha256=WAKZkD6p_xoX6-cX3EM3d7-zFGO_ANJzlNvbSvozQFE,560
-pyhcal-1.0.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pyhcal-1.0.2.dist-info/RECORD,,