mpcaHydro 2.1.0__tar.gz → 2.2.1__tar.gz

mpcahydro-2.2.1/ERROR.FIL

@@ -0,0 +1,6 @@
+15:21:44.605 : LOG_MSG:ERROR.FIL OPENED
+15:21:44.607 : HASS_ENT:F90_WDBOPNR:entr:WDMSFL,RWFLG:         100           1 C:\Users\mfratki\Documents\github\pyHSPF\src\hspf\bin\WinHSPFLt\hspfmsg.wdm
+15:21:44.608 : HASS_ENT:F90_WDBOPNR:exit:WDMSFL,RETCOD         100           0
+FILBLK RETCOD           0
+wdmfl           0           0           0           0
+FILBLK RETCOD           0

{mpcahydro-2.1.0 → mpcahydro-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mpcaHydro
-Version: 2.1.0
+Version: 2.2.1
 Summary: Python package for downloading MPCA hydrology data
 Project-URL: Homepage, https://github.com/mfratkin1/mpcaHydro
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>
@@ -10,7 +10,9 @@ Keywords: Hydrology,MPCA
 Classifier: Development Status :: 3 - Alpha
 Classifier: Programming Language :: Python
 Requires-Python: >=3.8
+Requires-Dist: baseflow
 Requires-Dist: duckdb
+Requires-Dist: oracledb
 Requires-Dist: pandas
 Requires-Dist: pathlib
 Requires-Dist: requests

mpcahydro-2.2.1/demo.py

@@ -0,0 +1,226 @@
+#%%
+from mpcaHydro.data_manager import dataManager
+from pyhcal.repository import Repository
+from mpcaHydro import outlets
+import duckdb
+from mpcaHydro import equis, warehouse, wiski
+from hspf.hspfModel import hspfModel
+from hspf.uci import UCI
+from mpcaHydro import etlSWD
+
+
+#%% 
+'''
+New approach. Directly load to warehouse from downloads. 
+Store raw and processed data in warehouse. For large timeseries I could store
+as parquet files. The transformations using pandas take a bit of time. I imagine doing them
+within duckdb would be faster.
+
+'''
+
+# with warehouse.connect(db_path) as con:
+#    df = con.execute("SELECT * FROM staging.wiski").df()
+#    df = wiski.transform(df,filter_qc_codes = False)
+
+#%%
+model_name = 'Nemadji'
+db_path = f'C:/Users/mfratki/Documents/{model_name}.duckdb'
+start_year = 1996
+end_year = 2030
+replace = True
+filter_qc_codes = True
+equis_stations = outlets.equis_stations(model_name)
+wiski_stations = outlets.wiski_stations(model_name)
+equis.connect('MFRATKI',password = 'DeltaT#MPCA3')
+warehouse.init_db(db_path,reset = True)
+
+
+#%% Old approach. Store as indvidual processed station files then load to warehouse
+#df_equis = equis.download(equis_stations)
+#df_wiski = wiski.download(wiski_stations,start_year = start_year, end_year = end_year)
+
+#%% equis
+
+
+
+
+def download_equis_data(db_path,station_ids,replace = False):
+  with warehouse.connect(db_path,read_only = False) as con:
+    df = equis.download(station_ids)
+    if not df.empty:
+      warehouse.load_df_to_table(con,df, 'staging.equis',replace = replace)
+      warehouse.load_df_to_table(con,equis.transform(df), 'analytics.equis',replace = replace)
+    else:
+      print('No data neccesary for HSPF calibration available from equis for stations:',station_ids)
+
+def download_wiski_data(db_path,station_ids,replace = False):
+  with warehouse.connect(db_path,read_only = False) as con:
+    df = wiski.download(station_ids,start_year = start_year, end_year = end_year)
+    if not df.empty:
+      warehouse.load_df_to_table(con,df, 'staging.wiski', replace = replace)
+      warehouse.load_df_to_table(con,wiski.transform(df), 'analytics.wiski',replace = replace)
+    else:
+      print('No data neccesary for HSPF calibration available from wiski for stations:',station_ids)
+
+
+# Add to warehouse from custom df. Must contain required normalized columns.
+with warehouse.connect(db_path,read_only = False) as con:
+  if replace:
+     warehouse.drop_station_id(con,station_id,station_origin='equis')
+  warehouse.add_to_table(con,df, 'staging','equis_normalized')
+
+
+  warehouse.load_df_to_staging(con,df, 'equis_raw',replace = replace)
+  df = equis.normalize(df.copy())
+  warehouse.add_to_table(con,df, 'staging','equis_normalized')
+  df = equis.transform(df)
+  warehouse.add_to_table(con,df, 'analytics','equis')
+
+
+
+#%% swd
+
+df = etlSWD.download(equis_stations)
+
+with warehouse.connect(db_path,read_only = False) as con:
+  warehouse.load_df_to_staging(con,df, 'equis_raw',replace = replace)
+  df = equis.normalize(df.copy())
+  warehouse.add_to_table(con,df, 'staging','equis_normalized')
+  df = equis.transform(df)
+  warehouse.add_to_table(con,df, 'analytics','equis')
+#%% wiski
+
+
+
+      if station_origin == 'wiski':
+          df = wiski.download(station_ids,start_year = start_year, end_year = end_year)
+          warehouse.load_df_to_staging(con,df, 'wiski_raw', replace = replace)
+          df = wiski.normalize(df.copy())
+          warehouse.add_to_table(con,df, 'staging','wiski_normalized')
+          df = wiski.transform(df,filter_qc_codes = filter_qc_codes)
+          warehouse.add_to_table(con,df, 'analytics','wiski') # method includes normalization
+
+      if station_origin == 'swd':
+          df = pd.concat([etlSWD.download(station_id) for station_id in station_ids])
+          warehouse.load_df_to_staging(con,df, 'equis_raw', replace = replace)
+          df = etlSWD.transform(df.copy())
+          warehouse.add_to_table(con,df, 'analytics','equis')
+      warehouse.update_views(con)
+
+with warehouse.connect(db_path) as con:
+  warehouse.update_views(con)
+
+
+#%%
+
+import requests
+url = 'http://ifrshiny.seas.umich.edu/mglp/'
+requests.get(url)
+
+
+ 
+db_path = 'C:/Users/mfratki/Documents/Rum.duckdb'
+modl_db.build_outlet_db(db_path)
+con = duckdb.connect(db_path)
+con.execute("SELECT * FROM station_reach_pairs").df()
+con.execute('SELECT * FROM station_reach_pairs WHERE outlet_id = 76').df()
+
+# Need to remove duplicates from MODL_DB
+modl_db.MODL_DB.loc[modl_db.MODL_DB.duplicated(['station_id','source'])]
+
+#%%
+dm = dataManager('C:/Users/mfratki/Documents/')
+dm._build_warehouse()
+equis_stations = modl_db.equis_stations('Nemadji')
+wiski_stations = modl_db.wiski_stations('Nemadji')
+
+#%% Old approach. Store as indvidual processed station files then load to warehouse
+for station_id in equis_stations:
+    dm._download_station_data(station_id,'equis', True)
+    
+for station_id in wiski_stations:
+    dm._download_station_data(station_id,'wiski', True)
+
+
+
+
+
+
+
+
+
+
+#%% Adding HSPF outputs to warehouse
+
+
+
+
+
+
+
+
+
+
+
+con = duckdb.connect(db_path)
+
+model_name = 'Nemadji'
+outlets = [group for _, group in modl_db.MODL_DB.query('repository_name == @model_name').groupby(by = ['opnids','repository_name'])]
+
+for outlet in outlets:
+    1+1
+
+
+dfs = []
+for constituent in ['Q','TSS','TP','N','OP','TKN']:
+    opnids = modl_db.split_opnids([opnid.split(',') for opnid in set(outlet['opnids'].tolist())])
+    for opnid in opnids:
+        df = mod.hbns.get_reach_constituent(constituent,opnids,time_step='h')
+        df.columns = ['value']
+        df['constituent'] = constituent
+        df['operation'] = operation
+        df['opnid'] = opnid
+        dfs.append(df)
+        
+df = pd.concat(dfs).reset_index()
+df['model_name'] = model_name
+
+
+
+station_ids = ['H05018001','S006-214','S015-102']
+target_constituent = 'TSS'
+flow_constituent = 'Q'
+
+# build placeholders for the IN list (one ? per station id)
+placeholders = ','.join(['?'] * len(station_ids))
+
+sql = f'''
+SELECT o.*, f.datetime AS flow_datetime, f.value AS flow, f.baseflow, f.station_id AS flow_station_id, f.station_origin AS flow_station_origin  
+FROM analytics.observations o
+JOIN analytics.observations f
+  ON o.datetime = f.datetime
+WHERE o.constituent = ?
+  AND o.station_id IN ({placeholders})
+  AND f.constituent = ?;
+'''
+
+# parameter order must match the ? positions in the query
+params = [target_constituent] + station_ids + [flow_constituent]
+
+df = con.execute(sql, params).df()
+
+outlet_id: station_ids
+
+outlet_id: opnid
+
+
+outlets = []
+for index, (_, group) in enumerate(modl_db.MODL_DB.groupby(by = ['opnids','repository_name'])):
+    group['outlet_id'] = index
+    group.reset_index(drop=True, inplace=True)
+    outlets.append(group)
+
+
+    for _, row in group.iterrows():
+        opnids = group.split_opnids(row['opnids'].str.split(',').to_list())
+        row*len(opnids)

{mpcahydro-2.1.0 → mpcahydro-2.2.1}/pyproject.toml

@@ -5,12 +5,14 @@ build-backend = "hatchling.build"
 [project]
 name = "mpcaHydro"
 urls = { "Homepage" = "https://github.com/mfratkin1/mpcaHydro" }  # ? Add this!
-version = "2.1.0"
+version = "2.2.1"
 dependencies = [
   "pandas",
   "requests",
   "pathlib",
-  "duckdb"
+  "duckdb",
+  "oracledb",
+  "baseflow"
 ]
 requires-python = ">=3.8"
 authors = [

mpcahydro-2.2.1/src/mpcaHydro/data/WISKI_QUALITY_CODES.csv

@@ -0,0 +1,71 @@
+quality_code,Text,Description,Active
+0,Unchecked,Unchecked data in progress or data that is not quality coded as part of the workup.  Default coding for shifts so the quality codes from Level and Ratings are used for discharges. ,1
+3,Instantaneous,Instantaneous groundwater measurements or sampled date for load stations.,1
+5,Excellent,Discharge measurements that are excellent.,1
+8,Reliable Interpolation,The value of the data point is an interpolation between adjacent points.  Code used for filling gaps less than 4 hours or with no change in data trend likely based on reference information.,1
+10,Good,Time series data that tracks well and requires no corrections or corrections of very small magnitude or  timeseries data that has been reviewed and accepted for precipitation and groundwater level. Also used for discharge measurements and rating points.  ,1
+15,Fair,Time series data that tracks fairly well and requires some corrections of relatively small magnitude. Also used for discharge measurements and rating points.  ,1
+20,Poor,Time series data that tracks poorly and requires significant or many corrections. Also used for discharge measurements and rating points. ,1
+27,Questionable,"Timeseries data or discharge measurements that are questionable due to operator error, equipment error, etc). Extra scrutiny should be used for these data. ",1
+28,Unknown data quality,"Unknown quality of time series data, ratings or discharge measurements.",1
+29,Modeled,"Time-series data, rating point or discharge from a reliable mathematical and\or computer model. ",1
+34,Estimated,"Time-series data estimated from reference traces, models or extrapolation of the rating curve using supporting data and up to two times the maximum measured discharge.",1
+35,Unreliable,Time-series data computed with a rating extrapolated without supporting data or beyond two times the maximum measured discharge without a model.,1
+36,Threshold Exceedance,"Time-series data may be beyond the measuring limits of the monitoring equipment, or outside the bounds of historical extremes.",1
+40,Default import code,WISKI default coding for gaugings. ,1
+45,Approved Ext Data,"External data that has been graded externally as ""Approved"".",1
+48,Unknown Ext Data,External data that has been graded internally as “Unknown”.,1
+49,Estimated Ext Data,External data that has been graded externally as “Estimated.”  Typically this is finalized ice data.,1
+50,Provisional Ext Data,External data that has been graded internally or externally as “Provisional”.,1
+80,Ice - Estimated,Ice affected time series data. Discharge computed with ice affected stage data is considered estimated.,1
+199,199-Logger Unknown,Initial code for data coming to the system from the logger.,1
+200,200,Initial code for data coming to the system from telemetry or default coding for WISKI timeseries. ,1
+228,Info Parameter,This parameter is collected for informational purposes only.  Data has been through a cursory check only.  This is stored in the database and available upon request.,1
+255,---,System assigned code for gaps in the data set.  Records with null values. ,1
+1,Continuous Data,~Discontinued~ Good TS data that requires no correction.,0
+2,Edited Data,~Discontinued~ TS data that has been edited.  Typically used when spikes are removed or when points are edited manual for datum corrections.,0
+3,Instantaneous Data,Final WQ data.,0
+4,Questionable data,~Discontinued~,0
+5,Excellent measurment,Used to indicated discharge measurements that are excellent as well as excellent sections of the rating.,0
+10,Good measurement,Used to indicated discharge measurements and sections of the rating that are good and time series data that tracks well and requires no corrections or corrections of very small magnitude.,0
+12,Modeled measurement,~Discontinued~ Rating point or discharge was obtained from a relizble mathematical and/or computer model. After 3/1/11 use QC148.,0
+15,Fair measurement,Used to indicated discharge measurements and sections of the rating that are fair and time series data that tracks fairly well and requires some corrections of relatively small magnitude.,0
+20,Poor measurement,Used to indicated discharge measurements and sections of the rating that are poor and time series data that tracks poorly and requires significant or many corrections.,0
+25,Unknown measurement,Measurement data not available.,0
+27,Questionable data,"Flow measurement is very poor and should be given extra scrutiny or time series data that is questionable due to operator error, equipment error, etc.",0
+30,Good Archived Daily Value,This code is used for archived daily value data that is considered “Good”.,0
+31,Fair Archived Daily Value,This code is used for archived daily value data that is considered “Fair”.,0
+32,Poor Archived Daily Value,This code is used for archived daily value data that is considered “Poor”.,0
+33,Unknown Archived Daily Value,This code is used for archived daily value data that has unknown quality based on lack of documentation.,0
+34,Estimated Archived Daily Value,This code is used for archived daily value data that has been estimated.,0
+35,Unreliable Archived Daily Value,This code is used for archived daily value data that is unreliable based on the quality of the supporting time series data and/or rating.,0
+45,Good External Data,This code is used for external data that has been graded internally as “Good”.,0
+46,Fair External Data,This code is used for external data that has been graded internally as “Fair”.,0
+47,Poor External Data,This code is used for external data that has been graded internally as “Poor”.,0
+48,Unknown External Data,This code is used for external data that has been graded internally as “Unknown”,0
+49,Estimated External Data,This code is used for external data that has been graded externally as “Estimated.”  Typically this is finalized ice data.,0
+50,Provisional External Data,This code is used for external data that has been graded internally as “Provisional”,0
+51,Telemetry data - DCP,This code is used for time-series data when imported into hydstra using an automated telemetry method that accesses a DCP through the GOES network.  The “questionable measurement” flag is set through the shef code that accompanies the DCP data.,0
+60,Above rating,~Discontinued~,0
+70,Estimated Data,Value of the data point is estimated.,0
+76,Reliable interpolation,Value of the data point is an interpolation between adjacent points. ,0
+80,Ice,"(DISCONTINUED) Used to indicate ice conditions when the data should not be exported. Use in conjunction with 80 to code 232.00 values, run USDAY to compute daily flow, then recode 232.00 80 values to 180 so unit value export cannot occur.",0
+82,Linear interpolation across a gap in records,~Discontinued~  Points that were added to fill a gap in the data record.  The points fall on a straight line between the end points of the gap. This code was changed to 8 in WISKI.,0
+103,Provisional Instantaneous Data,Provisional WQ data.,0
+130,Good Provisional Daily Value,This code is used for archived daily value data that is considered “Good” but Provisional because there is only one year of gaging measurements.,0
+131,Fair Provisional Daily Value,This code is used for archived daily value data that is considered “Fair” but Provisional because there is only one year of gaging measurements.,0
+132,Poor Provisional Daily Value,This code is used for archived daily value data that is considered “Poor” but Provisional because there is only one year of gaging measurements.,0
+133,Unknown Provisional Archived Daily Value,This code is used for archived daily value data that has unknown quality based on lack of documentation but Provisional because there is only one year of gaging measurements.,0
+134,Estimated Provisional Archived Daily Value,This code is used for archived daily value data that has been estimated but Provisional because there is only one year of gaging measurements.,0
+135,Unreliable Provisional Archived Daily Value,This code is used for archived daily value data that is unreliable based on the quality of the supporting time series data and/or rating but Provisional because there is only one year of gaging measurements.,0
+140,Data not yet checked,This code is used for time-series data when it is initially imported into hydstra using manual import methods.  ,0
+141,Telemetry data - not yet checked,This code is used for time-series data when it is imported into hydstra using an automated telemetry method.,0
+148,Modeled measurement,Rating point or discharge was obtained from a reliable mathematical and/or computer model.,0
+149,Extrapolated rating point,Rating point accurately extrapolated using supporting data and is less than two times the maxiumum measured discharge.,0
+150,Over-extrapolated rating point,Rating point extrapolated without supporting data or beyone two times the maximum measured discharge without a mathematical model.,0
+151,Data Missing,"This code is used to flag the end of a period of missing time-series data, before the next good data value.",0
+160,Above rating,~Discontinued~,0
+169,Datalogger Hardware Error Code 6999,"This code is used to indicate that a time-series point had a value of 6999 or -6999, a typical hardware error code, and the value was changed.",0
+170,Estimated Data,"Used to indicate estimated data when the data should not be exported.  Often used in conjunction with 70 to code 232.00 values, run USDAY to compute daily flow, then recode 232.00 70 values to 170 so unit value export can not occur.",0
+180,Ice,Used to indicate ice conditions.,0
+255,Data Missing,This code is used when data is exported and does not exist for a given time period.,0

mpcahydro-2.2.1/src/mpcaHydro/data_manager.py

@@ -0,0 +1,292 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun  3 10:01:14 2022
+
+@author: mfratki
+"""
+
+from copy import replace
+import pandas as pd
+#from abc import abstractmethod
+from pathlib import Path
+from mpcaHydro import etlSWD
+from mpcaHydro import equis, wiski, warehouse
+from mpcaHydro import xref
+from mpcaHydro import outlets
+from mpcaHydro.reports import reportManager
+import duckdb
+
+AGG_DEFAULTS = {'cfs':'mean',
+                'mg/l':'mean',
+                'degf': 'mean',
+                'lb':'sum'}
+
+UNIT_DEFAULTS = {'Q': 'cfs',
+                 'QB': 'cfs',
+                 'TSS': 'mg/l',
+                 'TP' : 'mg/l',
+                 'OP' : 'mg/l',
+                 'TKN': 'mg/l',
+                 'N'  : 'mg/l',
+                 'WT' : 'degf',
+                 'WL' : 'ft'}
+
+
+def validate_constituent(constituent):
+    assert constituent in ['Q','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
+
+def validate_unit(unit):
+    assert(unit in ['mg/l','lb','cfs','degF'])
+
+
+def build_warehouse(folderpath):
+    folderpath = Path(folderpath)
+    db_path = folderpath.joinpath('observations.duckdb').as_posix()
+    warehouse.init_db(db_path)
+
+def constituent_summary(db_path):
+    with duckdb.connect(db_path) as con:
+        query = '''
+        SELECT
+          station_id,
+          station_origin,
+          constituent,
+          COUNT(*) AS sample_count,
+          year(MIN(datetime)) AS start_date,
+          year(MAX(datetime)) AS end_date
+        FROM
+          observations
+        GROUP BY
+          constituent, station_id,station_origin
+        ORDER BY
+          sample_count;'''
+          
+        res = con.execute(query)
+        return res.fetch_df()
+
+
+
+
+class dataManager():
+
+    def __init__(self,folderpath, oracle_username = None, oracle_password =None, reset = False):
+        
+        self.data = {}
+        self.folderpath = Path(folderpath)
+        self.db_path = self.folderpath.joinpath('observations.duckdb')
+        self.oracle_username = oracle_username
+        self.oracle_password = oracle_password
+
+        if not self.db_path.exists() or reset:
+            self._build_warehouse()
+
+        self.xref = xref #TODO: implement xref manager class
+        self.outlets = outlets #TODO: implement outlets manager class
+        self.reports = reportManager(self.db_path)
+
+    
+    def connect_to_oracle(self):
+        assert (self.credentials_exist(), 'Oracle credentials not found. Set ORACLE_USER and ORACLE_PASSWORD environment variables or use swd as station_origin')
+        equis.connect(user = self.oracle_username, password = self.oracle_password)
+    
+    def credentials_exist(self):
+        if (self.oracle_username is not None) & (self.oracle_password is not None):
+            return True
+        else:
+            return False
+        
+    def _build_warehouse(self):
+        warehouse.init_db(self.db_path.as_posix(),True)
+
+    def _process_wiski_data(self,filter_qc_codes = True, data_codes = None, baseflow_method = 'Boughton'):
+        with warehouse.connect(self.db_path,read_only = False) as con:
+            df = con.execute("SELECT * FROM staging.wiski").df()
+            df_transformed = wiski.transform(df, filter_qc_codes, data_codes, baseflow_method)
+            warehouse.load_df_to_table(con,df_transformed, 'analytics.wiski')
+            warehouse.update_views(con)
+
+    def _process_equis_data(self):
+        with warehouse.connect(self.db_path,read_only = False) as con:
+            df = con.execute("SELECT * FROM staging.equis").df()
+            df_transformed = equis.transform(df)
+            warehouse.load_df_to_table(con,df_transformed, 'analytics.equis')
+            warehouse.update_views(con)
+
+    def _process_data(self,filter_qc_codes = True, data_codes = None, baseflow_method = 'Boughton'):
+        self._process_wiski_data(filter_qc_codes, data_codes, baseflow_method)
+        self._process_equis_data()
+
+    def _update_views(self):
+        with warehouse.connect(self.db_path,read_only = False) as con:
+            warehouse.update_views(con)
+
+    def _download_wiski_data(self,station_ids,start_year = 1996, end_year = 2030, filter_qc_codes = True, data_codes = None, baseflow_method = 'Boughton'):
+        with warehouse.connect(self.db_path,read_only = False) as con:
+            df = wiski.download(station_ids,start_year = start_year, end_year = end_year)
+            if not df.empty:
+                warehouse.load_df_to_table(con,df, 'staging.wiski')
+                warehouse.load_df_to_table(con,wiski.transform(df, filter_qc_codes,data_codes,baseflow_method), 'analytics.wiski')
+                warehouse.update_views(con)
+            else:
+                print('No data neccesary for HSPF calibration available from wiski for stations:',station_ids)     
+
+    def _download_equis_data(self,station_ids):
+        if self.credentials_exist():
+            self.connect_to_oracle()
+            print('Connected to Oracle database.')
+            with warehouse.connect(self.db_path,read_only = False) as con:
+                df = equis.download(station_ids)
+                if not df.empty:
+                    warehouse.load_df_to_table(con,df, 'staging.equis')
+                    warehouse.load_df_to_table(con,equis.transform(df.copy()), 'analytics.equis')
+                    warehouse.update_views(con)
+                else:
+                    print('No data neccesary for HSPF calibration available from equis for stations:',station_ids)
+        else:
+            raise ValueError('Oracle credentials not found. Set ORACLE_USER and ORACLE_PASSWORD environment variables or use swd as station_origin')
+        
+
+    def _get_equis_template(self):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            query = '''
+            SELECT *
+            FROM staging.equis
+            LIMIT 0'''
+            df = con.execute(query).fetch_df().to_csv(self.folderpath.joinpath('equis_template.csv'), index=False)
+        return df
+    
+    def _get_wiski_template(self):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            query = '''
+            SELECT *
+            FROM staging.wiski
+            LIMIT 0'''
+            df = con.execute(query).fetch_df().to_csv(self.folderpath.joinpath('wiski_template.csv'), index=False)
+        return df
+
+    def get_outlets(self,model_name):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            query = '''
+            SELECT *
+            FROM outlets.station_reach_pairs
+            WHERE repository_name = ?
+            ORDER BY outlet_id'''
+            df = con.execute(query,[model_name]).fetch_df()
+        return df
+    
+    def get_station_ids(self,station_origin = None):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            if station_origin is None:
+                query = '''
+                SELECT DISTINCT station_id, station_origin
+                FROM analytics.observations'''
+                df = con.execute(query).fetch_df()
+            else:
+                query = '''
+                SELECT DISTINCT station_id
+                FROM analytics.observations
+                WHERE station_origin = ?'''
+                df = con.execute(query,[station_origin]).fetch_df()
+        
+        return df['station_id'].to_list()
+    
+
+    def get_observation_data(self,station_ids,constituent,agg_period = None):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            query = '''
+            SELECT *
+            FROM analytics.observations
+            WHERE station_id IN ? AND constituent = ?'''
+            df = con.execute(query,[station_ids,constituent]).fetch_df()
+        
+        unit = UNIT_DEFAULTS[constituent]
+        agg_func = AGG_DEFAULTS[unit]
+
+        df.set_index('datetime',inplace=True)
+        df.attrs['unit'] = unit
+        df.attrs['constituent'] = constituent
+        if agg_period is not None:
+            df = df[['value']].resample(agg_period).agg(agg_func)
+            df.attrs['agg_period'] = agg_period
+
+        df.rename(columns={'value': 'observed'}, inplace=True) 
+        return df.dropna(subset=['observed'])
+    
+    def get_outlet_data(self,outlet_id,constituent,agg_period = 'D',to_csv = False):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            query = '''
+            SELECT *
+            FROM analytics.outlet_observations_with_flow
+            WHERE outlet_id = ? AND constituent = ?'''
+            df = con.execute(query,[outlet_id,constituent]).fetch_df()    
+
+        unit = UNIT_DEFAULTS[constituent]
+        agg_func = AGG_DEFAULTS[unit]
+
+        df.set_index('datetime',inplace=True)
+        df.attrs['unit'] = unit
+        df.attrs['constituent'] = constituent
+        if agg_period is not None:
+            df = df[['value','flow_value','baseflow_value']].resample(agg_period).agg(agg_func)
+            df.attrs['agg_period'] = agg_period
+
+        df.rename(columns={'value': 'observed',
+                           'flow_value': 'observed_flow',
+                           'baseflow_value': 'observed_baseflow'}, inplace=True) 
+        return df.dropna(subset=['observed'])
+    
+    def get_raw_data(self,station_id,station_origin, to_csv = False):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            if station_origin.lower() == 'equis':
+                query = '''
+                SELECT *
+                FROM staging.equis_raw
+                WHERE station_id = ?'''
+            elif station_origin.lower() == 'wiski':
+                query = '''
+                SELECT *
+                FROM staging.wiski_raw
+                WHERE station_id = ?'''
+            else:
+                raise ValueError(f'Station origin {station_origin} not recognized. Valid options are equis or wiski.')
+            
+            df = con.execute(query,[station_id]).fetch_df()    
+
+        if to_csv:
+            df.to_csv(self.folderpath.joinpath(f'{station_id}_raw.csv'), index=False)
+        return df
+
+    def to_csv(self,station_id  ,station_origin,folderpath = None):
+        if folderpath is None:
+            folderpath = self.folderpath
+        else:
+            folderpath = Path(folderpath)
+        df = self.get_station_data([station_id],constituent = 'Q',agg_period = None)
+        if len(df) > 0:
+            df.to_csv(folderpath.joinpath(station_id + '.csv'))
+        else:
+            print(f'No {station_id} calibration data available at Station {station_id}')
+        
+        df.to_csv(folderpath.joinpath(station_id + '.csv'))
+
+
+# class database():
+#     def __init__(self,db_path):
+#         self.dbm = MonitoringDatabase(db_path)
+        
+    
+#     def get_timeseries(self,station_ds, constituent,agg_period):      
+#         validate_constituent(constituent)
+#         unit = UNIT_DEFAULTS[constituent]
+#         agg_func = AGG_DEFAULTS[unit]
+#         return odm.get_timeseries(station_id,constituent)
+
+    
+#     def get_samples(self,station_ds, constituent,agg_period):
+#         validate_constituent(constituent)
+#         unit = UNIT_DEFAULTS[constituent]
+#         agg_func = AGG_DEFAULTS[unit]
+#         return odm.get_sample(station_id,constituent)
+
+#     def get_samples_and_timeseries(self,station_ds, constituent,agg_period)
+