mpcaHydro 2.1.0__py3-none-any.whl → 2.2.1__py3-none-any.whl

mpcaHydro/data/WISKI_QUALITY_CODES.csv

@@ -0,0 +1,71 @@
+quality_code,Text,Description,Active
+0,Unchecked,Unchecked data in progress or data that is not quality coded as part of the workup.  Default coding for shifts so the quality codes from Level and Ratings are used for discharges. ,1
+3,Instantaneous,Instantaneous groundwater measurements or sampled date for load stations.,1
+5,Excellent,Discharge measurements that are excellent.,1
+8,Reliable Interpolation,The value of the data point is an interpolation between adjacent points.  Code used for filling gaps less than 4 hours or with no change in data trend likely based on reference information.,1
+10,Good,Time series data that tracks well and requires no corrections or corrections of very small magnitude or  timeseries data that has been reviewed and accepted for precipitation and groundwater level. Also used for discharge measurements and rating points.  ,1
+15,Fair,Time series data that tracks fairly well and requires some corrections of relatively small magnitude. Also used for discharge measurements and rating points.  ,1
+20,Poor,Time series data that tracks poorly and requires significant or many corrections. Also used for discharge measurements and rating points. ,1
+27,Questionable,"Timeseries data or discharge measurements that are questionable due to operator error, equipment error, etc). Extra scrutiny should be used for these data. ",1
+28,Unknown data quality,"Unknown quality of time series data, ratings or discharge measurements.",1
+29,Modeled,"Time-series data, rating point or discharge from a reliable mathematical and\or computer model. ",1
+34,Estimated,"Time-series data estimated from reference traces, models or extrapolation of the rating curve using supporting data and up to two times the maximum measured discharge.",1
+35,Unreliable,Time-series data computed with a rating extrapolated without supporting data or beyond two times the maximum measured discharge without a model.,1
+36,Threshold Exceedance,"Time-series data may be beyond the measuring limits of the monitoring equipment, or outside the bounds of historical extremes.",1
+40,Default import code,WISKI default coding for gaugings. ,1
+45,Approved Ext Data,"External data that has been graded externally as ""Approved"".",1
+48,Unknown Ext Data,External data that has been graded internally as “Unknown”.,1
+49,Estimated Ext Data,External data that has been graded externally as “Estimated.”  Typically this is finalized ice data.,1
+50,Provisional Ext Data,External data that has been graded internally or externally as “Provisional”.,1
+80,Ice - Estimated,Ice affected time series data. Discharge computed with ice affected stage data is considered estimated.,1
+199,199-Logger Unknown,Initial code for data coming to the system from the logger.,1
+200,200,Initial code for data coming to the system from telemetry or default coding for WISKI timeseries. ,1
+228,Info Parameter,This parameter is collected for informational purposes only.  Data has been through a cursory check only.  This is stored in the database and available upon request.,1
+255,---,System assigned code for gaps in the data set.  Records with null values. ,1
+1,Continuous Data,~Discontinued~ Good TS data that requires no correction.,0
+2,Edited Data,~Discontinued~ TS data that has been edited.  Typically used when spikes are removed or when points are edited manual for datum corrections.,0
+3,Instantaneous Data,Final WQ data.,0
+4,Questionable data,~Discontinued~,0
+5,Excellent measurment,Used to indicated discharge measurements that are excellent as well as excellent sections of the rating.,0
+10,Good measurement,Used to indicated discharge measurements and sections of the rating that are good and time series data that tracks well and requires no corrections or corrections of very small magnitude.,0
+12,Modeled measurement,~Discontinued~ Rating point or discharge was obtained from a relizble mathematical and/or computer model. After 3/1/11 use QC148.,0
+15,Fair measurement,Used to indicated discharge measurements and sections of the rating that are fair and time series data that tracks fairly well and requires some corrections of relatively small magnitude.,0
+20,Poor measurement,Used to indicated discharge measurements and sections of the rating that are poor and time series data that tracks poorly and requires significant or many corrections.,0
+25,Unknown measurement,Measurement data not available.,0
+27,Questionable data,"Flow measurement is very poor and should be given extra scrutiny or time series data that is questionable due to operator error, equipment error, etc.",0
+30,Good Archived Daily Value,This code is used for archived daily value data that is considered “Good”.,0
+31,Fair Archived Daily Value,This code is used for archived daily value data that is considered “Fair”.,0
+32,Poor Archived Daily Value,This code is used for archived daily value data that is considered “Poor”.,0
+33,Unknown Archived Daily Value,This code is used for archived daily value data that has unknown quality based on lack of documentation.,0
+34,Estimated Archived Daily Value,This code is used for archived daily value data that has been estimated.,0
+35,Unreliable Archived Daily Value,This code is used for archived daily value data that is unreliable based on the quality of the supporting time series data and/or rating.,0
+45,Good External Data,This code is used for external data that has been graded internally as “Good”.,0
+46,Fair External Data,This code is used for external data that has been graded internally as “Fair”.,0
+47,Poor External Data,This code is used for external data that has been graded internally as “Poor”.,0
+48,Unknown External Data,This code is used for external data that has been graded internally as “Unknown”,0
+49,Estimated External Data,This code is used for external data that has been graded externally as “Estimated.”  Typically this is finalized ice data.,0
+50,Provisional External Data,This code is used for external data that has been graded internally as “Provisional”,0
+51,Telemetry data - DCP,This code is used for time-series data when imported into hydstra using an automated telemetry method that accesses a DCP through the GOES network.  The “questionable measurement” flag is set through the shef code that accompanies the DCP data.,0
+60,Above rating,~Discontinued~,0
+70,Estimated Data,Value of the data point is estimated.,0
+76,Reliable interpolation,Value of the data point is an interpolation between adjacent points. ,0
+80,Ice,"(DISCONTINUED) Used to indicate ice conditions when the data should not be exported. Use in conjunction with 80 to code 232.00 values, run USDAY to compute daily flow, then recode 232.00 80 values to 180 so unit value export cannot occur.",0
+82,Linear interpolation across a gap in records,~Discontinued~  Points that were added to fill a gap in the data record.  The points fall on a straight line between the end points of the gap. This code was changed to 8 in WISKI.,0
+103,Provisional Instantaneous Data,Provisional WQ data.,0
+130,Good Provisional Daily Value,This code is used for archived daily value data that is considered “Good” but Provisional because there is only one year of gaging measurements.,0
+131,Fair Provisional Daily Value,This code is used for archived daily value data that is considered “Fair” but Provisional because there is only one year of gaging measurements.,0
+132,Poor Provisional Daily Value,This code is used for archived daily value data that is considered “Poor” but Provisional because there is only one year of gaging measurements.,0
+133,Unknown Provisional Archived Daily Value,This code is used for archived daily value data that has unknown quality based on lack of documentation but Provisional because there is only one year of gaging measurements.,0
+134,Estimated Provisional Archived Daily Value,This code is used for archived daily value data that has been estimated but Provisional because there is only one year of gaging measurements.,0
+135,Unreliable Provisional Archived Daily Value,This code is used for archived daily value data that is unreliable based on the quality of the supporting time series data and/or rating but Provisional because there is only one year of gaging measurements.,0
+140,Data not yet checked,This code is used for time-series data when it is initially imported into hydstra using manual import methods.  ,0
+141,Telemetry data - not yet checked,This code is used for time-series data when it is imported into hydstra using an automated telemetry method.,0
+148,Modeled measurement,Rating point or discharge was obtained from a reliable mathematical and/or computer model.,0
+149,Extrapolated rating point,Rating point accurately extrapolated using supporting data and is less than two times the maxiumum measured discharge.,0
+150,Over-extrapolated rating point,Rating point extrapolated without supporting data or beyone two times the maximum measured discharge without a mathematical model.,0
+151,Data Missing,"This code is used to flag the end of a period of missing time-series data, before the next good data value.",0
+160,Above rating,~Discontinued~,0
+169,Datalogger Hardware Error Code 6999,"This code is used to indicate that a time-series point had a value of 6999 or -6999, a typical hardware error code, and the value was changed.",0
+170,Estimated Data,"Used to indicate estimated data when the data should not be exported.  Often used in conjunction with 70 to code 232.00 values, run USDAY to compute daily flow, then recode 232.00 70 values to 170 so unit value export can not occur.",0
+180,Ice,Used to indicate ice conditions.,0
+255,Data Missing,This code is used when data is exported and does not exist for a given time period.,0

mpcaHydro/data_manager.py

@@ -5,20 +5,20 @@ Created on Fri Jun  3 10:01:14 2022
 @author: mfratki
 """
 
+from copy import replace
 import pandas as pd
 #from abc import abstractmethod
 from pathlib import Path
 from mpcaHydro import etlSWD
 from mpcaHydro import equis, wiski, warehouse
+from mpcaHydro import xref
+from mpcaHydro import outlets
+from mpcaHydro.reports import reportManager
 import duckdb
 
-
-WISKI_EQUIS_XREF = pd.read_csv(Path(__file__).parent/'data/WISKI_EQUIS_XREF.csv')
-#WISKI_EQUIS_XREF = pd.read_csv('C:/Users/mfratki/Documents/GitHub/hspf_tools/WISKI_EQUIS_XREF.csv')
-
 AGG_DEFAULTS = {'cfs':'mean',
                 'mg/l':'mean',
-                'degF': 'mean',
+                'degf': 'mean',
                 'lb':'sum'}
 
 UNIT_DEFAULTS = {'Q': 'cfs',
@@ -28,29 +28,15 @@ UNIT_DEFAULTS = {'Q': 'cfs',
                  'OP' : 'mg/l',
                  'TKN': 'mg/l',
                  'N'  : 'mg/l',
-                 'WT' : 'degF',
+                 'WT' : 'degf',
                  'WL' : 'ft'}
 
-def are_lists_identical(nested_list):
-    # Sort each sublist
-    sorted_sublists = [sorted(sublist) for sublist in nested_list]
-    # Compare all sublists to the first one
-    return all(sublist == sorted_sublists[0] for sublist in sorted_sublists)                                                                                               
 
-def construct_database(folderpath):
-    folderpath = Path(folderpath)
-    db_path = folderpath.joinpath('observations.duckdb').as_posix()
-    with duckdb.connect(db_path) as con:
-        con.execute("DROP TABLE IF EXISTS observations")
-        datafiles = folderpath.joinpath('*.csv').as_posix()
-        query = '''
-        CREATE TABLE observations AS SELECT * 
-        FROM
-        read_csv_auto(?,
-                        union_by_name = true);
-        
-        '''
-        con.execute(query,[datafiles])
+def validate_constituent(constituent):
+    assert constituent in ['Q','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
+
+def validate_unit(unit):
+    assert(unit in ['mg/l','lb','cfs','degF'])
 
 
 def build_warehouse(folderpath):
@@ -79,315 +65,209 @@ def constituent_summary(db_path):
         return res.fetch_df()
 
 
+
+
 class dataManager():
 
-    def __init__(self,folderpath, oracle_user = None, oracle_password =None):
+    def __init__(self,folderpath, oracle_username = None, oracle_password =None, reset = False):
         
         self.data = {}
         self.folderpath = Path(folderpath)
         self.db_path = self.folderpath.joinpath('observations.duckdb')
-        self.oracle_user = oracle_user
+        self.oracle_username = oracle_username
         self.oracle_password = oracle_password
+
+        if not self.db_path.exists() or reset:
+            self._build_warehouse()
+
+        self.xref = xref #TODO: implement xref manager class
+        self.outlets = outlets #TODO: implement outlets manager class
+        self.reports = reportManager(self.db_path)
+
     
     def connect_to_oracle(self):
         assert (self.credentials_exist(), 'Oracle credentials not found. Set ORACLE_USER and ORACLE_PASSWORD environment variables or use swd as station_origin')
-        equis.connect(user = self.oracle_user, password = self.oracle_password)
+        equis.connect(user = self.oracle_username, password = self.oracle_password)
     
     def credentials_exist(self):
-        if (self.oracle_user is not None) & (self.oracle_password is not None):
+        if (self.oracle_username is not None) & (self.oracle_password is not None):
             return True
         else:
             return False
         
-    def _reconstruct_database(self):
-        construct_database(self.folderpath)
-    
     def _build_warehouse(self):
-        build_warehouse(self.folderpath)
+        warehouse.init_db(self.db_path.as_posix(),True)
+
+    def _process_wiski_data(self,filter_qc_codes = True, data_codes = None, baseflow_method = 'Boughton'):
+        with warehouse.connect(self.db_path,read_only = False) as con:
+            df = con.execute("SELECT * FROM staging.wiski").df()
+            df_transformed = wiski.transform(df, filter_qc_codes, data_codes, baseflow_method)
+            warehouse.load_df_to_table(con,df_transformed, 'analytics.wiski')
+            warehouse.update_views(con)
+
+    def _process_equis_data(self):
+        with warehouse.connect(self.db_path,read_only = False) as con:
+            df = con.execute("SELECT * FROM staging.equis").df()
+            df_transformed = equis.transform(df)
+            warehouse.load_df_to_table(con,df_transformed, 'analytics.equis')
+            warehouse.update_views(con)
+
+    def _process_data(self,filter_qc_codes = True, data_codes = None, baseflow_method = 'Boughton'):
+        self._process_wiski_data(filter_qc_codes, data_codes, baseflow_method)
+        self._process_equis_data()
+
+    def _update_views(self):
+        with warehouse.connect(self.db_path,read_only = False) as con:
+            warehouse.update_views(con)
+
+    def _download_wiski_data(self,station_ids,start_year = 1996, end_year = 2030, filter_qc_codes = True, data_codes = None, baseflow_method = 'Boughton'):
+        with warehouse.connect(self.db_path,read_only = False) as con:
+            df = wiski.download(station_ids,start_year = start_year, end_year = end_year)
+            if not df.empty:
+                warehouse.load_df_to_table(con,df, 'staging.wiski')
+                warehouse.load_df_to_table(con,wiski.transform(df, filter_qc_codes,data_codes,baseflow_method), 'analytics.wiski')
+                warehouse.update_views(con)
+            else:
+                print('No data neccesary for HSPF calibration available from wiski for stations:',station_ids)     
+
+    def _download_equis_data(self,station_ids):
+        if self.credentials_exist():
+            self.connect_to_oracle()
+            print('Connected to Oracle database.')
+            with warehouse.connect(self.db_path,read_only = False) as con:
+                df = equis.download(station_ids)
+                if not df.empty:
+                    warehouse.load_df_to_table(con,df, 'staging.equis')
+                    warehouse.load_df_to_table(con,equis.transform(df.copy()), 'analytics.equis')
+                    warehouse.update_views(con)
+                else:
+                    print('No data neccesary for HSPF calibration available from equis for stations:',station_ids)
+        else:
+            raise ValueError('Oracle credentials not found. Set ORACLE_USER and ORACLE_PASSWORD environment variables or use swd as station_origin')
         
-    def constituent_summary(self,constituents = None):
-        with duckdb.connect(self.db_path) as con:
-            if constituents is None:
-                constituents = con.query('''
-                                        SELECT DISTINCT
-                                        constituent
-                                        FROM observations''').to_df()['constituent'].to_list()
 
+    def _get_equis_template(self):
+        with duckdb.connect(self.db_path,read_only=True) as con:
             query = '''
-            SELECT
-            station_id,
-            station_origin,
-            constituent,
-            COUNT(*) AS sample_count,
-            year(MIN(datetime)) AS start_date,
-            year(MAX(datetime)) AS end_date
-            FROM
-            observations
-            WHERE
-            constituent in (SELECT UNNEST(?))
-            GROUP BY
-            constituent,station_id,station_origin
-            ORDER BY
-            constituent,sample_count;'''
-        
-            df = con.execute(query,[constituents]).fetch_df()
+            SELECT *
+            FROM staging.equis
+            LIMIT 0'''
+            df = con.execute(query).fetch_df().to_csv(self.folderpath.joinpath('equis_template.csv'), index=False)
         return df
-
-    def get_wiski_stations(self):
-        return list(WISKI_EQUIS_XREF['WISKI_STATION_NO'].unique())
-    
-    def get_equis_stations(self):
-        return list(WISKI_EQUIS_XREF['EQUIS_STATION_ID'].unique())
     
-    def wiski_equis_alias(self,wiski_station_id):
-        equis_ids =  list(set(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_STATION_NO'] == wiski_station_id,'WISKI_EQUIS_ID'].to_list()))
-        equis_ids = [equis_id for equis_id in equis_ids if not pd.isna(equis_id)]
-        if len(equis_ids) == 0:
-            return []
-        elif len(equis_ids) > 1:
-            print(f'Too Many Equis Stations for {wiski_station_id}')
-            raise 
-        else:
-            return equis_ids[0]
-
-    def wiski_equis_associations(self,wiski_station_id):
-        equis_ids =  list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_STATION_NO'] == wiski_station_id,'EQUIS_STATION_ID'].unique())
-        equis_ids =  [equis_id for equis_id in equis_ids if not pd.isna(equis_id)]
-        if len(equis_ids) == 0:
-            return []
-        else:
-            return equis_ids
-        
-    def equis_wiski_associations(self,equis_station_id):
-        wiski_ids = list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['EQUIS_STATION_ID'] == equis_station_id,'WISKI_STATION_NO'].unique())
-        wiski_ids = [wiski_id for wiski_id in wiski_ids if not pd.isna(wiski_id)]
-        if len(wiski_ids) == 0:
-            return []
-        else:
-            return wiski_ids
-        
-    def equis_wiski_alias(self,equis_station_id):
-        wiski_ids =  list(set(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_EQUIS_ID'] == equis_station_id,'WISKI_STATION_NO'].to_list()))
-        wiski_ids = [wiski_id for wiski_id in wiski_ids if not pd.isna(wiski_id)]
-        if len(wiski_ids) == 0:
-            return []
-        elif len(wiski_ids) > 1:
-            print(f'Too Many WISKI Stations for {equis_station_id}')
-            raise 
-        else:
-            return wiski_ids[0]
-
-    def _equis_wiski_associations(self,equis_station_ids):
-        wiski_stations = [self.equis_wiski_associations(equis_station_id) for equis_station_id in equis_station_ids]
-        if are_lists_identical(wiski_stations):
-            return wiski_stations[0]
-        else:
-            return []
-            
-    def _stations_by_wid(self,wid_no,station_origin):
-        if station_origin in ['wiski','wplmn']:
-            station_col = 'WISKI_STATION_NO'
-        elif station_origin in ['equis','swd']:
-            station_col = 'EQUIS_STATION_ID'
-        else:
-            raise
-            
-        return list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WID'] == wid_no,station_col].unique())
+    def _get_wiski_template(self):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            query = '''
+            SELECT *
+            FROM staging.wiski
+            LIMIT 0'''
+            df = con.execute(query).fetch_df().to_csv(self.folderpath.joinpath('wiski_template.csv'), index=False)
+        return df
 
+    def get_outlets(self,model_name):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            query = '''
+            SELECT *
+            FROM outlets.station_reach_pairs
+            WHERE repository_name = ?
+            ORDER BY outlet_id'''
+            df = con.execute(query,[model_name]).fetch_df()
+        return df
     
-    def download_stations_by_wid(self, wid_no,station_origin, folderpath = None, overwrite = False):
-
-        station_ids = self._station_by_wid(wid_no,station_origin)
-        
-        if not station_ids.empty:
-            for _, row in station_ids.iterrows():
-                self.download_station_data(row['station_id'],station_origin, folderpath, overwrite)
-
-    def _download_station_data(self,station_id,station_origin,overwrite=False): 
-        assert(station_origin in ['wiski','equis','swd','wplmn'])
-        if station_origin == 'wiski':
-            self.download_station_data(station_id,'wiski',overwrite = overwrite)
-        elif station_origin == 'wplmn':
-            self.download_station_data(station_id,'wplmn',overwrite = overwrite)
-        elif station_origin == 'swd':
-            self.download_station_data(station_id,'swd',overwrite = overwrite)
-        else:
-            self.download_station_data(station_id,'equis',overwrite = overwrite)
-
-
-       
-
-    def download_station_data(self,station_id,station_origin,start_year = 1996, end_year = 2030,folderpath=None,overwrite = False,baseflow_method = 'Boughton'):
-        assert(station_origin in ['wiski','equis','swd','wplmn'])
-        station_id = str(station_id)
-        save_name = station_id
-        if station_origin == 'wplmn':
-            save_name = station_id + '_wplmn'
-        
-        if folderpath is None:
-            folderpath = self.folderpath
-        else:
-            folderpath = Path(folderpath)
-        
-        
-        if (folderpath.joinpath(save_name + '.csv').exists()) & (not overwrite):
-            print (f'{station_id} data already downloaded')
-            return
-        
-        if station_origin == 'wiski':
-            data = wiski.transform(wiski.download([station_id],wplmn=False, baseflow_method = baseflow_method))
-        elif station_origin == 'swd':
-            data = etlSWD.download(station_id)
-        elif station_origin == 'equis':
-            assert (self.credentials_exist(), 'Oracle credentials not found. Set ORACLE_USER and ORACLE_PASSWORD environment variables or use swd as station_origin')
-            data = equis.transform(equis.download([station_id]))
-        else:
-            data = wiski.transform(wiski.download([station_id],wplmn=True, baseflow_method = baseflow_method))
+    def get_station_ids(self,station_origin = None):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            if station_origin is None:
+                query = '''
+                SELECT DISTINCT station_id, station_origin
+                FROM analytics.observations'''
+                df = con.execute(query).fetch_df()
+            else:
+                query = '''
+                SELECT DISTINCT station_id
+                FROM analytics.observations
+                WHERE station_origin = ?'''
+                df = con.execute(query,[station_origin]).fetch_df()
         
+        return df['station_id'].to_list()
+    
 
-       
-        
-        if len(data) > 0:
-            data.to_csv(folderpath.joinpath(save_name + '.csv'))
-            self.data[station_id] = data
-        else:
-            print(f'No {station_origin} calibration cata available at Station {station_id}')
-        
-    def _load(self,station_id):
-        with duckdb.connect(self.db_path) as con:
+    def get_observation_data(self,station_ids,constituent,agg_period = None):
+        with duckdb.connect(self.db_path,read_only=True) as con:
             query = '''
             SELECT *
             FROM analytics.observations
-            WHERE station_id = ?'''
-            df = con.execute(query,[station_id]).fetch_df()
-            df.set_index('datetime',inplace=True)
-            self.data[station_id] = df
-            return df
-
-    def _load2(self,station_id):
-        df =  pd.read_csv(self.folderpath.joinpath(station_id + '.csv'), 
-                          index_col='datetime', 
-                          parse_dates=['datetime'], 
-                          #usecols=['Ts Date','Station number','variable', 'value','reach_id'],
-                          dtype={'station_id': str, 'value': float, 'variable': str,'constituent':str,'unit':str})
-        self.data[station_id] = df
-        return df
-    
-    def load(self,station_id):
-        try:
-            df = self.data[station_id]
-        except:
-            df = self._load(station_id)
-        return df
-    
-    def info(self,constituent):
-        return pd.concat([self._load(file.stem) for file in self.folderpath.iterdir() if file.suffix == '.csv'])[['station_id','constituent','value']].groupby(by = ['station_id','constituent']).count()
-        
-    def get_wplmn_data(self,station_id,constituent,unit = 'mg/l', agg_period = 'YE', samples_only = True):
-        
-        assert constituent in ['Q','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
-        station_id = station_id + '_wplmn'
-        dfsub = self._load(station_id)
-        
-        if samples_only:
-            dfsub = dfsub.loc[dfsub['quality_id'] == 3]
-        agg_func = 'mean'
+            WHERE station_id IN ? AND constituent = ?'''
+            df = con.execute(query,[station_ids,constituent]).fetch_df()
         
-        dfsub = dfsub.loc[(dfsub['constituent'] == constituent) & 
-                              (dfsub['unit'] == unit),
-                              ['value','station_origin']]
+        unit = UNIT_DEFAULTS[constituent]
+        agg_func = AGG_DEFAULTS[unit]
 
-        
-        df = dfsub[['value']].resample(agg_period).agg(agg_func)
-        
-        if df.empty:
-            dfsub = df
-        else:
-            
-            df['station_origin'] = dfsub['station_origin'].iloc[0]
-            
-            #if (constituent == 'TSS') & (unit == 'lb'): #convert TSS from lbs to us tons
-            #    dfsub['value'] = dfsub['value']/2000
-    
-            #dfsub = dfsub.resample('H').mean().dropna()
-        
+        df.set_index('datetime',inplace=True)
         df.attrs['unit'] = unit
         df.attrs['constituent'] = constituent
-        return df['value'].to_frame().dropna()
-    
-    def get_data(self,station_id,constituent,agg_period = 'D'):
-        return self._get_data([station_id],constituent,agg_period)
+        if agg_period is not None:
+            df = df[['value']].resample(agg_period).agg(agg_func)
+            df.attrs['agg_period'] = agg_period
+
+        df.rename(columns={'value': 'observed'}, inplace=True) 
+        return df.dropna(subset=['observed'])
     
-    def _get_data(self,station_ids,constituent,agg_period = 'D',tz_offset = '-6'):
-        '''
-        
-        Returns the processed observational data associated with the calibration specific id. 
-            
+    def get_outlet_data(self,outlet_id,constituent,agg_period = 'D',to_csv = False):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            query = '''
+            SELECT *
+            FROM analytics.outlet_observations_with_flow
+            WHERE outlet_id = ? AND constituent = ?'''
+            df = con.execute(query,[outlet_id,constituent]).fetch_df()    
 
-        Parameters
-        ----------
-        station_id : STR
-            Station ID as a string
-        constituent : TYPE
-            Constituent abbreviation used for calibration. Valid options:
-                'Q',
-                'TSS',
-                'TP',
-                'OP',
-                'TKN',
-                'N',
-                'WT',
-                'DO',
-                'WL']
-        unit : TYPE, optional
-            Units of data. The default is 'mg/l'.
-        sample_flag : TYPE, optional
-            For WPLMN data this flag determines modeled loads are returned. The default is False.
-
-        Returns
-        -------
-        dfsub : Pands.Series
-            Pandas series of data. Note that no metadata is returned.
-
-        '''
-        
-        assert constituent in ['Q','QB','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
-        
         unit = UNIT_DEFAULTS[constituent]
         agg_func = AGG_DEFAULTS[unit]
-            
-        dfsub = pd.concat([self.load(station_id) for station_id in station_ids]) # Check cache
-        dfsub.index = dfsub.index.tz_localize(None) # Drop timezone info
-        #dfsub.set_index('datetime',drop=True,inplace=True)
-        dfsub.rename(columns={'source':'station_origin'},inplace=True)
-        dfsub = dfsub.loc[(dfsub['constituent'] == constituent) &
-                              (dfsub['unit'] == unit),
-                              ['value','station_origin']]   
-        
-        df = dfsub[['value']].resample(agg_period).agg(agg_func)
+
+        df.set_index('datetime',inplace=True)
         df.attrs['unit'] = unit
         df.attrs['constituent'] = constituent
-        
-        if df.empty:
-            
-            return df
-        else:
-            
-            df['station_origin'] = dfsub['station_origin'].iloc[0]
-
-
-        # convert to desired timzone before stripping timezone information.
-        #df.index.tz_convert('UTC-06:00').tz_localize(None)
-    
-        return df['value'].to_frame().dropna()
+        if agg_period is not None:
+            df = df[['value','flow_value','baseflow_value']].resample(agg_period).agg(agg_func)
+            df.attrs['agg_period'] = agg_period
+
+        df.rename(columns={'value': 'observed',
+                           'flow_value': 'observed_flow',
+                           'baseflow_value': 'observed_baseflow'}, inplace=True) 
+        return df.dropna(subset=['observed'])
     
+    def get_raw_data(self,station_id,station_origin, to_csv = False):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            if station_origin.lower() == 'equis':
+                query = '''
+                SELECT *
+                FROM staging.equis_raw
+                WHERE station_id = ?'''
+            elif station_origin.lower() == 'wiski':
+                query = '''
+                SELECT *
+                FROM staging.wiski_raw
+                WHERE station_id = ?'''
+            else:
+                raise ValueError(f'Station origin {station_origin} not recognized. Valid options are equis or wiski.')
+            
+            df = con.execute(query,[station_id]).fetch_df()    
 
-def validate_constituent(constituent):
-    assert constituent in ['Q','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
-
-def validate_unit(unit):
-    assert(unit in ['mg/l','lb','cfs','degF'])
+        if to_csv:
+            df.to_csv(self.folderpath.joinpath(f'{station_id}_raw.csv'), index=False)
+        return df
 
+    def to_csv(self,station_id  ,station_origin,folderpath = None):
+        if folderpath is None:
+            folderpath = self.folderpath
+        else:
+            folderpath = Path(folderpath)
+        df = self.get_station_data([station_id],constituent = 'Q',agg_period = None)
+        if len(df) > 0:
+            df.to_csv(folderpath.joinpath(station_id + '.csv'))
+        else:
+            print(f'No {station_id} calibration data available at Station {station_id}')
+        
+        df.to_csv(folderpath.joinpath(station_id + '.csv'))
 
 
 # class database():