mpcaHydro 2.0.4__py3-none-any.whl → 2.0.5__py3-none-any.whl

mpcaHydro/data/WISKI_QUALITY_CODES.csv

@@ -0,0 +1,71 @@
+quality_code,Text,Description,Active
+0,Unchecked,Unchecked data in progress or data that is not quality coded as part of the workup.  Default coding for shifts so the quality codes from Level and Ratings are used for discharges. ,1
+3,Instantaneous,Instantaneous groundwater measurements or sampled date for load stations.,1
+5,Excellent,Discharge measurements that are excellent.,1
+8,Reliable Interpolation,The value of the data point is an interpolation between adjacent points.  Code used for filling gaps less than 4 hours or with no change in data trend likely based on reference information.,1
+10,Good,Time series data that tracks well and requires no corrections or corrections of very small magnitude or  timeseries data that has been reviewed and accepted for precipitation and groundwater level. Also used for discharge measurements and rating points.  ,1
+15,Fair,Time series data that tracks fairly well and requires some corrections of relatively small magnitude. Also used for discharge measurements and rating points.  ,1
+20,Poor,Time series data that tracks poorly and requires significant or many corrections. Also used for discharge measurements and rating points. ,1
+27,Questionable,"Timeseries data or discharge measurements that are questionable due to operator error, equipment error, etc). Extra scrutiny should be used for these data. ",1
+28,Unknown data quality,"Unknown quality of time series data, ratings or discharge measurements.",1
+29,Modeled,"Time-series data, rating point or discharge from a reliable mathematical and\or computer model. ",1
+34,Estimated,"Time-series data estimated from reference traces, models or extrapolation of the rating curve using supporting data and up to two times the maximum measured discharge.",1
+35,Unreliable,Time-series data computed with a rating extrapolated without supporting data or beyond two times the maximum measured discharge without a model.,1
+36,Threshold Exceedance,"Time-series data may be beyond the measuring limits of the monitoring equipment, or outside the bounds of historical extremes.",1
+40,Default import code,WISKI default coding for gaugings. ,1
+45,Approved Ext Data,"External data that has been graded externally as ""Approved"".",1
+48,Unknown Ext Data,External data that has been graded internally as “Unknown”.,1
+49,Estimated Ext Data,External data that has been graded externally as “Estimated.”  Typically this is finalized ice data.,1
+50,Provisional Ext Data,External data that has been graded internally or externally as “Provisional”.,1
+80,Ice - Estimated,Ice affected time series data. Discharge computed with ice affected stage data is considered estimated.,1
+199,199-Logger Unknown,Initial code for data coming to the system from the logger.,1
+200,200,Initial code for data coming to the system from telemetry or default coding for WISKI timeseries. ,1
+228,Info Parameter,This parameter is collected for informational purposes only.  Data has been through a cursory check only.  This is stored in the database and available upon request.,1
+255,---,System assigned code for gaps in the data set.  Records with null values. ,1
+1,Continuous Data,~Discontinued~ Good TS data that requires no correction.,0
+2,Edited Data,~Discontinued~ TS data that has been edited.  Typically used when spikes are removed or when points are edited manual for datum corrections.,0
+3,Instantaneous Data,Final WQ data.,0
+4,Questionable data,~Discontinued~,0
+5,Excellent measurment,Used to indicated discharge measurements that are excellent as well as excellent sections of the rating.,0
+10,Good measurement,Used to indicated discharge measurements and sections of the rating that are good and time series data that tracks well and requires no corrections or corrections of very small magnitude.,0
+12,Modeled measurement,~Discontinued~ Rating point or discharge was obtained from a relizble mathematical and/or computer model. After 3/1/11 use QC148.,0
+15,Fair measurement,Used to indicated discharge measurements and sections of the rating that are fair and time series data that tracks fairly well and requires some corrections of relatively small magnitude.,0
+20,Poor measurement,Used to indicated discharge measurements and sections of the rating that are poor and time series data that tracks poorly and requires significant or many corrections.,0
+25,Unknown measurement,Measurement data not available.,0
+27,Questionable data,"Flow measurement is very poor and should be given extra scrutiny or time series data that is questionable due to operator error, equipment error, etc.",0
+30,Good Archived Daily Value,This code is used for archived daily value data that is considered “Good”.,0
+31,Fair Archived Daily Value,This code is used for archived daily value data that is considered “Fair”.,0
+32,Poor Archived Daily Value,This code is used for archived daily value data that is considered “Poor”.,0
+33,Unknown Archived Daily Value,This code is used for archived daily value data that has unknown quality based on lack of documentation.,0
+34,Estimated Archived Daily Value,This code is used for archived daily value data that has been estimated.,0
+35,Unreliable Archived Daily Value,This code is used for archived daily value data that is unreliable based on the quality of the supporting time series data and/or rating.,0
+45,Good External Data,This code is used for external data that has been graded internally as “Good”.,0
+46,Fair External Data,This code is used for external data that has been graded internally as “Fair”.,0
+47,Poor External Data,This code is used for external data that has been graded internally as “Poor”.,0
+48,Unknown External Data,This code is used for external data that has been graded internally as “Unknown”,0
+49,Estimated External Data,This code is used for external data that has been graded externally as “Estimated.”  Typically this is finalized ice data.,0
+50,Provisional External Data,This code is used for external data that has been graded internally as “Provisional”,0
+51,Telemetry data - DCP,This code is used for time-series data when imported into hydstra using an automated telemetry method that accesses a DCP through the GOES network.  The “questionable measurement” flag is set through the shef code that accompanies the DCP data.,0
+60,Above rating,~Discontinued~,0
+70,Estimated Data,Value of the data point is estimated.,0
+76,Reliable interpolation,Value of the data point is an interpolation between adjacent points. ,0
+80,Ice,"(DISCONTINUED) Used to indicate ice conditions when the data should not be exported. Use in conjunction with 80 to code 232.00 values, run USDAY to compute daily flow, then recode 232.00 80 values to 180 so unit value export cannot occur.",0
+82,Linear interpolation across a gap in records,~Discontinued~  Points that were added to fill a gap in the data record.  The points fall on a straight line between the end points of the gap. This code was changed to 8 in WISKI.,0
+103,Provisional Instantaneous Data,Provisional WQ data.,0
+130,Good Provisional Daily Value,This code is used for archived daily value data that is considered “Good” but Provisional because there is only one year of gaging measurements.,0
+131,Fair Provisional Daily Value,This code is used for archived daily value data that is considered “Fair” but Provisional because there is only one year of gaging measurements.,0
+132,Poor Provisional Daily Value,This code is used for archived daily value data that is considered “Poor” but Provisional because there is only one year of gaging measurements.,0
+133,Unknown Provisional Archived Daily Value,This code is used for archived daily value data that has unknown quality based on lack of documentation but Provisional because there is only one year of gaging measurements.,0
+134,Estimated Provisional Archived Daily Value,This code is used for archived daily value data that has been estimated but Provisional because there is only one year of gaging measurements.,0
+135,Unreliable Provisional Archived Daily Value,This code is used for archived daily value data that is unreliable based on the quality of the supporting time series data and/or rating but Provisional because there is only one year of gaging measurements.,0
+140,Data not yet checked,This code is used for time-series data when it is initially imported into hydstra using manual import methods.  ,0
+141,Telemetry data - not yet checked,This code is used for time-series data when it is imported into hydstra using an automated telemetry method.,0
+148,Modeled measurement,Rating point or discharge was obtained from a reliable mathematical and/or computer model.,0
+149,Extrapolated rating point,Rating point accurately extrapolated using supporting data and is less than two times the maxiumum measured discharge.,0
+150,Over-extrapolated rating point,Rating point extrapolated without supporting data or beyone two times the maximum measured discharge without a mathematical model.,0
+151,Data Missing,"This code is used to flag the end of a period of missing time-series data, before the next good data value.",0
+160,Above rating,~Discontinued~,0
+169,Datalogger Hardware Error Code 6999,"This code is used to indicate that a time-series point had a value of 6999 or -6999, a typical hardware error code, and the value was changed.",0
+170,Estimated Data,"Used to indicate estimated data when the data should not be exported.  Often used in conjunction with 70 to code 232.00 values, run USDAY to compute daily flow, then recode 232.00 70 values to 170 so unit value export can not occur.",0
+180,Ice,Used to indicate ice conditions.,0
+255,Data Missing,This code is used when data is exported and does not exist for a given time period.,0

mpcaHydro/data_manager.py

@@ -8,89 +8,47 @@ Created on Fri Jun  3 10:01:14 2022
 import pandas as pd
 #from abc import abstractmethod
 from pathlib import Path
-from mpcaHydro import etlWISKI, etlSWD#, etlEQUIS
+from mpcaHydro import etlSWD
+from mpcaHydro import equis, wiski, warehouse
+from mpcaHydro import xref
+from mpcaHydro import outlets
+from mpcaHydro.reports import reportManager
 import duckdb
 
-#
-'''
-Q
-WT
-TSS
-N
-TKN
-OP
-TP
-CHLA
-DO
-
-
-class Station
-
-- id
-- name
-- source
-- data
-
-
-
-
-
-'''
-WISKI_EQUIS_XREF = pd.read_csv(Path(__file__).parent/'data/WISKI_EQUIS_XREF.csv')
-#WISKI_EQUIS_XREF = pd.read_csv('C:/Users/mfratki/Documents/GitHub/hspf_tools/WISKI_EQUIS_XREF.csv')
-
 AGG_DEFAULTS = {'cfs':'mean',
                 'mg/l':'mean',
-                'degF': 'mean',
+                'degf': 'mean',
                 'lb':'sum'}
 
 UNIT_DEFAULTS = {'Q': 'cfs',
+                 'QB': 'cfs',
                  'TSS': 'mg/l',
                  'TP' : 'mg/l',
                  'OP' : 'mg/l',
                  'TKN': 'mg/l',
                  'N'  : 'mg/l',
-                 'WT' : 'degF',
+                 'WT' : 'degf',
                  'WL' : 'ft'}
 
-# VALID_UNITS = {'Q': 'cfs',
-#                  'TSS': 'mg/l','lb',
-#                  'TP' : 'mg/l',
-#                  'OP' : 'mg/l',
-#                  'TKN': 'mg/l',
-#                  'N'  : 'mg/l',
-#                  'WT' : 'degF',
-#                  'WL' : 'ft'}
 
+def validate_constituent(constituent):
+    assert constituent in ['Q','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
+
+def validate_unit(unit):
+    assert(unit in ['mg/l','lb','cfs','degF'])
 
-def are_lists_identical(nested_list):
-    # Sort each sublist
-    sorted_sublists = [sorted(sublist) for sublist in nested_list]
-    # Compare all sublists to the first one
-    return all(sublist == sorted_sublists[0] for sublist in sorted_sublists)                                                                                               
 
-def construct_database(folderpath):
+def build_warehouse(folderpath):
     folderpath = Path(folderpath)
     db_path = folderpath.joinpath('observations.duckdb').as_posix()
-    with duckdb.connect(db_path) as con:
-        con.execute("DROP TABLE IF EXISTS observations")
-        datafiles = folderpath.joinpath('*.csv').as_posix()
-        query = '''
-        CREATE TABLE observations AS SELECT * 
-        FROM
-        read_csv_auto(?,
-                        union_by_name = true);
-        
-        '''
-        con.execute(query,[datafiles])
-
+    warehouse.init_db(db_path)
 
 def constituent_summary(db_path):
     with duckdb.connect(db_path) as con:
         query = '''
         SELECT
           station_id,
-          source,
+          station_origin,
           constituent,
           COUNT(*) AS sample_count,
           year(MIN(datetime)) AS start_date,
@@ -98,7 +56,7 @@ def constituent_summary(db_path):
         FROM
           observations
         GROUP BY
-          constituent, station_id,source
+          constituent, station_id,station_origin
         ORDER BY
           sample_count;'''
           
@@ -108,293 +66,163 @@ def constituent_summary(db_path):
 
 class dataManager():
 
-    def __init__(self,folderpath):
+    def __init__(self,folderpath, oracle_user = None, oracle_password =None):
         
         self.data = {}
         self.folderpath = Path(folderpath)
         self.db_path = self.folderpath.joinpath('observations.duckdb')
-
-    def _reconstruct_database(self):
-        construct_database(self.folderpath)
         
-        
-    def constituent_summary(self,constituents = None):
-        with duckdb.connect(self.db_path) as con:
-            if constituents is None:
-                constituents = con.query('''
-                                        SELECT DISTINCT
-                                        constituent
-                                        FROM observations''').to_df()['constituent'].to_list()
-
-            query = '''
-            SELECT
-            station_id,
-            source,
-            constituent,
-            COUNT(*) AS sample_count,
-            year(MIN(datetime)) AS start_date,
-            year(MAX(datetime)) AS end_date
-            FROM
-            observations
-            WHERE
-            constituent in (SELECT UNNEST(?))
-            GROUP BY
-            constituent,station_id,source
-            ORDER BY
-            constituent,sample_count;'''
-        
-            df = con.execute(query,[constituents]).fetch_df()
-        return df
+        self.oracle_user = oracle_user
+        self.oracle_password = oracle_password
+        warehouse.init_db(self.db_path,reset = False)
+        self.xref = xref
+        self.outlets = outlets
+        self.reports = reportManager(self.db_path)
 
-    def get_wiski_stations(self):
-        return list(WISKI_EQUIS_XREF['WISKI_STATION_NO'].unique())
     
-    def get_equis_stations(self):
-        return list(WISKI_EQUIS_XREF['EQUIS_STATION_ID'].unique())
+    def connect_to_oracle(self):
+        assert (self.credentials_exist(), 'Oracle credentials not found. Set ORACLE_USER and ORACLE_PASSWORD environment variables or use swd as station_origin')
+        equis.connect(user = self.oracle_user, password = self.oracle_password)
     
-    def wiski_equis_alias(self,wiski_station_id):
-        equis_ids =  list(set(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_STATION_NO'] == wiski_station_id,'WISKI_EQUIS_ID'].to_list()))
-        equis_ids = [equis_id for equis_id in equis_ids if not pd.isna(equis_id)]
-        if len(equis_ids) == 0:
-            return []
-        elif len(equis_ids) > 1:
-            print(f'Too Many Equis Stations for {wiski_station_id}')
-            raise 
+    def credentials_exist(self):
+        if (self.oracle_user is not None) & (self.oracle_password is not None):
+            return True
         else:
-            return equis_ids[0]
-
-    def wiski_equis_associations(self,wiski_station_id):
-        equis_ids =  list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_STATION_NO'] == wiski_station_id,'EQUIS_STATION_ID'].unique())
-        equis_ids =  [equis_id for equis_id in equis_ids if not pd.isna(equis_id)]
-        if len(equis_ids) == 0:
-            return []
-        else:
-            return equis_ids
-        
-    def equis_wiski_associations(self,equis_station_id):
-        wiski_ids = list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['EQUIS_STATION_ID'] == equis_station_id,'WISKI_STATION_NO'].unique())
-        wiski_ids = [wiski_id for wiski_id in wiski_ids if not pd.isna(wiski_id)]
-        if len(wiski_ids) == 0:
-            return []
-        else:
-            return wiski_ids
+            return False
         
-    def equis_wiski_alias(self,equis_station_id):
-        wiski_ids =  list(set(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_EQUIS_ID'] == equis_station_id,'WISKI_STATION_NO'].to_list()))
-        wiski_ids = [wiski_id for wiski_id in wiski_ids if not pd.isna(wiski_id)]
-        if len(wiski_ids) == 0:
-            return []
-        elif len(wiski_ids) > 1:
-            print(f'Too Many WISKI Stations for {equis_station_id}')
-            raise 
-        else:
-            return wiski_ids[0]
-
-    def _equis_wiski_associations(self,equis_station_ids):
-        wiski_stations = [self.equis_wiski_associations(equis_station_id) for equis_station_id in equis_station_ids]
-        if are_lists_identical(wiski_stations):
-            return wiski_stations[0]
-        else:
-            return []
-            
-    def _stations_by_wid(self,wid_no,station_origin):
-        if station_origin in ['wiski','wplmn']:
-            station_col = 'WISKI_STATION_NO'
-        elif station_origin in ['equis','swd']:
-            station_col = 'EQUIS_STATION_ID'
-        else:
-            raise
-            
-        return list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WID'] == wid_no,station_col].unique())
+    def _build_warehouse(self):
+        build_warehouse(self.folderpath)
 
+    def download_station_data(self,station_id,station_origin,overwrite=True,to_csv = False,filter_qc_codes = True, start_year = 1996, end_year = 2030,baseflow_method = 'Boughton'):
+        '''
+        Method to download data for a specific station and load it into the warehouse.
+        
+        :param self: Description
+        :param station_id: Station identifier
+        :param station_origin: source of station data: wiski, equis, or swd
+        :param overwrite: Whether to overwrite existing data
+        :param to_csv: Whether to export data to CSV
+        :param filter_qc_codes: Whether to filter quality control codes
+        :param start_year: Start year for data download
+        :param end_year: End year for data download
+        :param baseflow_method: Method for baseflow calculation
+        '''
+        with duckdb.connect(self.db_path,read_only=False) as con:
+            if overwrite:
+                warehouse.drop_station_id(con,station_id,station_origin)
+                warehouse.update_views(con)
+
+            if station_origin == 'wiski':
+                df = wiski.download([station_id],start_year = start_year, end_year = end_year)
+                warehouse.load_df_to_staging(con,df, 'wiski_raw', replace = overwrite)
+                warehouse.load_df_to_analytics(con,wiski.transform(df,filter_qc_codes = filter_qc_codes,baseflow_method = baseflow_method),'wiski') # method includes normalization
+                
+            elif station_origin == 'equis':
+                assert (self.credentials_exist(), 'Oracle credentials not found. Set ORACLE_USER and ORACLE_PASSWORD environment variables or use swd as station_origin')
+                df = equis.download([station_id])
+                warehouse.load_df_to_staging(con,df, 'equis_raw',replace = overwrite)
+                warehouse.load_df_to_analytics(con,equis.transform(df),'equis')
+
+            elif station_origin == 'swd':
+                df = etlSWD.download(station_id)
+                warehouse.load_df_to_staging(con,df, 'swd_raw', replace = overwrite)
+                warehouse.load_df_to_analytics(con,etlSWD.transform(df),'swd')
+            else:
+                raise ValueError('station_origin must be wiski, equis, or swd')    
     
-    def download_stations_by_wid(self, wid_no,station_origin, folderpath = None, overwrite = False):
-
-        station_ids = self._station_by_wid(wid_no,station_origin)
-        
-        if not station_ids.empty:
-            for _, row in station_ids.iterrows():
-                self.download_station_data(row['station_id'],station_origin, folderpath, overwrite)
-
-    def _download_station_data(self,station_id,station_origin,overwrite=False): 
-        assert(station_origin in ['wiski','equis','swd','wplmn'])
-        if station_origin == 'wiski':
-            #equis_stations = list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_STATION_NO'] == station_id,'WISKI_EQUIS_ID'].unique())
-            #[self.download_station_data(equis_station,'equis',overwrite = overwrite) for equis_station in equis_stations]
-            self.download_station_data(station_id,'wiski',overwrite = overwrite)
-            equis_alias = self.wiski_equis_alias(station_id)
-            self.download_station_data(equis_alias,'swd',overwrite = overwrite)
-        elif station_origin == 'wplmn':
-            self.download_station_data(station_id,'wplmn',overwrite = overwrite)
-            equis_alias = self.wiski_equis_alias(station_id)
-            self.download_station_data(equis_alias,'swd',overwrite = overwrite)
-        else:
-            wiski_station = self.equis_wiski_associations(station_id)
-            #wiski_station = WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['EQUIS_STATION_ID'] == station_id,'WISKI_STATION_NO']
-            self.download_station_data(station_id,'equis',overwrite = overwrite)
-            self.download_station_data(wiski_station,'wiski',overwrite = overwrite)
-        
-
-    def download_station_data(self,station_id,source,folderpath=None,overwrite = False):
-        assert(source in ['wiski','equis','swd','wplmn'])
-        station_id = str(station_id)
-        save_name = station_id
-        if source == 'wplmn':
-            save_name = station_id + '_wplmn'
-        
-        if folderpath is None:
-            folderpath = self.folderpath
-        else:
-            folderpath = Path(folderpath)
-        
-        
-        if (folderpath.joinpath(save_name + '.csv').exists()) & (not overwrite):
-            print (f'{station_id} data already downloaded')
-            return
-        
-        if source == 'wiski':
-            data = etlWISKI.download(station_id)
-        elif source == 'swd':
-            data = etlSWD.download(station_id)
-        elif source == 'equis':
-            data = etlSWD.download(station_id)
-        else:
-            data = etlWISKI.download(station_id,wplmn=True)
-            #raise NotImplementedError()
-            #data = etlEQUIS.download(station_id)
+        with duckdb.connect(self.db_path,read_only=False) as con:
+            warehouse.update_views(con)
 
-       
-        
-        if len(data) > 0:
-            data.to_csv(folderpath.joinpath(save_name + '.csv'))
-            self.data[station_id] = data
-        else:
-            print(f'No {source} calibration cata available at Station {station_id}')
-        
-        
-    def _load(self,station_id):
-        df =  pd.read_csv(self.folderpath.joinpath(station_id + '.csv'), 
-                          index_col='datetime', 
-                          parse_dates=['datetime'], 
-                          #usecols=['Ts Date','Station number','variable', 'value','reach_id'],
-                          dtype={'station_id': str, 'value': float, 'variable': str,'constituent':str,'unit':str})
-        self.data[station_id] = df
+        if to_csv:
+            self.to_csv(station_id)
+            
         return df
     
-    def load(self,station_id):
-        try:
-            df = self.data[station_id]
-        except:
-            self._load(station_id)
+    def get_outlets(self):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            query = '''
+            SELECT *
+            FROM outlets.station_reach_pairs
+            ORDER BY outlet_id'''
+            df = con.execute(query).fetch_df()
         return df
+    def get_station_ids(self,station_origin = None):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            if station_origin is None:
+                query = '''
+                SELECT DISTINCT station_id, station_origin
+                FROM analytics.observations'''
+                df = con.execute(query).fetch_df()
+            else:
+                query = '''
+                SELECT DISTINCT station_id
+                FROM analytics.observations
+                WHERE station_origin = ?'''
+                df = con.execute(query,[station_origin]).fetch_df()
+        
+        return df['station_id'].to_list()
     
-    def info(self,constituent):
-        return pd.concat([self._load(file.stem) for file in self.folderpath.iterdir() if file.suffix == '.csv'])[['station_id','constituent','value']].groupby(by = ['station_id','constituent']).count()
-        
-    def get_wplmn_data(self,station_id,constituent,unit = 'mg/l', agg_period = 'YE', samples_only = True):
-        
-        assert constituent in ['Q','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
-        station_id = station_id + '_wplmn'
-        dfsub = self._load(station_id)
-        
-        if samples_only:
-            dfsub = dfsub.loc[dfsub['quality_id'] == 3]
-        agg_func = 'mean'
-        
-        dfsub = dfsub.loc[(dfsub['constituent'] == constituent) & 
-                              (dfsub['unit'] == unit),
-                              ['value','data_format','source']]
 
+    def get_station_data(self,station_ids,constituent,agg_period = None):
         
-        df = dfsub[['value']].resample(agg_period).agg(agg_func)
-        
-        if df.empty:
-            dfsub = df
-        else:
-            
-            df['data_format'] = dfsub['data_format'].iloc[0]
-            df['source'] = dfsub['source'].iloc[0]
-            
-            #if (constituent == 'TSS') & (unit == 'lb'): #convert TSS from lbs to us tons
-            #    dfsub['value'] = dfsub['value']/2000
-    
-            #dfsub = dfsub.resample('H').mean().dropna()
+
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            query = '''
+            SELECT *
+            FROM analytics.observations
+            WHERE station_id IN ? AND constituent = ?'''
+            df = con.execute(query,[station_ids,constituent]).fetch_df()
         
+        unit = UNIT_DEFAULTS[constituent]
+        agg_func = AGG_DEFAULTS[unit]
+
+        df.set_index('datetime',inplace=True)
         df.attrs['unit'] = unit
         df.attrs['constituent'] = constituent
-        return df['value'].to_frame().dropna()
-    
-    def get_data(self,station_id,constituent,agg_period = 'D'):
-        return self._get_data([station_id],constituent,agg_period)
-    
-    def _get_data(self,station_ids,constituent,agg_period = 'D',tz_offset = '-6'):
-        '''
-        
-        Returns the processed observational data associated with the calibration specific id. 
-            
-
-        Parameters
-        ----------
-        station_id : STR
-            Station ID as a string
-        constituent : TYPE
-            Constituent abbreviation used for calibration. Valid options:
-                'Q',
-                'TSS',
-                'TP',
-                'OP',
-                'TKN',
-                'N',
-                'WT',
-                'DO',
-                'WL']
-        unit : TYPE, optional
-            Units of data. The default is 'mg/l'.
-        sample_flag : TYPE, optional
-            For WPLMN data this flag determines modeled loads are returned. The default is False.
+        if agg_period is not None:
+            df = df[['value']].resample(agg_period).agg(agg_func)
+            df.attrs['agg_period'] = agg_period
 
-        Returns
-        -------
-        dfsub : Pands.Series
-            Pandas series of data. Note that no metadata is returned.
+        df.rename(columns={'value': 'observed'}, inplace=True) 
+        return df
+    
+    def get_outlet_data(self,outlet_id,constituent,agg_period = 'D'):
+        with duckdb.connect(self.db_path,read_only=True) as con:
+            query = '''
+            SELECT *
+            FROM analytics.outlet_observations_with_flow
+            WHERE outlet_id = ? AND constituent = ?'''
+            df = con.execute(query,[outlet_id,constituent]).fetch_df()    
 
-        '''
-        
-        assert constituent in ['Q','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
-        
         unit = UNIT_DEFAULTS[constituent]
         agg_func = AGG_DEFAULTS[unit]
-            
-        dfsub = pd.concat([self.load(station_id) for station_id in station_ids]) # Check cache
-        dfsub = dfsub.loc[(dfsub['constituent'] == constituent) &
-                              (dfsub['unit'] == unit),
-                              ['value','data_format','source']]   
-        
-        df = dfsub[['value']].resample(agg_period).agg(agg_func)
+
+        df.set_index('datetime',inplace=True)
         df.attrs['unit'] = unit
         df.attrs['constituent'] = constituent
-        
-        if df.empty:
-            
-            return df
-        else:
-            
-            df['data_format'] = dfsub['data_format'].iloc[0]
-            df['source'] = dfsub['source'].iloc[0]
+        if agg_period is not None:
+            df = df[['value','flow_value','baseflow_value']].resample(agg_period).agg(agg_func)
+            df.attrs['agg_period'] = agg_period
 
+        df.rename(columns={'value': 'observed',
+                           'flow_value': 'observed_flow',
+                           'baseflow_value': 'observed_baseflow'}, inplace=True) 
+        return df
 
-        # convert to desired timzone before stripping timezone information.
-        #df.index.tz_convert('UTC-06:00').tz_localize(None)
-        df.index = df.index.tz_localize(None)
-        return df['value'].to_frame().dropna()
     
 
-def validate_constituent(constituent):
-    assert constituent in ['Q','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
-
-def validate_unit(unit):
-    assert(unit in ['mg/l','lb','cfs','degF'])
-
+    def to_csv(self,station_id,folderpath = None):
+        if folderpath is None:
+            folderpath = self.folderpath
+        else:
+            folderpath = Path(folderpath)
+        df = self._load(station_id)
+        if len(df) > 0:
+            df.to_csv(folderpath.joinpath(station_id + '.csv'))
+        else:
+            print(f'No {station_id} calibration data available at Station {station_id}')
+        
+        df.to_csv(folderpath.joinpath(station_id + '.csv'))
 
 
 # class database():