mpcaHydro 2.0.5__tar.gz → 2.1.0__tar.gz

{mpcahydro-2.0.5 → mpcahydro-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mpcaHydro
-Version: 2.0.5
+Version: 2.1.0
 Summary: Python package for downloading MPCA hydrology data
 Project-URL: Homepage, https://github.com/mfratkin1/mpcaHydro
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>
@@ -11,7 +11,6 @@ Classifier: Development Status :: 3 - Alpha
 Classifier: Programming Language :: Python
 Requires-Python: >=3.8
 Requires-Dist: duckdb
-Requires-Dist: oracledb
 Requires-Dist: pandas
 Requires-Dist: pathlib
 Requires-Dist: requests

{mpcahydro-2.0.5 → mpcahydro-2.1.0}/pyproject.toml

@@ -5,13 +5,12 @@ build-backend = "hatchling.build"
 [project]
 name = "mpcaHydro"
 urls = { "Homepage" = "https://github.com/mfratkin1/mpcaHydro" }  # ? Add this!
-version = "2.0.5"
+version = "2.1.0"
 dependencies = [
   "pandas",
   "requests",
   "pathlib",
-  "duckdb",
-  "oracledb"
+  "duckdb"
 ]
 requires-python = ">=3.8"
 authors = [

mpcahydro-2.0.5/src/mpcaHydro/wiski.py → mpcahydro-2.1.0/src/mpcaHydro/WISKI.py

@@ -157,7 +157,7 @@ def download_chunk(ts_id,start_year = 1996,end_year = 2030, interval = 4, as_jso
             end = end_year
         df = pywisk.get_ts(ts_id,start_date = f'{start}-01-01',end_date = f'{end}-12-31',as_json = as_json)
         if not df.empty: frames.append(df)
-        df['Timestamp'] = pd.to_datetime(df['Timestamp']).dt.tz_localize(None)
+        df.index = pd.to_datetime(df['Timestamp'])
         time.sleep(.1)   
     return pd.concat(frames)
 
@@ -197,8 +197,11 @@ def tkn(station_nos,start_year = 1996,end_year = 2030):
     return _download('TKN',station_nos,start_year,end_year)
 
 
-
-
+def filter_quality_codes(df):
+    '''
+    Filter dataframe by valid quality codes
+    '''
+    return df.loc[df['Quality Code'].isin(DATA_CODES)]
 
 def convert_units(df):
     '''
@@ -230,22 +233,12 @@ def normalize_columns(df):
         'station_no':'station_id',
         'Timestamp':'datetime',
         'Value':'value',
-        'ts_unitsymbol':'unit',
-        'Quality Code':'quality_code',
-        'Quality Code Name':'quality_code_name'}, inplace=True)
+        'ts_unitsymbol':'unit'}, inplace=True)
     return df
     
-
-
-def filter_quality_codes(df, data_codes):
-    '''
-    Filter dataframe by valid quality codes
-    '''
-    return df.loc[df['quality_code'].isin(data_codes)]
-
 def average_results(df):
-    #df['datetime'] = pd.to_datetime(df.loc[:,'datetime'])
-    df.loc[:,'datetime'] = df.loc[:,'datetime'].dt.round('h')
+    df['datetime'] = pd.to_datetime(df.loc[:,'datetime'])
+    df['datetime'] = df['datetime'].dt.round('h')
     return df.groupby(['station_id', 'datetime', 'constituent', 'unit']).agg(value=('value', 'mean')).reset_index()
     # Convert units
 
@@ -274,35 +267,14 @@ def calculate_baseflow(df, method = 'Boughton'):
     return pd.concat(dfs)
 
 
-def normalize(df):
+def transform(df, baseflow_method = 'Boughton'):
     '''
-    Standardize raw WISKI data into standardized format without transformations.
-    The standardized format includes normalized column names and units.
-    ---
-    Parameters:
-    df (pandas.DataFrame): Raw WISKI data
-    Returns:
-    pandas.DataFrame: Normalized WISKI data
+    Transform raw WISKI data into standardized format
     '''
-
+    df = filter_quality_codes(df)
     df = convert_units(df)
     df = normalize_columns(df)
-    return df
-
-def transform(df, filter_qc_codes = True, data_codes = None, baseflow_method = 'Boughton'):
-    '''
-    Transform normalized WISKI data into standardized format
-    '''
-    df = normalize(df)
-    if filter_qc_codes:
-        if data_codes is None:
-            data_codes = DATA_CODES
-        df = filter_quality_codes(df, data_codes)
     df = average_results(df)
     df = calculate_baseflow(df, method = baseflow_method)
     df['station_origin'] = 'wiski'
-    #df.set_index('datetime',inplace=True)
     return df
-
-
-

mpcahydro-2.1.0/src/mpcaHydro/data_manager.py

@@ -0,0 +1,412 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun  3 10:01:14 2022
+
+@author: mfratki
+"""
+
+import pandas as pd
+#from abc import abstractmethod
+from pathlib import Path
+from mpcaHydro import etlSWD
+from mpcaHydro import equis, wiski, warehouse
+import duckdb
+
+
+WISKI_EQUIS_XREF = pd.read_csv(Path(__file__).parent/'data/WISKI_EQUIS_XREF.csv')
+#WISKI_EQUIS_XREF = pd.read_csv('C:/Users/mfratki/Documents/GitHub/hspf_tools/WISKI_EQUIS_XREF.csv')
+
+AGG_DEFAULTS = {'cfs':'mean',
+                'mg/l':'mean',
+                'degF': 'mean',
+                'lb':'sum'}
+
+UNIT_DEFAULTS = {'Q': 'cfs',
+                 'QB': 'cfs',
+                 'TSS': 'mg/l',
+                 'TP' : 'mg/l',
+                 'OP' : 'mg/l',
+                 'TKN': 'mg/l',
+                 'N'  : 'mg/l',
+                 'WT' : 'degF',
+                 'WL' : 'ft'}
+
+def are_lists_identical(nested_list):
+    # Sort each sublist
+    sorted_sublists = [sorted(sublist) for sublist in nested_list]
+    # Compare all sublists to the first one
+    return all(sublist == sorted_sublists[0] for sublist in sorted_sublists)                                                                                               
+
+def construct_database(folderpath):
+    folderpath = Path(folderpath)
+    db_path = folderpath.joinpath('observations.duckdb').as_posix()
+    with duckdb.connect(db_path) as con:
+        con.execute("DROP TABLE IF EXISTS observations")
+        datafiles = folderpath.joinpath('*.csv').as_posix()
+        query = '''
+        CREATE TABLE observations AS SELECT * 
+        FROM
+        read_csv_auto(?,
+                        union_by_name = true);
+        
+        '''
+        con.execute(query,[datafiles])
+
+
+def build_warehouse(folderpath):
+    folderpath = Path(folderpath)
+    db_path = folderpath.joinpath('observations.duckdb').as_posix()
+    warehouse.init_db(db_path)
+
+def constituent_summary(db_path):
+    with duckdb.connect(db_path) as con:
+        query = '''
+        SELECT
+          station_id,
+          station_origin,
+          constituent,
+          COUNT(*) AS sample_count,
+          year(MIN(datetime)) AS start_date,
+          year(MAX(datetime)) AS end_date
+        FROM
+          observations
+        GROUP BY
+          constituent, station_id,station_origin
+        ORDER BY
+          sample_count;'''
+          
+        res = con.execute(query)
+        return res.fetch_df()
+
+
+class dataManager():
+
+    def __init__(self,folderpath, oracle_user = None, oracle_password =None):
+        
+        self.data = {}
+        self.folderpath = Path(folderpath)
+        self.db_path = self.folderpath.joinpath('observations.duckdb')
+        self.oracle_user = oracle_user
+        self.oracle_password = oracle_password
+    
+    def connect_to_oracle(self):
+        assert (self.credentials_exist(), 'Oracle credentials not found. Set ORACLE_USER and ORACLE_PASSWORD environment variables or use swd as station_origin')
+        equis.connect(user = self.oracle_user, password = self.oracle_password)
+    
+    def credentials_exist(self):
+        if (self.oracle_user is not None) & (self.oracle_password is not None):
+            return True
+        else:
+            return False
+        
+    def _reconstruct_database(self):
+        construct_database(self.folderpath)
+    
+    def _build_warehouse(self):
+        build_warehouse(self.folderpath)
+        
+    def constituent_summary(self,constituents = None):
+        with duckdb.connect(self.db_path) as con:
+            if constituents is None:
+                constituents = con.query('''
+                                        SELECT DISTINCT
+                                        constituent
+                                        FROM observations''').to_df()['constituent'].to_list()
+
+            query = '''
+            SELECT
+            station_id,
+            station_origin,
+            constituent,
+            COUNT(*) AS sample_count,
+            year(MIN(datetime)) AS start_date,
+            year(MAX(datetime)) AS end_date
+            FROM
+            observations
+            WHERE
+            constituent in (SELECT UNNEST(?))
+            GROUP BY
+            constituent,station_id,station_origin
+            ORDER BY
+            constituent,sample_count;'''
+        
+            df = con.execute(query,[constituents]).fetch_df()
+        return df
+
+    def get_wiski_stations(self):
+        return list(WISKI_EQUIS_XREF['WISKI_STATION_NO'].unique())
+    
+    def get_equis_stations(self):
+        return list(WISKI_EQUIS_XREF['EQUIS_STATION_ID'].unique())
+    
+    def wiski_equis_alias(self,wiski_station_id):
+        equis_ids =  list(set(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_STATION_NO'] == wiski_station_id,'WISKI_EQUIS_ID'].to_list()))
+        equis_ids = [equis_id for equis_id in equis_ids if not pd.isna(equis_id)]
+        if len(equis_ids) == 0:
+            return []
+        elif len(equis_ids) > 1:
+            print(f'Too Many Equis Stations for {wiski_station_id}')
+            raise 
+        else:
+            return equis_ids[0]
+
+    def wiski_equis_associations(self,wiski_station_id):
+        equis_ids =  list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_STATION_NO'] == wiski_station_id,'EQUIS_STATION_ID'].unique())
+        equis_ids =  [equis_id for equis_id in equis_ids if not pd.isna(equis_id)]
+        if len(equis_ids) == 0:
+            return []
+        else:
+            return equis_ids
+        
+    def equis_wiski_associations(self,equis_station_id):
+        wiski_ids = list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['EQUIS_STATION_ID'] == equis_station_id,'WISKI_STATION_NO'].unique())
+        wiski_ids = [wiski_id for wiski_id in wiski_ids if not pd.isna(wiski_id)]
+        if len(wiski_ids) == 0:
+            return []
+        else:
+            return wiski_ids
+        
+    def equis_wiski_alias(self,equis_station_id):
+        wiski_ids =  list(set(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_EQUIS_ID'] == equis_station_id,'WISKI_STATION_NO'].to_list()))
+        wiski_ids = [wiski_id for wiski_id in wiski_ids if not pd.isna(wiski_id)]
+        if len(wiski_ids) == 0:
+            return []
+        elif len(wiski_ids) > 1:
+            print(f'Too Many WISKI Stations for {equis_station_id}')
+            raise 
+        else:
+            return wiski_ids[0]
+
+    def _equis_wiski_associations(self,equis_station_ids):
+        wiski_stations = [self.equis_wiski_associations(equis_station_id) for equis_station_id in equis_station_ids]
+        if are_lists_identical(wiski_stations):
+            return wiski_stations[0]
+        else:
+            return []
+            
+    def _stations_by_wid(self,wid_no,station_origin):
+        if station_origin in ['wiski','wplmn']:
+            station_col = 'WISKI_STATION_NO'
+        elif station_origin in ['equis','swd']:
+            station_col = 'EQUIS_STATION_ID'
+        else:
+            raise
+            
+        return list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WID'] == wid_no,station_col].unique())
+
+    
+    def download_stations_by_wid(self, wid_no,station_origin, folderpath = None, overwrite = False):
+
+        station_ids = self._station_by_wid(wid_no,station_origin)
+        
+        if not station_ids.empty:
+            for _, row in station_ids.iterrows():
+                self.download_station_data(row['station_id'],station_origin, folderpath, overwrite)
+
+    def _download_station_data(self,station_id,station_origin,overwrite=False): 
+        assert(station_origin in ['wiski','equis','swd','wplmn'])
+        if station_origin == 'wiski':
+            self.download_station_data(station_id,'wiski',overwrite = overwrite)
+        elif station_origin == 'wplmn':
+            self.download_station_data(station_id,'wplmn',overwrite = overwrite)
+        elif station_origin == 'swd':
+            self.download_station_data(station_id,'swd',overwrite = overwrite)
+        else:
+            self.download_station_data(station_id,'equis',overwrite = overwrite)
+
+
+       
+
+    def download_station_data(self,station_id,station_origin,start_year = 1996, end_year = 2030,folderpath=None,overwrite = False,baseflow_method = 'Boughton'):
+        assert(station_origin in ['wiski','equis','swd','wplmn'])
+        station_id = str(station_id)
+        save_name = station_id
+        if station_origin == 'wplmn':
+            save_name = station_id + '_wplmn'
+        
+        if folderpath is None:
+            folderpath = self.folderpath
+        else:
+            folderpath = Path(folderpath)
+        
+        
+        if (folderpath.joinpath(save_name + '.csv').exists()) & (not overwrite):
+            print (f'{station_id} data already downloaded')
+            return
+        
+        if station_origin == 'wiski':
+            data = wiski.transform(wiski.download([station_id],wplmn=False, baseflow_method = baseflow_method))
+        elif station_origin == 'swd':
+            data = etlSWD.download(station_id)
+        elif station_origin == 'equis':
+            assert (self.credentials_exist(), 'Oracle credentials not found. Set ORACLE_USER and ORACLE_PASSWORD environment variables or use swd as station_origin')
+            data = equis.transform(equis.download([station_id]))
+        else:
+            data = wiski.transform(wiski.download([station_id],wplmn=True, baseflow_method = baseflow_method))
+        
+
+       
+        
+        if len(data) > 0:
+            data.to_csv(folderpath.joinpath(save_name + '.csv'))
+            self.data[station_id] = data
+        else:
+            print(f'No {station_origin} calibration cata available at Station {station_id}')
+        
+    def _load(self,station_id):
+        with duckdb.connect(self.db_path) as con:
+            query = '''
+            SELECT *
+            FROM analytics.observations
+            WHERE station_id = ?'''
+            df = con.execute(query,[station_id]).fetch_df()
+            df.set_index('datetime',inplace=True)
+            self.data[station_id] = df
+            return df
+
+    def _load2(self,station_id):
+        df =  pd.read_csv(self.folderpath.joinpath(station_id + '.csv'), 
+                          index_col='datetime', 
+                          parse_dates=['datetime'], 
+                          #usecols=['Ts Date','Station number','variable', 'value','reach_id'],
+                          dtype={'station_id': str, 'value': float, 'variable': str,'constituent':str,'unit':str})
+        self.data[station_id] = df
+        return df
+    
+    def load(self,station_id):
+        try:
+            df = self.data[station_id]
+        except:
+            df = self._load(station_id)
+        return df
+    
+    def info(self,constituent):
+        return pd.concat([self._load(file.stem) for file in self.folderpath.iterdir() if file.suffix == '.csv'])[['station_id','constituent','value']].groupby(by = ['station_id','constituent']).count()
+        
+    def get_wplmn_data(self,station_id,constituent,unit = 'mg/l', agg_period = 'YE', samples_only = True):
+        
+        assert constituent in ['Q','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
+        station_id = station_id + '_wplmn'
+        dfsub = self._load(station_id)
+        
+        if samples_only:
+            dfsub = dfsub.loc[dfsub['quality_id'] == 3]
+        agg_func = 'mean'
+        
+        dfsub = dfsub.loc[(dfsub['constituent'] == constituent) & 
+                              (dfsub['unit'] == unit),
+                              ['value','station_origin']]
+
+        
+        df = dfsub[['value']].resample(agg_period).agg(agg_func)
+        
+        if df.empty:
+            dfsub = df
+        else:
+            
+            df['station_origin'] = dfsub['station_origin'].iloc[0]
+            
+            #if (constituent == 'TSS') & (unit == 'lb'): #convert TSS from lbs to us tons
+            #    dfsub['value'] = dfsub['value']/2000
+    
+            #dfsub = dfsub.resample('H').mean().dropna()
+        
+        df.attrs['unit'] = unit
+        df.attrs['constituent'] = constituent
+        return df['value'].to_frame().dropna()
+    
+    def get_data(self,station_id,constituent,agg_period = 'D'):
+        return self._get_data([station_id],constituent,agg_period)
+    
+    def _get_data(self,station_ids,constituent,agg_period = 'D',tz_offset = '-6'):
+        '''
+        
+        Returns the processed observational data associated with the calibration specific id. 
+            
+
+        Parameters
+        ----------
+        station_id : STR
+            Station ID as a string
+        constituent : TYPE
+            Constituent abbreviation used for calibration. Valid options:
+                'Q',
+                'TSS',
+                'TP',
+                'OP',
+                'TKN',
+                'N',
+                'WT',
+                'DO',
+                'WL']
+        unit : TYPE, optional
+            Units of data. The default is 'mg/l'.
+        sample_flag : TYPE, optional
+            For WPLMN data this flag determines modeled loads are returned. The default is False.
+
+        Returns
+        -------
+        dfsub : Pands.Series
+            Pandas series of data. Note that no metadata is returned.
+
+        '''
+        
+        assert constituent in ['Q','QB','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
+        
+        unit = UNIT_DEFAULTS[constituent]
+        agg_func = AGG_DEFAULTS[unit]
+            
+        dfsub = pd.concat([self.load(station_id) for station_id in station_ids]) # Check cache
+        dfsub.index = dfsub.index.tz_localize(None) # Drop timezone info
+        #dfsub.set_index('datetime',drop=True,inplace=True)
+        dfsub.rename(columns={'source':'station_origin'},inplace=True)
+        dfsub = dfsub.loc[(dfsub['constituent'] == constituent) &
+                              (dfsub['unit'] == unit),
+                              ['value','station_origin']]   
+        
+        df = dfsub[['value']].resample(agg_period).agg(agg_func)
+        df.attrs['unit'] = unit
+        df.attrs['constituent'] = constituent
+        
+        if df.empty:
+            
+            return df
+        else:
+            
+            df['station_origin'] = dfsub['station_origin'].iloc[0]
+
+
+        # convert to desired timzone before stripping timezone information.
+        #df.index.tz_convert('UTC-06:00').tz_localize(None)
+    
+        return df['value'].to_frame().dropna()
+    
+
+def validate_constituent(constituent):
+    assert constituent in ['Q','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
+
+def validate_unit(unit):
+    assert(unit in ['mg/l','lb','cfs','degF'])
+
+
+
+# class database():
+#     def __init__(self,db_path):
+#         self.dbm = MonitoringDatabase(db_path)
+        
+    
+#     def get_timeseries(self,station_ds, constituent,agg_period):      
+#         validate_constituent(constituent)
+#         unit = UNIT_DEFAULTS[constituent]
+#         agg_func = AGG_DEFAULTS[unit]
+#         return odm.get_timeseries(station_id,constituent)
+
+    
+#     def get_samples(self,station_ds, constituent,agg_period):
+#         validate_constituent(constituent)
+#         unit = UNIT_DEFAULTS[constituent]
+#         agg_func = AGG_DEFAULTS[unit]
+#         return odm.get_sample(station_id,constituent)
+
+#     def get_samples_and_timeseries(self,station_ds, constituent,agg_period)
+        

{mpcahydro-2.0.5 → mpcahydro-2.1.0}/src/mpcaHydro/equis.py

@@ -164,25 +164,26 @@ def as_utc_offset(naive_dt: Union[datetime, str], tz_label: str, target_offset:
     aware_src = naive.replace(tzinfo=src_tz)
 
     # convert the instant to fixed UTC-6
-    return aware_src.astimezone(target_offset).tz_localize(None)
+    return aware_src.astimezone(target_offset)
 
 
 def normalize_columns(df):
     '''Select relevant columns from Equis data.'''
     return df[['SYS_LOC_CODE',
-               'constituent',
-               'CAS_RN',
                'datetime',
                'RESULT_NUMERIC',
                'RESULT_UNIT',
+               'constituent'
                ]].rename(columns={
                    'SYS_LOC_CODE':'station_id',
                    'RESULT_NUMERIC':'value',
-                   'RESULT_UNIT':'unit',
-                   'CAS_RN':'cas_rn'
+                   'RESULT_UNIT':'unit'
                })
 
-
+def replace_nondetects(df):
+    '''Replace non-detect results with 0 in Equis data.'''
+    df.loc[df['RESULT_NUMERIC'].isna(), 'RESULT_NUMERIC'] = 0
+    return df
 
 def normalize_timezone(df):
     '''Normalize datetime to UTC in Equis data.'''
@@ -193,27 +194,27 @@ def normalize_timezone(df):
         except Exception:
             return pd.NaT
 
-    df.loc[:,'datetime'] = df.apply(_conv, axis=1)
+    df['datetime'] = df.apply(_conv, axis=1)
     return df
 
 def convert_units(df):
     '''Convert units in Equis data to standard units.'''
     # Convert ug/L to mg/L
-    df['unit'] = df['unit'].str.lower()
+    df['RESULT_UNIT'] = df['RESULT_UNIT'].str.lower()
 
-    mask_ugL = df['unit'] == 'ug/l'
-    df.loc[mask_ugL, 'value'] = df.loc[mask_ugL, 'value'] / 1000
-    df.loc[mask_ugL, 'unit'] = 'mg/l'
+    mask_ugL = df['RESULT_UNIT'] == 'ug/l'
+    df.loc[mask_ugL, 'RESULT_NUMERIC'] = df.loc[mask_ugL, 'RESULT_NUMERIC'] / 1000
+    df.loc[mask_ugL, 'RESULT_UNIT'] = 'mg/l'
 
     # Convert mg/g to mg/L (assuming density of 1 g/mL)
-    mask_mgg = df['unit'] == 'mg/g'
-    df.loc[mask_mgg, 'value'] = df.loc[mask_mgg, 'value'] * 1000
-    df.loc[mask_mgg, 'unit'] = 'mg/l'
+    mask_mgg = df['RESULT_UNIT'] == 'mg/g'
+    df.loc[mask_mgg, 'RESULT_NUMERIC'] = df.loc[mask_mgg, 'RESULT_NUMERIC'] * 1000
+    df.loc[mask_mgg, 'RESULT_UNIT'] = 'mg/l'
 
     # Convert deg C to degF
-    mask_degC = df['unit'].isin(['deg c', 'degc'])
-    df.loc[mask_degC, 'value'] = (df.loc[mask_degC, 'value'] * 9/5) + 32
-    df.loc[mask_degC, 'unit'] = 'degf'
+    mask_degC = df['RESULT_UNIT'].isin(['deg c', 'degc'])
+    df.loc[mask_degC, 'RESULT_NUMERIC'] = (df.loc[mask_degC, 'RESULT_NUMERIC'] * 9/5) + 32
+    df.loc[mask_degC, 'RESULT_UNIT'] = 'degf'
 
     return df
 
@@ -231,25 +232,15 @@ def average_results(df):
         value=('value', 'mean')
     ).reset_index()
 
-def replace_nondetects(df):
-    '''Replace non-detect results with 0 in Equis data.'''
-    df.loc[df['value'].isna(), 'value'] = 0
-    return df
-
-def normalize(df):
-    '''Normalize Equis data: select relevant columns.'''
-    df = map_constituents(df)
-    df = normalize_timezone(df)
-    df = normalize_columns(df)
-    df = convert_units(df)
-    return df
-
 def transform(df):
     '''Transform Equis data: handle non-detects, convert units, map constituents.'''
     
-    df = normalize(df)
     df = replace_nondetects(df)
     if not df.empty:
+        df = normalize_timezone(df)
+        df = convert_units(df)
+        df = map_constituents(df)
+        df = normalize_columns(df)
         df = average_results(df)
     return df