mpcaHydro 2.1.0__py3-none-any.whl → 2.2.1__py3-none-any.whl

mpcaHydro/{WISKI.py → wiski.py}

@@ -19,12 +19,9 @@ PARAMETERTYPE_MAP ={'11522': 'TP',
                     '11504': 'WT',
                     '11533': 'DO',
                     '11507':'WL'}
-#STATIONPARAMETER_NOS = ['262*','450*','451*','863*','866*','5034' ,'5035','5005', '5004','5014' ,'5015','5024'  ,'5025','5044' ,'5045']
-STATIONPARAMETER_NOS = ['262*','450*','451*','863*','866*']
 
 DATA_CODES = [1,3,10,12,15,20,29,30,31,32,34,45,46,47,48,49]
 
-
 TS_NAME_SELECTOR = {'Q':{'Internal':{'daily':'20.Day.Mean.Archive',
                                      'unit': '15.Rated'},
                          'External': {'daily': '20.Day.Mean',
@@ -62,7 +59,8 @@ TS_NAME_SELECTOR = {'Q':{'Internal':{'daily':'20.Day.Mean.Archive',
                         'External': {'daily': '20.Day.Mean',
                                     'unit': '08.Provisional.Edited'}}}
 
-
+#STATIONPARAMETER_NOS = ['262*','450*','451*','863*','866*','5034' ,'5035','5005', '5004','5014' ,'5015','5024'  ,'5025','5044' ,'5045']
+STATIONPARAMETER_NOS = ['262*','450*','451*','863*','866*']
 
 CONSTITUENT_NAME_NO = {'Q'  :['262*'],#,'263'],
                        'WT' :['450*', '451*'], # '450.42','451.42'],
@@ -74,6 +72,13 @@ CONSTITUENT_NAME_NO = {'Q'  :['262*'],#,'263'],
                        'N'  :None,
                        'TKN':None}
 
+STATIONPARAMETER_NOS_MAP = {'262*':'Q',
+                            '450*':'WT',
+                            '451*':'WT',
+                            '863*':'OP',
+                            '866*':'DO',
+                            '811*':'TRB'}
+
 CONSTITUENT_NAME_NO_WPLMN = {'Q'  :['262*'],#,'263'],
                        'WT' :['450*', '451*'], # '450.42','451.42'],
                        'OP' :['863*','5034' ,'5035'],
@@ -91,6 +96,38 @@ def test_connection():
     '''
     return pywisk.test_connection()
 
+def info(station_ids: list,constituent = None):
+    '''
+    Fetch metadata for given station IDs from WISKI database using the KISTERS API.
+    '''
+    if constituent is not None:
+        stationparameter_nos = CONSTITUENT_NAME_NO[constituent]
+    else:
+        stationparameter_nos = STATIONPARAMETER_NOS
+    
+    df = pywisk.get_ts_ids(station_nos = station_ids,
+                            stationparameter_no = stationparameter_nos,
+                            ts_name = ['15.Rated','09.Archive','08.Provisional.Edited'])
+
+    df = normalize_columns(df)
+
+    # rows = []
+    # for station_id in df['station_id'].unique():            
+    #     for constituent in df.loc[df['station_id'] == station_id,'constituent'].unique():
+    #         df_station_constituent = df.loc[(df['station_id'] == station_id) & (df['constituent'] == constituent) & df['ts_name'].isin(['15.Rated','09.Archive','08.Provisional.Edited'])]
+    #         if not df_station_constituent.empty:
+    #             if station_id.lower().startswith('e'):
+    #                 ts_names = TS_NAME_SELECTOR[constituent]['External']['unit']
+    #             else:
+    #                 ts_names = TS_NAME_SELECTOR[constituent]['Internal']['unit']
+    #             rows.append(df_station_constituent.loc[df_station_constituent['ts_name'] == ts_names,:])
+ 
+    return df
+
+
+
+
+
 def download(station_ids: list, start_year: int = 1996, end_year: int = 2030,wplmn: bool = False):
     '''
     Fetch data for given station IDs from WISKI database using the KISTERS API.
@@ -157,7 +194,7 @@ def download_chunk(ts_id,start_year = 1996,end_year = 2030, interval = 4, as_jso
             end = end_year
         df = pywisk.get_ts(ts_id,start_date = f'{start}-01-01',end_date = f'{end}-12-31',as_json = as_json)
         if not df.empty: frames.append(df)
-        df.index = pd.to_datetime(df['Timestamp'])
+        df['Timestamp'] = pd.to_datetime(df['Timestamp']).dt.tz_localize(None)
         time.sleep(.1)   
     return pd.concat(frames)
 
@@ -197,11 +234,8 @@ def tkn(station_nos,start_year = 1996,end_year = 2030):
     return _download('TKN',station_nos,start_year,end_year)
 
 
-def filter_quality_codes(df):
-    '''
-    Filter dataframe by valid quality codes
-    '''
-    return df.loc[df['Quality Code'].isin(DATA_CODES)]
+
+
 
 def convert_units(df):
     '''
@@ -222,23 +256,48 @@ def convert_units(df):
     return df
 
 
+def map_constituents(df):
+    '''
+    Map stationparameter_no to constituent names
+    '''
+    def map_values(value):
+        for key, replacement in STATIONPARAMETER_NOS_MAP.items():
+            if value.startswith(key.rstrip('*')):  # Match prefix without the wildcard '*'
+                return replacement
+        return value  # If no match, return the original value
+
+    df['constituent'] = df['stationparameter_no'].apply(map_values)
+    return df
+
 def normalize_columns(df):
     '''
     Normalize column names and units
     '''
     # Map parameter numbers to constituent names
-    df['constituent'] = df['parametertype_id'].map(PARAMETERTYPE_MAP)
+    #df['constituent'] = df['stationparameter_no'].map(STATIONPARAMETER_NOS_MAP,regex=True)
     
+    df = map_constituents(df)
+
     df.rename(columns={
         'station_no':'station_id',
         'Timestamp':'datetime',
         'Value':'value',
-        'ts_unitsymbol':'unit'}, inplace=True)
+        'ts_unitsymbol':'unit',
+        'Quality Code':'quality_code',
+        'Quality Code Name':'quality_code_name'}, inplace=True)
     return df
     
+
+
+def filter_quality_codes(df, data_codes):
+    '''
+    Filter dataframe by valid quality codes
+    '''
+    return df.loc[df['quality_code'].isin(data_codes)]
+
 def average_results(df):
-    df['datetime'] = pd.to_datetime(df.loc[:,'datetime'])
-    df['datetime'] = df['datetime'].dt.round('h')
+    #df['datetime'] = pd.to_datetime(df.loc[:,'datetime'])
+    df.loc[:,'datetime'] = df.loc[:,'datetime'].dt.round('h')
     return df.groupby(['station_id', 'datetime', 'constituent', 'unit']).agg(value=('value', 'mean')).reset_index()
     # Convert units
 
@@ -267,14 +326,35 @@ def calculate_baseflow(df, method = 'Boughton'):
     return pd.concat(dfs)
 
 
-def transform(df, baseflow_method = 'Boughton'):
+def normalize(df):
     '''
-    Transform raw WISKI data into standardized format
+    Standardize raw WISKI data into standardized format without transformations.
+    The standardized format includes normalized column names and units.
+    ---
+    Parameters:
+    df (pandas.DataFrame): Raw WISKI data
+    Returns:
+    pandas.DataFrame: Normalized WISKI data
     '''
-    df = filter_quality_codes(df)
+
     df = convert_units(df)
     df = normalize_columns(df)
+    return df
+
+def transform(df, filter_qc_codes = True, data_codes = None, baseflow_method = 'Boughton'):
+    '''
+    Transform normalized WISKI data into standardized format
+    '''
+    df = normalize(df)
+    if filter_qc_codes:
+        if data_codes is None:
+            data_codes = DATA_CODES
+        df = filter_quality_codes(df, data_codes)
     df = average_results(df)
     df = calculate_baseflow(df, method = baseflow_method)
     df['station_origin'] = 'wiski'
+    #df.set_index('datetime',inplace=True)
     return df
+
+
+

mpcaHydro/xref.py

@@ -0,0 +1,74 @@
+import pandas as pd
+from pathlib import Path
+
+WISKI_EQUIS_XREF = pd.read_csv(Path(__file__).parent/'data/WISKI_EQUIS_XREF.csv')
+#WISKI_EQUIS_XREF = pd.read_csv('C:/Users/mfratki/Documents/GitHub/hspf_tools/WISKI_EQUIS_XREF.csv')
+
+
+def are_lists_identical(nested_list):
+    # Sort each sublist
+    sorted_sublists = [sorted(sublist) for sublist in nested_list]
+    # Compare all sublists to the first one
+    return all(sublist == sorted_sublists[0] for sublist in sorted_sublists)                                                                                               
+
+def get_wiski_stations():
+    return list(WISKI_EQUIS_XREF['WISKI_STATION_NO'].unique())
+
+def get_equis_stations():
+    return list(WISKI_EQUIS_XREF['EQUIS_STATION_ID'].unique())
+
+def wiski_equis_alias(wiski_station_id):
+    equis_ids =  list(set(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_STATION_NO'] == wiski_station_id,'WISKI_EQUIS_ID'].to_list()))
+    equis_ids = [equis_id for equis_id in equis_ids if not pd.isna(equis_id)]
+    if len(equis_ids) == 0:
+        return []
+    elif len(equis_ids) > 1:
+        print(f'Too Many Equis Stations for {wiski_station_id}')
+        raise 
+    else:
+        return equis_ids[0]
+
+def wiski_equis_associations(wiski_station_id):
+    equis_ids =  list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_STATION_NO'] == wiski_station_id,'EQUIS_STATION_ID'].unique())
+    equis_ids =  [equis_id for equis_id in equis_ids if not pd.isna(equis_id)]
+    if len(equis_ids) == 0:
+        return []
+    else:
+        return equis_ids
+    
+def equis_wiski_associations(equis_station_id):
+    wiski_ids = list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['EQUIS_STATION_ID'] == equis_station_id,'WISKI_STATION_NO'].unique())
+    wiski_ids = [wiski_id for wiski_id in wiski_ids if not pd.isna(wiski_id)]
+    if len(wiski_ids) == 0:
+        return []
+    else:
+        return wiski_ids
+    
+def equis_wiski_alias(equis_station_id):
+    wiski_ids =  list(set(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_EQUIS_ID'] == equis_station_id,'WISKI_STATION_NO'].to_list()))
+    wiski_ids = [wiski_id for wiski_id in wiski_ids if not pd.isna(wiski_id)]
+    if len(wiski_ids) == 0:
+        return []
+    elif len(wiski_ids) > 1:
+        print(f'Too Many WISKI Stations for {equis_station_id}')
+        raise ValueError(f'Too Many WISKI Stations for {equis_station_id}')
+    else:
+        return wiski_ids[0]
+
+def _equis_wiski_associations(equis_station_ids):
+    wiski_stations = [equis_wiski_associations(equis_station_id) for equis_station_id in equis_station_ids]
+    if are_lists_identical(wiski_stations):
+        return wiski_stations[0]
+    else:
+        return []
+        
+def _stations_by_wid(wid_no,station_origin):
+    if station_origin in ['wiski','wplmn']:
+        station_col = 'WISKI_STATION_NO'
+    elif station_origin in ['equis','swd']:
+        station_col = 'EQUIS_STATION_ID'
+    else:
+        raise
+        
+    return list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WID'] == wid_no,station_col].unique())
+

{mpcahydro-2.1.0.dist-info → mpcahydro-2.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mpcaHydro
-Version: 2.1.0
+Version: 2.2.1
 Summary: Python package for downloading MPCA hydrology data
 Project-URL: Homepage, https://github.com/mfratkin1/mpcaHydro
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>
@@ -10,7 +10,9 @@ Keywords: Hydrology,MPCA
 Classifier: Development Status :: 3 - Alpha
 Classifier: Programming Language :: Python
 Requires-Python: >=3.8
+Requires-Dist: baseflow
 Requires-Dist: duckdb
+Requires-Dist: oracledb
 Requires-Dist: pandas
 Requires-Dist: pathlib
 Requires-Dist: requests

mpcahydro-2.2.1.dist-info/RECORD

@@ -0,0 +1,23 @@
+mpcaHydro/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mpcaHydro/data_manager.py,sha256=COJF4U5qMbYXcalZKThYcyxgJClvnaLUEZm_wWEsXvs,11504
+mpcaHydro/equis.py,sha256=G4pCjfdDggLTrL0lQeGKAqmPsyO4p-sb1DS4KycUInc,18012
+mpcaHydro/etlCSG.py,sha256=5QT6V2dHvNKC9r5-dspt-NpOmECP2LFw1Lyq1zdkqps,2630
+mpcaHydro/etlSWD.py,sha256=F1KmvIlMplbMjjrRk8uyKFTjLp7tiP3uwbuL1rqsPmU,6516
+mpcaHydro/etlWISKI.py,sha256=S1dNUe08Y0riJNBaEJDfgmewR8CwPtIaB_3Vw8JujkM,22201
+mpcaHydro/etlWPLMN.py,sha256=avLJvWRRxsG35w--OVq0qSCrFjO6G2x0aQ31d9kcYHg,4179
+mpcaHydro/outlets.py,sha256=EvdSoUztV4Lz86SaDiFD6g5jpmrdsXMLpN85x9qChdQ,13761
+mpcaHydro/pywisk.py,sha256=kaxJCPCZHy9oEo9VnoSmFC58qm1sX9fVbtp6nXs7934,13290
+mpcaHydro/reports.py,sha256=acNQ37LeAbRPfsSmMEw0LxpZrgMvqRNAnlz93hlzMn8,2047
+mpcaHydro/warehouse.py,sha256=wp8LYE71faudzUDLnCQ1npGS7hZtFLdeQBX8ux6Ak84,26024
+mpcaHydro/warehouseManager.py,sha256=zrQIdzVm09rS5pLNr2szaGUS1-036AAW3D-Xf7xUeSs,1432
+mpcaHydro/wiski.py,sha256=pPW7HjABdyvaa-XvpPmBmHvhVTicv5MF7rjkSvJEDtI,13901
+mpcaHydro/xref.py,sha256=TxMVtUhHKVJ3wZ-445KJaKSoTU3B0VlmSVgT9hX4lLk,3085
+mpcaHydro/data/EQUIS_PARAMETER_XREF.csv,sha256=XZPrcZan9irSqFd4UasnPo_NQBcjyFodi0q3FGQphjI,5667
+mpcaHydro/data/WISKI_EQUIS_XREF.csv,sha256=bPYq-f4-Qc6jsvUgl81lwXBeFamfDe5TjohqUV1XJlg,1244704
+mpcaHydro/data/WISKI_QUALITY_CODES.csv,sha256=PvKBMAUj6pmbhaMUUkAOk0CnxM6RN5oIaXY7MJ_x4T8,9812
+mpcaHydro/data/outlet.duckdb,sha256=yAme7aTdbaMFc_wf005Tzc1Dhu0UAkDSVFNiaa01Y68,2109440
+mpcaHydro/data/stations_EQUIS.gpkg,sha256=unAeRksLe33LuA1vkWaBurenZwxs0ON6fkn7OQvETzY,2072576
+mpcaHydro/data/stations_wiski.gpkg,sha256=SKcjP1oi3qbpZVGWX_VrnM5gjG9o-_PbsG77owKzcWg,937984
+mpcahydro-2.2.1.dist-info/METADATA,sha256=wIS1JMa4S-Kfz4Fs-PklRpw1JvTO_ozeLQxpmqjMQwc,591
+mpcahydro-2.2.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+mpcahydro-2.2.1.dist-info/RECORD,,

mpcahydro-2.1.0.dist-info/RECORD

@@ -1,15 +0,0 @@
-mpcaHydro/WISKI.py,sha256=VWPwmDGv6rKFHgiQrcn4fAyx-h43Tyf8Vjtewa49yj4,11022
-mpcaHydro/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mpcaHydro/data_manager.py,sha256=LofBoNfhfStQlzD8EK1in_15BtNv_8q8057cjK4TmbA,15611
-mpcaHydro/equis.py,sha256=R4BEAkj9I6hVhSmd4WjjMLyQXBcOL5B2YIZjwm4EtqE,17943
-mpcaHydro/etlCSG.py,sha256=5QT6V2dHvNKC9r5-dspt-NpOmECP2LFw1Lyq1zdkqps,2630
-mpcaHydro/etlSWD.py,sha256=FvFP5lIOxtzF3eEgUDGw-C2BJTRfxXxkbt3hYl8PCZQ,6367
-mpcaHydro/etlWISKI.py,sha256=S1dNUe08Y0riJNBaEJDfgmewR8CwPtIaB_3Vw8JujkM,22201
-mpcaHydro/etlWPLMN.py,sha256=avLJvWRRxsG35w--OVq0qSCrFjO6G2x0aQ31d9kcYHg,4179
-mpcaHydro/pywisk.py,sha256=kaxJCPCZHy9oEo9VnoSmFC58qm1sX9fVbtp6nXs7934,13290
-mpcaHydro/warehouse.py,sha256=Rn8onCs9R-EnU9XNA1Gy53B-PAIhW_YaCLBDO29zyMY,7786
-mpcaHydro/data/EQUIS_PARAMETER_XREF.csv,sha256=XZPrcZan9irSqFd4UasnPo_NQBcjyFodi0q3FGQphjI,5667
-mpcaHydro/data/WISKI_EQUIS_XREF.csv,sha256=bPYq-f4-Qc6jsvUgl81lwXBeFamfDe5TjohqUV1XJlg,1244704
-mpcahydro-2.1.0.dist-info/METADATA,sha256=LyNxLeMkwdtq1ox-2ygVqPjqUPk5rFCseONZfziJEwg,543
-mpcahydro-2.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-mpcahydro-2.1.0.dist-info/RECORD,,