mpcaHydro 2.0.1__py3-none-any.whl → 2.0.3__py3-none-any.whl

mpcaHydro/data_manager.py

@@ -9,7 +9,7 @@ import pandas as pd
 #from abc import abstractmethod
 from pathlib import Path
 from mpcaHydro import etlWISKI, etlSWD#, etlEQUIS
-
+import duckdb
 
 #
 '''
@@ -69,12 +69,82 @@ def are_lists_identical(nested_list):
     # Compare all sublists to the first one
     return all(sublist == sorted_sublists[0] for sublist in sorted_sublists)                                                                                               
 
+def construct_database(folderpath):
+    folderpath = Path(folderpath)
+    db_path = folderpath.joinpath('observations.duckdb').as_posix()
+    with duckdb.connect(db_path) as con:
+        con.execute("DROP TABLE IF EXISTS observations")
+        datafiles = folderpath.joinpath('*.csv').as_posix()
+        query = '''
+        CREATE TABLE observations AS SELECT * 
+        FROM
+        read_csv_auto(?,
+                        union_by_name = true);
+        
+        '''
+        con.execute(query,[datafiles])
+
+
+def constituent_summary(db_path):
+    with duckdb.connect(db_path) as con:
+        query = '''
+        SELECT
+          station_id,
+          source,
+          constituent,
+          COUNT(*) AS sample_count,
+          year(MIN(datetime)) AS start_date,
+          year(MAX(datetime)) AS end_date
+        FROM
+          observations
+        GROUP BY
+          constituent, station_id,source
+        ORDER BY
+          sample_count;'''
+          
+        res = con.execute(query)
+        return res.fetch_df()
+
+
 class dataManager():
 
     def __init__(self,folderpath):
         
         self.data = {}
         self.folderpath = Path(folderpath)
+        self.db_path = self.folderpath.joinpath('observations.duckdb')
+
+    def _reconstruct_database(self):
+        construct_database(self.folderpath)
+        
+        
+    def constituent_summary(self,constituents = None):
+        with duckdb.connect(self.db_path) as con:
+            if constituents is None:
+                constituents = con.query('''
+                                        SELECT DISTINCT
+                                        constituent
+                                        FROM observations''').to_df()['constituent'].to_list()
+
+            query = '''
+            SELECT
+            station_id,
+            source,
+            constituent,
+            COUNT(*) AS sample_count,
+            year(MIN(datetime)) AS start_date,
+            year(MAX(datetime)) AS end_date
+            FROM
+            observations
+            WHERE
+            constituent in (SELECT UNNEST(?))
+            GROUP BY
+            constituent,station_id,source
+            ORDER BY
+            constituent,sample_count;'''
+        
+            df = con.execute(query,[constituents]).fetch_df()
+        return df
 
     def get_wiski_stations(self):
         return list(WISKI_EQUIS_XREF['WISKI_STATION_NO'].unique())
@@ -108,6 +178,17 @@ class dataManager():
             return []
         else:
             return wiski_ids
+        
+    def equis_wiski_alias(self,equis_station_id):
+        wiski_ids =  list(set(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WISKI_EQUIS_ID'] == equis_station_id,'WISKI_STATION_NO'].to_list()))
+        wiski_ids = [wiski_id for wiski_id in wiski_ids if not pd.isna(wiski_id)]
+        if len(wiski_ids) == 0:
+            return []
+        elif len(wiski_ids) > 1:
+            print(f'Too Many WISKI Stations for {equis_station_id}')
+            raise 
+        else:
+            return wiski_ids[0]
 
     def _equis_wiski_associations(self,equis_station_ids):
         wiski_stations = [self.equis_wiski_associations(equis_station_id) for equis_station_id in equis_station_ids]
@@ -115,6 +196,25 @@ class dataManager():
             return wiski_stations[0]
         else:
             return []
+            
+    def _stations_by_wid(self,wid_no,station_origin):
+        if station_origin in ['wiski','wplmn']:
+            station_col = 'WISKI_STATION_NO'
+        elif station_origin in ['equis','swd']:
+            station_col = 'EQUIS_STATION_ID'
+        else:
+            raise
+            
+        return list(WISKI_EQUIS_XREF.loc[WISKI_EQUIS_XREF['WID'] == wid_no,station_col].unique())
+
+    
+    def download_stations_by_wid(self, wid_no,station_origin, folderpath = None, overwrite = False):
+
+        station_ids = self._station_by_wid(wid_no,station_origin)
+        
+        if not station_ids.empty:
+            for _, row in station_ids.iterrows():
+                self.download_station_data(row['station_id'],station_origin, folderpath, overwrite)
 
     def _download_station_data(self,station_id,station_origin,overwrite=False): 
         assert(station_origin in ['wiski','equis','swd','wplmn'])
@@ -232,7 +332,7 @@ class dataManager():
     def get_data(self,station_id,constituent,agg_period = 'D'):
         return self._get_data([station_id],constituent,agg_period)
     
-    def _get_data(self,station_ids,constituent,agg_period = 'D'):
+    def _get_data(self,station_ids,constituent,agg_period = 'D',tz_offset = '-6'):
         '''
         
         Returns the processed observational data associated with the calibration specific id. 
@@ -287,7 +387,10 @@ class dataManager():
             df['data_format'] = dfsub['data_format'].iloc[0]
             df['source'] = dfsub['source'].iloc[0]
 
-       
+
+        # convert to desired timzone before stripping timezone information.
+        #df.index.tz_convert('UTC-06:00').tz_localize(None)
+        df.index = df.index.tz_localize(None)
         return df['value'].to_frame().dropna()
     
 

mpcaHydro/etlCSG.py

@@ -6,6 +6,9 @@ Created on Tue Oct 10 14:13:23 2023
 """
 
 import pandas as pd
+import requests
+import zipfile
+import io
 # import geopandas as gpd
 
 
@@ -14,20 +17,30 @@ CONSITUENT_MAP = {'Water Temp. (C)': 'WT',
                  'DO (mg/L)': 'DO'
     }
 
-def download(station_no):
-    # save_path = Path(save_path)
-    # file_path = save_path.joinpath('csg.csv')
-
-    station = station_no[1:]
-    df = pd.read_csv(f'https://maps2.dnr.state.mn.us/cgi-bin/csg.cgi?mode=dump_hydro_data_as_csv&site={station}&startdate=1996-1-1&enddate=2050-1-1')
-    df['station_id'] = station_no
+# def download(station_no):
+#     # save_path = Path(save_path)
+#     # file_path = save_path.joinpath('csg.csv')
 
-    return df
+#     station = station_no[1:]
+#     df = pd.read_csv(f'https://maps2.dnr.state.mn.us/cgi-bin/csg.cgi?mode=dump_hydro_data_as_csv&site={station}&startdate=1996-1-1&enddate=2050-1-1')
+#     df = pd.read_csv(f'https://apps.dnr.state.mn.us/csg/api/v1/download?callback=json&ids=66050001&vars=262')
+#     df['station_id'] = station_no
 
+#     return df
 
+def download(station_no):
+    station = station_no[1:]
+    url = f'https://apps.dnr.state.mn.us/csg/api/v1/download?ids={station}&vars=262'
+    response = requests.get(url)
+    with zipfile.ZipFile(io.BytesIO(response.content)) as zip_ref:
+        df = pd.read_csv(zip_ref.open(zip_ref.namelist()[0]))
+        df['station_id'] = station_no
+      
+    return df
+      
 
     # def process(df):
-    #     df['Timestamp'] = pd.to_datetime(df['Timestamp'])
+    #     
     #     df.set_index('Timestamp',inplace=True)
     #     value_variables = [column for column in df.columns if (column not in ['Site','Timestamp','station_no']) & ~(column.endswith('Quality'))]
         
@@ -35,45 +48,24 @@ def download(station_no):
     #     df = df['Value'].resample(rule='1H', kind='interval').mean().to_frame()
         
 def transform(data):
-    
-    
-    data['Timestamp'] = pd.to_datetime(data['Timestamp'])
-    data['Timestamp'].dt.tz_localize('UTC')
-    
-    id_columns = ['Timestamp','station_id']
-    quality_columns = ['Water Temp. (C) Quality',
-                     'Discharge (cfs) Quality',
-                     'DO (mg/L) Quality']
-    
-    value_columns = ['Water Temp. (C)',
-                     'Discharge (cfs)',
-                     'DO (mg/L)']
-
-    value_columns = [column for column in data.columns if column in value_columns]
-    quality_columns = [column for column in data.columns if column in quality_columns]
+    data.rename(columns = {'tstamp': 'datetime',
+                                'var_name': 'variable',
+                                'station_no': 'station_id'}, inplace = True)
 
-
-    
-    data_melt = pd.melt(data,col_level=0,id_vars = id_columns,value_vars = value_columns)
-    data_melt['Quality'] = pd.melt(data,col_level=0,id_vars = id_columns,value_vars = quality_columns)['value']
-
-    data_melt.rename(columns = {'Timestamp': 'datetime',
-                                'Value': 'value',
-                                'stationparameter_name': 'variable',
-                                'station_no': 'station_id',
-                                'Quality' : 'quality'},inplace = True)
-    
-    data_melt['unit'] = data_melt['variable'].map({'Water Temp. (C)' : 'C',
+    data['unit'] = data['variable'].map({'Water Temp. (C)' : 'C',
                                                    'Discharge (cfs)' : 'cfs',
                                                    'DO (mg/L)' : 'mg/L'})   
     
-    data_melt['constituent'] = data_melt['variable'].map({'Water Temp. (C)' : 'WT',
+    data['constituent'] = data['variable'].map({'Water Temp. (C)' : 'WT',
                                                           'Discharge (cfs)' : 'Q',
                                                           'DO (mg/L)' : 'DO'})
   
-    data_melt.dropna(subset = 'value',inplace=True)
+    data['datetime'] = pd.to_datetime(data['datetime'])
+    data.set_index('datetime',drop=True,inplace=True)
+    data.index = data.index.tz_localize('UTC-06:00')
+    data.dropna(subset = 'value',inplace=True)
     data['source'] = 'csg'
-    return data_melt
+    return data
 
 
 

mpcaHydro/etlSWD.py

@@ -14,16 +14,25 @@ import pandas as pd
 
                    
 CONSTITUENT_MAP = {'Total suspended solids':'TSS',
+                   'Total solids': 'TSS',
+                   'Solids, Suspended' : 'TSS',
+                   'Solids, Total Suspended' : 'TSS',
                   'Residue - nonfilterable (TSS)': 'TSS',
                  'Kjeldahl nitrogen as N': 'TKN',
+                 'Inorganic nitrogen (nitrate and nitrate) as N': 'N',
                  'Nitrogen, Total Kjeldahl (TKN) as N': 'TKN',
                  'Nitrate + Nitrite Nitrogen, Total as N': 'N',
                  'Nitrate/Nitrite as N (N+N) as N': 'N',
                  'Nutrient-nitrogen as N': 'N',
+                 'Nitrate/Nitrite as N': 'N',
                  'Phosphorus, Total as P as P':'TP',
+                 'Phosphorus, Total as P' : 'TP',
                  'Phosphorus as P': 'TP',
+                 'Total Phosphorus as P': 'TP',
+                 'Orthophosphate as P': 'OP',
                  'Carbonaceous biochemical oxygen demand, standard conditions': 'BOD',
                  'Chemical oxygen demand':'BOD',
+                 'Biochemical oxygen demand, standard conditions': 'BOD',
                  'Chlorophyll a, corrected for pheophytin':'CHLA',
                  'Chlorophyll-A':'CHLA',
                  'Chlorophyll-a, Pheophytin Corrected':'CHLA',
@@ -145,7 +154,7 @@ def transform(df):
     df.set_index('datetime',drop=True,inplace=True)
     df.index = df.index.tz_localize('UTC-06:00')
     
-    df.index = df.index.round('H').round('H')
+    df.index = df.index.round('h').round('h')
     df = df.reset_index()
     df = df.groupby(['datetime','variable','unit','station_id','station_name','constituent','data_format','data_type','source']).mean()
     df = df.reset_index()

{mpcahydro-2.0.1.dist-info → mpcahydro-2.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mpcaHydro
-Version: 2.0.1
+Version: 2.0.3
 Summary: Python package for downloading MPCA hydrology data
 Project-URL: Homepage, https://github.com/mfratkin1/mpcaHydro
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>
@@ -10,6 +10,7 @@ Keywords: Hydrology,MPCA
 Classifier: Development Status :: 3 - Alpha
 Classifier: Programming Language :: Python
 Requires-Python: >=3.8
+Requires-Dist: duckdb
 Requires-Dist: pandas
 Requires-Dist: pathlib
 Requires-Dist: requests

{mpcahydro-2.0.1.dist-info → mpcahydro-2.0.3.dist-info}/RECORD

@@ -1,11 +1,11 @@
 mpcaHydro/WISKI.py,sha256=yqsljbx8TlFA8HIrXFGs5meO0RcTis5Px3__UUzrtiI,13303
 mpcaHydro/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mpcaHydro/data_manager.py,sha256=QAjWBwSAd2ziQ7KbujzedOnYy6YJiRXJ3-4imCpNwys,11682
-mpcaHydro/etlCSG.py,sha256=gPk6D2r0R0Okx-S0C9vLtRlmGzf9tExtVoJZrj8IA8U,2950
-mpcaHydro/etlSWD.py,sha256=rn71939arFQ08gSrRMKg1JbTBH_4GV4d0zBPp-opH18,7021
+mpcaHydro/data_manager.py,sha256=UR4mE93eUUXXs74qnJCFstNt_z0yaX1IB8USD4-XkTc,15396
+mpcaHydro/etlCSG.py,sha256=5QT6V2dHvNKC9r5-dspt-NpOmECP2LFw1Lyq1zdkqps,2630
+mpcaHydro/etlSWD.py,sha256=FnpFv-LjK2zAvI2-wrN_4YaS70bI1AGi-aX5lEevkrc,7509
 mpcaHydro/etlWISKI.py,sha256=6I1uTJfM-yL_hY0q-X0JKFqz9DVDaFR7wt4ssmjbcEU,19645
 mpcaHydro/etlWPLMN.py,sha256=b44xvx4s7lwXhpRtfR6rj7RnBpbVKXaYqZCr26BexUI,4160
 mpcaHydro/data/WISKI_EQUIS_XREF.csv,sha256=bPYq-f4-Qc6jsvUgl81lwXBeFamfDe5TjohqUV1XJlg,1244704
-mpcahydro-2.0.1.dist-info/METADATA,sha256=hG1tAHrflPN5fBcHhxHVwG6C0lL6WEqeGrrVz-oLsx4,521
-mpcahydro-2.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-mpcahydro-2.0.1.dist-info/RECORD,,
+mpcahydro-2.0.3.dist-info/METADATA,sha256=FKpSp78k6axfes_kk4NL_-VdsyuSKeGRla3ZC5lxY8M,543
+mpcahydro-2.0.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+mpcahydro-2.0.3.dist-info/RECORD,,