mpcaHydro 2.0.3__tar.gz → 2.0.4__tar.gz

{mpcahydro-2.0.3 → mpcahydro-2.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mpcaHydro
-Version: 2.0.3
+Version: 2.0.4
 Summary: Python package for downloading MPCA hydrology data
 Project-URL: Homepage, https://github.com/mfratkin1/mpcaHydro
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>

{mpcahydro-2.0.3 → mpcahydro-2.0.4}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "mpcaHydro"
 urls = { "Homepage" = "https://github.com/mfratkin1/mpcaHydro" }  # ? Add this!
-version = "2.0.3"
+version = "2.0.4"
 dependencies = [
   "pandas",
   "requests",
@@ -27,3 +27,12 @@ classifiers = [
   "Development Status :: 3 - Alpha",
   "Programming Language :: Python"
 ]
+
+[tool.pixi.workspace]
+channels = ["conda-forge"]
+platforms = ["win-64"]
+
+[tool.pixi.pypi-dependencies]
+mpcaHydro = { path = ".", editable = true }
+
+[tool.pixi.tasks]

{mpcahydro-2.0.3 → mpcahydro-2.0.4}/src/mpcaHydro/WISKI.py

@@ -71,22 +71,19 @@ class Service():
         self._url = url
         return url
     
-
-    def df(self,args_dict):
-        
+    def get_json(self,args_dict):
         # Download request
-        # print('Downloading')
         response = requests.get(self.url(args_dict))
-        
         response.raise_for_status()  # raises exception when not a 2xx response
-        
-        
-        
         if response.status_code != 200:
             print('Error: ' + response.json()['message'])
             return 1
+        return  response.json()  
+    
+    def df(self,args_dict):
 
-        get_requests = response.json()    
+
+        get_requests = self.get_json(args_dict)
         # Convert to dataframe
         if args_dict['request'] in ['getTimeseriesValues']:
             dfs = []
@@ -164,7 +161,8 @@ class pyWISK():
                start_date = '1996-01-01',
                end_date = '2050-12-31',
                stationgroup_id = None,
-               timezone = 'GMT-6'):
+               timezone = 'GMT-6',
+               as_json = False):
         
         if ts_ids is None:
             print('Determing Timeseries IDs')
@@ -192,9 +190,12 @@ class pyWISK():
                 'timezone':timezone,
                 'ca_sta_returnfields': ['stn_HUC12','stn_EQuIS_ID']}
         
-        df = self.service.get(args)
+        if as_json:
+            output = self.service.get_json(args)
+        else: 
+            output = self.service.get(args)
         #print('Done!')
-        return df
+        return output
         
     def get_stations(self,
                      huc_id = None, 

mpcahydro-2.0.4/src/mpcaHydro/data/EQUIS_PARAMETER_XREF.csv

@@ -0,0 +1,88 @@
+CAS_RN,CHEMICAL_NAME,PARAMETER,constituent
+BOD,"Biochemical oxygen demand, standard conditions",Biochemical Oxygen Demand,BOD
+BOD,"Biochemical oxygen demand, standard conditions",Biochemical Oxygen Demand (BOD),BOD
+BOD,"Biochemical oxygen demand, standard conditions","Biochemical oxygen demand, standard conditions",BOD
+479-61-8,Chlorophyll a,Chlorophyll a,CHLA
+479-61-8,Chlorophyll a,Chlorophyll A,CHLA
+CHLA-CORR,"Chlorophyll a, corrected for pheophytin",Chlorophyll A,CHLA
+CHLA-CORR,"Chlorophyll a, corrected for pheophytin",Chlorophyll-A,CHLA
+CHLA-CORR,"Chlorophyll a, corrected for pheophytin",Chlorophyll,CHLA
+CHLA-CORR,"Chlorophyll a, corrected for pheophytin","Chlorophyll a, pheophytin-adjusted",CHLA
+CHLA-CORR,"Chlorophyll a, corrected for pheophytin","Chlorophyll a, corrected for pheophytin",CHLA
+CHLA-CORR,"Chlorophyll a, corrected for pheophytin","Chlorophyll-a, Pheophytin Corrected",CHLA
+CHLA-CORR,"Chlorophyll a, corrected for pheophytin","Chlorophyll a, corrected for pheophytin - Periphyton (attached)",CHLA
+CHLA-CORR,"Chlorophyll a, corrected for pheophytin",Chlorophyll a,CHLA
+CHLA-C-PERI,"Chlorophyll a, corrected for pheophytin - Periphyton (attached)","Chlorophyll a, corrected for pheophytin - Periphyton (attached)",CHLA
+FECAL-COLIF,Fecal Coliform,Fecal Coliform,ECOLI
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),Nitrate/Nitrite as N (N+N) as N,N
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),"Nitrate + Nitrite Nitrogen, Dissolved as N",N
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),Nitrate/Nitrite as N as N,N
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),"Nitrogen, NO2 plus NO3 as N",N
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),N+N as N,N
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),Nitrate/Nitrite as N,N
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),Inorganic nitrogen (nitrate and nitrite) as N,N
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),"Nitrate + Nitrite Nitrogen, Total",N
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),"Nitrate + Nitrite Nitrogen, Total as N",N
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),Nitrate+Nitrite Nitrogen as N,N
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),Inorganic nitrogen (nitrate and nitrate) as N,N
+NO2NO3,Inorganic nitrogen (nitrate and nitrite),Nitrate+Nitrite as N,N
+14797-55-8,Nitrate,"Nitrogen, nitrate, as NO3 as N",NO3
+14797-55-8,Nitrate,Nitrate as N as N,NO3
+14797-55-8,Nitrate,Nitrate-N as N,NO3
+14797-55-8,Nitrate,"Nitrogen, nitrate, as N as N",NO3
+14797-55-8,Nitrate,Nitrate as N,NO3
+14797-65-0,Nitrite,"Nitrogen, nitrite, as N as N",NO2
+14797-65-0,Nitrite,Nitrite as N,NO2
+14797-65-0,Nitrite,Nitrite as N as N,NO2
+14797-65-0,Nitrite,Nitrite-N as N,NO2
+14797-65-0,Nitrite,"Nitrogen, nitrite, as NO2 as N",NO2
+14265-44-2,Orthophosphate,"Orthophosphate, as P as P",OP
+14265-44-2,Orthophosphate,Orthophosphate as P as P,OP
+14265-44-2,Orthophosphate,"Orthophosphate as P, Dissolved as P",OP
+14265-44-2,Orthophosphate,Orthophosphate as P (LF) as P,OP
+14265-44-2,Orthophosphate,phosphate-P as P,OP
+14265-44-2,Orthophosphate,"Orthophosphate Phosphorus, Total as P",OP
+14265-44-2,Orthophosphate,"Orthophosphate Phosphorus, Dissolved as P",OP
+14265-44-2,Orthophosphate,"Ortho Phosphorus, Dissolved as P",OP
+14265-44-2,Orthophosphate,Orthophosphate as P,OP
+14265-44-2,Orthophosphate,Ortho Phosphorus as P,OP
+14265-44-2,Orthophosphate,"Phosphorus, Soluble Ortho as P",OP
+14265-44-2,Orthophosphate,Orthophosphate as P (FF) as P,OP
+14265-44-2,Orthophosphate,Phosphorus as P,OP
+SSC,Suspended Sediment Concentration (SSC),Suspended Sediment Concentration (SSC),SSC
+SOLIDS-TDS,Total dissolved solids,Residue - filterable (TDS),TDS
+SOLIDS-TDS,Total dissolved solids,Total dissolved solids (TDS),TDS
+SOLIDS-TDS,Total dissolved solids,"Solids, Total Dissolved",TDS
+SOLIDS-TDS,Total dissolved solids,Total dissolved solids,TDS
+SOLIDS-TDS,Total dissolved solids,Total Dissolved Solids,TDS
+N-KJEL,Kjeldahl nitrogen,"Kjeldahl Nitrogen, Total as N",TKN
+N-KJEL,Kjeldahl nitrogen,Total Kjeldahl Nitrogen as N,TKN
+N-KJEL,Kjeldahl nitrogen,TKN as N,TKN
+N-KJEL,Kjeldahl nitrogen,"Nitrogen, Total Kjeldahl (TKN) as N",TKN
+N-KJEL,Kjeldahl nitrogen,"Nitrogen, Total Kjeldahl as N",TKN
+N-KJEL,Kjeldahl nitrogen,Kjeldahl nitrogen as N,TKN
+PHOSPHATE-P,Phosphate-phosphorus,Phosphate-phosphorus as P,TP
+7723-14-0,Phosphorus,Orthophosphate as P,TP
+7723-14-0,Phosphorus,"Phosphorus, Total as P",TP
+7723-14-0,Phosphorus,"Phosphorus, Total as P as P",TP
+7723-14-0,Phosphorus,"Phosphorus, Total Dissolved, as P (FF) as P",TP
+7723-14-0,Phosphorus,Total Diss Phosphorus as P,TP
+7723-14-0,Phosphorus,"Phosphorus, Dissolved as P",TP
+7723-14-0,Phosphorus,"Phosphorus, Total Dissolved, as P (LF) as P",TP
+7723-14-0,Phosphorus,Total Phosphorus as P,TP
+7723-14-0,Phosphorus,total phosphorus as P,TP
+7723-14-0,Phosphorus,Phosphorus as P,TP
+7723-14-0,Phosphorus,"Phosphorus, Total",TP
+SOLIDS-TS,Total solids,Total solids,TS
+SOLIDS-TS,Total solids,Total Solids,TS
+SOLIDS-TSS,Total suspended solids,"Solids, Total Suspended",TSS
+SOLIDS-TSS,Total suspended solids,Total Suspended Solids in Water,TSS
+SOLIDS-TSS,Total suspended solids,Total suspended solids,TSS
+SOLIDS-TSS,Total suspended solids,Residue - nonfilterable (TSS),TSS
+SOLIDS-TSS,Total suspended solids,total suspended solids,TSS
+SOLIDS-TSS,Total suspended solids,"Solids, Suspended",TSS
+SOLIDS-TSS,Total suspended solids,Total Suspended Solids,TSS
+TEMP-W,"Temperature, water","Temperature, water",WT
+TEMP-W,"Temperature, water","Temperature, field",WT
+TEMP-W,"Temperature, water","Temperature, Water",WT
+TEMP-W,"Temperature, water",Field Temperature,WT

{mpcahydro-2.0.3 → mpcahydro-2.0.4}/src/mpcaHydro/data_manager.py

@@ -285,12 +285,7 @@ class dataManager():
         try:
             df = self.data[station_id]
         except:
-            df =  pd.read_csv(self.folderpath.joinpath(station_id + '.csv'), 
-                              index_col='datetime', 
-                              parse_dates=['datetime'], 
-                              #usecols=['Ts Date','Station number','variable', 'value','reach_id'],
-                              dtype={'station_id': str, 'value': float, 'variable': str,'constituent':str,'unit':str})
-            self.data[station_id] = df
+            self._load(station_id)
         return df
     
     def info(self,constituent):

{mpcahydro-2.0.3 → mpcahydro-2.0.4}/src/mpcaHydro/etlSWD.py

@@ -6,41 +6,15 @@ Created on Tue Oct 10 14:13:23 2023
 """
 
 import pandas as pd
+from pathlib import Path
 #from hspf_tools.orm.models import Station
 # import geopandas as gpd
 
+EQUIS_PARAMETER_XREF = pd.read_csv(Path(__file__).parent/'data/EQUIS_PARAMETER_XREF.csv')
 
 
-
-                   
-CONSTITUENT_MAP = {'Total suspended solids':'TSS',
-                   'Total solids': 'TSS',
-                   'Solids, Suspended' : 'TSS',
-                   'Solids, Total Suspended' : 'TSS',
-                  'Residue - nonfilterable (TSS)': 'TSS',
-                 'Kjeldahl nitrogen as N': 'TKN',
-                 'Inorganic nitrogen (nitrate and nitrate) as N': 'N',
-                 'Nitrogen, Total Kjeldahl (TKN) as N': 'TKN',
-                 'Nitrate + Nitrite Nitrogen, Total as N': 'N',
-                 'Nitrate/Nitrite as N (N+N) as N': 'N',
-                 'Nutrient-nitrogen as N': 'N',
-                 'Nitrate/Nitrite as N': 'N',
-                 'Phosphorus, Total as P as P':'TP',
-                 'Phosphorus, Total as P' : 'TP',
-                 'Phosphorus as P': 'TP',
-                 'Total Phosphorus as P': 'TP',
-                 'Orthophosphate as P': 'OP',
-                 'Carbonaceous biochemical oxygen demand, standard conditions': 'BOD',
-                 'Chemical oxygen demand':'BOD',
-                 'Biochemical oxygen demand, standard conditions': 'BOD',
-                 'Chlorophyll a, corrected for pheophytin':'CHLA',
-                 'Chlorophyll-A':'CHLA',
-                 'Chlorophyll-a, Pheophytin Corrected':'CHLA',
-                 'Flow':'Q',
-                 'Temperature, water': 'WT',
-                 'Dissolved oxygen': 'DO',
-                 'Dissolved oxygen (DO)': 'DO',
-                 'Suspended Sediment Concentration': 'SSC'}    
+CONSTITUENT_MAP = {i[0]:i[1] for i in EQUIS_PARAMETER_XREF[['PARAMETER','constituent']].values}
+  
 
 # station_no  = 	'S010-822'
 # data = download(station_no)
@@ -164,6 +138,14 @@ def transform(df):
     return df
 
 def load(df,file_path):
+    '''
+    date, time, value, variable, unit, station_id, station_name, constituent, source, data_format, data_type, quality_code, 
+    
+    
+    
+    '''
+
+
     df.to_csv(file_path)      
     
     

{mpcahydro-2.0.3 → mpcahydro-2.0.4}/src/mpcaHydro/etlWISKI.py

@@ -152,7 +152,60 @@ VALID_CONSTITUENTS = ['Q','WT','OP','DO','TP','TSS','N','TKN']
 #                    latitude = station_info.iloc[0]['station_latitude'],
 #                    longitude = station_info.iloc[0]['station_longitude'],
 #                    station_type = 'River')
+
+
+def extract(station_nos, constituent, dbpath, start_year = 1996, end_year = 2030, wplmn = False):
+    '''
+    given a list of station_nos, download all data relevent to HSPF from MPCA WISKI and store in a duckdb database
+    
+    1. Find relevent timeseries ids for each constituent
+    2. Download data for each timeseries id
+    3. Store data in duckdb database
+    
+    '''
+    #1. Find relevent timeseries ids for each constituent
+    if station_nos[0] == 'E':
+        ts_names = TS_NAME_SELECTOR[constituent]['External']
+    else:
+        ts_names =TS_NAME_SELECTOR[constituent]['Internal']
+    
+    if wplmn:
+        constituent_nos = CONSTITUENT_NAME_NO_WPLMN[constituent]
+    else:
+        constituent_nos = CONSTITUENT_NAME_NO[constituent]
         
+    ts_ids = wiski.get_ts_ids(station_nos = station_nos,
+                        stationparameter_no = constituent_nos,
+                        ts_name = ts_names['unit'])
+    
+    jsons = []
+    for ts_id in ts_ids:
+        jsons.append(download_chunk(ts_id,start_year,end_year,as_json = True))
+        time.sleep(.1)
+    return jsons
+
+
+    # with duckdb.connect(db_path) as con:
+    #     con.execute("DROP TABLE IF EXISTS observations")
+    #     datafiles = folderpath.joinpath('*.csv').as_posix()
+    #     query = '''
+    #     CREATE TABLE observations AS SELECT * 
+    #     FROM
+    #     read_csv_auto(?,
+    #                     union_by_name = true);
+        
+    #     '''
+    #     con.execute(query,[datafiles])
+
+    
+    # con = duckdb.connect(database=db_path))
+    # print('Downloading Timeseries Data')
+    # df = pd.concat([_download(constituent,station_nos,start_year,end_year,raw = True, wplmn = False) for constituent in VALID_CONSTITUENTS])
+    # df.to_csv(filepath,index = False)
+    # print('Timeseries Data Downloaded!')
+
+    
+
 
 def info(station_nos):
     ts_ids = wiski.get_ts_ids(station_nos = station_nos,
@@ -263,14 +316,14 @@ def _download(constituent,station_nos,start_year = 1996,end_year = 2030, raw = F
     return df
 
 
-def download_chunk(ts_id,start_year = 1996,end_year = 2030, interval = 5):
+def download_chunk(ts_id,start_year = 1996,end_year = 2030, interval = 5, as_json = False):
     frames = [pd.DataFrame()]
 
     for start in range(start_year,end_year,interval):
         end = int(start + interval-1)
         if end > end_year:
             end = end_year
-        df = wiski.get_ts(ts_id,start_date = f'{start}-01-01',end_date = f'{end}-12-31')
+        df = wiski.get_ts(ts_id,start_date = f'{start}-01-01',end_date = f'{end}-12-31',as_json = as_json)
         if not df.empty: frames.append(df)
         df.index = pd.to_datetime(df['Timestamp'])
         time.sleep(.1)   

mpcahydro-2.0.3/tests/pixi.toml

@@ -1,25 +0,0 @@
-[workspace]
-channels   = ["https://prefix.dev/conda-forge"]
-platforms  = ["linux-64", "osx-64", "win-64"]
-
-
-[dependencies]
-requests    = "*"
-pandas       = "*"
-time = 			"*"
-pathlib = 		"*"
-spyder = "*"
-jupyter = "*"
-
-[package]
-name        = "mpcaHydro"
-version     = "0.1.0"
-
-[package.build]
-backend     = { name = "pixi-build-python", version = "0.1.*" }
-
-[package.run-dependencies]
-requests    = "*"
-pandas       = "*"
-time = 			"*"
-pathlib = 		"*"