mpcaHydro 2.0.4__tar.gz → 2.0.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. mpcahydro-2.0.6/ERROR.FIL +6 -0
  2. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/PKG-INFO +3 -1
  3. mpcahydro-2.0.6/demo.py +167 -0
  4. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/pyproject.toml +4 -11
  5. mpcahydro-2.0.6/src/mpcaHydro/data/WISKI_QUALITY_CODES.csv +71 -0
  6. mpcahydro-2.0.6/src/mpcaHydro/data/outlets.duckdb +0 -0
  7. mpcahydro-2.0.6/src/mpcaHydro/data/stations_EQUIS.gpkg +0 -0
  8. mpcahydro-2.0.6/src/mpcaHydro/data/stations_wiski.gpkg +0 -0
  9. mpcahydro-2.0.6/src/mpcaHydro/data_manager.py +247 -0
  10. mpcahydro-2.0.6/src/mpcaHydro/equis.py +488 -0
  11. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/src/mpcaHydro/etlSWD.py +4 -5
  12. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/src/mpcaHydro/etlWISKI.py +39 -23
  13. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/src/mpcaHydro/etlWPLMN.py +2 -2
  14. mpcahydro-2.0.6/src/mpcaHydro/outlets.py +371 -0
  15. mpcahydro-2.0.6/src/mpcaHydro/pywisk.py +381 -0
  16. mpcahydro-2.0.6/src/mpcaHydro/reports.py +80 -0
  17. mpcahydro-2.0.6/src/mpcaHydro/warehouse.py +581 -0
  18. mpcahydro-2.0.6/src/mpcaHydro/warehouseManager.py +47 -0
  19. mpcahydro-2.0.6/src/mpcaHydro/wiski.py +308 -0
  20. mpcahydro-2.0.6/src/mpcaHydro/xref.py +74 -0
  21. mpcahydro-2.0.4/src/mpcaHydro/WISKI.py +0 -352
  22. mpcahydro-2.0.4/src/mpcaHydro/data_manager.py +0 -419
  23. mpcahydro-2.0.4/src/mpcaHydro/helpers.py +0 -0
  24. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/.gitattributes +0 -0
  25. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/.gitignore +0 -0
  26. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/README.md +0 -0
  27. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/src/mpcaHydro/__init__.py +0 -0
  28. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/src/mpcaHydro/data/EQUIS_PARAMETER_XREF.csv +0 -0
  29. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/src/mpcaHydro/data/WISKI_EQUIS_XREF.csv +0 -0
  30. {mpcahydro-2.0.4 → mpcahydro-2.0.6}/src/mpcaHydro/etlCSG.py +0 -0
@@ -0,0 +1,6 @@
1
+ 15:21:44.605 : LOG_MSG:ERROR.FIL OPENED
2
+ 15:21:44.607 : HASS_ENT:F90_WDBOPNR:entr:WDMSFL,RWFLG: 100 1 C:\Users\mfratki\Documents\github\pyHSPF\src\hspf\bin\WinHSPFLt\hspfmsg.wdm
3
+ 15:21:44.608 : HASS_ENT:F90_WDBOPNR:exit:WDMSFL,RETCOD 100 0
4
+ FILBLK RETCOD 0
5
+ wdmfl 0 0 0 0
6
+ FILBLK RETCOD 0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mpcaHydro
3
- Version: 2.0.4
3
+ Version: 2.0.6
4
4
  Summary: Python package for downloading MPCA hydrology data
5
5
  Project-URL: Homepage, https://github.com/mfratkin1/mpcaHydro
6
6
  Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>
@@ -10,7 +10,9 @@ Keywords: Hydrology,MPCA
10
10
  Classifier: Development Status :: 3 - Alpha
11
11
  Classifier: Programming Language :: Python
12
12
  Requires-Python: >=3.8
13
+ Requires-Dist: baseflow
13
14
  Requires-Dist: duckdb
15
+ Requires-Dist: oracledb
14
16
  Requires-Dist: pandas
15
17
  Requires-Dist: pathlib
16
18
  Requires-Dist: requests
@@ -0,0 +1,167 @@
1
+ #%%
2
+ from mpcaHydro.data_manager import dataManager
3
+ from pyhcal.repository import Repository
4
+ from mpcaHydro import outlets
5
+ import duckdb
6
+ from mpcaHydro import equis, warehouse, wiski
7
+ from hspf.hspfModel import hspfModel
8
+ from hspf.uci import UCI
9
+
10
+
11
+
12
+ #%%
13
+ '''
14
+ New approach. Directly load to warehouse from downloads.
15
+ Store raw and processed data in warehouse. For large timeseries I could store
16
+ as parquet files. The transformations using pandas take a bit of time. I imagine doing them
17
+ within duckdb would be faster.
18
+
19
+ '''
20
+
21
+ # with warehouse.connect(db_path) as con:
22
+ # df = con.execute("SELECT * FROM staging.wiski").df()
23
+ # df = wiski.transform(df,filter_qc_codes = False)
24
+
25
+ #%%
26
+ model_name = 'Nemadji'
27
+ db_path = f'C:/Users/mfratki/Documents/{model_name}.duckdb'
28
+ start_year = 1996
29
+ end_year = 2030
30
+ replace = True
31
+ filter_qc_codes = True
32
+ equis_stations = outlets.equis_stations(model_name)
33
+ wiski_stations = outlets.wiski_stations(model_name)
34
+ equis.connect('MFRATKI',password = 'DeltaT#MPCA3')
35
+ warehouse.init_db(db_path,reset = True)
36
+
37
+ #%%
38
+
39
+ with warehouse.connect(db_path) as con:
40
+ df = equis.download(equis_stations)
41
+ warehouse.load_df_to_staging(con,df, 'equis_raw',replace = replace)
42
+ warehouse.load_df_to_analytics(con,equis.transform(df),'equis')
43
+
44
+ df = wiski.download(wiski_stations,start_year = start_year, end_year = end_year)
45
+ warehouse.load_df_to_staging(con,df, 'wiski_raw', replace = replace)
46
+ warehouse.load_df_to_analytics(con,wiski.transform(df,filter_qc_codes = filter_qc_codes),'wiski') # method includes normalization
47
+
48
+ outlets.build_outlets(con, model_name)
49
+
50
+
51
+ with warehouse.connect(db_path) as con:
52
+ warehouse.update_views(con)
53
+
54
+
55
+ #%%
56
+
57
+ import requests
58
+ url = 'http://ifrshiny.seas.umich.edu/mglp/'
59
+ requests.get(url)
60
+
61
+
62
+
63
+ db_path = 'C:/Users/mfratki/Documents/Rum.duckdb'
64
+ modl_db.build_outlet_db(db_path)
65
+ con = duckdb.connect(db_path)
66
+ con.execute("SELECT * FROM station_reach_pairs").df()
67
+ con.execute('SELECT * FROM station_reach_pairs WHERE outlet_id = 76').df()
68
+
69
+ # Need to remove duplicates from MODL_DB
70
+ modl_db.MODL_DB.loc[modl_db.MODL_DB.duplicated(['station_id','source'])]
71
+
72
+ #%%
73
+ dm = dataManager('C:/Users/mfratki/Documents/')
74
+ dm._build_warehouse()
75
+ equis_stations = modl_db.equis_stations('Nemadji')
76
+ wiski_stations = modl_db.wiski_stations('Nemadji')
77
+
78
+ #%% Old approach. Store as indvidual processed station files then load to warehouse
79
+ for station_id in equis_stations:
80
+ dm._download_station_data(station_id,'equis', True)
81
+
82
+ for station_id in wiski_stations:
83
+ dm._download_station_data(station_id,'wiski', True)
84
+
85
+
86
+
87
+
88
+
89
+
90
+
91
+
92
+
93
+
94
+ #%% Adding HSPF outputs to warehouse
95
+
96
+
97
+
98
+
99
+
100
+
101
+
102
+
103
+
104
+
105
+
106
+ con = duckdb.connect(db_path)
107
+
108
+ model_name = 'Nemadji'
109
+ outlets = [group for _, group in modl_db.MODL_DB.query('repository_name == @model_name').groupby(by = ['opnids','repository_name'])]
110
+
111
+ for outlet in outlets:
112
+ 1+1
113
+
114
+
115
+ dfs = []
116
+ for constituent in ['Q','TSS','TP','N','OP','TKN']:
117
+ opnids = modl_db.split_opnids([opnid.split(',') for opnid in set(outlet['opnids'].tolist())])
118
+ for opnid in opnids:
119
+ df = mod.hbns.get_reach_constituent(constituent,opnids,time_step='h')
120
+ df.columns = ['value']
121
+ df['constituent'] = constituent
122
+ df['operation'] = operation
123
+ df['opnid'] = opnid
124
+ dfs.append(df)
125
+
126
+ df = pd.concat(dfs).reset_index()
127
+ df['model_name'] = model_name
128
+
129
+
130
+
131
+ station_ids = ['H05018001','S006-214','S015-102']
132
+ target_constituent = 'TSS'
133
+ flow_constituent = 'Q'
134
+
135
+ # build placeholders for the IN list (one ? per station id)
136
+ placeholders = ','.join(['?'] * len(station_ids))
137
+
138
+ sql = f'''
139
+ SELECT o.*, f.datetime AS flow_datetime, f.value AS flow, f.baseflow, f.station_id AS flow_station_id, f.station_origin AS flow_station_origin
140
+ FROM analytics.observations o
141
+ JOIN analytics.observations f
142
+ ON o.datetime = f.datetime
143
+ WHERE o.constituent = ?
144
+ AND o.station_id IN ({placeholders})
145
+ AND f.constituent = ?;
146
+ '''
147
+
148
+ # parameter order must match the ? positions in the query
149
+ params = [target_constituent] + station_ids + [flow_constituent]
150
+
151
+ df = con.execute(sql, params).df()
152
+
153
+ outlet_id: station_ids
154
+
155
+ outlet_id: opnid
156
+
157
+
158
+ outlets = []
159
+ for index, (_, group) in enumerate(modl_db.MODL_DB.groupby(by = ['opnids','repository_name'])):
160
+ group['outlet_id'] = index
161
+ group.reset_index(drop=True, inplace=True)
162
+ outlets.append(group)
163
+
164
+
165
+ for _, row in group.iterrows():
166
+ opnids = group.split_opnids(row['opnids'].str.split(',').to_list())
167
+ row*len(opnids)
@@ -5,12 +5,14 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "mpcaHydro"
7
7
  urls = { "Homepage" = "https://github.com/mfratkin1/mpcaHydro" } # ? Add this!
8
- version = "2.0.4"
8
+ version = "2.0.6"
9
9
  dependencies = [
10
10
  "pandas",
11
11
  "requests",
12
12
  "pathlib",
13
- "duckdb"
13
+ "duckdb",
14
+ "oracledb",
15
+ "baseflow"
14
16
  ]
15
17
  requires-python = ">=3.8"
16
18
  authors = [
@@ -27,12 +29,3 @@ classifiers = [
27
29
  "Development Status :: 3 - Alpha",
28
30
  "Programming Language :: Python"
29
31
  ]
30
-
31
- [tool.pixi.workspace]
32
- channels = ["conda-forge"]
33
- platforms = ["win-64"]
34
-
35
- [tool.pixi.pypi-dependencies]
36
- mpcaHydro = { path = ".", editable = true }
37
-
38
- [tool.pixi.tasks]
@@ -0,0 +1,71 @@
1
+ quality_code,Text,Description,Active
2
+ 0,Unchecked,Unchecked data in progress or data that is not quality coded as part of the workup. Default coding for shifts so the quality codes from Level and Ratings are used for discharges. ,1
3
+ 3,Instantaneous,Instantaneous groundwater measurements or sampled date for load stations.,1
4
+ 5,Excellent,Discharge measurements that are excellent.,1
5
+ 8,Reliable Interpolation,The value of the data point is an interpolation between adjacent points. Code used for filling gaps less than 4 hours or with no change in data trend likely based on reference information.,1
6
+ 10,Good,Time series data that tracks well and requires no corrections or corrections of very small magnitude or timeseries data that has been reviewed and accepted for precipitation and groundwater level. Also used for discharge measurements and rating points. ,1
7
+ 15,Fair,Time series data that tracks fairly well and requires some corrections of relatively small magnitude. Also used for discharge measurements and rating points. ,1
8
+ 20,Poor,Time series data that tracks poorly and requires significant or many corrections. Also used for discharge measurements and rating points. ,1
9
+ 27,Questionable,"Timeseries data or discharge measurements that are questionable due to operator error, equipment error, etc). Extra scrutiny should be used for these data. ",1
10
+ 28,Unknown data quality,"Unknown quality of time series data, ratings or discharge measurements.",1
11
+ 29,Modeled,"Time-series data, rating point or discharge from a reliable mathematical and\or computer model. ",1
12
+ 34,Estimated,"Time-series data estimated from reference traces, models or extrapolation of the rating curve using supporting data and up to two times the maximum measured discharge.",1
13
+ 35,Unreliable,Time-series data computed with a rating extrapolated without supporting data or beyond two times the maximum measured discharge without a model.,1
14
+ 36,Threshold Exceedance,"Time-series data may be beyond the measuring limits of the monitoring equipment, or outside the bounds of historical extremes.",1
15
+ 40,Default import code,WISKI default coding for gaugings. ,1
16
+ 45,Approved Ext Data,"External data that has been graded externally as ""Approved"".",1
17
+ 48,Unknown Ext Data,External data that has been graded internally as “Unknown”.,1
18
+ 49,Estimated Ext Data,External data that has been graded externally as “Estimated.” Typically this is finalized ice data.,1
19
+ 50,Provisional Ext Data,External data that has been graded internally or externally as “Provisional”.,1
20
+ 80,Ice - Estimated,Ice affected time series data. Discharge computed with ice affected stage data is considered estimated.,1
21
+ 199,199-Logger Unknown,Initial code for data coming to the system from the logger.,1
22
+ 200,200,Initial code for data coming to the system from telemetry or default coding for WISKI timeseries. ,1
23
+ 228,Info Parameter,This parameter is collected for informational purposes only. Data has been through a cursory check only. This is stored in the database and available upon request.,1
24
+ 255,---,System assigned code for gaps in the data set. Records with null values. ,1
25
+ 1,Continuous Data,~Discontinued~ Good TS data that requires no correction.,0
26
+ 2,Edited Data,~Discontinued~ TS data that has been edited. Typically used when spikes are removed or when points are edited manual for datum corrections.,0
27
+ 3,Instantaneous Data,Final WQ data.,0
28
+ 4,Questionable data,~Discontinued~,0
29
+ 5,Excellent measurment,Used to indicated discharge measurements that are excellent as well as excellent sections of the rating.,0
30
+ 10,Good measurement,Used to indicated discharge measurements and sections of the rating that are good and time series data that tracks well and requires no corrections or corrections of very small magnitude.,0
31
+ 12,Modeled measurement,~Discontinued~ Rating point or discharge was obtained from a relizble mathematical and/or computer model. After 3/1/11 use QC148.,0
32
+ 15,Fair measurement,Used to indicated discharge measurements and sections of the rating that are fair and time series data that tracks fairly well and requires some corrections of relatively small magnitude.,0
33
+ 20,Poor measurement,Used to indicated discharge measurements and sections of the rating that are poor and time series data that tracks poorly and requires significant or many corrections.,0
34
+ 25,Unknown measurement,Measurement data not available.,0
35
+ 27,Questionable data,"Flow measurement is very poor and should be given extra scrutiny or time series data that is questionable due to operator error, equipment error, etc.",0
36
+ 30,Good Archived Daily Value,This code is used for archived daily value data that is considered “Good”.,0
37
+ 31,Fair Archived Daily Value,This code is used for archived daily value data that is considered “Fair”.,0
38
+ 32,Poor Archived Daily Value,This code is used for archived daily value data that is considered “Poor”.,0
39
+ 33,Unknown Archived Daily Value,This code is used for archived daily value data that has unknown quality based on lack of documentation.,0
40
+ 34,Estimated Archived Daily Value,This code is used for archived daily value data that has been estimated.,0
41
+ 35,Unreliable Archived Daily Value,This code is used for archived daily value data that is unreliable based on the quality of the supporting time series data and/or rating.,0
42
+ 45,Good External Data,This code is used for external data that has been graded internally as “Good”.,0
43
+ 46,Fair External Data,This code is used for external data that has been graded internally as “Fair”.,0
44
+ 47,Poor External Data,This code is used for external data that has been graded internally as “Poor”.,0
45
+ 48,Unknown External Data,This code is used for external data that has been graded internally as “Unknown”,0
46
+ 49,Estimated External Data,This code is used for external data that has been graded externally as “Estimated.” Typically this is finalized ice data.,0
47
+ 50,Provisional External Data,This code is used for external data that has been graded internally as “Provisional”,0
48
+ 51,Telemetry data - DCP,This code is used for time-series data when imported into hydstra using an automated telemetry method that accesses a DCP through the GOES network. The “questionable measurement” flag is set through the shef code that accompanies the DCP data.,0
49
+ 60,Above rating,~Discontinued~,0
50
+ 70,Estimated Data,Value of the data point is estimated.,0
51
+ 76,Reliable interpolation,Value of the data point is an interpolation between adjacent points. ,0
52
+ 80,Ice,"(DISCONTINUED) Used to indicate ice conditions when the data should not be exported. Use in conjunction with 80 to code 232.00 values, run USDAY to compute daily flow, then recode 232.00 80 values to 180 so unit value export cannot occur.",0
53
+ 82,Linear interpolation across a gap in records,~Discontinued~ Points that were added to fill a gap in the data record. The points fall on a straight line between the end points of the gap. This code was changed to 8 in WISKI.,0
54
+ 103,Provisional Instantaneous Data,Provisional WQ data.,0
55
+ 130,Good Provisional Daily Value,This code is used for archived daily value data that is considered “Good” but Provisional because there is only one year of gaging measurements.,0
56
+ 131,Fair Provisional Daily Value,This code is used for archived daily value data that is considered “Fair” but Provisional because there is only one year of gaging measurements.,0
57
+ 132,Poor Provisional Daily Value,This code is used for archived daily value data that is considered “Poor” but Provisional because there is only one year of gaging measurements.,0
58
+ 133,Unknown Provisional Archived Daily Value,This code is used for archived daily value data that has unknown quality based on lack of documentation but Provisional because there is only one year of gaging measurements.,0
59
+ 134,Estimated Provisional Archived Daily Value,This code is used for archived daily value data that has been estimated but Provisional because there is only one year of gaging measurements.,0
60
+ 135,Unreliable Provisional Archived Daily Value,This code is used for archived daily value data that is unreliable based on the quality of the supporting time series data and/or rating but Provisional because there is only one year of gaging measurements.,0
61
+ 140,Data not yet checked,This code is used for time-series data when it is initially imported into hydstra using manual import methods. ,0
62
+ 141,Telemetry data - not yet checked,This code is used for time-series data when it is imported into hydstra using an automated telemetry method.,0
63
+ 148,Modeled measurement,Rating point or discharge was obtained from a reliable mathematical and/or computer model.,0
64
+ 149,Extrapolated rating point,Rating point accurately extrapolated using supporting data and is less than two times the maxiumum measured discharge.,0
65
+ 150,Over-extrapolated rating point,Rating point extrapolated without supporting data or beyone two times the maximum measured discharge without a mathematical model.,0
66
+ 151,Data Missing,"This code is used to flag the end of a period of missing time-series data, before the next good data value.",0
67
+ 160,Above rating,~Discontinued~,0
68
+ 169,Datalogger Hardware Error Code 6999,"This code is used to indicate that a time-series point had a value of 6999 or -6999, a typical hardware error code, and the value was changed.",0
69
+ 170,Estimated Data,"Used to indicate estimated data when the data should not be exported. Often used in conjunction with 70 to code 232.00 values, run USDAY to compute daily flow, then recode 232.00 70 values to 170 so unit value export can not occur.",0
70
+ 180,Ice,Used to indicate ice conditions.,0
71
+ 255,Data Missing,This code is used when data is exported and does not exist for a given time period.,0
@@ -0,0 +1,247 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Created on Fri Jun 3 10:01:14 2022
4
+
5
+ @author: mfratki
6
+ """
7
+
8
+ import pandas as pd
9
+ #from abc import abstractmethod
10
+ from pathlib import Path
11
+ from mpcaHydro import etlSWD
12
+ from mpcaHydro import equis, wiski, warehouse
13
+ from mpcaHydro import xref
14
+ from mpcaHydro import outlets
15
+ from mpcaHydro.reports import reportManager
16
+ import duckdb
17
+
18
+ AGG_DEFAULTS = {'cfs':'mean',
19
+ 'mg/l':'mean',
20
+ 'degf': 'mean',
21
+ 'lb':'sum'}
22
+
23
+ UNIT_DEFAULTS = {'Q': 'cfs',
24
+ 'QB': 'cfs',
25
+ 'TSS': 'mg/l',
26
+ 'TP' : 'mg/l',
27
+ 'OP' : 'mg/l',
28
+ 'TKN': 'mg/l',
29
+ 'N' : 'mg/l',
30
+ 'WT' : 'degf',
31
+ 'WL' : 'ft'}
32
+
33
+
34
+ def validate_constituent(constituent):
35
+ assert constituent in ['Q','TSS','TP','OP','TKN','N','WT','DO','WL','CHLA']
36
+
37
+ def validate_unit(unit):
38
+ assert(unit in ['mg/l','lb','cfs','degF'])
39
+
40
+
41
+ def build_warehouse(folderpath):
42
+ folderpath = Path(folderpath)
43
+ db_path = folderpath.joinpath('observations.duckdb').as_posix()
44
+ warehouse.init_db(db_path)
45
+
46
+ def constituent_summary(db_path):
47
+ with duckdb.connect(db_path) as con:
48
+ query = '''
49
+ SELECT
50
+ station_id,
51
+ station_origin,
52
+ constituent,
53
+ COUNT(*) AS sample_count,
54
+ year(MIN(datetime)) AS start_date,
55
+ year(MAX(datetime)) AS end_date
56
+ FROM
57
+ observations
58
+ GROUP BY
59
+ constituent, station_id,station_origin
60
+ ORDER BY
61
+ sample_count;'''
62
+
63
+ res = con.execute(query)
64
+ return res.fetch_df()
65
+
66
+
67
+ class dataManager():
68
+
69
+ def __init__(self,folderpath, oracle_user = None, oracle_password =None):
70
+
71
+ self.data = {}
72
+ self.folderpath = Path(folderpath)
73
+ self.db_path = self.folderpath.joinpath('observations.duckdb')
74
+
75
+ self.oracle_user = oracle_user
76
+ self.oracle_password = oracle_password
77
+ warehouse.init_db(self.db_path,reset = False)
78
+ self.xref = xref
79
+ self.outlets = outlets
80
+ self.reports = reportManager(self.db_path)
81
+
82
+
83
+ def connect_to_oracle(self):
84
+ assert (self.credentials_exist(), 'Oracle credentials not found. Set ORACLE_USER and ORACLE_PASSWORD environment variables or use swd as station_origin')
85
+ equis.connect(user = self.oracle_user, password = self.oracle_password)
86
+
87
+ def credentials_exist(self):
88
+ if (self.oracle_user is not None) & (self.oracle_password is not None):
89
+ return True
90
+ else:
91
+ return False
92
+
93
+ def _build_warehouse(self):
94
+ build_warehouse(self.folderpath)
95
+
96
+ def download_station_data(self,station_id,station_origin,overwrite=True,to_csv = False,filter_qc_codes = True, start_year = 1996, end_year = 2030,baseflow_method = 'Boughton'):
97
+ '''
98
+ Method to download data for a specific station and load it into the warehouse.
99
+
100
+ :param self: Description
101
+ :param station_id: Station identifier
102
+ :param station_origin: source of station data: wiski, equis, or swd
103
+ :param overwrite: Whether to overwrite existing data
104
+ :param to_csv: Whether to export data to CSV
105
+ :param filter_qc_codes: Whether to filter quality control codes
106
+ :param start_year: Start year for data download
107
+ :param end_year: End year for data download
108
+ :param baseflow_method: Method for baseflow calculation
109
+ '''
110
+ with duckdb.connect(self.db_path,read_only=False) as con:
111
+ if overwrite:
112
+ warehouse.drop_station_id(con,station_id,station_origin)
113
+ warehouse.update_views(con)
114
+
115
+ if station_origin == 'wiski':
116
+ df = wiski.download([station_id],start_year = start_year, end_year = end_year)
117
+ warehouse.load_df_to_staging(con,df, 'wiski_raw', replace = overwrite)
118
+ warehouse.load_df_to_analytics(con,wiski.transform(df,filter_qc_codes = filter_qc_codes,baseflow_method = baseflow_method),'wiski') # method includes normalization
119
+
120
+ elif station_origin == 'equis':
121
+ assert (self.credentials_exist(), 'Oracle credentials not found. Set ORACLE_USER and ORACLE_PASSWORD environment variables or use swd as station_origin')
122
+ df = equis.download([station_id])
123
+ warehouse.load_df_to_staging(con,df, 'equis_raw',replace = overwrite)
124
+ warehouse.load_df_to_analytics(con,equis.transform(df),'equis')
125
+
126
+ elif station_origin == 'swd':
127
+ df = etlSWD.download(station_id)
128
+ warehouse.load_df_to_staging(con,df, 'swd_raw', replace = overwrite)
129
+ warehouse.load_df_to_analytics(con,etlSWD.transform(df),'swd')
130
+ else:
131
+ raise ValueError('station_origin must be wiski, equis, or swd')
132
+
133
+ with duckdb.connect(self.db_path,read_only=False) as con:
134
+ warehouse.update_views(con)
135
+
136
+ if to_csv:
137
+ self.to_csv(station_id)
138
+
139
+ return df
140
+
141
+ def get_outlets(self):
142
+ with duckdb.connect(self.db_path,read_only=True) as con:
143
+ query = '''
144
+ SELECT *
145
+ FROM outlets.station_reach_pairs
146
+ ORDER BY outlet_id'''
147
+ df = con.execute(query).fetch_df()
148
+ return df
149
+ def get_station_ids(self,station_origin = None):
150
+ with duckdb.connect(self.db_path,read_only=True) as con:
151
+ if station_origin is None:
152
+ query = '''
153
+ SELECT DISTINCT station_id, station_origin
154
+ FROM analytics.observations'''
155
+ df = con.execute(query).fetch_df()
156
+ else:
157
+ query = '''
158
+ SELECT DISTINCT station_id
159
+ FROM analytics.observations
160
+ WHERE station_origin = ?'''
161
+ df = con.execute(query,[station_origin]).fetch_df()
162
+
163
+ return df['station_id'].to_list()
164
+
165
+
166
+ def get_station_data(self,station_ids,constituent,agg_period = None):
167
+
168
+
169
+ with duckdb.connect(self.db_path,read_only=True) as con:
170
+ query = '''
171
+ SELECT *
172
+ FROM analytics.observations
173
+ WHERE station_id IN ? AND constituent = ?'''
174
+ df = con.execute(query,[station_ids,constituent]).fetch_df()
175
+
176
+ unit = UNIT_DEFAULTS[constituent]
177
+ agg_func = AGG_DEFAULTS[unit]
178
+
179
+ df.set_index('datetime',inplace=True)
180
+ df.attrs['unit'] = unit
181
+ df.attrs['constituent'] = constituent
182
+ if agg_period is not None:
183
+ df = df[['value']].resample(agg_period).agg(agg_func)
184
+ df.attrs['agg_period'] = agg_period
185
+
186
+ df.rename(columns={'value': 'observed'}, inplace=True)
187
+ return df
188
+
189
+ def get_outlet_data(self,outlet_id,constituent,agg_period = 'D'):
190
+ with duckdb.connect(self.db_path,read_only=True) as con:
191
+ query = '''
192
+ SELECT *
193
+ FROM analytics.outlet_observations_with_flow
194
+ WHERE outlet_id = ? AND constituent = ?'''
195
+ df = con.execute(query,[outlet_id,constituent]).fetch_df()
196
+
197
+ unit = UNIT_DEFAULTS[constituent]
198
+ agg_func = AGG_DEFAULTS[unit]
199
+
200
+ df.set_index('datetime',inplace=True)
201
+ df.attrs['unit'] = unit
202
+ df.attrs['constituent'] = constituent
203
+ if agg_period is not None:
204
+ df = df[['value','flow_value','baseflow_value']].resample(agg_period).agg(agg_func)
205
+ df.attrs['agg_period'] = agg_period
206
+
207
+ df.rename(columns={'value': 'observed',
208
+ 'flow_value': 'observed_flow',
209
+ 'baseflow_value': 'observed_baseflow'}, inplace=True)
210
+ return df
211
+
212
+
213
+
214
+ def to_csv(self,station_id,folderpath = None):
215
+ if folderpath is None:
216
+ folderpath = self.folderpath
217
+ else:
218
+ folderpath = Path(folderpath)
219
+ df = self._load(station_id)
220
+ if len(df) > 0:
221
+ df.to_csv(folderpath.joinpath(station_id + '.csv'))
222
+ else:
223
+ print(f'No {station_id} calibration data available at Station {station_id}')
224
+
225
+ df.to_csv(folderpath.joinpath(station_id + '.csv'))
226
+
227
+
228
+ # class database():
229
+ # def __init__(self,db_path):
230
+ # self.dbm = MonitoringDatabase(db_path)
231
+
232
+
233
+ # def get_timeseries(self,station_ds, constituent,agg_period):
234
+ # validate_constituent(constituent)
235
+ # unit = UNIT_DEFAULTS[constituent]
236
+ # agg_func = AGG_DEFAULTS[unit]
237
+ # return odm.get_timeseries(station_id,constituent)
238
+
239
+
240
+ # def get_samples(self,station_ds, constituent,agg_period):
241
+ # validate_constituent(constituent)
242
+ # unit = UNIT_DEFAULTS[constituent]
243
+ # agg_func = AGG_DEFAULTS[unit]
244
+ # return odm.get_sample(station_id,constituent)
245
+
246
+ # def get_samples_and_timeseries(self,station_ds, constituent,agg_period)
247
+