cwms-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- CWMS/.ipynb_checkpoints/__init__-checkpoint.py +11 -0
- CWMS/.ipynb_checkpoints/core-checkpoint.py +0 -0
- CWMS/.ipynb_checkpoints/cwms_loc-checkpoint.py +38 -0
- CWMS/.ipynb_checkpoints/cwms_ts-checkpoint.py +47 -0
- CWMS/.ipynb_checkpoints/utils-checkpoint.py +52 -0
- CWMS/__init__.py +11 -0
- CWMS/__pycache__/__init__.cpython-39.pyc +0 -0
- CWMS/__pycache__/core.cpython-39.pyc +0 -0
- CWMS/__pycache__/cwms_loc.cpython-39.pyc +0 -0
- CWMS/__pycache__/cwms_ts.cpython-39.pyc +0 -0
- CWMS/__pycache__/utils.cpython-39.pyc +0 -0
- CWMS/core.py +21 -0
- CWMS/cwms_loc.py +58 -0
- CWMS/cwms_ts.py +182 -0
- CWMS/utils.py +79 -0
- cwms_python-0.1.0.dist-info/LICENSE +24 -0
- cwms_python-0.1.0.dist-info/METADATA +72 -0
- cwms_python-0.1.0.dist-info/RECORD +20 -0
- cwms_python-0.1.0.dist-info/WHEEL +5 -0
- cwms_python-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
2
|
+
|
|
3
|
+
from CWMSpy.cwms_loc import *
|
|
4
|
+
from CWMSpy.cwm,s_ts import *
|
|
5
|
+
from CWMSpy.utils import *
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
__version__ = version('CWMSpy')
|
|
10
|
+
except PackageNotFoundError:
|
|
11
|
+
__version__ = 'version-unknown'
|
|
File without changes
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from utils import queryCDA
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
def retreive_loc_group(apiRoot,p_loc_group_id,p_category_id,p_office_id,output = 'dataframe'):
|
|
6
|
+
|
|
7
|
+
endPoint = f'/location/group/{p_loc_group_id}'
|
|
8
|
+
|
|
9
|
+
params = {
|
|
10
|
+
"office": p_office_id,
|
|
11
|
+
"category-id": p_category_id
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
headerList={
|
|
15
|
+
"Accept": "application/json"
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
responce = queryCDA(apiRoot+endPoint,params,headerList,output,dict_key = 'assigned-locations')
|
|
19
|
+
|
|
20
|
+
#if dataframe:
|
|
21
|
+
# responce = pd.DataFrame(responce['assigned-locations'])
|
|
22
|
+
|
|
23
|
+
return responce
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def ExpandAliases(df):
|
|
27
|
+
|
|
28
|
+
df_alias = pd.DataFrame()
|
|
29
|
+
temp = df.aliases.apply(pd.Series)
|
|
30
|
+
for i in temp.columns:
|
|
31
|
+
temp2 = temp[i].apply(pd.Series).dropna(how='all')
|
|
32
|
+
temp2 = temp2.dropna(how='all', axis = 'columns')
|
|
33
|
+
temp2 = temp2.reset_index()
|
|
34
|
+
df_alias = pd.concat([df_alias,temp2], ignore_index=True)
|
|
35
|
+
df_alias = df_alias.drop_duplicates(subset=['locID','name'], keep='last')
|
|
36
|
+
df_alias = df_alias.pivot(index='locID', columns='name',values = 'value')
|
|
37
|
+
df_alias = pd.concat([df, df_alias], axis=1)
|
|
38
|
+
return df_alias
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from utils import queryCDA
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
def retreive_ts_group(apiRoot,p_group_id,p_category_id,p_office_id, output = 'dataframe'):
|
|
6
|
+
|
|
7
|
+
endPoint = f'/timeseries/group/{p_group_id}'
|
|
8
|
+
|
|
9
|
+
params = {
|
|
10
|
+
"office": p_office_id,
|
|
11
|
+
"category-id": p_category_id
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
headerList={
|
|
15
|
+
"Accept": "application/json"
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
responce = queryCDA(apiRoot+endPoint, params, headerList, output, dict_key = 'assigned-time-series')
|
|
19
|
+
|
|
20
|
+
#if dataframe:
|
|
21
|
+
# responce = pd.DataFrame(responce['assigned-time-series'])
|
|
22
|
+
|
|
23
|
+
return responce
|
|
24
|
+
|
|
25
|
+
def retrieve_ts(apiRoot, p_tsId, p_office_id=None, p_unit=None, p_datum=None, p_start_date=None, p_end_date=None, p_timezone=None, p_page_size=500, output='dataframe'):
|
|
26
|
+
#creates the dataframe from the timeseries data
|
|
27
|
+
endPoint = '/timeseries'
|
|
28
|
+
if p_start_date is not None: p_start_date = p_start_date.strftime('%Y-%m-%dT%H:%M:%S')
|
|
29
|
+
|
|
30
|
+
if p_end_date is not None: p_end_date = p_end_date.strftime('%Y-%m-%dT%H:%M:%S')
|
|
31
|
+
|
|
32
|
+
params = {
|
|
33
|
+
"office": p_office_id,
|
|
34
|
+
"name": p_tsId,
|
|
35
|
+
"unit": p_unit,
|
|
36
|
+
"begin": p_start_date,
|
|
37
|
+
"end": p_end_date,
|
|
38
|
+
"page-size" : p_page_size
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
headerList={
|
|
42
|
+
"Accept": "application/json;version=2"
|
|
43
|
+
}
|
|
44
|
+
responce = queryCDA(apiRoot+endPoint,params,headerList,output, dict_key = 'values')
|
|
45
|
+
|
|
46
|
+
return responce
|
|
47
|
+
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
def queryCDA(url, payload, headerList, output, dict_key):
|
|
6
|
+
"""Send a query.
|
|
7
|
+
|
|
8
|
+
Wrapper for requests.get that handles errors and returns response.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
url: string
|
|
13
|
+
URL to query
|
|
14
|
+
payload: dict
|
|
15
|
+
query parameters passed to ``requests.get``
|
|
16
|
+
|
|
17
|
+
Returns
|
|
18
|
+
-------
|
|
19
|
+
string: query response
|
|
20
|
+
The response from the API query ``requests.get`` function call.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
response = requests.get(url, params=payload, headers=headerList)
|
|
25
|
+
|
|
26
|
+
if response.status_code == 400:
|
|
27
|
+
raise ValueError(
|
|
28
|
+
f'Bad Request, check that your parameters are correct. URL: {response.url}'
|
|
29
|
+
)
|
|
30
|
+
elif response.status_code == 404:
|
|
31
|
+
raise ValueError(
|
|
32
|
+
'Page Not Found Error. May be the result of an empty query. '
|
|
33
|
+
+ f'URL: {response.url}'
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
return output_type(response, output, dict_key)
|
|
37
|
+
|
|
38
|
+
def output_type(response, output, dict_key):
|
|
39
|
+
|
|
40
|
+
if output in ['dataframe','dictionary']:
|
|
41
|
+
response = response.json()
|
|
42
|
+
|
|
43
|
+
if output == 'dataframe':
|
|
44
|
+
temp = pd.DataFrame(response[dict_key])
|
|
45
|
+
if dict_key == 'values':
|
|
46
|
+
temp.columns = [sub['name'] for sub in response['value-columns']]
|
|
47
|
+
|
|
48
|
+
if 'date-time' in temp.columns:
|
|
49
|
+
temp['date-time'] = pd.to_datetime(temp['date-time'], unit='ms')
|
|
50
|
+
response = temp
|
|
51
|
+
|
|
52
|
+
return response
|
CWMS/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
2
|
+
|
|
3
|
+
#from CWMSpy.cwms_loc import *
|
|
4
|
+
#from CWMSpy.cwm,s_ts import *
|
|
5
|
+
#from CWMSpy.utils import *
|
|
6
|
+
from .core import CWMS
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
__version__ = version('cwms-python')
|
|
10
|
+
except PackageNotFoundError:
|
|
11
|
+
__version__ = 'version-unknown'
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
CWMS/core.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
from requests_toolbelt import sessions
|
|
3
|
+
|
|
4
|
+
from .cwms_ts import CwmsTsMixin
|
|
5
|
+
from .cwms_loc import CwmsLocMixin
|
|
6
|
+
#from .cwms_level import CwmsLevelMixin
|
|
7
|
+
#from .cwms_sec import CwmsSecMixin
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class CWMS(CwmsLocMixin, CwmsTsMixin): #CwmsLevelMixin, CwmsSecMixin):
|
|
11
|
+
|
|
12
|
+
def __init__(self, conn=None):
|
|
13
|
+
self.conn = conn
|
|
14
|
+
|
|
15
|
+
def connect(self,apiRoot,apiKey=None):
|
|
16
|
+
self.s = sessions.BaseUrlSession(base_url=apiRoot)
|
|
17
|
+
if apiKey is not None:
|
|
18
|
+
self.s.headers.update({'Authorization': apiKey})
|
|
19
|
+
|
|
20
|
+
def close(self):
|
|
21
|
+
self.s.close()
|
CWMS/cwms_loc.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from .utils import queryCDA
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class CwmsLocMixin:
|
|
7
|
+
def retreive_loc_group(self,p_loc_group_id,p_category_id,p_office_id, return_type='df'):
|
|
8
|
+
|
|
9
|
+
endPoint = f'location/group/{p_loc_group_id}'
|
|
10
|
+
|
|
11
|
+
params = {
|
|
12
|
+
"office": p_office_id,
|
|
13
|
+
"category-id": p_category_id
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
headerList={
|
|
17
|
+
"Accept": "application/json"
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
responce = queryCDA(self,endPoint,params,headerList,return_type,dict_key = ['assigned-locations'])
|
|
21
|
+
|
|
22
|
+
#if dataframe:
|
|
23
|
+
# responce = pd.DataFrame(responce['assigned-locations'])
|
|
24
|
+
|
|
25
|
+
return responce
|
|
26
|
+
|
|
27
|
+
def retreive_locs(self, p_office_id=None, p_loc_ids = None, p_units = None, p_datum = None, return_type='df'):
|
|
28
|
+
|
|
29
|
+
endPoint = 'locations'
|
|
30
|
+
|
|
31
|
+
params = {
|
|
32
|
+
'office': p_office_id,
|
|
33
|
+
'names' : p_loc_ids,
|
|
34
|
+
'units' : p_units,
|
|
35
|
+
'datum' : p_datum,
|
|
36
|
+
}
|
|
37
|
+
headerList={
|
|
38
|
+
"Accept": "application/json;version=2"
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
responce = queryCDA(self,endPoint,params,headerList,return_type,dict_key = ['locations','locations'])
|
|
42
|
+
#if output = 'dataframe':
|
|
43
|
+
#responce =
|
|
44
|
+
return responce
|
|
45
|
+
|
|
46
|
+
def ExpandLocations(df):
|
|
47
|
+
|
|
48
|
+
df_alias = pd.DataFrame()
|
|
49
|
+
temp = df.aliases.apply(pd.Series)
|
|
50
|
+
for i in temp.columns:
|
|
51
|
+
temp2 = temp[i].apply(pd.Series).dropna(how='all')
|
|
52
|
+
temp2 = temp2.dropna(how='all', axis = 'columns')
|
|
53
|
+
temp2 = temp2.reset_index()
|
|
54
|
+
df_alias = pd.concat([df_alias,temp2], ignore_index=True)
|
|
55
|
+
df_alias = df_alias.drop_duplicates(subset=['locID','name'], keep='last')
|
|
56
|
+
df_alias = df_alias.pivot(index='locID', columns='name',values = 'value')
|
|
57
|
+
df_alias = pd.concat([df, df_alias], axis=1)
|
|
58
|
+
return df_alias
|
CWMS/cwms_ts.py
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
from .utils import queryCDA
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class CwmsTsMixin:
|
|
7
|
+
|
|
8
|
+
def retreive_ts_group(self,p_group_id,p_category_id,p_office_id, return_type='df'):
|
|
9
|
+
"""Retreives time series stored in the requested time series group
|
|
10
|
+
|
|
11
|
+
Parameters
|
|
12
|
+
-------
|
|
13
|
+
p_group_id : str
|
|
14
|
+
Specifies the timeseries group whose data is to be included in the response (required)
|
|
15
|
+
p_category_id : str
|
|
16
|
+
Specifies the category containing the timeseries group whose data is to be included in the response. (required)
|
|
17
|
+
p_office_id : str
|
|
18
|
+
Specifies the owning office of the timeseries group whose data is to be included in the response. (required)
|
|
19
|
+
return_type : str
|
|
20
|
+
output type to return values as. 1. 'df' will return a pandas dataframe. 2. 'dict' will return a json decoded dictionay. 3. all other values will return Responce object from request package.
|
|
21
|
+
|
|
22
|
+
Returns
|
|
23
|
+
-------
|
|
24
|
+
pandas df, json decoded dictionay, or Responce object from request package
|
|
25
|
+
|
|
26
|
+
Examples
|
|
27
|
+
-------
|
|
28
|
+
"""
|
|
29
|
+
endPoint = f'timeseries/group/{p_group_id}'
|
|
30
|
+
|
|
31
|
+
params = {
|
|
32
|
+
"office": p_office_id,
|
|
33
|
+
"category-id": p_category_id
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
headerList={
|
|
37
|
+
"Accept": "application/json"
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
responce = queryCDA(self, endPoint, params, headerList, return_type, dict_key = ['assigned-time-series'])
|
|
41
|
+
|
|
42
|
+
#if dataframe:
|
|
43
|
+
# responce = pd.DataFrame(responce['assigned-time-series'])
|
|
44
|
+
|
|
45
|
+
return responce
|
|
46
|
+
|
|
47
|
+
def retrieve_ts(self, p_tsId, p_office_id, p_unit='EN', p_datum=None, p_start_date=None, p_end_date=None, p_timezone=None, p_page_size=500000, return_type='df'):
|
|
48
|
+
"""Retrieves time series data from a specified time series and time window. Value date-times obtained are always in UTC.
|
|
49
|
+
|
|
50
|
+
Parameters
|
|
51
|
+
-----------
|
|
52
|
+
p_tsId : str (required)
|
|
53
|
+
Specifies the name(s) of the time series whose data is to be included in the response. A case insensitive comparison is used to match names.
|
|
54
|
+
p_office_id : str
|
|
55
|
+
Specifies the owning office of the time series(s) whose data is to be included in the response. If this field is not specified, matching location level information from all offices shall be returned.
|
|
56
|
+
p_unit : str
|
|
57
|
+
Specifies the unit or unit system of the response. Valid values for the unit field are: 1. EN. (default) Specifies English unit system. 2. SI. Specifies the SI unit system. 3. Other. Any unit returned in the response to the units URI request that is appropriate for the requested parameters.
|
|
58
|
+
p_datum: str
|
|
59
|
+
Specifies the elevation datum of the response. This field affects only elevation location levels. Valid values for this field are: 1. NAVD88. The elevation values will in the specified or default units above the NAVD-88 datum. 2. NGVD29. The elevation values will be in the specified or default units above the NGVD-29 datum.
|
|
60
|
+
p_start_date : datetime
|
|
61
|
+
Specifies the start of the time window for data to be included in the response. If this field is not specified, any required time window begins 24 hours prior to the specified or default end time.
|
|
62
|
+
p_end_date : datetime
|
|
63
|
+
Specifies the end of the time window for data to be included in the response. If this field is not specified, any required time window ends at the current time.
|
|
64
|
+
p_timezone : string
|
|
65
|
+
Specifies the time zone of the values of the begin and end fields. UTC is the default. This value does not impact the values in response. response is always in UTC.
|
|
66
|
+
return_type : str
|
|
67
|
+
output type to return values as. 1. 'df' will return a pandas dataframe. 2. 'dict' will return a json decoded dictionay. 3. all other values will return Responce object from request package.
|
|
68
|
+
Returns
|
|
69
|
+
--------
|
|
70
|
+
pandas df, json decoded dictionay, or Responce object from request package. Dates in responce are in UTC.
|
|
71
|
+
|
|
72
|
+
Examples
|
|
73
|
+
-------
|
|
74
|
+
"""
|
|
75
|
+
#creates the dataframe from the timeseries data
|
|
76
|
+
endPoint = 'timeseries'
|
|
77
|
+
if p_start_date is not None: p_start_date = p_start_date.strftime('%Y-%m-%dT%H:%M:%S')
|
|
78
|
+
|
|
79
|
+
if p_end_date is not None: p_end_date = p_end_date.strftime('%Y-%m-%dT%H:%M:%S')
|
|
80
|
+
|
|
81
|
+
params = {
|
|
82
|
+
"office": p_office_id,
|
|
83
|
+
"name": p_tsId,
|
|
84
|
+
"unit": p_unit,
|
|
85
|
+
"datum": p_datum,
|
|
86
|
+
"begin": p_start_date,
|
|
87
|
+
"end": p_end_date,
|
|
88
|
+
"timezone" : p_timezone,
|
|
89
|
+
"page-size" : p_page_size
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
headerList={
|
|
93
|
+
"Accept": "application/json;version=2"
|
|
94
|
+
}
|
|
95
|
+
responce = queryCDA(self,endPoint,params,headerList,return_type, dict_key = ['values'])
|
|
96
|
+
|
|
97
|
+
return responce
|
|
98
|
+
|
|
99
|
+
def write_ts(self, data, version_date = None, timezone = None, create_as_ltrs = False, store_rule = None, override_protection = None):
|
|
100
|
+
"""Will Create new TimeSeries if not already present. Will store any data provided.
|
|
101
|
+
|
|
102
|
+
Parameters
|
|
103
|
+
-------
|
|
104
|
+
data : pd.Dataframe, or Dictionay
|
|
105
|
+
Time Series data to be stored. If dataframe data must be provided in the following format
|
|
106
|
+
df.tsId = timeseried id:specified name of the time series to be posted to
|
|
107
|
+
df.office = the owning office of the time series
|
|
108
|
+
df.units = units of values to be stored (ie. ft, in, m, cfs....)
|
|
109
|
+
dataframe should have three columns date-time, value, quality-code. date-time values can be a string in ISO8601 formate or a datetime field.
|
|
110
|
+
if quality-code column is not present is will be set to 0.
|
|
111
|
+
date-time value quality-code
|
|
112
|
+
0 2023-12-20T14:45:00.000-05:00 93.1 0
|
|
113
|
+
1 2023-12-20T15:00:00.000-05:00 99.8 0
|
|
114
|
+
2 2023-12-20T15:15:00.000-05:00 98.5 0
|
|
115
|
+
3 2023-12-20T15:30:00.000-05:00 98.5 0
|
|
116
|
+
|
|
117
|
+
version_date : str
|
|
118
|
+
Specifies the version date for the timeseries to create. If this field is not specified, a null version date will be used. The format for this field is ISO 8601 extended, with optional timezone, i.e., 'format', e.g., '2021-06-10T13:00:00-0700[PST8PDT]'.
|
|
119
|
+
timezone : str
|
|
120
|
+
Specifies the time zone of the version-date field (unless otherwise specified). If this field is not specified, the default time zone of UTC shall be used. Ignored if version-date was specified with offset and timezone.
|
|
121
|
+
create_as_lrts : bool
|
|
122
|
+
Flag indicating if timeseries should be created as Local Regular Time Series. 'True' or 'False', default is 'False'
|
|
123
|
+
store_rule : str
|
|
124
|
+
The business rule to use when merging the incoming with existing data. Available values : REPLACE_ALL, DO_NOT_REPLACE, REPLACE_MISSING_VALUES_ONLY, REPLACE_WITH_NON_MISSING, DELETE_INSERT
|
|
125
|
+
override_protection : bool
|
|
126
|
+
A flag to ignore the protected data quality when storing data. 'True' or 'False'
|
|
127
|
+
|
|
128
|
+
Returns
|
|
129
|
+
-------
|
|
130
|
+
Responce object from request package
|
|
131
|
+
|
|
132
|
+
Examples
|
|
133
|
+
-------
|
|
134
|
+
|
|
135
|
+
"""
|
|
136
|
+
endPoint = 'timeseries'
|
|
137
|
+
params = {
|
|
138
|
+
'version-date': version_date,
|
|
139
|
+
'timezone': timezone,
|
|
140
|
+
'create-as-lrts' : create_as_ltrs,
|
|
141
|
+
'store-rule' : store_rule,
|
|
142
|
+
'override-protection' : override_protection
|
|
143
|
+
}
|
|
144
|
+
headerList={
|
|
145
|
+
'accept': '*/*',
|
|
146
|
+
'Content-Type': 'application/json;version=2',
|
|
147
|
+
}
|
|
148
|
+
if isinstance(data, pd.DataFrame):
|
|
149
|
+
#grab time series information
|
|
150
|
+
tsId = data.tsId
|
|
151
|
+
office = data.office
|
|
152
|
+
units = data.units
|
|
153
|
+
|
|
154
|
+
#check dataframe columns
|
|
155
|
+
if 'quality-code' not in data:
|
|
156
|
+
values['quality-code'] = 0
|
|
157
|
+
if 'date-time' not in data:
|
|
158
|
+
raise TypeError("date-time is a required column in data when posting as a dateframe")
|
|
159
|
+
if 'value' not in data:
|
|
160
|
+
raise TypeError("value is a required column when posting data when posting as a dataframe")
|
|
161
|
+
|
|
162
|
+
#make sure that dataTime column is in iso8601 formate.
|
|
163
|
+
data['date-time'] = pd.to_datetime(data['date-time']).apply(pd.Timestamp.isoformat)
|
|
164
|
+
data = data.reindex(columns=['date-time','value','quality-code'])
|
|
165
|
+
if data.isnull().values.any():
|
|
166
|
+
raise ValueError("Null/NaN data must be removed from the dataframe")
|
|
167
|
+
|
|
168
|
+
ts_dict = {"name": tsId,
|
|
169
|
+
"office-id": office,
|
|
170
|
+
"units": units,
|
|
171
|
+
"values": data.values.tolist()
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
elif isinstance(data, dict):
|
|
175
|
+
ts_dict = data
|
|
176
|
+
|
|
177
|
+
else: raise TypeError("data is not of type dataframe or dictionary")
|
|
178
|
+
|
|
179
|
+
#print(ts_dict)
|
|
180
|
+
response = self.s.post(endPoint, headers = headerList, data = json.dumps(ts_dict))
|
|
181
|
+
return response
|
|
182
|
+
|
CWMS/utils.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
def queryCDA(self, endpoint, payload, headerList, return_type, dict_key):
|
|
6
|
+
"""Send a query.
|
|
7
|
+
|
|
8
|
+
Wrapper for requests.get that handles errors and returns response.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
endpoint: string
|
|
13
|
+
URL to query
|
|
14
|
+
payload: dict
|
|
15
|
+
query parameters passed to ``requests.get``
|
|
16
|
+
headerList: dict
|
|
17
|
+
headers
|
|
18
|
+
return_type : str
|
|
19
|
+
output type to return values as. 1. 'df' will return a pandas dataframe. 2. 'dict' will return a json decoded dictionay. 3. all other values will return Responce object from request package.
|
|
20
|
+
dict_key : str
|
|
21
|
+
key needed to grab correct values from json decoded dictionary.
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
Returns
|
|
25
|
+
-------
|
|
26
|
+
string: query response
|
|
27
|
+
The response from the API query ``requests.get`` function call.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
response = self.s.get(endpoint, params=payload, headers=headerList)
|
|
32
|
+
|
|
33
|
+
if response.status_code == 400:
|
|
34
|
+
raise ValueError(
|
|
35
|
+
f'Bad Request, check that your parameters are correct. URL: {response.url}'
|
|
36
|
+
)
|
|
37
|
+
elif response.status_code == 404:
|
|
38
|
+
raise ValueError(
|
|
39
|
+
'Page Not Found Error. May be the result of an empty query. '
|
|
40
|
+
+ f'URL: {response.url}'
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
return output_type(response, return_type, dict_key)
|
|
44
|
+
|
|
45
|
+
def output_type(response, return_type, dict_key):
|
|
46
|
+
"""Convert output to correct format requested by user
|
|
47
|
+
Parameters
|
|
48
|
+
----------
|
|
49
|
+
response : Request object
|
|
50
|
+
response from get request
|
|
51
|
+
return_type : str
|
|
52
|
+
output type to return values as. 1. 'df' will return a pandas dataframe. 2. 'dict' will return a json decoded dictionay. 3. all other values will return Responce object from request package.
|
|
53
|
+
dict_key : str
|
|
54
|
+
key needed to grab correct values from json decoded dictionary.
|
|
55
|
+
|
|
56
|
+
Returns
|
|
57
|
+
-------
|
|
58
|
+
pandas df, json decoded dictionay, or Responce object from request package
|
|
59
|
+
"""
|
|
60
|
+
#converts responce object to dictionary if output is df or dict
|
|
61
|
+
if return_type in ['df','dict']:
|
|
62
|
+
response = response.json()
|
|
63
|
+
|
|
64
|
+
#converts dictionary to df based on the key provided for the endpoint
|
|
65
|
+
if return_type == 'df':
|
|
66
|
+
temp = response
|
|
67
|
+
for key in dict_key:
|
|
68
|
+
temp = temp[key]
|
|
69
|
+
temp_df = pd.DataFrame(temp)
|
|
70
|
+
|
|
71
|
+
#if timeseries values are present then grab the values and put into dataframe
|
|
72
|
+
if dict_key[-1] == 'values':
|
|
73
|
+
temp_df.columns = [sub['name'] for sub in response['value-columns']]
|
|
74
|
+
|
|
75
|
+
if 'date-time' in temp_df.columns:
|
|
76
|
+
temp_df['date-time'] = pd.to_datetime(temp_df['date-time'], unit='ms')
|
|
77
|
+
response = temp_df
|
|
78
|
+
|
|
79
|
+
return response
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
This is free and unencumbered software released into the public domain.
|
|
2
|
+
|
|
3
|
+
Anyone is free to copy, modify, publish, use, compile, sell, or
|
|
4
|
+
distribute this software, either in source code form or as a compiled
|
|
5
|
+
binary, for any purpose, commercial or non-commercial, and by any
|
|
6
|
+
means.
|
|
7
|
+
|
|
8
|
+
In jurisdictions that recognize copyright laws, the author or authors
|
|
9
|
+
of this software dedicate any and all copyright interest in the
|
|
10
|
+
software to the public domain. We make this dedication for the benefit
|
|
11
|
+
of the public at large and to the detriment of our heirs and
|
|
12
|
+
successors. We intend this dedication to be an overt act of
|
|
13
|
+
relinquishment in perpetuity of all present and future rights to this
|
|
14
|
+
software under copyright law.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
19
|
+
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
|
20
|
+
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
21
|
+
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
22
|
+
OTHER DEALINGS IN THE SOFTWARE.
|
|
23
|
+
|
|
24
|
+
For more information, please refer to <https://unlicense.org>
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: cwms-python
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Corps water managerment systems (CWMS) REST API for Data Retrieval of USACE water data
|
|
5
|
+
Home-page: https://github.com/HydrologicEngineeringCenter/cwms-python
|
|
6
|
+
Author: Eric Novotny
|
|
7
|
+
Maintainer-email: Eric Novotny <eric.v.novotny@usace.army.mil>
|
|
8
|
+
Project-URL: homepage, https://github.com/HydrologicEngineeringCenter/cwms-python
|
|
9
|
+
Project-URL: repository, https://github.com/HydrologicEngineeringCenter/cwms-python.git
|
|
10
|
+
Keywords: USACE,water data
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Requires-Python: >=3.8
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: requests-toolbelt
|
|
16
|
+
Requires-Dist: requests
|
|
17
|
+
Requires-Dist: pandas
|
|
18
|
+
Provides-Extra: doc
|
|
19
|
+
Requires-Dist: sphinx ; extra == 'doc'
|
|
20
|
+
Provides-Extra: test
|
|
21
|
+
Requires-Dist: pytest >5.0.0 ; extra == 'test'
|
|
22
|
+
Requires-Dist: pytest-cov[all] ; extra == 'test'
|
|
23
|
+
|
|
24
|
+
# CWMSpy
|
|
25
|
+
CWMS REST API for Data Retrieval
|
|
26
|
+
|
|
27
|
+
## Requirements.
|
|
28
|
+
|
|
29
|
+
Python 3.8+
|
|
30
|
+
|
|
31
|
+
## Installation & Usage
|
|
32
|
+
### pip install
|
|
33
|
+
|
|
34
|
+
If the python package is hosted on Github, you can install directly from Github
|
|
35
|
+
|
|
36
|
+
```sh
|
|
37
|
+
pip install git+https://github.com/HydrologicEngineeringCenter/cwms-python.git
|
|
38
|
+
```
|
|
39
|
+
(you may need to run `pip` with root permission: `sudo pip install git+https://github.com/HydrologicEngineeringCenter/cwms-python.git`)
|
|
40
|
+
|
|
41
|
+
Then import the package:
|
|
42
|
+
```python
|
|
43
|
+
from CWMS import CWMS
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Getting Started
|
|
47
|
+
|
|
48
|
+
Please follow the [installation procedure](#installation--usage) and then run the following:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
from CWMS import CWMS
|
|
52
|
+
from datetime import datetime, timedelta
|
|
53
|
+
|
|
54
|
+
apiRoot = 'CDA url to connect to'
|
|
55
|
+
|
|
56
|
+
cwms = CWMS()
|
|
57
|
+
cwms.connect(apiRoot)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
end = datetime.now()
|
|
61
|
+
start = end - timedelta(days = 10)
|
|
62
|
+
df cwms.retrieve_ts(p_tsId='Some.Fully.Qualified.Ts.Id',p_start_date = start, p_end_date = end)
|
|
63
|
+
ts_df.head()
|
|
64
|
+
```
|
|
65
|
+
```
|
|
66
|
+
date-time value quality-code
|
|
67
|
+
0 2023-12-25 06:00:00 1432.82 0
|
|
68
|
+
1 2023-12-28 06:00:00 1432.86 0
|
|
69
|
+
2 2023-12-29 06:00:00 1432.92 0
|
|
70
|
+
3 2023-12-30 06:00:00 1432.92 0
|
|
71
|
+
4 2023-12-31 06:00:00 1432.91 0
|
|
72
|
+
```
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
CWMS/__init__.py,sha256=YCtFb6lu2H2z5rXf7h-VaNEBPZRSpOIaniMXS1kBuGU,287
|
|
2
|
+
CWMS/core.py,sha256=DmWXKS8SZ5t4kWFyoA8g7M-bk94t6DO9oUL-Yaa69ww,589
|
|
3
|
+
CWMS/cwms_loc.py,sha256=Z4RoQ7aGLGSndQs27IBTTSBwJIRXWa3JmRBw10lxIDM,1915
|
|
4
|
+
CWMS/cwms_ts.py,sha256=m4x_PZx29xtXTatJFnGCtgLC8DAA-e2lnvaGcePfwAI,9217
|
|
5
|
+
CWMS/utils.py,sha256=431HQF5q4-yhvDd14izImPUBUYu2uQMZQ7bo7GCkpNE,2703
|
|
6
|
+
CWMS/.ipynb_checkpoints/__init__-checkpoint.py,sha256=UpWwPd1fnkkw5A2DoBiCRoUihAOPo4t0Frj8pZlQmFU,256
|
|
7
|
+
CWMS/.ipynb_checkpoints/core-checkpoint.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
CWMS/.ipynb_checkpoints/cwms_loc-checkpoint.py,sha256=_-emS967g6fd-aE8V61ZO-veoQZj3csboTJmrjX0qrA,1156
|
|
9
|
+
CWMS/.ipynb_checkpoints/cwms_ts-checkpoint.py,sha256=xZzMCkpyX5u-D3CIlW2Xw4cJP--aYsd9P7Tay3eS4pE,1413
|
|
10
|
+
CWMS/.ipynb_checkpoints/utils-checkpoint.py,sha256=C-vl6hw4-yTTKuRiuLgRebwjN5Gev3mmeI-d2tkygPQ,1426
|
|
11
|
+
CWMS/__pycache__/__init__.cpython-39.pyc,sha256=kr79NF4C3O_3YDyKMV41opWA70R6dadJV6e0FyOP9c0,339
|
|
12
|
+
CWMS/__pycache__/core.cpython-39.pyc,sha256=R41CnTg3rsCk59J0X_OJ-miSu8Zwz843SjqrHewau3c,952
|
|
13
|
+
CWMS/__pycache__/cwms_loc.cpython-39.pyc,sha256=uFEt74k_Xh4Q_WScwlML3sbcsEUJraAFtid3CkIuyq0,1692
|
|
14
|
+
CWMS/__pycache__/cwms_ts.cpython-39.pyc,sha256=eHru4Z4gMk4WjpTQut6ETrCsBPzmrXqj_6LJFNznjNs,8012
|
|
15
|
+
CWMS/__pycache__/utils.cpython-39.pyc,sha256=njjuT3WLXmlvoFEjQ7Yq4M3hoLIKaQUpYP7g_8gPak8,2487
|
|
16
|
+
cwms_python-0.1.0.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
|
|
17
|
+
cwms_python-0.1.0.dist-info/METADATA,sha256=TCHfldJxpgfNBqZNn_6lWd_PFftqsgPW6wyBU-OM7eQ,2002
|
|
18
|
+
cwms_python-0.1.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
19
|
+
cwms_python-0.1.0.dist-info/top_level.txt,sha256=AFOyUiYroqfw6wVz7vGxaPdXdacJGr4LLQ0pY5An9P8,5
|
|
20
|
+
cwms_python-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
CWMS
|