cwms-cli 0.3.7__tar.gz → 0.3.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/PKG-INFO +1 -1
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/usgs/getusgs_cda.py +79 -16
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/__init__.py +3 -3
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/pyproject.toml +1 -1
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/LICENSE +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/README.md +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/__init__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/__main__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/callbacks/__init__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/blob.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/commands_cwms.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/.gitignore +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/README.md +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/__init__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/__main__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/config.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/doclinks.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/examples/complete_config.json +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/parser.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/__init__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/data/.gitignore +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/data/expected_brok_output.json +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/data/sample_brok.csv +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/data/sample_config.json +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/skip_test_integration_pipeline.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/test_dateutils.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/test_expressions.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/test_fileio.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/test_main.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/transform.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/utils/__init__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/utils/dateutils.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/utils/expression.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/utils/fileio.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/utils/logging.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/writer.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/shef_critfile_import.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/load/README.md +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/load/__init__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/load/__main__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/load/location/location.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/load/location/location_ids.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/load/location/location_ids_bygroup.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/load/root.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/load/timeseries/timeseries.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/load/timeseries/timeseries_ids.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/requirements.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/usgs/__init__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/usgs/__main__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/usgs/getUSGS_ratings_cda.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/usgs/getusgs_measurements_cda.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/usgs/rating_ini_file_import.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/click_help.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/colors.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/deps.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/friendly_errors.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/intervals.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/io.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/links.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/logging/__init__.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/logging/formatters.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/ssl_errors.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/update.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/version.py +0 -0
- {cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/utils/version_cli.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cwms-cli
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.8
|
|
4
4
|
Summary: Command line utilities for Corps Water Management Systems (CWMS) python scripts. This is a collection of shared scripts across the enterprise Water Management Enterprise System (WMES) teams.
|
|
5
5
|
License: LICENSE
|
|
6
6
|
License-File: LICENSE
|
|
@@ -6,6 +6,36 @@ import numpy as np
|
|
|
6
6
|
import pandas as pd
|
|
7
7
|
import requests
|
|
8
8
|
|
|
9
|
+
from cwmscli.utils import colors
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _log_error_and_exit(message: str, hint: str | None = None, *, exit_code: int = 1):
|
|
13
|
+
logging.error(colors.c(message, "red", bright=True))
|
|
14
|
+
if hint:
|
|
15
|
+
logging.error(colors.c(f"Hint: {hint}", "yellow", bright=True))
|
|
16
|
+
raise SystemExit(exit_code)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _require_group_dataframe(response, *, resource_name: str, office: str):
|
|
20
|
+
payload = getattr(response, "json", None)
|
|
21
|
+
|
|
22
|
+
try:
|
|
23
|
+
df = response.df
|
|
24
|
+
except AttributeError as exc:
|
|
25
|
+
if isinstance(payload, str):
|
|
26
|
+
_log_error_and_exit(
|
|
27
|
+
f"CWMS returned an unexpected response while reading {resource_name} for office {office}.",
|
|
28
|
+
"Check that --api-root points to the CDA API endpoint ending in /cwms-data.",
|
|
29
|
+
)
|
|
30
|
+
raise
|
|
31
|
+
|
|
32
|
+
if df is None:
|
|
33
|
+
_log_error_and_exit(
|
|
34
|
+
f"CWMS did not return any {resource_name} data for office {office}."
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
return df
|
|
38
|
+
|
|
9
39
|
|
|
10
40
|
def getusgs_cda(
|
|
11
41
|
api_root: str,
|
|
@@ -58,12 +88,13 @@ def getusgs_cda(
|
|
|
58
88
|
CWMS_writeData(USGS_ts, USGS_data, USGS_data_method, days_back)
|
|
59
89
|
else:
|
|
60
90
|
if backfill_tsids:
|
|
61
|
-
|
|
62
|
-
f"The following
|
|
91
|
+
_log_error_and_exit(
|
|
92
|
+
f"The following backfill time series ids were not present in the USGS time series or location alias groups: {backfill_tsids}"
|
|
63
93
|
)
|
|
64
94
|
else:
|
|
65
|
-
|
|
66
|
-
f"
|
|
95
|
+
_log_error_and_exit(
|
|
96
|
+
f"No eligible USGS time series were found for office {office_id}.",
|
|
97
|
+
"Confirm that time series exist in Data Acquisition / USGS TS Data Acquisition and that matching entries exist in Agency Aliases / USGS Station Number.",
|
|
67
98
|
)
|
|
68
99
|
|
|
69
100
|
|
|
@@ -115,18 +146,36 @@ def get_CMWS_TS_Loc_Data(office):
|
|
|
115
146
|
usgs_param = "Not Found"
|
|
116
147
|
return usgs_param
|
|
117
148
|
|
|
118
|
-
df =
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
149
|
+
df = _require_group_dataframe(
|
|
150
|
+
cwms.get_timeseries_group(
|
|
151
|
+
group_id="USGS TS Data Acquisition",
|
|
152
|
+
category_id="Data Acquisition",
|
|
153
|
+
office_id=office,
|
|
154
|
+
category_office_id="CWMS",
|
|
155
|
+
group_office_id="CWMS",
|
|
156
|
+
),
|
|
157
|
+
resource_name="USGS TS Data Acquisition group",
|
|
158
|
+
office=office,
|
|
159
|
+
)
|
|
160
|
+
if df is None or df.empty:
|
|
161
|
+
_log_error_and_exit(
|
|
162
|
+
f"No time series are defined in Data Acquisition / USGS TS Data Acquisition for office {office}.",
|
|
163
|
+
"Add one or more time series to that group, then rerun the command.",
|
|
164
|
+
)
|
|
165
|
+
if "timeseries-id" not in df.columns:
|
|
166
|
+
_log_error_and_exit(
|
|
167
|
+
"The USGS TS Data Acquisition group response did not include a 'timeseries-id' column."
|
|
168
|
+
)
|
|
125
169
|
df[["location-id", "param", "type", "int", "dur", "ver"]] = df[
|
|
126
170
|
"timeseries-id"
|
|
127
171
|
].str.split(".", expand=True)
|
|
128
172
|
|
|
129
173
|
df = df[df["office-id"] == office]
|
|
174
|
+
if df.empty:
|
|
175
|
+
_log_error_and_exit(
|
|
176
|
+
f"No time series are defined in Data Acquisition / USGS TS Data Acquisition for office {office}.",
|
|
177
|
+
"Add one or more office-specific time series to that group, then rerun the command.",
|
|
178
|
+
)
|
|
130
179
|
df["base-loc"] = df["location-id"].str.split("-", expand=True)[0]
|
|
131
180
|
if "alias-id" not in df.columns:
|
|
132
181
|
df["alias-id"] = np.nan
|
|
@@ -135,17 +184,31 @@ def get_CMWS_TS_Loc_Data(office):
|
|
|
135
184
|
df = df.rename(columns={"alias-id": "USGS_Method_TS"})
|
|
136
185
|
|
|
137
186
|
# error in CDA with category_office_id and group_office_id. need to fix once CDA is updated
|
|
138
|
-
Locdf =
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
187
|
+
Locdf = _require_group_dataframe(
|
|
188
|
+
cwms.get_location_group(
|
|
189
|
+
loc_group_id="USGS Station Number",
|
|
190
|
+
category_id="Agency Aliases",
|
|
191
|
+
office_id="CWMS",
|
|
192
|
+
),
|
|
193
|
+
resource_name="USGS Station Number alias group",
|
|
194
|
+
office=office,
|
|
195
|
+
).set_index("location-id")
|
|
143
196
|
|
|
144
197
|
Locdf = Locdf[Locdf["office-id"] == office]
|
|
198
|
+
if Locdf.empty:
|
|
199
|
+
_log_error_and_exit(
|
|
200
|
+
f"No USGS location aliases are defined in Agency Aliases / USGS Station Number for office {office}.",
|
|
201
|
+
"Add one or more location aliases to that group, then rerun the command.",
|
|
202
|
+
)
|
|
145
203
|
if "attribute" not in Locdf.columns:
|
|
146
204
|
Locdf["attribute"] = np.nan
|
|
147
205
|
# Grab all of the locations that have a USGS station number assigned to them
|
|
148
206
|
USGS_alias = Locdf[Locdf["alias-id"].notnull()]
|
|
207
|
+
if USGS_alias.empty:
|
|
208
|
+
_log_error_and_exit(
|
|
209
|
+
f"No USGS location aliases are defined in Agency Aliases / USGS Station Number for office {office}.",
|
|
210
|
+
"Add one or more alias-id values for that office, then rerun the command.",
|
|
211
|
+
)
|
|
149
212
|
# rename the columns
|
|
150
213
|
USGS_alias = USGS_alias.rename(
|
|
151
214
|
columns={"alias-id": "USGS_St_Num", "attribute": "Loc_attribute"}
|
|
@@ -88,11 +88,11 @@ log_level_option = click.option(
|
|
|
88
88
|
|
|
89
89
|
|
|
90
90
|
def get_api_key(api_key: str, api_key_loc: str) -> str:
|
|
91
|
-
if
|
|
92
|
-
return api_key
|
|
93
|
-
elif api_key_loc is not None:
|
|
91
|
+
if api_key_loc is not None:
|
|
94
92
|
with open(api_key_loc, "r") as f:
|
|
95
93
|
return f.readline().strip()
|
|
94
|
+
elif api_key is not None:
|
|
95
|
+
return api_key
|
|
96
96
|
else:
|
|
97
97
|
raise Exception(
|
|
98
98
|
"must add a value to either --api-key(-k) or --api-key-loc(-kl)"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/data/expected_brok_output.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cwms_cli-0.3.7 → cwms_cli-0.3.8}/cwmscli/commands/csv2cwms/tests/skip_test_integration_pipeline.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|