cwms-cli 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cwms_cli-0.1.1.dist-info/METADATA +40 -0
- cwms_cli-0.1.1.dist-info/RECORD +41 -0
- cwms_cli-0.1.1.dist-info/WHEEL +4 -0
- cwms_cli-0.1.1.dist-info/entry_points.txt +3 -0
- cwms_cli-0.1.1.dist-info/licenses/LICENSE +21 -0
- cwmscli/__init__.py +12 -0
- cwmscli/__main__.py +15 -0
- cwmscli/callbacks/__init__.py +18 -0
- cwmscli/commands/blob.py +439 -0
- cwmscli/commands/commands_cwms.py +227 -0
- cwmscli/commands/csv2cwms/.gitignore +3 -0
- cwmscli/commands/csv2cwms/README.md +51 -0
- cwmscli/commands/csv2cwms/__init__.py +5 -0
- cwmscli/commands/csv2cwms/__main__.py +265 -0
- cwmscli/commands/csv2cwms/examples/complete_config.json +19 -0
- cwmscli/commands/csv2cwms/examples/hourly.json +243 -0
- cwmscli/commands/csv2cwms/examples/minutes.json +315 -0
- cwmscli/commands/csv2cwms/tests/__init__.py +0 -0
- cwmscli/commands/csv2cwms/tests/data/.gitignore +1 -0
- cwmscli/commands/csv2cwms/tests/data/expected_brok_output.json +278 -0
- cwmscli/commands/csv2cwms/tests/data/sample_brok.csv +9 -0
- cwmscli/commands/csv2cwms/tests/data/sample_config.json +45 -0
- cwmscli/commands/csv2cwms/tests/skip_test_integration_pipeline.py +35 -0
- cwmscli/commands/csv2cwms/tests/test_dateutils.py +68 -0
- cwmscli/commands/csv2cwms/tests/test_expressions.py +49 -0
- cwmscli/commands/csv2cwms/tests/test_fileio.py +43 -0
- cwmscli/commands/csv2cwms/utils/__init__.py +5 -0
- cwmscli/commands/csv2cwms/utils/dateutils.py +105 -0
- cwmscli/commands/csv2cwms/utils/expression.py +39 -0
- cwmscli/commands/csv2cwms/utils/fileio.py +26 -0
- cwmscli/commands/csv2cwms/utils/logging.py +80 -0
- cwmscli/commands/csv2cwms/utils/terminal.py +45 -0
- cwmscli/commands/shef_critfile_import.py +146 -0
- cwmscli/requirements.py +25 -0
- cwmscli/usgs/__init__.py +161 -0
- cwmscli/usgs/getUSGS_ratings_cda.py +346 -0
- cwmscli/usgs/getusgs_cda.py +345 -0
- cwmscli/usgs/getusgs_measurements_cda.py +961 -0
- cwmscli/usgs/rating_ini_file_import.py +130 -0
- cwmscli/utils/__init__.py +68 -0
- cwmscli/utils/deps.py +102 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
import cwms
|
|
4
|
+
|
|
5
|
+
rating_types = {
|
|
6
|
+
"store_corr": {"db_type": "db_corr", "db_disc": "USGS-CORR"},
|
|
7
|
+
"store_base": {"db_type": "db_base", "db_disc": "USGS-BASE"},
|
|
8
|
+
"store_exsa": {"db_type": "db_exsa", "db_disc": "USGS-EXSA"},
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def rating_ini_file_import(api_root, api_key, ini_filename):
|
|
13
|
+
|
|
14
|
+
api_key = "apikey " + api_key
|
|
15
|
+
cwms.api.init_session(api_root=api_root, api_key=api_key)
|
|
16
|
+
|
|
17
|
+
logging.info(f"CDA connection: {api_root}")
|
|
18
|
+
logging.info(f"Opening ini file: {ini_filename}")
|
|
19
|
+
ini_file = open(ini_filename, "r")
|
|
20
|
+
lines = ini_file.readlines()
|
|
21
|
+
ini_file.close()
|
|
22
|
+
|
|
23
|
+
params = {}
|
|
24
|
+
keywords = ["cwms_office", "db_base", "db_exsa", "db_corr", "localid"]
|
|
25
|
+
rating_errors = []
|
|
26
|
+
for i in range(len(lines)):
|
|
27
|
+
line = lines[i][:-1].strip()
|
|
28
|
+
try:
|
|
29
|
+
line = line[: line.index("#")].strip() # strip comments
|
|
30
|
+
except:
|
|
31
|
+
pass
|
|
32
|
+
if not line:
|
|
33
|
+
continue
|
|
34
|
+
if "=" in line:
|
|
35
|
+
fields = line.split("=")
|
|
36
|
+
if fields[0] in keywords:
|
|
37
|
+
if fields[0] == "cwms_office":
|
|
38
|
+
fields[1] = fields[1].upper()
|
|
39
|
+
params[fields[0]] = fields[1]
|
|
40
|
+
else:
|
|
41
|
+
fields = parse_ini_line(line)
|
|
42
|
+
if fields[0] in rating_types.keys():
|
|
43
|
+
rating_db_type = rating_types[fields[0]]["db_type"]
|
|
44
|
+
if f"$(${rating_db_type})" in fields:
|
|
45
|
+
rating_spec = params[rating_db_type].replace(
|
|
46
|
+
"\$localid", params["localid"]
|
|
47
|
+
)
|
|
48
|
+
logging.info(f"Updating rating specification: {rating_spec}")
|
|
49
|
+
try:
|
|
50
|
+
update_rating_spec(
|
|
51
|
+
rating_spec,
|
|
52
|
+
params["cwms_office"],
|
|
53
|
+
rating_types[fields[0]]["db_disc"],
|
|
54
|
+
)
|
|
55
|
+
logging.info("SUCCESS: rating specification changes stored")
|
|
56
|
+
except:
|
|
57
|
+
logging.error(
|
|
58
|
+
"ERROR: rating specificataion could not be update"
|
|
59
|
+
)
|
|
60
|
+
rating_errors.append(
|
|
61
|
+
[rating_spec, rating_types[fields[0]]["db_disc"]]
|
|
62
|
+
)
|
|
63
|
+
logging.info(
|
|
64
|
+
f"ERRORS: The following rating specifications could not be updated {rating_errors}"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def parse_ini_line(line):
|
|
69
|
+
"""
|
|
70
|
+
Parses a line in the ini_file into fields
|
|
71
|
+
"""
|
|
72
|
+
if line.find("'") > 0 or line.find('"') > 0:
|
|
73
|
+
# ---------------------------#
|
|
74
|
+
# fields with spaces quoted #
|
|
75
|
+
# ---------------------------#
|
|
76
|
+
c1 = [c for c in line]
|
|
77
|
+
escape = False
|
|
78
|
+
quote = None
|
|
79
|
+
c2 = []
|
|
80
|
+
for c in c1:
|
|
81
|
+
if c == "\\":
|
|
82
|
+
escape = not escape
|
|
83
|
+
if not escape:
|
|
84
|
+
c2.append(c)
|
|
85
|
+
continue
|
|
86
|
+
if c in ('"', "'"):
|
|
87
|
+
if not quote:
|
|
88
|
+
quote = c
|
|
89
|
+
continue
|
|
90
|
+
if c == quote:
|
|
91
|
+
quote = None
|
|
92
|
+
continue
|
|
93
|
+
if c.isspace() and quote:
|
|
94
|
+
c = chr(0)
|
|
95
|
+
c2.append(c)
|
|
96
|
+
fields = "".join(c2).split()
|
|
97
|
+
for i in range(len(fields)):
|
|
98
|
+
fields[i] = fields[i].replace(chr(0), " ")
|
|
99
|
+
elif line.find("\t") > 0:
|
|
100
|
+
# ------------------------------------------------------------#
|
|
101
|
+
# all fields without spaces separated by tabs (version < 5.0 #
|
|
102
|
+
# ------------------------------------------------------------#
|
|
103
|
+
fields = line.split("\t")
|
|
104
|
+
else:
|
|
105
|
+
# -----------------------#
|
|
106
|
+
# no fields with spaces #
|
|
107
|
+
# -----------------------#
|
|
108
|
+
fields = line.split()
|
|
109
|
+
return fields
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def update_rating_spec(rating_id, office_id, db_disc):
|
|
113
|
+
rating_spec = cwms.get_rating_spec(rating_id=rating_id, office_id=office_id)
|
|
114
|
+
data = rating_spec.df
|
|
115
|
+
data = data.drop("effective-dates", axis=1)
|
|
116
|
+
logging.info(f"Setting source-agency to USGS")
|
|
117
|
+
data["source-agency"] = "USGS"
|
|
118
|
+
logging.info(f"Setting Active, Auto-update, Auto-Activate to True")
|
|
119
|
+
data["active"] = True
|
|
120
|
+
data["auto-update"] = True
|
|
121
|
+
data["auto-activate"] = True
|
|
122
|
+
if "description" in data.columns:
|
|
123
|
+
if db_disc not in data.loc[0, "description"]:
|
|
124
|
+
data["description"] = data["description"] + " " + db_disc
|
|
125
|
+
else:
|
|
126
|
+
data["description"] = db_disc
|
|
127
|
+
disc = data.loc[0, "description"]
|
|
128
|
+
logging.info(f"Saving specification discription as: {disc}")
|
|
129
|
+
data_xml = cwms.rating_spec_df_to_xml(data)
|
|
130
|
+
cwms.store_rating_spec(data=data_xml, fail_if_exists=False)
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def to_uppercase(ctx, param, value):
|
|
5
|
+
if value is None:
|
|
6
|
+
return None
|
|
7
|
+
return value.upper()
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
office_option = click.option(
|
|
11
|
+
"-o",
|
|
12
|
+
"--office",
|
|
13
|
+
required=True,
|
|
14
|
+
envvar="OFFICE",
|
|
15
|
+
type=str,
|
|
16
|
+
callback=to_uppercase,
|
|
17
|
+
help="Office to grab data for",
|
|
18
|
+
)
|
|
19
|
+
api_root_option = click.option(
|
|
20
|
+
"-a",
|
|
21
|
+
"--api_root",
|
|
22
|
+
required=True,
|
|
23
|
+
envvar="CDA_API_ROOT",
|
|
24
|
+
type=str,
|
|
25
|
+
help="Api Root for CDA. Can be user defined or placed in a env variable CDA_API_ROOT",
|
|
26
|
+
)
|
|
27
|
+
api_coop_root_option = click.option(
|
|
28
|
+
"--coop",
|
|
29
|
+
is_flag=True,
|
|
30
|
+
envvar="CDA_API_COOP_ROOT",
|
|
31
|
+
type=str,
|
|
32
|
+
help="Use CDA_API_COOP_ROOT from env",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
api_key_option = click.option(
|
|
36
|
+
"-k",
|
|
37
|
+
"--api_key",
|
|
38
|
+
default=None,
|
|
39
|
+
type=str,
|
|
40
|
+
envvar="CDA_API_KEY",
|
|
41
|
+
help="api key for CDA. Can be user defined or place in env variable CDA_API_KEY. one of api_key or api_key_loc are required",
|
|
42
|
+
)
|
|
43
|
+
api_key_loc_option = click.option(
|
|
44
|
+
"-kl",
|
|
45
|
+
"--api_key_loc",
|
|
46
|
+
default=None,
|
|
47
|
+
type=str,
|
|
48
|
+
help="file storing Api Key. One of api_key or api_key_loc are required",
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def get_api_key(api_key: str, api_key_loc: str) -> str:
|
|
53
|
+
if api_key is not None:
|
|
54
|
+
return api_key
|
|
55
|
+
elif api_key_loc is not None:
|
|
56
|
+
with open(api_key_loc, "r") as f:
|
|
57
|
+
return f.readline().strip()
|
|
58
|
+
else:
|
|
59
|
+
raise Exception(
|
|
60
|
+
"must add a value to either --api_key(-k) or --api_key_loc(-kl)"
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def common_api_options(f):
|
|
65
|
+
f = office_option(f)
|
|
66
|
+
f = api_root_option(f)
|
|
67
|
+
f = api_key_option(f)
|
|
68
|
+
return f
|
cwmscli/utils/deps.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
import importlib.metadata
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def requires(*requirements):
|
|
8
|
+
"""
|
|
9
|
+
Decorator that ensures required Python modules are installed and meet optional minimum version constraints.
|
|
10
|
+
|
|
11
|
+
Parameters:
|
|
12
|
+
*requirements: One or more dictionaries describing a module requirement.
|
|
13
|
+
Each dictionary may contain the following keys:
|
|
14
|
+
|
|
15
|
+
- module (str): The importable module name (e.g., "requests").
|
|
16
|
+
|
|
17
|
+
- package (str, optional): The name of the package to install via pip.
|
|
18
|
+
Use this if the pip install name differs from the import name
|
|
19
|
+
(e.g., module="cwms", package="cwms-python").
|
|
20
|
+
|
|
21
|
+
- version (str, optional): A minimum required version string (e.g., "2.30.0").
|
|
22
|
+
|
|
23
|
+
- desc (str, optional): A short description of what the module is or why it's needed.
|
|
24
|
+
Included in the error message to help users understand the dependency.
|
|
25
|
+
|
|
26
|
+
- link (str, optional): A URL pointing to documentation or the package's homepage.
|
|
27
|
+
|
|
28
|
+
Example:
|
|
29
|
+
@requires(
|
|
30
|
+
{
|
|
31
|
+
"module": "cwms",
|
|
32
|
+
"package": "cwms-python",
|
|
33
|
+
"version": "0.8.0",
|
|
34
|
+
"desc": "CWMS REST API Python client",
|
|
35
|
+
"link": "https://github.com/hydrologicengineeringcenter/cwms-python"
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"module": "requests",
|
|
39
|
+
"version": "2.30.0",
|
|
40
|
+
"desc": "Required for HTTP API access"
|
|
41
|
+
}
|
|
42
|
+
)
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def decorator(func):
|
|
46
|
+
def wrapper(*args, **kwargs):
|
|
47
|
+
missing = []
|
|
48
|
+
version_issues = []
|
|
49
|
+
|
|
50
|
+
for req in requirements:
|
|
51
|
+
mod = req["module"]
|
|
52
|
+
pkg = req.get("package", mod)
|
|
53
|
+
min_version = req.get("version")
|
|
54
|
+
desc = req.get("desc")
|
|
55
|
+
link = req.get("link")
|
|
56
|
+
# Check if the provided requirement is already imported
|
|
57
|
+
try:
|
|
58
|
+
importlib.import_module(mod)
|
|
59
|
+
except ImportError:
|
|
60
|
+
msg = f"- `{mod}` (install: `{pkg}`)"
|
|
61
|
+
if desc:
|
|
62
|
+
msg += f" — {desc}"
|
|
63
|
+
if link:
|
|
64
|
+
msg += f" [docs]({link})"
|
|
65
|
+
missing.append((msg, pkg))
|
|
66
|
+
continue
|
|
67
|
+
# Confirm the minimum version is met
|
|
68
|
+
if min_version:
|
|
69
|
+
try:
|
|
70
|
+
actual_version = importlib.metadata.version(pkg)
|
|
71
|
+
if actual_version < min_version:
|
|
72
|
+
version_issues.append(
|
|
73
|
+
f"- `{pkg}` version `{actual_version}` found, "
|
|
74
|
+
f"but `{min_version}` or higher is required"
|
|
75
|
+
)
|
|
76
|
+
except importlib.metadata.PackageNotFoundError:
|
|
77
|
+
version_issues.append(
|
|
78
|
+
f"- `{pkg}` is installed but version could not be verified"
|
|
79
|
+
)
|
|
80
|
+
# Build out the error response
|
|
81
|
+
if missing or version_issues:
|
|
82
|
+
error_lines = []
|
|
83
|
+
if missing:
|
|
84
|
+
error_lines.append("Missing module(s):")
|
|
85
|
+
for msg, _ in missing:
|
|
86
|
+
error_lines.append(msg)
|
|
87
|
+
install_cmd = "pip install " + " ".join(pkg for _, pkg in missing)
|
|
88
|
+
error_lines.append(
|
|
89
|
+
f"\nInstall missing packages:\n {install_cmd}"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
if version_issues:
|
|
93
|
+
error_lines.append("\nVersion issues:")
|
|
94
|
+
error_lines.extend(version_issues)
|
|
95
|
+
|
|
96
|
+
raise click.ClickException("\n".join(error_lines))
|
|
97
|
+
|
|
98
|
+
return func(*args, **kwargs)
|
|
99
|
+
|
|
100
|
+
return wrapper
|
|
101
|
+
|
|
102
|
+
return decorator
|