cwms-cli 0.1.1__py3-none-any.whl

cwmscli/commands/commands_cwms.py

@@ -0,0 +1,227 @@
+import click
+
+from cwmscli import requirements as reqs
+from cwmscli.callbacks import csv_to_list
+from cwmscli.commands import csv2cwms
+from cwmscli.utils import api_key_loc_option, common_api_options
+from cwmscli.utils.deps import requires
+
+
+@click.command(
+    "shefcritimport",
+    help="Import SHEF crit file into timeseries group for SHEF file processing",
+)
+@click.option(
+    "-f",
+    "--filename",
+    required=True,
+    type=str,
+    help="filename of SHEF crit file to be processed",
+)
+@common_api_options
+@api_key_loc_option
+@requires(reqs.cwms)
+def shefcritimport(filename, office, api_root, api_key, api_key_loc):
+    from cwmscli.commands.shef_critfile_import import import_shef_critfile
+
+    api_key = get_api_key(api_key, api_key_loc)
+    import_shef_critfile(
+        file_path=filename,
+        office_id=office,
+        api_root=api_root,
+        api_key=api_key,
+    )
+
+
+@click.command("csv2cwms", help="Store CSV TimeSeries data to CWMS using a config file")
+@common_api_options
+@click.option(
+    "--input-keys",
+    "input_keys",
+    default="all",
+    show_default=True,
+    help='Input keys. Defaults to all keys/files with --input-keys=all. These are the keys under "input_files" in a given config file. This option lets you run a single file from a config that contains multiple files. Example: --input-keys=file1',
+)
+@click.option(
+    "-lb",
+    "--lookback",
+    type=int,
+    default=24 * 5,
+    show_default=True,
+    help="Lookback period in HOURS",
+)
+@click.option("-v", "--verbose", is_flag=True, help="Verbose logging")
+@click.option(
+    "-c",
+    "--config",
+    "config_path",
+    required=True,
+    type=click.Path(exists=True),
+    help="Path to JSON config file",
+)
+@click.option(
+    "-df",
+    "--data-file",
+    "data_file",
+    type=str,
+    help="Override CSV file (else use config)",
+)
+@click.option("--log", show_default=True, help="Path to the log file.")
+@click.option("--dry-run", is_flag=True, help="Log only (no HTTP calls)")
+@click.option("--begin", type=str, help="YYYY-MM-DDTHH:MM (local to --tz)")
+@click.option("-tz", "--timezone", "tz", default="GMT", show_default=True)
+@click.option(
+    "--ignore-ssl-errors", is_flag=True, help="Ignore TLS errors (testing only)"
+)
+@click.version_option(version=csv2cwms.__version__)
+@requires(reqs.cwms)
+def csv2cwms_cmd(**kwargs):
+    from cwmscli.commands.csv2cwms.__main__ import main as csv2_main
+
+    # Handle the version for this specific command
+    if kwargs.pop("version", False):
+        from cwmscli.commands.csv2cwms import __version__
+
+        click.echo(f"csv2cwms v{__version__}")
+        return
+    csv2_main(**kwargs)
+
+
+# region Blob
+# ================================================================================
+#  BLOB
+# ================================================================================
+@click.group(
+    "blob",
+    help="Manage CWMS Blobs (upload, download, delete, update, list)",
+    epilog="""
+  * Store a PDF/image as a CWMS blob with optional description
+  * Download a blob by id to your local filesystem
+  * Update a blob's name/description
+  * Bulk list blobs for an office
+""",
+)
+@requires(reqs.cwms)
+def blob_group():
+    pass
+
+
+# ================================================================================
+#       Upload
+# ================================================================================
+@blob_group.command("upload", help="Upload a file as a blob")
+@click.option(
+    "--input-file",
+    required=True,
+    type=click.Path(exists=True, dir_okay=False, readable=True, path_type=str),
+    help="Path to the file to upload.",
+)
+@click.option("--blob-id", required=True, type=str, help="Blob ID to create.")
+@click.option("--description", default=None, help="Optional description JSON or text.")
+@click.option(
+    "--media-type",
+    default=None,
+    help="Override media type (guessed from file if omitted).",
+)
+@click.option(
+    "--overwrite/--no-overwrite",
+    default=False,
+    show_default=True,
+    help="If true, replace existing blob.",
+)
+@click.option("--dry-run", is_flag=True, help="Show request; do not send.")
+@common_api_options
+def blob_upload(**kwargs):
+    from cwmscli.commands.blob import upload_cmd
+
+    upload_cmd(**kwargs)
+
+
+# ================================================================================
+#       Download
+# ================================================================================
+@blob_group.command("download", help="Download a blob by ID")
+# TODO: test XML
+@click.option("--blob-id", required=True, type=str, help="Blob ID to download.")
+@click.option(
+    "--dest",
+    default=None,
+    help="Destination file path. Defaults to blob-id.",
+)
+@common_api_options
+def blob_download(**kwargs):
+    from cwmscli.commands.blob import download_cmd
+
+    download_cmd(**kwargs)
+
+
+# ================================================================================
+#       Delete
+# ================================================================================
+@blob_group.command("delete", help="[Not implemented] Delete a blob by ID")
+@click.option("--blob-id", required=True, type=str, help="Blob ID to delete.")
+@common_api_options
+def delete_cmd(**kwargs):
+    from cwmscli.commands.blob import delete_cmd
+
+    delete_cmd(**kwargs)
+
+
+# ================================================================================
+#       Update
+# ================================================================================
+@blob_group.command("update", help="[Not implemented] Update/patch a blob by ID")
+@click.option("--blob-id", required=True, type=str, help="Blob ID to update.")
+@click.option(
+    "--input-file",
+    required=False,
+    type=click.Path(exists=True, dir_okay=False, readable=True, path_type=str),
+    help="Optional file content to upload with update.",
+)
+@common_api_options
+def update_cmd(**kwargs):
+    from cwmscli.commands.blob import update_cmd
+
+    update_cmd(**kwargs)
+
+
+# ================================================================================
+#       List
+# ================================================================================
+@blob_group.command("list", help="List blobs with optional filters and sorting")
+# TODO: Add link to regex docs when new CWMS-DATA site is deployed to PROD
+@click.option(
+    "--blob-id-like", help="LIKE filter for blob ID (e.g., ``*PNG``)."
+)  # Escape the wildcard/asterisk for RTD generation with double backticks
+@click.option(
+    "--columns",
+    multiple=True,
+    callback=csv_to_list,
+    help="Columns to show (repeat or comma-separate).",
+)
+@click.option(
+    "--sort-by",
+    multiple=True,
+    callback=csv_to_list,
+    help="Columns to sort by (repeat or comma-separate).",
+)
+@click.option(
+    "--desc/--asc",
+    default=False,
+    show_default=True,
+    help="Sort descending instead of ascending.",
+)
+@click.option("--limit", type=int, default=None, help="Max rows to show.")
+@click.option(
+    "--to-csv",
+    type=click.Path(dir_okay=False, writable=True, path_type=str),
+    help="If set, write results to this CSV file.",
+)
+@common_api_options
+def list_cmd(**kwargs):
+    from cwmscli.commands.blob import list_cmd
+
+    list_cmd(**kwargs)
+
+
+# endregion

cwmscli/commands/csv2cwms/.gitignore

@@ -0,0 +1,3 @@
+# CSV2CWMS Ignore Files
+!timeseries_map.csv
+*.log*

cwmscli/commands/csv2cwms/README.md

@@ -0,0 +1,51 @@
+# CSV2CWMS
+
+Writes CSV timeseries data to CDA using a configuration file.
+
+To View the Help: `cwms-cli csv2cwms --help`
+
+## USAGE (--help)
+
+Usage: cwms-cli csv2cwms [OPTIONS]
+
+Store CSV TimeSeries data to CWMS using a config file
+
+Options:
+-o, --office TEXT Office to grab data for [required]
+-a, --api_root TEXT Api Root for CDA. Can be user defined or placed
+in a env variable CDA_API_ROOT [required]
+-k, --api_key TEXT api key for CDA. Can be user defined or place in
+env variable CDA_API_KEY. one of api_key or
+api_key_loc are required
+-l, --location TEXT Location ID. Use "-p=all" for all locations.
+[default: all]
+-lb, --lookback INTEGER Lookback period in HOURS [default: 120]
+-v, --verbose Verbose logging
+-c, --config PATH Path to JSON config file [required]
+[default: all]
+-lb, --lookback INTEGER Lookback period in HOURS [default: 120]
+-v, --verbose Verbose logging
+[default: all]
+[default: all]
+-lb, --lookback INTEGER Lookback period in HOURS [default: 120]
+-v, --verbose Verbose logging
+-c, --config PATH Path to JSON config file [required]
+-df, --data-file TEXT Override CSV file (else use config)
+--log TEXT Path to the log file.
+-dp, --data-path DIRECTORY Directory where csv files are stored [default:
+.]
+--dry-run Log only (no HTTP calls)
+--begin TEXT YYYY-MM-DDTHH:MM (local to --tz)
+-tz, --timezone TEXT [default: GMT]
+--ignore-ssl-errors Ignore TLS errors (testing only)
+--version Show the version and exit.
+--help Show this message and exit.
+
+## Features
+
+- Allow for specifying one or more date formats that might be seen per input csv file
+- Allow mathematical operations across multiple columns and storing into one timeseries
+- Store one column of data with a user-specified precision and units to a timeseries identifier
+- Dry runs to test what data might look like prior to database storage
+- Verbose logging via the -v flag
+- Colored terminal output for user readability

cwmscli/commands/csv2cwms/__init__.py

@@ -0,0 +1,5 @@
+__version__ = "0.0.1"
+__name__ = "csv2cwms"
+__author__ = "Charles Graham - USACE"
+__license__ = "MIT"
+__description__ = "Parses Comma Separated Value (CSV) files and uses a config to store TimeSeries to CDA"

cwmscli/commands/csv2cwms/__main__.py

@@ -0,0 +1,265 @@
+# Script Entry File
+import os
+import sys
+import time
+import traceback
+from datetime import datetime, timedelta
+
+import cwms
+
+# Add the current directory to the path
+# This is necessary for the script to be run as a standalone script
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
+
+
+# Handle imports for local and package use
+# This is necessary for the script to be run as a package or as a standalone script
+# The script can be run as a standalone script by running `python -m scada_ts` from the parent directory
+# or as a package by running `python scada_ts` from the parent directory
+try:
+    # Relative imports for modules
+    from . import __author__, __license__, __version__
+    from .utils import (
+        colorize,
+        colorize_count,
+        determine_interval,
+        eval_expression,
+        load_csv,
+        logger,
+        parse_date,
+        read_config,
+        safe_zoneinfo,
+        setup_logger,
+    )
+except ImportError:
+    from __init__ import __author__, __license__, __version__
+    from utils import (
+        colorize,
+        colorize_count,
+        determine_interval,
+        eval_expression,
+        load_csv,
+        logger,
+        parse_date,
+        read_config,
+        safe_zoneinfo,
+        setup_logger,
+    )
+
+# Load environment variables
+API_KEY = os.getenv("CDA_API_KEY")
+OFFICE = os.getenv("CDA_OFFICE", "SWT")
+HOST = os.getenv("CDA_HOST")
+
+if [API_KEY, OFFICE, HOST].count(None) > 0:
+    raise ValueError(
+        "Environment variables CDA_API_KEY, CDA_OFFICE, and CDA_HOST must be set."
+    )
+
+
+def parse_file(file_path, begin_time, date_format, timezone="GMT"):
+    csv_data = load_csv(file_path)
+    header = csv_data[0]
+    data = csv_data[1:]
+    ts_data = {}
+    logger.debug(f"Begin time: {begin_time}")
+    for row in data:
+        # Skip empty rows or rows without a timestamp
+        if not row:
+            continue
+        row_datetime = parse_date(row[0], tz_str=timezone, date_format=date_format)
+        # Guarantee only one entry per timestamp
+        ts_data[int(row_datetime.timestamp())] = row
+    return {"header": header, "data": ts_data}
+
+
+def load_timeseries(file_data, file_key, config):
+    header = file_data.get("header", [])
+    data = file_data.get("data", {})
+
+    if not header or not data:
+        raise ValueError(
+            "No data found in the CSV file for the range selected. Please ensure you set the timezone of the CSV file with --tz America/Chicago or similar."
+        )
+
+    ts_config = config["input_files"][file_key]["timeseries"]
+    file_ts = []
+
+    # Interval in seconds
+    interval = config.get("interval")
+    if not interval:
+        interval = determine_interval(data, 10)
+        logger.warning(
+            f"Interval not found in configuration. Determined interval: {interval} seconds."
+        )
+    start_epoch = min(data.keys())
+    end_epoch = max(data.keys())
+
+    # Map column names to indexes (case-insensitive)
+    header_map = {col.strip().lower(): i for i, col in enumerate(header)}
+    logger.debug(f"Header map (column name -> index): {header_map}")
+
+    for name, meta in ts_config.items():
+        expr = meta["columns"]
+        units = meta.get("units", "")
+        precision = meta.get("precision", 2)
+        values = []
+        epoch = start_epoch
+        while epoch <= end_epoch:
+            row = data.get(epoch)
+            if row:
+                value = eval_expression(expr, row, header_map)
+                value = round(value, precision) if value is not None else None
+                quality = 3 if value is not None else 5
+            else:
+                value = None
+                quality = 5
+            logger.debug(
+                f"[{name}] {datetime.fromtimestamp(epoch)} -> {value} (quality: {quality})"
+            )
+            values.append([epoch * 1000, value, quality])
+            # Convert seconds to minutes
+            epoch += interval
+
+        ts_obj = {"name": name, "units": units, "values": values}
+        valid = sum(1 for _, v, _ in values if v is not None)
+        total = len(values)
+        logger.info(
+            f"Built timeseries {colorize(name, 'blue')} with {colorize_count(valid, total)} valid points."
+        )
+        logger.debug(
+            f"Timeseries {name} data range: {colorize(datetime.fromtimestamp(start_epoch), 'blue')} to {colorize(datetime.fromtimestamp(end_epoch), 'blue')}"
+        )
+        file_ts.append(ts_obj)
+
+    return file_ts
+
+
+def config_check(config):
+    """Checks a configuration file for required keys"""
+    if not config.get("interval"):
+        logger.warning(
+            "Configuration file does not contain an 'interval' key (and value in seconds), this is recommended per CSV file to avoid ambiguity."
+        )
+    if config.get("projects"):
+        logger.warning(
+            "Configuration file contains a 'projects' key, this has been renamed to 'input_files' for clarity. Continuing for backwards compatibility."
+        )
+        config["input_files"] = config.pop("projects")
+    if not config.get("input_files"):
+        raise ValueError("Configuration file must contain an 'input_files' key.")
+    for file_key, file_data in config.get("input_files").items():
+        # Only check the specified keys or if all keys are specified
+        if file_key != "all" and file_key != file_key.lower():
+            continue
+        if not file_data.get("timeseries"):
+            raise ValueError(
+                f"Configuration file must contain a 'timeseries' key for file '{file_key}'."
+            )
+        for ts_name, ts_data in file_data.get("timeseries").items():
+            if not ts_data.get("columns"):
+                raise ValueError(
+                    f"Configuration file must contain a 'columns' key for timeseries '{ts_name}' in file '{file_key}'."
+                )
+
+
+def main(*args, **kwargs):
+    """
+    Main function to execute the scada_ts script.
+    This function serves as the entry point for the script.
+    """
+    start_time = time.time()
+    tz = safe_zoneinfo(kwargs.get("tz"))
+    if kwargs.get("begin"):
+        try:
+            begin_time = datetime.strptime(
+                kwargs.get("begin"), "%Y-%m-%dT%H:%M"
+            ).replace(tzinfo=tz)
+        except ValueError:
+            raise ValueError("--begin must be in format YYYY-MM-DDTHH:MM")
+    else:
+        begin_time = datetime.now(tz)
+
+    cwms.api.init_session(
+        api_root=kwargs.get("api_root"), api_key=kwargs.get("api_key")
+    )
+    # Setup the logger if a path is provided
+    setup_logger(kwargs.get("log"), verbose=kwargs.get("verbose"))
+    logger.info(f"Begin time: {begin_time}")
+    logger.debug(f"Timezone: {tz}")
+    # Override environment variables if provided in CLI
+    if kwargs.get("coop"):
+        HOST = os.getenv("CDA_COOP_HOST")
+        if not HOST:
+            raise ValueError(
+                "Environment variable CDA_COOP_HOST must be set to use --coop flag."
+            )
+    config_path = kwargs.get("config_path")
+    config = read_config(config_path)
+    config_check(config)
+    INPUT_FILES = config.get("input_files", {})
+    # Override file names if one is specified in CLI
+    if kwargs.get("input_keys"):
+        if kwargs.get("input_keys") == "all":
+            INPUT_FILES = config.get("input_files", {}).keys()
+        else:
+            INPUT_FILES = kwargs.get("input_keys").split(",")
+    logger.info(f"Started for {','.join(INPUT_FILES)} input files.")
+    # Input checks
+    # if kwargs.get("file_name") != "all" and kwargs.get("file_name") not in INPUT_FILES:
+    #     raise ValueError(
+    #         f"Invalid file name '{kwargs.get("file_name")}'. Valid options are: {', '.join(INPUT_FILES)}"
+    #     )
+
+    # Loop the file names and post the data
+    for file_name in INPUT_FILES:
+        # Grab the csv file path from the config
+        CONFIG_ITEM = config.get("input_files", {}).get(file_name, {})
+        DATA_FILE = CONFIG_ITEM.get("data_path", "")
+        if not DATA_FILE:
+            logger.warning(
+                # TODO: List URL to example in doc site once available
+                f"No data file specified for input-keys '{file_name}' in {config_path}. {colorize(f'Skipping {file_name}', 'red')}. Please provide a valid CSV file path by ensuring the 'data_path' key is set in the config."
+            )
+            continue
+        csv_data = parse_file(
+            DATA_FILE,
+            begin_time,
+            CONFIG_ITEM.get("date_format"),
+            kwargs.get("tz"),
+        )
+        try:
+            ts_min_data = load_timeseries(csv_data, file_name, config)
+        except ValueError as e:
+            logger.error(f"Error loading timeseries for {file_name}: {e}")
+            continue
+
+        if kwargs.get("dry_run"):
+            logger.info("DRY RUN enabled. No data will be posted")
+        for ts_object in ts_min_data:
+            try:
+                ts_object.update({"office-id": kwargs.get("office")})
+                logger.info(
+                    "Store Rule: " + CONFIG_ITEM.get("store_rule", "")
+                    if CONFIG_ITEM.get("store_rule", "")
+                    else f"No Store Rule specified, will default to REPLACE_ALL in {config_path}."
+                )
+                if kwargs.get("dry_run"):
+                    logger.info(f"DRY RUN: {ts_object}")
+                else:
+                    cwms.store_timeseries(
+                        data=ts_object,
+                        store_rule=CONFIG_ITEM.get("store_rule", "REPLACE_ALL"),
+                    )
+                    logger.info(f"Stored {ts_object['name']} values")
+            except Exception as e:
+                logger.error(
+                    f"Error posting data for {file_name}: {e}\n{traceback.format_exc()}"
+                )
+
+    logger.debug(f"\tExecution time: {round(time.time() - start_time, 3)} seconds.")
+    logger.debug(f"\tMemory usage: {round(os.sys.getsizeof(locals()) / 1024, 2)} KB")
+
+
+if __name__ == "__main__":
+    main()

cwmscli/commands/csv2cwms/examples/complete_config.json

@@ -0,0 +1,19 @@
+{
+    "interval": 3600,
+    "input_files": {
+        "BROK": {
+            "data_path": "cwmscli/commands/csv2cwms/tests/data/sample_brok.csv",
+            "date_format": [
+                "%m/%d/%Y %H:%M:%S",
+                "%m/%d/%Y %H:%M"
+            ],
+            "timeseries": {
+                "BROK.Elev.Inst.15Minutes.0.Rev-SCADA-cda": {
+                    "columns": "Headwater",
+                    "units": "ft",
+                    "precision": 2
+                }
+            }
+        }
+    }
+}