cwms-cli 0.1.1__py3-none-any.whl

cwmscli/commands/csv2cwms/utils/fileio.py

@@ -0,0 +1,26 @@
+import csv
+import json
+import os
+
+
+def read_config(config_path):
+    """Read the configuration file"""
+    if not config_path.endswith(".json"):
+        raise ValueError("Configuration file must be a JSON file.")
+    with open(config_path, "r") as f:
+        config = json.load(f)
+    return config
+
+
+def load_csv(file_path):
+    """Load the CSV file and return the data."""
+    if not file_path.endswith(".csv"):
+        raise ValueError("Data file must be a CSV file.")
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(
+            f"File not found: {file_path}. Be sure to set the path correctly in the configuration file. And use the argument --data-path to specify the directory containing the data files."
+        )
+    with open(file_path, "r") as f:
+        reader = csv.reader(f)
+        data = list(reader)
+    return data

cwmscli/commands/csv2cwms/utils/logging.py

@@ -0,0 +1,80 @@
+import logging
+from logging.handlers import RotatingFileHandler
+
+from .terminal import colorize
+
+logger = logging.getLogger("csv2cwms")
+
+
+class ColorFormatter(logging.Formatter):
+    """Logging Formatter that colorizes the level name."""
+
+    LEVEL_COLORS = {
+        logging.DEBUG: "cyan",
+        logging.INFO: "blue",
+        logging.WARNING: "yellow",
+        logging.ERROR: "red",
+        logging.CRITICAL: "magenta",
+    }
+
+    def formatTime(self, record, datefmt=None):
+        """Override to colorize asctime in gray."""
+        timestr = super().formatTime(record, datefmt)
+        return colorize(timestr, "gray")
+
+    def format(self, record):
+        level_color = self.LEVEL_COLORS.get(record.levelno, "reset")
+        record.levelname = colorize(record.levelname, level_color)
+        return super().format(record)
+
+
+def setup_logger(
+    log_path: str = "",
+    max_log_size_mb: int = 5,
+    backup_count: int = 3,
+    show_console: bool = True,
+    verbose: bool = False,
+):
+    """Set up logger with rotating file handler
+
+    Args:
+        log_path (str): Path to the log file
+        max_log_size_mb (int): Maximum log file size in MB before rotating
+        backup_count (int): Number of backup log files to keep
+        show_console (bool): Log to console as well as file
+
+    # Example usage
+    logger.info("This is an info message")
+    logger.error("This is an error message")
+
+    Returns:
+        logger: logging.Logger
+    """
+
+    # Remove the default logger handlers from cwms-cli so we can set up our own
+    root = logging.getLogger()
+    for h in root.handlers[:]:
+        root.removeHandler(h)
+    # Create formatter and attach to handler
+    formatter = ColorFormatter(
+        "[%(asctime)s] [%(levelname)s] %(message)s", "%Y-%m-%d %H:%M:%S"
+    )
+    # Setup the logger if it has not already been configured
+    logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+    if log_path:
+        # Create rotating file handler
+        file_handler = RotatingFileHandler(
+            log_path, maxBytes=max_log_size_mb * 1024 * 1024, backupCount=backup_count
+        )
+        file_handler.setLevel(logging.DEBUG)
+
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+
+    # Setup console handler if needed
+    if not logger.hasHandlers() or show_console:
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+    logger.debug(f"Logger configured with log file: {log_path}")
+    return logger

cwmscli/commands/csv2cwms/utils/terminal.py

@@ -0,0 +1,45 @@
+def colorize_count(valid: int, total: int) -> str:
+    """Colorize the count based on the percentage of valid data.
+    Args:
+        valid (int): Number of valid data.
+        total (int): Total number of data.
+    Returns:
+        str: ANSI Escape Code representing the colorized count status.
+    """
+    percent = valid / total if total else 0
+
+    if valid == 0:
+        color = "red"
+    elif percent >= 0.95:
+        color = "green"
+    else:
+        color = "yellow"
+
+    return f"{colorize(f'{valid}/{total}', color)}"
+
+
+def colorize(text: str, color: str) -> str:
+    """Colorize the text with ANSI Escape Codes provided a color
+    Args:
+        text (str): Text to colorize.
+        color (str): Color to use.
+
+    Returns:
+        str: ANSI Escape Code representing the colorized count status.
+    """
+    COLORS = {
+        "red": "\033[91m",
+        "green": "\033[92m",
+        "yellow": "\033[93m",
+        "blue": "\033[94m",
+        "magenta": "\033[95m",
+        "cyan": "\033[96m",
+        "white": "\033[97m",
+        "gray": "\033[90m",
+        "reset": "\033[0m",
+    }
+
+    if color not in COLORS:
+        return text
+
+    return f"{COLORS.get(color.lower(), COLORS['reset'])}{text}{COLORS['reset']}"

cwmscli/commands/shef_critfile_import.py

@@ -0,0 +1,146 @@
+import logging
+import re
+from typing import Dict, List
+
+import cwms
+import pandas as pd
+
+
+def import_shef_critfile(
+    file_path: str,
+    office_id: str,
+    api_root: str,
+    api_key: str,
+    group_id: str = "SHEF Data Acquisition",
+    category_id: str = "Data Acquisition",
+    group_office_id: str = "CWMS",
+    category_office_id: str = "CWMS",
+    replace_assigned_ts: bool = False,
+) -> None:
+    """
+    Processes a .crit file and saves the information to the SHEF Data Acquisition time series group.
+
+    Parameters
+    ----------
+    file_path : str
+        Path to the .crit file.
+    office_id : str
+        The ID of the office associated with the specified timeseries.
+    group_id : str, optional
+        The specified group associated with the timeseries data. Defaults to "SHEF Data Acquisition".
+    category_id : str, optional
+        The category ID that contains the timeseries group. Defaults to "Data Acquisition".
+    group_office_id : str, optional
+        The specified office group associated with the timeseries data. Defaults to "CWMS".
+    replace_assigned_ts : bool, optional
+        Specifies whether to unassign all existing time series before assigning new time series specified in the content body. Default is False.
+
+    Returns
+    -------
+    None
+    """
+
+    api_key = "apikey " + api_key
+    cwms.api.init_session(api_root=api_root, api_key=api_key)
+    logging.info(f"CDA connection: {api_root}")
+
+    # Parse the file and get the parsed data
+    parsed_data = parse_crit_file(file_path)
+    logging.info("CRIT file has been parsed")
+    # df = pd.DataFrame()
+    logging.info(f"Saving Timeseries IDs to group: {group_id}")
+    for data in parsed_data:
+        # Create DataFrame for the current row
+        try:
+            df = create_df(office_id, data["Timeseries ID"], data["Alias"])
+
+            # Generate JSON dictionary
+            json_dict = cwms.timeseries_group_df_to_json(
+                data=df,
+                group_id=group_id,
+                group_office_id=group_office_id,
+                category_office_id=category_office_id,
+                category_id=category_id,
+            )
+
+            cwms.update_timeseries_groups(
+                group_id=group_id,
+                office_id=office_id,
+                replace_assigned_ts=replace_assigned_ts,
+                data=json_dict,
+            )
+            logging.info(
+                f'SUCCESS Stored timeseries ID {data["Timeseries ID"]} to {group_id}'
+            )
+        except Exception as error:
+            logging.error(
+                f'FAIL Data could not be stored to CWMS database for -->  {data["Timeseries ID"]},{data["Alias"]} error = {error}'
+            )
+
+
+def parse_crit_file(file_path: str) -> List[Dict[str, str]]:
+    """
+    Parses a .crit file into a dictionary containing timeseries ID and Alias.
+
+    Parameters
+    ----------
+        file_path : str
+                Path to the .crit file.
+
+    Returns
+    -------
+    List[Dict[str, str]]
+        A list of dictionaries with "Alias" and "Timeseries ID" as keys.
+    """
+    parsed_data = []
+    with open(file_path, "r") as file:
+        for line in file:
+            # Ignore comment lines and empty lines
+            if line.startswith("#") or not line.strip():
+                continue
+
+            # Extract alias, timeseries ID, and TZ
+            match = re.match(r"([^=]+)=([^;]+);(.+)", line.strip())
+
+            if match:
+                alias = match.group(1).strip()
+                timeseries_id = match.group(2).strip()
+                alias2 = match.group(3).strip()
+
+                parsed_data.append(
+                    {
+                        "Alias": alias + ":" + alias2,
+                        "Timeseries ID": timeseries_id,
+                    }
+                )
+
+    return parsed_data
+
+
+def create_df(office_id: str, ts_id: str, alias: str) -> pd.DataFrame:
+    """
+    Appends a row to the DataFrame.
+
+    Parameters
+    ----------
+        df : pandas.DataFrame
+            The DataFrame to append to.
+        office_id : str
+            The ID of the office associated with the specified timeseries.
+        tsId : str
+            The timeseries ID from the file.
+        alias : str
+            The alias from the file.
+    Returns
+    -------
+    pandas.DataFrame
+        The updated DataFrame.
+    """
+    data = {
+        "office-id": [office_id],
+        "timeseries-id": [ts_id],
+        "alias-id": [alias],
+    }
+    # df = pd.concat([df, pd.DataFrame(data)])
+    df = pd.DataFrame(data)
+    return df

cwmscli/requirements.py

@@ -0,0 +1,25 @@
+# A file containing shared minimum version requirements for dependencies
+# used by the `@requires` decorator in `cwmscli.utils.deps`.
+
+
+cwms = {
+    "module": "cwms",
+    "package": "cwms-python",
+    "version": "0.8.0",
+    "desc": "CWMS REST API Python client",
+    "link": "https://github.com/HydrologicEngineeringCenter/cwms-python",
+}
+
+requests = {
+    "module": "requests",
+    "version": "2.30.0",
+    "desc": "Required for HTTP API access",
+}
+
+dataretrieval = {
+    "module": "dataretrieval",
+    "package": "dataretrieval",
+    "version": "1.0.10",
+    "desc": "Loading hydrologic data from USGS",
+    "link": "https://github.com/DOI-USGS/dataretrieval-python",
+}

cwmscli/usgs/__init__.py

@@ -0,0 +1,161 @@
+import click
+
+from cwmscli import requirements as reqs
+from cwmscli.utils.deps import requires
+
+
+@click.group()
+def usgs_group():
+    """USGS utilities"""
+    pass
+
+
+import click
+
+from cwmscli import requirements as reqs
+from cwmscli.utils import (
+    api_key_loc_option,
+    api_key_option,
+    api_root_option,
+    get_api_key,
+    office_option,
+)
+from cwmscli.utils.deps import requires
+
+days_back_option = click.option(
+    "-d",
+    "--days_back",
+    default="1",
+    type=float,
+    help="Days back from current time to get data.  Can be decimal and integer values",
+)
+
+
+@usgs_group.command(
+    "timeseries", help="Get USGS timeseries values and store into CWMS database"
+)
+@office_option
+@days_back_option
+@api_root_option
+@api_key_option
+@api_key_loc_option
+@click.option(
+    "-b",
+    "--backfill",
+    default=None,
+    type=str,
+    help='Backfill timeseries ids, use list of timeseries ids (e.g. "ts_id1, ts_id2") to attempt to backfill a subset of timeseries with USGS data',
+)
+@requires(reqs.cwms, reqs.requests)
+def getusgs_timeseries(office, days_back, api_root, api_key, api_key_loc, backfill):
+    from cwmscli.usgs.getusgs_cda import getusgs_cda
+
+    if backfill is not None:
+        backfill_list = backfill.replace(" ", "").split(",")
+    else:
+        backfill_list = None
+
+    api_key = get_api_key(api_key, api_key_loc)
+    getusgs_cda(
+        api_root=api_root,
+        office_id=office,
+        days_back=days_back,
+        api_key=api_key,
+        backfill_tsids=backfill_list,
+    )
+
+
+@usgs_group.command("ratings", help="Get USGS ratings and store into CWMS database")
+@office_option
+@days_back_option
+@api_root_option
+@api_key_option
+@api_key_loc_option
+@requires(reqs.cwms, reqs.requests, reqs.dataretrieval)
+def getusgs_ratings(office, days_back, api_root, api_key, api_key_loc):
+    from cwmscli.usgs.getUSGS_ratings_cda import getusgs_rating_cda
+
+    api_key = get_api_key(api_key, api_key_loc)
+    getusgs_rating_cda(
+        api_root=api_root,
+        office_id=office,
+        days_back=days_back,
+        api_key=api_key,
+    )
+
+
+@usgs_group.command(
+    "ratings-ini-file-import",
+    help="Store rating ini file information into database to be used with getusgs_ratings",
+)
+@click.option(
+    "-f",
+    "--filename",
+    required=True,
+    type=str,
+    help="filename of ratings ini file to be processed",
+)
+@api_root_option
+@api_key_option
+@api_key_loc_option
+@requires(reqs.cwms, reqs.requests)
+def ratingsinifileimport(filename, api_root, api_key, api_key_loc):
+    from cwmscli.usgs.rating_ini_file_import import rating_ini_file_import
+
+    api_key = get_api_key(api_key, api_key_loc)
+    rating_ini_file_import(api_root=api_root, api_key=api_key, ini_filename=filename)
+
+
+@usgs_group.command("measurements", help="Store USGS measurements into CWMS database")
+@click.option(
+    "-d",
+    "--days_back_modified",
+    default="2",
+    help="Days back from current time measurements have been modified in USGS database. Can be integer value",
+)
+@click.option(
+    "-c",
+    "--days_back_collected",
+    default="365",
+    help="Days back from current time measurements have been collected. Can be integer value",
+)
+@office_option
+@api_root_option
+@api_key_option
+@api_key_loc_option
+@click.option(
+    "-b",
+    "--backfill",
+    default=None,
+    type=str,
+    help="Backfill POR data, use list of USGS IDs (e.g. 05057200, 05051300) or the word 'group' to attempt to backfill all sites in the OFFICE id's Data Acquisition->USGS Measurements group",
+)
+@requires(reqs.cwms, reqs.requests, reqs.dataretrieval)
+def getusgs_measurements(
+    days_back_modified,
+    days_back_collected,
+    office,
+    api_root,
+    api_key,
+    api_key_loc,
+    backfill,
+):
+    from cwmscli.usgs.getusgs_measurements_cda import getusgs_measurement_cda
+
+    backfill_group = False
+    backfill_list = False
+    if backfill is not None:
+        if "group" in backfill:
+            backfill_group = True
+        elif type(backfill) == str:
+            backfill_list = backfill.replace(" ", "").split(",")
+    api_key = get_api_key(api_key, api_key_loc)
+    getusgs_measurement_cda(
+        api_root=api_root,
+        office_id=office,
+        api_key=api_key,
+        days_back_modified=days_back_modified,
+        days_back_collected=days_back_collected,
+        backfill_list=backfill_list,
+        backfill_group=backfill_group,
+    )