cwms-cli 0.1.1__py3-none-any.whl

cwmscli/commands/csv2cwms/tests/data/expected_brok_output.json

@@ -0,0 +1,278 @@
+[
+    {
+        "name": "BROK.Elev.Inst.15Minutes.0.Rev-SCADA-cda",
+        "units": "ft",
+        "values": [
+            [
+                1742952600000,
+                599.95,
+                3
+            ],
+            [
+                1742953500000,
+                599.92,
+                3
+            ],
+            [
+                1742954400000,
+                599.91,
+                3
+            ],
+            [
+                1742955300000,
+                599.93,
+                3
+            ],
+            [
+                1742956200000,
+                599.95,
+                3
+            ],
+            [
+                1742957100000,
+                599.92,
+                3
+            ],
+            [
+                1742958000000,
+                599.91,
+                3
+            ],
+            [
+                1742958900000,
+                599.93,
+                3
+            ]
+        ]
+    },
+    {
+        "name": "BROK.Elev-Tailwater.Inst.15Minutes.0.Rev-SCADA-cda",
+        "units": "ft",
+        "values": [
+            [
+                1742952600000,
+                405.54,
+                3
+            ],
+            [
+                1742953500000,
+                405.53,
+                3
+            ],
+            [
+                1742954400000,
+                405.57,
+                3
+            ],
+            [
+                1742955300000,
+                405.29,
+                3
+            ],
+            [
+                1742956200000,
+                405.54,
+                3
+            ],
+            [
+                1742957100000,
+                405.53,
+                3
+            ],
+            [
+                1742958000000,
+                405.57,
+                3
+            ],
+            [
+                1742958900000,
+                405.29,
+                3
+            ]
+        ]
+    },
+    {
+        "name": "BROK.Flow-Power.Inst.15Minutes.0.Rev-SCADA-cda",
+        "units": "cfs",
+        "values": [
+            [
+                1742952600000,
+                6700.0,
+                3
+            ],
+            [
+                1742953500000,
+                6772.0,
+                3
+            ],
+            [
+                1742954400000,
+                4027.0,
+                3
+            ],
+            [
+                1742955300000,
+                0.0,
+                3
+            ],
+            [
+                1742956200000,
+                6700.0,
+                3
+            ],
+            [
+                1742957100000,
+                6772.0,
+                3
+            ],
+            [
+                1742958000000,
+                4027.0,
+                3
+            ],
+            [
+                1742958900000,
+                0.0,
+                3
+            ]
+        ]
+    },
+    {
+        "name": "BROK.Power-Gen.Inst.15Minutes.0.Rev-SCADA-cda",
+        "units": "MW",
+        "values": [
+            [
+                1742952600000,
+                99.93,
+                3
+            ],
+            [
+                1742953500000,
+                100.62,
+                3
+            ],
+            [
+                1742954400000,
+                52.87,
+                3
+            ],
+            [
+                1742955300000,
+                -0.07,
+                3
+            ],
+            [
+                1742956200000,
+                99.93,
+                3
+            ],
+            [
+                1742957100000,
+                100.62,
+                3
+            ],
+            [
+                1742958000000,
+                52.87,
+                3
+            ],
+            [
+                1742958900000,
+                -0.07,
+                3
+            ]
+        ]
+    },
+    {
+        "name": "BROK.Turbine1.Power-Gen.Inst.15Minutes.0.Rev-SCADA-cda",
+        "units": "MW",
+        "values": [
+            [
+                1742952600000,
+                50.04,
+                3
+            ],
+            [
+                1742953500000,
+                50.6,
+                3
+            ],
+            [
+                1742954400000,
+                26.89,
+                3
+            ],
+            [
+                1742955300000,
+                -0.01,
+                3
+            ],
+            [
+                1742956200000,
+                50.04,
+                3
+            ],
+            [
+                1742957100000,
+                50.6,
+                3
+            ],
+            [
+                1742958000000,
+                26.89,
+                3
+            ],
+            [
+                1742958900000,
+                -0.01,
+                3
+            ]
+        ]
+    },
+    {
+        "name": "BROK.Turbine2.Power-Gen.Inst.15Minutes.0.Rev-SCADA-cda",
+        "units": "MW",
+        "values": [
+            [
+                1742952600000,
+                49.89,
+                3
+            ],
+            [
+                1742953500000,
+                50.02,
+                3
+            ],
+            [
+                1742954400000,
+                25.98,
+                3
+            ],
+            [
+                1742955300000,
+                -0.06,
+                3
+            ],
+            [
+                1742956200000,
+                49.89,
+                3
+            ],
+            [
+                1742957100000,
+                50.02,
+                3
+            ],
+            [
+                1742958000000,
+                25.98,
+                3
+            ],
+            [
+                1742958900000,
+                -0.06,
+                3
+            ]
+        ]
+    }
+]

cwmscli/commands/csv2cwms/tests/data/sample_brok.csv

@@ -0,0 +1,9 @@
+"Time","Headwater","Tailwater","U1_MW","U1_MVAR","U2_MW","U2_Mvar","total_Pwr_discharge","Tot_Tainter_Opening","SG1"
+03/25/2025 20:30:00,599.95,405.54,50.04,4.93,49.89,5.09,6700,0.00,21.00
+03/25/2025 20:45:00,599.92,405.53,50.6,2.57,50.02,3.33,6772,0.00,21.00
+03/25/2025 21:00:00,599.91,405.57,26.89,1.21,25.98,1.22,4027,0.00,21.00
+03/25/2025 21:15:00,599.93,405.29,-0.01,-0.01,-0.06,-0.01,0,0.00,21.00
+03/25/2025 21:30:00,599.95,405.54,50.04,4.93,49.89,5.09,6700,0.00,21.00
+03/25/2025 21:45:00,599.92,405.53,50.6,2.57,50.02,3.33,6772,0.00,21.00
+03/25/2025 22:00:00,599.91,405.57,26.89,1.21,25.98,1.22,4027,0.00,21.00
+03/25/2025 22:15:00,599.93,405.29,-0.01,-0.01,-0.06,-0.01,0,0.00,21.00

cwmscli/commands/csv2cwms/tests/data/sample_config.json

@@ -0,0 +1,45 @@
+{
+    "interval": null,
+    "input_files": {
+        "BROK": {
+            "data_path": "cwmscli/commands/csv2cwms/tests/data/sample_brok.csv",
+            "store_rule": "REPLACE_ALL",
+            "date_format": [
+                "%m/%d/%Y %H:%M:%S",
+                "%m/%d/%Y %H:%M"
+            ],
+            "timeseries": {
+                "BROK.Elev.Inst.15Minutes.0.Rev-SCADA-cda": {
+                    "columns": "Headwater",
+                    "units": "ft",
+                    "precision": 2
+                },
+                "BROK.Elev-Tailwater.Inst.15Minutes.0.Rev-SCADA-cda": {
+                    "columns": "Tailwater",
+                    "units": "ft",
+                    "precision": 2
+                },
+                "BROK.Flow-Power.Inst.15Minutes.0.Rev-SCADA-cda": {
+                    "columns": "total_Pwr_discharge",
+                    "units": "cfs",
+                    "precision": 0
+                },
+                "BROK.Power-Gen.Inst.15Minutes.0.Rev-SCADA-cda": {
+                    "columns": "U1_MW+U2_MW",
+                    "units": "MW",
+                    "precision": 2
+                },
+                "BROK-Turbine1.Power-Gen.Inst.15Minutes.0.Rev-SCADA-cda": {
+                    "columns": "U1_MW",
+                    "units": "MW",
+                    "precision": 2
+                },
+                "BROK-Turbine2.Power-Gen.Inst.15Minutes.0.Rev-SCADA-cda": {
+                    "columns": "U2_MW",
+                    "units": "MW",
+                    "precision": 2
+                }
+            }
+        }
+    }
+}

cwmscli/commands/csv2cwms/tests/skip_test_integration_pipeline.py

@@ -0,0 +1,35 @@
+import json
+import os
+
+from scada_ts import __main__ as main
+from utils.dateutils import parse_date
+from utils.fileio import load_csv, read_config
+
+
+def test_brok_pipeline_matches_expected():
+    # Setup paths to our test files
+    base = os.path.join(os.path.dirname(__file__), "data")
+    csv_path = os.path.join(base, "sample_brok.csv")
+    config_path = os.path.join(base, "sample_config.json")
+    expected_path = os.path.join(base, "expected_brok_output.json")
+
+    # Load all of our test files
+    config = read_config(config_path)
+    raw_csv = load_csv(csv_path)
+    header = raw_csv[0]
+    rows = raw_csv[1:]
+
+    # Load the data into a dictionary with timestamps as keys
+    parsed_data = {}
+    for row in rows:
+        dt = parse_date(row[0])
+        parsed_data[int(dt.timestamp())] = row
+
+    file_data = {"header": header, "data": parsed_data}
+
+    actual = main.load_timeseries(file_data, "BROK", config)
+
+    with open(expected_path) as f:
+        expected = json.load(f)
+
+    assert actual == expected

cwmscli/commands/csv2cwms/tests/test_dateutils.py

@@ -0,0 +1,68 @@
+from datetime import datetime, timedelta
+
+import pytest
+
+from ..utils.dateutils import determine_interval, parse_date, safe_zoneinfo
+
+
+def test_parse_date_valid_formats():
+    tz = safe_zoneinfo("UTC")
+    expected = datetime(2025, 3, 25, 14, 30, tzinfo=tz)
+    assert parse_date("03/25/2025 14:30:00") == expected
+    assert parse_date("03/25/2025 14:30") == expected
+    assert parse_date("03/25/2025 14") == datetime(2025, 3, 25, 14, tzinfo=tz)
+
+
+def test_parse_date_invalid_format():
+    with pytest.raises(ValueError):
+        parse_date("25-03-2025")
+
+
+def test_determine_interval_regular_spacing():
+    now = datetime(2025, 3, 25, 14, 30, tzinfo=safe_zoneinfo("UTC"))
+    interval = 900  # 15 minutes
+    csv_data = {
+        int((now + timedelta(seconds=i * interval)).timestamp()): [] for i in range(5)
+    }
+    assert determine_interval(csv_data) == 900
+
+
+def test_determine_interval_mixed_spacing():
+    now = datetime(2025, 3, 25, 14, 30, tzinfo=safe_zoneinfo("UTC"))
+    timestamps = [
+        now,
+        now + timedelta(minutes=15),
+        now + timedelta(minutes=30),
+        now + timedelta(minutes=60),  # outlier
+        now + timedelta(minutes=45),
+    ]
+    csv_data = {int(dt.timestamp()): [] for dt in timestamps}
+    assert determine_interval(csv_data) == 900
+
+
+def test_determine_interval_duplicate_timestamps():
+    now = datetime(2025, 3, 25, 14, 30, tzinfo=safe_zoneinfo("UTC"))
+    timestamps = [
+        now,
+        now + timedelta(minutes=15),
+        now + timedelta(minutes=15),  # duplicate
+        now + timedelta(minutes=30),
+    ]
+    csv_data = {int(dt.timestamp()): [] for dt in timestamps}
+    assert determine_interval(csv_data) == 900
+
+
+def test_determine_interval_missing_values():
+    now = datetime(2025, 3, 25, 14, 30, tzinfo=safe_zoneinfo("UTC"))
+    csv_data = {
+        int((now + timedelta(minutes=15 * i)).timestamp()): []
+        for i in [0, 1, 3, 4]  # skip index 2 (30-minute gap)
+    }
+    assert determine_interval(csv_data) == 900
+
+
+def test_determine_interval_insufficient_data():
+    now = datetime(2025, 3, 25, 14, 30, tzinfo=safe_zoneinfo("UTC"))
+    csv_data = {int(now.timestamp()): []}  # only one row
+    with pytest.raises(ValueError):
+        determine_interval(csv_data)

cwmscli/commands/csv2cwms/tests/test_expressions.py

@@ -0,0 +1,49 @@
+import pytest
+
+from ..utils.expression import eval_expression
+
+
+@pytest.mark.parametrize(
+    "expr,row,header_map,expected",
+    [
+        (
+            "U1_MW+U2_MW",
+            ["599.95", "405.54", "50.04", "4.93", "49.89"],
+            {"u1_mw": 2, "u2_mw": 4},
+            99.93,
+        ),
+        ("U1_MW-U2_MW", ["", "", "60.0", "", "15.5"], {"u1_mw": 2, "u2_mw": 4}, 44.5),
+        ("U1_MW*U2_MW", ["", "", "5.0", "", "2.0"], {"u1_mw": 2, "u2_mw": 4}, 10.0),
+        ("MISSING+U1_MW", ["", "", "50.0"], {"u1_mw": 2}, None),
+    ],
+)
+def test_eval_expression(expr, row, header_map, expected):
+    result = eval_expression(expr, row, header_map)
+    if expected is None:
+        assert result is None
+    else:
+        assert round(result, 2) == pytest.approx(expected, abs=1e-2)
+
+
+def test_eval_expression_invalid_token():
+    row = ["", "", "5.0", "", "2.0"]
+    header_map = {"u1_mw": 2, "u2_mw": 4}
+    assert eval_expression("U1_MW+@U2_MW", row, header_map) is None
+
+
+def test_eval_expression_division_by_zero():
+    row = ["", "", "5.0", "", "0"]
+    header_map = {"u1_mw": 2, "u2_mw": 4}
+    assert eval_expression("U1_MW/U2_MW", row, header_map) is None
+
+
+def test_eval_expression_missing_header_map():
+    row = ["", "", "5.0", "", "2.0"]
+    header_map = {}  # empty
+    assert eval_expression("U1_MW+U2_MW", row, header_map) is None
+
+
+def test_eval_expression_non_numeric():
+    row = ["", "", "fifty", "", "two"]
+    header_map = {"u1_mw": 2, "u2_mw": 4}
+    assert eval_expression("U1_MW+U2_MW", row, header_map) is None

cwmscli/commands/csv2cwms/tests/test_fileio.py

@@ -0,0 +1,43 @@
+import os
+
+import pytest
+
+from ..utils.fileio import load_csv, read_config
+
+
+def test_load_csv_valid():
+    path = os.path.join(os.path.dirname(__file__), "data", "sample_brok.csv")
+    result = load_csv(path)
+    assert isinstance(result, list)
+    assert isinstance(result[0], list)
+    assert len(result[0]) > 1  # header row should have multiple columns
+
+
+def test_load_csv_nonexistent():
+    path = os.path.join(os.path.dirname(__file__), "data", "does_not_exist.csv")
+    with pytest.raises(FileNotFoundError):
+        load_csv(path)
+
+
+def test_load_csv_malformed_row(tmp_path):
+    malformed = tmp_path / "bad.csv"
+    malformed.write_text("Time,Value\n2025-01-01 00:00\n2025-01-01 00:15,42,Extra")
+    result = load_csv(str(malformed))
+    assert len(result) == 3
+    assert result[1] == ["2025-01-01 00:00"]
+    assert result[2] == ["2025-01-01 00:15", "42", "Extra"]
+
+
+def test_read_config_valid():
+    path = os.path.join(os.path.dirname(__file__), "data", "sample_config.json")
+    config = read_config(path)
+    assert isinstance(config, dict)
+    assert "input_files" in config
+    assert "BROK" in config["input_files"]
+
+
+def test_read_config_invalid_json(tmp_path):
+    bad_json = tmp_path / "bad.json"
+    bad_json.write_text("{invalid_json: true,}")
+    with pytest.raises(Exception):
+        read_config(str(bad_json))

cwmscli/commands/csv2cwms/utils/__init__.py

@@ -0,0 +1,5 @@
+from .dateutils import DATE_STRINGS, determine_interval, parse_date, safe_zoneinfo
+from .expression import eval_expression
+from .fileio import load_csv, read_config
+from .logging import logger, setup_logger
+from .terminal import colorize, colorize_count

cwmscli/commands/csv2cwms/utils/dateutils.py

@@ -0,0 +1,105 @@
+import logging
+from collections import Counter
+from datetime import datetime, timezone
+from typing import List
+
+try:
+    from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+except ImportError:
+    # Python < 3.9 does not support zoneinfo
+    ZoneInfo = None
+    ZoneInfoNotFoundError = Exception
+
+logger = logging.getLogger(__name__)
+
+DATE_STRINGS = [
+    "%m/%d/%Y %H:%M:%S",
+    "%m/%d/%Y %H:%M",
+    "%m/%d/%Y %H",
+    "%Y-%m-%dT%H:%M:%S",
+    "%Y-%m-%dT%H:%M",
+    "%Y-%m-%dT%H",
+    "%Y-%m-%d %H:%M:%S",
+    "%Y-%m-%d %H:%M",
+    "%Y-%m-%d %H",
+]
+
+
+def safe_zoneinfo(key: str):
+    """
+    Attempts to return ZoneInfo(key); falls back to UTC if unavailable.
+    """
+    if ZoneInfo is None:
+        return timezone.utc  # fallback for very old Python
+
+    try:
+        return ZoneInfo(key)
+    except ZoneInfoNotFoundError:
+        return timezone.utc
+
+
+def parse_date(date, tz_str="UTC", date_format: str = "") -> datetime:
+    """Handle all date types seen in hydropower files
+    NOTE: TimeZone naive - assumes all timestamps are in the same timezone
+    Args:
+        date (str): Date string to parse
+    """
+    if isinstance(date, int):
+        return datetime.fromtimestamp(date, tz=safe_zoneinfo(tz_str))
+
+    if isinstance(date_format, str):
+        # Handle comma-separated list of formats
+        if date_format.find(",") >= 0:
+            date_format = [fmt.strip() for fmt in date_format.split(",") if fmt.strip()]
+        date_format = [date_format]
+
+    # Include the user-specified date format first, if provided
+    for idx, fmt in enumerate(date_format + DATE_STRINGS):
+        try:
+            if not fmt:
+                continue
+            dt_naive = datetime.strptime(date, fmt)
+            if idx > 0:
+                # Only log if using a fallback format
+                if not date_format:
+                    logger.warning(
+                        f"Using fallback date format '{fmt}' for date '{date}'. No user-specified format was provided."
+                    )
+                else:
+                    logger.warning(
+                        f"Using fallback date format '{fmt}' for date '{date}'. The user-specified format is '{date_format}'."
+                    )
+            return dt_naive.replace(tzinfo=safe_zoneinfo(tz_str))
+        except ValueError:
+            continue
+    raise ValueError(f"Invalid date format: {date}")
+
+
+def determine_interval(csv_data: List[list], sample_size=10) -> int:
+    """
+    Determine the most common interval (in seconds) between timestamps in the first few rows of CSV data.
+    Args:
+        `csv_data` is the raw list-of-lists from your CSV (NOT including header).
+        `sample_size` is the number of rows to sample from the CSV data.
+    Returns:
+        [int] The most common interval between timestamps in seconds
+
+    """
+
+    timestamps = []
+    dates = list(csv_data.keys())
+    sample_idx = min(sample_size, len(dates) - 1)
+    if sample_idx < 0:
+        raise ValueError("No data found in CSV file for the given lookback period.")
+    for row in dates[0:sample_idx]:
+        try:
+            timestamps.append(parse_date(row))
+        except Exception as err:
+            continue
+
+    if len(timestamps) < 2:
+        raise ValueError("Not enough valid timestamps to determine interval.")
+
+    diffs = [int((b - a).total_seconds()) for a, b in zip(timestamps, timestamps[1:])]
+    most_common = Counter(diffs).most_common(1)[0][0]
+    return most_common

cwmscli/commands/csv2cwms/utils/expression.py

@@ -0,0 +1,39 @@
+import re
+
+
+def eval_expression(expr, row, header_map):
+    """
+    Evaluate simple math expressions (+, -, *) using values from the row based on column names in the expression.
+    """
+    tokens = re.findall(
+        r'"[^"]+"|\'[^\']+\'|\+|\-|\*|[^\+\-\*]+', expr.replace(" ", "")
+    )
+    result = None
+    for i, token in enumerate(tokens):
+        if token in {"+", "-", "*"}:
+            continue
+
+        col_name = token.strip('"').strip("'").lower()
+        idx = header_map.get(col_name)
+        if idx is None or idx >= len(row):
+            # Immediately return to prevent adding None (0) to result
+            return None
+        else:
+            try:
+                val = float(row[idx])
+            except ValueError:
+                val = None
+
+        if result is None:
+            result = val
+        else:
+            op = tokens[i - 1]
+            if val is None or result is None:
+                result = None
+            elif op == "+":
+                result += val
+            elif op == "-":
+                result -= val
+            elif op == "*":
+                result *= val
+    return result