cwms-cli 0.6.0__tar.gz → 0.7.1__tar.gz

{cwms_cli-0.6.0 → cwms_cli-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cwms-cli
-Version: 0.6.0
+Version: 0.7.1
 Summary: Command line utilities for Corps Water Management Systems (CWMS) python scripts. This is a collection of shared scripts across the enterprise Water Management Enterprise System (WMES) teams.
 License: LICENSE
 License-File: LICENSE

{cwms_cli-0.6.0 → cwms_cli-0.7.1}/cwmscli/commands/blob.py

@@ -11,10 +11,12 @@ from typing import Optional, Sequence, Tuple, Union
 
 from cwmscli.utils import (
     colors,
+    format_local_download_error,
     get_api_key,
     has_invalid_chars,
     init_cwms_session,
     log_scoped_read_hint,
+    validate_default_download_dest,
 )
 from cwmscli.utils.click_help import DOCS_BASE_URL
 from cwmscli.utils.deps import requires
@@ -109,6 +111,14 @@ def _save_blob_content(
     return dest
 
 
+def _default_download_dest(blob_id: str) -> str:
+    return validate_default_download_dest(
+        blob_id,
+        resource_name="Blob",
+        docs_url=BLOB_DOCS_URL,
+    )
+
+
 def _blob_media_type(cwms_module, office: str, blob_id: str) -> Optional[str]:
     try:
         result = cwms_module.get_blobs(office_id=office, blob_id_like=blob_id)
@@ -602,7 +612,7 @@ def download_cmd(
 
     try:
         blob_content = cwms.get_blob(office_id=office, blob_id=bid)
-        target = dest or bid
+        target = dest or _default_download_dest(bid)
         _save_blob_content(
             blob_content,
             dest=target,
@@ -621,14 +631,15 @@ def download_cmd(
         )
         sys.exit(1)
     except Exception as e:
-        logging.error(f"Failed to download: {e}")
-        log_scoped_read_hint(
-            credential_kind=credential_kind,
-            anonymous=anonymous,
-            office=office,
-            action="download",
-            resource="blob content",
-        )
+        logging.error(format_local_download_error(e, BLOB_DOCS_URL))
+        if not isinstance(e, (OSError, ValueError)):
+            log_scoped_read_hint(
+                credential_kind=credential_kind,
+                anonymous=anonymous,
+                office=office,
+                action="download",
+                resource="blob content",
+            )
         sys.exit(1)
 
 

{cwms_cli-0.6.0 → cwms_cli-0.7.1}/cwmscli/commands/clob.py

@@ -9,7 +9,13 @@ import pandas as pd
 import requests
 from cwms import api as cwms_api
 
-from cwmscli.utils import get_api_key, has_invalid_chars, log_scoped_read_hint
+from cwmscli.utils import (
+    format_local_download_error,
+    get_api_key,
+    has_invalid_chars,
+    log_scoped_read_hint,
+    validate_default_download_dest,
+)
 
 
 def _join_api_url(api_root: str, path: str) -> str:
@@ -29,6 +35,10 @@ def _write_clob_content(content: str, dest: str) -> str:
     return dest
 
 
+def _default_download_dest(clob_id: str) -> str:
+    return validate_default_download_dest(clob_id, resource_name="Clob")
+
+
 def _clob_endpoint_id(clob_id: str) -> tuple[str, Optional[str]]:
     normalized = clob_id.upper()
     if has_invalid_chars(normalized):
@@ -198,7 +208,7 @@ def download_cmd(
                 content = str(payload)
         else:
             content = _get_special_clob_text(office=office, clob_id=query_id)
-        target = dest or bid
+        target = dest or _default_download_dest(bid)
         _write_clob_content(content, target)
         logging.info(f"Downloaded clob to: {target}")
     except requests.HTTPError as e:
@@ -213,14 +223,7 @@ def download_cmd(
         )
         sys.exit(1)
     except Exception as e:
-        logging.error(f"Failed to download: {e}")
-        log_scoped_read_hint(
-            api_key=resolved_api_key,
-            anonymous=anonymous,
-            office=office,
-            action="download",
-            resource="clob content",
-        )
+        logging.error(format_local_download_error(e, ""))
         sys.exit(1)
 
 

{cwms_cli-0.6.0 → cwms_cli-0.7.1}/cwmscli/load/location/location.py

@@ -5,6 +5,7 @@ import click
 
 from cwmscli import requirements as reqs
 from cwmscli.load.root import (
+    csv_source_target_options,
     load_group,
     shared_source_target_options,
     validate_cda_targets,
@@ -24,12 +25,14 @@ def location(ctx):
 @location.command(
     "ids-all",
     help=(
-        "Copy locations from a source CDA catalog to a target CDA. "
+        "Copy locations from a source CDA catalog to a target CDA, "
+        "or to/from a CSV file via --source-csv / --target-csv. "
         "The --like and --location-kind-like filters use CDA regex semantics. "
         f"Regex guide: {CDA_REGEXP_GUIDE_URL}"
     ),
 )
 @shared_source_target_options
+@csv_source_target_options(allow_source_csv=True, allow_target_csv=True)
 @click.option(
     "--like",
     default=None,
@@ -57,14 +60,16 @@ def location(ctx):
 @requires(reqs.cwms)
 @validate_cda_targets
 def load_locations(
-    source_cda: str,
-    source_office: str,
-    target_cda: str,
+    source_cda: Optional[str],
+    source_office: Optional[str],
+    target_cda: Optional[str],
     target_api_key: Optional[str],
     verbose: int,
     dry_run: bool,
     like: Optional[str],
     location_kind_like: Optional[Iterable[str]] = None,
+    source_csv: Optional[str] = None,
+    target_csv: Optional[str] = None,
 ):
     from cwmscli.load.location.location_ids import load_locations as _load_locations
 
@@ -77,14 +82,20 @@ def load_locations(
         dry_run=dry_run,
         like=like,
         location_kind_like=location_kind_like,
+        source_csv=source_csv,
+        target_csv=target_csv,
     )
 
 
 @location.command(
     "ids-bygroup",
-    help="Copy locations from a CWMS Location Group (source CDA) to a target CDA.",
+    help=(
+        "Copy locations from a CWMS Location Group (source CDA) to a target CDA, "
+        "or export the resolved members to a CSV file via --target-csv."
+    ),
 )
 @shared_source_target_options
+@csv_source_target_options(allow_source_csv=False, allow_target_csv=True)
 @click.option(
     "--group-id", required=True, help="Location Group ID (e.g., 'Ark Basin')."
 )
@@ -112,7 +123,7 @@ def load_locations(
 def load_locations_from_group(
     source_cda: str,
     source_office: str,
-    target_cda: str,
+    target_cda: Optional[str],
     target_api_key: Optional[str],
     verbose: int,
     group_id: str,
@@ -121,6 +132,7 @@ def load_locations_from_group(
     category_office_id: Optional[str],
     filter_office: bool,
     dry_run: bool,
+    target_csv: Optional[str] = None,
 ):
     from cwmscli.load.location.location_ids_bygroup import copy_from_group
 
@@ -136,4 +148,5 @@ def load_locations_from_group(
         category_office_id=category_office_id,
         filter_office=filter_office,
         dry_run=dry_run,
+        target_csv=target_csv,
     )

cwms_cli-0.7.1/cwmscli/load/location/location_ids.py

@@ -0,0 +1,147 @@
+import logging
+import math
+import re
+from typing import Iterable, Optional
+
+import click
+import cwms
+import pandas as pd
+
+from cwmscli.utils import init_cwms_session
+from cwmscli.utils.links import CDA_REGEXP_GUIDE_URL
+
+logger = logging.getLogger(__name__)
+
+
+def load_locations(
+    source_cda: Optional[str],
+    source_office: Optional[str],
+    target_cda: Optional[str],
+    target_api_key: Optional[str],
+    verbose: int,
+    dry_run: bool,
+    like: Optional[str],
+    location_kind_like: Optional[Iterable[str]] = "ALL",
+    source_csv: Optional[str] = None,
+    target_csv: Optional[str] = None,
+):
+    src_label = source_csv or source_cda or "-"
+    tgt_label = target_csv or target_cda or "-"
+
+    if verbose:
+        logger.info(
+            f"[load locations] source={src_label} ({source_office or '-'}) -> target={tgt_label}"
+        )
+        logger.info(
+            f"  like={like or '-'}  kinds={list(location_kind_like) or '-'}  dry_run={dry_run}"
+        )
+        if like or (
+            location_kind_like
+            and list(location_kind_like) != ["ALL"]
+            and list(location_kind_like) != []
+        ):
+            logger.info("  CDA regex guide: %s", CDA_REGEXP_GUIDE_URL)
+
+    if source_csv:
+        df = pd.read_csv(source_csv)
+        locations = df.to_dict(orient="records")
+    else:
+        init_cwms_session(cwms, api_root=source_cda)
+        locations = _fetch_locations_from_cda(
+            source_office=source_office,
+            like=like,
+            location_kind_like=location_kind_like,
+            verbose=verbose,
+        )
+
+    if verbose:
+        logger.info("Got %s locations from source", len(locations))
+
+    if dry_run:
+        for loc in locations:
+            logger.info(
+                f"[dry-run] would store Location(name={loc['name']}) to {tgt_label} "
+                f"({source_office or loc.get('office-id') or '-'})"
+            )
+        return
+
+    if target_csv:
+        pd.DataFrame(locations).to_csv(target_csv, index=False)
+        click.echo(f"Wrote {len(locations)} locations to {target_csv}")
+        return
+
+    init_cwms_session(cwms, api_root=target_cda, api_key=target_api_key)
+
+    errors = 0
+    for loc in locations:
+        loc = _clean_row(loc)
+        try:
+            if loc.get("active") is True:
+                result = cwms.store_location(data=loc, fail_if_exists=False)
+                if verbose:
+                    logger.info("%s", result)
+        except Exception as e:
+            errors += 1
+            click.echo(f"Error storing location {loc}: \n\t{e}", err=True)
+
+    if errors:
+        raise click.ClickException(f"Completed with {errors} error(s).")
+
+    click.echo("Done.")
+
+
+def _fetch_locations_from_cda(
+    source_office: str,
+    like: Optional[str],
+    location_kind_like: Optional[Iterable[str]],
+    verbose: int,
+) -> list:
+    cat_kwargs = {"office_id": source_office}
+    if like:
+        cat_kwargs["like"] = like
+    kinds = list(location_kind_like) if location_kind_like else ["ALL"]
+    if "ALL" in kinds:
+        kinds = ["ALL"]
+
+    if kinds == ["ALL"] and not like:
+        return cwms.get_locations(office_id=source_office).json
+
+    locations = []
+    seen_location_ids = set()
+    for kind in kinds:
+        cat_kwargs_k = dict(cat_kwargs)
+        if kind != "ALL":
+            cat_kwargs_k["location_kind_like"] = kind
+
+        if verbose >= 2:
+            logger.debug("  > catalog query: %s", cat_kwargs_k)
+
+        resp = cwms.get_locations_catalog(**cat_kwargs_k)
+        if resp.df.empty:
+            continue
+
+        for location_id in resp.df["name"].tolist():
+            if location_id in seen_location_ids:
+                continue
+            seen_location_ids.add(location_id)
+            if verbose >= 2:
+                logger.debug("  > location fetch: %s", location_id)
+            detail_resp = cwms.get_locations(
+                office_id=source_office,
+                location_ids=rf"^{re.escape(location_id)}$",
+            )
+            if detail_resp and detail_resp.json:
+                locations.extend(detail_resp.json)
+    return locations
+
+
+def _clean_row(loc: dict) -> dict:
+    cleaned = {}
+    for k, v in loc.items():
+        if isinstance(v, float) and math.isnan(v):
+            cleaned[k] = None
+        elif isinstance(v, str) and v in ("True", "False"):
+            cleaned[k] = v == "True"
+        else:
+            cleaned[k] = v
+    return cleaned

{cwms_cli-0.6.0 → cwms_cli-0.7.1}/cwmscli/load/location/location_ids_bygroup.py

@@ -5,6 +5,7 @@ from typing import Optional
 
 import click
 import cwms
+import pandas as pd
 
 from cwmscli.utils import init_cwms_session
 
@@ -22,7 +23,7 @@ def exact_or_regex(ids: list[str]) -> str:
 def copy_from_group(
     source_cda: str,
     source_office: str,
-    target_cda: str,
+    target_cda: Optional[str],
     target_api_key: Optional[str],
     verbose: int,
     group_id: str,
@@ -31,6 +32,7 @@ def copy_from_group(
     category_office_id: Optional[str],
     filter_office: bool,
     dry_run: bool,
+    target_csv: Optional[str] = None,
 ):
     group_office_id = group_office_id or source_office
     category_office_id = category_office_id or source_office
@@ -103,10 +105,16 @@ def copy_from_group(
     if dry_run:
         for loc in locations:
             logger.info(
-                f"[dry-run] would store Location(name={loc['name']}) to {target_cda} ({source_office})"
+                f"[dry-run] would store Location(name={loc['name']}) to "
+                f"{target_csv or target_cda} ({source_office})"
             )
         return
 
+    if target_csv:
+        pd.DataFrame(locations).to_csv(target_csv, index=False)
+        click.echo(f"Wrote {len(locations)} locations to {target_csv}")
+        return
+
     try:
         init_cwms_session(cwms, api_root=target_cda, api_key=target_api_key)
     except Exception as e:

{cwms_cli-0.6.0 → cwms_cli-0.7.1}/cwmscli/load/root.py

@@ -43,11 +43,39 @@ def _norm_office(o: Optional[str]) -> str:
 def validate_cda_targets(func):
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
+        source_csv = kwargs.get("source_csv")
+        target_csv = kwargs.get("target_csv")
+
+        if source_csv and target_csv:
+            raise click.ClickException(
+                "--source-csv and --target-csv are both set, but no CDA is involved. "
+                "Use a plain file copy instead."
+            )
+
+        if source_csv:
+            if kwargs.get("source_cda") and _param_was_explicit("source_cda"):
+                raise click.ClickException(
+                    "--source-csv and --source-cda are mutually exclusive."
+                )
+            kwargs["source_cda"] = None
+
+        if target_csv:
+            if kwargs.get("target_cda") and _param_was_explicit("target_cda"):
+                raise click.ClickException(
+                    "--target-csv and --target-cda are mutually exclusive."
+                )
+            kwargs["target_cda"] = None
+
         source_cda = _normalize_url(kwargs.get("source_cda"))
         target_cda = _normalize_url(kwargs.get("target_cda"))
         source_office = _norm_office(kwargs.get("source_office"))
         target_office = _norm_office(kwargs.get("target_office"))
 
+        if source_cda and not source_office:
+            raise click.ClickException(
+                "--source-office is required when reading from a source CDA."
+            )
+
         same_root = source_cda == target_cda and bool(source_cda)
         same_office = source_office == target_office and bool(source_office)
 
@@ -64,33 +92,40 @@ def validate_cda_targets(func):
                 "This is allowed, but double-check intent.",
             )
 
+        src_label = source_csv or source_cda or "-"
+        tgt_label = target_csv or target_cda or "-"
         logger.info(
-            f"Source: {source_cda} (office={source_office or '-'})\n"
-            f"Target: {target_cda} (office={source_office or '-'})",
+            f"Source: {src_label} (office={source_office or '-'})\n"
+            f"Target: {tgt_label} (office={target_office or source_office or '-'})",
         )
         return func(*args, **kwargs)
 
     return wrapper
 
 
+def _param_was_explicit(name: str) -> bool:
+    ctx = click.get_current_context(silent=True)
+    if ctx is None:
+        return False
+    src = ctx.get_parameter_source(name)
+    return src is not None and src.name != "DEFAULT"
+
+
 def shared_source_target_options(f):
     f = click.option(
         "--source-cda",
         envvar="CDA_SOURCE_URL",
-        required=True,
         default="https://cwms-data.usace.army.mil/cwms-data/",
         help="Source CWMS Data API root. Default: https://cwms-data.usace.army.mil/cwms-data/",
     )(f)
     f = click.option(
         "--source-office",
         envvar="CDA_SOURCE_OFFICE",
-        required=True,
-        help="Source office ID (e.g. SWT, SWL).",
+        help="Source office ID (e.g. SWT, SWL). Required when reading from a CDA.",
     )(f)
     f = click.option(
         "--target-cda",
         envvar="CDA_TARGET_URL",
-        required=True,
         default="http://localhost:8081/cwms-data/",
         help="Target CWMS Data API root. Default: http://localhost:8081/cwms-data/",
     )(f)
@@ -115,6 +150,37 @@ def shared_source_target_options(f):
     return f
 
 
+def csv_source_target_options(*, allow_source_csv: bool, allow_target_csv: bool):
+    """Add --source-csv and/or --target-csv to a command, depending on flags."""
+
+    def decorator(f):
+        if allow_target_csv:
+            f = click.option(
+                "--target-csv",
+                "target_csv",
+                type=click.Path(dir_okay=False, writable=True),
+                default=None,
+                help=(
+                    "Write fetched locations to this CSV file instead of POSTing "
+                    "to a target CDA. Mutually exclusive with --target-cda."
+                ),
+            )(f)
+        if allow_source_csv:
+            f = click.option(
+                "--source-csv",
+                "source_csv",
+                type=click.Path(exists=True, dir_okay=False, readable=True),
+                default=None,
+                help=(
+                    "Read locations from this CSV file instead of fetching from "
+                    "a source CDA. Mutually exclusive with --source-cda."
+                ),
+            )(f)
+        return f
+
+    return decorator
+
+
 @click.group(
     name="load",
     help="Load data from one CWMS Data API instance to another.",

{cwms_cli-0.6.0 → cwms_cli-0.7.1}/cwmscli/utils/__init__.py

@@ -1,4 +1,5 @@
 import logging as py_logging
+import re
 import time
 from pathlib import Path
 from typing import Optional, Union
@@ -235,6 +236,66 @@ def log_scoped_read_hint(
     )
 
 
+def format_local_download_error(error: Exception, docs_url: str) -> str:
+    if isinstance(error, (OSError, ValueError)):
+        message = (
+            f"{colors.c('Failed to download:', 'red', bright=True)} {error}. "
+            f"If this is a local destination/path issue, pass "
+            f"{colors.c('--dest', 'cyan', bright=True)} explicitly."
+        )
+        if docs_url:
+            message = (
+                f"{message} {colors.c('Docs:', 'blue', bright=True)} "
+                f"{colors.c(docs_url, 'blue', bright=True)}"
+            )
+        return message
+    return f"{colors.c('Failed to download:', 'red', bright=True)} {error}"
+
+
+def validate_default_download_dest(
+    raw_id: str,
+    *,
+    resource_name: str,
+    docs_url: str = "",
+) -> str:
+    if raw_id is None:
+        raise ValueError(
+            f"{resource_name} ID must include a non-root destination name. "
+            f"Pass --dest explicitly if needed."
+        )
+
+    if raw_id.startswith("//") or raw_id.startswith("\\\\"):
+        raise ValueError(
+            f"{resource_name} ID must resolve to a relative local path. "
+            f"Pass --dest explicitly if needed."
+        )
+
+    target = raw_id.lstrip("/\\")
+    if not target:
+        message = (
+            f"{resource_name} ID must include a non-root destination name. "
+            f"Pass --dest explicitly if needed."
+        )
+        if docs_url:
+            message = f"{message} Docs: {docs_url}"
+        raise ValueError(message)
+
+    if re.match(r"^[A-Za-z]:", target):
+        raise ValueError(
+            f"{resource_name} ID must resolve to a relative local path. "
+            f"Pass --dest explicitly if needed."
+        )
+
+    parts = re.split(r"[\\/]", target)
+    if any(part in {"", ".", ".."} for part in parts):
+        raise ValueError(
+            f"{resource_name} ID must resolve to a relative local path. "
+            f"Pass --dest explicitly if needed."
+        )
+
+    return target
+
+
 def common_api_options(f):
     f = log_level_option(f)
     f = office_option(f)

{cwms_cli-0.6.0 → cwms_cli-0.7.1}/pyproject.toml

@@ -2,7 +2,7 @@
 name = "cwms-cli"
 repository = "https://github.com/HydrologicEngineeringCenter/cwms-cli"
 
-version = "0.6.0"
+version = "0.7.1"
 
 
 packages = [

cwms_cli-0.6.0/cwmscli/load/location/location_ids.py

@@ -1,105 +0,0 @@
-import logging
-import re
-from typing import Iterable, Optional
-
-import click
-import cwms
-
-from cwmscli.utils import init_cwms_session
-from cwmscli.utils.links import CDA_REGEXP_GUIDE_URL
-
-logger = logging.getLogger(__name__)
-
-
-def load_locations(
-    source_cda: str,
-    source_office: str,
-    target_cda: str,
-    target_api_key: Optional[str],
-    verbose: int,
-    dry_run: bool,
-    like: Optional[str],
-    location_kind_like: Optional[Iterable[str]] = "ALL",
-):
-    if verbose:
-        logger.info(
-            f"[load locations] source={source_cda} ({source_office}) -> target={target_cda}"
-        )
-        logger.info(
-            f"  like={like or '-'}  kinds={list(location_kind_like) or '-'}  dry_run={dry_run}"
-        )
-        if like or (
-            location_kind_like
-            and list(location_kind_like) != ["ALL"]
-            and list(location_kind_like) != []
-        ):
-            logger.info("  CDA regex guide: %s", CDA_REGEXP_GUIDE_URL)
-
-    init_cwms_session(cwms, api_root=source_cda)
-
-    cat_kwargs = {"office_id": source_office}
-    if like:
-        cat_kwargs["like"] = like
-    kinds = list(location_kind_like) if location_kind_like else ["ALL"]
-    if "ALL" in kinds:
-        kinds = ["ALL"]
-
-    locations = []
-
-    if kinds == ["ALL"] and not like:
-        locations = cwms.get_locations(office_id=source_office).json
-    else:
-        seen_location_ids = set()
-        for kind in kinds:
-            cat_kwargs_k = dict(cat_kwargs)
-            if kind != "ALL":
-                cat_kwargs_k["location_kind_like"] = kind
-
-            if verbose >= 2:
-                logger.debug("  > catalog query: %s", cat_kwargs_k)
-
-            resp = cwms.get_locations_catalog(**cat_kwargs_k)
-            if resp.df.empty:
-                continue
-
-            for location_id in resp.df["name"].tolist():
-                if location_id in seen_location_ids:
-                    continue
-                seen_location_ids.add(location_id)
-                if verbose >= 2:
-                    logger.debug("  > location fetch: %s", location_id)
-                detail_resp = cwms.get_locations(
-                    office_id=source_office,
-                    location_ids=rf"^{re.escape(location_id)}$",
-                )
-                if detail_resp and detail_resp.json:
-                    locations.extend(detail_resp.json)
-
-    if verbose:
-        logger.info("Fetched %s locations from source", len(locations))
-
-    if dry_run:
-        for loc in locations:
-            logger.info(
-                f"[dry-run] would store Location(name={loc['name']}) to {target_cda} ({source_office})"
-            )
-        return
-
-    # init target once
-    init_cwms_session(cwms, api_root=target_cda, api_key=target_api_key)
-
-    errors = 0
-    for loc in locations:
-        try:
-            if loc["active"] is True:
-                result = cwms.store_location(data=loc, fail_if_exists=False)
-                if verbose:
-                    logger.info("%s", result)
-        except Exception as e:
-            errors += 1
-            click.echo(f"Error storing location {loc}: \n\t{e}", err=True)
-
-    if errors:
-        raise click.ClickException(f"Completed with {errors} error(s).")
-
-    click.echo("Done.")