cwms-cli 0.6.0__tar.gz → 0.7.0__tar.gz

{cwms_cli-0.6.0 → cwms_cli-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cwms-cli
-Version: 0.6.0
+Version: 0.7.0
 Summary: Command line utilities for Corps Water Management Systems (CWMS) python scripts. This is a collection of shared scripts across the enterprise Water Management Enterprise System (WMES) teams.
 License: LICENSE
 License-File: LICENSE

{cwms_cli-0.6.0 → cwms_cli-0.7.0}/cwmscli/load/location/location.py

@@ -5,6 +5,7 @@ import click
 
 from cwmscli import requirements as reqs
 from cwmscli.load.root import (
+    csv_source_target_options,
     load_group,
     shared_source_target_options,
     validate_cda_targets,
@@ -24,12 +25,14 @@ def location(ctx):
 @location.command(
     "ids-all",
     help=(
-        "Copy locations from a source CDA catalog to a target CDA. "
+        "Copy locations from a source CDA catalog to a target CDA, "
+        "or to/from a CSV file via --source-csv / --target-csv. "
         "The --like and --location-kind-like filters use CDA regex semantics. "
         f"Regex guide: {CDA_REGEXP_GUIDE_URL}"
     ),
 )
 @shared_source_target_options
+@csv_source_target_options(allow_source_csv=True, allow_target_csv=True)
 @click.option(
     "--like",
     default=None,
@@ -57,14 +60,16 @@ def location(ctx):
 @requires(reqs.cwms)
 @validate_cda_targets
 def load_locations(
-    source_cda: str,
-    source_office: str,
-    target_cda: str,
+    source_cda: Optional[str],
+    source_office: Optional[str],
+    target_cda: Optional[str],
     target_api_key: Optional[str],
     verbose: int,
     dry_run: bool,
     like: Optional[str],
     location_kind_like: Optional[Iterable[str]] = None,
+    source_csv: Optional[str] = None,
+    target_csv: Optional[str] = None,
 ):
     from cwmscli.load.location.location_ids import load_locations as _load_locations
 
@@ -77,14 +82,20 @@ def load_locations(
         dry_run=dry_run,
         like=like,
         location_kind_like=location_kind_like,
+        source_csv=source_csv,
+        target_csv=target_csv,
     )
 
 
 @location.command(
     "ids-bygroup",
-    help="Copy locations from a CWMS Location Group (source CDA) to a target CDA.",
+    help=(
+        "Copy locations from a CWMS Location Group (source CDA) to a target CDA, "
+        "or export the resolved members to a CSV file via --target-csv."
+    ),
 )
 @shared_source_target_options
+@csv_source_target_options(allow_source_csv=False, allow_target_csv=True)
 @click.option(
     "--group-id", required=True, help="Location Group ID (e.g., 'Ark Basin')."
 )
@@ -112,7 +123,7 @@ def load_locations(
 def load_locations_from_group(
     source_cda: str,
     source_office: str,
-    target_cda: str,
+    target_cda: Optional[str],
     target_api_key: Optional[str],
     verbose: int,
     group_id: str,
@@ -121,6 +132,7 @@ def load_locations_from_group(
     category_office_id: Optional[str],
     filter_office: bool,
     dry_run: bool,
+    target_csv: Optional[str] = None,
 ):
     from cwmscli.load.location.location_ids_bygroup import copy_from_group
 
@@ -136,4 +148,5 @@ def load_locations_from_group(
         category_office_id=category_office_id,
         filter_office=filter_office,
         dry_run=dry_run,
+        target_csv=target_csv,
     )

cwms_cli-0.7.0/cwmscli/load/location/location_ids.py

@@ -0,0 +1,147 @@
+import logging
+import math
+import re
+from typing import Iterable, Optional
+
+import click
+import cwms
+import pandas as pd
+
+from cwmscli.utils import init_cwms_session
+from cwmscli.utils.links import CDA_REGEXP_GUIDE_URL
+
+logger = logging.getLogger(__name__)
+
+
+def load_locations(
+    source_cda: Optional[str],
+    source_office: Optional[str],
+    target_cda: Optional[str],
+    target_api_key: Optional[str],
+    verbose: int,
+    dry_run: bool,
+    like: Optional[str],
+    location_kind_like: Optional[Iterable[str]] = "ALL",
+    source_csv: Optional[str] = None,
+    target_csv: Optional[str] = None,
+):
+    src_label = source_csv or source_cda or "-"
+    tgt_label = target_csv or target_cda or "-"
+
+    if verbose:
+        logger.info(
+            f"[load locations] source={src_label} ({source_office or '-'}) -> target={tgt_label}"
+        )
+        logger.info(
+            f"  like={like or '-'}  kinds={list(location_kind_like) or '-'}  dry_run={dry_run}"
+        )
+        if like or (
+            location_kind_like
+            and list(location_kind_like) != ["ALL"]
+            and list(location_kind_like) != []
+        ):
+            logger.info("  CDA regex guide: %s", CDA_REGEXP_GUIDE_URL)
+
+    if source_csv:
+        df = pd.read_csv(source_csv)
+        locations = df.to_dict(orient="records")
+    else:
+        init_cwms_session(cwms, api_root=source_cda)
+        locations = _fetch_locations_from_cda(
+            source_office=source_office,
+            like=like,
+            location_kind_like=location_kind_like,
+            verbose=verbose,
+        )
+
+    if verbose:
+        logger.info("Got %s locations from source", len(locations))
+
+    if dry_run:
+        for loc in locations:
+            logger.info(
+                f"[dry-run] would store Location(name={loc['name']}) to {tgt_label} "
+                f"({source_office or loc.get('office-id') or '-'})"
+            )
+        return
+
+    if target_csv:
+        pd.DataFrame(locations).to_csv(target_csv, index=False)
+        click.echo(f"Wrote {len(locations)} locations to {target_csv}")
+        return
+
+    init_cwms_session(cwms, api_root=target_cda, api_key=target_api_key)
+
+    errors = 0
+    for loc in locations:
+        loc = _clean_row(loc)
+        try:
+            if loc.get("active") is True:
+                result = cwms.store_location(data=loc, fail_if_exists=False)
+                if verbose:
+                    logger.info("%s", result)
+        except Exception as e:
+            errors += 1
+            click.echo(f"Error storing location {loc}: \n\t{e}", err=True)
+
+    if errors:
+        raise click.ClickException(f"Completed with {errors} error(s).")
+
+    click.echo("Done.")
+
+
+def _fetch_locations_from_cda(
+    source_office: str,
+    like: Optional[str],
+    location_kind_like: Optional[Iterable[str]],
+    verbose: int,
+) -> list:
+    cat_kwargs = {"office_id": source_office}
+    if like:
+        cat_kwargs["like"] = like
+    kinds = list(location_kind_like) if location_kind_like else ["ALL"]
+    if "ALL" in kinds:
+        kinds = ["ALL"]
+
+    if kinds == ["ALL"] and not like:
+        return cwms.get_locations(office_id=source_office).json
+
+    locations = []
+    seen_location_ids = set()
+    for kind in kinds:
+        cat_kwargs_k = dict(cat_kwargs)
+        if kind != "ALL":
+            cat_kwargs_k["location_kind_like"] = kind
+
+        if verbose >= 2:
+            logger.debug("  > catalog query: %s", cat_kwargs_k)
+
+        resp = cwms.get_locations_catalog(**cat_kwargs_k)
+        if resp.df.empty:
+            continue
+
+        for location_id in resp.df["name"].tolist():
+            if location_id in seen_location_ids:
+                continue
+            seen_location_ids.add(location_id)
+            if verbose >= 2:
+                logger.debug("  > location fetch: %s", location_id)
+            detail_resp = cwms.get_locations(
+                office_id=source_office,
+                location_ids=rf"^{re.escape(location_id)}$",
+            )
+            if detail_resp and detail_resp.json:
+                locations.extend(detail_resp.json)
+    return locations
+
+
+def _clean_row(loc: dict) -> dict:
+    cleaned = {}
+    for k, v in loc.items():
+        if isinstance(v, float) and math.isnan(v):
+            cleaned[k] = None
+        elif isinstance(v, str) and v in ("True", "False"):
+            cleaned[k] = v == "True"
+        else:
+            cleaned[k] = v
+    return cleaned

{cwms_cli-0.6.0 → cwms_cli-0.7.0}/cwmscli/load/location/location_ids_bygroup.py

@@ -5,6 +5,7 @@ from typing import Optional
 
 import click
 import cwms
+import pandas as pd
 
 from cwmscli.utils import init_cwms_session
 
@@ -22,7 +23,7 @@ def exact_or_regex(ids: list[str]) -> str:
 def copy_from_group(
     source_cda: str,
     source_office: str,
-    target_cda: str,
+    target_cda: Optional[str],
     target_api_key: Optional[str],
     verbose: int,
     group_id: str,
@@ -31,6 +32,7 @@ def copy_from_group(
     category_office_id: Optional[str],
     filter_office: bool,
     dry_run: bool,
+    target_csv: Optional[str] = None,
 ):
     group_office_id = group_office_id or source_office
     category_office_id = category_office_id or source_office
@@ -103,10 +105,16 @@ def copy_from_group(
     if dry_run:
         for loc in locations:
             logger.info(
-                f"[dry-run] would store Location(name={loc['name']}) to {target_cda} ({source_office})"
+                f"[dry-run] would store Location(name={loc['name']}) to "
+                f"{target_csv or target_cda} ({source_office})"
             )
         return
 
+    if target_csv:
+        pd.DataFrame(locations).to_csv(target_csv, index=False)
+        click.echo(f"Wrote {len(locations)} locations to {target_csv}")
+        return
+
     try:
         init_cwms_session(cwms, api_root=target_cda, api_key=target_api_key)
     except Exception as e:

{cwms_cli-0.6.0 → cwms_cli-0.7.0}/cwmscli/load/root.py

@@ -43,11 +43,39 @@ def _norm_office(o: Optional[str]) -> str:
 def validate_cda_targets(func):
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
+        source_csv = kwargs.get("source_csv")
+        target_csv = kwargs.get("target_csv")
+
+        if source_csv and target_csv:
+            raise click.ClickException(
+                "--source-csv and --target-csv are both set, but no CDA is involved. "
+                "Use a plain file copy instead."
+            )
+
+        if source_csv:
+            if kwargs.get("source_cda") and _param_was_explicit("source_cda"):
+                raise click.ClickException(
+                    "--source-csv and --source-cda are mutually exclusive."
+                )
+            kwargs["source_cda"] = None
+
+        if target_csv:
+            if kwargs.get("target_cda") and _param_was_explicit("target_cda"):
+                raise click.ClickException(
+                    "--target-csv and --target-cda are mutually exclusive."
+                )
+            kwargs["target_cda"] = None
+
         source_cda = _normalize_url(kwargs.get("source_cda"))
         target_cda = _normalize_url(kwargs.get("target_cda"))
         source_office = _norm_office(kwargs.get("source_office"))
         target_office = _norm_office(kwargs.get("target_office"))
 
+        if source_cda and not source_office:
+            raise click.ClickException(
+                "--source-office is required when reading from a source CDA."
+            )
+
         same_root = source_cda == target_cda and bool(source_cda)
         same_office = source_office == target_office and bool(source_office)
 
@@ -64,33 +92,40 @@ def validate_cda_targets(func):
                 "This is allowed, but double-check intent.",
             )
 
+        src_label = source_csv or source_cda or "-"
+        tgt_label = target_csv or target_cda or "-"
         logger.info(
-            f"Source: {source_cda} (office={source_office or '-'})\n"
-            f"Target: {target_cda} (office={source_office or '-'})",
+            f"Source: {src_label} (office={source_office or '-'})\n"
+            f"Target: {tgt_label} (office={target_office or source_office or '-'})",
         )
         return func(*args, **kwargs)
 
     return wrapper
 
 
+def _param_was_explicit(name: str) -> bool:
+    ctx = click.get_current_context(silent=True)
+    if ctx is None:
+        return False
+    src = ctx.get_parameter_source(name)
+    return src is not None and src.name != "DEFAULT"
+
+
 def shared_source_target_options(f):
     f = click.option(
         "--source-cda",
         envvar="CDA_SOURCE_URL",
-        required=True,
         default="https://cwms-data.usace.army.mil/cwms-data/",
         help="Source CWMS Data API root. Default: https://cwms-data.usace.army.mil/cwms-data/",
     )(f)
     f = click.option(
         "--source-office",
         envvar="CDA_SOURCE_OFFICE",
-        required=True,
-        help="Source office ID (e.g. SWT, SWL).",
+        help="Source office ID (e.g. SWT, SWL). Required when reading from a CDA.",
     )(f)
     f = click.option(
         "--target-cda",
         envvar="CDA_TARGET_URL",
-        required=True,
         default="http://localhost:8081/cwms-data/",
         help="Target CWMS Data API root. Default: http://localhost:8081/cwms-data/",
     )(f)
@@ -115,6 +150,37 @@ def shared_source_target_options(f):
     return f
 
 
+def csv_source_target_options(*, allow_source_csv: bool, allow_target_csv: bool):
+    """Add --source-csv and/or --target-csv to a command, depending on flags."""
+
+    def decorator(f):
+        if allow_target_csv:
+            f = click.option(
+                "--target-csv",
+                "target_csv",
+                type=click.Path(dir_okay=False, writable=True),
+                default=None,
+                help=(
+                    "Write fetched locations to this CSV file instead of POSTing "
+                    "to a target CDA. Mutually exclusive with --target-cda."
+                ),
+            )(f)
+        if allow_source_csv:
+            f = click.option(
+                "--source-csv",
+                "source_csv",
+                type=click.Path(exists=True, dir_okay=False, readable=True),
+                default=None,
+                help=(
+                    "Read locations from this CSV file instead of fetching from "
+                    "a source CDA. Mutually exclusive with --source-cda."
+                ),
+            )(f)
+        return f
+
+    return decorator
+
+
 @click.group(
     name="load",
     help="Load data from one CWMS Data API instance to another.",

{cwms_cli-0.6.0 → cwms_cli-0.7.0}/pyproject.toml

@@ -2,7 +2,7 @@
 name = "cwms-cli"
 repository = "https://github.com/HydrologicEngineeringCenter/cwms-cli"
 
-version = "0.6.0"
+version = "0.7.0"
 
 
 packages = [

cwms_cli-0.6.0/cwmscli/load/location/location_ids.py

@@ -1,105 +0,0 @@
-import logging
-import re
-from typing import Iterable, Optional
-
-import click
-import cwms
-
-from cwmscli.utils import init_cwms_session
-from cwmscli.utils.links import CDA_REGEXP_GUIDE_URL
-
-logger = logging.getLogger(__name__)
-
-
-def load_locations(
-    source_cda: str,
-    source_office: str,
-    target_cda: str,
-    target_api_key: Optional[str],
-    verbose: int,
-    dry_run: bool,
-    like: Optional[str],
-    location_kind_like: Optional[Iterable[str]] = "ALL",
-):
-    if verbose:
-        logger.info(
-            f"[load locations] source={source_cda} ({source_office}) -> target={target_cda}"
-        )
-        logger.info(
-            f"  like={like or '-'}  kinds={list(location_kind_like) or '-'}  dry_run={dry_run}"
-        )
-        if like or (
-            location_kind_like
-            and list(location_kind_like) != ["ALL"]
-            and list(location_kind_like) != []
-        ):
-            logger.info("  CDA regex guide: %s", CDA_REGEXP_GUIDE_URL)
-
-    init_cwms_session(cwms, api_root=source_cda)
-
-    cat_kwargs = {"office_id": source_office}
-    if like:
-        cat_kwargs["like"] = like
-    kinds = list(location_kind_like) if location_kind_like else ["ALL"]
-    if "ALL" in kinds:
-        kinds = ["ALL"]
-
-    locations = []
-
-    if kinds == ["ALL"] and not like:
-        locations = cwms.get_locations(office_id=source_office).json
-    else:
-        seen_location_ids = set()
-        for kind in kinds:
-            cat_kwargs_k = dict(cat_kwargs)
-            if kind != "ALL":
-                cat_kwargs_k["location_kind_like"] = kind
-
-            if verbose >= 2:
-                logger.debug("  > catalog query: %s", cat_kwargs_k)
-
-            resp = cwms.get_locations_catalog(**cat_kwargs_k)
-            if resp.df.empty:
-                continue
-
-            for location_id in resp.df["name"].tolist():
-                if location_id in seen_location_ids:
-                    continue
-                seen_location_ids.add(location_id)
-                if verbose >= 2:
-                    logger.debug("  > location fetch: %s", location_id)
-                detail_resp = cwms.get_locations(
-                    office_id=source_office,
-                    location_ids=rf"^{re.escape(location_id)}$",
-                )
-                if detail_resp and detail_resp.json:
-                    locations.extend(detail_resp.json)
-
-    if verbose:
-        logger.info("Fetched %s locations from source", len(locations))
-
-    if dry_run:
-        for loc in locations:
-            logger.info(
-                f"[dry-run] would store Location(name={loc['name']}) to {target_cda} ({source_office})"
-            )
-        return
-
-    # init target once
-    init_cwms_session(cwms, api_root=target_cda, api_key=target_api_key)
-
-    errors = 0
-    for loc in locations:
-        try:
-            if loc["active"] is True:
-                result = cwms.store_location(data=loc, fail_if_exists=False)
-                if verbose:
-                    logger.info("%s", result)
-        except Exception as e:
-            errors += 1
-            click.echo(f"Error storing location {loc}: \n\t{e}", err=True)
-
-    if errors:
-        raise click.ClickException(f"Completed with {errors} error(s).")
-
-    click.echo("Done.")