cwms-cli 0.4.0__tar.gz → 0.5.0__tar.gz

{cwms_cli-0.4.0 → cwms_cli-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cwms-cli
-Version: 0.4.0
+Version: 0.5.0
 Summary: Command line utilities for Corps Water Management Systems (CWMS) python scripts. This is a collection of shared scripts across the enterprise Water Management Enterprise System (WMES) teams.
 License: LICENSE
 License-File: LICENSE

{cwms_cli-0.4.0 → cwms_cli-0.5.0}/cwmscli/__main__.py

@@ -85,6 +85,7 @@ cli.add_command(commands_cwms.shefcritimport)
 cli.add_command(commands_cwms.csv2cwms_cmd)
 cli.add_command(commands_cwms.update_cli_cmd)
 cli.add_command(commands_cwms.blob_group)
+cli.add_command(commands_cwms.clob_group)
 cli.add_command(commands_cwms.users_group)
 cli.add_command(load.load_group)
 add_version_to_help_tree(cli)

cwms_cli-0.5.0/cwmscli/_generated/__init__.py

@@ -0,0 +1 @@
+# Generated files used at runtime and in docs.

cwms_cli-0.5.0/cwmscli/_generated/ownership_data.py

@@ -0,0 +1,57 @@
+# DO NOT EDIT THIS FILE MANUALLY.
+# Generated from maintainers.toml by scripts/sync_ownership.py.
+
+OWNERSHIP_DATA = {
+    "commands": {
+        "cwms-cli blob": [
+            {
+                "email": "charles.r.graham@usace.army.mil",
+                "name": "Charles Graham"
+            }
+        ],
+        "cwms-cli csv2cwms": [
+            {
+                "email": "charles.r.graham@usace.army.mil",
+                "name": "Charles Graham"
+            }
+        ],
+        "cwms-cli load": [
+            {
+                "email": "charles.r.graham@usace.army.mil",
+                "name": "Charles Graham"
+            },
+            {
+                "email": "eric.v.novotny@usace.army.mil",
+                "name": "Eric Novotny"
+            }
+        ],
+        "cwms-cli shefcritimport": [
+            {
+                "email": "eric.v.novotny@usace.army.mil",
+                "name": "Eric Novotny"
+            }
+        ],
+        "cwms-cli update": [
+            {
+                "email": "charles.r.graham@usace.army.mil",
+                "name": "Charles Graham"
+            }
+        ],
+        "cwms-cli usgs": [
+            {
+                "email": "eric.v.novotny@usace.army.mil",
+                "name": "Eric Novotny"
+            }
+        ]
+    },
+    "default": [
+        {
+            "email": "charles.r.graham@usace.army.mil",
+            "name": "Charles Graham"
+        },
+        {
+            "email": "eric.v.novotny@usace.army.mil",
+            "name": "Eric Novotny"
+        }
+    ]
+}

{cwms_cli-0.4.0 → cwms_cli-0.5.0}/cwmscli/commands/blob.py

@@ -7,7 +7,7 @@ import os
 import re
 import sys
 from collections import defaultdict
-from typing import Optional, Sequence, Tuple
+from typing import Optional, Sequence, Tuple, Union
 
 from cwmscli.utils import colors, get_api_key, log_scoped_read_hint
 from cwmscli.utils.click_help import DOCS_BASE_URL
@@ -64,12 +64,12 @@ def _looks_like_base64(raw: str) -> bool:
 
 
 def _save_blob_content(
-    content: bytes | str,
+    content: Union[bytes, str],
     dest: str,
     media_type_hint: Optional[str] = None,
 ) -> str:
     media_type = media_type_hint
-    data: bytes | str = content
+    data: Union[bytes, str] = content
 
     if isinstance(content, str):
         m = DATA_URL_RE.match(content.strip())
@@ -260,12 +260,19 @@ def list_blobs(
     sort_by: Optional[Sequence[str]] = None,
     ascending: bool = True,
     limit: Optional[int] = None,
+    page_size: Optional[int] = None,
 ):
     logging.info(f"Listing blobs for office: {office!r}...")
     import cwms
     import pandas as pd
 
-    result = cwms.get_blobs(office_id=office, blob_id_like=blob_id_like)
+    # Use page size if it's provided per #184
+    fetch_page_size = page_size if page_size is not None else limit
+    result = cwms.get_blobs(
+        office_id=office,
+        blob_id_like=blob_id_like,
+        page_size=fetch_page_size,
+    )
 
     # Accept either a DataFrame or a JSON/dict-like response
     if isinstance(result, pd.DataFrame):
@@ -678,6 +685,7 @@ def list_cmd(
     sort_by: list[str],
     desc: bool,
     limit: int,
+    page_size: int,
     to_csv: str,
     office: str,
     api_root: str,
@@ -697,6 +705,7 @@ def list_cmd(
             sort_by=sort_by,
             ascending=not desc,
             limit=limit,
+            page_size=page_size,
         )
     except Exception:
         log_scoped_read_hint(
@@ -713,4 +722,12 @@ def list_cmd(
     else:
         # Friendly console preview
         with pd.option_context("display.max_rows", 500, "display.max_columns", None):
-            logging.info(df.to_string(index=False))
+            # Left-align all columns
+            logging.info(
+                "\n"
+                + df.apply(
+                    lambda s: (s := s.astype(str).str.strip()).str.ljust(
+                        s.str.len().max()
+                    )
+                ).to_string(index=False, justify="left")
+            )

cwms_cli-0.5.0/cwmscli/commands/clob.py

@@ -0,0 +1,340 @@
+import json
+import logging
+import os
+import sys
+from typing import Optional, Sequence
+
+import cwms
+import pandas as pd
+import requests
+from cwms import api as cwms_api
+
+from cwmscli.utils import get_api_key, has_invalid_chars, log_scoped_read_hint
+
+
+def _join_api_url(api_root: str, path: str) -> str:
+    return f"{api_root.rstrip('/')}/{path.lstrip('/')}"
+
+
+def _resolve_optional_api_key(api_key: Optional[str], anonymous: bool) -> Optional[str]:
+    if anonymous or not api_key:
+        return None
+    return get_api_key(api_key, None)
+
+
+def _write_clob_content(content: str, dest: str) -> str:
+    os.makedirs(os.path.dirname(dest) or ".", exist_ok=True)
+    with open(dest, "w", encoding="utf-8", newline="") as f:
+        f.write(content)
+    return dest
+
+
+def _clob_endpoint_id(clob_id: str) -> tuple[str, Optional[str]]:
+    normalized = clob_id.upper()
+    if has_invalid_chars(normalized):
+        return "ignored", normalized
+    return normalized, None
+
+
+def _get_special_clob_text(*, office: str, clob_id: str) -> str:
+    with cwms_api.SESSION.get(
+        "clobs/ignored",
+        params={"office": office, "clob-id": clob_id},
+        headers={"Accept": "text/plain"},
+    ) as response:
+        response.raise_for_status()
+        return response.text
+
+
+def list_clobs(
+    office: Optional[str] = None,
+    clob_id_like: Optional[str] = None,
+    columns: Optional[Sequence[str]] = None,
+    sort_by: Optional[Sequence[str]] = None,
+    ascending: bool = True,
+    limit: Optional[int] = None,
+    page_size: Optional[int] = None,
+) -> pd.DataFrame:
+    logging.info(f"Listing clobs for office: {office!r}...")
+    fetch_page_size = page_size if page_size is not None else limit
+    result = cwms.get_clobs(
+        office_id=office,
+        clob_id_like=clob_id_like,
+        page_size=fetch_page_size,
+    )
+
+    # Accept either a DataFrame or a JSON/dict-like response
+    if isinstance(result, pd.DataFrame):
+        df = result.copy()
+    else:
+        # Expecting normal clob return structure
+        data = getattr(result, "json", None)
+        if callable(data):
+            data = result.json()
+        df = pd.DataFrame((data or {}).get("clobs", []))
+
+    # Allow column filtering
+    if columns:
+        keep = [c for c in columns if c in df.columns]
+        if keep:
+            df = df[keep]
+
+    # Sort by option
+    if sort_by:
+        by = [c for c in sort_by if c in df.columns]
+        if by:
+            df = df.sort_values(by=by, ascending=ascending, kind="stable")
+
+    # Optional limit
+    if limit is not None:
+        df = df.head(limit)
+
+    logging.info(f"Found {len(df):,} clob(s)")
+    # List the clobs in the logger
+    for _, row in df.iterrows():
+        logging.info(f"clob ID: {row['id']}, Description: {row.get('description')}")
+    return df
+
+
+def upload_cmd(
+    input_file: str,
+    clob_id: str,
+    description: str,
+    overwrite: bool,
+    dry_run: bool,
+    office: str,
+    api_root: str,
+    api_key: str,
+):
+    cwms.init_session(api_root=api_root, api_key=get_api_key(api_key, None))
+    try:
+        file_size = os.path.getsize(input_file)
+        with open(input_file, "r", encoding="utf-8") as f:
+            file_data = f.read()
+        logging.info(f"Read file: {input_file} ({file_size} bytes)")
+    except Exception as e:
+        logging.error(f"Failed to read file: {e}")
+        sys.exit(1)
+
+    clob_id_up = clob_id.upper()
+    logging.debug(f"Office={office} clobID={clob_id_up}")
+
+    clob = {
+        "office-id": office,
+        "id": clob_id_up,
+        "description": (
+            json.dumps(description)
+            if isinstance(description, (dict, list))
+            else description
+        ),
+        "value": file_data,
+    }
+    params = {"fail-if-exists": not overwrite}
+    view_url = _join_api_url(api_root, f"clobs/{clob_id_up}?office={office}")
+
+    if dry_run:
+        logging.info(
+            f"DRY RUN: would POST {_join_api_url(api_root, 'clobs')} with params={params}"
+        )
+        logging.info(
+            json.dumps(
+                {
+                    "url": _join_api_url(api_root, "clobs"),
+                    "params": params,
+                    "clob": {**clob, "value": f'<{len(clob["value"])} chars>'},
+                },
+                indent=2,
+            )
+        )
+        return
+
+    try:
+        cwms.store_clobs(clob, fail_if_exists=not overwrite)
+        logging.info(f"Uploaded clob: {clob_id_up}")
+        if has_invalid_chars(clob_id_up):
+            logging.info(
+                f"View: {_join_api_url(api_root, f'clobs/ignored?clob-id={clob_id_up}&office={office}')}"
+            )
+        else:
+            logging.info(f"View: {view_url}")
+    except requests.HTTPError as e:
+        detail = getattr(e.response, "text", "") or str(e)
+        logging.error(f"Failed to upload (HTTP): {detail}")
+        sys.exit(1)
+    except Exception as e:
+        logging.error(f"Failed to upload: {e}")
+        sys.exit(1)
+
+
+def download_cmd(
+    clob_id: str,
+    dest: str,
+    office: str,
+    api_root: str,
+    api_key: str,
+    dry_run: bool,
+    anonymous: bool = False,
+):
+    if dry_run:
+        logging.info(
+            f"DRY RUN: would GET {api_root} clob with clob-id={clob_id} office={office}."
+        )
+        return
+    resolved_api_key = _resolve_optional_api_key(api_key, anonymous)
+    cwms.init_session(api_root=api_root, api_key=resolved_api_key)
+    bid = clob_id.upper()
+    logging.debug(f"Office={office} clobID={bid}")
+
+    try:
+        path_id, query_id = _clob_endpoint_id(bid)
+        if query_id is None:
+            clob = cwms.get_clob(office_id=office, clob_id=path_id)
+            payload = getattr(clob, "json", clob)
+            if callable(payload):
+                payload = payload()
+            if isinstance(payload, dict):
+                content = payload.get("value", "")
+            else:
+                content = str(payload)
+        else:
+            content = _get_special_clob_text(office=office, clob_id=query_id)
+        target = dest or bid
+        _write_clob_content(content, target)
+        logging.info(f"Downloaded clob to: {target}")
+    except requests.HTTPError as e:
+        detail = getattr(e.response, "text", "") or str(e)
+        logging.error(f"Failed to download (HTTP): {detail}")
+        log_scoped_read_hint(
+            api_key=resolved_api_key,
+            anonymous=anonymous,
+            office=office,
+            action="download",
+            resource="clob content",
+        )
+        sys.exit(1)
+    except Exception as e:
+        logging.error(f"Failed to download: {e}")
+        log_scoped_read_hint(
+            api_key=resolved_api_key,
+            anonymous=anonymous,
+            office=office,
+            action="download",
+            resource="clob content",
+        )
+        sys.exit(1)
+
+
+def delete_cmd(clob_id: str, office: str, api_root: str, api_key: str, dry_run: bool):
+
+    if dry_run:
+        logging.info(
+            f"DRY RUN: would DELETE {api_root} clob with clob-id={clob_id} office={office}"
+        )
+        return
+    cwms.init_session(api_root=api_root, api_key=get_api_key(api_key, None))
+    cid = clob_id.upper()
+    path_id, query_id = _clob_endpoint_id(cid)
+    if query_id is None:
+        cwms.delete_clob(office_id=office, clob_id=cid)
+    else:
+        cwms_api.delete(
+            f"clobs/{path_id}", params={"office": office, "clob-id": query_id}
+        )
+    logging.info(f"Deleted clob: {clob_id} for office: {office}")
+
+
+def update_cmd(
+    input_file: str,
+    clob_id: str,
+    description: str,
+    ignore_nulls: bool,
+    dry_run: bool,
+    office: str,
+    api_root: str,
+    api_key: str,
+):
+    if dry_run:
+        logging.info(
+            f"DRY RUN: would PATCH {api_root} clob with clob-id={clob_id} office={office}"
+        )
+        return
+    file_data = None
+    if input_file:
+        try:
+            file_size = os.path.getsize(input_file)
+            with open(input_file, "r", encoding="utf-8") as f:
+                file_data = f.read()
+            logging.info(f"Read file: {input_file} ({file_size} bytes)")
+        except Exception as e:
+            logging.error(f"Failed to read file: {e}")
+            sys.exit(1)
+    # Setup minimum required payload
+    clob = {"office-id": office, "id": clob_id.upper()}
+    if description:
+        clob["description"] = description
+
+    if file_data:
+        clob["value"] = file_data
+    cwms.init_session(api_root=api_root, api_key=get_api_key(api_key, None))
+    cid = clob_id.upper()
+    path_id, query_id = _clob_endpoint_id(cid)
+    if query_id is None:
+        cwms.update_clob(clob, cid, ignore_nulls=ignore_nulls)
+    else:
+        cwms_api.patch(
+            f"clobs/{path_id}",
+            data=clob,
+            params={"clob-id": query_id, "ignore-nulls": ignore_nulls},
+        )
+    logging.info(f"Updated clob: {clob_id} for office: {office}")
+
+
+def list_cmd(
+    clob_id_like: str,
+    columns: list[str],
+    sort_by: list[str],
+    desc: bool,
+    limit: int,
+    page_size: int,
+    to_csv: str,
+    office: str,
+    api_root: str,
+    api_key: str,
+    anonymous: bool = False,
+):
+    resolved_api_key = _resolve_optional_api_key(api_key, anonymous)
+    cwms.init_session(api_root=api_root, api_key=resolved_api_key)
+    try:
+        df = list_clobs(
+            office=office,
+            clob_id_like=clob_id_like,
+            columns=columns,
+            sort_by=sort_by,
+            ascending=not desc,
+            limit=limit,
+            page_size=page_size,
+        )
+    except Exception:
+        log_scoped_read_hint(
+            api_key=resolved_api_key,
+            anonymous=anonymous,
+            office=office,
+            action="list",
+            resource="clob content",
+        )
+        raise
+    if to_csv:
+        df.to_csv(to_csv, index=False)
+        logging.info(f"Wrote {len(df)} rows to {to_csv}")
+    else:
+        # Friendly console preview
+        with pd.option_context("display.max_rows", 500, "display.max_columns", None):
+            # Left-align all columns
+            logging.info(
+                "\n"
+                + df.apply(
+                    lambda s: (s := s.astype(str).str.strip()).str.ljust(
+                        s.str.len().max()
+                    )
+                ).to_string(index=False, justify="left")
+            )

{cwms_cli-0.4.0 → cwms_cli-0.5.0}/cwmscli/commands/commands_cwms.py

@@ -387,6 +387,12 @@ def update_cmd(**kwargs):
     help="Sort descending instead of ascending.",
 )
 @click.option("--limit", type=int, default=None, help="Max rows to show.")
+@click.option(
+    "--page-size",
+    type=int,
+    default=None,
+    help="Max rows to request from the blob endpoint. Defaults to --limit if set, otherwise no pagination (all results in one page).",
+)
 @click.option(
     "--to-csv",
     type=click.Path(dir_okay=False, writable=True, path_type=str),
@@ -405,6 +411,174 @@ def list_cmd(**kwargs):
     list_cmd(**kwargs)
 
 
+# endregion
+
+
+# region Clob
+# ================================================================================
+#  CLOB
+# ================================================================================
+@click.group(
+    "clob",
+    help="Manage CWMS Clobs (upload, download, delete, update, list)",
+    epilog=textwrap.dedent(
+        """
+    Example Usage:\n
+    - Download a clob by id to your local filesystem\n
+    - Update a clob's name/description/mime-type\n
+    - Bulk list clobs for an office  
+"""
+    ),
+)
+@requires(reqs.cwms)
+def clob_group():
+    pass
+
+
+# ================================================================================
+#       Upload
+# ================================================================================
+@clob_group.command("upload", help="Upload a file as a clob")
+@click.option(
+    "--input-file",
+    required=True,
+    type=click.Path(exists=True, dir_okay=False, readable=True, path_type=str),
+    help="Path to the file to upload.",
+)
+@click.option("--clob-id", required=True, type=str, help="Clob ID to create.")
+@click.option("--description", default=None, help="Optional description JSON or text.")
+@click.option(
+    "--overwrite/--no-overwrite",
+    default=False,
+    show_default=True,
+    help="If true, replace existing clob.",
+)
+@click.option("--dry-run", is_flag=True, help="Show request; do not send.")
+@common_api_options
+def clob_upload(**kwargs):
+    from cwmscli.commands.clob import upload_cmd
+
+    upload_cmd(**kwargs)
+
+
+# ================================================================================
+#       Download
+# ================================================================================
+@clob_group.command("download", help="Download a clob by ID")
+# TODO: test XML
+@click.option("--clob-id", required=True, type=str, help="Clob ID to download.")
+@click.option(
+    "--dest",
+    default=None,
+    help="Destination file path. Defaults to clob-id.",
+)
+@click.option(
+    "--anonymous",
+    is_flag=True,
+    help="Do not send credentials for this read request, even if they are configured.",
+)
+@click.option("--dry-run", is_flag=True, help="Show request; do not send.")
+@common_api_options
+def clob_download(**kwargs):
+    from cwmscli.commands.clob import download_cmd
+
+    download_cmd(**kwargs)
+
+
+# ================================================================================
+#       Delete
+# ================================================================================
+@clob_group.command("delete", help="Delete a clob by ID")
+@click.option("--clob-id", required=True, type=str, help="Clob ID to delete.")
+@click.option("--dry-run", is_flag=True, help="Show request; do not send.")
+@common_api_options
+def delete_cmd(**kwargs):
+    from cwmscli.commands.clob import delete_cmd
+
+    delete_cmd(**kwargs)
+
+
+# ================================================================================
+#       Update
+# ================================================================================
+@clob_group.command("update", help="Update/patch a clob by ID")
+@click.option("--clob-id", required=True, type=str, help="Clob ID to update.")
+@click.option("--dry-run", is_flag=True, help="Show request; do not send.")
+@click.option(
+    "--description",
+    default=None,
+    help="New description JSON or text.",
+)
+@click.option(
+    "--input-file",
+    required=False,
+    type=click.Path(exists=True, dir_okay=False, readable=True, path_type=str),
+    help="Optional file content to upload with update.",
+)
+@click.option(
+    "--ignore-nulls/--no-ignore-nulls",
+    default=True,
+    show_default=True,
+    help="If true, null and empty fields in the provided clob will be ignored and the existing value of those fields left in place.",
+)
+@common_api_options
+def update_cmd(**kwargs):
+    from cwmscli.commands.clob import update_cmd
+
+    update_cmd(**kwargs)
+
+
+# ================================================================================
+#       List
+# ================================================================================
+@clob_group.command("list", help="List clobs with optional filters and sorting")
+# TODO: Add link to regex docs when new CWMS-DATA site is deployed to PROD
+@click.option(
+    "--clob-id-like", help="LIKE filter for clob ID (e.g., ``*PNG``)."
+)  # Escape the wildcard/asterisk for RTD generation with double backticks
+@click.option(
+    "--columns",
+    multiple=True,
+    callback=csv_to_list,
+    help="Columns to show (repeat or comma-separate).",
+)
+@click.option(
+    "--sort-by",
+    multiple=True,
+    callback=csv_to_list,
+    help="Columns to sort by (repeat or comma-separate).",
+)
+@click.option(
+    "--desc/--asc",
+    default=False,
+    show_default=True,
+    help="Sort descending instead of ascending.",
+)
+@click.option("--limit", type=int, default=None, help="Max rows to show.")
+@click.option(
+    "--page-size",
+    type=int,
+    default=None,
+    help="Max rows to request from the clob endpoint. Defaults to --limit when set.",
+)
+@click.option(
+    "--to-csv",
+    type=click.Path(dir_okay=False, writable=True, path_type=str),
+    help="If set, write results to this CSV file.",
+)
+@click.option(
+    "--anonymous",
+    is_flag=True,
+    help="Do not send credentials for this read request, even if they are configured.",
+)
+@common_api_options
+def list_cmd(**kwargs):
+    from cwmscli.commands.clob import list_cmd
+
+    list_cmd(**kwargs)
+
+
+# endregion
 # ================================================================================
 #       USERS
 # ================================================================================