cwms-python 1.0.3__tar.gz → 1.0.7__tar.gz

{cwms_python-1.0.3 → cwms_python-1.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cwms-python
-Version: 1.0.3
+Version: 1.0.7
 Summary: Corps water management systems (CWMS) REST API for Data Retrieval of USACE water data
 License: LICENSE
 License-File: LICENSE
@@ -44,6 +44,24 @@ Then import the package:
 import cwms
 ```
 
+### Authentication
+
+`cwms.init_session()` supports both CDA API keys and Keycloak access tokens.
+Use `api_key=` for the headless CDA API key flow, or `token=` for an OIDC access
+token such as one saved by [`cwms-cli login`]().
+
+```python
+import cwms
+
+cwms.init_session(
+    api_root="https://cwms-data.usace.army.mil/cwms-data/",
+    token="ACCESS_TOKEN",
+)
+```
+
+If both `token` and `api_key` are provided, `cwms-python` will use the token
+and log a warning.
+
 ## Getting Started
 
 ```python

{cwms_python-1.0.3 → cwms_python-1.0.7}/README.md

@@ -20,6 +20,24 @@ Then import the package:
 import cwms
 ```
 
+### Authentication
+
+`cwms.init_session()` supports both CDA API keys and Keycloak access tokens.
+Use `api_key=` for the headless CDA API key flow, or `token=` for an OIDC access
+token such as one saved by [`cwms-cli login`]().
+
+```python
+import cwms
+
+cwms.init_session(
+    api_root="https://cwms-data.usace.army.mil/cwms-data/",
+    token="ACCESS_TOKEN",
+)
+```
+
+If both `token` and `api_key` are provided, `cwms-python` will use the token
+and log a warning.
+
 ## Getting Started
 
 ```python

{cwms_python-1.0.3 → cwms_python-1.0.7}/cwms/api.py

@@ -5,9 +5,9 @@ functions should be used internally to interact with the API. The user should no
 interact with these directly.
 
 The `init_session()` function can be used to specify an alternative root URL, and to
-provide an authentication key (if required). If `init_session()` is not called, the
-default root URL (see `API_ROOT` below) will be used, and no authentication keys will be
-included when making API calls.
+provide an authentication key or bearer token (if required). If `init_session()` is not
+called, the default root URL (see `API_ROOT` below) will be used, and no authentication
+headers will be included when making API calls.
 
 Example: Initializing a session
 
@@ -17,6 +17,9 @@ Example: Initializing a session
     # Specify an alternate URL and an auth key
     init_session(api_root="https://example.com/cwms-data", api_key="API_KEY")
 
+    # Specify an alternate URL and an OIDC bearer token
+    init_session(api_root="https://example.com/cwms-data", token="ACCESS_TOKEN")
+
 Functions which make API calls that _may_ return a JSON response will return a `dict`
 containing the deserialized data. If the API response does not include data, an empty
 `dict` will be returned.
@@ -34,6 +37,7 @@ from json import JSONDecodeError
 from typing import Any, Optional, cast
 
 from requests import Response, adapters
+from requests.exceptions import RetryError as RequestsRetryError
 from requests_toolbelt import sessions  # type: ignore
 from requests_toolbelt.sessions import BaseUrlSession  # type: ignore
 from urllib3.util.retry import Retry
@@ -52,12 +56,12 @@ retry_strategy = Retry(
     status_forcelist=[
         403,
         429,
-        500,
         502,
         503,
         504,
     ],  # Example: also retry on these HTTP status codes
     allowed_methods=["GET", "PUT", "POST", "PATCH", "DELETE"],  # Methods to retry
+    raise_on_status=False,
 )
 SESSION = sessions.BaseUrlSession(base_url=API_ROOT)
 adapter = adapters.HTTPAdapter(
@@ -137,21 +141,46 @@ class PermissionError(ApiError):
     """Raised when the CDA request is not authorized for the current caller."""
 
 
+def _unwrap_retry_error(error: RequestsRetryError) -> Exception:
+    """Return the original retry cause when requests wraps it in RetryError."""
+
+    current: Exception = error
+    cause = error.__cause__
+    while isinstance(cause, Exception):
+        current = cause
+        cause = cause.__cause__
+
+    if current is error and error.args:
+        first_arg = error.args[0]
+        if isinstance(first_arg, Exception):
+            current = first_arg
+            reason = getattr(current, "reason", None)
+            while isinstance(reason, Exception):
+                current = reason
+                reason = getattr(current, "reason", None)
+
+    return current
+
+
 def init_session(
     *,
     api_root: Optional[str] = None,
     api_key: Optional[str] = None,
+    token: Optional[str] = None,
     pool_connections: int = 100,
 ) -> BaseUrlSession:
-    """Specify a root URL and authentication key for the CWMS Data API.
+    """Specify a root URL and authentication credentials for the CWMS Data API.
 
     This function can be used to change the root URL used when interacting with the CDA.
-    All API calls made after this function is called will use the specified URL. If an
-    authentication key is given it will be included in all future request headers.
+    All API calls made after this function is called will use the specified URL. If
+    authentication credentials are given they will be included in all future request
+    headers.
 
     Keyword Args:
         api_root (optional): The root URL for the CWMS Data API.
         api_key (optional): An authentication key.
+        token (optional): A Keycloak access token. If both token and api_key are
+            provided, token is used.
 
     Returns:
         Returns the updated session object.
@@ -169,7 +198,16 @@ def init_session(
             max_retries=retry_strategy,
         )
         SESSION.mount("https://", adapter)
-    if api_key:
+    if token:
+        if api_key:
+            logging.warning(
+                "Both token and api_key were provided to init_session(); using token for Authorization."
+            )
+        # Ensure we don't provide the bearer text twice
+        if token.lower().startswith("bearer "):
+            token = token[7:]
+        SESSION.headers.update({"Authorization": "Bearer " + token})
+    elif api_key:
         if api_key.startswith("apikey "):
             api_key = api_key.replace("apikey ", "")
         SESSION.headers.update({"Authorization": "apikey " + api_key})
@@ -292,11 +330,14 @@ def get(
     """
 
     headers = {"Accept": api_version_text(api_version)}
-    with SESSION.get(endpoint, params=params, headers=headers) as response:
-        if not response.ok:
-            logging.error(f"CDA Error: response={response}")
-            raise ApiError(response)
-        return _process_response(response)
+    try:
+        with SESSION.get(endpoint, params=params, headers=headers) as response:
+            if not response.ok:
+                logging.error(f"CDA Error: response={response}")
+                raise ApiError(response)
+            return _process_response(response)
+    except RequestsRetryError as error:
+        raise _unwrap_retry_error(error) from None
 
 
 def get_with_paging(
@@ -351,11 +392,16 @@ def _post_function(
     headers = {"accept": "*/*", "Content-Type": api_version_text(api_version)}
     if isinstance(data, dict) or isinstance(data, list):
         data = json.dumps(data)
-    with SESSION.post(endpoint, params=params, headers=headers, data=data) as response:
-        if not response.ok:
-            logging.error(f"CDA Error: response={response}")
-            raise ApiError(response)
-        return response
+    try:
+        with SESSION.post(
+            endpoint, params=params, headers=headers, data=data
+        ) as response:
+            if not response.ok:
+                logging.error(f"CDA Error: response={response}")
+                raise ApiError(response)
+            return response
+    except RequestsRetryError as error:
+        raise _unwrap_retry_error(error) from None
 
 
 def post(
@@ -445,10 +491,15 @@ def patch(
 
     if data and isinstance(data, dict) or isinstance(data, list):
         data = json.dumps(data)
-    with SESSION.patch(endpoint, params=params, headers=headers, data=data) as response:
-        if not response.ok:
-            logging.error(f"CDA Error: response={response}")
-            raise ApiError(response)
+    try:
+        with SESSION.patch(
+            endpoint, params=params, headers=headers, data=data
+        ) as response:
+            if not response.ok:
+                logging.error(f"CDA Error: response={response}")
+                raise ApiError(response)
+    except RequestsRetryError as error:
+        raise _unwrap_retry_error(error) from None
 
 
 def delete(
@@ -472,7 +523,10 @@ def delete(
     """
 
     headers = {"Accept": api_version_text(api_version)}
-    with SESSION.delete(endpoint, params=params, headers=headers) as response:
-        if not response.ok:
-            logging.error(f"CDA Error: response={response}")
-            raise ApiError(response)
+    try:
+        with SESSION.delete(endpoint, params=params, headers=headers) as response:
+            if not response.ok:
+                logging.error(f"CDA Error: response={response}")
+                raise ApiError(response)
+    except RequestsRetryError as error:
+        raise _unwrap_retry_error(error) from None

{cwms_python-1.0.3 → cwms_python-1.0.7}/cwms/catalog/blobs.py

@@ -1,9 +1,9 @@
 import base64
-from typing import Optional
+from typing import Any, Optional
 
 import cwms.api as api
 from cwms.cwms_types import JSON, Data
-from cwms.utils.checks import is_base64
+from cwms.utils.checks import has_invalid_chars, is_base64
 
 STORE_DICT = """data = {
     "office-id": "SWT",
@@ -14,6 +14,8 @@ STORE_DICT = """data = {
 }
 """
 
+IGNORED_ID = "ignored"
+
 
 def get_blob(blob_id: str, office_id: str) -> str:
     """Get a single BLOB (Binary Large Object).
@@ -29,8 +31,13 @@ def get_blob(blob_id: str, office_id: str) -> str:
         str: the value returned based on the content-type it was stored with as a string
     """
 
-    endpoint = f"blobs/{blob_id}"
-    params = {"office": office_id}
+    params: dict[str, Any] = {}
+    if has_invalid_chars(blob_id):
+        endpoint = f"blobs/{IGNORED_ID}"
+        params["blob-id"] = blob_id
+    else:
+        endpoint = f"blobs/{blob_id}"
+    params["office"] = office_id
     response = api.get(endpoint, params, api_version=1)
     return str(response)
 
@@ -107,8 +114,13 @@ def delete_blob(blob_id: str, office_id: str) -> None:
         None
     """
 
-    endpoint = f"blobs/{blob_id}"
-    params = {"office": office_id}
+    params: dict[str, Any] = {}
+    if has_invalid_chars(blob_id):
+        endpoint = f"blobs/{IGNORED_ID}"
+        params["blob-id"] = blob_id
+    else:
+        endpoint = f"blobs/{blob_id}"
+    params["office"] = office_id
     return api.delete(endpoint, params, api_version=1)
 
 
@@ -143,6 +155,11 @@ def update_blob(data: JSON, fail_if_not_exists: Optional[bool] = True) -> None:
 
     blob_id = data.get("id", "").upper()
 
-    endpoint = f"blobs/{blob_id}"
-    params = {"fail-if-not-exists": fail_if_not_exists}
+    params: dict[str, Any] = {}
+    if has_invalid_chars(blob_id):
+        endpoint = f"blobs/{IGNORED_ID}"
+        params["blob-id"] = blob_id
+    else:
+        endpoint = f"blobs/{blob_id}"
+    params["fail-if-not-exists"] = fail_if_not_exists
     return api.patch(endpoint, data, params, api_version=1)

{cwms_python-1.0.3 → cwms_python-1.0.7}/cwms/catalog/catalog.py

@@ -67,7 +67,9 @@ def get_locations_catalog(
         "location-kind-like": location_kind_like,
     }
 
-    response = api.get(endpoint=endpoint, params=params, api_version=2)
+    response = api.get_with_paging(
+        endpoint=endpoint, selector="entries", params=params, api_version=2
+    )
     return Data(response, selector="entries")
 
 
@@ -131,7 +133,9 @@ def get_timeseries_catalog(
         "include-extents": include_extents,
     }
 
-    response = api.get(endpoint=endpoint, params=params, api_version=2)
+    response = api.get_with_paging(
+        endpoint=endpoint, selector="entries", params=params, api_version=2
+    )
     return Data(response, selector="entries")
 
 

{cwms_python-1.0.3 → cwms_python-1.0.7}/cwms/catalog/clobs.py

@@ -1,10 +1,21 @@
-from typing import Optional
+from typing import Any, Optional
 
 import cwms.api as api
 from cwms.cwms_types import JSON, Data
+from cwms.utils.checks import has_invalid_chars
 
+STORE_DICT = """data = {
+    "office-id": "SWT",
+    "id": "CLOB_ID",
+    "description": "Your description here",
+    "value": "STRING of content"
+}
+"""
 
-def get_clob(clob_id: str, office_id: str, clob_id_query: Optional[str] = None) -> Data:
+IGNORED_ID = "ignored"
+
+
+def get_clob(clob_id: str, office_id: str) -> Data:
     """Get a single clob.
 
     Parameters
@@ -13,16 +24,6 @@ def get_clob(clob_id: str, office_id: str, clob_id_query: Optional[str] = None)
                 Specifies the id of the clob
             office_id: string
                 Specifies the office of the clob.
-            clob_id_query: string
-                If this query parameter is provided the id path parameter is ignored and the
-                value of the query parameter is used. Note: this query parameter is necessary
-                for id's that contain '/' or other special characters. Because of abuse even
-                properly escaped '/' in url paths are blocked. When using this query parameter
-                a valid path parameter must still be provided for the request to be properly
-                routed. If your clob id contains '/' you can't specify the clob-id query
-                parameter and also specify the id path parameter because firewall and/or server
-                rules will deny the request even though you are specifying this override. "ignored"
-                is suggested.
 
 
         Returns
@@ -30,11 +31,13 @@ def get_clob(clob_id: str, office_id: str, clob_id_query: Optional[str] = None)
             cwms data type.  data.json will return the JSON output and data.df will return a dataframe
     """
 
-    endpoint = f"clobs/{clob_id}"
-    params = {
-        "office": office_id,
-        "clob-id-query": clob_id_query,
-    }
+    params: dict[str, Any] = {}
+    if has_invalid_chars(clob_id):
+        endpoint = f"clobs/{IGNORED_ID}"
+        params["clob-id"] = clob_id
+    else:
+        endpoint = f"clobs/{clob_id}"
+    params["office"] = office_id
     response = api.get(endpoint, params)
     return Data(response)
 
@@ -90,13 +93,20 @@ def delete_clob(clob_id: str, office_id: str) -> None:
             None
     """
 
-    endpoint = f"clobs/{clob_id}"
-    params = {"office": office_id}
+    params: dict[str, Any] = {}
+    if has_invalid_chars(clob_id):
+        endpoint = f"clobs/{IGNORED_ID}"
+        params["clob-id"] = clob_id
+    else:
+        endpoint = f"clobs/{clob_id}"
+    params["office"] = office_id
 
     return api.delete(endpoint, params=params, api_version=1)
 
 
-def update_clob(data: JSON, clob_id: str, ignore_nulls: Optional[bool] = True) -> None:
+def update_clob(
+    data: JSON, clob_id: Optional[str] = None, ignore_nulls: Optional[bool] = True
+) -> None:
     """Updates clob
 
     Parameters
@@ -110,7 +120,7 @@ def update_clob(data: JSON, clob_id: str, ignore_nulls: Optional[bool] = True) -
                         "value": "string"
                     }
             clob_id: string
-                Specifies the id of the clob to be deleted
+                Specifies the id of the clob to be deleted. Unused if "id" is present in JSON data.
             ignore_nulls: Boolean
                 If true, null and empty fields in the provided clob will be ignored and the existing value of those fields left in place. Default: true
 
@@ -122,8 +132,19 @@ def update_clob(data: JSON, clob_id: str, ignore_nulls: Optional[bool] = True) -
     if not isinstance(data, dict):
         raise ValueError("Cannot store a Clob without a JSON data dictionary")
 
-    endpoint = f"clobs/{clob_id}"
-    params = {"ignore-nulls": ignore_nulls}
+    if "id" in data:
+        clob_id = data.get("id", "").upper()
+
+    if clob_id is None:
+        raise ValueError(f"Cannot update a Clob without an 'id' field:\n{STORE_DICT}")
+
+    params: dict[str, Any] = {}
+    if has_invalid_chars(clob_id):
+        endpoint = f"clobs/{IGNORED_ID}"
+        params["clob-id"] = clob_id
+    else:
+        endpoint = f"clobs/{clob_id}"
+    params["ignore-nulls"] = ignore_nulls
 
     return api.patch(endpoint, data, params, api_version=1)
 

{cwms_python-1.0.3 → cwms_python-1.0.7}/cwms/timeseries/timeseries.py

@@ -233,6 +233,14 @@ def combine_timeseries_results(results: List[Data]) -> Data:
     )
     combined_df["date-time"] = combined_df["date-time"].astype("Int64")
     combined_df = combined_df.reindex(columns=["date-time", "value", "quality-code"])
+
+    # Replace NaN in value column with None so they serialize as JSON null
+    # rather than the invalid JSON literal NaN.
+    combined_df["value"] = (
+        combined_df["value"]
+        .astype(object)
+        .where(combined_df["value"].notna(), other=None)
+    )
     # Update the "values" key in the JSON to include the combined data
     combined_json["values"] = combined_df.values.tolist()
 
@@ -438,8 +446,11 @@ def timeseries_df_to_json(
         pd.Timestamp.isoformat
     )
     df = df.reindex(columns=["date-time", "value", "quality-code"])
-    if df.isnull().values.any():
-        raise ValueError("Null/NaN data must be removed from the dataframe")
+
+    # Replace NaN/NA/NaT in value column with None so they serialize as JSON
+    # null rather than the invalid JSON literal NaN.
+    df["value"] = df["value"].astype(object).where(df["value"].notna(), other=None)
+
     if version_date:
         version_date_iso = version_date.isoformat()
     else:
@@ -458,6 +469,10 @@ def timeseries_df_to_json(
 def store_multi_timeseries_df(
     data: pd.DataFrame,
     office_id: str,
+    create_as_ltrs: Optional[bool] = False,
+    store_rule: Optional[str] = None,
+    override_protection: Optional[bool] = False,
+    multithread: Optional[bool] = True,
     max_workers: Optional[int] = 30,
 ) -> None:
     """stored mulitple timeseries from a dataframe.  The dataframe must be a metled dataframe with columns
@@ -478,6 +493,19 @@ def store_multi_timeseries_df(
                 2   2023-12-20T15:15:00.000-05:00  98.5           0   OMA.Stage.Inst.6Hours.0.Fcst-MRBWM-GRFT ft     2024-04-22 07:15:00-05:00
         office_id: string
             The owning office of the time series(s).
+        create_as_ltrs: bool, optional, default is False
+            Flag indicating if timeseries should be created as Local Regular Time Series.
+        store_rule: str, optional, default is None:
+            The business rule to use when merging the incoming with existing data. Available values :
+                REPLACE_ALL,
+                DO_NOT_REPLACE,
+                REPLACE_MISSING_VALUES_ONLY,
+                REPLACE_WITH_NON_MISSING,
+                DELETE_INSERT.
+        override_protection: bool, optional, default is False
+            A flag to ignore the protected data quality flag when storing data.
+        multithread: bool, default is false
+            Specifies whether to store chunked time series values using multiple threads.
         max_workers: Int, Optional, default is None
             It is a number of Threads aka size of pool in concurrent.futures.ThreadPoolExecutor.
 
@@ -491,7 +519,6 @@ def store_multi_timeseries_df(
         ts_id: str,
         office_id: str,
         version_date: Optional[datetime] = None,
-        multithread: bool = False,
     ) -> None:
         try:
             units = data["units"].iloc[0]
@@ -502,7 +529,13 @@ def store_multi_timeseries_df(
                 office_id=office_id,
                 version_date=version_date,
             )
-            store_timeseries(data=data_json, multithread=multithread)
+            store_timeseries(
+                data=data_json,
+                create_as_ltrs=create_as_ltrs,
+                store_rule=store_rule,
+                override_protection=override_protection,
+                multithread=multithread,
+            )
         except Exception as e:
             print(f"Error processing {ts_id}: {e}")
         return None

{cwms_python-1.0.3 → cwms_python-1.0.7}/cwms/users/users.py

@@ -35,18 +35,66 @@ def get_user_profile() -> dict[str, Any]:
     return dict(response)
 
 
+def filter_users_by_office(data: dict[str, Any], office: str) -> dict[str, Any]:
+    """
+    Filter users JSON to only include users that have roles for the specified office.
+    Each user's roles dict will only contain the entry for that office.
+
+    Args:
+        data:   The full users JSON as a Python dict.
+        office: The office key to filter by (e.g., 'MVP', 'LRL').
+
+    Returns:
+        A new dict with the same structure, filtered to the specified office.
+    """
+    filtered_users = []
+
+    for user in data.get("users", []):
+        roles = user.get("roles", {})
+
+        if office in roles:
+            # Build a copy of the user with only the target office's roles
+            filtered_user = {k: v for k, v in user.items() if k != "roles"}
+            filtered_user["roles"] = {office: roles[office]}
+            filtered_users.append(filtered_user)
+
+    return {
+        "page": data.get("page"),
+        "page-size": data.get("page-size"),
+        "total": len(filtered_users),
+        "users": filtered_users,
+    }
+
+
 def get_users(
     office_id: Optional[str] = None,
+    username_like: Optional[str] = None,
+    include_roles: Optional[bool] = None,
     page: Optional[str] = None,
-    page_size: Optional[int] = None,
+    page_size: Optional[int] = 5000,
 ) -> Data:
     """Retrieve users with optional office and paging filters."""
 
-    params = {"office": office_id, "page": page, "page-size": page_size}
+    endpoint = "users"
+    params = {
+        "office": office_id,
+        "username-like": username_like,
+        "include-roles": include_roles,
+        "page": page,
+        "page-size": page_size,
+    }
     try:
-        response = api.get("users", params=params, api_version=1)
+        response = api.get_with_paging(
+            endpoint=endpoint, selector="users", params=params, api_version=1
+        )
     except api.ApiError as error:
         _raise_user_management_error(error, "User list lookup")
+
+    # filter by office if office_id is provided since the API does not
+    # currently support filtering by office on the backend. This is a
+    # temporary workaround until the API supports office filtering.
+    if office_id:
+        response = filter_users_by_office(response, office_id)
     return Data(response, selector="users")
 
 

{cwms_python-1.0.3 → cwms_python-1.0.7}/cwms/utils/checks.py

@@ -8,3 +8,15 @@ def is_base64(s: str) -> bool:
         return base64.b64encode(decoded).decode("utf-8") == s
     except (ValueError, TypeError):
         return False
+
+
+def has_invalid_chars(id: str) -> bool:
+    """
+    Checks if ID contains any invalid web path characters.
+    """
+    INVALID_PATH_CHARS = ["/", "\\", "&", "?", "="]
+
+    for char in INVALID_PATH_CHARS:
+        if char in id:
+            return True
+    return False

{cwms_python-1.0.3 → cwms_python-1.0.7}/pyproject.toml

@@ -2,8 +2,7 @@
 name = "cwms-python"
 repository = "https://github.com/HydrologicEngineeringCenter/cwms-python"
 
-version = "1.0.3"
-
+version = "1.0.7"
 
 packages = [
     { include = "cwms" },