cwms-python 0.3.0__tar.gz → 0.5.0__tar.gz

{cwms_python-0.3.0 → cwms_python-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cwms-python
-Version: 0.3.0
+Version: 0.5.0
 Summary: Corps water managerment systems (CWMS) REST API for Data Retrieval of USACE water data
 License: LICENSE
 Keywords: USACE,water data
@@ -49,7 +49,7 @@ from datetime import datetime, timedelta
 
 end = datetime.now()
 begin = end - timedelta(days = 10)
-data = cwms.get_timeseries(p_tsId='Some.Fully.Qualified.Ts.Id',begin = begin, end = end)
+data = cwms.get_timeseries(ts_id='Some.Fully.Qualified.Ts.Id',office_id='OFFICE1' , begin = begin, end = end)
 
 #a cwms data object will be provided this object containes both the JSON as well
 #as the values converted into a dataframe

{cwms_python-0.3.0 → cwms_python-0.5.0}/README.md

@@ -28,7 +28,7 @@ from datetime import datetime, timedelta
 
 end = datetime.now()
 begin = end - timedelta(days = 10)
-data = cwms.get_timeseries(p_tsId='Some.Fully.Qualified.Ts.Id',begin = begin, end = end)
+data = cwms.get_timeseries(ts_id='Some.Fully.Qualified.Ts.Id',office_id='OFFICE1' , begin = begin, end = end)
 
 #a cwms data object will be provided this object containes both the JSON as well
 #as the values converted into a dataframe

cwms_python-0.5.0/cwms/__init__.py

@@ -0,0 +1,27 @@
+from importlib.metadata import PackageNotFoundError, version
+
+from cwms.api import *
+from cwms.catalog.catalog import *
+from cwms.forecast.forecast_instance import *
+from cwms.forecast.forecast_spec import *
+from cwms.levels.location_levels import *
+from cwms.levels.specified_levels import *
+from cwms.locations.physical_locations import *
+from cwms.outlets.outlets import *
+from cwms.outlets.virtual_outlets import *
+from cwms.projects.project_lock_rights import *
+from cwms.projects.project_locks import *
+from cwms.projects.projects import *
+from cwms.ratings.ratings import *
+from cwms.ratings.ratings_spec import *
+from cwms.ratings.ratings_template import *
+from cwms.standard_text.standard_text import *
+from cwms.timeseries.timerseries_identifier import *
+from cwms.timeseries.timeseries import *
+from cwms.timeseries.timeseries_bin import *
+from cwms.timeseries.timeseries_txt import *
+
+try:
+    __version__ = version("cwms-python")
+except PackageNotFoundError:
+    __version__ = "version-unknown"

{cwms_python-0.3.0 → cwms_python-0.5.0}/cwms/api.py

@@ -29,20 +29,22 @@ the error.
 import json
 import logging
 from json import JSONDecodeError
-from typing import Optional, cast
+from typing import Any, Optional, cast
 
-from requests import Response
+from requests import Response, adapters
 from requests_toolbelt import sessions  # type: ignore
 from requests_toolbelt.sessions import BaseUrlSession  # type: ignore
 
-from cwms.types import JSON, RequestParams
+from cwms.cwms_types import JSON, RequestParams
 
 # Specify the default API root URL and version.
 API_ROOT = "https://cwms-data.usace.army.mil/cwms-data/"
 API_VERSION = 2
 
-# Initialize a non-authenticated session with the default root URL.
+# Initialize a non-authenticated session with the default root URL and set default pool connections.
 SESSION = sessions.BaseUrlSession(base_url=API_ROOT)
+adapter = adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)
+SESSION.mount("https://", adapter)
 
 
 class InvalidVersion(Exception):
@@ -91,7 +93,10 @@ class ApiError(Exception):
 
 
 def init_session(
-    *, api_root: Optional[str] = None, api_key: Optional[str] = None
+    *,
+    api_root: Optional[str] = None,
+    api_key: Optional[str] = None,
+    pool_connections: int = 100,
 ) -> BaseUrlSession:
     """Specify a root URL and authentication key for the CWMS Data API.
 
@@ -112,7 +117,10 @@ def init_session(
     if api_root:
         logging.debug(f"Initializing root URL: api_root={api_root}")
         SESSION = sessions.BaseUrlSession(base_url=api_root)
-
+        adapter = adapters.HTTPAdapter(
+            pool_connections=pool_connections, pool_maxsize=pool_connections
+        )
+        SESSION.mount("https://", adapter)
     if api_key:
         logging.debug(f"Setting authorization key: api_key={api_key}")
         SESSION.headers.update({"Authorization": api_key})
@@ -150,12 +158,51 @@ def api_version_text(api_version: int) -> str:
         version = "application/json"
     elif api_version == 2:
         version = "application/json;version=2"
+    elif api_version == 102:
+        version = "application/xml;version=2"
     else:
         raise InvalidVersion(f"API version {api_version} is not supported.")
 
     return version
 
 
+def get_xml(
+    endpoint: str,
+    params: Optional[RequestParams] = None,
+    *,
+    api_version: int = API_VERSION,
+) -> Any:
+    """Make a GET request to the CWMS Data API.
+
+    Args:
+        endpoint: The CDA endpoint for the record(s).
+        params (optional): Query parameters for the request.
+
+    Keyword Args:
+        api_version (optional): The CDA version to use for the request. If not specified,
+            the default API_VERSION will be used.
+
+    Returns:
+        The deserialized JSON response data.
+
+    Raises:
+        ApiError: If an error response is return by the API.
+    """
+
+    headers = {"Accept": api_version_text(api_version)}
+    response = SESSION.get(endpoint, params=params, headers=headers)
+
+    if response.status_code < 200 or response.status_code >= 300:
+        logging.error(f"CDA Error: response={response}")
+        raise ApiError(response)
+
+    try:
+        return response.content.decode("utf-8")
+    except JSONDecodeError as error:
+        logging.error(f"Error decoding CDA response as xml: {error}")
+        return {}
+
+
 def get(
     endpoint: str,
     params: Optional[RequestParams] = None,
@@ -181,21 +228,60 @@ def get(
 
     headers = {"Accept": api_version_text(api_version)}
     response = SESSION.get(endpoint, params=params, headers=headers)
-
-    if response.status_code != 200:
+    if response.status_code < 200 or response.status_code >= 300:
         logging.error(f"CDA Error: response={response}")
         raise ApiError(response)
 
     try:
         return cast(JSON, response.json())
     except JSONDecodeError as error:
-        logging.error(f"Error decoding CDA response: {error}")
+        logging.error(f"Error decoding CDA response as json: {error}")
         return {}
 
 
+def get_with_paging(
+    selector: str,
+    endpoint: str,
+    params: RequestParams,
+    *,
+    api_version: int = API_VERSION,
+) -> JSON:
+    """Make a GET request to the CWMS Data API with paging.
+
+    Args:
+        endpoint: The CDA endpoint for the record(s).
+        selector: The json key that will be merged though each page call
+        params (optional): Query parameters for the request.
+
+    Keyword Args:
+        api_version (optional): The CDA version to use for the request. If not specified,
+            the default API_VERSION will be used.
+
+    Returns:
+        The deserialized JSON response data.
+
+    Raises:
+        ApiError: If an error response is return by the API.
+    """
+
+    first_pass = True
+    while (params["page"] is not None) or first_pass:
+        temp = get(endpoint, params, api_version=api_version)
+        if first_pass:
+            response = temp
+        else:
+            response[selector] = response[selector] + temp[selector]
+        if "next-page" in temp.keys():
+            params["page"] = temp["next-page"]
+        else:
+            params["page"] = None
+        first_pass = False
+    return response
+
+
 def post(
     endpoint: str,
-    data: JSON,
+    data: Any,
     params: Optional[RequestParams] = None,
     *,
     api_version: int = API_VERSION,
@@ -221,18 +307,19 @@ def post(
     # post requires different headers than get for
     headers = {"accept": "*/*", "Content-Type": api_version_text(api_version)}
 
-    response = SESSION.post(
-        endpoint, params=params, headers=headers, data=json.dumps(data)
-    )
+    if isinstance(data, dict):
+        data = json.dumps(data)
 
-    if response.status_code != 200:
+    response = SESSION.post(endpoint, params=params, headers=headers, data=data)
+
+    if response.status_code < 200 or response.status_code >= 300:
         logging.error(f"CDA Error: response={response}")
         raise ApiError(response)
 
 
 def patch(
     endpoint: str,
-    data: JSON,
+    data: Optional[Any] = None,
     params: Optional[RequestParams] = None,
     *,
     api_version: int = API_VERSION,
@@ -256,12 +343,14 @@ def patch(
     """
 
     headers = {"accept": "*/*", "Content-Type": api_version_text(api_version)}
+    if data is None:
+        response = SESSION.patch(endpoint, params=params, headers=headers)
+    else:
+        if isinstance(data, dict):
+            data = json.dumps(data)
+        response = SESSION.patch(endpoint, params=params, headers=headers, data=data)
 
-    response = SESSION.patch(
-        endpoint, params=params, headers=headers, data=json.dumps(data)
-    )
-
-    if response.status_code != 200:
+    if response.status_code < 200 or response.status_code >= 300:
         logging.error(f"CDA Error: response={response}")
         raise ApiError(response)
 
@@ -289,6 +378,6 @@ def delete(
     headers = {"Accept": api_version_text(api_version)}
     response = SESSION.delete(endpoint, params=params, headers=headers)
 
-    if response.status_code != 200:
+    if response.status_code < 200 or response.status_code >= 300:
         logging.error(f"CDA Error: response={response}")
         raise ApiError(response)

cwms_python-0.5.0/cwms/catalog/catalog.py

@@ -0,0 +1,128 @@
+from typing import Optional
+
+import cwms.api as api
+from cwms.cwms_types import Data
+
+
+def get_locations_catalog(
+    office_id: str,
+    page: Optional[str] = None,
+    page_size: Optional[int] = 5000,
+    unit_system: Optional[str] = None,
+    like: Optional[str] = None,
+    location_category_like: Optional[str] = None,
+    location_group_like: Optional[str] = None,
+    bounding_office_like: Optional[str] = None,
+    location_kind_like: Optional[str] = None,
+) -> Data:
+    """Retrieves filters for a locations catalog
+
+    Parameters
+    ----------
+        page: string
+            The endpoint used to identify where the request is located.
+        page_size: integer
+            The entries per page returned. The default value is 5000.
+        unit_system: string
+            The unit system desired in response. Valid values for this
+            field are:
+                1. SI
+                2. EN
+        office_id: string
+            The owning office of the timeseries group.
+        like: string
+            The regex for matching against the id
+        location_category_like: string
+            The regex for matching against the location category id
+        location_group_like: string
+            The regex for matching against the location group id
+        bounding_office_like: string
+            The regex for matching against the location bounding office
+        location_kind_like: string
+            Posix regular expression matching against the location kind. The location-kind is typically unset or one of the following: {"SITE", "EMBANKMENT", "OVERFLOW", "TURBINE", "STREAM", "PROJECT", "STREAMGAGE", "BASIN", "OUTLET", "LOCK", "GATE"}. Multiple kinds can be matched by using Regular Expression OR clauses. For example: "(SITE|STREAM)"
+
+    Returns
+    -------
+        cwms data type
+    """
+
+    # CHECKS
+    if office_id is None:
+        raise ValueError("Retrieve locations catalog requires an office")
+
+    dataset = "LOCATIONS"
+    endpoint = f"catalog/{dataset}"
+    params = {
+        "page": page,
+        "page-size": page_size,
+        "units": unit_system,
+        "office": office_id,
+        "like": like,
+        "location-category-like": location_category_like,
+        "location-group-like": location_group_like,
+        "bounding-office-like": bounding_office_like,
+        "location-kind-like": location_kind_like,
+    }
+
+    response = api.get(endpoint=endpoint, params=params, api_version=2)
+    return Data(response, selector="entries")
+
+
+def get_timeseries_catalog(
+    office_id: str,
+    page: Optional[str] = None,
+    page_size: Optional[int] = 5000,
+    unit_system: Optional[str] = None,
+    like: Optional[str] = None,
+    timeseries_category_like: Optional[str] = None,
+    timeseries_group_like: Optional[str] = "DMZ Include List",
+    bounding_office_like: Optional[str] = None,
+) -> Data:
+    """Retrieves filters for the timeseries catalog
+
+    Parameters
+    ----------
+        page: string
+            The endpoint used to identify where the request is located.
+        page_size: integer
+            The entries per page returned. The default value is 500.
+        unit_system: string
+            The unit system desired in response. Valid values for this
+            field are:
+                1. SI
+                2. EN
+        office_id: string
+            The owning office of the timeseries group.
+        like: string
+            The regex for matching against the id
+        timeseries_category_like: string
+            The regex for matching against the category id
+        timeseries_group_like: string
+            The regex for matching against the timeseries group id. This will default to pull only public datasets
+        bounding_office_like: string
+            The regex for matching against the location bounding office
+
+    Returns
+    -------
+        cwms data type
+    """
+
+    # CHECKS
+    if office_id is None:
+        raise ValueError("Retrieve timeseries catalog requires an office")
+
+    dataset = "TIMESERIES"
+    endpoint = f"catalog/{dataset}"
+    params = {
+        "page": page,
+        "page-size": page_size,
+        "unit-system": unit_system,
+        "office": office_id,
+        "like": like,
+        "timeseries-category-like": timeseries_category_like,
+        "timeseries-group-like": timeseries_group_like,
+        "bounding-office-like": bounding_office_like,
+    }
+
+    response = api.get(endpoint=endpoint, params=params, api_version=2)
+    return Data(response, selector="entries")

cwms_python-0.5.0/cwms/cwms_types.py

@@ -0,0 +1,107 @@
+from copy import deepcopy
+from enum import Enum, auto
+from typing import Any, Optional
+
+from pandas import DataFrame, Index, json_normalize, to_datetime
+
+# Describes generic JSON serializable data.
+JSON = dict[str, Any]
+
+# Describes request parameters.
+RequestParams = dict[str, Any]
+
+
+class DeleteMethod(Enum):
+    DELETE_ALL = auto()
+    DELETE_KEY = auto()
+    DELETE_DATA = auto()
+
+
+class RatingMethod(Enum):
+    EAGER = auto()
+    LAZY = auto()
+    REFERENCE = auto()
+
+
+class Data:
+    """Wrapper for CWMS API data."""
+
+    def __init__(self, json: JSON, *, selector: Optional[str] = None):
+        """Wrap CWMS API Data.
+
+        Args:
+            data:
+            selector:
+        """
+
+        self.json = json
+        self.selector = selector
+
+        self._df: Optional[DataFrame] = None
+
+    @staticmethod
+    def to_df(json: JSON, selector: Optional[str]) -> DataFrame:
+        """Create a data frame from JSON data.
+
+        Args:
+            json: JSON data returned in the API response.
+            selector: Dot separated string of keys used to extract data for data frame.
+
+        Returns:
+            A data frame containing the data located
+        """
+
+        def get_df_data(data: JSON, selector: str) -> JSON:
+            # get the data that will be stored in the dataframe using the selectors
+            df_data = data
+            for key in selector.split("."):
+                if key in df_data.keys():
+                    df_data = df_data[key]
+            return df_data
+
+        def rating_type(data: JSON) -> DataFrame:
+            # grab the correct point values for a rating table
+            df = DataFrame(data["point"]) if data["point"] else DataFrame()
+            return df
+
+        def timeseries_type(orig_json: JSON, value_json: JSON) -> DataFrame:
+            # if timeseries values are present then grab the values and put into
+            # dataframe else create empty dataframe
+            columns = Index([sub["name"] for sub in orig_json["value-columns"]])
+            if value_json:
+                df = DataFrame(value_json)
+                df.columns = columns
+            else:
+                df = DataFrame(columns=columns)
+
+            if "date-time" in df.columns:
+                df["date-time"] = to_datetime(df["date-time"], unit="ms", utc=True)
+            return df
+
+        data = deepcopy(json)
+
+        if selector:
+            df_data = get_df_data(data, selector)
+
+            # if the dataframe is for a rating table
+            if ("rating-points" in selector) and ("point" in df_data.keys()):
+                df = rating_type(df_data)
+
+            elif selector == "values":
+                df = timeseries_type(data, df_data)
+
+            else:
+                df = json_normalize(df_data) if df_data else DataFrame()
+        else:
+            df = json_normalize(data)
+
+        return df
+
+    @property
+    def df(self) -> DataFrame:
+        """Return the data frame."""
+
+        if not isinstance(self._df, DataFrame):
+            self._df = Data.to_df(self.json, self.selector)
+
+        return self._df