cwms-python 0.6.0__tar.gz → 0.7.0__tar.gz

{cwms_python-0.6.0 → cwms_python-0.7.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.3
 Name: cwms-python
-Version: 0.6.0
-Summary: Corps water managerment systems (CWMS) REST API for Data Retrieval of USACE water data
+Version: 0.7.0
+Summary: Corps water management systems (CWMS) REST API for Data Retrieval of USACE water data
 License: LICENSE
 Keywords: USACE,water data,CWMS
 Author: Eric Novotny

{cwms_python-0.6.0 → cwms_python-0.7.0}/cwms/__init__.py

@@ -4,7 +4,6 @@ from cwms.api import *
 from cwms.catalog.blobs import *
 from cwms.catalog.catalog import *
 from cwms.catalog.clobs import *
-from cwms.datafile_imports.shef_critfile_import import *
 from cwms.forecast.forecast_instance import *
 from cwms.forecast.forecast_spec import *
 from cwms.levels.location_levels import *
@@ -12,6 +11,7 @@ from cwms.levels.specified_levels import *
 from cwms.locations.gate_changes import *
 from cwms.locations.location_groups import *
 from cwms.locations.physical_locations import *
+from cwms.measurements.measurements import *
 from cwms.outlets.outlets import *
 from cwms.outlets.virtual_outlets import *
 from cwms.projects.project_lock_rights import *

{cwms_python-0.6.0 → cwms_python-0.7.0}/cwms/api.py

@@ -26,6 +26,7 @@ which includes the response object and provides some hints to the user on how to
 the error.
 """
 
+import base64
 import json
 import logging
 from json import JSONDecodeError
@@ -34,6 +35,7 @@ from typing import Any, Optional, cast
 from requests import Response, adapters
 from requests_toolbelt import sessions  # type: ignore
 from requests_toolbelt.sessions import BaseUrlSession  # type: ignore
+from urllib3.util.retry import Retry
 
 from cwms.cwms_types import JSON, RequestParams
 
@@ -42,8 +44,24 @@ API_ROOT = "https://cwms-data.usace.army.mil/cwms-data/"
 API_VERSION = 2
 
 # Initialize a non-authenticated session with the default root URL and set default pool connections.
+
+retry_strategy = Retry(
+    total=6,
+    backoff_factor=0.5,
+    status_forcelist=[
+        403,
+        429,
+        500,
+        502,
+        503,
+        504,
+    ],  # Example: also retry on these HTTP status codes
+    allowed_methods=["GET", "PUT", "POST", "PATCH", "DELETE"],  # Methods to retry
+)
 SESSION = sessions.BaseUrlSession(base_url=API_ROOT)
-adapter = adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)
+adapter = adapters.HTTPAdapter(
+    pool_connections=100, pool_maxsize=100, max_retries=retry_strategy
+)
 SESSION.mount("https://", adapter)
 
 
@@ -118,7 +136,9 @@ def init_session(
         logging.debug(f"Initializing root URL: api_root={api_root}")
         SESSION = sessions.BaseUrlSession(base_url=api_root)
         adapter = adapters.HTTPAdapter(
-            pool_connections=pool_connections, pool_maxsize=pool_connections
+            pool_connections=pool_connections,
+            pool_maxsize=pool_connections,
+            max_retries=retry_strategy,
         )
         SESSION.mount("https://", adapter)
     if api_key:
@@ -188,20 +208,8 @@ def get_xml(
     Raises:
         ApiError: If an error response is return by the API.
     """
-
-    headers = {"Accept": api_version_text(api_version)}
-    response = SESSION.get(endpoint, params=params, headers=headers)
-    response.close()
-
-    if response.status_code < 200 or response.status_code >= 300:
-        logging.error(f"CDA Error: response={response}")
-        raise ApiError(response)
-
-    try:
-        return response.content.decode("utf-8")
-    except JSONDecodeError as error:
-        logging.error(f"Error decoding CDA response as xml: {error}")
-        return {}
+    # Wrap the primary get for backwards compatibility
+    return get(endpoint=endpoint, params=params, api_version=api_version)
 
 
 def get(
@@ -209,7 +217,7 @@ def get(
     params: Optional[RequestParams] = None,
     *,
     api_version: int = API_VERSION,
-) -> JSON:
+) -> Any:
     """Make a GET request to the CWMS Data API.
 
     Args:
@@ -228,17 +236,28 @@ def get(
     """
 
     headers = {"Accept": api_version_text(api_version)}
-    response = SESSION.get(endpoint, params=params, headers=headers)
-    response.close()
-    if response.status_code < 200 or response.status_code >= 300:
-        logging.error(f"CDA Error: response={response}")
-        raise ApiError(response)
-
-    try:
-        return cast(JSON, response.json())
-    except JSONDecodeError as error:
-        logging.error(f"Error decoding CDA response as json: {error}")
-        return {}
+    with SESSION.get(endpoint, params=params, headers=headers) as response:
+        if not response.ok:
+            logging.error(f"CDA Error: response={response}")
+            raise ApiError(response)
+        try:
+            # Avoid case sensitivity issues with the content type header
+            content_type = response.headers.get("Content-Type", "").lower()
+            # Most CDA content is JSON
+            if "application/json" in content_type or not content_type:
+                return cast(JSON, response.json())
+            # Use automatic charset detection with .text
+            if "text/plain" in content_type or "text/" in content_type:
+                return response.text
+            if content_type.startswith("image/"):
+                return base64.b64encode(response.content).decode("utf-8")
+            # Fallback for remaining content types
+            return response.content.decode("utf-8")
+        except JSONDecodeError as error:
+            logging.error(
+                f"Error decoding CDA response as JSON: {error} on line {error.lineno}\n\tFalling back to text"
+            )
+            return response.text
 
 
 def get_with_paging(
@@ -247,7 +266,7 @@ def get_with_paging(
     params: RequestParams,
     *,
     api_version: int = API_VERSION,
-) -> JSON:
+) -> Any:
     """Make a GET request to the CWMS Data API with paging.
 
     Args:
@@ -312,12 +331,10 @@ def post(
     if isinstance(data, dict) or isinstance(data, list):
         data = json.dumps(data)
 
-    response = SESSION.post(endpoint, params=params, headers=headers, data=data)
-    response.close()
-
-    if response.status_code < 200 or response.status_code >= 300:
-        logging.error(f"CDA Error: response={response}")
-        raise ApiError(response)
+    with SESSION.post(endpoint, params=params, headers=headers, data=data) as response:
+        if not response.ok:
+            logging.error(f"CDA Error: response={response}")
+            raise ApiError(response)
 
 
 def patch(
@@ -346,16 +363,13 @@ def patch(
     """
 
     headers = {"accept": "*/*", "Content-Type": api_version_text(api_version)}
-    if data is None:
-        response = SESSION.patch(endpoint, params=params, headers=headers)
-    else:
-        if isinstance(data, dict) or isinstance(data, list):
-            data = json.dumps(data)
-        response = SESSION.patch(endpoint, params=params, headers=headers, data=data)
-    response.close()
-    if response.status_code < 200 or response.status_code >= 300:
-        logging.error(f"CDA Error: response={response}")
-        raise ApiError(response)
+
+    if data and isinstance(data, dict) or isinstance(data, list):
+        data = json.dumps(data)
+    with SESSION.patch(endpoint, params=params, headers=headers, data=data) as response:
+        if not response.ok:
+            logging.error(f"CDA Error: response={response}")
+            raise ApiError(response)
 
 
 def delete(
@@ -379,8 +393,7 @@ def delete(
     """
 
     headers = {"Accept": api_version_text(api_version)}
-    response = SESSION.delete(endpoint, params=params, headers=headers)
-    response.close()
-    if response.status_code < 200 or response.status_code >= 300:
-        logging.error(f"CDA Error: response={response}")
-        raise ApiError(response)
+    with SESSION.delete(endpoint, params=params, headers=headers) as response:
+        if not response.ok:
+            logging.error(f"CDA Error: response={response}")
+            raise ApiError(response)

{cwms_python-0.6.0 → cwms_python-0.7.0}/cwms/catalog/blobs.py

@@ -1,29 +1,40 @@
+import base64
 from typing import Optional
 
 import cwms.api as api
 from cwms.cwms_types import JSON, Data
+from cwms.utils.checks import is_base64
 
+STORE_DICT = """data = {
+    "office-id": "SWT",
+    "id": "MYFILE_OR_BLOB_ID.TXT",
+    "description": "Your description here",
+    "media-type-id": "application/octet-stream",
+    "value": "STRING of content or BASE64_ENCODED_STRING"
+}
+"""
 
-def get_blob(blob_id: str, office_id: str) -> Data:
-    """Get a single clob.
+
+def get_blob(blob_id: str, office_id: str) -> str:
+    """Get a single BLOB (Binary Large Object).
 
     Parameters
         ----------
             blob_id: string
-                Specifies the id of the blob
+                Specifies the id of the blob. ALL blob ids are UPPERCASE.
             office_id: string
                 Specifies the office of the blob.
 
 
         Returns
         -------
-            cwms data type.  data.json will return the JSON output and data.df will return a dataframe
+            str: the value returned based on the content-type it was stored with as a string
     """
 
     endpoint = f"blobs/{blob_id}"
     params = {"office": office_id}
     response = api.get(endpoint, params, api_version=1)
-    return Data(response)
+    return str(response)
 
 
 def get_blobs(
@@ -50,36 +61,39 @@ def get_blobs(
     endpoint = "blobs"
     params = {"office": office_id, "page-size": page_size, "like": blob_id_like}
 
-    response = api.get(endpoint, params, api_version=1)
+    response = api.get(endpoint, params, api_version=2)
     return Data(response, selector="blobs")
 
 
 def store_blobs(data: JSON, fail_if_exists: Optional[bool] = True) -> None:
-    """Create New Blob
+    f"""Create New Blob
 
     Parameters
-        ----------
-            Data: JSON dictionary
-                JSON containing information of Blob to be updated
-                    {
-                    "office-id": "string",
-                    "id": "string",
-                    "description": "string",
-                    "media-type-id": "string",
-                    "value": "string"
-                    }
-            fail_if_exists: Boolean
-                Create will fail if provided ID already exists. Default: true
+    ----------
+        **Note**: The "id" field is automatically cast to uppercase.
 
-        Returns
-        -------
-            None
+        Data: JSON dictionary
+            JSON containing information of Blob to be updated.
+
+            {STORE_DICT}
+        fail_if_exists: Boolean
+            Create will fail if the provided ID already exists. Default: True
+
+    Returns
+    -------
+        None
     """
 
     if not isinstance(data, dict):
-        raise ValueError("Cannot store a Blob without a JSON data dictionary")
+        raise ValueError(
+            f"Cannot store a Blob without a JSON data dictionary:\n{STORE_DICT}"
+        )
+
+    # Encode value if it's not already Base64-encoded
+    if "value" in data and not is_base64(data["value"]):
+        # Encode to bytes, then Base64, then decode to string for storing
+        data["value"] = base64.b64encode(data["value"].encode("utf-8")).decode("utf-8")
 
     endpoint = "blobs"
     params = {"fail-if-exists": fail_if_exists}
-
     return api.post(endpoint, data, params, api_version=1)

{cwms_python-0.6.0 → cwms_python-0.7.0}/cwms/catalog/catalog.py

@@ -77,6 +77,7 @@ def get_timeseries_catalog(
     timeseries_category_like: Optional[str] = None,
     timeseries_group_like: Optional[str] = "DMZ Include List",
     bounding_office_like: Optional[str] = None,
+    include_extents: Optional[bool] = False,
 ) -> Data:
     """Retrieves filters for the timeseries catalog
 
@@ -101,6 +102,8 @@ def get_timeseries_catalog(
             The regex for matching against the timeseries group id. This will default to pull only public datasets
         bounding_office_like: string
             The regex for matching against the location bounding office
+        include_extents: bool
+            Whether to include the time series extents in the catalog
 
     Returns
     -------
@@ -122,6 +125,7 @@ def get_timeseries_catalog(
         "timeseries-category-like": timeseries_category_like,
         "timeseries-group-like": timeseries_group_like,
         "bounding-office-like": bounding_office_like,
+        "include-extents": include_extents,
     }
 
     response = api.get(endpoint=endpoint, params=params, api_version=2)

{cwms_python-0.6.0 → cwms_python-0.7.0}/cwms/locations/physical_locations.py

@@ -70,7 +70,7 @@ def get_locations(
     params = {
         "office": office_id,
         "names": location_ids,
-        "units": units,
+        "unit": units,
         "datum": datum,
     }
 
@@ -95,6 +95,7 @@ def ExpandLocations(df: DataFrame) -> DataFrame:
 def delete_location(
     location_id: str,
     office_id: Optional[str] = None,
+    cascade_delete: Optional[bool] = False,
 ) -> None:
     """
     Deletes location data with the given ID and office ID.
@@ -105,6 +106,8 @@ def delete_location(
             The ID of the office that the data belongs to.
         loc_ids : str
             The ID of the location that the data belongs to.
+        cascade_delete: bool
+            Whether to delete all data associated with location.
 
     Returns
     -------
@@ -119,6 +122,7 @@ def delete_location(
     endpoint = f"locations/{location_id}"
     params = {
         "office": office_id,
+        "cascade-delete": cascade_delete,
     }
 
     return api.delete(endpoint, params=params)

cwms_python-0.7.0/cwms/measurements/measurements.py

@@ -0,0 +1,177 @@
+from datetime import datetime
+from typing import Optional
+
+import cwms.api as api
+from cwms.cwms_types import JSON, Data
+
+
+def get_measurements(
+    office_id: Optional[str] = None,
+    location_id_mask: Optional[str] = None,
+    min_number_id: Optional[str] = None,
+    max_number_id: Optional[str] = None,
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    timezone: Optional[str] = None,
+    min_height: Optional[float] = None,
+    max_height: Optional[float] = None,
+    min_flow: Optional[float] = None,
+    max_flow: Optional[float] = None,
+    agency: Optional[str] = None,
+    quality: Optional[str] = None,
+    unit: Optional[str] = "EN",
+) -> Data:
+    """Returns matching measurement data
+
+    Parameters
+    ----------
+        office_id: string, optional, default is None
+            Office id mask for filtering measurements.
+        location_id_mask: string, optional, default is None
+            Location id mask for filtering measurements. Use null to retrieve measurements for all locations.
+        min_number_id: sting, optional, default is None
+            Minimum measurement number-id for filtering measurements.
+        max_number_id: string, optional, default is None
+            Maximum measurement number-id for filtering measurements.
+        begin: datetime, optional, default is None
+            Start of the time window for data to be included in the response. If this field is
+            not specified, then begin time will be unbounded. Any timezone information should be
+            passed within the datetime object. If no timezone information is given, default will be UTC.
+        end: datetime, optional, default is None
+            End of the time window for data to be included in the response. If this field is
+            not specified, then begin time will be unbounded. Any timezone information should
+            be passed within the datetime object. If no timezone information is given, default will be UTC.
+        timezone: string, optional, default is None
+            This field specifies a default timezone to be used if the format of the begin and end
+            parameters do not include offset or time zone information. Defaults to UTC
+        min_height: float, optional, default is None
+            Minimum height for filtering measurements.
+        max_height: float, optional, default is None
+            Maximum flow for filtering measurements.
+        min_flow: float, optional, default is None
+            Minimum flow for filtering measurements.
+        max_flow: float, optional, default is None
+            Maximum flow for filtering measurements.
+        agency: string, optional, default is None
+            Agency for filtering measurements
+        quality: string, optional, default is None
+            Quality for filtering measurements
+        unit_systems: string, optional, default is EN
+            Specifies the unit system of the response. Valid values for the unit field are:
+                1. EN. English unit system.
+                2. SI. SI unit system.
+    Returns
+    -------
+        cwms data type.  data.json will return the JSON output and data.df will return a dataframe. Dates returned are all in UTC.
+    """
+
+    # creates the dataframe from the timeseries data
+    endpoint = "measurements"
+    if begin and not isinstance(begin, datetime):
+        raise ValueError("begin needs to be in datetime")
+    if end and not isinstance(end, datetime):
+        raise ValueError("end needs to be in datetime")
+
+    params = {
+        "office-mask": office_id,
+        "id-mask": location_id_mask,
+        "min-number": min_number_id,
+        "max-number": max_number_id,
+        "begin": begin.isoformat() if begin else None,
+        "end": end.isoformat() if end else None,
+        "timezone": timezone,
+        "min-height": min_height,
+        "max-height": max_height,
+        "min-flow": min_flow,
+        "max-flow": max_flow,
+        "agency": agency,
+        "quality": quality,
+        "unit-system": unit,
+    }
+
+    response = api.get(endpoint, params, api_version=1)
+    return Data(response)  # , selector=selector)
+
+
+def store_measurements(
+    data: JSON,
+    fail_if_exists: Optional[bool] = True,
+) -> None:
+    """Will Create new measurement(s)
+
+    Parameters
+    ----------
+        data: JSON dictionary
+            measurement data to be stored.
+        fail_if_exists: bool, optional, default is True
+            Create will fail if provided Measurement(s) already exist.
+
+    Returns
+    -------
+    response
+    """
+
+    endpoint = "measurements"
+    params = {
+        "fail-if-exists": fail_if_exists,
+    }
+
+    if not isinstance(data, dict):
+        raise ValueError("Cannot store a timeseries without a JSON data dictionary")
+
+    return api.post(endpoint, data, params, api_version=1)
+
+
+def delete_measurements(
+    location_id: str,
+    office_id: str,
+    begin: datetime,
+    end: datetime,
+    timezone: Optional[str] = None,
+    min_number_id: Optional[str] = None,
+    max_number_id: Optional[str] = None,
+) -> None:
+    """Delete an existing measurement
+
+    Parameters
+    ----------
+        office_id: string
+            Specifies the office of the measurements to delete
+        location_id: string
+            Specifies the location-id of the measurement(s) to be deleted.
+        begin: datetime
+            Start of the time window to delete. Any timezone information should be
+            passed within the datetime object. If no timezone information is given, default will be UTC.
+        end: datetime
+            End of the time window to delete. Any timezone information should
+            be passed within the datetime object. If no timezone information is given, default will be UTC.
+        timezone: string, optional, default is None
+            This field specifies a default timezone to be used if the format of the begin and end
+            parameters do not include offset or time zone information. Defaults to UTC
+        min_number_id: sting, optional, default is None
+            Minimum measurement number-id of the measurement to be deleted.
+        max_number_id: string, optional, default is None
+            Maximum measurement number-id of the measurement to be deleted.
+
+    Returns
+    -------
+        None
+    """
+
+    if location_id is None:
+        raise ValueError("Deleting measurements requires a location id")
+    if office_id is None:
+        raise ValueError("Deleting measurements requires an office")
+
+    endpoint = f"measurements/{location_id}"
+
+    params = {
+        "office": office_id,
+        "begin": begin.isoformat() if begin else None,
+        "end": end.isoformat() if end else None,
+        "timezone": timezone,
+        "min-number": min_number_id,
+        "max-number": max_number_id,
+    }
+
+    return api.delete(endpoint, params, api_version=1)

{cwms_python-0.6.0 → cwms_python-0.7.0}/cwms/ratings/ratings.py

@@ -15,7 +15,7 @@ def rating_current_effective_date(rating_id: str, office_id: str) -> Any:
     """Retrieve the most recent effective date for a specific rating id.
 
     Returns
-        datatime
+        Any
             the datetime of the most recent effective date for a rating id. If max effective date is
             not present for rating_id then None will be returned
 
@@ -46,7 +46,7 @@ def get_current_rating(
             The owning office of the rating specifications. If no office is provided information from all offices will
             be returned
         rating_table_in_df: Bool, Optional Default = True
-            define if the independant and dependant variables should be stored as a dataframe
+            define if the independent and dependant variables should be stored as a dataframe
     Returns
     -------
         Data : Data
@@ -112,7 +112,7 @@ def get_ratings_xml(
     timezone: Optional[str] = None,
     method: Optional[str] = "EAGER",
 ) -> Any:
-    """Retrives ratings for a specific rating-id
+    """Retrieves ratings for a specific rating-id
 
     Parameters
     ----------
@@ -124,7 +124,7 @@ def get_ratings_xml(
         begin: datetime, optional
             the start of the time window for data to be included in the response.  This is based on the effective date of the ratings
         end: datetime, optional
-            the end of the time window for data to be included int he reponse. This is based on the effective date of the ratings
+            the end of the time window for data to be included int he response. This is based on the effective date of the ratings
         timezone:
             the time zone of the values in the being and end fields if not specified UTC is used
         method:
@@ -225,13 +225,13 @@ def rating_simple_df_to_json(
     active: Optional[bool] = True,
 ) -> JSON:
     """This function converts a dataframe to a json dictionary in the correct format to be posted using the store_ratings function. Can
-    only be used for simple ratings with a indenpendant and 1 dependant variable.
+    only be used for simple ratings with a independent and 1 dependant variable.
 
     Parameters
     ----------
         data: pd.Dataframe
             Rating Table to be stored to an exiting rating specification and template.  Can only have 2 columns ind and dep. ind
-            contained the indenpendant variable and dep contains the dependent variable.
+            contained the independent variable and dep contains the dependent variable.
                         ind	dep
                     0	9.62	0.01
                     1	9.63	0.01
@@ -249,7 +249,7 @@ def rating_simple_df_to_json(
         office_id: str
             the owning office of the rating
         units: str
-            units for both the independant and dependent variable seperated by ; i.e. ft;cfs or ft;ft.
+            units for both the independent and dependent variable separated by ; i.e. ft;cfs or ft;ft.
         effective_date: datetime,
             The effective date of the rating curve to be stored.
         transition_start_date: datetime Optional = None
@@ -384,7 +384,7 @@ def delete_ratings(
 
 
 def store_rating(data: Any, store_template: Optional[bool] = True) -> None:
-    """Will create a new ratingset including template/spec and rating
+    """Will create a new rating-set including template/spec and rating
 
     Parameters
     ----------
@@ -403,7 +403,7 @@ def store_rating(data: Any, store_template: Optional[bool] = True) -> None:
 
     if not isinstance(data, dict) and xml_heading not in data:
         raise ValueError(
-            "Cannot store a timeseries without a JSON data dictionaryor in XML"
+            "Cannot store a timeseries without a JSON data dictionary or in XML"
         )
 
     if xml_heading in data:

{cwms_python-0.6.0 → cwms_python-0.7.0}/cwms/ratings/ratings_spec.py

@@ -8,7 +8,7 @@ from cwms.cwms_types import JSON, Data
 
 
 def get_rating_spec(rating_id: str, office_id: str) -> Data:
-    """Retrives a single rating spec
+    """Retrieves a single rating spec
 
       Parameters
       ----------
@@ -37,7 +37,7 @@ def get_rating_specs(
     rating_id_mask: Optional[str] = None,
     page_size: int = 500000,
 ) -> Data:
-    """Retrives a list of rating specification
+    """Retrieves a list of rating specification
 
       Parameters
       ----------
@@ -45,7 +45,7 @@ def get_rating_specs(
               The owning office of the rating specifications. If no office is provided information from all offices will
               be returned
           rating-id-mask: string, optional
-              Posix regular expression that specifies the rating ids to be included in the reponce.  If not specified all
+              Posix regular expression that specifies the rating ids to be included in the response.  If not specified all
               rating specs shall be returned.
           page-size: int, optional, default is 5000000: Specifies the number of records to obtain in
               a single call.
@@ -111,7 +111,7 @@ def rating_spec_df_to_xml(data: pd.DataFrame) -> str:
     Parameters
     ----------
     data : pd_dataframe
-        pandas dataframe that contrains rating specification paramters
+        pandas dataframe that contains rating specification parameters
         should follow same formate the is returned from get_rating_spec function
     Returns
     -------
@@ -134,10 +134,10 @@ def rating_spec_df_to_xml(data: pd.DataFrame) -> str:
       <auto-migrate-extension>{str(data.loc[0,'auto-migrate-extension']).lower()}</auto-migrate-extension>
       <ind-rounding-specs>"""
 
-    ind_rouding = data.loc[0, "independent-rounding-specs"]
-    if isinstance(ind_rouding, list):
+    ind_rounding = data.loc[0, "independent-rounding-specs"]
+    if isinstance(ind_rounding, list):
         i = 1
-        for rounding in ind_rouding:
+        for rounding in ind_rounding:
             spec_xml = (
                 spec_xml
                 + f"""\n   <ind-rounding-spec position="{i}">{rounding['value']}</ind-rounding-spec>"""

{cwms_python-0.6.0 → cwms_python-0.7.0}/cwms/timeseries/timeseries.py

@@ -1,4 +1,4 @@
-import threading
+import concurrent.futures
 from datetime import datetime
 from typing import Any, Dict, Optional
 
@@ -16,13 +16,14 @@ def get_multi_timeseries_df(
     begin: Optional[datetime] = None,
     end: Optional[datetime] = None,
     melted: Optional[bool] = False,
+    max_workers: Optional[int] = 30,
 ) -> DataFrame:
     """gets multiple timeseries and stores into a single dataframe
 
     Parameters
     ----------
-        ts_ids: linst
-            a list of timeseries to get.  If the timeseries is a verioned timeseries then serpeate the ts_id from the
+        ts_ids: list
+            a list of timeseries to get.  If the timeseries is a versioned timeseries then separate the ts_id from the
             version_date using a :.  Example "OMA.Stage.Inst.6Hours.0.Fcst-MRBWM-GRFT:2024-04-22 07:00:00-05:00".  Make
             sure that the version date include the timezone offset if not in UTC.
         office_id: string
@@ -46,6 +47,9 @@ def get_multi_timeseries_df(
         melted: Boolean, optional, default is false
             if set to True a melted dataframe will be provided. By default a multi-index column dataframe will be
             returned.
+        max_workers: Int, Optional, default is None
+            It is a number of Threads aka size of pool in concurrent.futures.ThreadPoolExecutor. From 3.8 onwards
+            default value is min(32, os.cpu_count() + 4). Out of these 5 threads are preserved for I/O bound task.
 
 
         Returns
@@ -53,60 +57,47 @@ def get_multi_timeseries_df(
             dataframe
     """
 
-    def get_ts_ids(
-        result_dict: list[Dict[str, Any]],
-        ts_id: str,
-        office_id: str,
-        begin: datetime,
-        end: datetime,
-        unit: str,
-        version_date: datetime,
-    ) -> None:
-        data = get_timeseries(
-            ts_id=ts_id,
-            office_id=office_id,
-            unit=unit,
-            begin=begin,
-            end=end,
-            version_date=version_date,
-        )
-        result_dict.append(
-            {
+    def get_ts_ids(ts_id: str) -> Any:
+        try:
+            if ":" in ts_id:
+                ts_id, version_date = ts_id.split(":", 1)
+                version_date_dt = pd.to_datetime(version_date)
+            else:
+                version_date_dt = None
+            data = get_timeseries(
+                ts_id=ts_id,
+                office_id=office_id,
+                unit=unit,
+                begin=begin,
+                end=end,
+                version_date=version_date_dt,
+            )
+            result_dict = {
                 "ts_id": ts_id,
                 "unit": data.json["units"],
-                "version_date": version_date,
+                "version_date": version_date_dt,
                 "values": data.df,
             }
-        )
-
-    result_dict = []  # type: list[Dict[str,Any]]
-    threads = []
-    for ts_id in ts_ids:
-        if ":" in ts_id:
-            ts_id, version_date = ts_id.split(":", 1)
-            version_date_dt = pd.to_datetime(version_date)
-        else:
-            version_date_dt = None
-        t = threading.Thread(
-            target=get_ts_ids,
-            args=(result_dict, ts_id, office_id, begin, end, unit, version_date_dt),
-        )
-        threads.append(t)
-        t.start()
+            return result_dict
+        except Exception as e:
+            print(f"Error processing {ts_id}: {e}")
+            return None
 
-    for t in threads:
-        t.join()
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        results = executor.map(get_ts_ids, ts_ids)
 
+    result_dict = list(results)
     data = pd.DataFrame()
     for row in result_dict:
-        temp_df = row["values"]
-        temp_df = temp_df.assign(ts_id=row["ts_id"], units=row["unit"])
-        if "version_date" in row.keys():
-            temp_df = temp_df.assign(version_date=row["version_date"])
-        temp_df.dropna(how="all", axis=1, inplace=True)
-        data = pd.concat([data, temp_df], ignore_index=True)
-
-    if not melted:
+        if row:
+            temp_df = row["values"]
+            temp_df = temp_df.assign(ts_id=row["ts_id"], units=row["unit"])
+            if "version_date" in row.keys():
+                temp_df = temp_df.assign(version_date=row["version_date"])
+            temp_df.dropna(how="all", axis=1, inplace=True)
+            data = pd.concat([data, temp_df], ignore_index=True)
+
+    if not melted and "date-time" in data.columns:
         cols = ["ts_id", "units"]
         if "version_date" in data.columns:
             cols.append("version_date")
@@ -129,7 +120,7 @@ def get_timeseries(
     datum: Optional[str] = None,
     begin: Optional[datetime] = None,
     end: Optional[datetime] = None,
-    page_size: Optional[int] = 500000,
+    page_size: Optional[int] = 300000,
     version_date: Optional[datetime] = None,
     trim: Optional[bool] = True,
 ) -> Data:
@@ -163,7 +154,7 @@ def get_timeseries(
             not specified, any required time window ends at the current time. Any timezone
             information should be passed within the datetime object. If no timezone information
             is given, default will be UTC.
-        page_size: int, optional, default is 5000000: Sepcifies the number of records to obtain in
+        page_size: int, optional, default is 300000: Specifies the number of records to obtain in
             a single call.
         version_date: datetime, optional, default is None
             Version date of time series values being requested. If this field is not specified and
@@ -208,7 +199,7 @@ def timeseries_df_to_json(
     office_id: str,
     version_date: Optional[datetime] = None,
 ) -> JSON:
-    """This function converts a dataframe to a json dictionary in the correct format to be posted using the store_timeseries fucntion.
+    """This function converts a dataframe to a json dictionary in the correct format to be posted using the store_timeseries function.
 
     Parameters
     ----------
@@ -223,7 +214,7 @@ def timeseries_df_to_json(
                 2   2023-12-20T15:15:00.000-05:00  98.5           0
                 3   2023-12-20T15:30:00.000-05:00  98.5           0
         ts_id: str
-            timeseried id:specified name of the timeseries to be posted to
+            timeseries id:specified name of the timeseries to be posted to
         office_id: str
             the owning office of the time series
         units: str
@@ -242,7 +233,7 @@ def timeseries_df_to_json(
         df["quality-code"] = 0
     if "date-time" not in df:
         raise TypeError(
-            "date-time is a required column in data when posting as a dateframe"
+            "date-time is a required column in data when posting as a dataframe"
         )
     if "value" not in df:
         raise TypeError(
@@ -268,6 +259,54 @@ def timeseries_df_to_json(
     return ts_dict
 
 
+def store_multi_timeseries_df(
+    ts_data: pd.DataFrame, office_id: str, max_workers: Optional[int] = 30
+) -> None:
+
+    def store_ts_ids(
+        data: pd.DataFrame,
+        ts_id: str,
+        office_id: str,
+        version_date: Optional[datetime] = None,
+    ) -> None:
+        units = data["units"].iloc[0]
+        data_json = timeseries_df_to_json(
+            data=data,
+            ts_id=ts_id,
+            units=units,
+            office_id=office_id,
+            version_date=version_date,
+        )
+        store_timeseries(data=data_json)
+        return None
+
+    unique_tsids = (
+        ts_data["ts_id"].astype(str) + ":" + ts_data["version_date"].astype(str)
+    ).unique()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        for ts_id_all in unique_tsids:
+            try:
+                ts_id, version_date = ts_id_all.split(":", 1)
+                if version_date != "NaT":
+                    version_date_dt = pd.to_datetime(version_date)
+                    data = ts_data[
+                        (ts_data["ts_id"] == ts_id)
+                        & (ts_data["version_date"] == version_date_dt)
+                    ]
+                else:
+                    version_date_dt = None
+                    data = ts_data[
+                        (ts_data["ts_id"] == ts_id) & ts_data["version_date"].isna()
+                    ]
+                if not data.empty:
+                    executor.submit(
+                        store_ts_ids, data, ts_id, office_id, version_date_dt
+                    )
+            except Exception as e:
+                print(f"Error processing {ts_id}: {e}")
+
+
 def store_timeseries(
     data: JSON,
     create_as_ltrs: Optional[bool] = False,
@@ -280,7 +319,7 @@ def store_timeseries(
     ----------
         data: JSON dictionary
             Time Series data to be stored.
-        create_as_ltrs: bool, optional, defualt is False
+        create_as_ltrs: bool, optional, default is False
             Flag indicating if timeseries should be created as Local Regular Time Series.
         store_rule: str, optional, default is None:
             The business rule to use when merging the incoming with existing data. Available values :

{cwms_python-0.6.0 → cwms_python-0.7.0}/cwms/timeseries/timeseries_group.py

@@ -12,9 +12,9 @@ from cwms.cwms_types import JSON, Data
 def get_timeseries_group(
     group_id: str,
     category_id: str,
-    office_id: str,
-    group_office_id: str,
     category_office_id: str,
+    office_id: Optional[str] = None,
+    group_office_id: Optional[str] = None,
 ) -> Data:
     """Retreives time series stored in the requested time series group
 

cwms_python-0.7.0/cwms/utils/checks.py

@@ -0,0 +1,10 @@
+import base64
+
+
+def is_base64(s: str) -> bool:
+    """Check if a string is Base64 encoded."""
+    try:
+        decoded = base64.b64decode(s, validate=True)
+        return base64.b64encode(decoded).decode("utf-8") == s
+    except (ValueError, TypeError):
+        return False

{cwms_python-0.6.0 → cwms_python-0.7.0}/pyproject.toml

@@ -1,12 +1,14 @@
 [tool.poetry]
 name = "cwms-python"
 repository = "https://github.com/HydrologicEngineeringCenter/cwms-python"
-version = "0.6.0"
+
+version = "0.7.0"
+
 
 packages = [
     { include = "cwms" },
 ]
-description = "Corps water managerment systems (CWMS) REST API for Data Retrieval of USACE water data"
+description = "Corps water management systems (CWMS) REST API for Data Retrieval of USACE water data"
 readme = "README.md"
 license = "LICENSE"
 keywords = ["USACE", "water data", "CWMS"]

cwms_python-0.6.0/cwms/datafile_imports/shef_critfile_import.py

@@ -1,130 +0,0 @@
-import re
-from typing import Dict, List
-
-import pandas as pd
-
-import cwms
-
-
-def import_critfile_to_ts_group(
-    file_path: str,
-    office_id: str,
-    group_id: str = "SHEF Data Acquisition",
-    category_id: str = "Data Acquisition",
-    group_office_id: str = "CWMS",
-    category_office_id: str = "CWMS",
-    replace_assigned_ts: bool = False,
-) -> None:
-    """
-    Processes a .crit file and saves the information to the SHEF Data Acquisition time series group.
-
-    Parameters
-    ----------
-    file_path : str
-        Path to the .crit file.
-    office_id : str
-        The ID of the office associated with the specified timeseries.
-    group_id : str, optional
-        The specified group associated with the timeseries data. Defaults to "SHEF Data Acquisition".
-    category_id : str, optional
-        The category ID that contains the timeseries group. Defaults to "Data Acquisition".
-    group_office_id : str, optional
-        The specified office group associated with the timeseries data. Defaults to "CWMS".
-    replace_assigned_ts : bool, optional
-        Specifies whether to unassign all existing time series before assigning new time series specified in the content body. Default is False.
-
-    Returns
-    -------
-    None
-    """
-
-    def parse_crit_file(file_path: str) -> List[Dict[str, str]]:
-        """
-        Parses a .crit file into a dictionary containing timeseries ID and Alias.
-
-        Parameters
-        ----------
-            file_path : str
-                   Path to the .crit file.
-
-        Returns
-        -------
-        List[Dict[str, str]]
-            A list of dictionaries with "Alias" and "Timeseries ID" as keys.
-        """
-        parsed_data = []
-        with open(file_path, "r") as file:
-            for line in file:
-                # Ignore comment lines and empty lines
-                if line.startswith("#") or not line.strip():
-                    continue
-
-                # Extract alias, timeseries ID, and TZ
-                match = re.match(r"([^=]+)=([^;]+);(.+)", line.strip())
-
-                if match:
-                    alias = match.group(1).strip()
-                    timeseries_id = match.group(2).strip()
-                    alias2 = match.group(3).strip()
-
-                    parsed_data.append(
-                        {
-                            "Alias": alias + ":" + alias2,
-                            "Timeseries ID": timeseries_id,
-                        }
-                    )
-
-        return parsed_data
-
-    def append_df(
-        df: pd.DataFrame, office_id: str, ts_id: str, alias: str
-    ) -> pd.DataFrame:
-        """
-        Appends a row to the DataFrame.
-
-        Parameters
-        ----------
-            df : pandas.DataFrame
-                The DataFrame to append to.
-            office_id : str
-                The ID of the office associated with the specified timeseries.
-            tsId : str
-                The timeseries ID from the file.
-            alias : str
-                The alias from the file.
-        Returns
-        -------
-        pandas.DataFrame
-            The updated DataFrame.
-        """
-        data = {
-            "office-id": [office_id],
-            "timeseries-id": [ts_id],
-            "alias-id": [alias],
-        }
-        df = pd.concat([df, pd.DataFrame(data)])
-        return df
-
-    # Parse the file and get the parsed data
-    parsed_data = parse_crit_file(file_path)
-
-    df = pd.DataFrame()
-    for data in parsed_data:
-        # Create DataFrame for the current row
-        df = append_df(df, office_id, data["Timeseries ID"], data["Alias"])
-
-    # Generate JSON dictionary
-    json_dict = cwms.timeseries_group_df_to_json(
-        data=df,
-        group_id=group_id,
-        group_office_id=group_office_id,
-        category_office_id=category_office_id,
-        category_id=category_id,
-    )
-
-    cwms.update_timeseries_groups(
-        group_id=group_id,
-        office_id=office_id,
-        replace_assigned_ts=replace_assigned_ts,
-        data=json_dict,
-    )