cwms-python 1.0.7__tar.gz → 1.0.8__tar.gz

{cwms_python-1.0.7 → cwms_python-1.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cwms-python
-Version: 1.0.7
+Version: 1.0.8
 Summary: Corps water management systems (CWMS) REST API for Data Retrieval of USACE water data
 License: LICENSE
 License-File: LICENSE

{cwms_python-1.0.7 → cwms_python-1.0.8}/cwms/timeseries/timeseries.py

@@ -154,6 +154,23 @@ def get_timeseries_chunk(
     return Data(response, selector=selector)
 
 
+# Number of attempts for a single chunked timeseries request. CDA occasionally
+# returns 500s caused by connection-pool exhaustion that succeed on retry; 500
+# is intentionally not in the session-level status_forcelist (see PR #282), so
+# we retry here, scoped to the chunked store/fetch paths only.
+_CHUNK_ATTEMPTS = 6
+
+
+def _call_with_retry(fn: Any, *args: Any, attempts: int = _CHUNK_ATTEMPTS) -> Any:
+    for i in range(attempts):
+        try:
+            return fn(*args)
+        except Exception as e:
+            if i == attempts - 1:
+                raise
+            logging.warning(f"chunk attempt {i + 1}/{attempts} failed: {e}")
+
+
 def fetch_timeseries_chunks(
     chunks: List[Tuple[datetime, datetime]],
     params: Dict[str, Any],
@@ -161,14 +178,13 @@ def fetch_timeseries_chunks(
     endpoint: str,
     max_workers: int,
 ) -> List[Data]:
-    # Initialize an empty list to store results
-    results = []
+    results: List[Data] = []
+    errors: List[str] = []
 
-    # Create a ThreadPoolExecutor to manage multithreading
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-        # Submit tasks for each chunk to the api
         future_to_chunk = {
             executor.submit(
+                _call_with_retry,
                 get_timeseries_chunk,
                 selector,
                 endpoint,
@@ -179,18 +195,23 @@ def fetch_timeseries_chunks(
             for chunk_start, chunk_end in chunks
         }
 
-        # Process completed threads as they finish
         for future in concurrent.futures.as_completed(future_to_chunk):
+            chunk_start, chunk_end = future_to_chunk[future]
             try:
-                # Retrieve the result of the completed future
-                result = future.result()
-                results.append(result)
+                results.append(future.result())
             except Exception as e:
-                chunk_start, chunk_end = future_to_chunk[future]
-                # Log or handle any errors that occur during execution
-                logging.error(
+                error_msg = (
                     f"Failed to fetch data from {chunk_start} to {chunk_end}: {e}"
                 )
+                logging.error(error_msg)
+                errors.append(error_msg)
+
+    if errors:
+        raise RuntimeError(
+            f"{len(errors)} of {len(chunks)} chunk(s) failed to fetch:\n"
+            + "\n".join(errors)
+        )
+
     return results
 
 
@@ -669,29 +690,31 @@ def store_timeseries(
 
     # Store chunks concurrently
     responses: List[Dict[str, Any]] = []
-    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-        # Initialize an empty list to store futures
-        futures = []
-        # Submit each chunk as a separate task to the executor
-        for chunk in chunks:
-            future = executor.submit(
-                api.post,  # The function to execute
-                endpoint,
-                chunk,  # The chunk of data to store
-                params,
-            )
-            futures.append(future)  # Add the future to the list
+    errors: List[str] = []
 
-        for future in concurrent.futures.as_completed(futures):
+    with concurrent.futures.ThreadPoolExecutor(max_workers=actual_workers) as executor:
+        future_to_chunk = {
+            executor.submit(_call_with_retry, api.post, endpoint, chunk, params): chunk
+            for chunk in chunks
+        }
+
+        for future in concurrent.futures.as_completed(future_to_chunk):
+            chunk = future_to_chunk[future]
             try:
-                responses.append({"success:": future.result()})
+                responses.append({"success": future.result()})
             except Exception as e:
                 start_time = chunk["values"][0][0]
                 end_time = chunk["values"][-1][0]
-                logging.error(
-                    f"Error storing chunk from {start_time} to {end_time}: {e}"
-                )
-                responses.append({"error": str(e)})
+                error_msg = f"Error storing chunk from {start_time} to {end_time}: {e}"
+                logging.error(error_msg)
+                errors.append(error_msg)
+                responses.append({"error": error_msg})
+
+    if errors:
+        raise RuntimeError(
+            f"{len(errors)} of {len(chunks)} chunk(s) failed to store:\n"
+            + "\n".join(errors)
+        )
 
     return
 

{cwms_python-1.0.7 → cwms_python-1.0.8}/cwms/timeseries/timeseries_group.py

@@ -11,8 +11,8 @@ from cwms.cwms_types import JSON, Data
 
 def get_timeseries_group(
     group_id: str,
-    category_id: str,
-    category_office_id: str,
+    category_id: Optional[str] = None,
+    category_office_id: Optional[str] = None,
     office_id: Optional[str] = None,
     group_office_id: Optional[str] = None,
 ) -> Data:
@@ -54,14 +54,16 @@ def get_timeseries_groups(
     timeseries_category_like: Optional[str] = None,
     timeseries_group_like: Optional[str] = None,
     category_office_id: Optional[str] = None,
+    group_office_id: Optional[str] = None,
 ) -> Data:
     """
     Retreives a list of time series groups.
 
     Parameters
     ----------
-        category_id: string
-            The category id that contains the timeseries group.
+        office_id: string
+            Specifies the owning office of the timeseries assigned to the group(s).
+            If not specified, group information for all assigned TS offices is returned.
         include_assigned: Boolean
             Include the assigned timeseries in the returned timeseries groups. (default: true)
         timeseries_category_like: string
@@ -70,6 +72,8 @@ def get_timeseries_groups(
             Posix regular expression matching against the timeseries group id
         category_office_id: string
             Specifies the owning office of the timeseries group category
+        group_office_id: string
+            Specifies the owning office of the timeseries group
     Returns
     -------
         cwms data type.  data.json will return the JSON output and data.df will return a dataframe
@@ -78,9 +82,10 @@ def get_timeseries_groups(
     endpoint = "timeseries/group"
     params = {
         "office": office_id,
+        "group-office-id": group_office_id,
         "include-assigned": include_assigned,
         "timeseries-category-like": timeseries_category_like,
-        "timeseries_group_like": timeseries_group_like,
+        "timeseries-group-like": timeseries_group_like,
         "category-office-id": category_office_id,
     }
     response = api.get(endpoint=endpoint, params=params, api_version=1)
@@ -165,7 +170,11 @@ def timeseries_group_df_to_json(
     return json_dict
 
 
-def store_timeseries_groups(data: JSON, fail_if_exists: Optional[bool] = True) -> None:
+def store_timeseries_groups(
+    data: JSON,
+    fail_if_exists: Optional[bool] = True,
+    ignore_nulls: Optional[bool] = True,
+) -> None:
     """
     Create new TimeSeriesGroup
     Parameters
@@ -174,6 +183,11 @@ def store_timeseries_groups(data: JSON, fail_if_exists: Optional[bool] = True) -
             Time Series data to be stored.
         fail_if_exists: Boolean Defualt = True
             Create will fail if provided ID already exists.
+        ignore_nulls: Boolean Default = True
+            Ignore null values in the request body. If fail_if_exists is False
+            and ignore_nulls is False, an existing group's description or
+            assigned time series list may be replaced with null/empty values
+            from the request body.
 
     Returns
     -------
@@ -184,7 +198,7 @@ def store_timeseries_groups(data: JSON, fail_if_exists: Optional[bool] = True) -
         raise ValueError("Cannot store a standard text without timeseries group JSON")
 
     endpoint = "timeseries/group"
-    params = {"fail-if-exists": fail_if_exists}
+    params = {"fail-if-exists": fail_if_exists, "ignore-nulls": ignore_nulls}
 
     return api.post(endpoint, data, params, api_version=1)
 
@@ -227,7 +241,12 @@ def update_timeseries_groups(
     api.patch(endpoint=endpoint, data=data, params=params, api_version=1)
 
 
-def delete_timeseries_group(group_id: str, category_id: str, office_id: str) -> None:
+def delete_timeseries_group(
+    group_id: str,
+    category_id: str,
+    office_id: str,
+    cascade_delete: Optional[bool] = False,
+) -> None:
     """Deletes requested time series group
 
     Parameters
@@ -238,6 +257,8 @@ def delete_timeseries_group(group_id: str, category_id: str, office_id: str) ->
                 Specifies the time series category of the time series group to be deleted
             office_id: string
                 Specifies the owning office of the time series group to be deleted
+            cascade_delete: Boolean Default = False
+                Specifies whether to unassign time series in this group before deleting.
 
         Returns
         -------
@@ -248,6 +269,7 @@ def delete_timeseries_group(group_id: str, category_id: str, office_id: str) ->
     params = {
         "office": office_id,
         "category-id": category_id,
+        "cascade-delete": cascade_delete,
     }
 
     return api.delete(endpoint, params=params, api_version=1)

{cwms_python-1.0.7 → cwms_python-1.0.8}/pyproject.toml

@@ -2,7 +2,7 @@
 name = "cwms-python"
 repository = "https://github.com/HydrologicEngineeringCenter/cwms-python"
 
-version = "1.0.7"
+version = "1.0.8"
 
 packages = [
     { include = "cwms" },