cwms-cli 0.1.1__py3-none-any.whl

cwmscli/usgs/getUSGS_ratings_cda.py

@@ -0,0 +1,346 @@
+import logging
+import sys
+from datetime import datetime, timedelta
+from json import loads
+
+import cwms
+import numpy as np
+import pandas as pd
+import requests
+from dataretrieval import nwis
+
+
+def getusgs_rating_cda(api_root, office_id, days_back, api_key):
+    api_key = "apikey " + api_key
+    cwms.api.init_session(api_root=api_root, api_key=api_key)
+    logging.info(f"CDA connection: {api_root}")
+    logging.info(
+        f"Updated Ratings will be obtained from the USGS for the past {days_back} days"
+    )
+    execution_date = datetime.now()
+    logging.info(f"Execution date {execution_date}")
+
+    logging.info("Getting Rating Specification information from CWMS Database")
+    rating_specs = get_rating_ids_from_specs(office_id)
+    USGS_ratings = get_location_aliases(
+        rating_specs, "USGS Station Number", "Agency Aliases", "CWMS", None, None
+    )
+
+    # grab ratings that don't have an existing rating curve.  ie new specs.
+    USGS_ratings_empty = USGS_ratings[USGS_ratings["effective-dates"].isna()]
+    USGS_ratings = USGS_ratings[USGS_ratings["effective-dates"].notna()]
+
+    logging.info(f"Getting list of ratings updated by USGS in past {days_back} days")
+    df = get_usgs_updated_ratings(days_back * 24)
+
+    updated_ratings = pd.merge(
+        USGS_ratings,
+        df,
+        how="inner",
+        left_on=["USGS_St_Num", "rating-type"],
+        right_on=["USGS_St_Num", "rating-type"],
+    )
+
+    updated_ratings.loc[:, "effective-dates"] = updated_ratings[
+        "effective-dates"
+    ].apply(lambda x: [pd.to_datetime(d) for d in x])
+    updated_ratings.loc[:, "cwms_max_effective_date"] = updated_ratings[
+        "effective-dates"
+    ].apply(max)
+
+    # merge the new specs without an existing curve back into the update ratings df
+    if not USGS_ratings_empty.empty:
+        updated_ratings = pd.concat(
+            [updated_ratings, USGS_ratings_empty], ignore_index=True
+        )
+
+    cwms_write_ratings(updated_ratings)
+
+
+def get_rating_ids_from_specs(office_id):
+    rating_types = ["EXSA", "CORR", "BASE"]
+    rating_specs = cwms.get_rating_specs(office_id=office_id).df
+    if "effective-dates" not in rating_specs.columns:
+        rating_specs["effective-dates"] = np.nan
+    # Determine if any specs return
+    if rating_specs.empty:
+        logging.warning(f"No rating specifications found for office {office_id}")
+        sys.exit()
+    rating_specs = rating_specs.dropna(subset=["description"])
+    for rating_type in rating_types:
+        rating_specs.loc[
+            rating_specs["description"].str.contains(f"USGS-{rating_type}"),
+            "rating-type",
+        ] = rating_type
+    rating_specs = rating_specs[
+        (rating_specs["rating-type"].isin(rating_types))
+        & (rating_specs["active"])
+        & (rating_specs["auto-update"])
+    ]
+    return rating_specs
+
+
+def get_location_aliases(
+    df, loc_group_id, category_id, office_id, category_office_id, group_office_id
+):
+    # CDA get location group endpoint has an error with category and group office ids.  need to update when error is fixed.
+    Locdf = cwms.get_location_group(
+        loc_group_id=loc_group_id,
+        category_id=category_id,
+        office_id=office_id,
+        category_office_id=category_office_id,
+        group_office_id=group_office_id,
+    ).df
+    USGS_alias = Locdf[Locdf["alias-id"].notnull()]
+    USGS_alias = USGS_alias.rename(
+        columns={"alias-id": "USGS_St_Num", "attribute": "Loc_attribute"}
+    )
+    USGS_alias.USGS_St_Num = USGS_alias.USGS_St_Num.str.rjust(8, "0")
+    USGS_ratings = pd.merge(
+        df, USGS_alias, how="inner", on=["location-id", "office-id"]
+    )
+    return USGS_ratings
+
+
+def get_usgs_updated_ratings(period):
+    """
+    Function to grab data from the USGS based off of dataretieve-python
+    """
+    # Get USGS data
+    base_url = "https://nwis.waterdata.usgs.gov/nwisweb/get_ratings"
+
+    query_dict = {"period": period, "format": "rdb"}
+
+    r = requests.get(base_url, params=query_dict)
+    temp = pd.DataFrame(r.text.split("\n"))
+    temp = temp[temp[0].str.startswith("USGS")]
+    updated_ratings = temp[0].str.split("\t", expand=True)
+    updated_ratings.columns = [
+        "org",
+        "USGS_St_Num",
+        "rating-type",
+        "date_updated",
+        "url",
+    ]
+    updated_ratings["rating-type"] = updated_ratings["rating-type"].str.upper()
+    return updated_ratings
+
+
+def convert_tz(tz: str):
+    if tz in ("AST", "ADT"):
+        tzid = "America/Halifax"
+    elif tz in ("EST", "EDT"):
+        tzid = "US/Eastern"
+    elif tz in ("CST", "CDT"):
+        tzid = "US/Central"
+    elif tz in ("MST", "MDT"):
+        tzid = "US/Mountain"
+    elif tz in ("PST", "PDT"):
+        tzid = "US/Pacific"
+    elif tz in ("AKST", "AKDT"):
+        tzid = "America/Anchorage"
+    elif tz in ("UTC", "GMT"):
+        tzid = "UTC"
+    else:
+        tzid = tz
+    return tzid
+
+
+def get_usgs_tz(data):
+    line = data[data[0].str.startswith("# //STATION AGENCY=")].iloc[0, 0]
+    timezone = line.split("TIME_ZONE=")[1].split()[0].replace('"', "")
+    timezone = convert_tz(timezone)
+    return timezone
+
+
+def get_begin_with_date(data, str_starts):
+    date_string = None
+    lines = data[data[0].str.startswith(str_starts)]
+    for _, line in lines.iterrows():
+        timestr = line[0].split("BEGIN=")[1].split()[0].strip().replace('"', "")
+        if timestr.isdigit():
+            date_string = timestr
+    return date_string
+
+
+def get_usgs_effective_date(data, rating_type):
+
+    date_string = None
+    if rating_type == "EXSA":
+        line = data[data[0].str.startswith("# //RATING SHIFTED=")].iloc[0, 0]
+        rating_shifted_date = line.split("=")[1].replace('"', "")
+        date_string = rating_shifted_date.split()[0]
+
+    elif rating_type == "BASE":
+        date_string = get_begin_with_date(data, ("# //RATING_DATETIME BEGIN="))
+
+    elif rating_type == "CORR":
+        date_string = get_begin_with_date(
+            data,
+            ("# //CORR1_PREV BEGIN=", "# //CORR2_PREV BEGIN=", "# //CORR3_PREV BEGIN="),
+        )
+
+    if date_string is None:
+        line = data[data[0].str.startswith("# //RETRIEVED:")].iloc[0, 0]
+        date_string = line.split("RETRIEVED: ")[1]
+
+    timezone = get_usgs_tz(data)
+    dt = pd.to_datetime(date_string).tz_localize(timezone).floor("Min")
+    return dt
+
+
+def convert_usgs_rating_df(df, rating_type):
+    if rating_type == "CORR":
+        df = df.groupby("CORR")
+        df = pd.concat([df.first(), df.last()], ignore_index=True, join="inner")
+        df = df.sort_values(by=["INDEP"], ignore_index=True)
+    df = df.rename(columns={"INDEP": "ind", "CORRINDEP": "dep", "DEP": "dep"})
+    df_out = df[["ind", "dep"]].copy()
+    return df_out
+
+
+def cwms_write_ratings(updated_ratings):
+    storErr = []
+    usgsapiErr = []
+    usgsemptyErr = []
+    usgseffectiveErr = []
+    total_recs = len(updated_ratings.index)
+    saved = 0
+    saved_ratings = []
+    same_effective = 0
+
+    rating_units = {"EXSA": "ft;cfs", "BASE": "ft;cfs", "CORR": "ft;ft"}
+    for _, row in updated_ratings.iterrows():
+        logging.info(f'Getting data for rating ID = {row["rating-id"]}')
+        logging.info(
+            f'Getting data from USGS for USGS ID = {row["USGS_St_Num"]}, Rating Type = {row["rating-type"]}'
+        )
+        try:
+            usgs_rating, meta = nwis.get_ratings(
+                site=row["USGS_St_Num"], file_type=str(row["rating-type"]).lower()
+            )
+            url = meta.url
+        except Exception as error:
+            usgsapiErr.append(
+                [row["rating-id"], row["USGS_St_Num"], row["rating-type"], error]
+            )
+            logging.error(
+                f'FAIL Error collecting rating data from USGS for -->  {row["rating-id"]},{row["USGS_St_Num"]}, {row["rating-type"]} USGS error = {error}'
+            )
+            continue
+        if usgs_rating.empty:
+            logging.warning(
+                f'Empty rating obtained from USGS for USGS ID = {row["USGS_St_Num"]}, Rating Type = {row["rating-type"]}, url'
+            )
+            usgsemptyErr.append(
+                [row["rating-id"], row["USGS_St_Num"], row["rating-type"]]
+            )
+        else:
+            try:
+                response = requests.get(url)
+                temp = pd.DataFrame(response.text.split("\n"))
+                usgs_effective_date = get_usgs_effective_date(temp, row["rating-type"])
+            except Exception as error:
+                usgseffectiveErr.append(
+                    [row["rating-id"], row["USGS_St_Num"], row["rating-type"], error]
+                )
+                logging.error(
+                    f'FAIL Error collecting effective date from USGS rating -->  {row["rating-id"]},{row["USGS_St_Num"]}, {row["rating-type"]} CDA error = {error}'
+                )
+                continue
+            cwms_effective_date = row["cwms_max_effective_date"]
+            logging.info(
+                f"Effective dates: cwms = {cwms_effective_date}, usgs = {usgs_effective_date}"
+            )
+            if (cwms_effective_date == usgs_effective_date) or (
+                cwms_effective_date == (usgs_effective_date + timedelta(hours=1))
+            ):
+                logging.info(
+                    "Effective dates are the same rating curve will not be saved"
+                )
+                same_effective = same_effective + 1
+            else:
+                try:
+                    usgs_store_rating = convert_usgs_rating_df(
+                        usgs_rating, row["rating-type"]
+                    )
+
+                    if row["auto-migrate-extension"] and pd.notna(cwms_effective_date):
+                        current_rating = cwms.get_ratings(
+                            rating_id=row["rating-id"],
+                            office_id=row["office-id"],
+                            begin=cwms_effective_date,
+                            end=cwms_effective_date,
+                            method="EAGER",
+                            single_rating_df=True,
+                        )
+                        rating_json = current_rating.json
+                        points_json = loads(usgs_store_rating.to_json(orient="records"))
+                        rating_json["simple-rating"]["rating-points"] = {
+                            "point": points_json
+                        }
+                        rating_json["simple-rating"][
+                            "effective-date"
+                        ] = usgs_effective_date.isoformat()
+                        del rating_json["simple-rating"]["create-date"]
+                        rating_json["simple-rating"]["active"] = row["auto-activate"]
+                    else:
+                        rating_json = cwms.rating_simple_df_to_json(
+                            data=usgs_store_rating,
+                            rating_id=row["rating-id"],
+                            office_id=row["office-id"],
+                            units=rating_units[row["rating-type"]],
+                            effective_date=usgs_effective_date,
+                            active=row["auto-activate"],
+                        )
+                    response = cwms.update_ratings(
+                        data=rating_json, rating_id=row["rating-id"]
+                    )
+                    logging.info(
+                        f'SUCCESS Stored rating for rating id = {row["rating-id"]}, effective date = {usgs_effective_date}'
+                    )
+                    saved = saved + 1
+                    saved_ratings.append(
+                        [row["rating-id"], row["USGS_St_Num"], row["rating-type"]]
+                    )
+                except Exception as error:
+                    storErr.append(
+                        [
+                            row["rating-id"],
+                            row["USGS_St_Num"],
+                            row["rating-type"],
+                            error,
+                        ]
+                    )
+                    logging.error(
+                        f'FAIL Data could not be stored to CWMS database for -->  {row["rating-id"]},{row["USGS_St_Num"]}, {row["rating-type"]} CDA error = {error}'
+                    )
+    logging.info(
+        f"A total of {total_recs} ratings were updated by the USGS over the lookback period."
+    )
+    logging.info(
+        f"Of those {total_recs} ratings {same_effective} were already stored in the CWMS database"
+    )
+    if len(saved_ratings) > 0:
+        logging.info(
+            f"A total of {saved} ratings were new and saved successfully to the database"
+        )
+        logging.info(
+            f"Rating ids saved successfully to the database were: {saved_ratings}"
+        )
+    if len(usgsapiErr) > 0:
+        logging.info(
+            f"The following ratings errored out when accessing the USGS API: {usgsapiErr}"
+        )
+    if len(usgsemptyErr) > 0:
+        logging.info(
+            f"The following ratings had an empty rating curve returned from the usgs: {usgsemptyErr}"
+        )
+    if len(usgseffectiveErr) > 0:
+        logging.info(
+            f"The following ratings errored when trying to determine the effective date from the USGS: {usgseffectiveErr}"
+        )
+    if len(storErr) > 0:
+        logging.info(
+            f"The following ratings errored when trying to store to CDA: {storErr}"
+        )

cwmscli/usgs/getusgs_cda.py

@@ -0,0 +1,345 @@
+import logging
+from datetime import datetime, timedelta
+
+import cwms
+import numpy as np
+import pandas as pd
+import requests
+
+
+def getusgs_cda(api_root, office_id, days_back, api_key, backfill_tsids: list = None):
+    api_key = "apikey " + api_key
+    cwms.api.init_session(api_root=api_root, api_key=api_key)
+    logging.info(f"CDA connection: {api_root}")
+    logging.info(
+        f"Data will be grabbed and stored from USGS for past {days_back} days for office: {office_id}"
+    )
+    execution_date = datetime.now()
+
+    USGS_ts = get_CMWS_TS_Loc_Data(office_id)
+
+    if backfill_tsids:
+        USGS_ts = USGS_ts[USGS_ts["timeseries-id"].isin(backfill_tsids)]
+
+    if len(USGS_ts) > 0:
+        # grab all of the unique USGS stations numbers to be sent to USGS api
+        sites = USGS_ts[USGS_ts["USGS_Method_TS"].isna()].USGS_St_Num.unique()
+        method_sites = USGS_ts[USGS_ts["USGS_Method_TS"].notna()].USGS_St_Num.unique()
+        logging.info(f"Execution date {execution_date}")
+
+        # This is added to the 'startDT'
+        tw_delta = -timedelta(days_back)
+
+        # Set the execution date and time window for URL
+        startDT = execution_date + tw_delta
+
+        # Airflow only looks at the last period during an execution run,
+        # so to ensure the latest data is retrieved, add 2 hours to end date
+        endDT = execution_date + timedelta(hours=2)
+
+        logging.info(f"Grabing data from USGS between {startDT} and {endDT}")
+
+        USGS_data = pd.DataFrame()
+        USGS_data_method = pd.DataFrame()
+
+        if len(sites) > 0:
+            USGS_data = getUSGS_ts(sites, startDT, endDT)
+        # sites with a method_id or usgs tsid are retrieved from a seperate database. this is access using 3 as access in
+        # usgs API call.
+        if len(method_sites) > 0:
+            USGS_data_method = getUSGS_ts(method_sites, startDT, endDT, 3)
+
+        CWMS_writeData(USGS_ts, USGS_data, USGS_data_method, days_back)
+    else:
+        if backfill_tsids:
+            logging.error(
+                f"The following backload timeseries ids were not present in the USGS timeseries or Locations groups: {backfill_tsids}"
+            )
+        else:
+            logging.error(
+                f"USGS data was present in the timeseries or locations groups"
+            )
+
+
+def get_USGS_params():
+    # defines USGS standard parameters.
+    columns = [
+        "USGS_PARAMETER",
+        "USGS_Alias",
+        "CWMS_PARAMETER",
+        "CWMS_FACTOR",
+        "CWMS_UNIT",
+        "CWMS_TYPE",
+    ]
+    data = [
+        ["00010", "Water Temp", "Temp-Water", 1, "C", "Inst"],
+        ["00021", "Air Temp", "Temp-Air", 1, "F", "Inst"],
+        ["00035", "Wind Speed", "Speed-Wind", 1, "mph", "Inst"],
+        ["00036", "Wind Dir", "Dir-Wind", 1, "deg", "Inst"],
+        ["00045", "Precip", "Precip-Inc", 1, "in", "Total"],
+        ["00045", "Precip", "Precip", 1, "in", "Total"],
+        ["00052", "RelHumidity", "%-Humidity", 1, "%", "Inst"],
+        ["00060", "Flow", "Flow", 1, "cfs", "Inst"],
+        # ['00061','Flow',1,'cfs','Inst'],
+        ["00065", "Stage", "Stage", 1, "ft", "Inst"],
+        ["00095", "Sp Cond", "Cond", 1, "umho/cm", "Inst"],
+        ["00096", "Salinity", "Conc-Salinity", 0.001, "mg/l", "Inst"],
+        # ['00062','Elevation','Elev',1,'ft','Inst'],
+        ["72036", "Res Storage", "Stor", 1000, "ac-ft", "Inst"],
+        ["62608", "Sol Rad", "Irrad-Solar", 1, "W/m2", "Inst"],
+        # ['62614','Elev-Lake','Elev',1,'ft','Inst'],
+        ["63160", "Elev-NAVD88", "Elev", 1, "ft", "Inst"],
+    ]
+    USGS_Params = pd.DataFrame(data, columns=columns).set_index("CWMS_PARAMETER")
+    return USGS_Params
+
+
+def get_CMWS_TS_Loc_Data(office):
+    """
+    get time series group and location alias information and combine into singe dataframe
+
+    """
+
+    def find_usgsparam(attribute, param):
+        if attribute > 0:
+            usgs_param = str(attribute).split(".")[0]
+        elif param in USGS_Params.index:
+            usgs_param = USGS_Params.at[param, "USGS_PARAMETER"]
+        else:
+            usgs_param = "Not Found"
+        return usgs_param
+
+    df = cwms.get_timeseries_group(
+        group_id="USGS TS Data Acquisition",
+        category_id="Data Acquisition",
+        office_id=office,
+        category_office_id="CWMS",
+        group_office_id="CWMS",
+    ).df
+    df[["location-id", "param", "type", "int", "dur", "ver"]] = df[
+        "timeseries-id"
+    ].str.split(".", expand=True)
+
+    df = df[df["office-id"] == office]
+    df["base-loc"] = df["location-id"].str.split("-", expand=True)[0]
+    if "alias-id" not in df.columns:
+        df["alias-id"] = np.nan
+    if "attribute" not in df.columns:
+        df["attribute"] = np.nan
+    df = df.rename(columns={"alias-id": "USGS_Method_TS"})
+
+    # error in CDA with category_office_id and group_office_id. need to fix once CDA is updated
+    Locdf = cwms.get_location_group(
+        loc_group_id="USGS Station Number",
+        category_id="Agency Aliases",
+        office_id="CWMS",
+    ).df.set_index("location-id")
+
+    Locdf = Locdf[Locdf["office-id"] == office]
+    if "attribute" not in Locdf.columns:
+        Locdf["attribute"] = np.nan
+    # Grab all of the locations that have a USGS station number assigned to them
+    USGS_alias = Locdf[Locdf["alias-id"].notnull()]
+    # rename the columns
+    USGS_alias = USGS_alias.rename(
+        columns={"alias-id": "USGS_St_Num", "attribute": "Loc_attribute"}
+    )
+    # pad the USGS id with 0s if they are not 8 digits long
+    USGS_alias.USGS_St_Num = USGS_alias.USGS_St_Num.str.rjust(8, "0")
+
+    # do an inner join with the time series that are in the USGS time series group and the location group.  Join based on the Location ID and office if
+    USGS_ts = pd.merge(df, USGS_alias, how="left", on=["location-id", "office-id"])
+    # grab time series with missing USGS_St_Num and check to see if the base location has an assigned USGS station.
+    if USGS_ts.USGS_St_Num.isnull().any():
+        USGS_ts_base = pd.merge(
+            USGS_ts[USGS_ts.USGS_St_Num.isnull()].drop(
+                ["USGS_St_Num", "Loc_attribute"], axis=1
+            ),
+            USGS_alias,
+            left_on=["base-loc", "office-id"],
+            right_on=["location-id", "office-id"],
+        )
+        # merge with existing dataframe
+        USGS_ts = pd.concat(
+            [USGS_ts[USGS_ts["USGS_St_Num"].notnull()], USGS_ts_base], axis=0
+        )
+
+    USGS_Params = get_USGS_params()
+    # this code fills in the USGS_Params field with values in the Time Series Group Attribute if it exists.  If it does not exist it
+    # grabs the default USGS paramter for the coresponding CWMS parameter
+    USGS_ts.attribute = USGS_ts.apply(
+        lambda x: find_usgsparam(x.attribute, x.param), axis=1
+    ).astype("string")
+    USGS_ts.attribute = USGS_ts.attribute.str.rjust(5, "0")
+    # renames the attribute column to USGS_PARAMETER
+    USGS_ts = USGS_ts.rename(columns={"attribute": "USGS_PARAMETER"})
+
+    logging.info("CWMS TS Groups and Location Data Obtained")
+    return USGS_ts
+
+
+def getUSGS_ts(sites, startDT, endDT, access=None):
+    """
+    Function to grab data from the USGS based off of dataretieve-python
+    """
+
+    # Get USGS data
+    base_url = "https://waterservices.usgs.gov/nwis/iv/?"
+
+    query_dict = {
+        "format": "json",
+        "sites": ",".join(sites),
+        "startDT": startDT.isoformat(),
+        "endDT": endDT.isoformat(),
+        "access": access,
+        # "parameterCd": ",".join(unique_param_codes),
+        # 'period': 'P1D',
+        # "modifiedSince": "PT6H",
+        "siteStatus": "active",
+    }
+
+    r = requests.get(base_url, params=query_dict).json()
+
+    # format the responce from USGS API into dataframe
+    USGS_data = pd.DataFrame(r["value"]["timeSeries"])
+    USGS_data["Id.param"] = (
+        USGS_data.name.str.split(":").str[1]
+        + "."
+        + USGS_data.name.str.split(":").str[2]
+    )
+    USGS_data = USGS_data.set_index("Id.param")
+
+    logging.info("Data obtained from USGS")
+    return USGS_data
+
+
+def CWMS_writeData(USGS_ts, USGS_data, USGS_data_method, days_back):
+    # lists to hold time series that fail
+    # noData -> usgs location and parameter were present in USGS api but the values were empty
+    # NotinAPI -> usgs location and parameter were not retrieved from USGS api
+    # storErr -> an error occured when saving data to CWMS database
+    noData = []
+    NotinAPI = []
+    storErr = []
+    mult_ids = []
+    total_recs = len(USGS_ts.index)
+    saved = 0
+
+    # loop through all rows in the USGS_ts dataframe
+    for index, row in USGS_ts.iterrows():
+        # grab the CWMS time series if and the USGS station numbuer plus USGS parameter code
+        ts_id = row["timeseries-id"]
+        USGS_Id_param = f"{row.USGS_St_Num}.{row.USGS_PARAMETER}"
+        # check if the USGS st number and para code are in the data obtain from USGS api
+        logging.info(
+            f"Attempting to write values for ts_id -->  {ts_id},{USGS_Id_param}"
+        )
+        values = pd.DataFrame()
+        USGS_data_row = None
+        if (USGS_Id_param in USGS_data.index) and pd.isna(row.USGS_Method_TS):
+            USGS_data_row = USGS_data.loc[USGS_Id_param]
+        elif USGS_Id_param in USGS_data_method.index:
+            USGS_data_row = USGS_data_method.loc[USGS_Id_param]
+        if USGS_data_row is not None:
+            try:
+
+                # grab the time series values obtained from USGS API.
+                values_df = pd.DataFrame(USGS_data_row["values"])
+                if values_df.shape[0] > 1:
+                    if pd.isna(row.USGS_Method_TS):
+                        logging.warning(
+                            f"FAIL there are multiple time series for {USGS_Id_param} need to specify the USGS method TSID for {ts_id}"
+                        )
+                        mult_ids.append([ts_id, USGS_Id_param])
+                    else:
+                        temp = values_df.method.apply(pd.Series)
+                        temp = values_df.join(pd.json_normalize(temp.pop(0)))
+                        try:
+                            values = pd.DataFrame(
+                                temp.query(f"methodID == {row.USGS_Method_TS}")[
+                                    "value"
+                                ].item()
+                            )
+                        except Exception as error:
+                            mult_ids.append([ts_id, USGS_Id_param])
+                            logging.error(
+                                f"The USGS method ID defined could not be found from the USGS API check that it is correct for -->  {ts_id},{USGS_Id_param},{row.USGS_Method_TS}"
+                            )
+                else:
+                    values = pd.DataFrame(values_df.loc[0, "value"])
+                # if values array is empty then append infor to noData list
+                if values.empty:
+                    noData.append([ts_id, USGS_Id_param])
+                    logging.warning(
+                        f"FAIL No Data obtained from USGS for ts_id: Values array is empty in USGS API output-->  {ts_id},{USGS_Id_param}"
+                    )
+                else:
+                    # grab value  and for no data (ie -999999) remove from dataset
+                    nodata_val = USGS_data_row["variable"]["noDataValue"]
+                    values = values[values.value != str(int(nodata_val))]
+                    # check again if values dataframe is empty after removing nodata_vals
+                    if values.empty:
+                        noData.append([ts_id, USGS_Id_param])
+                        logging.warning(
+                            f"FAIL No Data obtained from USGS for ts_id: Values array is empty after removing -999999 values-->  {ts_id},{USGS_Id_param}"
+                        )
+                    # if values are present grab information needed to save to CWMS database using CDA
+                    else:
+                        values = values.reindex(
+                            columns=["dateTime", "value", "qualifiers"]
+                        )
+
+                        # adjust column names to fit cwms-python format.
+                        values = values.rename(
+                            columns={
+                                "dateTime": "date-time",
+                                "qualifiers": "quality-code",
+                            }
+                        )
+                        units = USGS_data_row["variable"]["unit"]["unitCode"]
+                        office = row["office-id"]
+                        values["quality-code"] = 0
+
+                        # write values to CWMS database
+                        try:
+                            data = cwms.timeseries_df_to_json(
+                                data=values, ts_id=ts_id, units=units, office_id=office
+                            )
+                            if days_back < 365:
+                                cwms.store_timeseries(data)
+                            else:
+                                cwms.store_timeseries(
+                                    data, max_workers=30, chunk_size=30 * 24 * 4
+                                )
+                            logging.info(
+                                f"SUCCESS Data stored in CWMS database for -->  {ts_id},{USGS_Id_param}"
+                            )
+                            saved = saved + 1
+                        except Exception as error:
+                            storErr.append([ts_id, USGS_Id_param, error])
+                            logging.error(
+                                f"FAIL Data could not be stored to CWMS database for -->  {ts_id},{USGS_Id_param} CDA error = {error}"
+                            )
+            except Exception as error:
+                logging.error(
+                    f"FAIL Unspecified Error when trying to save USGS data -->  {ts_id},{USGS_Id_param} error = {error}"
+                )
+        else:
+            NotinAPI.append([ts_id, USGS_Id_param])
+            logging.warning(
+                f"FAIL USGS ID and parameter were not present in USGS API for-->  {ts_id},{USGS_Id_param}"
+            )
+
+    logging.info(
+        f"A total of {saved} records were successfully saved out of {total_recs}"
+    )
+    logging.info(
+        f"The following ts_ids errored due to no data received from USGS for the time period requested: {noData}"
+    )
+    logging.info(
+        f"The following ts_ids errored because the USGS ID and parameter were not found in USGS API {NotinAPI}"
+    )
+    logging.info(f"The following ts_ids errored when storing into CDA {storErr}")
+    logging.info(
+        f"The following ts_ids errored because multiple method TSID were present for the USGS station. A USGS method TSID needs to be defined in the time series group in CWMS or an incorrect TSID is defined. {mult_ids}"
+    )