loone-data-prep 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl

loone_data_prep/utils.py

@@ -663,6 +663,75 @@ def nutrient_prediction(
         out_dataframe.to_csv(os.path.join(input_dir, f"{station}_PHOSPHATE_predicted.csv"))
 
 
+def find_last_date_in_csv(workspace: str, file_name: str) -> str:
+    """
+    Gets the most recent date from the last line of a .csv file.
+    Assumes the file is formatted as a .csv file, encoded in UTF-8, 
+    and the rows in the file are sorted by date in ascending order.
+
+    Args:
+        workspace (str): The directory where the file is located.
+        file_name (str): The name of the file.
+
+    Returns:
+        str: The most recent date as a string in YYYY-MM-DD format, or None if the file does not exist or the date cannot be found.
+    """
+    # Helper Functions
+    def is_valid_date(date_string):
+        try:
+            datetime.datetime.strptime(date_string, '%Y-%m-%d')
+            return True
+        except ValueError:
+            return False
+    
+    # Check that file exists
+    file_path = os.path.join(workspace, file_name)
+    if not os.path.exists(file_path):
+        return None
+    
+    # Attempt to extract the date of the last line in the file
+    try:
+        with open(file_path, 'rb') as file:
+            # Go to the end of the file
+            file.seek(-2, os.SEEK_END)
+            
+            # Loop backwards until you find the first newline character
+            while file.read(1) != b'\n':
+                file.seek(-2, os.SEEK_CUR)
+                
+            # Read the last line
+            last_line = file.readline().decode()
+            
+            # Extract the date from the last line
+            date = None
+            
+            for value in last_line.split(','):
+                if is_valid_date(value):
+                    date = value
+                    break
+            
+            # Return date
+            return date
+    except OSError as e:
+        print(f"Error reading file {file_name}: {e}")
+        return None
+
+
+def dbhydro_data_is_latest(date_latest: str):
+    """
+    Checks whether the given date is the most recent date possible to get data from dbhydro.
+    Can be used to check whether dbhydro data is up-to-date.
+    
+    Args:
+        date_latest (str): The date of the most recent data of the dbhydro data you have
+    
+    Returns:
+        bool: True if the date_latest is the most recent date possible to get data from dbhydro, False otherwise
+    """
+    date_latest_object = datetime.datetime.strptime(date_latest, "%Y-%m-%d").date()
+    return date_latest_object == (datetime.datetime.now().date() - datetime.timedelta(days=1))
+
+
 if __name__ == "__main__":
     if sys.argv[1] == "get_dbkeys":
         get_dbkeys(sys.argv[2].strip("[]").replace(" ", "").split(","), *sys.argv[3:])

loone_data_prep/water_level_data/get_all.py

@@ -1,34 +1,231 @@
 import sys
 import os
+import requests
+import uuid
+from datetime import datetime
 from loone_data_prep.water_level_data import hydro
+from loone_data_prep.flow_data.get_forecast_flows import get_stations_latitude_longitude
+from loone_data_prep.utils import find_last_date_in_csv, dbhydro_data_is_latest
+import pandas as pd
 
+DATE_NOW = datetime.now().date().strftime("%Y-%m-%d")
 
 D = {
-    "LO_Stage": {"dbkeys": ["16022", "12509", "12519", "16265", "15611"]},
-    "Stg_3ANW": {"dbkeys": ["LA369"], "date_min": "1972-01-01", "date_max": "2023-04-30"},
-    "Stg_2A17": {"dbkeys": ["16531"], "date_min": "1972-01-01", "date_max": "2023-04-30"},
-    "Stg_3A3": {"dbkeys": ["16532"], "date_min": "1972-01-01", "date_max": "2023-04-30"},
-    "Stg_3A4": {"dbkeys": ["16537"], "date_min": "1972-01-01", "date_max": "2023-04-30"},
-    "Stg_3A28": {"dbkeys": ["16538"], "date_min": "1972-01-01", "date_max": "2023-04-30"}
+    "LO_Stage": {"dbkeys": ["16022", "12509", "12519", "16265", "15611"], "datum": "NGVD29"},
+    "LO_Stage_2": {"dbkeys": ["94832"], "date_min": "2024-04-30", "datum": "NAVD88"},
+    "Stg_3ANW": {"dbkeys": ["LA369"], "date_min": "1972-01-01", "date_max": "2023-04-30", "datum": "NGVD29"},
+    "Stg_2A17": {"dbkeys": ["16531"], "date_min": "1972-01-01", "date_max": "2023-04-30", "datum": "NGVD29"},
+    "Stg_3A3": {"dbkeys": ["16532"], "date_min": "1972-01-01", "date_max": "2023-04-30", "datum": "NGVD29"},
+    "Stg_3A4": {"dbkeys": ["16537"], "date_min": "1972-01-01", "date_max": "2023-04-30", "datum": "NGVD29"},
+    "Stg_3A28": {"dbkeys": ["16538"], "date_min": "1972-01-01", "date_max": "2023-04-30", "datum": "NGVD29"}
 }
 
 
 def main(workspace: str, d: dict = D) -> dict:
     missing_files = []
+    failed_downloads = []   # List of file names that the script failed to get the latest data for (but the files still exist)
+    
+    # Get the date of the latest data in LO_Stage_2.csv
+    date_latest_lo_stage_2 = find_last_date_in_csv(workspace, "LO_Stage_2.csv")
+    
     for name, params in d.items():
-        print(f"Getting {name}.")
-        hydro.get(workspace, name, **params)
+        # Get the date of the latest data in the csv file
+        date_latest = find_last_date_in_csv(workspace, f"{name}.csv")
+        
+        # File with data for this dbkey does NOT already exist (or possibly some other error occurred)
+        if date_latest is None:
+            print(f"Getting all water level data for {name}.")
+            hydro.get(workspace, name, **params)
+        else:
+            # Check whether the latest data is already up to date.
+            if dbhydro_data_is_latest(date_latest):
+                # Notify that the data is already up to date
+                print(f'Downloading of new water level data skipped for {name}. Data is already up to date.')
+                continue
+            
+            # Temporarily rename current data file so it isn't over written
+            original_file_name = f"{name}.csv"
+            original_file_name_temp = f"{name}_{uuid.uuid4()}.csv"
+            os.rename(os.path.join(workspace, original_file_name), os.path.join(workspace, original_file_name_temp))
+            
+            try:
+                # Download only the new data
+                print(f'Downloading new water level data for {name} starting from date {date_latest}')
+                hydro.get(workspace, name, dbkeys=params['dbkeys'], date_min=date_latest, date_max=DATE_NOW, datum=params['datum'])
+                
+                # Read in the original data and the newly downloaded data
+                df_original = pd.read_csv(os.path.join(workspace, original_file_name_temp), index_col=0)
+                df_new = pd.read_csv(os.path.join(workspace, original_file_name), index_col=0)
+                
+                # For get_hydro() calls with multiple dbkeys, remove the row corresponding to the latest date from the downloaded data.
+                # When get_hydro() is given multiple keys its returned data starts from the date given instead of the day after like it
+                # does when given a single key.
+                if len(params['dbkeys']) > 1:
+                    df_new = df_new[df_new['date'] != date_latest]
+                
+                # Merge the new data with the original data
+                df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                
+                # Write out the merged data
+                df_merged.to_csv(os.path.join(workspace, original_file_name))
+                
+                # Remove the original renamed data file
+                os.remove(os.path.join(workspace, original_file_name_temp))
+            except Exception as e:
+                # Notify of the error
+                print(f"Error occurred while downloading new water level data: {e}")
+                
+                # Remove the newly downloaded data file if it exists
+                if os.path.exists(os.path.join(workspace, original_file_name)):
+                    os.remove(os.path.join(workspace, original_file_name))
+                
+                # Rename the original renamed file back to its original name
+                if os.path.exists(os.path.join(workspace, original_file_name_temp)):
+                    os.rename(os.path.join(workspace, original_file_name_temp), os.path.join(workspace, original_file_name))
+                
+                # Add the file name to the list of failed downloads
+                failed_downloads.append(original_file_name)
+            
         if os.path.exists(os.path.join(workspace, f"{name}.csv")):
             print(f"{name} downloaded successfully.")
         else:
             missing_files.append(f"{name}.csv")
             print(f"{name} could not be downloaded after various tries.")
 
-    if missing_files:
-        return {"error": f"The following files could not be downloaded: {missing_files}"}
-
+    # Merge data from old and new dbkey for station "L OKEE"
+    convert_failure = False
+    if os.path.exists(os.path.join(workspace, "LO_Stage.csv")) and os.path.exists(os.path.join(workspace, "LO_Stage_2.csv")):
+        # Output Progress
+        print("\nMerging data for station 'L OKEE'...")
+        
+        # Get the latitude and longitude of the "L OKEE" station
+        lat_long_map = get_stations_latitude_longitude(["L OKEE"])
+        latitude, longitude = lat_long_map["L OKEE"]
+        
+        # Load the LO_Stage_2.csv file
+        df_lo_stage_2 = pd.read_csv(os.path.join(workspace, "LO_Stage_2.csv"), index_col="date")
+        df_lo_stage_2.index = pd.to_datetime(df_lo_stage_2.index)
+        
+        # Output Progress
+        print("Converting NAVD88 to NGVD29 for 'L OKEE's new dbkey...\n")
+        
+        # Use only the data that is not already in the LO_Stage.csv file
+        if date_latest_lo_stage_2 is not None:
+            date_start = datetime.strptime(date_latest_lo_stage_2, "%Y-%m-%d") + pd.DateOffset(days=1)
+            df_lo_stage_2 = df_lo_stage_2.loc[date_start:]
+        
+        # Convert the stage values from NAVD88 to NGVD29
+        lo_stage_2_dates = df_lo_stage_2.index.tolist()
+        lo_stage_2_values_navd88 = df_lo_stage_2["L OKEE_STG_ft NGVD29"].tolist()
+        lo_stage_2_values_ngvd29 = []
+        
+        for i in range(0, len(lo_stage_2_values_navd88)):
+            date = lo_stage_2_dates[i]
+            value = lo_stage_2_values_navd88[i]
+            try:
+                lo_stage_2_values_ngvd29.append(_convert_navd88_to_ngvd29(latitude, longitude, value, date.year))
+            except Exception as e:
+                convert_failure = True
+                print(str(e))
+                break
+        
+        # Check for conversion failure
+        if not convert_failure:        
+            # Update the LO_Stage.csv file with the converted values
+            df_lo_stage = pd.read_csv(os.path.join(workspace, "LO_Stage.csv"), index_col="date")
+            df_lo_stage.index = pd.to_datetime(df_lo_stage.index)
+            
+            for i in range(0, len(lo_stage_2_values_ngvd29)):
+                # Get the current date and value
+                date = lo_stage_2_dates[i]
+                value = lo_stage_2_values_ngvd29[i]
+                
+                # Update the value in the LO_Stage dataframe
+                df_lo_stage.at[date, "L OKEE_STG_ft NGVD29"] = value
+            
+            # Reset the index
+            df_lo_stage.reset_index(inplace=True)
+            df_lo_stage.drop(columns=["Unnamed: 0"], inplace=True)
+            
+            # Save the updated LO_Stage.csv file
+            df_lo_stage.to_csv(os.path.join(workspace, "LO_Stage.csv"))
+    else:
+        # Conversion failed due to missing files
+        convert_failure = True
+        print("Error: Missing LO_Stage.csv or LO_Stage_2.csv file, cannot convert and merge.")
+            
+    if missing_files or convert_failure:
+        error_string = ""
+        
+        if missing_files:
+            error_string += f"The following files could not be downloaded: {missing_files}"
+        
+        if failed_downloads:
+            error_string += f"\nFailed to download the latest data for the following files: {failed_downloads}"
+        
+        if convert_failure:
+            error_string += "\nFailed to convert NAVD88 to NGVD29 for 'L OKEE' station."
+            
+        return {"error": error_string}
+    
     return {"success": "Completed water level data download."}
 
+def _convert_navd88_to_ngvd29(latitude: float, longitude: float, stage: float, year: int) -> float:
+    """Converts a stage value from NAVD88 to NGVD29 using NCAT.
+    
+    Args:
+        latitude (float): The latitude of the station (in decimal degrees format).
+        longitude (float): The longitude of the station (in decimal degrees format).
+        stage (float): The stage (water level) value to convert (in feet).
+        year (int): The year when the stage value was recorded.
+        
+    Returns:
+        float: The converted stage value in feet (NGVD29).
+    """
+    # Helper functions
+    def _feet_to_meters(feet: float) -> float:
+        return feet * 0.3048
+    
+    def _meters_to_feet(meters: float) -> float:
+        return meters / 0.3048
+    
+    # Check for NA value
+    if pd.isna(stage):
+        return stage
+    
+    # Convert stage to meters
+    stage_meters = _feet_to_meters(stage)
+    
+    # Make request
+    base_url = "https://geodesy.noaa.gov/api/ncat/llh"
+    
+    params = {
+        "lat": latitude,            # latitude
+        "lon": longitude,           # longitude
+        "orthoHt": stage_meters,    # orthometric height in NAVD88
+        "year": year,               # year of observation
+        "inDatum": "NAD83(1986)",   # Datum used for input latitude and longitude
+        "outDatum": "NAD83(1986)",  # Datum used for output latitude and longitude
+        "inVertDatum": "NAVD88",    # vertical datum of input orthometric height
+        "outVertDatum": "NGVD29",   # vertical datum of output orthometric height (desired vertical datum)
+    }
+    
+    try:
+        response = requests.get(base_url, params=params)
+    except Exception as e:
+        raise Exception(f"Error converting NAVD88 to NGVD29: {e}")
+    
+    # Check for failure
+    if response.status_code != 200:
+        raise Exception(f"Error converting NAVD88 to NGVD29: {response.text}")
+    
+    # Return converted stage in feet
+    try:
+        value = _meters_to_feet(float(response.json()["destOrthoht"]))
+    except Exception as e:
+        raise Exception(f"Error converting NAVD88 to NGVD29: {e}")
+    
+    return value
 
 if __name__ == "__main__":
     workspace = sys.argv[1].rstrip("/")

loone_data_prep/water_level_data/hydro.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from retry import retry
 from rpy2.robjects import r
 from rpy2.rinterface_lib.embedded import RRuntimeError
-
+import pandas as pd
 
 DEFAULT_DBKEYS = ["16022", "12509", "12519", "16265", "15611"]
 DATE_NOW = datetime.now().strftime("%Y-%m-%d")
@@ -16,20 +16,88 @@ def get(
     dbkeys: list = DEFAULT_DBKEYS,
     date_min: str = "1950-01-01",
     date_max: str = DATE_NOW,
+    datum: str = "",
     **kwargs: str | list
 ) -> None:
+    # Get the type and units for the station
+    data_type = "STG"
+    units = "ft NGVD29"
+    
+    if name in ["Stg_3A3", "Stg_2A17", "Stg_3A4", "Stg_3A28"]:
+        data_type = "GAGHT"
+        units = "feet"
+    
     dbkeys_str = "\"" + "\", \"".join(dbkeys) + "\""
     r(
         f"""
         # Load the required libraries
         library(rio)
         library(dbhydroR)
-        #Stage Data
-        {name} = get_hydro(dbkey = c({dbkeys_str}), date_min = "{date_min}", date_max = "{date_max}")
+        library(dplyr)
+        
+        # Stage Data
+        if ("{datum}" == "")
+        {{
+            {name} <- get_hydro(dbkey = c({dbkeys_str}), date_min = "{date_min}", date_max = "{date_max}", raw = TRUE)
+        }}
+        
+        if (nchar("{datum}") > 0)
+        {{
+            {name} <- get_hydro(dbkey = c({dbkeys_str}), date_min = "{date_min}", date_max = "{date_max}", raw = TRUE, datum = "{datum}")
+        }}
+        
+        # Give data.frame correct column names so it can be cleaned using the clean_hydro function
+        colnames({name}) <- c("station", "dbkey", "date", "data.value", "qualifer", "revision.date")
+        
+        # Check if the data.frame has any rows
+        if (nrow({name}) == 0) 
+        {{
+            # No data given back, It's possible that the dbkey has reached its end date.
+            print(paste("Empty data.frame returned for dbkeys", "{dbkeys}", "It's possible that the dbkey has reached its end date. Skipping to the next dbkey."))
+            return(list(success = FALSE, dbkey = "{dbkeys}"))
+        }}
+        
+        # Get the station
+        station <- {name}$station[1]
+        
+        # Add a type and units column to data so it can be cleaned using the clean_hydro function
+        {name}$type <- "{data_type}"
+        {name}$units <- "{units}"
+        
+        # Clean the data.frame
+        {name} <- clean_hydro({name})
+        
+        # Drop the " _STG_ft NGVD29" column
+        {name} <- {name} %>% select(-` _{data_type}_{units}`)
+        
+        # Write the data to a csv file
         write.csv({name},file ='{workspace}/{name}.csv')
         """
     )
+    
+    _reformat_water_level_file(workspace, name)
 
+def _reformat_water_level_file(workspace: str, name: str):
+    # Read in the data
+    df = pd.read_csv(f"{workspace}/{name}.csv")
+    
+    # Drop the "Unnamed: 0" column
+    df.drop(columns=['Unnamed: 0'], inplace=True)
+    
+    # Convert date column to datetime
+    df['date'] = pd.to_datetime(df['date'], format='%d-%b-%Y')
+    
+    # Sort the data by date
+    df.sort_values('date', inplace=True)
+    
+    # Renumber the index
+    df.reset_index(drop=True, inplace=True)
+    
+    # Drop rows that are missing all their values
+    df.dropna(how='all', inplace=True)
+    
+    # Write the updated data back to the file
+    df.to_csv(f"{workspace}/{name}.csv")
 
 if __name__ == "__main__":
     args = [sys.argv[1].rstrip("/"), sys.argv[2]]

loone_data_prep/water_quality_data/get_inflows.py

@@ -1,6 +1,10 @@
 import sys
 import os
+import uuid
+from datetime import datetime, timedelta
+import pandas as pd
 from loone_data_prep.water_quality_data import wq
+from loone_data_prep.utils import find_last_date_in_csv, dbhydro_data_is_latest
 
 
 D = {
@@ -21,18 +25,99 @@ D = {
 }
 
 
+DEFAULT_DATE_MIN = "1950-01-01"
+
+
 def main(workspace: str, d: dict = D) -> dict:
     missing_files = []
+    failed_downloads = []   # List of file names that the script failed to get the latest data for (but the files still exist)
     for name, params in d.items():
         print(f"Getting {name} for the following station IDs: {params['station_ids']}.")
-        wq.get(workspace, name, **params)
+        
+        # Get the date of the latest data in the csv file for each station id
+        station_date_latest = {}
+        for station_id in params["station_ids"]:
+            station_date_latest[station_id] = find_last_date_in_csv(workspace, f"water_quality_{station_id}_{name}.csv")
+        
+        # Get the water quality data
+        for station_id, date_latest in station_date_latest.items():
+            # File with data for this station/name combination does NOT already exist (or possibly some other error occurred)
+            if date_latest is None:
+                # Get all the water quality data for the name/station combination
+                print(f"Getting all {name} data for station ID: {station_id}.")
+                wq.get(workspace, name, [station_id])
+            else:
+                # Check whether we already have the latest data
+                if dbhydro_data_is_latest(date_latest):
+                    # Notify that the data is already up to date
+                    print(f'Downloading of new water quality data for test name: {name} station: {station} skipped. Data is already up to date.')
+                    continue
+                
+                # Temporarily rename current data file so it isn't over written
+                original_file_name = f"water_quality_{station_id}_{name}.csv"
+                original_file_name_temp = f"water_quality_{station_id}_{name}_{uuid.uuid4()}.csv"
+                os.rename(os.path.join(workspace, original_file_name), os.path.join(workspace, original_file_name_temp))
+                
+                try:
+                    # Get only the water quality data that is newer than the latest data in the csv file
+                    print(f"Downloading new water quality data for test name: {name} station ID: {station_id} starting from date: {date_latest}.")
+                    date_latest = (datetime.strptime(date_latest, "%Y-%m-%d") + timedelta(days=1)).strftime("%Y-%m-%d")
+                    wq.get(workspace, name, [station_id], date_min=date_latest)
+                    
+                    # Data failed to download - It's possible the data's end date has been reached
+                    if not os.path.exists(os.path.join(workspace, original_file_name)):
+                        raise Exception(f"It's possible that the data for test name: {name} station ID: {station_id} has reached its end date.")
+                    
+                    # Read in the original data
+                    df_original = pd.read_csv(os.path.join(workspace, original_file_name_temp), index_col=0)
+                    
+                    # Calculate the days column for the newly downloaded data
+                    df_original_date_min = df_original['date'].min()
+                    wq._calculate_days_column(workspace, original_file_name, df_original_date_min)
+                    
+                    # Read in the newly downloaded data
+                    df_new = pd.read_csv(os.path.join(workspace, original_file_name), index_col=0)
+                    df_new.reset_index(inplace=True)
+                    
+                    # Merge the new data with the original data
+                    df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                    
+                    # Write out the merged data
+                    df_merged.to_csv(os.path.join(workspace, original_file_name))
+                    
+                    # Remove the original renamed data file
+                    os.remove(os.path.join(workspace, original_file_name_temp))
+                except Exception as e:
+                    # Notify of the error
+                    print(f"Error occurred while downloading new water quality data: {e}")
+                    
+                    # Remove the newly downloaded data file if it exists
+                    if os.path.exists(os.path.join(workspace, original_file_name)):
+                        os.remove(os.path.join(workspace, original_file_name))
+                    
+                    # Rename the original renamed file back to its original name
+                    if os.path.exists(os.path.join(workspace, original_file_name_temp)):
+                        os.rename(os.path.join(workspace, original_file_name_temp), os.path.join(workspace, original_file_name))
+                    
+                    # Add the file name to the list of failed downloads
+                    failed_downloads.append(original_file_name)
+        
+        # Check for any download failures
         for station in params["station_ids"]:
             if not os.path.exists(os.path.join(workspace, f"water_quality_{station}_{name}.csv")):
                 missing_files.append(f"water_quality_{station}_{name}.csv")
                 print(f"{name} station ID: {station} could not be downloaded after various tries.")
 
-    if missing_files:
-        return {"error": f"The following files could not be downloaded: {missing_files}"}
+    if missing_files or failed_downloads:
+        error_string = ""
+        
+        if missing_files:
+            error_string += f"The following files could not be downloaded: {missing_files}"
+            
+        if failed_downloads:
+            error_string += f"\nThe following files could not be updated: {failed_downloads}"
+            
+        return {"error": error_string}
 
     return {"success": "Completed water quality data download."}
 

loone_data_prep/water_quality_data/get_lake_wq.py

@@ -1,6 +1,10 @@
 import sys
 import os
+import uuid
+from datetime import datetime, timedelta
+import pandas as pd
 from loone_data_prep.water_quality_data import wq
+from loone_data_prep.utils import find_last_date_in_csv,  dbhydro_data_is_latest
 
 
 D = {
@@ -28,16 +32,94 @@ D = {
 
 def main(workspace: str, d: dict = D) -> dict:
     missing_files = []
+    failed_downloads = []   # List of file names that the script failed to get the latest data for (but the files still exist)
     for name, params in d.items():
         print(f"Getting {name} for the following station IDs: {params['station_ids']}.")
-        wq.get(workspace, name, **params)
+        
+        # Get the date of the latest data in the csv file for each station id
+        station_date_latest = {}
+        for station_id in params["station_ids"]:
+            station_date_latest[station_id] = find_last_date_in_csv(workspace, f"water_quality_{station_id}_{name}.csv")
+        
+        # Get the water quality data
+        for station_id, date_latest in station_date_latest.items():
+            # File with data for this station/name combination does NOT already exist (or possibly some other error occurred)
+            if date_latest is None:
+                # Get all the water quality data for the name/station combination
+                print(f"Getting all {name} data for station ID: {station_id}.")
+                wq.get(workspace, name, [station_id])
+            else:
+                # Check whether we already have the latest data
+                if dbhydro_data_is_latest(date_latest):
+                    # Notify that the data is already up to date
+                    print(f'Downloading of new water quality data for test name: {name} station: {station} skipped. Data is already up to date.')
+                    continue
+                
+                # Temporarily rename current data file so it isn't over written
+                original_file_name = f"water_quality_{station_id}_{name}.csv"
+                original_file_name_temp = f"water_quality_{station_id}_{name}_{uuid.uuid4()}.csv"
+                os.rename(os.path.join(workspace, original_file_name), os.path.join(workspace, original_file_name_temp))
+                
+                try:
+                    # Get only the water quality data that is newer than the latest data in the csv file
+                    print(f"Downloading new water quality data for test name: {name} station ID: {station_id} starting from date: {date_latest}.")
+                    date_latest = (datetime.strptime(date_latest, "%Y-%m-%d") + timedelta(days=1)).strftime("%Y-%m-%d")
+                    wq.get(workspace, name, [station_id], date_min=date_latest)
+                    
+                    # Data failed to download - It's possible the data's end date has been reached
+                    if not os.path.exists(os.path.join(workspace, original_file_name)):
+                        raise Exception(f"It's possible that the data for test name: {name} station ID: {station_id} has reached its end date.")
+                    
+                    # Read in the original data
+                    df_original = pd.read_csv(os.path.join(workspace, original_file_name_temp), index_col=0)
+                    
+                    # Calculate the days column for the newly downloaded data
+                    df_original_date_min = df_original['date'].min()
+                    wq._calculate_days_column(workspace, original_file_name, df_original_date_min)
+                    
+                    # Read in the newly downloaded data
+                    df_new = pd.read_csv(os.path.join(workspace, original_file_name), index_col=0)
+                    df_new.reset_index(inplace=True)
+                    
+                    # Merge the new data with the original data
+                    df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                    
+                    # Write out the merged data
+                    df_merged.to_csv(os.path.join(workspace, original_file_name))
+                    
+                    # Remove the original renamed data file
+                    os.remove(os.path.join(workspace, original_file_name_temp))
+                except Exception as e:
+                    # Notify of the error
+                    print(f"Error occurred while downloading new water quality data: {e}")
+                    
+                    # Remove the newly downloaded data file if it exists
+                    if os.path.exists(os.path.join(workspace, original_file_name)):
+                        os.remove(os.path.join(workspace, original_file_name))
+                    
+                    # Rename the original renamed file back to its original name
+                    if os.path.exists(os.path.join(workspace, original_file_name_temp)):
+                        os.rename(os.path.join(workspace, original_file_name_temp), os.path.join(workspace, original_file_name))
+                    
+                    # Add the file name to the list of failed downloads
+                    failed_downloads.append(original_file_name)
+        
+        # Check for missing files
         for station in params["station_ids"]:
             if not os.path.exists(os.path.join(workspace, f"water_quality_{station}_{name}.csv")):
                 missing_files.append(f"water_quality_{station}_{name}.csv")
                 print(f"{name} station ID: {station} could not be downloaded after various tries.")
 
-    if missing_files:
-        return {"error": f"The following files could not be downloaded: {missing_files}"}
+    if missing_files or failed_downloads:
+        error_string = ""
+        
+        if missing_files:
+            error_string += f"The following files could not be downloaded: {missing_files}"
+            
+        if failed_downloads:
+            error_string += f"\nThe following files could not be updated: {failed_downloads}"
+            
+        return {"error": error_string}
 
     return {"success": "Completed water quality data download."}