loone-data-prep 0.1.6__tar.gz → 0.1.7__tar.gz

{loone_data_prep-0.1.6 → loone_data_prep-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: loone_data_prep
-Version: 0.1.6
+Version: 0.1.7
 Summary: Prepare data to run the LOONE model.
 Author-email: Osama Tarabih <osamatarabih@usf.edu>
 Maintainer-email: Michael Souffront <msouffront@aquaveo.com>, James Dolinar <jdolinar@aquaveo.com>

{loone_data_prep-0.1.6 → loone_data_prep-0.1.7}/loone_data_prep/LOONE_DATA_PREP.py

@@ -383,7 +383,6 @@ def main(input_dir: str, output_dir: str) -> None:
     LOWS['LZ40WS'] = LZ40WS['LZ40_WNDS_MPH']
     LOWS['LO_Avg_WS_MPH'] = LOWS.mean(axis=1, numeric_only=True)
     LOWS.to_csv(f'{output_dir}/LOWS.csv', index=False)
-    LOWS.to_csv(f'{input_dir}/LOWS.csv', index=False)  # Also needed in temporary directory by utils.py's wind_induced_waves()
 
     # RFVol acft
     # Create File (RF_Volume)
@@ -897,7 +896,6 @@ def main(input_dir: str, output_dir: str) -> None:
     # Write Data into csv files
     # write Avg Stage (ft, m) Storage (acft, m3) SA (acres) to csv
     LO_Stg_Sto_SA_df.to_csv(f'{output_dir}/Average_LO_Storage_3MLag.csv', index=False)
-    LO_Stg_Sto_SA_df.to_csv(f'{input_dir}/Average_LO_Storage_3MLag.csv', index=False)   # Also needed in temporary directory by utils.py's wind_induced_waves()
     # Write S65 TP concentrations (mg/L)
     S65_total_TP.to_csv(f'{output_dir}/S65_TP_3MLag.csv', index=False)
     # TP External Loads 3 Months Lag (mg)

loone_data_prep-0.1.7/loone_data_prep/flow_data/S65E_total.py

@@ -0,0 +1,89 @@
+import sys
+from retry import retry
+from rpy2.robjects import r
+from rpy2.rinterface_lib.embedded import RRuntimeError
+import pandas as pd
+
+
+@retry(RRuntimeError, tries=5, delay=15, max_delay=60, backoff=2)
+def get(
+    workspace, 
+    date_min: str = "1972-01-01", 
+    date_max: str = "2023-06-30"
+) -> None:
+    r(
+        f"""
+        # Load the required libraries
+        library(dbhydroR)
+        library(dplyr)
+        
+        # Helper Functions
+        retrieve_data <- function(dbkey, date_min, date_max) 
+        {{
+            # Get the data from dbhydro
+            df = get_hydro(dbkey = dbkey, date_min = date_min, date_max = date_max, raw = TRUE)
+        
+            # Give data.frame correct column names so it can be cleaned using the clean_hydro function
+            colnames(df) <- c("station", "dbkey", "date", "data.value", "qualifer", "revision.date")
+            
+            # Add a type and units column to data so it can be cleaned using the clean_hydro function
+            df$type <- "FLOW"
+            df$units <- "cfs"
+            
+            # Clean the data.frame
+            df <- clean_hydro(df)
+            
+            # Drop the " _FLOW_cfs" column
+            df <- df %>% select(-` _FLOW_cfs`)
+            
+            # Convert Flow rate from cfs to m³/day
+            df[, -1] <- df[, -1] * (0.0283168466 * 86400)
+            
+            # Return resulting data.frame
+            return(df)
+        }}
+        
+        # S65E_S
+        S65E_S <- retrieve_data(dbkey = "91656", date_min = "{date_min}", date_max = "{date_max}")
+        
+        # Wait five seconds before next request to avoid "too many requests" error
+        Sys.sleep(5)
+        
+        # S65EX1_S
+        S65EX1_S <- retrieve_data(dbkey = "AL760", date_min = "{date_min}", date_max = "{date_max}")
+        
+        # Merge the data from each dbkey
+        result <- merge(S65E_S, S65EX1_S, by = "date", all = TRUE)
+        
+        # Write the data to a file
+        write.csv(result, file = '{workspace}/S65E_total.csv')
+        """
+    )
+    
+    _reformat_s65e_total_file(workspace)
+
+def _reformat_s65e_total_file(workspace: str):
+    # Read in the data
+    df = pd.read_csv(f"{workspace}/S65E_total.csv")
+    
+    # Drop unused columns
+    df.drop('Unnamed: 0', axis=1, inplace=True)
+    
+    # Convert date column to datetime
+    df['date'] = pd.to_datetime(df['date'], format='%d-%b-%Y')
+    
+    # Sort the data by date
+    df.sort_values('date', inplace=True)
+    
+    # Renumber the index
+    df.reset_index(drop=True, inplace=True)
+    
+    # Drop rows that are missing all their values
+    df.dropna(how='all', inplace=True)
+    
+    # Write the updated data back to the file
+    df.to_csv(f"{workspace}/S65E_total.csv")
+
+if __name__ == "__main__":
+    workspace = sys.argv[1].rstrip("/")
+    get(workspace)

loone_data_prep-0.1.7/loone_data_prep/flow_data/get_inflows.py

@@ -0,0 +1,159 @@
+import sys
+import os
+from glob import glob
+from datetime import datetime
+import uuid
+import pandas as pd
+from loone_data_prep.flow_data import hydro, S65E_total
+from loone_data_prep.utils import find_last_date_in_csv, dbhydro_data_is_latest
+
+
+# Database keys for needed inflow data mapped to their stations
+DBKEYS = {
+    "91370": "S127_C",
+    "91371": "S127_P",
+    "91373": "S129_C",
+    "91377": "S133_P",
+    "91379": "S135_C",
+    "91401": "S154_C",
+    "91429": "S191_S",
+    "91473": "S2_P",
+    "91508": "S351_S",
+    "91510": "S352_S",
+    "91513": "S354_S",
+    "91599": "S3_P",
+    "91608": "S4_P",
+    "91656": "S65E_S",
+    "91668": "S71_S",
+    "91675": "S72_S",
+    "91687": "S84_S",
+    "15627": "FISHP",
+    "15640": "L8.441",
+    "15626": "S308.DS",
+    "15642": "S129 PMP_P",
+    "15638": "S135 PMP_P",
+}
+
+def main(workspace: str, dbkeys: dict = DBKEYS) -> dict:
+    """
+    Retrieve the inflows data used by LOONE. 
+    
+    Args:
+        workspace (str): Path to workspace where data will be downloaded.
+        dbkeys (dict): Dictionary of dbkeys and corresponding station names.
+        
+    Returns:
+        dict: Success or error message
+    """
+    
+    # Retrieve inflow data
+    for dbkey, station in dbkeys.copy().items():
+        file_name = f"{station}_FLOW_cmd.csv"
+        date_latest = find_last_date_in_csv(workspace, file_name)
+        
+        # File with data for this dbkey does NOT already exist (or possibly some other error occurred)
+        if date_latest is None:
+            # Download all the data
+            print(f'Downloading all inflow data for {station}')
+            hydro.get(workspace, dbkey)
+        else:
+            # Check whether the latest data is already up to date.
+            if dbhydro_data_is_latest(date_latest):
+                # Notify that the data is already up to date
+                print(f'Downloading of new inflow data skipped for Station {station} (dbkey: {dbkey}). Data is already up to date.')
+                
+                # Remove dbkey from dbkeys so we know it didn't fail
+                del dbkeys[dbkey]
+                continue
+            
+            # Download only the new data
+            print(f'Downloading new inflow data for {station} starting from date {date_latest}')
+            hydro.get(workspace, dbkey, date_latest)
+            
+            # Make sure both our original data and newly downloaded data exist
+            df_original_path = os.path.join(workspace, f"{station}_FLOW_cmd.csv")
+            df_new_path = os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv")
+            
+            if os.path.exists(df_original_path) and os.path.exists(df_new_path):
+                # Merge the new data with the old data
+                df_original = pd.read_csv(df_original_path, index_col=0)
+                df_new = pd.read_csv(df_new_path, index_col=0)
+                df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                
+                # Write the merged data to the new file
+                df_merged.to_csv(os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv"))
+                
+                # Remove the old file
+                os.remove(os.path.join(workspace, f"{station}_FLOW_cmd.csv"))
+
+    # Download S65E_total.csv Data
+    date_latest = find_last_date_in_csv(workspace, "S65E_total.csv")
+
+    if date_latest is None:
+        print('Downloading all S65E_total data')
+        S65E_total.get(workspace, date_max=datetime.now().strftime("%Y-%m-%d"))
+    else:
+        # Check whether the latest data is already up to date.
+        if dbhydro_data_is_latest(date_latest):
+            # Notify that the data is already up to date
+            print(f'Downloading of new inflow data skipped for S65E_total. Data is already up to date.')
+        else:
+            # Temporarily rename current data file so it isn't over written
+            original_file_name = F"S65E_total_old_{uuid.uuid4()}.csv"
+            os.rename(os.path.join(workspace, "S65E_total.csv"), os.path.join(workspace, original_file_name))
+            
+            try:
+                # Download only the new data
+                print(f'Downloading new S65E_total data starting from date {date_latest}')
+                S65E_total.get(workspace, date_min=date_latest, date_max=datetime.now().strftime("%Y-%m-%d"))
+                
+                # Merge the new data with the original data
+                df_original = pd.read_csv(os.path.join(workspace, original_file_name), index_col=0)
+                df_new = pd.read_csv(os.path.join(workspace, "S65E_total.csv"), index_col=0)
+                df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                
+                # Write out the merged data
+                df_merged.to_csv(os.path.join(workspace, original_file_name))
+                
+                # Remove the newly downloaded data file
+                os.remove(os.path.join(workspace, "S65E_total.csv"))
+            except Exception as e:
+                print(f"Error occurred while downloading new S65E_total data: {e}")
+            finally:
+                # Rename the original updated file back to its original name
+                os.rename(os.path.join(workspace, original_file_name), os.path.join(workspace, "S65E_total.csv"))
+                
+    # Check if all files were downloaded
+    files = glob(f"{workspace}/*FLOW*_cmd.csv")
+
+    for file in files:
+        file_dbkey = file.split("_")[-2]
+
+        if file_dbkey in dbkeys:
+            # Remove dbkey from file name
+            new_file_name = file.replace(f"_{file_dbkey}", "")
+            os.rename(file, new_file_name)
+
+            # Remove dbkey from dbkeys so we know it successfully downloaded
+            del dbkeys[file_dbkey]
+    
+    # Check for failed downloads
+    if len(dbkeys) > 0 or not os.path.exists(f"{workspace}/S65E_total.csv"):
+        error_message = ""
+        
+        # dbkeys
+        if len(dbkeys) > 0:
+            error_message += f"The data from the following dbkeys could not be downloaded: {list(dbkeys.keys())}\n"
+        
+        # S65E_total.csv
+        if not os.path.exists(f"{workspace}/S65E_total.csv"):
+            error_message += "S65E_total.csv file could not be downloaded.\n"
+        
+        return {"error": error_message}
+
+    return {"success": "Completed inflow flow data download."}
+
+
+if __name__ == "__main__":
+    workspace = sys.argv[1].rstrip("/")
+    main(workspace)

loone_data_prep-0.1.7/loone_data_prep/flow_data/get_outflows.py

@@ -0,0 +1,164 @@
+import sys
+import os
+from glob import glob
+import pandas as pd
+from loone_data_prep.utils import get_dbkeys, find_last_date_in_csv, dbhydro_data_is_latest
+from loone_data_prep.flow_data import hydro
+
+STATION_IDS = [
+    "S308.DS",
+    "S77_S",
+    "L8.441",
+    "S127_C",
+    "S129_C",
+    "S135_C",
+    "S351_S",
+    "S352_S",
+    "S354_S",
+    "INDUST",
+    "S79",
+    "S80_S",
+    "S2_NNR",
+    "S3",
+    "S48_S",
+    "S49_S",
+]
+
+
+DBKEYS = {
+    "91370": "S127_C",
+    "91373": "S129_C",
+    "91379": "S135_C",
+    "91508": "S351_S",
+    "91510": "S352_S",
+    "91513": "S354_S",
+    "91677": "S77_S",
+    "15628": "INDUST",
+    "15640": "L8.441",
+    "15626": "S308.DS",
+    "00865": "S79_TOT",
+    "JW224": "S80_S",
+    "00436": "S2 NNR",
+    "15018": "S3",
+    "91606": "S48_S",
+    "JW223": "S49_S",
+}
+
+
+def _get_outflow_data_from_station_ids(workspace: str, station_ids: list) -> dict:
+    """Attempt to download outflow data from station ids.
+    
+    Args:
+        workspace (str): Path to workspace where data will be downloaded.
+        station_ids (list): List of station ids to download data for.
+        
+    Returns:
+        dict: Success or error message
+    """
+    # Get dbkeys from station ids
+    dbkeys = list(get_dbkeys(station_ids, "SW", "FLOW", "MEAN", "PREF", detail_level="dbkey"))
+    dbkeys.extend(list(get_dbkeys(station_ids, "SW", "FLOW", "MEAN", "DRV", detail_level="dbkey")))
+
+    for dbkey in dbkeys:
+        hydro.get(workspace, dbkey, "2000-01-01")
+
+    # Check if all files were downloaded
+    files = glob(f"{workspace}/*FLOW*_cmd.csv")
+
+    for file in files:
+        file_dbkey = file.split("_")[-2]
+
+        if file_dbkey in dbkeys:
+            # Remove dbkey from file name
+            new_file_name = file.replace(f"_{file_dbkey}", "")
+            os.rename(file, new_file_name)
+
+            # Remove dbkey from dbkeys so we know it successfully downloaded
+            dbkeys.remove(file_dbkey)
+
+    if len(dbkeys) > 0:
+        return {"error": f"The data from the following dbkeys could not be downloaded: {dbkeys}"}
+
+    return {"success": "Completed outflow flow data download."}
+
+
+def main(workspace: str, dbkeys: dict = DBKEYS, station_ids: list = STATION_IDS) -> dict:
+    """
+    Retrieve the outflow data used by LOONE. 
+    
+    Args:
+        workspace (str): Path to workspace where data will be downloaded.
+        dbkeys (dict): Dictionary of dbkeys and corresponding station names.
+        station_ids (list): List of station ids to download data for if the dbkeys argument is not provided.
+        
+    Returns:
+        dict: Success or error message
+    """
+    
+    # No dbkeys given, attempt to get data from station ids
+    if dbkeys is None:
+        return _get_outflow_data_from_station_ids(workspace, station_ids)
+
+    # Get outflow data from dbkeys
+    for dbkey, station in dbkeys.copy().items():
+        # Get the date of the latest data in the csv file (if any)
+        date_latest = find_last_date_in_csv(workspace, f"{station}_FLOW_cmd.csv")
+        
+        # File with data for this dbkey does NOT already exist (or possibly some other error occurred)
+        if date_latest is None:
+            # Download all data
+            print(f'Downloading all outflow data for {station}')
+            hydro.get(workspace, dbkey, "2000-01-01")
+        else:
+            # Check whether the latest data is already up to date.
+            if dbhydro_data_is_latest(date_latest):
+                # Notify that the data is already up to date
+                print(f'Downloading of new outflow data skipped for Station {station} (dbkey: {dbkey}). Data is already up to date.')
+                
+                # Remove dbkey from dbkeys so we know it didn't fail
+                del dbkeys[dbkey]
+                continue
+            
+            # Download only the new data
+            print(f'Downloading new outflow data for {station} starting from date {date_latest}')
+            hydro.get(workspace, dbkey, date_latest)
+            
+            # Make sure both our original data and newly downloaded data exist
+            df_old_path = os.path.join(workspace, f"{station}_FLOW_cmd.csv")
+            df_new_path = os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv")
+            
+            if os.path.exists(df_old_path) and os.path.exists(df_new_path):
+                # Merge the new data with the old data
+                df_original = pd.read_csv(df_old_path, index_col=0)
+                df_new = pd.read_csv(df_new_path, index_col=0)
+                df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                
+                # Write the merged data to the new file
+                df_merged.to_csv(os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv"))
+                
+                # Remove the old file
+                os.remove(os.path.join(workspace, f"{station}_FLOW_cmd.csv"))
+
+    # Check if all files were downloaded
+    files = glob(f"{workspace}/*FLOW*_cmd.csv")
+
+    for file in files:
+        file_dbkey = file.split("_")[-2]
+
+        if file_dbkey in dbkeys:
+            # Remove dbkey from file name
+            new_file_name = file.replace(f"_{file_dbkey}", "")
+            os.rename(file, new_file_name)
+
+            # Remove dbkey from dbkeys so we know it successfully downloaded
+            del dbkeys[file_dbkey]
+
+    if len(dbkeys) > 0:
+        return {"error": f"The data from the following dbkeys could not be downloaded: {dbkeys}"}
+
+    return {"success": "Completed outflow flow data download."}
+
+
+if __name__ == "__main__":
+    workspace = sys.argv[1].rstrip("/")
+    main(workspace)

loone_data_prep-0.1.7/loone_data_prep/flow_data/hydro.py

@@ -0,0 +1,155 @@
+import sys
+from datetime import datetime
+from glob import glob
+from retry import retry
+import os
+import pandas as pd
+from rpy2.robjects import r
+from rpy2.rinterface_lib.embedded import RRuntimeError
+
+
+DATE_NOW = datetime.now().strftime("%Y-%m-%d")
+
+
+@retry(RRuntimeError, tries=5, delay=15, max_delay=60, backoff=2)
+def get(
+    workspace: str,
+    dbkey: str,
+    date_min: str = "1990-01-01",
+    date_max: str = DATE_NOW
+) -> None:
+    r_str = f"""
+    download_flow_data <- function(workspace, dbkey, date_min, date_max) 
+    {{
+        # Load the required libraries
+        library(dbhydroR)
+        library(dplyr)
+
+        # Retrieve data for the dbkey
+        data <- get_hydro(dbkey = "{dbkey}", date_min = "{date_min}", date_max = "{date_max}", raw = TRUE)
+        
+        # Check if data is empty or contains only the "date" column
+        if (ncol(data) <= 1) {{
+            cat("No data found for dbkey", "{dbkey}", "Skipping to the next dbkey.\n")
+        }}
+        
+        # Give data.frame correct column names so it can be cleaned using the clean_hydro function
+        colnames(data) <- c("station", "dbkey", "date", "data.value", "qualifer", "revision.date")
+        
+        # Check if the data.frame has any rows
+        if (nrow(data) == 0) 
+        {{
+            # No data given back, It's possible that the dbkey has reached its end date.
+            print(paste("Empty data.frame returned for dbkey", "{dbkey}", "It's possible that the dbkey has reached its end date. Skipping to the next dbkey."))
+            return(list(success = FALSE, dbkey = "{dbkey}"))
+        }}
+        
+        # Add a type and units column to data so it can be cleaned using the clean_hydro function
+        data$type <- "FLOW"
+        data$units <- "cfs"
+        
+        # Get the station
+        station <- data$station[1]
+        
+        # Clean the data.frame
+        data <- clean_hydro(data)
+
+        # Multiply all columns except "date" column by 0.0283168466 * 86400 to convert Flow rate from cfs to m³/day
+        data[, -1] <- data[, -1] * (0.0283168466 * 86400)
+        
+        # Drop the " _FLOW_cfs" column
+        data <- data %>% select(-` _FLOW_cfs`)    
+        
+        # Sort the data by date
+        data <- data[order(data$date), ]
+        
+        # Get the filename for the output CSV file
+        filename <- paste0(station, "_FLOW", "_{dbkey}_cmd.csv")
+        
+        # Save data to a CSV file
+        write.csv(data, file = paste0("{workspace}/", filename))
+
+        # Print a message indicating the file has been saved
+        cat("CSV file", filename, "has been saved.\n")
+
+        # Add a delay between requests
+        Sys.sleep(1)  # Wait for 1 second before the next iteration
+        
+        # Return the station and dbkey to the python code
+        list(success = TRUE, station = station, dbkey = "{dbkey}")
+    }}
+    """
+
+    r(r_str)
+    
+    # Call the R function to download the flow data
+    result = r.download_flow_data(workspace, dbkey, date_min, date_max)
+    
+    # Check for failure
+    success = result.rx2("success")[0]
+
+    if not success:
+        return
+    
+    # Get the station name for _reformat_flow_file()
+    station = result.rx2("station")[0]
+    
+    # Reformat the flow data file to the expected layout
+    _reformat_flow_file(workspace, station, dbkey)
+    
+    # Check if the station name contains a space
+    if " " in station:
+        # Replace space with underscore in the station name
+        station_previous = station
+        station = station.replace(" ", "_")
+        
+        # Rename the file
+        os.rename(f"{workspace}/{station_previous}_FLOW_{dbkey}_cmd.csv", f"{workspace}/{station}_FLOW_{dbkey}_cmd.csv")
+
+    # column values are converted to cmd in R. This snippet makes sure column names are updated accordingly.
+    file = glob(f'{workspace}/*FLOW*{dbkey}_cmd.csv')[0]
+    df = pd.read_csv(file, index_col=False)
+    df.columns = df.columns.astype(str).str.replace("_cfs", "_cmd")
+    df.to_csv(file, index=False)
+
+
+def _reformat_flow_file(workspace:str, station: str, dbkey: str):
+    '''
+    Reformat the flow data file to the expected layout.
+    Converts the format of the dates in the file to 'YYYY-MM-DD' then sorts the data by date.
+    Reads and writes to a .CSV file.
+    
+    Args:
+        workspace (str): The path to the workspace directory.
+        station (str): The station name.
+        dbkey (str): The dbkey for the station.
+        
+    Returns:
+        None
+    '''
+    # Read in the data
+    df = pd.read_csv(f"{workspace}/{station}_FLOW_{dbkey}_cmd.csv")
+    
+    # Grab only the columns we need
+    df = df[['date', f'{station}_FLOW_cfs']]
+    
+    # Convert date column to datetime
+    df['date'] = pd.to_datetime(df['date'], format='%d-%b-%Y')
+    
+    # Sort the data by date
+    df.sort_values('date', inplace=True)
+    
+    # Renumber the index
+    df.reset_index(drop=True, inplace=True)
+    
+    # Drop rows that are missing values for both the date and value columns
+    df = df.drop(df[(df['date'].isna()) & (df[f'{station}_FLOW_cfs'].isna())].index)
+    
+    # Write the updated data back to the file
+    df.to_csv(f"{workspace}/{station}_FLOW_{dbkey}_cmd.csv")
+
+
+if __name__ == "__main__":
+    workspace = sys.argv[1].rstrip("/")
+    dbkey = sys.argv[2]
+    get(workspace, dbkey)

{loone_data_prep-0.1.6 → loone_data_prep-0.1.7}/loone_data_prep/utils.py

@@ -663,6 +663,75 @@ def nutrient_prediction(
         out_dataframe.to_csv(os.path.join(input_dir, f"{station}_PHOSPHATE_predicted.csv"))
 
 
+def find_last_date_in_csv(workspace: str, file_name: str) -> str:
+    """
+    Gets the most recent date from the last line of a .csv file.
+    Assumes the file is formatted as a .csv file, encoded in UTF-8, 
+    and the rows in the file are sorted by date in ascending order.
+
+    Args:
+        workspace (str): The directory where the file is located.
+        file_name (str): The name of the file.
+
+    Returns:
+        str: The most recent date as a string in YYYY-MM-DD format, or None if the file does not exist or the date cannot be found.
+    """
+    # Helper Functions
+    def is_valid_date(date_string):
+        try:
+            datetime.datetime.strptime(date_string, '%Y-%m-%d')
+            return True
+        except ValueError:
+            return False
+    
+    # Check that file exists
+    file_path = os.path.join(workspace, file_name)
+    if not os.path.exists(file_path):
+        return None
+    
+    # Attempt to extract the date of the last line in the file
+    try:
+        with open(file_path, 'rb') as file:
+            # Go to the end of the file
+            file.seek(-2, os.SEEK_END)
+            
+            # Loop backwards until you find the first newline character
+            while file.read(1) != b'\n':
+                file.seek(-2, os.SEEK_CUR)
+                
+            # Read the last line
+            last_line = file.readline().decode()
+            
+            # Extract the date from the last line
+            date = None
+            
+            for value in last_line.split(','):
+                if is_valid_date(value):
+                    date = value
+                    break
+            
+            # Return date
+            return date
+    except OSError as e:
+        print(f"Error reading file {file_name}: {e}")
+        return None
+
+
+def dbhydro_data_is_latest(date_latest: str):
+    """
+    Checks whether the given date is the most recent date possible to get data from dbhydro.
+    Can be used to check whether dbhydro data is up-to-date.
+    
+    Args:
+        date_latest (str): The date of the most recent data of the dbhydro data you have
+    
+    Returns:
+        bool: True if the date_latest is the most recent date possible to get data from dbhydro, False otherwise
+    """
+    date_latest_object = datetime.datetime.strptime(date_latest, "%Y-%m-%d").date()
+    return date_latest_object == (datetime.datetime.now().date() - datetime.timedelta(days=1))
+
+
 if __name__ == "__main__":
     if sys.argv[1] == "get_dbkeys":
         get_dbkeys(sys.argv[2].strip("[]").replace(" ", "").split(","), *sys.argv[3:])