loone-data-prep 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl

loone_data_prep/LOONE_DATA_PREP.py

@@ -383,7 +383,6 @@ def main(input_dir: str, output_dir: str) -> None:
     LOWS['LZ40WS'] = LZ40WS['LZ40_WNDS_MPH']
     LOWS['LO_Avg_WS_MPH'] = LOWS.mean(axis=1, numeric_only=True)
     LOWS.to_csv(f'{output_dir}/LOWS.csv', index=False)
-    LOWS.to_csv(f'{input_dir}/LOWS.csv', index=False)  # Also needed in temporary directory by utils.py's wind_induced_waves()
 
     # RFVol acft
     # Create File (RF_Volume)
@@ -897,7 +896,6 @@ def main(input_dir: str, output_dir: str) -> None:
     # Write Data into csv files
     # write Avg Stage (ft, m) Storage (acft, m3) SA (acres) to csv
     LO_Stg_Sto_SA_df.to_csv(f'{output_dir}/Average_LO_Storage_3MLag.csv', index=False)
-    LO_Stg_Sto_SA_df.to_csv(f'{input_dir}/Average_LO_Storage_3MLag.csv', index=False)   # Also needed in temporary directory by utils.py's wind_induced_waves()
     # Write S65 TP concentrations (mg/L)
     S65_total_TP.to_csv(f'{output_dir}/S65_TP_3MLag.csv', index=False)
     # TP External Loads 3 Months Lag (mg)

loone_data_prep/flow_data/S65E_total.py

@@ -2,22 +2,87 @@ import sys
 from retry import retry
 from rpy2.robjects import r
 from rpy2.rinterface_lib.embedded import RRuntimeError
+import pandas as pd
 
 
 @retry(RRuntimeError, tries=5, delay=15, max_delay=60, backoff=2)
-def get(workspace):
+def get(
+    workspace, 
+    date_min: str = "1972-01-01", 
+    date_max: str = "2023-06-30"
+) -> None:
     r(
         f"""
         # Load the required libraries
         library(dbhydroR)
-
-        #S65E_Total
-        S65E_total = get_hydro(dbkey = c("91656", "AL760"), date_min = "1972-01-01", date_max = "2023-06-30")
-        S65E_total[, -1] <- S65E_total[, -1] * (0.0283168466 * 86400)
-        write.csv(S65E_total,file ='{workspace}/S65E_total.csv')
+        library(dplyr)
+        
+        # Helper Functions
+        retrieve_data <- function(dbkey, date_min, date_max) 
+        {{
+            # Get the data from dbhydro
+            df = get_hydro(dbkey = dbkey, date_min = date_min, date_max = date_max, raw = TRUE)
+        
+            # Give data.frame correct column names so it can be cleaned using the clean_hydro function
+            colnames(df) <- c("station", "dbkey", "date", "data.value", "qualifer", "revision.date")
+            
+            # Add a type and units column to data so it can be cleaned using the clean_hydro function
+            df$type <- "FLOW"
+            df$units <- "cfs"
+            
+            # Clean the data.frame
+            df <- clean_hydro(df)
+            
+            # Drop the " _FLOW_cfs" column
+            df <- df %>% select(-` _FLOW_cfs`)
+            
+            # Convert Flow rate from cfs to m³/day
+            df[, -1] <- df[, -1] * (0.0283168466 * 86400)
+            
+            # Return resulting data.frame
+            return(df)
+        }}
+        
+        # S65E_S
+        S65E_S <- retrieve_data(dbkey = "91656", date_min = "{date_min}", date_max = "{date_max}")
+        
+        # Wait five seconds before next request to avoid "too many requests" error
+        Sys.sleep(5)
+        
+        # S65EX1_S
+        S65EX1_S <- retrieve_data(dbkey = "AL760", date_min = "{date_min}", date_max = "{date_max}")
+        
+        # Merge the data from each dbkey
+        result <- merge(S65E_S, S65EX1_S, by = "date", all = TRUE)
+        
+        # Write the data to a file
+        write.csv(result, file = '{workspace}/S65E_total.csv')
         """
     )
+    
+    _reformat_s65e_total_file(workspace)
 
+def _reformat_s65e_total_file(workspace: str):
+    # Read in the data
+    df = pd.read_csv(f"{workspace}/S65E_total.csv")
+    
+    # Drop unused columns
+    df.drop('Unnamed: 0', axis=1, inplace=True)
+    
+    # Convert date column to datetime
+    df['date'] = pd.to_datetime(df['date'], format='%d-%b-%Y')
+    
+    # Sort the data by date
+    df.sort_values('date', inplace=True)
+    
+    # Renumber the index
+    df.reset_index(drop=True, inplace=True)
+    
+    # Drop rows that are missing all their values
+    df.dropna(how='all', inplace=True)
+    
+    # Write the updated data back to the file
+    df.to_csv(f"{workspace}/S65E_total.csv")
 
 if __name__ == "__main__":
     workspace = sys.argv[1].rstrip("/")

loone_data_prep/flow_data/get_inflows.py

@@ -1,43 +1,128 @@
 import sys
 import os
 from glob import glob
+from datetime import datetime
+import uuid
+import pandas as pd
 from loone_data_prep.flow_data import hydro, S65E_total
+from loone_data_prep.utils import find_last_date_in_csv, dbhydro_data_is_latest
 
 
-# Database keys for needed inflow data
-DBKEYS = [
-    "91370",
-    "91371",
-    "91373",
-    "91377",
-    "91379",
-    "91401",
-    "91429",
-    "91473",
-    "91508",
-    "91510",
-    "91513",
-    "91599",
-    "91608",
-    "91656",
-    "91668",
-    "91675",
-    "91687",
-    "15627",
-    "15640",
-    "15626",
-    "15642",
-    "15638",
-]
+# Database keys for needed inflow data mapped to their stations
+DBKEYS = {
+    "91370": "S127_C",
+    "91371": "S127_P",
+    "91373": "S129_C",
+    "91377": "S133_P",
+    "91379": "S135_C",
+    "91401": "S154_C",
+    "91429": "S191_S",
+    "91473": "S2_P",
+    "91508": "S351_S",
+    "91510": "S352_S",
+    "91513": "S354_S",
+    "91599": "S3_P",
+    "91608": "S4_P",
+    "91656": "S65E_S",
+    "91668": "S71_S",
+    "91675": "S72_S",
+    "91687": "S84_S",
+    "15627": "FISHP",
+    "15640": "L8.441",
+    "15626": "S308.DS",
+    "15642": "S129 PMP_P",
+    "15638": "S135 PMP_P",
+}
 
-
-def main(workspace: str, dbkeys: list = DBKEYS) -> dict:
+def main(workspace: str, dbkeys: dict = DBKEYS) -> dict:
+    """
+    Retrieve the inflows data used by LOONE. 
+    
+    Args:
+        workspace (str): Path to workspace where data will be downloaded.
+        dbkeys (dict): Dictionary of dbkeys and corresponding station names.
+        
+    Returns:
+        dict: Success or error message
+    """
+    
     # Retrieve inflow data
-    for dbkey in dbkeys:
-        hydro.get(workspace, dbkey)
+    for dbkey, station in dbkeys.copy().items():
+        file_name = f"{station}_FLOW_cmd.csv"
+        date_latest = find_last_date_in_csv(workspace, file_name)
+        
+        # File with data for this dbkey does NOT already exist (or possibly some other error occurred)
+        if date_latest is None:
+            # Download all the data
+            print(f'Downloading all inflow data for {station}')
+            hydro.get(workspace, dbkey)
+        else:
+            # Check whether the latest data is already up to date.
+            if dbhydro_data_is_latest(date_latest):
+                # Notify that the data is already up to date
+                print(f'Downloading of new inflow data skipped for Station {station} (dbkey: {dbkey}). Data is already up to date.')
+                
+                # Remove dbkey from dbkeys so we know it didn't fail
+                del dbkeys[dbkey]
+                continue
+            
+            # Download only the new data
+            print(f'Downloading new inflow data for {station} starting from date {date_latest}')
+            hydro.get(workspace, dbkey, date_latest)
+            
+            # Make sure both our original data and newly downloaded data exist
+            df_original_path = os.path.join(workspace, f"{station}_FLOW_cmd.csv")
+            df_new_path = os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv")
+            
+            if os.path.exists(df_original_path) and os.path.exists(df_new_path):
+                # Merge the new data with the old data
+                df_original = pd.read_csv(df_original_path, index_col=0)
+                df_new = pd.read_csv(df_new_path, index_col=0)
+                df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                
+                # Write the merged data to the new file
+                df_merged.to_csv(os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv"))
+                
+                # Remove the old file
+                os.remove(os.path.join(workspace, f"{station}_FLOW_cmd.csv"))
 
-    S65E_total.get(workspace)
+    # Download S65E_total.csv Data
+    date_latest = find_last_date_in_csv(workspace, "S65E_total.csv")
 
+    if date_latest is None:
+        print('Downloading all S65E_total data')
+        S65E_total.get(workspace, date_max=datetime.now().strftime("%Y-%m-%d"))
+    else:
+        # Check whether the latest data is already up to date.
+        if dbhydro_data_is_latest(date_latest):
+            # Notify that the data is already up to date
+            print(f'Downloading of new inflow data skipped for S65E_total. Data is already up to date.')
+        else:
+            # Temporarily rename current data file so it isn't over written
+            original_file_name = F"S65E_total_old_{uuid.uuid4()}.csv"
+            os.rename(os.path.join(workspace, "S65E_total.csv"), os.path.join(workspace, original_file_name))
+            
+            try:
+                # Download only the new data
+                print(f'Downloading new S65E_total data starting from date {date_latest}')
+                S65E_total.get(workspace, date_min=date_latest, date_max=datetime.now().strftime("%Y-%m-%d"))
+                
+                # Merge the new data with the original data
+                df_original = pd.read_csv(os.path.join(workspace, original_file_name), index_col=0)
+                df_new = pd.read_csv(os.path.join(workspace, "S65E_total.csv"), index_col=0)
+                df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                
+                # Write out the merged data
+                df_merged.to_csv(os.path.join(workspace, original_file_name))
+                
+                # Remove the newly downloaded data file
+                os.remove(os.path.join(workspace, "S65E_total.csv"))
+            except Exception as e:
+                print(f"Error occurred while downloading new S65E_total data: {e}")
+            finally:
+                # Rename the original updated file back to its original name
+                os.rename(os.path.join(workspace, original_file_name), os.path.join(workspace, "S65E_total.csv"))
+                
     # Check if all files were downloaded
     files = glob(f"{workspace}/*FLOW*_cmd.csv")
 
@@ -50,17 +135,21 @@ def main(workspace: str, dbkeys: list = DBKEYS) -> dict:
             os.rename(file, new_file_name)
 
             # Remove dbkey from dbkeys so we know it successfully downloaded
-            dbkeys.remove(file_dbkey)
-
-    if len(dbkeys) > 0:
-        return {
-            "error": (
-                "The data from the following dbkeys could not be "
-                f"downloaded: {dbkeys}"
-            )
-        }
-    elif not os.path.exists(f"{workspace}/S65E_total.csv"):
-        return {"error": "S65E_total.csv file could not be downloaded."}
+            del dbkeys[file_dbkey]
+    
+    # Check for failed downloads
+    if len(dbkeys) > 0 or not os.path.exists(f"{workspace}/S65E_total.csv"):
+        error_message = ""
+        
+        # dbkeys
+        if len(dbkeys) > 0:
+            error_message += f"The data from the following dbkeys could not be downloaded: {list(dbkeys.keys())}\n"
+        
+        # S65E_total.csv
+        if not os.path.exists(f"{workspace}/S65E_total.csv"):
+            error_message += "S65E_total.csv file could not be downloaded.\n"
+        
+        return {"error": error_message}
 
     return {"success": "Completed inflow flow data download."}
 

loone_data_prep/flow_data/get_outflows.py

@@ -1,7 +1,8 @@
 import sys
 import os
 from glob import glob
-from loone_data_prep.utils import get_dbkeys
+import pandas as pd
+from loone_data_prep.utils import get_dbkeys, find_last_date_in_csv, dbhydro_data_is_latest
 from loone_data_prep.flow_data import hydro
 
 STATION_IDS = [
@@ -24,33 +25,39 @@ STATION_IDS = [
 ]
 
 
-DBKEYS = [
-    "91370",
-    "91373",
-    "91379",
-    "91508",
-    "91510",
-    "91513",
-    "91677",
-    "15628",
-    "15640",
-    "15626",
-    "00865",
-    "JW224",
-    "00436",
-    "15018",
-    "91606",
-    "JW223",
-]
-
+DBKEYS = {
+    "91370": "S127_C",
+    "91373": "S129_C",
+    "91379": "S135_C",
+    "91508": "S351_S",
+    "91510": "S352_S",
+    "91513": "S354_S",
+    "91677": "S77_S",
+    "15628": "INDUST",
+    "15640": "L8.441",
+    "15626": "S308.DS",
+    "00865": "S79_TOT",
+    "JW224": "S80_S",
+    "00436": "S2 NNR",
+    "15018": "S3",
+    "91606": "S48_S",
+    "JW223": "S49_S",
+}
 
-def main(workspace: str, dbkeys: list = DBKEYS, station_ids: list = STATION_IDS) -> dict:
-    # Retrieve outflow data
 
-    if dbkeys is None:
-        # Get dbkeys from station ids
-        dbkeys = list(get_dbkeys(station_ids, "SW", "FLOW", "MEAN", "PREF", detail_level="dbkey"))
-        dbkeys.extend(list(get_dbkeys(station_ids, "SW", "FLOW", "MEAN", "DRV", detail_level="dbkey")))
+def _get_outflow_data_from_station_ids(workspace: str, station_ids: list) -> dict:
+    """Attempt to download outflow data from station ids.
+    
+    Args:
+        workspace (str): Path to workspace where data will be downloaded.
+        station_ids (list): List of station ids to download data for.
+        
+    Returns:
+        dict: Success or error message
+    """
+    # Get dbkeys from station ids
+    dbkeys = list(get_dbkeys(station_ids, "SW", "FLOW", "MEAN", "PREF", detail_level="dbkey"))
+    dbkeys.extend(list(get_dbkeys(station_ids, "SW", "FLOW", "MEAN", "DRV", detail_level="dbkey")))
 
     for dbkey in dbkeys:
         hydro.get(workspace, dbkey, "2000-01-01")
@@ -75,6 +82,83 @@ def main(workspace: str, dbkeys: list = DBKEYS, station_ids: list = STATION_IDS)
     return {"success": "Completed outflow flow data download."}
 
 
+def main(workspace: str, dbkeys: dict = DBKEYS, station_ids: list = STATION_IDS) -> dict:
+    """
+    Retrieve the outflow data used by LOONE. 
+    
+    Args:
+        workspace (str): Path to workspace where data will be downloaded.
+        dbkeys (dict): Dictionary of dbkeys and corresponding station names.
+        station_ids (list): List of station ids to download data for if the dbkeys argument is not provided.
+        
+    Returns:
+        dict: Success or error message
+    """
+    
+    # No dbkeys given, attempt to get data from station ids
+    if dbkeys is None:
+        return _get_outflow_data_from_station_ids(workspace, station_ids)
+
+    # Get outflow data from dbkeys
+    for dbkey, station in dbkeys.copy().items():
+        # Get the date of the latest data in the csv file (if any)
+        date_latest = find_last_date_in_csv(workspace, f"{station}_FLOW_cmd.csv")
+        
+        # File with data for this dbkey does NOT already exist (or possibly some other error occurred)
+        if date_latest is None:
+            # Download all data
+            print(f'Downloading all outflow data for {station}')
+            hydro.get(workspace, dbkey, "2000-01-01")
+        else:
+            # Check whether the latest data is already up to date.
+            if dbhydro_data_is_latest(date_latest):
+                # Notify that the data is already up to date
+                print(f'Downloading of new outflow data skipped for Station {station} (dbkey: {dbkey}). Data is already up to date.')
+                
+                # Remove dbkey from dbkeys so we know it didn't fail
+                del dbkeys[dbkey]
+                continue
+            
+            # Download only the new data
+            print(f'Downloading new outflow data for {station} starting from date {date_latest}')
+            hydro.get(workspace, dbkey, date_latest)
+            
+            # Make sure both our original data and newly downloaded data exist
+            df_old_path = os.path.join(workspace, f"{station}_FLOW_cmd.csv")
+            df_new_path = os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv")
+            
+            if os.path.exists(df_old_path) and os.path.exists(df_new_path):
+                # Merge the new data with the old data
+                df_original = pd.read_csv(df_old_path, index_col=0)
+                df_new = pd.read_csv(df_new_path, index_col=0)
+                df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                
+                # Write the merged data to the new file
+                df_merged.to_csv(os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv"))
+                
+                # Remove the old file
+                os.remove(os.path.join(workspace, f"{station}_FLOW_cmd.csv"))
+
+    # Check if all files were downloaded
+    files = glob(f"{workspace}/*FLOW*_cmd.csv")
+
+    for file in files:
+        file_dbkey = file.split("_")[-2]
+
+        if file_dbkey in dbkeys:
+            # Remove dbkey from file name
+            new_file_name = file.replace(f"_{file_dbkey}", "")
+            os.rename(file, new_file_name)
+
+            # Remove dbkey from dbkeys so we know it successfully downloaded
+            del dbkeys[file_dbkey]
+
+    if len(dbkeys) > 0:
+        return {"error": f"The data from the following dbkeys could not be downloaded: {dbkeys}"}
+
+    return {"success": "Completed outflow flow data download."}
+
+
 if __name__ == "__main__":
     workspace = sys.argv[1].rstrip("/")
     main(workspace)

loone_data_prep/flow_data/hydro.py

@@ -2,6 +2,7 @@ import sys
 from datetime import datetime
 from glob import glob
 from retry import retry
+import os
 import pandas as pd
 from rpy2.robjects import r
 from rpy2.rinterface_lib.embedded import RRuntimeError
@@ -18,36 +19,92 @@ def get(
     date_max: str = DATE_NOW
 ) -> None:
     r_str = f"""
-    # Load the required libraries
-    library(dbhydroR)
+    download_flow_data <- function(workspace, dbkey, date_min, date_max) 
+    {{
+        # Load the required libraries
+        library(dbhydroR)
+        library(dplyr)
 
-    # Retrieve data for the dbkey
-    data <- get_hydro(dbkey = "{dbkey}", date_min = "{date_min}", date_max = "{date_max}")
+        # Retrieve data for the dbkey
+        data <- get_hydro(dbkey = "{dbkey}", date_min = "{date_min}", date_max = "{date_max}", raw = TRUE)
+        
+        # Check if data is empty or contains only the "date" column
+        if (ncol(data) <= 1) {{
+            cat("No data found for dbkey", "{dbkey}", "Skipping to the next dbkey.\n")
+        }}
+        
+        # Give data.frame correct column names so it can be cleaned using the clean_hydro function
+        colnames(data) <- c("station", "dbkey", "date", "data.value", "qualifer", "revision.date")
+        
+        # Check if the data.frame has any rows
+        if (nrow(data) == 0) 
+        {{
+            # No data given back, It's possible that the dbkey has reached its end date.
+            print(paste("Empty data.frame returned for dbkey", "{dbkey}", "It's possible that the dbkey has reached its end date. Skipping to the next dbkey."))
+            return(list(success = FALSE, dbkey = "{dbkey}"))
+        }}
+        
+        # Add a type and units column to data so it can be cleaned using the clean_hydro function
+        data$type <- "FLOW"
+        data$units <- "cfs"
+        
+        # Get the station
+        station <- data$station[1]
+        
+        # Clean the data.frame
+        data <- clean_hydro(data)
 
-    # Check if data is empty or contains only the "date" column
-    if (ncol(data) <= 1) {{
-        cat("No data found for dbkey", "{dbkey}", "Skipping to the next dbkey.\n")
-    }}
-
-    # Multiply all columns except "date" column by 0.0283168466 * 86400 to convert Flow rate from cfs to m³/day
-    data[, -1] <- data[, -1] * (0.0283168466 * 86400)
-
-    # Extract the column names excluding the date column
-    column_names <- names(data)[-1]
+        # Multiply all columns except "date" column by 0.0283168466 * 86400 to convert Flow rate from cfs to m³/day
+        data[, -1] <- data[, -1] * (0.0283168466 * 86400)
+        
+        # Drop the " _FLOW_cfs" column
+        data <- data %>% select(-` _FLOW_cfs`)    
+        
+        # Sort the data by date
+        data <- data[order(data$date), ]
+        
+        # Get the filename for the output CSV file
+        filename <- paste0(station, "_FLOW", "_{dbkey}_cmd.csv")
+        
+        # Save data to a CSV file
+        write.csv(data, file = paste0("{workspace}/", filename))
 
-    # Generate the filename based on the column names
-    filename <- paste0( gsub(" ", "_", sub("_[^_]*$", "", paste(column_names, collapse = "_"))), "_{dbkey}_cmd.csv")
-    # Save data to a CSV file
-    write.csv(data, file = paste0("{workspace}/", filename))
+        # Print a message indicating the file has been saved
+        cat("CSV file", filename, "has been saved.\n")
 
-    # Print a message indicating the file has been saved
-    cat("CSV file", filename, "has been saved.\n")
-
-    # Add a delay between requests
-    Sys.sleep(1)  # Wait for 1 second before the next iteration
+        # Add a delay between requests
+        Sys.sleep(1)  # Wait for 1 second before the next iteration
+        
+        # Return the station and dbkey to the python code
+        list(success = TRUE, station = station, dbkey = "{dbkey}")
+    }}
     """
 
     r(r_str)
+    
+    # Call the R function to download the flow data
+    result = r.download_flow_data(workspace, dbkey, date_min, date_max)
+    
+    # Check for failure
+    success = result.rx2("success")[0]
+
+    if not success:
+        return
+    
+    # Get the station name for _reformat_flow_file()
+    station = result.rx2("station")[0]
+    
+    # Reformat the flow data file to the expected layout
+    _reformat_flow_file(workspace, station, dbkey)
+    
+    # Check if the station name contains a space
+    if " " in station:
+        # Replace space with underscore in the station name
+        station_previous = station
+        station = station.replace(" ", "_")
+        
+        # Rename the file
+        os.rename(f"{workspace}/{station_previous}_FLOW_{dbkey}_cmd.csv", f"{workspace}/{station}_FLOW_{dbkey}_cmd.csv")
 
     # column values are converted to cmd in R. This snippet makes sure column names are updated accordingly.
     file = glob(f'{workspace}/*FLOW*{dbkey}_cmd.csv')[0]
@@ -55,7 +112,44 @@ def get(
     df.columns = df.columns.astype(str).str.replace("_cfs", "_cmd")
     df.to_csv(file, index=False)
 
-    if __name__ == "__main__":
-        workspace = sys.argv[1].rstrip("/")
-        dbkey = sys.argv[2]
-        get(workspace, dbkey)
+
+def _reformat_flow_file(workspace:str, station: str, dbkey: str):
+    '''
+    Reformat the flow data file to the expected layout.
+    Converts the format of the dates in the file to 'YYYY-MM-DD' then sorts the data by date.
+    Reads and writes to a .CSV file.
+    
+    Args:
+        workspace (str): The path to the workspace directory.
+        station (str): The station name.
+        dbkey (str): The dbkey for the station.
+        
+    Returns:
+        None
+    '''
+    # Read in the data
+    df = pd.read_csv(f"{workspace}/{station}_FLOW_{dbkey}_cmd.csv")
+    
+    # Grab only the columns we need
+    df = df[['date', f'{station}_FLOW_cfs']]
+    
+    # Convert date column to datetime
+    df['date'] = pd.to_datetime(df['date'], format='%d-%b-%Y')
+    
+    # Sort the data by date
+    df.sort_values('date', inplace=True)
+    
+    # Renumber the index
+    df.reset_index(drop=True, inplace=True)
+    
+    # Drop rows that are missing values for both the date and value columns
+    df = df.drop(df[(df['date'].isna()) & (df[f'{station}_FLOW_cfs'].isna())].index)
+    
+    # Write the updated data back to the file
+    df.to_csv(f"{workspace}/{station}_FLOW_{dbkey}_cmd.csv")
+
+
+if __name__ == "__main__":
+    workspace = sys.argv[1].rstrip("/")
+    dbkey = sys.argv[2]
+    get(workspace, dbkey)