loone-data-prep 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

loone_data_prep/LOONE_DATA_PREP.py

@@ -383,7 +383,6 @@ def main(input_dir: str, output_dir: str) -> None:
     LOWS['LZ40WS'] = LZ40WS['LZ40_WNDS_MPH']
     LOWS['LO_Avg_WS_MPH'] = LOWS.mean(axis=1, numeric_only=True)
     LOWS.to_csv(f'{output_dir}/LOWS.csv', index=False)
-    LOWS.to_csv(f'{input_dir}/LOWS.csv', index=False)  # Also needed in temporary directory by utils.py's wind_induced_waves()
 
     # RFVol acft
     # Create File (RF_Volume)
@@ -418,7 +417,7 @@ def main(input_dir: str, output_dir: str) -> None:
     Stg_3A28 = pd.read_csv(f'{input_dir}/Stg_3A28.csv')
     Stg_3A28 = DF_Date_Range(Stg_3A28, St_Yr, St_M, St_D, En_Yr, En_M, En_D)
     WCA_Stg = pd.DataFrame(Stg_3A28['date'], columns=['date'])
-    WCA_Stg['3A-NW'] = Stg_3ANW['3A-NW_STG_ft NGVD29'].values
+    WCA_Stg['3A-NW'] = Stg_3ANW.iloc[:, -1].values
     WCA_Stg['2A-17'] = Stg_2A17.iloc[:, -1].values
     WCA_Stg['3A-3'] = Stg_3A3.iloc[:, -1].values
     WCA_Stg['3A-4'] = Stg_3A4.iloc[:, -1].values
@@ -897,7 +896,6 @@ def main(input_dir: str, output_dir: str) -> None:
     # Write Data into csv files
     # write Avg Stage (ft, m) Storage (acft, m3) SA (acres) to csv
     LO_Stg_Sto_SA_df.to_csv(f'{output_dir}/Average_LO_Storage_3MLag.csv', index=False)
-    LO_Stg_Sto_SA_df.to_csv(f'{input_dir}/Average_LO_Storage_3MLag.csv', index=False)   # Also needed in temporary directory by utils.py's wind_induced_waves()
     # Write S65 TP concentrations (mg/L)
     S65_total_TP.to_csv(f'{output_dir}/S65_TP_3MLag.csv', index=False)
     # TP External Loads 3 Months Lag (mg)

loone_data_prep/flow_data/S65E_total.py

@@ -2,22 +2,87 @@ import sys
 from retry import retry
 from rpy2.robjects import r
 from rpy2.rinterface_lib.embedded import RRuntimeError
+import pandas as pd
 
 
 @retry(RRuntimeError, tries=5, delay=15, max_delay=60, backoff=2)
-def get(workspace):
+def get(
+    workspace, 
+    date_min: str = "1972-01-01", 
+    date_max: str = "2023-06-30"
+) -> None:
     r(
         f"""
         # Load the required libraries
         library(dbhydroR)
-
-        #S65E_Total
-        S65E_total = get_hydro(dbkey = c("91656", "AL760"), date_min = "1972-01-01", date_max = "2023-06-30")
-        S65E_total[, -1] <- S65E_total[, -1] * (0.0283168466 * 86400)
-        write.csv(S65E_total,file ='{workspace}/S65E_total.csv')
+        library(dplyr)
+        
+        # Helper Functions
+        retrieve_data <- function(dbkey, date_min, date_max) 
+        {{
+            # Get the data from dbhydro
+            df = get_hydro(dbkey = dbkey, date_min = date_min, date_max = date_max, raw = TRUE)
+        
+            # Give data.frame correct column names so it can be cleaned using the clean_hydro function
+            colnames(df) <- c("station", "dbkey", "date", "data.value", "qualifer", "revision.date")
+            
+            # Add a type and units column to data so it can be cleaned using the clean_hydro function
+            df$type <- "FLOW"
+            df$units <- "cfs"
+            
+            # Clean the data.frame
+            df <- clean_hydro(df)
+            
+            # Drop the " _FLOW_cfs" column
+            df <- df %>% select(-` _FLOW_cfs`)
+            
+            # Convert Flow rate from cfs to m³/day
+            df[, -1] <- df[, -1] * (0.0283168466 * 86400)
+            
+            # Return resulting data.frame
+            return(df)
+        }}
+        
+        # S65E_S
+        S65E_S <- retrieve_data(dbkey = "91656", date_min = "{date_min}", date_max = "{date_max}")
+        
+        # Wait five seconds before next request to avoid "too many requests" error
+        Sys.sleep(5)
+        
+        # S65EX1_S
+        S65EX1_S <- retrieve_data(dbkey = "AL760", date_min = "{date_min}", date_max = "{date_max}")
+        
+        # Merge the data from each dbkey
+        result <- merge(S65E_S, S65EX1_S, by = "date", all = TRUE)
+        
+        # Write the data to a file
+        write.csv(result, file = '{workspace}/S65E_total.csv')
         """
     )
+    
+    _reformat_s65e_total_file(workspace)
 
+def _reformat_s65e_total_file(workspace: str):
+    # Read in the data
+    df = pd.read_csv(f"{workspace}/S65E_total.csv")
+    
+    # Drop unused columns
+    df.drop('Unnamed: 0', axis=1, inplace=True)
+    
+    # Convert date column to datetime
+    df['date'] = pd.to_datetime(df['date'], format='%d-%b-%Y')
+    
+    # Sort the data by date
+    df.sort_values('date', inplace=True)
+    
+    # Renumber the index
+    df.reset_index(drop=True, inplace=True)
+    
+    # Drop rows that are missing all their values
+    df.dropna(how='all', inplace=True)
+    
+    # Write the updated data back to the file
+    df.to_csv(f"{workspace}/S65E_total.csv")
 
 if __name__ == "__main__":
     workspace = sys.argv[1].rstrip("/")

loone_data_prep/flow_data/forecast_bias_correction.py

@@ -1,4 +1,5 @@
 import sys
+import os
 import pandas as pd
 import geoglows
 
@@ -12,6 +13,7 @@ def get_bias_corrected_data(
     observed_data_path: str,
     station_ensembles: pd.DataFrame,
     station_stats: pd.DataFrame,
+    cache_path: str = None,
 ) -> dict:
     # Load the observed data from a CSV file
     observed_data = pd.read_csv(
@@ -34,7 +36,23 @@ def get_bias_corrected_data(
     prepared_od = prep_observed_data(observed_data)
 
     # Get the historical simulation data for the given reach ID
-    historical_data = geoglows.streamflow.historic_simulation(reach_id)
+    historical_data = None
+    
+    if cache_path is None:
+        historical_data = geoglows.streamflow.historic_simulation(reach_id)
+    else:
+        # Create the geoglows cache directory if it doesn't exist
+        geoglows_cache_path = os.path.join(cache_path, 'geoglows_cache')
+        if not os.path.exists(geoglows_cache_path):
+            os.makedirs(geoglows_cache_path)
+        
+        # Check if the historical simulation data is already cached
+        if os.path.exists(os.path.join(geoglows_cache_path, f'{reach_id}_historic_simulation.csv')):
+            historical_data = pd.read_csv(os.path.join(geoglows_cache_path, f'{reach_id}_historic_simulation.csv'), index_col=0)
+            historical_data.index = pd.to_datetime(historical_data.index)
+        else:
+            historical_data = geoglows.streamflow.historic_simulation(reach_id)
+            historical_data.to_csv(os.path.join(geoglows_cache_path, f'{reach_id}_historic_simulation.csv'))
 
     # Correct the forecast bias in the station ensembles
     station_ensembles = geoglows.bias.correct_forecast(

loone_data_prep/flow_data/get_forecast_flows.py

@@ -366,6 +366,7 @@ def main(
     forecast_date: str = FORECAST_DATE,
     bias_corrected: bool = False,
     observed_data_dir: str | None = None,
+    cache_path: str | None = None,
 ):
     """Downloads the flow forecasts for the given station ids and writes them
         out as .csv files.
@@ -379,6 +380,8 @@ def main(
             Default is False.
         observed_data_dir (str): The path to the observed flow data directory
             (only needed if bias_corrected is True).
+        cache_path (str): The path to the cache directory for geoglows data. 
+            Should hold a directory named geoglows_cache that holds the cached files. Use None to not use a cache.
     """
     # Local Variables
     reach_ids = {}
@@ -428,6 +431,7 @@ def main(
                     observed_data_path,
                     station_ensembles,
                     station_stats,
+                    cache_path,
                 )
 
         ensembles_to_csv(

loone_data_prep/flow_data/get_inflows.py

@@ -1,43 +1,128 @@
 import sys
 import os
 from glob import glob
+from datetime import datetime
+import uuid
+import pandas as pd
 from loone_data_prep.flow_data import hydro, S65E_total
+from loone_data_prep.utils import find_last_date_in_csv, dbhydro_data_is_latest
 
 
-# Database keys for needed inflow data
-DBKEYS = [
-    "91370",
-    "91371",
-    "91373",
-    "91377",
-    "91379",
-    "91401",
-    "91429",
-    "91473",
-    "91508",
-    "91510",
-    "91513",
-    "91599",
-    "91608",
-    "91656",
-    "91668",
-    "91675",
-    "91687",
-    "15627",
-    "15640",
-    "15626",
-    "15642",
-    "15638",
-]
+# Database keys for needed inflow data mapped to their stations
+DBKEYS = {
+    "91370": "S127_C",
+    "91371": "S127_P",
+    "91373": "S129_C",
+    "91377": "S133_P",
+    "91379": "S135_C",
+    "91401": "S154_C",
+    "91429": "S191_S",
+    "91473": "S2_P",
+    "91508": "S351_S",
+    "91510": "S352_S",
+    "91513": "S354_S",
+    "91599": "S3_P",
+    "91608": "S4_P",
+    "91656": "S65E_S",
+    "91668": "S71_S",
+    "91675": "S72_S",
+    "91687": "S84_S",
+    "15627": "FISHP",
+    "15640": "L8.441",
+    "15626": "S308.DS",
+    "15642": "S129 PMP_P",
+    "15638": "S135 PMP_P",
+}
 
-
-def main(workspace: str, dbkeys: list = DBKEYS) -> dict:
+def main(workspace: str, dbkeys: dict = DBKEYS) -> dict:
+    """
+    Retrieve the inflows data used by LOONE. 
+    
+    Args:
+        workspace (str): Path to workspace where data will be downloaded.
+        dbkeys (dict): Dictionary of dbkeys and corresponding station names.
+        
+    Returns:
+        dict: Success or error message
+    """
+    
     # Retrieve inflow data
-    for dbkey in dbkeys:
-        hydro.get(workspace, dbkey)
+    for dbkey, station in dbkeys.copy().items():
+        file_name = f"{station}_FLOW_cmd.csv"
+        date_latest = find_last_date_in_csv(workspace, file_name)
+        
+        # File with data for this dbkey does NOT already exist (or possibly some other error occurred)
+        if date_latest is None:
+            # Download all the data
+            print(f'Downloading all inflow data for {station}')
+            hydro.get(workspace, dbkey)
+        else:
+            # Check whether the latest data is already up to date.
+            if dbhydro_data_is_latest(date_latest):
+                # Notify that the data is already up to date
+                print(f'Downloading of new inflow data skipped for Station {station} (dbkey: {dbkey}). Data is already up to date.')
+                
+                # Remove dbkey from dbkeys so we know it didn't fail
+                del dbkeys[dbkey]
+                continue
+            
+            # Download only the new data
+            print(f'Downloading new inflow data for {station} starting from date {date_latest}')
+            hydro.get(workspace, dbkey, date_latest)
+            
+            # Make sure both our original data and newly downloaded data exist
+            df_original_path = os.path.join(workspace, f"{station}_FLOW_cmd.csv")
+            df_new_path = os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv")
+            
+            if os.path.exists(df_original_path) and os.path.exists(df_new_path):
+                # Merge the new data with the old data
+                df_original = pd.read_csv(df_original_path, index_col=0)
+                df_new = pd.read_csv(df_new_path, index_col=0)
+                df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                
+                # Write the merged data to the new file
+                df_merged.to_csv(os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv"))
+                
+                # Remove the old file
+                os.remove(os.path.join(workspace, f"{station}_FLOW_cmd.csv"))
 
-    S65E_total.get(workspace)
+    # Download S65E_total.csv Data
+    date_latest = find_last_date_in_csv(workspace, "S65E_total.csv")
 
+    if date_latest is None:
+        print('Downloading all S65E_total data')
+        S65E_total.get(workspace, date_max=datetime.now().strftime("%Y-%m-%d"))
+    else:
+        # Check whether the latest data is already up to date.
+        if dbhydro_data_is_latest(date_latest):
+            # Notify that the data is already up to date
+            print(f'Downloading of new inflow data skipped for S65E_total. Data is already up to date.')
+        else:
+            # Temporarily rename current data file so it isn't over written
+            original_file_name = F"S65E_total_old_{uuid.uuid4()}.csv"
+            os.rename(os.path.join(workspace, "S65E_total.csv"), os.path.join(workspace, original_file_name))
+            
+            try:
+                # Download only the new data
+                print(f'Downloading new S65E_total data starting from date {date_latest}')
+                S65E_total.get(workspace, date_min=date_latest, date_max=datetime.now().strftime("%Y-%m-%d"))
+                
+                # Merge the new data with the original data
+                df_original = pd.read_csv(os.path.join(workspace, original_file_name), index_col=0)
+                df_new = pd.read_csv(os.path.join(workspace, "S65E_total.csv"), index_col=0)
+                df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                
+                # Write out the merged data
+                df_merged.to_csv(os.path.join(workspace, original_file_name))
+                
+                # Remove the newly downloaded data file
+                os.remove(os.path.join(workspace, "S65E_total.csv"))
+            except Exception as e:
+                print(f"Error occurred while downloading new S65E_total data: {e}")
+            finally:
+                # Rename the original updated file back to its original name
+                os.rename(os.path.join(workspace, original_file_name), os.path.join(workspace, "S65E_total.csv"))
+                
     # Check if all files were downloaded
     files = glob(f"{workspace}/*FLOW*_cmd.csv")
 
@@ -50,17 +135,21 @@ def main(workspace: str, dbkeys: list = DBKEYS) -> dict:
             os.rename(file, new_file_name)
 
             # Remove dbkey from dbkeys so we know it successfully downloaded
-            dbkeys.remove(file_dbkey)
-
-    if len(dbkeys) > 0:
-        return {
-            "error": (
-                "The data from the following dbkeys could not be "
-                f"downloaded: {dbkeys}"
-            )
-        }
-    elif not os.path.exists(f"{workspace}/S65E_total.csv"):
-        return {"error": "S65E_total.csv file could not be downloaded."}
+            del dbkeys[file_dbkey]
+    
+    # Check for failed downloads
+    if len(dbkeys) > 0 or not os.path.exists(f"{workspace}/S65E_total.csv"):
+        error_message = ""
+        
+        # dbkeys
+        if len(dbkeys) > 0:
+            error_message += f"The data from the following dbkeys could not be downloaded: {list(dbkeys.keys())}\n"
+        
+        # S65E_total.csv
+        if not os.path.exists(f"{workspace}/S65E_total.csv"):
+            error_message += "S65E_total.csv file could not be downloaded.\n"
+        
+        return {"error": error_message}
 
     return {"success": "Completed inflow flow data download."}
 

loone_data_prep/flow_data/get_outflows.py

@@ -1,7 +1,8 @@
 import sys
 import os
 from glob import glob
-from loone_data_prep.utils import get_dbkeys
+import pandas as pd
+from loone_data_prep.utils import get_dbkeys, find_last_date_in_csv, dbhydro_data_is_latest
 from loone_data_prep.flow_data import hydro
 
 STATION_IDS = [
@@ -24,33 +25,39 @@ STATION_IDS = [
 ]
 
 
-DBKEYS = [
-    "91370",
-    "91373",
-    "91379",
-    "91508",
-    "91510",
-    "91513",
-    "91677",
-    "15628",
-    "15640",
-    "15626",
-    "00865",
-    "JW224",
-    "00436",
-    "15018",
-    "91606",
-    "JW223",
-]
-
+DBKEYS = {
+    "91370": "S127_C",
+    "91373": "S129_C",
+    "91379": "S135_C",
+    "91508": "S351_S",
+    "91510": "S352_S",
+    "91513": "S354_S",
+    "91677": "S77_S",
+    "15628": "INDUST",
+    "15640": "L8.441",
+    "15626": "S308.DS",
+    "00865": "S79_TOT",
+    "JW224": "S80_S",
+    "00436": "S2 NNR",
+    "15018": "S3",
+    "91606": "S48_S",
+    "JW223": "S49_S",
+}
 
-def main(workspace: str, dbkeys: list = DBKEYS, station_ids: list = STATION_IDS) -> dict:
-    # Retrieve outflow data
 
-    if dbkeys is None:
-        # Get dbkeys from station ids
-        dbkeys = list(get_dbkeys(station_ids, "SW", "FLOW", "MEAN", "PREF", detail_level="dbkey"))
-        dbkeys.extend(list(get_dbkeys(station_ids, "SW", "FLOW", "MEAN", "DRV", detail_level="dbkey")))
+def _get_outflow_data_from_station_ids(workspace: str, station_ids: list) -> dict:
+    """Attempt to download outflow data from station ids.
+    
+    Args:
+        workspace (str): Path to workspace where data will be downloaded.
+        station_ids (list): List of station ids to download data for.
+        
+    Returns:
+        dict: Success or error message
+    """
+    # Get dbkeys from station ids
+    dbkeys = list(get_dbkeys(station_ids, "SW", "FLOW", "MEAN", "PREF", detail_level="dbkey"))
+    dbkeys.extend(list(get_dbkeys(station_ids, "SW", "FLOW", "MEAN", "DRV", detail_level="dbkey")))
 
     for dbkey in dbkeys:
         hydro.get(workspace, dbkey, "2000-01-01")
@@ -75,6 +82,83 @@ def main(workspace: str, dbkeys: list = DBKEYS, station_ids: list = STATION_IDS)
     return {"success": "Completed outflow flow data download."}
 
 
+def main(workspace: str, dbkeys: dict = DBKEYS, station_ids: list = STATION_IDS) -> dict:
+    """
+    Retrieve the outflow data used by LOONE. 
+    
+    Args:
+        workspace (str): Path to workspace where data will be downloaded.
+        dbkeys (dict): Dictionary of dbkeys and corresponding station names.
+        station_ids (list): List of station ids to download data for if the dbkeys argument is not provided.
+        
+    Returns:
+        dict: Success or error message
+    """
+    
+    # No dbkeys given, attempt to get data from station ids
+    if dbkeys is None:
+        return _get_outflow_data_from_station_ids(workspace, station_ids)
+
+    # Get outflow data from dbkeys
+    for dbkey, station in dbkeys.copy().items():
+        # Get the date of the latest data in the csv file (if any)
+        date_latest = find_last_date_in_csv(workspace, f"{station}_FLOW_cmd.csv")
+        
+        # File with data for this dbkey does NOT already exist (or possibly some other error occurred)
+        if date_latest is None:
+            # Download all data
+            print(f'Downloading all outflow data for {station}')
+            hydro.get(workspace, dbkey, "2000-01-01")
+        else:
+            # Check whether the latest data is already up to date.
+            if dbhydro_data_is_latest(date_latest):
+                # Notify that the data is already up to date
+                print(f'Downloading of new outflow data skipped for Station {station} (dbkey: {dbkey}). Data is already up to date.')
+                
+                # Remove dbkey from dbkeys so we know it didn't fail
+                del dbkeys[dbkey]
+                continue
+            
+            # Download only the new data
+            print(f'Downloading new outflow data for {station} starting from date {date_latest}')
+            hydro.get(workspace, dbkey, date_latest)
+            
+            # Make sure both our original data and newly downloaded data exist
+            df_old_path = os.path.join(workspace, f"{station}_FLOW_cmd.csv")
+            df_new_path = os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv")
+            
+            if os.path.exists(df_old_path) and os.path.exists(df_new_path):
+                # Merge the new data with the old data
+                df_original = pd.read_csv(df_old_path, index_col=0)
+                df_new = pd.read_csv(df_new_path, index_col=0)
+                df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                
+                # Write the merged data to the new file
+                df_merged.to_csv(os.path.join(workspace, f"{station}_FLOW_{dbkey}_cmd.csv"))
+                
+                # Remove the old file
+                os.remove(os.path.join(workspace, f"{station}_FLOW_cmd.csv"))
+
+    # Check if all files were downloaded
+    files = glob(f"{workspace}/*FLOW*_cmd.csv")
+
+    for file in files:
+        file_dbkey = file.split("_")[-2]
+
+        if file_dbkey in dbkeys:
+            # Remove dbkey from file name
+            new_file_name = file.replace(f"_{file_dbkey}", "")
+            os.rename(file, new_file_name)
+
+            # Remove dbkey from dbkeys so we know it successfully downloaded
+            del dbkeys[file_dbkey]
+
+    if len(dbkeys) > 0:
+        return {"error": f"The data from the following dbkeys could not be downloaded: {dbkeys}"}
+
+    return {"success": "Completed outflow flow data download."}
+
+
 if __name__ == "__main__":
     workspace = sys.argv[1].rstrip("/")
     main(workspace)