loone-data-prep 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

loone_data_prep/flow_data/hydro.py

@@ -2,6 +2,7 @@ import sys
 from datetime import datetime
 from glob import glob
 from retry import retry
+import os
 import pandas as pd
 from rpy2.robjects import r
 from rpy2.rinterface_lib.embedded import RRuntimeError
@@ -18,36 +19,92 @@ def get(
     date_max: str = DATE_NOW
 ) -> None:
     r_str = f"""
-    # Load the required libraries
-    library(dbhydroR)
+    download_flow_data <- function(workspace, dbkey, date_min, date_max) 
+    {{
+        # Load the required libraries
+        library(dbhydroR)
+        library(dplyr)
 
-    # Retrieve data for the dbkey
-    data <- get_hydro(dbkey = "{dbkey}", date_min = "{date_min}", date_max = "{date_max}")
+        # Retrieve data for the dbkey
+        data <- get_hydro(dbkey = "{dbkey}", date_min = "{date_min}", date_max = "{date_max}", raw = TRUE)
+        
+        # Check if data is empty or contains only the "date" column
+        if (ncol(data) <= 1) {{
+            cat("No data found for dbkey", "{dbkey}", "Skipping to the next dbkey.\n")
+        }}
+        
+        # Give data.frame correct column names so it can be cleaned using the clean_hydro function
+        colnames(data) <- c("station", "dbkey", "date", "data.value", "qualifer", "revision.date")
+        
+        # Check if the data.frame has any rows
+        if (nrow(data) == 0) 
+        {{
+            # No data given back, It's possible that the dbkey has reached its end date.
+            print(paste("Empty data.frame returned for dbkey", "{dbkey}", "It's possible that the dbkey has reached its end date. Skipping to the next dbkey."))
+            return(list(success = FALSE, dbkey = "{dbkey}"))
+        }}
+        
+        # Add a type and units column to data so it can be cleaned using the clean_hydro function
+        data$type <- "FLOW"
+        data$units <- "cfs"
+        
+        # Get the station
+        station <- data$station[1]
+        
+        # Clean the data.frame
+        data <- clean_hydro(data)
 
-    # Check if data is empty or contains only the "date" column
-    if (ncol(data) <= 1) {{
-        cat("No data found for dbkey", "{dbkey}", "Skipping to the next dbkey.\n")
-    }}
-
-    # Multiply all columns except "date" column by 0.0283168466 * 86400 to convert Flow rate from cfs to m³/day
-    data[, -1] <- data[, -1] * (0.0283168466 * 86400)
-
-    # Extract the column names excluding the date column
-    column_names <- names(data)[-1]
+        # Multiply all columns except "date" column by 0.0283168466 * 86400 to convert Flow rate from cfs to m³/day
+        data[, -1] <- data[, -1] * (0.0283168466 * 86400)
+        
+        # Drop the " _FLOW_cfs" column
+        data <- data %>% select(-` _FLOW_cfs`)    
+        
+        # Sort the data by date
+        data <- data[order(data$date), ]
+        
+        # Get the filename for the output CSV file
+        filename <- paste0(station, "_FLOW", "_{dbkey}_cmd.csv")
+        
+        # Save data to a CSV file
+        write.csv(data, file = paste0("{workspace}/", filename))
 
-    # Generate the filename based on the column names
-    filename <- paste0( gsub(" ", "_", sub("_[^_]*$", "", paste(column_names, collapse = "_"))), "_{dbkey}_cmd.csv")
-    # Save data to a CSV file
-    write.csv(data, file = paste0("{workspace}/", filename))
+        # Print a message indicating the file has been saved
+        cat("CSV file", filename, "has been saved.\n")
 
-    # Print a message indicating the file has been saved
-    cat("CSV file", filename, "has been saved.\n")
-
-    # Add a delay between requests
-    Sys.sleep(1)  # Wait for 1 second before the next iteration
+        # Add a delay between requests
+        Sys.sleep(1)  # Wait for 1 second before the next iteration
+        
+        # Return the station and dbkey to the python code
+        list(success = TRUE, station = station, dbkey = "{dbkey}")
+    }}
     """
 
     r(r_str)
+    
+    # Call the R function to download the flow data
+    result = r.download_flow_data(workspace, dbkey, date_min, date_max)
+    
+    # Check for failure
+    success = result.rx2("success")[0]
+
+    if not success:
+        return
+    
+    # Get the station name for _reformat_flow_file()
+    station = result.rx2("station")[0]
+    
+    # Reformat the flow data file to the expected layout
+    _reformat_flow_file(workspace, station, dbkey)
+    
+    # Check if the station name contains a space
+    if " " in station:
+        # Replace space with underscore in the station name
+        station_previous = station
+        station = station.replace(" ", "_")
+        
+        # Rename the file
+        os.rename(f"{workspace}/{station_previous}_FLOW_{dbkey}_cmd.csv", f"{workspace}/{station}_FLOW_{dbkey}_cmd.csv")
 
     # column values are converted to cmd in R. This snippet makes sure column names are updated accordingly.
     file = glob(f'{workspace}/*FLOW*{dbkey}_cmd.csv')[0]
@@ -55,7 +112,44 @@ def get(
     df.columns = df.columns.astype(str).str.replace("_cfs", "_cmd")
     df.to_csv(file, index=False)
 
-    if __name__ == "__main__":
-        workspace = sys.argv[1].rstrip("/")
-        dbkey = sys.argv[2]
-        get(workspace, dbkey)
+
+def _reformat_flow_file(workspace:str, station: str, dbkey: str):
+    '''
+    Reformat the flow data file to the expected layout.
+    Converts the format of the dates in the file to 'YYYY-MM-DD' then sorts the data by date.
+    Reads and writes to a .CSV file.
+    
+    Args:
+        workspace (str): The path to the workspace directory.
+        station (str): The station name.
+        dbkey (str): The dbkey for the station.
+        
+    Returns:
+        None
+    '''
+    # Read in the data
+    df = pd.read_csv(f"{workspace}/{station}_FLOW_{dbkey}_cmd.csv")
+    
+    # Grab only the columns we need
+    df = df[['date', f'{station}_FLOW_cfs']]
+    
+    # Convert date column to datetime
+    df['date'] = pd.to_datetime(df['date'], format='%d-%b-%Y')
+    
+    # Sort the data by date
+    df.sort_values('date', inplace=True)
+    
+    # Renumber the index
+    df.reset_index(drop=True, inplace=True)
+    
+    # Drop rows that are missing values for both the date and value columns
+    df = df.drop(df[(df['date'].isna()) & (df[f'{station}_FLOW_cfs'].isna())].index)
+    
+    # Write the updated data back to the file
+    df.to_csv(f"{workspace}/{station}_FLOW_{dbkey}_cmd.csv")
+
+
+if __name__ == "__main__":
+    workspace = sys.argv[1].rstrip("/")
+    dbkey = sys.argv[2]
+    get(workspace, dbkey)

loone_data_prep/utils.py

@@ -4,6 +4,7 @@ import datetime
 import math
 from glob import glob
 from calendar import monthrange
+import traceback
 import numpy as np
 import pandas as pd
 from retry import retry
@@ -635,8 +636,9 @@ def nutrient_prediction(
                 predicted_column.columns = [column_name]
 
                 prediction_columns.append(predicted_column)
-            except RuntimeWarning:
-                breakpoint()
+            except RuntimeWarning as e:
+                print(f"Unexpected RuntimeWarning: {str(e)}")
+                traceback.print_exc()
 
         # Concat individual ensemble columns together into one pandas DataFrame
         out_dataframe = pd.concat(objs=prediction_columns, axis="columns")
@@ -661,6 +663,75 @@ def nutrient_prediction(
         out_dataframe.to_csv(os.path.join(input_dir, f"{station}_PHOSPHATE_predicted.csv"))
 
 
+def find_last_date_in_csv(workspace: str, file_name: str) -> str:
+    """
+    Gets the most recent date from the last line of a .csv file.
+    Assumes the file is formatted as a .csv file, encoded in UTF-8, 
+    and the rows in the file are sorted by date in ascending order.
+
+    Args:
+        workspace (str): The directory where the file is located.
+        file_name (str): The name of the file.
+
+    Returns:
+        str: The most recent date as a string in YYYY-MM-DD format, or None if the file does not exist or the date cannot be found.
+    """
+    # Helper Functions
+    def is_valid_date(date_string):
+        try:
+            datetime.datetime.strptime(date_string, '%Y-%m-%d')
+            return True
+        except ValueError:
+            return False
+    
+    # Check that file exists
+    file_path = os.path.join(workspace, file_name)
+    if not os.path.exists(file_path):
+        return None
+    
+    # Attempt to extract the date of the last line in the file
+    try:
+        with open(file_path, 'rb') as file:
+            # Go to the end of the file
+            file.seek(-2, os.SEEK_END)
+            
+            # Loop backwards until you find the first newline character
+            while file.read(1) != b'\n':
+                file.seek(-2, os.SEEK_CUR)
+                
+            # Read the last line
+            last_line = file.readline().decode()
+            
+            # Extract the date from the last line
+            date = None
+            
+            for value in last_line.split(','):
+                if is_valid_date(value):
+                    date = value
+                    break
+            
+            # Return date
+            return date
+    except OSError as e:
+        print(f"Error reading file {file_name}: {e}")
+        return None
+
+
+def dbhydro_data_is_latest(date_latest: str):
+    """
+    Checks whether the given date is the most recent date possible to get data from dbhydro.
+    Can be used to check whether dbhydro data is up-to-date.
+    
+    Args:
+        date_latest (str): The date of the most recent data of the dbhydro data you have
+    
+    Returns:
+        bool: True if the date_latest is the most recent date possible to get data from dbhydro, False otherwise
+    """
+    date_latest_object = datetime.datetime.strptime(date_latest, "%Y-%m-%d").date()
+    return date_latest_object == (datetime.datetime.now().date() - datetime.timedelta(days=1))
+
+
 if __name__ == "__main__":
     if sys.argv[1] == "get_dbkeys":
         get_dbkeys(sys.argv[2].strip("[]").replace(" ", "").split(","), *sys.argv[3:])

loone_data_prep/water_level_data/get_all.py

@@ -1,34 +1,231 @@
 import sys
 import os
+import requests
+import uuid
+from datetime import datetime
 from loone_data_prep.water_level_data import hydro
+from loone_data_prep.flow_data.get_forecast_flows import get_stations_latitude_longitude
+from loone_data_prep.utils import find_last_date_in_csv, dbhydro_data_is_latest
+import pandas as pd
 
+DATE_NOW = datetime.now().date().strftime("%Y-%m-%d")
 
 D = {
-    "LO_Stage": {"dbkeys": ["16022", "12509", "12519", "16265", "15611"]},
-    "Stg_3ANW": {"dbkeys": ["LA369"], "date_min": "1972-01-01", "date_max": "2023-04-30"},
-    "Stg_2A17": {"dbkeys": ["16531"], "date_min": "1972-01-01", "date_max": "2023-04-30"},
-    "Stg_3A3": {"dbkeys": ["16532"], "date_min": "1972-01-01", "date_max": "2023-04-30"},
-    "Stg_3A4": {"dbkeys": ["16537"], "date_min": "1972-01-01", "date_max": "2023-04-30"},
-    "Stg_3A28": {"dbkeys": ["16538"], "date_min": "1972-01-01", "date_max": "2023-04-30"}
+    "LO_Stage": {"dbkeys": ["16022", "12509", "12519", "16265", "15611"], "datum": "NGVD29"},
+    "LO_Stage_2": {"dbkeys": ["94832"], "date_min": "2024-04-30", "datum": "NAVD88"},
+    "Stg_3ANW": {"dbkeys": ["LA369"], "date_min": "1972-01-01", "date_max": "2023-04-30", "datum": "NGVD29"},
+    "Stg_2A17": {"dbkeys": ["16531"], "date_min": "1972-01-01", "date_max": "2023-04-30", "datum": "NGVD29"},
+    "Stg_3A3": {"dbkeys": ["16532"], "date_min": "1972-01-01", "date_max": "2023-04-30", "datum": "NGVD29"},
+    "Stg_3A4": {"dbkeys": ["16537"], "date_min": "1972-01-01", "date_max": "2023-04-30", "datum": "NGVD29"},
+    "Stg_3A28": {"dbkeys": ["16538"], "date_min": "1972-01-01", "date_max": "2023-04-30", "datum": "NGVD29"}
 }
 
 
 def main(workspace: str, d: dict = D) -> dict:
     missing_files = []
+    failed_downloads = []   # List of file names that the script failed to get the latest data for (but the files still exist)
+    
+    # Get the date of the latest data in LO_Stage_2.csv
+    date_latest_lo_stage_2 = find_last_date_in_csv(workspace, "LO_Stage_2.csv")
+    
     for name, params in d.items():
-        print(f"Getting {name}.")
-        hydro.get(workspace, name, **params)
+        # Get the date of the latest data in the csv file
+        date_latest = find_last_date_in_csv(workspace, f"{name}.csv")
+        
+        # File with data for this dbkey does NOT already exist (or possibly some other error occurred)
+        if date_latest is None:
+            print(f"Getting all water level data for {name}.")
+            hydro.get(workspace, name, **params)
+        else:
+            # Check whether the latest data is already up to date.
+            if dbhydro_data_is_latest(date_latest):
+                # Notify that the data is already up to date
+                print(f'Downloading of new water level data skipped for {name}. Data is already up to date.')
+                continue
+            
+            # Temporarily rename current data file so it isn't over written
+            original_file_name = f"{name}.csv"
+            original_file_name_temp = f"{name}_{uuid.uuid4()}.csv"
+            os.rename(os.path.join(workspace, original_file_name), os.path.join(workspace, original_file_name_temp))
+            
+            try:
+                # Download only the new data
+                print(f'Downloading new water level data for {name} starting from date {date_latest}')
+                hydro.get(workspace, name, dbkeys=params['dbkeys'], date_min=date_latest, date_max=DATE_NOW, datum=params['datum'])
+                
+                # Read in the original data and the newly downloaded data
+                df_original = pd.read_csv(os.path.join(workspace, original_file_name_temp), index_col=0)
+                df_new = pd.read_csv(os.path.join(workspace, original_file_name), index_col=0)
+                
+                # For get_hydro() calls with multiple dbkeys, remove the row corresponding to the latest date from the downloaded data.
+                # When get_hydro() is given multiple keys its returned data starts from the date given instead of the day after like it
+                # does when given a single key.
+                if len(params['dbkeys']) > 1:
+                    df_new = df_new[df_new['date'] != date_latest]
+                
+                # Merge the new data with the original data
+                df_merged = pd.concat([df_original, df_new], ignore_index=True)
+                
+                # Write out the merged data
+                df_merged.to_csv(os.path.join(workspace, original_file_name))
+                
+                # Remove the original renamed data file
+                os.remove(os.path.join(workspace, original_file_name_temp))
+            except Exception as e:
+                # Notify of the error
+                print(f"Error occurred while downloading new water level data: {e}")
+                
+                # Remove the newly downloaded data file if it exists
+                if os.path.exists(os.path.join(workspace, original_file_name)):
+                    os.remove(os.path.join(workspace, original_file_name))
+                
+                # Rename the original renamed file back to its original name
+                if os.path.exists(os.path.join(workspace, original_file_name_temp)):
+                    os.rename(os.path.join(workspace, original_file_name_temp), os.path.join(workspace, original_file_name))
+                
+                # Add the file name to the list of failed downloads
+                failed_downloads.append(original_file_name)
+            
         if os.path.exists(os.path.join(workspace, f"{name}.csv")):
             print(f"{name} downloaded successfully.")
         else:
             missing_files.append(f"{name}.csv")
             print(f"{name} could not be downloaded after various tries.")
 
-    if missing_files:
-        return {"error": f"The following files could not be downloaded: {missing_files}"}
-
+    # Merge data from old and new dbkey for station "L OKEE"
+    convert_failure = False
+    if os.path.exists(os.path.join(workspace, "LO_Stage.csv")) and os.path.exists(os.path.join(workspace, "LO_Stage_2.csv")):
+        # Output Progress
+        print("\nMerging data for station 'L OKEE'...")
+        
+        # Get the latitude and longitude of the "L OKEE" station
+        lat_long_map = get_stations_latitude_longitude(["L OKEE"])
+        latitude, longitude = lat_long_map["L OKEE"]
+        
+        # Load the LO_Stage_2.csv file
+        df_lo_stage_2 = pd.read_csv(os.path.join(workspace, "LO_Stage_2.csv"), index_col="date")
+        df_lo_stage_2.index = pd.to_datetime(df_lo_stage_2.index)
+        
+        # Output Progress
+        print("Converting NAVD88 to NGVD29 for 'L OKEE's new dbkey...\n")
+        
+        # Use only the data that is not already in the LO_Stage.csv file
+        if date_latest_lo_stage_2 is not None:
+            date_start = datetime.strptime(date_latest_lo_stage_2, "%Y-%m-%d") + pd.DateOffset(days=1)
+            df_lo_stage_2 = df_lo_stage_2.loc[date_start:]
+        
+        # Convert the stage values from NAVD88 to NGVD29
+        lo_stage_2_dates = df_lo_stage_2.index.tolist()
+        lo_stage_2_values_navd88 = df_lo_stage_2["L OKEE_STG_ft NGVD29"].tolist()
+        lo_stage_2_values_ngvd29 = []
+        
+        for i in range(0, len(lo_stage_2_values_navd88)):
+            date = lo_stage_2_dates[i]
+            value = lo_stage_2_values_navd88[i]
+            try:
+                lo_stage_2_values_ngvd29.append(_convert_navd88_to_ngvd29(latitude, longitude, value, date.year))
+            except Exception as e:
+                convert_failure = True
+                print(str(e))
+                break
+        
+        # Check for conversion failure
+        if not convert_failure:        
+            # Update the LO_Stage.csv file with the converted values
+            df_lo_stage = pd.read_csv(os.path.join(workspace, "LO_Stage.csv"), index_col="date")
+            df_lo_stage.index = pd.to_datetime(df_lo_stage.index)
+            
+            for i in range(0, len(lo_stage_2_values_ngvd29)):
+                # Get the current date and value
+                date = lo_stage_2_dates[i]
+                value = lo_stage_2_values_ngvd29[i]
+                
+                # Update the value in the LO_Stage dataframe
+                df_lo_stage.at[date, "L OKEE_STG_ft NGVD29"] = value
+            
+            # Reset the index
+            df_lo_stage.reset_index(inplace=True)
+            df_lo_stage.drop(columns=["Unnamed: 0"], inplace=True)
+            
+            # Save the updated LO_Stage.csv file
+            df_lo_stage.to_csv(os.path.join(workspace, "LO_Stage.csv"))
+    else:
+        # Conversion failed due to missing files
+        convert_failure = True
+        print("Error: Missing LO_Stage.csv or LO_Stage_2.csv file, cannot convert and merge.")
+            
+    if missing_files or convert_failure:
+        error_string = ""
+        
+        if missing_files:
+            error_string += f"The following files could not be downloaded: {missing_files}"
+        
+        if failed_downloads:
+            error_string += f"\nFailed to download the latest data for the following files: {failed_downloads}"
+        
+        if convert_failure:
+            error_string += "\nFailed to convert NAVD88 to NGVD29 for 'L OKEE' station."
+            
+        return {"error": error_string}
+    
     return {"success": "Completed water level data download."}
 
+def _convert_navd88_to_ngvd29(latitude: float, longitude: float, stage: float, year: int) -> float:
+    """Converts a stage value from NAVD88 to NGVD29 using NCAT.
+    
+    Args:
+        latitude (float): The latitude of the station (in decimal degrees format).
+        longitude (float): The longitude of the station (in decimal degrees format).
+        stage (float): The stage (water level) value to convert (in feet).
+        year (int): The year when the stage value was recorded.
+        
+    Returns:
+        float: The converted stage value in feet (NGVD29).
+    """
+    # Helper functions
+    def _feet_to_meters(feet: float) -> float:
+        return feet * 0.3048
+    
+    def _meters_to_feet(meters: float) -> float:
+        return meters / 0.3048
+    
+    # Check for NA value
+    if pd.isna(stage):
+        return stage
+    
+    # Convert stage to meters
+    stage_meters = _feet_to_meters(stage)
+    
+    # Make request
+    base_url = "https://geodesy.noaa.gov/api/ncat/llh"
+    
+    params = {
+        "lat": latitude,            # latitude
+        "lon": longitude,           # longitude
+        "orthoHt": stage_meters,    # orthometric height in NAVD88
+        "year": year,               # year of observation
+        "inDatum": "NAD83(1986)",   # Datum used for input latitude and longitude
+        "outDatum": "NAD83(1986)",  # Datum used for output latitude and longitude
+        "inVertDatum": "NAVD88",    # vertical datum of input orthometric height
+        "outVertDatum": "NGVD29",   # vertical datum of output orthometric height (desired vertical datum)
+    }
+    
+    try:
+        response = requests.get(base_url, params=params)
+    except Exception as e:
+        raise Exception(f"Error converting NAVD88 to NGVD29: {e}")
+    
+    # Check for failure
+    if response.status_code != 200:
+        raise Exception(f"Error converting NAVD88 to NGVD29: {response.text}")
+    
+    # Return converted stage in feet
+    try:
+        value = _meters_to_feet(float(response.json()["destOrthoht"]))
+    except Exception as e:
+        raise Exception(f"Error converting NAVD88 to NGVD29: {e}")
+    
+    return value
 
 if __name__ == "__main__":
     workspace = sys.argv[1].rstrip("/")

loone_data_prep/water_level_data/hydro.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from retry import retry
 from rpy2.robjects import r
 from rpy2.rinterface_lib.embedded import RRuntimeError
-
+import pandas as pd
 
 DEFAULT_DBKEYS = ["16022", "12509", "12519", "16265", "15611"]
 DATE_NOW = datetime.now().strftime("%Y-%m-%d")
@@ -16,20 +16,88 @@ def get(
     dbkeys: list = DEFAULT_DBKEYS,
     date_min: str = "1950-01-01",
     date_max: str = DATE_NOW,
+    datum: str = "",
     **kwargs: str | list
 ) -> None:
+    # Get the type and units for the station
+    data_type = "STG"
+    units = "ft NGVD29"
+    
+    if name in ["Stg_3A3", "Stg_2A17", "Stg_3A4", "Stg_3A28"]:
+        data_type = "GAGHT"
+        units = "feet"
+    
     dbkeys_str = "\"" + "\", \"".join(dbkeys) + "\""
     r(
         f"""
         # Load the required libraries
         library(rio)
         library(dbhydroR)
-        #Stage Data
-        {name} = get_hydro(dbkey = c({dbkeys_str}), date_min = "{date_min}", date_max = "{date_max}")
+        library(dplyr)
+        
+        # Stage Data
+        if ("{datum}" == "")
+        {{
+            {name} <- get_hydro(dbkey = c({dbkeys_str}), date_min = "{date_min}", date_max = "{date_max}", raw = TRUE)
+        }}
+        
+        if (nchar("{datum}") > 0)
+        {{
+            {name} <- get_hydro(dbkey = c({dbkeys_str}), date_min = "{date_min}", date_max = "{date_max}", raw = TRUE, datum = "{datum}")
+        }}
+        
+        # Give data.frame correct column names so it can be cleaned using the clean_hydro function
+        colnames({name}) <- c("station", "dbkey", "date", "data.value", "qualifer", "revision.date")
+        
+        # Check if the data.frame has any rows
+        if (nrow({name}) == 0) 
+        {{
+            # No data given back, It's possible that the dbkey has reached its end date.
+            print(paste("Empty data.frame returned for dbkeys", "{dbkeys}", "It's possible that the dbkey has reached its end date. Skipping to the next dbkey."))
+            return(list(success = FALSE, dbkey = "{dbkeys}"))
+        }}
+        
+        # Get the station
+        station <- {name}$station[1]
+        
+        # Add a type and units column to data so it can be cleaned using the clean_hydro function
+        {name}$type <- "{data_type}"
+        {name}$units <- "{units}"
+        
+        # Clean the data.frame
+        {name} <- clean_hydro({name})
+        
+        # Drop the " _STG_ft NGVD29" column
+        {name} <- {name} %>% select(-` _{data_type}_{units}`)
+        
+        # Write the data to a csv file
         write.csv({name},file ='{workspace}/{name}.csv')
         """
     )
+    
+    _reformat_water_level_file(workspace, name)
 
+def _reformat_water_level_file(workspace: str, name: str):
+    # Read in the data
+    df = pd.read_csv(f"{workspace}/{name}.csv")
+    
+    # Drop the "Unnamed: 0" column
+    df.drop(columns=['Unnamed: 0'], inplace=True)
+    
+    # Convert date column to datetime
+    df['date'] = pd.to_datetime(df['date'], format='%d-%b-%Y')
+    
+    # Sort the data by date
+    df.sort_values('date', inplace=True)
+    
+    # Renumber the index
+    df.reset_index(drop=True, inplace=True)
+    
+    # Drop rows that are missing all their values
+    df.dropna(how='all', inplace=True)
+    
+    # Write the updated data back to the file
+    df.to_csv(f"{workspace}/{name}.csv")
 
 if __name__ == "__main__":
     args = [sys.argv[1].rstrip("/"), sys.argv[2]]