ecopipeline 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

ecopipeline/load/__init__.py

@@ -1,2 +1,2 @@
-from .load import check_table_exists, create_new_table, load_overwrite_database, load_event_table
-__all__ = ["check_table_exists", "create_new_table", "load_overwrite_database", "load_event_table"]
+from .load import check_table_exists, create_new_table, load_overwrite_database, load_event_table, report_data_loss
+__all__ = ["check_table_exists", "create_new_table", "load_overwrite_database", "load_event_table", "report_data_loss"]

ecopipeline/load/load.py

@@ -8,7 +8,7 @@ import math
 pd.set_option('display.max_columns', None)
 import mysql.connector.errors as mysqlerrors
 from ecopipeline import ConfigManager
-import datetime
+from datetime import datetime, timedelta
 import numpy as np
 
 data_map = {'int64':'float',
@@ -161,97 +161,123 @@ def create_new_columns(cursor : mysql.connector.cursor.MySQLCursor, table_name:
 
     return True
 
-def load_overwrite_database(cursor : mysql.connector.cursor.MySQLCursor, dataframe: pd.DataFrame, config_info: dict, data_type: str, primary_key: str = "time_pt", table_name: str = None):
+def load_overwrite_database(config : ConfigManager, dataframe: pd.DataFrame, config_info: dict, data_type: str, 
+                            primary_key: str = "time_pt", table_name: str = None, auto_log_data_loss : bool = False):
     """
     Loads given pandas DataFrame into a MySQL table overwriting any conflicting data. Uses an UPSERT strategy to ensure any gaps in data are filled.
     Note: will not overwrite values with NULL. Must have a new value to overwrite existing values in database
 
     Parameters
     ----------  
-    cursor : mysql.connector.cursor.MySQLCursor
-        A cursor object connected to the database where the data will land
+    config : ecopipeline.ConfigManager
+        The ConfigManager object that holds configuration data for the pipeline.
     dataframe: pd.DataFrame
         The pandas DataFrame to be written into the mySQL server.
     config_info: dict
         The dictionary containing the configuration information in the data upload. This can be aquired through the get_login_info() function in this package
     data_type: str
-        The header name corresponding to the table you wish to write data to.  
+        The header name corresponding to the table you wish to write data to. 
+    primary_key : str
+        The name of the primary key in the database to upload to. Default as 'time_pt'
+    table_name : str
+        overwrites table name from config_info if needed
+    auto_log_data_loss : bool
+        if set to True, a data loss event will be reported if no data exits in the dataframe 
+        for the last two days from the current date OR if an error occurs
 
     Returns
     ------- 
     bool: 
         A boolean value indicating if the data was successfully written to the database. 
     """
-    # Drop empty columns
-    dataframe = dataframe.dropna(axis=1, how='all')
-
-    dbname = config_info['database']
-    if table_name == None:
-        table_name = config_info[data_type]["table_name"]   
-    
-    if(len(dataframe.index) <= 0):
-        print(f"Attempted to write to {table_name} but dataframe was empty.")
-        return True
-
-    print(f"Attempting to write data for {dataframe.index[0]} to {dataframe.index[-1]} into {table_name}")
-    
-    # Get string of all column names for sql insert
-    sensor_names = primary_key
-    sensor_types = ["datetime"]
-    for column in dataframe.columns:
-        sensor_names += "," + column    
-        sensor_types.append(data_map[dataframe[column].dtype.name])
+    # Database Connection
+    db_connection, cursor = config.connect_db()
+    try:
 
-    # create SQL statement
-    insert_str = "INSERT INTO " + table_name + " (" + sensor_names + ") VALUES ("
-    for column in dataframe.columns:
-        insert_str += "%s, "
-    insert_str += "%s)"
-    
-    # last_time = datetime.datetime.strptime('20/01/1990', "%d/%m/%Y") # arbitrary past date
-    existing_rows_list = []
+        # Drop empty columns
+        dataframe = dataframe.dropna(axis=1, how='all')
 
-    # create db table if it does not exist, otherwise add missing columns to existing table
-    if not check_table_exists(cursor, table_name, dbname):
-        if not create_new_table(cursor, table_name, sensor_names.split(",")[1:], sensor_types[1:], primary_key=primary_key): #split on colums and remove first column aka time_pt
-            print(f"Could not create new table {table_name} in database {dbname}")
-            return False
-    else:
-        try:
-            # find existing times in database for upsert statement
-            cursor.execute(
-                f"SELECT {primary_key} FROM {table_name} WHERE {primary_key} >= '{dataframe.index.min()}'")
-            # Fetch the results into a DataFrame
-            existing_rows = pd.DataFrame(cursor.fetchall(), columns=[primary_key])
+        dbname = config_info['database']
+        if table_name == None:
+            table_name = config_info[data_type]["table_name"]   
+        
+        if(len(dataframe.index) <= 0):
+            print(f"Attempted to write to {table_name} but dataframe was empty.")
+            ret_value = True
+        else:
 
-            # Convert the primary_key column to a list
-            existing_rows_list = existing_rows[primary_key].tolist()
+            print(f"Attempting to write data for {dataframe.index[0]} to {dataframe.index[-1]} into {table_name}")
+            if auto_log_data_loss and dataframe.index[-1] < datetime.now() - timedelta(days=2):
+                report_data_loss(config)
+            
+            # Get string of all column names for sql insert
+            sensor_names = primary_key
+            sensor_types = ["datetime"]
+            for column in dataframe.columns:
+                sensor_names += "," + column    
+                sensor_types.append(data_map[dataframe[column].dtype.name])
+
+            # create SQL statement
+            insert_str = "INSERT INTO " + table_name + " (" + sensor_names + ") VALUES ("
+            for column in dataframe.columns:
+                insert_str += "%s, "
+            insert_str += "%s)"
+            
+            # last_time = datetime.strptime('20/01/1990', "%d/%m/%Y") # arbitrary past date
+            existing_rows_list = []
+
+            # create db table if it does not exist, otherwise add missing columns to existing table
+            if not check_table_exists(cursor, table_name, dbname):
+                if not create_new_table(cursor, table_name, sensor_names.split(",")[1:], sensor_types[1:], primary_key=primary_key): #split on colums and remove first column aka time_pt
+                    ret_value = False
+                    raise Exception(f"Could not create new table {table_name} in database {dbname}")
+            else:
+                try:
+                    # find existing times in database for upsert statement
+                    cursor.execute(
+                        f"SELECT {primary_key} FROM {table_name} WHERE {primary_key} >= '{dataframe.index.min()}'")
+                    # Fetch the results into a DataFrame
+                    existing_rows = pd.DataFrame(cursor.fetchall(), columns=[primary_key])
+
+                    # Convert the primary_key column to a list
+                    existing_rows_list = existing_rows[primary_key].tolist()
+
+                except mysqlerrors.Error:
+                    print(f"Table {table_name} has no data.")
+
+                missing_cols, missing_types = find_missing_columns(cursor, dataframe, config_info, table_name)
+                if len(missing_cols):
+                    if not create_new_columns(cursor, table_name, missing_cols, missing_types):
+                        print("Unable to add new columns due to database error.")
+            
+            updatedRows = 0
+            for index, row in dataframe.iterrows():
+                time_data = row.values.tolist()
+                #remove nans and infinites
+                time_data = [None if (x is None or pd.isna(x)) else x for x in time_data]
+                time_data = [None if (x == float('inf') or x == float('-inf')) else x for x in time_data]
+
+                if index in existing_rows_list:
+                    statement, values = _generate_mysql_update(row, index, table_name, primary_key)
+                    if statement != "":
+                        cursor.execute(statement, values)
+                        updatedRows += 1
+                else:
+                    cursor.execute(insert_str, (index, *time_data))
 
-        except mysqlerrors.Error:
-            print(f"Table {table_name} has no data.")
+            db_connection.commit()
+            print(f"Successfully wrote {len(dataframe.index)} rows to table {table_name} in database {dbname}. {updatedRows} existing rows were overwritten.")
+            ret_value = True
+    except Exception as e:
+        print(f"Unable to load data into database. Exception: {e}")
+        if auto_log_data_loss:
+            report_data_loss(config)
+        ret_value = False
 
-        missing_cols, missing_types = find_missing_columns(cursor, dataframe, config_info, table_name)
-        if len(missing_cols):
-            if not create_new_columns(cursor, table_name, missing_cols, missing_types):
-                print("Unable to add new columns due to database error.")
-    
-    updatedRows = 0
-    for index, row in dataframe.iterrows():
-        time_data = row.values.tolist()
-        #remove nans and infinites
-        time_data = [None if (x is None or pd.isna(x)) else x for x in time_data]
-        time_data = [None if (x == float('inf') or x == float('-inf')) else x for x in time_data]
-
-        if index in existing_rows_list:
-            statement, values = _generate_mysql_update(row, index, table_name, primary_key)
-            if statement != "":
-                cursor.execute(statement, values)
-                updatedRows += 1
-        else:
-            cursor.execute(insert_str, (index, *time_data))
+    db_connection.close()
+    cursor.close()
+    return ret_value
 
-    print(f"Successfully wrote {len(dataframe.index)} rows to table {table_name} in database {dbname}. {updatedRows} existing rows were overwritten.")
-    return True
 
 def load_event_table(config : ConfigManager, event_df: pd.DataFrame, site_name : str = None):
     """
@@ -299,7 +325,7 @@ def load_event_table(config : ConfigManager, event_df: pd.DataFrame, site_name :
         column_names += "," + column
 
     # create SQL statement
-    insert_str = "INSERT INTO " + table_name + " (" + column_names + ", last_modified_date, last_modified_by) VALUES (%s,%s,%s,%s,%s,'"+datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')+"','automatic_upload')"
+    insert_str = "INSERT INTO " + table_name + " (" + column_names + ", last_modified_date, last_modified_by) VALUES (%s,%s,%s,%s,%s,'"+datetime.now().strftime('%Y-%m-%d %H:%M:%S')+"','automatic_upload')"
 
     # add aditional columns for db creation
     full_column_names = column_names.split(",")[1:]
@@ -368,6 +394,71 @@ def load_event_table(config : ConfigManager, event_df: pd.DataFrame, site_name :
     cursor.close()
     return True
 
+def report_data_loss(config : ConfigManager, site_name : str = None):
+    """
+    Logs data loss event in event database (assumes one exists)
+
+    Parameters
+    ----------  
+    config : ecopipeline.ConfigManager
+        The ConfigManager object that holds configuration data for the pipeline.
+    site_name : str
+        the name of the site to correspond the events with. If left blank will default to minute table name
+
+    Returns
+    ------- 
+    bool: 
+        A boolean value indicating if the data was successfully written to the database. 
+    """
+    # Drop empty columns
+
+    dbname = config.get_db_name()
+    table_name = "site_events"
+    if site_name is None:
+        site_name = config.get_site_name()
+    error_string = "Error proccessing data. Please check logs to resolve."
+
+    print(f"logging DATA_LOSS into {table_name}")
+
+    # create SQL statement
+    insert_str = "INSERT INTO " + table_name + " (start_time_pt, site_name, event_detail, event_type, last_modified_date, last_modified_by) VALUES "
+    insert_str += f"('{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}','{site_name}','{error_string}','DATA_LOSS','{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}','automatic_upload')"
+
+    existing_rows = pd.DataFrame({
+        'id' : []
+    })
+
+    connection, cursor = config.connect_db() 
+
+    # create db table if it does not exist, otherwise add missing columns to existing table
+    if not check_table_exists(cursor, table_name, dbname):
+        print(f"Cannot log data loss. {table_name} does not exist in database {dbname}")
+        return False
+    else:
+        try:
+            # find existing times in database for upsert statement
+            cursor.execute(
+                f"SELECT id FROM {table_name} WHERE end_time_pt IS NULL AND site_name = '{site_name}' AND event_type = 'DATA_LOSS'")
+            # Fetch the results into a DataFrame
+            existing_rows = pd.DataFrame(cursor.fetchall(), columns=['id'])
+
+        except mysqlerrors.Error as e:
+            print(f"Retrieving data from {table_name} caused exception: {e}")
+    try:
+        
+        if existing_rows.empty:
+            cursor.execute(insert_str)
+            connection.commit()
+            print("Successfully logged data loss.")
+        else:
+            print("Data loss already logged.")
+    except Exception as e:
+        # Print the exception message
+        print(f"Caught an exception when uploading to site_events table: {e}")
+    connection.close()
+    cursor.close()
+    return True
+
 def _generate_mysql_update_event_table(row, id):
     statement = f"UPDATE site_events SET "
     statment_elems = []
@@ -379,7 +470,7 @@ def _generate_mysql_update_event_table(row, id):
 
     if values:
         statement += ", ".join(statment_elems)
-        statement += f", last_modified_by = 'automatic_upload', last_modified_date = '{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}'"
+        statement += f", last_modified_by = 'automatic_upload', last_modified_date = '{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}'"
         statement += f" WHERE id = {id};"
         # statement += f" WHERE start_time_pt = '{start_time_pt}' AND end_time_pt = '{end_time_pt}' AND event_type = '{event_type}' AND site_name = '{site_name}';"
     else:

{ecopipeline-0.5.0.dist-info → ecopipeline-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: ecopipeline
-Version: 0.5.0
+Version: 0.6.0
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

{ecopipeline-0.5.0.dist-info → ecopipeline-0.6.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
 ecopipeline/__init__.py,sha256=vCRzwd781ciCSXMP1ycM_BXAqxj3KVaNKIjsLOPcbwc,171
 ecopipeline/extract/__init__.py,sha256=3u_CUMdCguVewU3kN8x6xhVNyo1-p-gwTrhjOh7Psqg,645
 ecopipeline/extract/extract.py,sha256=ryorqnu1RgyNK7joZRcbMmTajlTlB5hwaYzzpo8Z8Ho,43369
-ecopipeline/load/__init__.py,sha256=7ipv7GJfZ5382lcrdNm4MyM-WiCEVuRWTqxyzDSZhqg,197
-ecopipeline/load/load.py,sha256=RbGZSsigkChZpX1SZzYvZuS6-DS0k-d2IYUsJvZDvmk,17485
+ecopipeline/load/__init__.py,sha256=oDAVF8AhK_qugqegjW7jK16p-nb9QzKhiNQOkEBniKM,235
+ecopipeline/load/load.py,sha256=SzbUSq7M2f2Ifj_bbHzDRP63-x6UBJbEZ1C4blCpC6U,21416
 ecopipeline/transform/__init__.py,sha256=DcIJfkRs4OmZzDeEfW_OiOIXNqN6CUl1_lW0SS7-eN8,2280
 ecopipeline/transform/bayview.py,sha256=TP24dnTsUD95X-f6732egPZKjepFLJgDm9ImGr-fppY,17899
 ecopipeline/transform/lbnl.py,sha256=EQ54G4rJXaZ7pwVusKcdK2KBehSdCsNo2ybphtMGs7o,33400
@@ -10,8 +10,8 @@ ecopipeline/transform/transform.py,sha256=uyBIXKCXUCT6zVnZyQohripGAzmY1yV9T1GxsX
 ecopipeline/utils/ConfigManager.py,sha256=t4sfTjGO0g5P50XBQqGVFWaXfAlW1GMDh1DLoBuFGks,9826
 ecopipeline/utils/__init__.py,sha256=ccWUR0m7gD9DfcgsxBCLOfi4lho6RdYuB2Ugy_g6ZdQ,28
 ecopipeline/utils/unit_convert.py,sha256=VFh1we2Y8KV3u21BeWb-U3TlZJXo83q5vdxxkpgcuME,3064
-ecopipeline-0.5.0.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ecopipeline-0.5.0.dist-info/METADATA,sha256=gb550ta4s3kOYQCt52JECXo_HaSEeXKkWGeIF3mTVr8,2307
-ecopipeline-0.5.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ecopipeline-0.5.0.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
-ecopipeline-0.5.0.dist-info/RECORD,,
+ecopipeline-0.6.0.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ecopipeline-0.6.0.dist-info/METADATA,sha256=HgOhlhoLCXiA1vSP42F7RlHURYBuCTiml5VR48BF7Jo,2307
+ecopipeline-0.6.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ecopipeline-0.6.0.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
+ecopipeline-0.6.0.dist-info/RECORD,,

{ecopipeline-0.5.0.dist-info → ecopipeline-0.6.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any