ecopipeline 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

ecopipeline/load/__init__.py

@@ -1,2 +1,2 @@
-from .load import check_table_exists, create_new_table, load_overwrite_database, load_event_table
-__all__ = ["check_table_exists", "create_new_table", "load_overwrite_database", "load_event_table"]
+from .load import check_table_exists, create_new_table, load_overwrite_database, load_event_table, report_data_loss
+__all__ = ["check_table_exists", "create_new_table", "load_overwrite_database", "load_event_table", "report_data_loss"]

ecopipeline/load/load.py

@@ -368,6 +368,69 @@ def load_event_table(config : ConfigManager, event_df: pd.DataFrame, site_name :
     cursor.close()
     return True
 
+def report_data_loss(config : ConfigManager, site_name : str = None):
+    """
+    Logs data loss event in event database (assumes one exists)
+
+    Parameters
+    ----------  
+    config : ecopipeline.ConfigManager
+        The ConfigManager object that holds configuration data for the pipeline.
+    site_name : str
+        the name of the site to correspond the events with. If left blank will default to minute table name
+
+    Returns
+    ------- 
+    bool: 
+        A boolean value indicating if the data was successfully written to the database. 
+    """
+    # Drop empty columns
+
+    dbname = config.get_db_name()
+    table_name = "site_events"
+    if site_name is None:
+        site_name = config.get_site_name()
+    error_string = "Error proccessing data. Please check logs to resolve."
+
+    print(f"logging DATA_LOSS into {table_name}")
+
+    # create SQL statement
+    insert_str = "INSERT INTO " + table_name + " (start_time_pt, site_name, event_detail, event_type, last_modified_date, last_modified_by) VALUES "
+    insert_str += f"('{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}','{site_name}','{error_string}','DATA_LOSS','{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}','automatic_upload')"
+
+    existing_rows = pd.DataFrame({
+        'id' : []
+    })
+
+    connection, cursor = config.connect_db() 
+
+    # create db table if it does not exist, otherwise add missing columns to existing table
+    if not check_table_exists(cursor, table_name, dbname):
+        print(f"Cannot log data loss. {table_name} does not exist in database {dbname}")
+        return False
+    else:
+        try:
+            # find existing times in database for upsert statement
+            cursor.execute(
+                f"SELECT id FROM {table_name} WHERE end_time_pt IS NULL AND site_name = '{site_name}' AND event_type = 'DATA_LOSS' and event_detail = '{error_string}'")
+            # Fetch the results into a DataFrame
+            existing_rows = pd.DataFrame(cursor.fetchall(), columns=['id'])
+
+        except mysqlerrors.Error as e:
+            print(f"Retrieving data from {table_name} caused exception: {e}")
+    try:
+        
+        if existing_rows.empty:
+            cursor.execute(insert_str)
+        connection.commit()
+        print("Successfully logged data loss.")
+    except Exception as e:
+        # Print the exception message
+        print(f"Caught an exception when uploading to site_events table: {e}")
+    connection.close()
+    cursor.close()
+    return True
+
 def _generate_mysql_update_event_table(row, id):
     statement = f"UPDATE site_events SET "
     statment_elems = []

{ecopipeline-0.5.0.dist-info → ecopipeline-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: ecopipeline
-Version: 0.5.0
+Version: 0.5.1
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

{ecopipeline-0.5.0.dist-info → ecopipeline-0.5.1.dist-info}/RECORD

@@ -1,8 +1,8 @@
 ecopipeline/__init__.py,sha256=vCRzwd781ciCSXMP1ycM_BXAqxj3KVaNKIjsLOPcbwc,171
 ecopipeline/extract/__init__.py,sha256=3u_CUMdCguVewU3kN8x6xhVNyo1-p-gwTrhjOh7Psqg,645
 ecopipeline/extract/extract.py,sha256=ryorqnu1RgyNK7joZRcbMmTajlTlB5hwaYzzpo8Z8Ho,43369
-ecopipeline/load/__init__.py,sha256=7ipv7GJfZ5382lcrdNm4MyM-WiCEVuRWTqxyzDSZhqg,197
-ecopipeline/load/load.py,sha256=RbGZSsigkChZpX1SZzYvZuS6-DS0k-d2IYUsJvZDvmk,17485
+ecopipeline/load/__init__.py,sha256=oDAVF8AhK_qugqegjW7jK16p-nb9QzKhiNQOkEBniKM,235
+ecopipeline/load/load.py,sha256=H9OKjE-EoqhZJ-5Xixqn5vvhvgUNFDdsVsX4fqht0hE,19975
 ecopipeline/transform/__init__.py,sha256=DcIJfkRs4OmZzDeEfW_OiOIXNqN6CUl1_lW0SS7-eN8,2280
 ecopipeline/transform/bayview.py,sha256=TP24dnTsUD95X-f6732egPZKjepFLJgDm9ImGr-fppY,17899
 ecopipeline/transform/lbnl.py,sha256=EQ54G4rJXaZ7pwVusKcdK2KBehSdCsNo2ybphtMGs7o,33400
@@ -10,8 +10,8 @@ ecopipeline/transform/transform.py,sha256=uyBIXKCXUCT6zVnZyQohripGAzmY1yV9T1GxsX
 ecopipeline/utils/ConfigManager.py,sha256=t4sfTjGO0g5P50XBQqGVFWaXfAlW1GMDh1DLoBuFGks,9826
 ecopipeline/utils/__init__.py,sha256=ccWUR0m7gD9DfcgsxBCLOfi4lho6RdYuB2Ugy_g6ZdQ,28
 ecopipeline/utils/unit_convert.py,sha256=VFh1we2Y8KV3u21BeWb-U3TlZJXo83q5vdxxkpgcuME,3064
-ecopipeline-0.5.0.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ecopipeline-0.5.0.dist-info/METADATA,sha256=gb550ta4s3kOYQCt52JECXo_HaSEeXKkWGeIF3mTVr8,2307
-ecopipeline-0.5.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ecopipeline-0.5.0.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
-ecopipeline-0.5.0.dist-info/RECORD,,
+ecopipeline-0.5.1.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ecopipeline-0.5.1.dist-info/METADATA,sha256=BMxe3qd_Ym_Ym5G3QN1zKwPJaZme5g4aEl6LUnaNPjc,2307
+ecopipeline-0.5.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ecopipeline-0.5.1.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
+ecopipeline-0.5.1.dist-info/RECORD,,

{ecopipeline-0.5.0.dist-info → ecopipeline-0.5.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any