ecopipeline 0.9.4__py3-none-any.whl → 0.10.1__py3-none-any.whl

ecopipeline/event_tracking/__init__.py

@@ -1,2 +1,2 @@
 from .event_tracking import *
-__all__ = ['flag_boundary_alarms','power_ratio_alarm']
+__all__ = ['central_alarm_df_creator','flag_boundary_alarms','power_ratio_alarm','flag_abnormal_COP']

ecopipeline/event_tracking/event_tracking.py

@@ -2,6 +2,107 @@ import pandas as pd
 import numpy as np
 import datetime as dt
 from ecopipeline import ConfigManager
+import re
+import mysql.connector.errors as mysqlerrors
+
+def central_alarm_df_creator(df: pd.DataFrame, daily_data : pd.DataFrame, config : ConfigManager, system: str = "", 
+                             default_cop_high_bound : float = 4.5, default_cop_low_bound : float = 0,
+                             default_boundary_fault_time : int = 15, site_name : str = None) -> pd.DataFrame:
+    day_list = daily_data.index.to_list()
+    print('Checking for alarms...')
+    alarm_df = _convert_silent_alarm_dict_to_df({})
+    boundary_alarm_df = flag_boundary_alarms(df, config, full_days=day_list, system=system, default_fault_time= default_boundary_fault_time)
+    pwr_alarm_df = power_ratio_alarm(daily_data, config, system=system)
+    abnormal_COP_df = flag_abnormal_COP(daily_data, config, system = system, default_high_bound=default_cop_high_bound, default_low_bound=default_cop_low_bound)
+
+    if len(boundary_alarm_df) > 0:
+        print("Boundary alarms detected. Adding them to event df...")
+        alarm_df = boundary_alarm_df
+    else:
+        print("No boundary alarms detected.")
+
+    if len(pwr_alarm_df) > 0:
+        print("Power alarms detected. Adding them to event df...")
+        alarm_df = pd.concat([alarm_df, pwr_alarm_df])
+    else:
+        print("No power alarms detected.")
+    
+    if _check_if_during_ongoing_cop_alarm(daily_data, config, site_name):
+        print("Ongoing DATA_LOSS_COP detected. No further DATA_LOSS_COP events will be uploaded")
+    elif len(abnormal_COP_df) > 0:
+        print("Abnormal COPs detected. Adding them to event df...")
+        alarm_df = pd.concat([alarm_df, abnormal_COP_df])
+    else:
+        print("No abnormal COPs.")
+
+    return alarm_df
+
+def flag_abnormal_COP(daily_data: pd.DataFrame, config : ConfigManager, system: str = "", default_high_bound : float = 4.5, default_low_bound : float = 0) -> pd.DataFrame:
+    variable_names_path = config.get_var_names_path()
+    try:
+        bounds_df = pd.read_csv(variable_names_path)
+    except FileNotFoundError:
+        print("File Not Found: ", variable_names_path)
+        return pd.DataFrame()
+
+    if (system != ""):
+        if not 'system' in bounds_df.columns:
+            raise Exception("system parameter is non null, however, system is not present in Variable_Names.csv")
+        bounds_df = bounds_df.loc[bounds_df['system'] == system]
+    if not "variable_name" in bounds_df.columns:
+        raise Exception(f"variable_name is not present in Variable_Names.csv")
+    if not 'pretty_name' in bounds_df.columns:
+        bounds_df['pretty_name'] = bounds_df['variable_name']
+    else:
+        bounds_df['pretty_name'] = bounds_df['pretty_name'].fillna(bounds_df['variable_name'])
+    if not 'high_alarm' in bounds_df.columns:
+        bounds_df['high_alarm'] = default_high_bound
+    else:
+        bounds_df['high_alarm'] = bounds_df['high_alarm'].fillna(default_high_bound)
+    if not 'low_alarm' in bounds_df.columns:
+        bounds_df['low_alarm'] = default_low_bound
+    else:
+        bounds_df['low_alarm'] = bounds_df['low_alarm'].fillna(default_low_bound)
+
+    bounds_df = bounds_df.loc[:, ["variable_name", "high_alarm", "low_alarm", "pretty_name"]]
+    bounds_df.dropna(axis=0, thresh=2, inplace=True)
+    bounds_df.set_index(['variable_name'], inplace=True)
+
+    cop_pattern = re.compile(r'^(COP\w*|SystemCOP\w*)$')
+    cop_columns = [col for col in daily_data.columns if re.match(cop_pattern, col)]
+
+    alarms_dict = {}
+    if not daily_data.empty and len(cop_columns) > 0:
+        for bound_var, bounds in bounds_df.iterrows():
+            if bound_var in cop_columns:
+                for day, day_values in daily_data.iterrows():
+                    if day_values[bound_var] > bounds['high_alarm'] or day_values[bound_var] < bounds['low_alarm']:
+                        alarm_str = f"{bounds['pretty_name']} = {round(day_values[bound_var],2)}"
+                        if day in alarms_dict:
+                            alarms_dict[day] = alarms_dict[day] + f", {alarm_str}"
+                        else:
+                            alarms_dict[day] = f"Unexpected COP Value(s) detected: {alarm_str}"
+    return _convert_event_type_dict_to_df(alarms_dict)
+
+def _check_if_during_ongoing_cop_alarm(daily_df : pd.DataFrame, config : ConfigManager, site_name : str = None) -> bool:
+    if site_name is None:
+        site_name = config.get_site_name()
+    connection, cursor = config.connect_db()
+    on_going_cop = False
+    try:
+        # find existing times in database for upsert statement
+        cursor.execute(
+            f"SELECT id FROM site_events WHERE start_time_pt <= '{daily_df.index.min()}' AND (end_time_pt IS NULL OR end_time_pt >= '{daily_df.index.max()}') AND site_name = '{site_name}' AND event_type = 'DATA_LOSS_COP'")
+        # Fetch the results into a DataFrame
+        existing_rows = pd.DataFrame(cursor.fetchall(), columns=['id'])
+        if not existing_rows.empty:
+            on_going_cop = True
+
+    except mysqlerrors.Error as e:
+        print(f"Retrieving data from site_events caused exception: {e}")
+    connection.close()
+    cursor.close()
+    return on_going_cop
 
 def flag_boundary_alarms(df: pd.DataFrame, config : ConfigManager, default_fault_time : int = 15, system: str = "", full_days : list = None) -> pd.DataFrame:
     """
@@ -102,6 +203,25 @@ def _convert_silent_alarm_dict_to_df(alarm_dict : dict) -> pd.DataFrame:
     event_df.set_index('start_time_pt', inplace=True)
     return event_df
 
+def _convert_event_type_dict_to_df(alarm_dict : dict, event_type = 'DATA_LOSS_COP') -> pd.DataFrame:
+    events = {
+        'start_time_pt' : [],
+        'end_time_pt' : [],
+        'event_type' : [],
+        'event_detail' : [],
+        'variable_name' : []
+    }
+    for key, value in alarm_dict.items():
+        events['start_time_pt'].append(key)
+        events['end_time_pt'].append(key)
+        events['event_type'].append(event_type)
+        events['event_detail'].append(value)
+        events['variable_name'].append(None)
+
+    event_df = pd.DataFrame(events)
+    event_df.set_index('start_time_pt', inplace=True)
+    return event_df
+
 def _check_and_add_alarm(df : pd.DataFrame, mask : pd.Series, alarms_dict, day, fault_time : int, var_name : str, pretty_name : str, alarm_type : str = 'Lower'):
     # KNOWN BUG : Avg value during fault time excludes the first (fault_time-1) minutes of each fault window
     next_day = day + pd.Timedelta(days=1)
@@ -130,7 +250,7 @@ def _check_and_add_alarm(df : pd.DataFrame, mask : pd.Series, alarms_dict, day,
         else:
             alarms_dict[day] = [[var_name, alarm_string]]
 
-def power_ratio_alarm(daily_df: pd.DataFrame, config : ConfigManager, system: str = "") -> pd.DataFrame:
+def power_ratio_alarm(daily_df: pd.DataFrame, config : ConfigManager, system: str = "", verbose : bool = False) -> pd.DataFrame:
     daily_df_copy = daily_df.copy()
     variable_names_path = config.get_var_names_path()
     try:
@@ -148,6 +268,8 @@ def power_ratio_alarm(daily_df: pd.DataFrame, config : ConfigManager, system: st
             raise Exception(f"{required_column} is not present in Variable_Names.csv")
     if not 'pretty_name' in ratios_df.columns:
         ratios_df['pretty_name'] = ratios_df['variable_name']
+    else:
+        ratios_df['pretty_name'] = ratios_df['pretty_name'].fillna(ratios_df['variable_name'])
     ratios_df = ratios_df.loc[:, ["variable_name", "alarm_codes", "pretty_name"]]
     ratios_df = ratios_df[ratios_df['alarm_codes'].str.contains('PR', na=False)]
     ratios_df.dropna(axis=0, thresh=2, inplace=True)
@@ -172,13 +294,16 @@ def power_ratio_alarm(daily_df: pd.DataFrame, config : ConfigManager, system: st
                     ratio_dict[pr_id][3].append(ratios['pretty_name'])
                 else:
                     ratio_dict[pr_id] = [[ratios_var],[float(low_high[0])],[float(low_high[1])],[ratios['pretty_name']]]
-
+    if verbose:
+        print("ratio_dict keys:", ratio_dict.keys())
     alarms = {}
     for key, value_list in ratio_dict.items():
         daily_df_copy[key] = daily_df_copy[value_list[0]].sum(axis=1)
         for i in range(len(value_list[0])):
             column_name = value_list[0][i]
             daily_df_copy[f'{column_name}_{key}'] = (daily_df_copy[column_name]/daily_df_copy[key]) * 100
+            if verbose:
+                print(f"Ratios for {column_name}_{key}",daily_df_copy[f'{column_name}_{key}'])
             _check_and_add_ratio_alarm(daily_df_copy, key, column_name, value_list[3][i], alarms, value_list[2][i], value_list[1][i])
     return _convert_silent_alarm_dict_to_df(alarms)      
 

ecopipeline/load/load.py

@@ -302,6 +302,11 @@ def load_event_table(config : ConfigManager, event_df: pd.DataFrame, site_name :
     bool: 
         A boolean value indicating if the data was successfully written to the database. 
     """
+    # define constants
+    proj_cop_filters = ['MV_COMMISSIONED','PLANT_COMMISSIONED','DATA_LOSS_COP','SYSTEM_MAINTENANCE','SYSTEM_TESTING']
+    optim_cop_filters = ['MV_COMMISSIONED','PLANT_COMMISSIONED','DATA_LOSS_COP','INSTALLATION_ERROR_COP',
+                            'PARTIAL_OCCUPANCY','SOO_PERIOD_COP','SYSTEM_TESTING','EQUIPMENT_MALFUNCTION',
+                            'SYSTEM_MAINTENANCE']
     # Drop empty columns
     event_df = event_df.dropna(axis=1, how='all')
 
@@ -329,7 +334,7 @@ def load_event_table(config : ConfigManager, event_df: pd.DataFrame, site_name :
         column_names += "," + column
 
     # create SQL statement
-    insert_str = "INSERT INTO " + table_name + " (" + column_names + ", variable_name, last_modified_date, last_modified_by) VALUES (%s,%s,%s,%s,%s,%s,'"+datetime.now().strftime('%Y-%m-%d %H:%M:%S')+"','automatic_upload')"
+    insert_str = "INSERT INTO " + table_name + " (" + column_names + ", variable_name, summary_filtered, optim_filtered, last_modified_date, last_modified_by) VALUES (%s, %s, %s,%s,%s,%s,%s,%s,'"+datetime.now().strftime('%Y-%m-%d %H:%M:%S')+"','automatic_upload')"
 
     if not 'variable_name' in event_df.columns:
         event_df['variable_name'] = None
@@ -338,11 +343,15 @@ def load_event_table(config : ConfigManager, event_df: pd.DataFrame, site_name :
     full_column_names.append('last_modified_date')
     full_column_names.append('last_modified_by')
     full_column_names.append('variable_name')
+    full_column_names.append('summary_filtered')
+    full_column_names.append('optim_filtered')
+
     full_column_types = column_types[1:]
     full_column_types.append('datetime')
     full_column_types.append('varchar(60)')
     full_column_types.append('varchar(70)')
-
+    full_column_types.append('tinyint(1)')
+    full_column_types.append('tinyint(1)')
 
     existing_rows = pd.DataFrame({
         'start_time_pt' : [],
@@ -376,7 +385,7 @@ def load_event_table(config : ConfigManager, event_df: pd.DataFrame, site_name :
     ignoredRows = 0
     try:
         for index, row in event_df.iterrows():
-            time_data = [index,site_name,row['end_time_pt'],row['event_type'],row['event_detail'],row['variable_name']]
+            time_data = [index,site_name,row['end_time_pt'],row['event_type'],row['event_detail'],row['variable_name'], row['event_type'] in proj_cop_filters, row['event_type'] in optim_cop_filters]
             #remove nans and infinites
             time_data = [None if (x is None or pd.isna(x)) else x for x in time_data]
             time_data = [None if (x == float('inf') or x == float('-inf')) else x for x in time_data]
@@ -436,8 +445,8 @@ def report_data_loss(config : ConfigManager, site_name : str = None):
     print(f"logging DATA_LOSS_COP into {table_name}")
 
     # create SQL statement
-    insert_str = "INSERT INTO " + table_name + " (start_time_pt, site_name, event_detail, event_type, last_modified_date, last_modified_by) VALUES "
-    insert_str += f"('{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}','{site_name}','{error_string}','DATA_LOSS_COP','{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}','automatic_upload')"
+    insert_str = "INSERT INTO " + table_name + " (start_time_pt, site_name, event_detail, event_type, summary_filtered, optim_filtered, last_modified_date, last_modified_by) VALUES "
+    insert_str += f"('{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}','{site_name}','{error_string}','DATA_LOSS_COP', true, true, '{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}','automatic_upload')"
 
     existing_rows = pd.DataFrame({
         'id' : []

{ecopipeline-0.9.4.dist-info → ecopipeline-0.10.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecopipeline
-Version: 0.9.4
+Version: 0.10.1
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

{ecopipeline-0.9.4.dist-info → ecopipeline-0.10.1.dist-info}/RECORD

@@ -1,10 +1,10 @@
 ecopipeline/__init__.py,sha256=d48mO5La6OrQDkRe_qqoY6lUx7x-e8krOH388jmWjwU,218
-ecopipeline/event_tracking/__init__.py,sha256=hVV2IHyt3pLSIOESpINr8-sQBmK98t7CiVb1IlvE5xQ,84
-ecopipeline/event_tracking/event_tracking.py,sha256=0w-dgsziC-JolqCPxiyryA9PiWKETQYZ3Gbt0faVgbI,17088
+ecopipeline/event_tracking/__init__.py,sha256=SV2kkvJgptjeyLQlqHWcDRpQO6-JC433_dRZ3H9-ZNU,131
+ecopipeline/event_tracking/event_tracking.py,sha256=ly6y6IM8_jlwa9dZ-FlAYEC-rbNKPc1NByYD12lyl2w,23168
 ecopipeline/extract/__init__.py,sha256=gQ3sak6NJ63Gpo-hZXrtZfeKOTHLRyAVXfTgxxRpqPo,675
 ecopipeline/extract/extract.py,sha256=y32feIIzgABwrwfduNQM1hICmkVOU4PYu6-M07zCLpU,51422
 ecopipeline/load/__init__.py,sha256=NLa_efQJZ8aP-J0Y5xx9DP7mtfRH9jY6Jz1ZMZN_BAA,292
-ecopipeline/load/load.py,sha256=Ptxr0MOjns_HeVSmZsLLApHJGB-z6XOB2m8LNiVaD7E,23860
+ecopipeline/load/load.py,sha256=PaSGWOZI0Xg44_SWN7htn2DPIAU_s8mOtCGibXq25tM,24614
 ecopipeline/transform/__init__.py,sha256=YveBLBsNhfI4qZP04doa0NrTbEKvjDAUDEKtEPdFPfU,2545
 ecopipeline/transform/bayview.py,sha256=TP24dnTsUD95X-f6732egPZKjepFLJgDm9ImGr-fppY,17899
 ecopipeline/transform/lbnl.py,sha256=EQ54G4rJXaZ7pwVusKcdK2KBehSdCsNo2ybphtMGs7o,33400
@@ -12,8 +12,8 @@ ecopipeline/transform/transform.py,sha256=wL4B00XBwLWVlf7goOLSHKgLFmIsXprQNepGLL
 ecopipeline/utils/ConfigManager.py,sha256=-g1wtExdvhYO5Y6Q3cRbywa__DxRMFruLrB4YanwaPY,12168
 ecopipeline/utils/__init__.py,sha256=ccWUR0m7gD9DfcgsxBCLOfi4lho6RdYuB2Ugy_g6ZdQ,28
 ecopipeline/utils/unit_convert.py,sha256=VFh1we2Y8KV3u21BeWb-U3TlZJXo83q5vdxxkpgcuME,3064
-ecopipeline-0.9.4.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ecopipeline-0.9.4.dist-info/METADATA,sha256=W1-fbJlxCCnlkqlSV5KgSvQlKuQTLYYxFvCgtQY8XFM,2329
-ecopipeline-0.9.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ecopipeline-0.9.4.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
-ecopipeline-0.9.4.dist-info/RECORD,,
+ecopipeline-0.10.1.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ecopipeline-0.10.1.dist-info/METADATA,sha256=LqVlpLeTsKuQTy_LhAcziKYvS_NgAiquydrzALKypDc,2330
+ecopipeline-0.10.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ecopipeline-0.10.1.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
+ecopipeline-0.10.1.dist-info/RECORD,,