ecopipeline 0.6.10__py3-none-any.whl → 0.7.1__py3-none-any.whl

ecopipeline/load/__init__.py

@@ -1,2 +1,3 @@
-from .load import check_table_exists, create_new_table, load_overwrite_database, load_event_table, report_data_loss
-__all__ = ["check_table_exists", "create_new_table", "load_overwrite_database", "load_event_table", "report_data_loss"]
+from .load import check_table_exists, create_new_table, load_overwrite_database, load_event_table, report_data_loss, load_data_statistics
+__all__ = ["check_table_exists", "create_new_table", "load_overwrite_database", "load_event_table", "report_data_loss",
+           "load_data_statistics"]

ecopipeline/load/load.py

@@ -315,7 +315,7 @@ def load_event_table(config : ConfigManager, event_df: pd.DataFrame, site_name :
         site_name = config.get_site_name()
     column_names = f"start_time_pt,site_name"
     column_types = ["datetime","varchar(25)","datetime",
-                    "ENUM('HW_OUTAGE', 'HW_LOSS','PIPELINE_STATUS', 'MISC_EVENT', 'PIPELINE_UPLOAD', 'DATA_LOSS', 'DATA_LOSS_COP', 'SITE_VISIT', 'COMMISIONING', 'SYSTEM_MAINTENENCE', 'POWER_OUTAGE', 'EQUIPMENT_MALFUNCTION','PARTIAL_OCCUPANCY','INSTALLATION_ERROR','SETPOINT_ADJUSTMENT')",
+                    "ENUM('MISC_EVENT','DATA_LOSS','DATA_LOSS_COP','SITE_VISIT','SYSTEM_MAINTENANCE','EQUIPMENT_MALFUNCTION','PARTIAL_OCCUPANCY','INSTALLATION_ERROR','ALARM','MV_COMMISSIONED','PLANT_COMMISSIONED','INSTALLATION_ERROR_COP','SOO_PERIOD','SOO_PERIOD_COP','SYSTEM_TESTING')",
                     "varchar(200)"]
     column_list = ['end_time_pt','event_type', 'event_detail']
     if not set(column_list).issubset(event_df.columns):
@@ -460,6 +460,27 @@ def report_data_loss(config : ConfigManager, site_name : str = None):
     cursor.close()
     return True
 
+def load_data_statistics(config : ConfigManager, daily_stats_df : pd.DataFrame, config_daily_indicator : str = "day"):
+    """
+    Logs data statistics for the site in a table with name "{daily table name}_stats"
+
+    Parameters
+    ----------  
+    config : ecopipeline.ConfigManager
+        The ConfigManager object that holds configuration data for the pipeline.
+    daily_stats_df : pd.DataFrame
+        dataframe created by the create_data_statistics_df() function in ecopipeline.transform
+    config_daily_indicator : str
+        the indicator of the daily_table name in the config.ini file of the data pipeline
+
+    Returns
+    ------- 
+    bool: 
+        A boolean value indicating if the data was successfully written to the database. 
+    """
+    table_name = f"{config.get_table_name(config_daily_indicator)}_stats"
+    return load_overwrite_database(config, daily_stats_df, config.get_db_table_info([]), config_daily_indicator, table_name=table_name)
+
 def _generate_mysql_update_event_table(row, id):
     statement = f"UPDATE site_events SET "
     statment_elems = []

ecopipeline/transform/__init__.py

@@ -1,9 +1,16 @@
-from .transform import rename_sensors, avg_duplicate_times, remove_outliers, ffill_missing, nullify_erroneous, sensor_adjustment, round_time, aggregate_df, join_to_hourly, concat_last_row, join_to_daily, cop_method_1, cop_method_2, create_summary_tables, remove_partial_days,convert_c_to_f,convert_l_to_g, convert_on_off_col_to_bool, flag_dhw_outage,generate_event_log_df,convert_time_zone, shift_accumulative_columns,heat_output_calc, add_relative_humidity, apply_equipment_cop_derate, delete_erroneous_from_time_pt
-from .lbnl import nclarity_filter_new, site_specific, condensate_calculations, gas_valve_diff, gather_outdoor_conditions, aqsuite_prep_time, nclarity_csv_to_df, _add_date, add_local_time, aqsuite_filter_new, get_refrig_charge, elev_correction, change_ID_to_HVAC, get_hvac_state, get_cop_values, get_cfm_values, replace_humidity, create_fan_curves, lbnl_temperature_conversions, lbnl_pressure_conversions, lbnl_sat_calculations, get_site_cfm_info, get_site_info, merge_indexlike_rows
+from .transform import rename_sensors, avg_duplicate_times, remove_outliers, ffill_missing, nullify_erroneous, sensor_adjustment, round_time, \
+    aggregate_df, join_to_hourly, concat_last_row, join_to_daily, cop_method_1, cop_method_2, create_summary_tables, remove_partial_days, \
+    convert_c_to_f,convert_l_to_g, convert_on_off_col_to_bool, flag_dhw_outage,generate_event_log_df,convert_time_zone, shift_accumulative_columns, \
+    heat_output_calc, add_relative_humidity, apply_equipment_cop_derate, create_data_statistics_df, delete_erroneous_from_time_pt
+from .lbnl import nclarity_filter_new, site_specific, condensate_calculations, gas_valve_diff, gather_outdoor_conditions, aqsuite_prep_time, \
+    nclarity_csv_to_df, _add_date, add_local_time, aqsuite_filter_new, get_refrig_charge, elev_correction, change_ID_to_HVAC, get_hvac_state, \
+    get_cop_values, get_cfm_values, replace_humidity, create_fan_curves, lbnl_temperature_conversions, lbnl_pressure_conversions, \
+    lbnl_sat_calculations, get_site_cfm_info, get_site_info, merge_indexlike_rows
 from .bayview import calculate_cop_values, aggregate_values, get_energy_by_min, verify_power_energy, get_temp_zones120, get_storage_gals120
 __all__ = ["rename_sensors", "avg_duplicate_times", "remove_outliers", "ffill_missing", "nullify_erroneous", "sensor_adjustment", "round_time", "aggregate_df", "join_to_hourly", "concat_last_row", "join_to_daily", 
            "cop_method_1", "cop_method_2", "create_summary_tables", "remove_partial_days", "nclarity_filter_new", "site_specific", "condensate_calculations", "gas_valve_diff", "gather_outdoor_conditions", "aqsuite_prep_time", 
            "nclarity_csv_to_df", "_add_date", "add_local_time", "aqsuite_filter_new", "get_refrig_charge", "elev_correction", "change_ID_to_HVAC", "get_hvac_state", "get_cop_values", "get_cfm_values", "replace_humidity", 
            "create_fan_curves", "lbnl_temperature_conversions", "lbnl_pressure_conversions", "lbnl_sat_calculations", "get_site_cfm_info", "get_site_info", "merge_indexlike_rows", "calculate_cop_values", "aggregate_values", 
            "get_energy_by_min", "verify_power_energy", "get_temp_zones120", "get_storage_gals120","convert_c_to_f","convert_l_to_g", "convert_on_off_col_to_bool", "flag_dhw_outage","generate_event_log_df","convert_time_zone",
-           "shift_accumulative_columns","heat_output_calc", "add_relative_humidity","apply_equipment_cop_derate","delete_erroneous_from_time_pt"]
+           "shift_accumulative_columns","heat_output_calc", "add_relative_humidity","apply_equipment_cop_derate","create_data_statistics_df",
+           "delete_erroneous_from_time_pt"]

ecopipeline/transform/transform.py

@@ -717,7 +717,7 @@ def flag_dhw_outage(df: pd.DataFrame, daily_df : pd.DataFrame, dhw_outlet_column
     Returns
     -------
     event_df : pd.DataFrame
-        Dataframe with 'HW_OUTAGE' events on the days in which there was a DHW Outage.
+        Dataframe with 'ALARM' events on the days in which there was a DHW Outage.
     """
     # TODO edge case for outage that spans over a day
     events = {
@@ -739,7 +739,7 @@ def flag_dhw_outage(df: pd.DataFrame, daily_df : pd.DataFrame, dhw_outlet_column
             adjusted_time = first_true_index - pd.Timedelta(minutes=consecutive_minutes-1)
             events['start_time_pt'].append(day)
             events['end_time_pt'].append(next_day - pd.Timedelta(minutes=1))
-            events['event_type'].append("HW_OUTAGE")
+            events['event_type'].append("ALARM")
             events['event_detail'].append(f"Hot Water Outage Occured (first one starting at {adjusted_time.strftime('%H:%M')})")
     event_df = pd.DataFrame(events)
     event_df.set_index('start_time_pt', inplace=True)
@@ -1027,7 +1027,7 @@ def join_to_daily(daily_data: pd.DataFrame, cop_data: pd.DataFrame) -> pd.DataFr
     out_df = daily_data.join(cop_data)
     return out_df
 
-def apply_equipment_cop_derate(df: pd.DataFrame, equip_cop_col: str, r_val : int = 16):
+def apply_equipment_cop_derate(df: pd.DataFrame, equip_cop_col: str, r_val : int = 16) -> pd.DataFrame:
     """
     Function derates equipment COP based on R value
     R12 - R16 : 12 %
@@ -1070,3 +1070,70 @@ def apply_equipment_cop_derate(df: pd.DataFrame, equip_cop_col: str, r_val : int
     
     df[equip_cop_col] =  df[equip_cop_col] * derate
     return df
+
+def create_data_statistics_df(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Function must be called on the raw minute data df after the rename_varriables() and before the ffill_missing() function has been called.
+    The function returns a dataframe indexed by day. Each column will expanded to 3 columns, appended with '_missing_mins', '_avg_gap', and
+    '_max_gap' respectively. the columns will carry the following statisctics:
+    _missing_mins -> the number of minutes in the day that have no reported data value for the column
+    _avg_gap -> the average gap (in minutes) between collected data values that day
+    _max_gap -> the maximum gap (in minutes) between collected data values that day
+
+    Parameters
+    ---------- 
+    df : pd.DataFrame
+        minute data df after the rename_varriables() and before the ffill_missing() function has been called
+
+    Returns
+    -------
+    daily_data_stats : pd.DataFrame
+        new dataframe with the columns descriped in the function's description
+    """
+    min_time = df.index.min()
+    start_day = min_time.floor('D')
+
+    # If min_time is not exactly at the start of the day, move to the next day
+    if min_time != start_day:
+        start_day = start_day + pd.tseries.offsets.Day(1)
+
+    # Build a complete minutely timestamp index over the full date range
+    full_index = pd.date_range(start=start_day,
+                               end=df.index.max().floor('D') - pd.Timedelta(minutes=1),
+                               freq='T')
+    
+    # Reindex to include any completely missing minutes
+    df_full = df.reindex(full_index)
+
+    # Resample daily to count missing values per column
+    total_missing = df_full.isna().resample('D').sum().astype(int)
+
+    # Function to calculate max consecutive missing values
+    def max_consecutive_nans(x):
+        is_na = x.isna()
+        groups = (is_na != is_na.shift()).cumsum()
+        return is_na.groupby(groups).sum().max() or 0
+
+    # Function to calculate average consecutive missing values
+    def avg_consecutive_nans(x):
+        is_na = x.isna()
+        groups = (is_na != is_na.shift()).cumsum()
+        gap_lengths = is_na.groupby(groups).sum()
+        gap_lengths = gap_lengths[gap_lengths > 0]
+        if len(gap_lengths) == 0:
+            return 0
+        return gap_lengths.mean()
+
+    # Apply daily, per column
+    max_consec_missing = df_full.resample('D').apply(lambda day: day.apply(max_consecutive_nans))
+    avg_consec_missing = df_full.resample('D').apply(lambda day: day.apply(avg_consecutive_nans))
+
+    # Rename columns to include a suffix
+    total_missing = total_missing.add_suffix('_missing_mins')
+    max_consec_missing = max_consec_missing.add_suffix('_max_gap')
+    avg_consec_missing = avg_consec_missing.add_suffix('_avg_gap')
+
+    # Concatenate along columns (axis=1)
+    combined_df = pd.concat([total_missing, max_consec_missing, avg_consec_missing], axis=1)
+
+    return combined_df

ecopipeline/utils/ConfigManager.py

@@ -134,15 +134,22 @@ class ConfigManager:
             tables. 
         """
 
-        configure = configparser.ConfigParser()
-        configure.read(self.config_directory)
-
-        db_table_info = {header: {"table_name": configure.get(header, 'table_name')} for header in table_headers}
+        db_table_info = {}
+        if len(table_headers) > 0:
+            configure = configparser.ConfigParser()
+            configure.read(self.config_directory)
+            db_table_info = {header: {"table_name": configure.get(header, 'table_name')} for header in table_headers}
         db_table_info["database"] = self.db_connection_info["database"]
 
         print(f"Successfully fetched configuration information from file path {self.config_directory}.")
         return db_table_info
     
+    def get_table_name(self, header):
+        configure = configparser.ConfigParser()
+        configure.read(self.config_directory)
+
+        return configure.get(header, 'table_name')
+    
     def get_db_name(self):
         """
         returns name of database that data will be uploaded to

{ecopipeline-0.6.10.dist-info → ecopipeline-0.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecopipeline
-Version: 0.6.10
+Version: 0.7.1
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

ecopipeline-0.7.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+ecopipeline/__init__.py,sha256=vCRzwd781ciCSXMP1ycM_BXAqxj3KVaNKIjsLOPcbwc,171
+ecopipeline/extract/__init__.py,sha256=3u_CUMdCguVewU3kN8x6xhVNyo1-p-gwTrhjOh7Psqg,645
+ecopipeline/extract/extract.py,sha256=heWcWTeRVTRITh_1sHVnkaKOOi5PwUOEVIi4k5tw2Z8,43384
+ecopipeline/load/__init__.py,sha256=NLa_efQJZ8aP-J0Y5xx9DP7mtfRH9jY6Jz1ZMZN_BAA,292
+ecopipeline/load/load.py,sha256=eNps857QRf0DZw_s90ShYbPMAyfOwbQJJfdKqYl7g-Y,22469
+ecopipeline/transform/__init__.py,sha256=hYb4F64fXdXtjBSYCqv6gLFBwKZjjnl0z7s291pFE98,2505
+ecopipeline/transform/bayview.py,sha256=TP24dnTsUD95X-f6732egPZKjepFLJgDm9ImGr-fppY,17899
+ecopipeline/transform/lbnl.py,sha256=EQ54G4rJXaZ7pwVusKcdK2KBehSdCsNo2ybphtMGs7o,33400
+ecopipeline/transform/transform.py,sha256=l1jw6uQ9Bqcb8tCydpxO3WEE5t_B3CKxfhohrlwCiXA,48944
+ecopipeline/utils/ConfigManager.py,sha256=E7e2RC2FveigjREMfeaiFu9fNQ8_b0xlWDfLCywS92k,10088
+ecopipeline/utils/__init__.py,sha256=ccWUR0m7gD9DfcgsxBCLOfi4lho6RdYuB2Ugy_g6ZdQ,28
+ecopipeline/utils/unit_convert.py,sha256=VFh1we2Y8KV3u21BeWb-U3TlZJXo83q5vdxxkpgcuME,3064
+ecopipeline-0.7.1.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ecopipeline-0.7.1.dist-info/METADATA,sha256=Jr0oFXTGHK2XGj0Nx6fnmp3NIba-Znjhxf9TinY1Sx4,2329
+ecopipeline-0.7.1.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+ecopipeline-0.7.1.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
+ecopipeline-0.7.1.dist-info/RECORD,,

{ecopipeline-0.6.10.dist-info → ecopipeline-0.7.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ecopipeline-0.6.10.dist-info/RECORD

@@ -1,17 +0,0 @@
-ecopipeline/__init__.py,sha256=vCRzwd781ciCSXMP1ycM_BXAqxj3KVaNKIjsLOPcbwc,171
-ecopipeline/extract/__init__.py,sha256=3u_CUMdCguVewU3kN8x6xhVNyo1-p-gwTrhjOh7Psqg,645
-ecopipeline/extract/extract.py,sha256=heWcWTeRVTRITh_1sHVnkaKOOi5PwUOEVIi4k5tw2Z8,43384
-ecopipeline/load/__init__.py,sha256=oDAVF8AhK_qugqegjW7jK16p-nb9QzKhiNQOkEBniKM,235
-ecopipeline/load/load.py,sha256=X7JIakIxyjzZbLuUjJ991kcQpyK4cFEZ0Lk36eXBEfI,21506
-ecopipeline/transform/__init__.py,sha256=7HuovqGHqrw0bZmeSCPMq1SkSRxJY8QBOBhR7y1JdBw,2400
-ecopipeline/transform/bayview.py,sha256=TP24dnTsUD95X-f6732egPZKjepFLJgDm9ImGr-fppY,17899
-ecopipeline/transform/lbnl.py,sha256=EQ54G4rJXaZ7pwVusKcdK2KBehSdCsNo2ybphtMGs7o,33400
-ecopipeline/transform/transform.py,sha256=kcJl6gzOmPx5K5pzcfQq17a5zInW4XfyjtwLxOMNlr4,46004
-ecopipeline/utils/ConfigManager.py,sha256=t4sfTjGO0g5P50XBQqGVFWaXfAlW1GMDh1DLoBuFGks,9826
-ecopipeline/utils/__init__.py,sha256=ccWUR0m7gD9DfcgsxBCLOfi4lho6RdYuB2Ugy_g6ZdQ,28
-ecopipeline/utils/unit_convert.py,sha256=VFh1we2Y8KV3u21BeWb-U3TlZJXo83q5vdxxkpgcuME,3064
-ecopipeline-0.6.10.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ecopipeline-0.6.10.dist-info/METADATA,sha256=7Ru_udzflx21RI3z8s7ZWrXh8bB2-5xXxYNsihAZIlY,2330
-ecopipeline-0.6.10.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ecopipeline-0.6.10.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
-ecopipeline-0.6.10.dist-info/RECORD,,