ecopipeline 0.8.11__py3-none-any.whl → 0.9.0__py3-none-any.whl

ecopipeline/event_tracking/__init__.py

@@ -1,2 +1,2 @@
 from .event_tracking import *
-__all__ = ['flag_boundary_alarms']
+__all__ = ['flag_boundary_alarms','power_ratio_alarm']

ecopipeline/event_tracking/event_tracking.py

@@ -77,6 +77,10 @@ def flag_boundary_alarms(df: pd.DataFrame, config : ConfigManager, default_fault
                     print(f"Could not process alarm for {bound_var}. Fault time must be greater than or equal to 1 minute.")
                 _check_and_add_alarm(df, lower_mask, alarms, day, bounds["fault_time"], bound_var, bounds['pretty_name'], 'Lower')
                 _check_and_add_alarm(df, upper_mask, alarms, day, bounds["fault_time"], bound_var, bounds['pretty_name'], 'Upper')
+
+    return _convert_silent_alarm_dict_to_df(alarms)
+
+def _convert_silent_alarm_dict_to_df(alarm_dict : dict) -> pd.DataFrame:
     events = {
         'start_time_pt' : [],
         'end_time_pt' : [],
@@ -84,7 +88,7 @@ def flag_boundary_alarms(df: pd.DataFrame, config : ConfigManager, default_fault
         'event_detail' : [],
         'variable_name' : []
     }
-    for key, value_list in alarms.items():
+    for key, value_list in alarm_dict.items():
         for value in value_list:
             events['start_time_pt'].append(key)
             events['end_time_pt'].append(key)
@@ -124,6 +128,68 @@ def _check_and_add_alarm(df : pd.DataFrame, mask : pd.Series, alarms_dict, day,
         else:
             alarms_dict[day] = [[var_name, alarm_string]]
 
+def power_ratio_alarm(daily_df: pd.DataFrame, config : ConfigManager, system: str = "") -> pd.DataFrame:
+    daily_df_copy = daily_df.copy()
+    variable_names_path = config.get_var_names_path()
+    try:
+        ratios_df = pd.read_csv(variable_names_path)
+    except FileNotFoundError:
+        print("File Not Found: ", variable_names_path)
+        return pd.DataFrame()
+    if (system != ""):
+        if not 'system' in ratios_df.columns:
+            raise Exception("system parameter is non null, however, system is not present in Variable_Names.csv")
+        ratios_df = ratios_df.loc[ratios_df['system'] == system]
+    required_columns = ["variable_name", "alarm_codes"]
+    for required_column in required_columns:
+        if not required_column in ratios_df.columns:
+            raise Exception(f"{required_column} is not present in Variable_Names.csv")
+    if not 'pretty_name' in ratios_df.columns:
+        ratios_df['pretty_name'] = ratios_df['variable_name']
+    ratios_df = ratios_df.loc[:, ["variable_name", "alarm_codes", "pretty_name"]]
+    ratios_df = ratios_df[ratios_df['alarm_codes'].str.contains('PR', na=False)]
+    ratios_df.dropna(axis=0, thresh=2, inplace=True)
+    ratios_df.set_index(['variable_name'], inplace=True)
+
+    ratio_dict = {}
+    for ratios_var, ratios in ratios_df.iterrows():
+        if not ratios_var in daily_df_copy.columns:
+                daily_df_copy[ratios_var] = 0
+        alarm_codes = str(ratios['alarm_codes']).split(";")
+        for alarm_code in alarm_codes:
+            if alarm_code[:2] == "PR":
+                split_out_alarm = alarm_code.split(":")
+                low_high = split_out_alarm[1].split("-")
+                pr_id = split_out_alarm[0].split("_")[1]
+                if len(low_high) != 2:
+                    raise Exception(f"Error processing alarm code {alarm_code}")
+                if pr_id in ratio_dict:
+                    ratio_dict[pr_id][0].append(ratios_var)
+                    ratio_dict[pr_id][1].append(float(low_high[0]))
+                    ratio_dict[pr_id][2].append(float(low_high[1]))
+                    ratio_dict[pr_id][3].append(ratios['pretty_name'])
+                else:
+                    ratio_dict[pr_id] = [[ratios_var],[float(low_high[0])],[float(low_high[1])],[ratios['pretty_name']]]
+
+    alarms = {}
+    for key, value_list in ratio_dict.items():
+        daily_df_copy[key] = daily_df_copy[value_list[0]].sum(axis=1)
+        for i in range(len(value_list[0])):
+            column_name = value_list[0][i]
+            daily_df_copy[f'{column_name}_{key}'] = (daily_df_copy[column_name]/daily_df_copy[key]) * 100
+            _check_and_add_ratio_alarm(daily_df_copy, key, column_name, value_list[3][i], alarms, value_list[2][i], value_list[1][i])
+    return _convert_silent_alarm_dict_to_df(alarms)      
+
+def _check_and_add_ratio_alarm(daily_df: pd.DataFrame, alarm_key : str, column_name : str, pretty_name : str, alarms_dict : dict, high_bound : float, low_bound : float):
+    alarm_daily_df = daily_df.loc[(daily_df[f"{column_name}_{alarm_key}"] < low_bound) | (daily_df[f"{column_name}_{alarm_key}"] > high_bound)]
+    if not alarm_daily_df.empty:
+        for day, values in alarm_daily_df.iterrows():
+            alarm_str = f"Power ratio alarm: {pretty_name} accounted for {round(values[f'{column_name}_{alarm_key}'], 2)}% of {alarm_key} energy use."
+            if day in alarms_dict:
+                alarms_dict[day].append([column_name, alarm_str])
+            else:
+                alarms_dict[day] = [[column_name, alarm_str]]
+
 # def flag_dhw_outage(df: pd.DataFrame, daily_df : pd.DataFrame, dhw_outlet_column : str, supply_temp : int = 110, consecutive_minutes : int = 15) -> pd.DataFrame:
 #     """
 #      Parameters

ecopipeline/extract/extract.py

@@ -767,7 +767,8 @@ def pull_egauge_data(config: ConfigManager, eGauge_ids: list, eGauge_usr : str,
 
     os.chdir(original_directory)
 
-def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: datetime = None, create_csv : bool = True, query_hours : int = 1):
+def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: datetime = None, create_csv : bool = True, query_hours : float = 1,
+                 sensor_keys : list = [], seperate_keys : bool = False):
     """
     Function connects to the things board manager api to pull data and returns a dataframe.
 
@@ -785,7 +786,7 @@ def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: dat
         is local time from the data's index. 
     create_csv : bool
         create csv files as you process such that API need not be relied upon for reprocessing
-    query_hours : int
+    query_hours : float
         number of hours to query at a time from ThingsBoard API
     
     Returns
@@ -794,80 +795,102 @@ def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: dat
         Pandas Dataframe containing data from the API pull with column headers the same as the variable names in the data from the pull.
         Will return with index in UTC so needs to be converted after to appropriate timezone
     """
-    if endTime is None:
-        endTime = datetime.now()
-    if startTime is None:
-        # 28 hours to ensure encapsulation of last day
-        startTime = endTime - timedelta(hours=28)
-
-    if endTime - timedelta(hours=query_hours) > startTime:
-        time_diff = endTime - startTime
-        midpointTime = startTime + time_diff / 2
-        # recursively construct the df
-        df_1 = tb_api_to_df(config, startTime, midpointTime, create_csv=False, query_hours=query_hours)
-        df_2 = tb_api_to_df(config, midpointTime, endTime, create_csv=False, query_hours=query_hours)
-        df = pd.concat([df_1, df_2])
-        df = df.sort_index()
-        df = df.groupby(df.index).mean()
-        if create_csv:
-                filename = f"{startTime.strftime('%Y%m%d%H%M%S')}.csv"
-                original_directory = os.getcwd()
-                os.chdir(config.data_directory)
-                df.to_csv(filename, index_label='time_pt')
-                os.chdir(original_directory)
-
-        return df
-    url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{config.api_device_id}/values/timeseries'
-    token = config.get_thingsboard_token()
-    keys = _get_tb_keys(config, token)
-    if len(keys) <= 0:
-        raise Exception(f"No sensors available at ThingsBoard site with id {config.api_device_id}")
-    key_string = ','.join(keys)
-    params = {
-        'keys': key_string,
-        'startTs': f'{int(startTime.timestamp())*1000}',
-        'endTs': f'{int(endTime.timestamp())*1000}',
-        'orderBy': 'ASC',
-        'useStrictDataTypes': 'false'
-    }
-
-    # Headers
-    headers = {
-        'accept': 'application/json',
-        'X-Authorization': f'Bearer {token}'
-    }
-
-    try:
-        response = requests.get(url, headers=headers, params=params)
-        if response.status_code == 200:
-            response_json = response.json()
-            data = {}
-            for key, records in response_json.items():
-                try:
-                    series = pd.Series(
-                        data={record['ts']: float(record['value']) for record in records}
-                    )
-                    data[key] = series
-                except:
-                    print_statement = f"Could not convert {key} values to floats."
-                    # print(print_statement)
-            df = pd.DataFrame(data)
-            df.index = pd.to_datetime(df.index, unit='ms')
+    df = pd.DataFrame()
+    if len(sensor_keys) <= 0:
+        token = config.get_thingsboard_token()
+        key_list = _get_tb_keys(config, token)
+        if len(key_list) <= 0:
+            raise Exception(f"No sensors available at ThingsBoard site with id {config.api_device_id}")
+        return tb_api_to_df(config, startTime, endTime, create_csv, query_hours, key_list, seperate_keys)
+    if seperate_keys:
+        df_list = []
+        for sensor_key in sensor_keys:
+            df_list.append(tb_api_to_df(config, startTime, endTime, False, query_hours, [sensor_key], False))
+        df = pd.concat(df_list)
+    else:    
+    # not seperate_keys:
+        if endTime is None:
+            endTime = datetime.now()
+        if startTime is None:
+            # 28 hours to ensure encapsulation of last day
+            startTime = endTime - timedelta(hours=28)
+
+        if endTime - timedelta(hours=query_hours) > startTime:
+            time_diff = endTime - startTime
+            midpointTime = startTime + time_diff / 2
+            df_1 = tb_api_to_df(config, startTime, midpointTime, query_hours=query_hours, sensor_keys=sensor_keys, create_csv=False)#True if startTime >= datetime(2025,7,13,9) and startTime <= datetime(2025,7,13,10) else csv_pass_down)
+            df_2 = tb_api_to_df(config, midpointTime, endTime, query_hours=query_hours, sensor_keys=sensor_keys,create_csv=False)#True if endTime >= datetime(2025,7,13,9) and endTime <= datetime(2025,7,13,10) else csv_pass_down)
+            df = pd.concat([df_1, df_2])
             df = df.sort_index()
-            # save to file
-            if create_csv:
-                filename = f"{startTime.strftime('%Y%m%d%H%M%S')}.csv"
-                original_directory = os.getcwd()
-                os.chdir(config.data_directory)
-                df.to_csv(filename, index_label='time_pt')
-                os.chdir(original_directory)
-            return df
-        print(f"Failed to make GET request. Status code: {response.status_code} {response.json()}")
-        return pd.DataFrame()
-    except Exception as e:
-        traceback.print_exc()
-        print(f"An error occurred: {e}")
-        return pd.DataFrame()
+            df = df.groupby(df.index).mean()
+        else:
+            url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{config.api_device_id}/values/timeseries'
+            token = config.get_thingsboard_token()
+            key_string = ','.join(sensor_keys)
+            params = {
+                'keys': key_string,
+                'startTs': f'{int(startTime.timestamp())*1000}',
+                'endTs': f'{int(endTime.timestamp())*1000}',
+                'orderBy': 'ASC',
+                'useStrictDataTypes': 'false',
+                'interval' : '0',
+                'agg' : 'NONE'
+            }
+
+            # Headers
+            headers = {
+                'accept': 'application/json',
+                'X-Authorization': f'Bearer {token}'
+            }
+
+            try:
+                response = requests.get(url, headers=headers, params=params)
+                if response.status_code == 200:
+                    response_json = response.json()
+                    # if create_csv:
+                    #     json_filename = f"{startTime.strftime('%Y%m%d%H%M%S')}.json"
+                    #     print(f"filename: {json_filename}, url: {url}, params: {params}")
+                    #     original_directory = os.getcwd()
+                    #     os.chdir(config.data_directory)
+                    #     with open(json_filename, 'w') as f:
+                    #         json.dump(response_json, f, indent=4)  # indent=4 makes it human-readable
+                    #     os.chdir(original_directory)
+                        
+                    data = {}
+                    for key, records in response_json.items():
+                        try:
+                            series = pd.Series(
+                                data={record['ts']: _get_float_value(record['value'])  for record in records}
+                            )
+                            data[key] = series
+                        except:
+                            print_statement = f"Could not convert {key} values to floats."
+                            print(print_statement)
+                    df = pd.DataFrame(data)
+                    df.index = pd.to_datetime(df.index, unit='ms')
+                    df = df.sort_index()
+                else:
+                    print(f"Failed to make GET request. Status code: {response.status_code} {response.json()}")
+                    df = pd.DataFrame()
+            except Exception as e:
+                traceback.print_exc()
+                print(f"An error occurred: {e}")
+                df = pd.DataFrame()
+    # save to file
+    if create_csv:
+        filename = f"{startTime.strftime('%Y%m%d%H%M%S')}.csv"
+        original_directory = os.getcwd()
+        os.chdir(config.data_directory)
+        df.to_csv(filename, index_label='time_pt')
+        os.chdir(original_directory)
+    return df
+    
+def _get_float_value(value):
+    try:
+        ret_val = float(value)
+        return ret_val
+    except (ValueError, TypeError):
+        return None
     
 def _get_tb_keys(config: ConfigManager, token : str) -> List[str]:
     url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{config.api_device_id}/keys/timeseries'

ecopipeline/transform/__init__.py

@@ -1,7 +1,7 @@
 from .transform import rename_sensors, avg_duplicate_times, remove_outliers, ffill_missing, nullify_erroneous, sensor_adjustment, round_time, \
     aggregate_df, join_to_hourly, concat_last_row, join_to_daily, cop_method_1, cop_method_2, create_summary_tables, remove_partial_days, \
     convert_c_to_f,convert_l_to_g, convert_on_off_col_to_bool, flag_dhw_outage,generate_event_log_df,convert_time_zone, shift_accumulative_columns, \
-    heat_output_calc, add_relative_humidity, apply_equipment_cop_derate, create_data_statistics_df, delete_erroneous_from_time_pt
+    heat_output_calc, add_relative_humidity, apply_equipment_cop_derate, create_data_statistics_df, delete_erroneous_from_time_pt,column_name_change
 from .lbnl import nclarity_filter_new, site_specific, condensate_calculations, gas_valve_diff, gather_outdoor_conditions, aqsuite_prep_time, \
     nclarity_csv_to_df, _add_date, add_local_time, aqsuite_filter_new, get_refrig_charge, elev_correction, change_ID_to_HVAC, get_hvac_state, \
     get_cop_values, get_cfm_values, replace_humidity, create_fan_curves, lbnl_temperature_conversions, lbnl_pressure_conversions, \
@@ -13,4 +13,4 @@ __all__ = ["rename_sensors", "avg_duplicate_times", "remove_outliers", "ffill_mi
            "create_fan_curves", "lbnl_temperature_conversions", "lbnl_pressure_conversions", "lbnl_sat_calculations", "get_site_cfm_info", "get_site_info", "merge_indexlike_rows", "calculate_cop_values", "aggregate_values", 
            "get_energy_by_min", "verify_power_energy", "get_temp_zones120", "get_storage_gals120","convert_c_to_f","convert_l_to_g", "convert_on_off_col_to_bool", "flag_dhw_outage","generate_event_log_df","convert_time_zone",
            "shift_accumulative_columns","heat_output_calc", "add_relative_humidity","apply_equipment_cop_derate","create_data_statistics_df",
-           "delete_erroneous_from_time_pt"]
+           "delete_erroneous_from_time_pt","column_name_change"]

ecopipeline/transform/transform.py

@@ -378,6 +378,31 @@ def nullify_erroneous(original_df: pd.DataFrame, config : ConfigManager) -> pd.D
 
     return df
 
+def column_name_change(df : pd.DataFrame, dt : pd.Timestamp, new_column : str, old_column : str, remove_old_column : bool = True) -> pd.DataFrame:
+    """
+    Overwrites values in `new_column` with values from `old_column` 
+    for all rows before `dt`, if `dt` is within the index range.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        Pandas dataframe with minute-to-minute data
+    dt: pd.Timestamp
+        timestamp of the varriable name change
+    new_column: str
+        column to be overwritten
+    old_column: str
+        column to copy from
+    remove_old_column : bool
+        remove old column when done
+    """
+    if df.index.min() < dt:
+        mask = df.index < dt
+        df.loc[mask, new_column] = df.loc[mask, old_column]
+    if remove_old_column:
+        df = df.drop(columns=[old_column])
+    return df
+
 def heat_output_calc(df: pd.DataFrame, flow_var : str, hot_temp : str, cold_temp : str, heat_out_col_name : str, return_as_kw : bool = True) -> pd.DataFrame:
     """
     Function will take a flow varriable and two temperature inputs to calculate heat output 
@@ -903,7 +928,8 @@ def shift_accumulative_columns(df : pd.DataFrame, column_names : list = []):
     if len(column_names) == 0:
         return df_diff
     for column_name in column_names:
-        df[column_name] = df_diff[column_name]
+        if column_name in df.columns:
+            df[column_name] = df_diff[column_name]
     return df
 
 def create_summary_tables(df: pd.DataFrame):

{ecopipeline-0.8.11.dist-info → ecopipeline-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecopipeline
-Version: 0.8.11
+Version: 0.9.0
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

ecopipeline-0.9.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+ecopipeline/__init__.py,sha256=d48mO5La6OrQDkRe_qqoY6lUx7x-e8krOH388jmWjwU,218
+ecopipeline/event_tracking/__init__.py,sha256=hVV2IHyt3pLSIOESpINr8-sQBmK98t7CiVb1IlvE5xQ,84
+ecopipeline/event_tracking/event_tracking.py,sha256=WAHyCZtr0UToB3Ll3F9MBXjewy7bIsvS5m7nZ0ljBkw,16901
+ecopipeline/extract/__init__.py,sha256=gQ3sak6NJ63Gpo-hZXrtZfeKOTHLRyAVXfTgxxRpqPo,675
+ecopipeline/extract/extract.py,sha256=i5e-mL4KiJfmWMFnKwvCW_BdPELGJiYKRk6FbvcEi1U,51106
+ecopipeline/load/__init__.py,sha256=NLa_efQJZ8aP-J0Y5xx9DP7mtfRH9jY6Jz1ZMZN_BAA,292
+ecopipeline/load/load.py,sha256=Ptxr0MOjns_HeVSmZsLLApHJGB-z6XOB2m8LNiVaD7E,23860
+ecopipeline/transform/__init__.py,sha256=YveBLBsNhfI4qZP04doa0NrTbEKvjDAUDEKtEPdFPfU,2545
+ecopipeline/transform/bayview.py,sha256=TP24dnTsUD95X-f6732egPZKjepFLJgDm9ImGr-fppY,17899
+ecopipeline/transform/lbnl.py,sha256=EQ54G4rJXaZ7pwVusKcdK2KBehSdCsNo2ybphtMGs7o,33400
+ecopipeline/transform/transform.py,sha256=1s4gtaiP8H6WLasOCntQZrx8Av_Y9pp1YvMnWZm_ya8,49975
+ecopipeline/utils/ConfigManager.py,sha256=-g1wtExdvhYO5Y6Q3cRbywa__DxRMFruLrB4YanwaPY,12168
+ecopipeline/utils/__init__.py,sha256=ccWUR0m7gD9DfcgsxBCLOfi4lho6RdYuB2Ugy_g6ZdQ,28
+ecopipeline/utils/unit_convert.py,sha256=VFh1we2Y8KV3u21BeWb-U3TlZJXo83q5vdxxkpgcuME,3064
+ecopipeline-0.9.0.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ecopipeline-0.9.0.dist-info/METADATA,sha256=t6AjFnEnFEhp1RV6TiN8N1ROF7H9R_97QlKlO5cl8Oo,2329
+ecopipeline-0.9.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ecopipeline-0.9.0.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
+ecopipeline-0.9.0.dist-info/RECORD,,

ecopipeline-0.8.11.dist-info/RECORD

@@ -1,19 +0,0 @@
-ecopipeline/__init__.py,sha256=d48mO5La6OrQDkRe_qqoY6lUx7x-e8krOH388jmWjwU,218
-ecopipeline/event_tracking/__init__.py,sha256=q49j46fXMUjNUPzL4FvXEppB93i3lUni-QUZpp61tt0,64
-ecopipeline/event_tracking/event_tracking.py,sha256=LOCLE7ju320O7CrwnWRIqHRa2uAqoq-KvXZ3zWQ2S74,13224
-ecopipeline/extract/__init__.py,sha256=gQ3sak6NJ63Gpo-hZXrtZfeKOTHLRyAVXfTgxxRpqPo,675
-ecopipeline/extract/extract.py,sha256=feqmSEUYAGn1gl_gLqTsY8qOYVyDuWVwfUdxRlEqX24,49495
-ecopipeline/load/__init__.py,sha256=NLa_efQJZ8aP-J0Y5xx9DP7mtfRH9jY6Jz1ZMZN_BAA,292
-ecopipeline/load/load.py,sha256=Ptxr0MOjns_HeVSmZsLLApHJGB-z6XOB2m8LNiVaD7E,23860
-ecopipeline/transform/__init__.py,sha256=hYb4F64fXdXtjBSYCqv6gLFBwKZjjnl0z7s291pFE98,2505
-ecopipeline/transform/bayview.py,sha256=TP24dnTsUD95X-f6732egPZKjepFLJgDm9ImGr-fppY,17899
-ecopipeline/transform/lbnl.py,sha256=EQ54G4rJXaZ7pwVusKcdK2KBehSdCsNo2ybphtMGs7o,33400
-ecopipeline/transform/transform.py,sha256=S8fpAb45XBcYzeGNkxELiHM8-1jlNQqADV7_m-2oeWI,49097
-ecopipeline/utils/ConfigManager.py,sha256=-g1wtExdvhYO5Y6Q3cRbywa__DxRMFruLrB4YanwaPY,12168
-ecopipeline/utils/__init__.py,sha256=ccWUR0m7gD9DfcgsxBCLOfi4lho6RdYuB2Ugy_g6ZdQ,28
-ecopipeline/utils/unit_convert.py,sha256=VFh1we2Y8KV3u21BeWb-U3TlZJXo83q5vdxxkpgcuME,3064
-ecopipeline-0.8.11.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ecopipeline-0.8.11.dist-info/METADATA,sha256=HFo9uw0zynyzql9HleX9KtUsZGZ7hkuyETw4mFhP8f0,2330
-ecopipeline-0.8.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ecopipeline-0.8.11.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
-ecopipeline-0.8.11.dist-info/RECORD,,