ecopipeline 0.11.1__py3-none-any.whl → 0.11.4__py3-none-any.whl

ecopipeline/extract/extract.py

@@ -295,6 +295,9 @@ def csv_to_df(csv_filenames: List[str], mb_prefix : bool = False, round_time_ind
                     continue
                 
             temp_dfs.append(data)
+    if len(temp_dfs) <= 0:
+        print("no data for timefarme.")
+        return pd.DataFrame()
     df = pd.concat(temp_dfs, ignore_index=False)
 
     if create_time_pt_idx:

ecopipeline/transform/__init__.py

@@ -1,7 +1,8 @@
 from .transform import rename_sensors, avg_duplicate_times, remove_outliers, ffill_missing, nullify_erroneous, sensor_adjustment, round_time, \
     aggregate_df, join_to_hourly, concat_last_row, join_to_daily, cop_method_1, cop_method_2, create_summary_tables, remove_partial_days, \
     convert_c_to_f,convert_l_to_g, convert_on_off_col_to_bool, flag_dhw_outage,generate_event_log_df,convert_time_zone, shift_accumulative_columns, \
-    heat_output_calc, add_relative_humidity, apply_equipment_cop_derate, create_data_statistics_df, delete_erroneous_from_time_pt,column_name_change
+    heat_output_calc, add_relative_humidity, apply_equipment_cop_derate, create_data_statistics_df, delete_erroneous_from_time_pt,column_name_change, \
+    process_ls_signal
 from .lbnl import nclarity_filter_new, site_specific, condensate_calculations, gas_valve_diff, gather_outdoor_conditions, aqsuite_prep_time, \
     nclarity_csv_to_df, _add_date, add_local_time, aqsuite_filter_new, get_refrig_charge, elev_correction, change_ID_to_HVAC, get_hvac_state, \
     get_cop_values, get_cfm_values, replace_humidity, create_fan_curves, lbnl_temperature_conversions, lbnl_pressure_conversions, \
@@ -13,4 +14,4 @@ __all__ = ["rename_sensors", "avg_duplicate_times", "remove_outliers", "ffill_mi
            "create_fan_curves", "lbnl_temperature_conversions", "lbnl_pressure_conversions", "lbnl_sat_calculations", "get_site_cfm_info", "get_site_info", "merge_indexlike_rows", "calculate_cop_values", "aggregate_values", 
            "get_energy_by_min", "verify_power_energy", "get_temp_zones120", "get_storage_gals120","convert_c_to_f","convert_l_to_g", "convert_on_off_col_to_bool", "flag_dhw_outage","generate_event_log_df","convert_time_zone",
            "shift_accumulative_columns","heat_output_calc", "add_relative_humidity","apply_equipment_cop_derate","create_data_statistics_df",
-           "delete_erroneous_from_time_pt","column_name_change"]
+           "delete_erroneous_from_time_pt","column_name_change","process_ls_signal"]

ecopipeline/transform/transform.py

@@ -245,7 +245,6 @@ def _ffill(col, ffill_df, previous_fill: pd.DataFrame = None):  # Helper functio
         elif (cp == 0):  # ffill only up to length
             col.fillna(method='ffill', inplace=True, limit=length)
 
-
 def ffill_missing(original_df: pd.DataFrame, config : ConfigManager, previous_fill: pd.DataFrame = None) -> pd.DataFrame:
     """
     Function will take a pandas dataframe and forward fill select variables with no entry. 
@@ -306,6 +305,86 @@ def ffill_missing(original_df: pd.DataFrame, config : ConfigManager, previous_fi
     df.apply(_ffill, args=(ffill_df,previous_fill))
     return df
 
+def process_ls_signal(df: pd.DataFrame, hourly_df: pd.DataFrame, daily_df: pd.DataFrame, load_dict: dict = {1: "normal", 2: "loadUp", 3 : "shed"}, ls_column: str = 'ls',
+                      drop_ls_from_df : bool = False):
+    """
+    Function takes aggregated dfs and adds loadshift signals to hourly df and loadshift days to daily_df
+
+    Parameters
+    ---------- 
+    df: pd.DataFrame
+        Timestamp indexed Pandas dataframe of minute by minute values
+    hourly_df: pd.DataFrame
+        Timestamp indexed Pandas dataframe of hourly average values
+    daily_df: pd.DataFrame
+        Timestamp indexed Pandas dataframe of daily average values
+    load_dict: dict
+        dictionary of what loadshift signal is indicated by a value of the ls_column column in df
+    ls_column: str
+        the name of the loadshift column in df
+    drop_ls_from_df: bool
+        Set to true to drop ls_column from df after processing
+    
+    Returns
+    ------- 
+    df: pd.DataFrame 
+        Timestamp indexed Pandas dataframe of minute by minute values with ls_column removed if drop_ls_from_df = True
+    hourly_df: pd.DataFrame
+        Timestamp indexed Pandas dataframe of hourly average values with added column 'system_state' which contains the 
+        loadshift command value from load_dict from the average (rounded to the nearest integer) key for all indexes in 
+        df within that load_dict key. If the integer is not a key in load_dict, the loadshift command value will be null 
+    daily_df: pd.DataFrame
+        Timestamp indexed Pandas dataframe of daily average values with added boolean column 'load_shift_day' which holds
+        the value True on days which contains hours in hourly_df in which there are loadshift commands other than normal
+        and Fals on days where the only command in normal unknown
+    """
+    # Make copies to avoid modifying original dataframes
+    df_copy = df.copy()
+
+    if ls_column in df_copy.columns:
+        # print("1",df_copy[np.isfinite(df_copy[ls_column])])
+        df_copy = df_copy[df_copy[ls_column].notna() & np.isfinite(df_copy[ls_column])]
+        # print("2",df_copy[np.isfinite(df_copy[ls_column])])
+
+    # Process hourly data - aggregate ls_column values by hour and map to system_state
+    if ls_column in df_copy.columns:
+        # Group by hour and calculate mean of ls_column, then round to nearest integer
+        hourly_ls = df_copy[ls_column].resample('H').mean().round()
+        
+        # Convert to int only for non-NaN values
+        hourly_ls = hourly_ls.apply(lambda x: int(x) if pd.notna(x) else x)
+        
+        # Map the rounded integer values to load_dict, using None for unmapped values
+        hourly_df['system_state'] = hourly_ls.map(load_dict)
+        
+        # For hours not present in the minute data, system_state will be NaN
+        hourly_df['system_state'] = hourly_df['system_state'].where(
+            hourly_df.index.isin(hourly_ls.index)
+        )
+    else:
+        # If ls_column doesn't exist, set all system_state to None
+        hourly_df['system_state'] = None
+    
+    # Process daily data - determine if any non-normal loadshift commands occurred
+    if 'system_state' in hourly_df.columns:
+        # Group by date and check if any non-"normal" and non-null system_state exists
+        daily_ls = hourly_df.groupby(hourly_df.index.date)['system_state'].apply(
+            lambda x: any((state != "normal") and (state is not None) for state in x.dropna())
+        )
+        
+        # Map the daily boolean results to the daily_df index
+        daily_df['load_shift_day'] = daily_df.index.date
+        daily_df['load_shift_day'] = daily_df['load_shift_day'].map(daily_ls).fillna(False)
+    else:
+        # If no system_state column, set all days to False
+        daily_df['load_shift_day'] = False
+    
+    # Drop ls_column from df if requested
+    if drop_ls_from_df and ls_column in df.columns:
+        df = df.drop(columns=[ls_column])
+    
+    return df, hourly_df, daily_df
+
 def delete_erroneous_from_time_pt(df: pd.DataFrame, time_point : pd.Timestamp, column_names : list, new_value = None) -> pd.DataFrame:
     """
     Function will take a pandas dataframe and delete specified erroneous values at a specified time point. 

{ecopipeline-0.11.1.dist-info → ecopipeline-0.11.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecopipeline
-Version: 0.11.1
+Version: 0.11.4
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

{ecopipeline-0.11.1.dist-info → ecopipeline-0.11.4.dist-info}/RECORD

@@ -2,19 +2,19 @@ ecopipeline/__init__.py,sha256=pjC00JWsjVAhS0jUKHD-wyi4UIpTsWbIg9JaxLS1mlc,275
 ecopipeline/event_tracking/__init__.py,sha256=SV2kkvJgptjeyLQlqHWcDRpQO6-JC433_dRZ3H9-ZNU,131
 ecopipeline/event_tracking/event_tracking.py,sha256=HffWAIAkNJ8INdG3_86RnDgw2bpHwv9hhkZ5oiiugZY,29653
 ecopipeline/extract/__init__.py,sha256=gQ3sak6NJ63Gpo-hZXrtZfeKOTHLRyAVXfTgxxRpqPo,675
-ecopipeline/extract/extract.py,sha256=y32feIIzgABwrwfduNQM1hICmkVOU4PYu6-M07zCLpU,51422
+ecopipeline/extract/extract.py,sha256=5C6KrfMAGQhTxHaWc1Lgm8yV5g994Fiinwk-IEVSHbM,51519
 ecopipeline/load/__init__.py,sha256=NLa_efQJZ8aP-J0Y5xx9DP7mtfRH9jY6Jz1ZMZN_BAA,292
 ecopipeline/load/load.py,sha256=PaSGWOZI0Xg44_SWN7htn2DPIAU_s8mOtCGibXq25tM,24614
-ecopipeline/transform/__init__.py,sha256=YveBLBsNhfI4qZP04doa0NrTbEKvjDAUDEKtEPdFPfU,2545
+ecopipeline/transform/__init__.py,sha256=9au1Rjw7SMtbIxpoq_5XWi6VWTxMU2CBjPksSh4LM1o,2590
 ecopipeline/transform/bayview.py,sha256=TP24dnTsUD95X-f6732egPZKjepFLJgDm9ImGr-fppY,17899
 ecopipeline/transform/lbnl.py,sha256=EQ54G4rJXaZ7pwVusKcdK2KBehSdCsNo2ybphtMGs7o,33400
-ecopipeline/transform/transform.py,sha256=wL4B00XBwLWVlf7goOLSHKgLFmIsXprQNepGLLO_wTk,50028
+ecopipeline/transform/transform.py,sha256=UF-sNw4zoxyXv0zsJZk6AqfQxXmAVQ_fsVZtjKiu1sk,54012
 ecopipeline/utils/ConfigManager.py,sha256=-g1wtExdvhYO5Y6Q3cRbywa__DxRMFruLrB4YanwaPY,12168
 ecopipeline/utils/NOAADataDownloader.py,sha256=iC2nl_O4PS1KFrchcPXRZxshwZwUMSqXy6BQBUwnOUU,20927
 ecopipeline/utils/__init__.py,sha256=7dT3tP6SMK4uBW6NBmQ8i6LaNTTuV6fpAZToBBlJ904,62
 ecopipeline/utils/unit_convert.py,sha256=VFh1we2Y8KV3u21BeWb-U3TlZJXo83q5vdxxkpgcuME,3064
-ecopipeline-0.11.1.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ecopipeline-0.11.1.dist-info/METADATA,sha256=_-HP7vfIrz6JBltdDkX4obF-AUJGrbxZfnFtrUBQ49k,2330
-ecopipeline-0.11.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ecopipeline-0.11.1.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
-ecopipeline-0.11.1.dist-info/RECORD,,
+ecopipeline-0.11.4.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ecopipeline-0.11.4.dist-info/METADATA,sha256=hpyr7M9aTCQo1MjbxuOeYwX9eiHnyR8XFVQnY5yWjzI,2330
+ecopipeline-0.11.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ecopipeline-0.11.4.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
+ecopipeline-0.11.4.dist-info/RECORD,,