teradataml 20.0.0.0__py3-none-any.whl → 20.0.0.2__py3-none-any.whl

teradataml/automl/feature_engineering.py

@@ -24,7 +24,7 @@ from teradataml.dataframe.dataframe import DataFrame
 from teradataml.dataframe.copy_to import copy_to_sql
 from teradataml import Antiselect
 from teradataml import BincodeFit, BincodeTransform
-from teradataml import ColumnSummary, CategoricalSummary, GetFutileColumns, FillRowId
+from teradataml import CategoricalSummary, ColumnSummary, ConvertTo, GetFutileColumns, FillRowId
 from teradataml import Fit, Transform
 from teradataml import NonLinearCombineFit, NonLinearCombineTransform
 from teradataml import NumApply
@@ -36,6 +36,8 @@ from teradataml import TargetEncodingFit, TargetEncodingTransform
 from sqlalchemy import literal_column
 from teradatasqlalchemy import INTEGER
 from teradataml import display
+from teradataml.common.garbagecollector import GarbageCollector
+from teradataml.dataframe.sql_functions import case
 from teradataml.hyperparameter_tuner.utils import _ProgressBar
 from teradataml.utils.validators import _Validators
 
@@ -48,7 +50,8 @@ class _FeatureEngineering:
                  model_list,
                  verbose = 0,
                  task_type = "Regression",
-                 custom_data = None):
+                 custom_data = None,
+                 **kwargs):
         """
         DESCRIPTION:
             Function initializes the data, target column and columns datatypes
@@ -61,12 +64,12 @@ class _FeatureEngineering:
                 Types: teradataml Dataframe
             
             target_column:
-                Required Arugment.
+                Required Argument.
                 Specifies the name of the target column in "data"..
                 Types: str
             
             model_list:
-                Required Arugment.
+                Required Argument.
                 Specifies the list of models to be used for model training.
                 Types: list
             
@@ -81,7 +84,7 @@ class _FeatureEngineering:
                 Types: int
 
             task_type:
-                Required Arugment.
+                Required Argument.
                 Specifies the task type for AutoML, whether to apply regresion OR classification
                 on the provived dataset.
                 Default Value: "Regression"
@@ -89,9 +92,31 @@ class _FeatureEngineering:
                 Types: str
 
             custom_data:
-                Optional Arugment.
+                Optional Argument.
                 Specifies json object containing user customized input.
                 Types: json object
+
+            **kwargs:
+                Specifies the additional arguments for feature engineering. Below
+                are the additional arguments:
+                    volatile:
+                        Optional Argument.
+                        Specifies whether to put the interim results of the
+                        functions in a volatile table or not. When set to
+                        True, results are stored in a volatile table,
+                        otherwise not.
+                        Default Value: False
+                        Types: bool
+
+                    persist:
+                        Optional Argument.
+                        Specifies whether to persist the interim results of the
+                        functions in a table or not. When set to True,
+                        results are persisted in a table; otherwise,
+                        results are garbage collected at the end of the
+                        session.
+                        Default Value: False
+                        Types: bool
         """
         # Instance variables
         self.data = data
@@ -106,6 +131,8 @@ class _FeatureEngineering:
         self.data_transform_dict = {}
         self.one_hot_obj_count = 0
         self.is_classification_type = lambda: self.task_type.upper() == 'CLASSIFICATION'
+        self.volatile = kwargs.get('volatile', False)
+        self.persist = kwargs.get('persist', False)
     
     # Method for doing feature engineering on data -> adding id, removing futile col, imputation, encoding(one hot)
     def feature_engineering(self, 
@@ -120,7 +147,7 @@ class _FeatureEngineering:
          
         PARAMETERS:   
             auto:
-                Optional Arugment.
+                Optional Argument.
                 Specifies whether to run AutoML in custom mode or auto mode.
                 When set to False, runs in custom mode. Otherwise, by default runs in auto mode.
                 Default Value: True
@@ -131,7 +158,7 @@ class _FeatureEngineering:
              second element represents list of columns which are not participating in outlier tranformation.
         """
         # Assigning number of base jobs for progress bar.
-        base_jobs = 14 if auto else 18
+        base_jobs = 13 if auto else 17
         
         # Updating model list based on distinct value of target column for classification type
         if self.is_classification_type():
@@ -181,9 +208,12 @@ class _FeatureEngineering:
             self._remove_duplicate_rows()
             self.progress_bar.update()
             
+            self._anti_select_columns()
+            self.progress_bar.update()
+
             self._remove_futile_columns()
             self.progress_bar.update()
-            
+
             self._handle_date_columns()
             self.progress_bar.update()
             
@@ -204,10 +234,7 @@ class _FeatureEngineering:
             
             self._non_linear_transformation()
             self.progress_bar.update()
-            
-            self._anti_select_columns()
-            self.progress_bar.update()
-            
+               
         return self.data, self.excluded_cols, self.target_label, self.data_transform_dict
     
     def _extract_list(self,
@@ -255,7 +282,7 @@ class _FeatureEngineering:
                                   f"Remaining Columns in the data: {self.data.shape[1]}",
                               progress_bar=self.progress_bar)
         else:
-            self._display_msg(inline_msg="Analysis complete. No action taken.",
+            self._display_msg(inline_msg="Analysis completed. No action taken.",
                               progress_bar=self.progress_bar)   
 
         end_time = time.time()
@@ -322,18 +349,22 @@ class _FeatureEngineering:
         if len(categorical_columns) != 0:
             
             obj = CategoricalSummary(data=self.data, 
-                                     target_columns=categorical_columns)
+                                     target_columns=categorical_columns,
+                                     volatile=self.volatile,
+                                     persist=self.persist)
             
             gfc_out = GetFutileColumns(data=self.data, 
                                        object=obj, 
                                        category_summary_column="ColumnName",
-                                       threshold_value =0.7)
+                                       threshold_value =0.7,
+                                       volatile=self.volatile,
+                                       persist=self.persist)
             
             # Extracting Futile columns
             f_cols = [row[0] for row in gfc_out.result.itertuples()]
             
             if len(f_cols) == 0:
-                self._display_msg(inline_msg="All categorical columns seem to be significant.",
+                self._display_msg(inline_msg="Analysis indicates all categorical columns are significant. No action Needed.",
                                   progress_bar=self.progress_bar)
             else:
 
@@ -350,128 +381,80 @@ class _FeatureEngineering:
         self._display_msg(msg="Total time to handle less significant features: {:.2f} sec  ".format( end_time - start_time),
                           progress_bar=self.progress_bar,
                           show_data=True)
-
-    def _handle_date_component(self,
-                               date_component_columns,
-                               date_component):  
         
+    def _fetch_date_component(self):
         """
         DESCRIPTION:
-            Function to handle newly generated date components, i.e., day , month and year diff.
-            Based on their distinct values, binning is done with predefined prefix. 
-            Binned component is used further as categorical features.
-            
-        PARAMETERS:
-            date_component_columns:
-                Required Argument.
-                Specifies the list of newly generated differnt component of date features.
-                Types: list
-                
-            date_component:
-                Required Argument.
-                Specifies identifier for the differnt component of date features, i.e., D - Days , M - Months and Y - Year diffs.
-                Types: str 
-            
-        """
-        # Check for day
-        if date_component == "D":
-            prefix_value = "Day_"
-        # Check for month
-        elif date_component == "M":
-            prefix_value = "Month_"
-        # Check for year diff
-        elif date_component == "Y":
-            prefix_value = "Year_diff_"
-        
-        # Deciding bins based on distinct value of date component features.
-        for col in date_component_columns:
-            data_size = self.data.drop_duplicate(col).size
-            if data_size < 4:
-                num_bins = data_size
-            else:
-                num_bins = 4
-            # Performing bincode for converting date component to specific labels
-            fit_params = {
-                        "data": self.data,
-                        "target_columns": col,
-                        "method_type":"Equal-Width",
-                        "nbins": num_bins,
-                        "label_prefix" : prefix_value
-                        }
-            bin_code_fit = BincodeFit(**fit_params)
-            
-            fit_params_map = {"D": "day_component_fit_object",
-                              "M": "month_component_fit_object",
-                              "Y": "year_diff_component_fit_object"}
-            
-            # Storing fit object for each date component in data transform dictionary
-            self.data_transform_dict[fit_params_map[date_component]][col] = bin_code_fit.output
-                
-            accumulate_columns = self._extract_list(self.data.columns, [col])
-            transform_params = {
-                        "data": self.data,
-                        "object": bin_code_fit.output,
-                        "accumulate": accumulate_columns,
-                        "persist": True
-                        }
-            self.data = BincodeTransform(**transform_params).result
-          
-    def _fetch_date_component(self, 
-                              process, 
-                              regex_str, 
-                              columns, 
-                              date_component):
+            Function to fetch day of week, week of month, month of quarter, quarter of year
+            component from date column. Generate weekend and month half details from day of week and
+            week of month columns respectively. Convert quarter of year and month of quarter
+            component columns to VARCHAR.
         
+        RETURNS:
+            List of newly generated date component features.
         """
-        DESCRIPTION:
-            Function to fetch newly generated date component features.
-            Passing ahead for performing binning.
-            
-        PARAMETERS:
-            process:
-                Required Argument.
-                Specifies date component of date feature which is going to be fetched and handled.
-                Types: str
-                
-            regex_str:
-                Required Argument.
-                Specifies regular expression for identifying newly generated date component features.
-                Types: str 
-            
-            columns:
-                Required Argument.
-                Specifies list of newly generated date component features.
-                Types: list
-                
-            date_component:
-                Required Argument.
-                Specifies identifier for the differnt component of date features, i.e., D - Days , M - Months and Y - Year diffs.
-                Types: str
+        # List for storing newly generated date component features
+        new_date_components=[]
+        # Extracting weekend, month, quarter details information from date columns
+        date_component_param={}
+        for col in self.date_column_list: 
+            # Generating new column names for extracted date components
+            weekend_col = f'{col}_weekend'
+            month_half_col = f'{col}_month_half'
+            month_of_quarter_col=f'{col}_month_of_quarter'
+            quarter_of_year_col=f'{col}_quarter_of_year'
             
-        """
-        date_component_columns = [col for col in columns if re.search(regex_str+"$", col)]
-        if len(date_component_columns) != 0:
-            self._handle_date_component(date_component_columns,date_component)
-            self._display_msg(msg="Useful {} features:".format(process),
-                              col_lst=date_component_columns,
-                              progress_bar=self.progress_bar)
-            self._display_msg(msg="Updated dataset sample:",
-                              data=self.data,
-                              progress_bar=self.progress_bar)
-                
-        else:
-            self._display_msg("\nNo useful feature found for {} component:".format(process),
-                              progress_bar=self.progress_bar)
+            date_component_param =  {
+                **date_component_param,
+                weekend_col: case([(self.data[col].day_of_week().isin([1, 7]), 'yes')], else_='no'),
+                month_half_col: case([(self.data[col].week_of_month().isin([1, 2]), 'first_half')], else_='second_half'),
+                month_of_quarter_col: self.data[col].month_of_quarter(),
+                quarter_of_year_col: self.data[col].quarter_of_year()
+            }
+            # Storing newly generated date component month and quarter columns.
+            # Skipping day of week and week of month columns as they will be used 
+            # later for extracting weekend and month part details.
+            new_date_components.extend([weekend_col, month_half_col, month_of_quarter_col, quarter_of_year_col])
+        # Adding new date component columns to dataset
+        self.data=self.data.assign(**date_component_param)
+        # Dropping date columns as different component columns are extracted.     
+        self.data = self.data.drop(self.date_column_list, axis=1)
+        
+        # Converting remaining component columns to VARCHAR 
+        # So that it will be treated as categorical columns
+        remaining_component_columns = [col for col in self.data.columns if re.search('month_of_quarter|quarter_of_year'+"$", col)]
+        accumulate_columns = self._extract_list(self.data.columns, remaining_component_columns)
+        convertto_params = {
+                            "data" : self.data,
+                            "target_columns" : remaining_component_columns,
+                            "target_datatype" : ["VARCHAR(charlen=20,charset=UNICODE,casespecific=NO)"],
+                            "accumulate" : accumulate_columns,
+                            "persist" : True
+                            }
+        # Disabling display table name if persist is True by default
+        if not self.volatile and not self.persist:
+            convertto_params["display_table_name"] = False
             
-        return date_component_columns
+        # Setting persist to False if volatile is True
+        if self.volatile:
+            convertto_params["persist"] = False
+            convertto_params["volatile"] = True
+
+        # returning dataset after performing string manipulation                 
+        self.data = ConvertTo(**convertto_params).result
+
+        # IF volatile is False and persist is False
+        if not self.volatile and not self.persist:
+            # Adding transformed data containing table to garbage collector
+            GarbageCollector._add_to_garbagecollector(self.data._table_name)
+        return new_date_components
   
     def _handle_date_columns_helper(self):
         
         """
         DESCRIPTION:
-            Function for dropping irrelevent date features.
-            Extracting day, month and year component from revelent date features.
-            Passing extracted component for performing binning.
+            Function for dropping irrelevent date features. Perform Extraction of different
+            component from revelent date features and transform them.
         """
         
         # Dropping missing value for all date columns
@@ -484,7 +467,7 @@ class _FeatureEngineering:
         # Date columns list eligible for dropping from dataset
         drop_date_cols = []     
         
-        # Checking for single valued date columns
+        # Checking for unique valued date columns
         for col in self.date_column_list:
             if self.data.drop_duplicate(col).size == self.data.shape[0]:
                 drop_date_cols.append(col)
@@ -496,46 +479,18 @@ class _FeatureEngineering:
             self._display_msg(msg='Dropping date features with all unique value:',
                               col_lst = drop_date_cols,
                               progress_bar=self.progress_bar)
-             
-        # Updated date columns list
-        self.date_column_list = [item for item in self.date_column_list if item not in drop_date_cols]
-        
-        # List for storing newly generated date component features
-        new_columns=[]
+            # Updated date column list after dropping irrelevant date columns
+            self.date_column_list = [item for item in self.date_column_list if item not in drop_date_cols]
         
-        # Extracting day, month and year difference from date columns
         if len(self.date_column_list) != 0:
             
-            component_param={}
-            for col in self.date_column_list: 
-                
-                day_column=str(col)+"_day_comp"
-                month_column=str(col)+"_month_comp"
-                year_diff_column=str(col)+"_year_diff_comp"
-                new_columns.extend([day_column,month_column,year_diff_column])
-                day_query=("EXTRACT(DAY FROM {0})".format(col))
-                month_query=("EXTRACT(MONTH FROM {0})".format(col))
-                year_query=("EXTRACT(YEAR FROM CURRENT_DATE) - EXTRACT(YEAR FROM {0})".format(col))
-                component_param[day_column]=literal_column(day_query,INTEGER())
-                component_param[month_column]=literal_column(month_query,INTEGER())
-                component_param[year_diff_column]=literal_column(year_query,INTEGER())
-                
-            self.data=self.data.assign(**component_param)
-            # Storing newly generated date component list along with parameters in data transform dictionary
-            self.data_transform_dict['extract_date_comp_col'] = self.date_column_list
-            self.data_transform_dict['extract_date_comp_param'] = component_param
-         
-            # Dropping date columns as we have already extracted day, month and year in new columns        
-            self.data = self.data.drop(self.date_column_list, axis=1)
+            # List for storing newly generated date component features
+            new_columns=self._fetch_date_component()
             self._display_msg(msg='List of newly generated features from existing date features:',
                               col_lst=new_columns,
                               progress_bar=self.progress_bar)
-            self._display_msg(msg='List of newly generated features from existing date features:',
-                              data=self.data,
-                              progress_bar=self.progress_bar)
-            
+            # Dropping columns with all unique values or single value
             drop_cols=[]
-            
             for col in new_columns:
                 distinct_rows = self.data.drop_duplicate(col).size
                 if  distinct_rows == self.data.shape[0]:
@@ -555,21 +510,11 @@ class _FeatureEngineering:
                 self.data = self.data.drop(drop_cols, axis=1)
                 # Storing extract date component list for drop in data transform dictionary
                 self.data_transform_dict['drop_extract_date_columns'] = drop_cols        
-                
-            # Extracting all newly generated columns    
-            new_columns = [item for item in new_columns if item not in drop_cols]
+                # Extracting all newly generated columns    
+                new_columns = [item for item in new_columns if item not in drop_cols]
             
-            # Storing each date component transformation fit object in data transform dictionary
-            self.data_transform_dict = {**self.data_transform_dict, 
-                                        'day_component_fit_object': {}, 
-                                        'month_component_fit_object': {}, 
-                                        'year_diff_component_fit_object': {}}
-            # Grouping date components based on types i.e., day, month, and year_diff for performing binning
-            if len(new_columns) != 0:
-                self.day_columns = self._fetch_date_component("day", "_day_comp", new_columns, "D")
-                self.month_columns = self._fetch_date_component("month", "_month_comp", new_columns, "M")
-                self.year_diff_columns = self._fetch_date_component("year_diff", "_year_diff_comp", new_columns, "Y")
-                self._display_msg(inline_msg="No useful date component found",
+                self._display_msg(msg='Updated list of newly generated features from existing date features :',
+                                  col_lst=new_columns,
                                   progress_bar=self.progress_bar)
             
             self._display_msg(msg='Updated dataset sample after handling date features:',
@@ -595,7 +540,7 @@ class _FeatureEngineering:
                                if d_type in ["datetime.date","datetime.datetime"]]
                  
         if len(self.date_column_list) == 0:
-            self._display_msg(inline_msg="Dataset does not contain any feature related to dates.",
+            self._display_msg(inline_msg="Analysis Completed. Dataset does not contain any feature related to dates. No action needed.",
                               progress_bar=self.progress_bar)
         else:
             # Storing date column list in data transform dictionary
@@ -622,8 +567,9 @@ class _FeatureEngineering:
         self.data = self.data.dropna(subset=[self.target_column])
         
         obj = ColumnSummary(data=self.data, 
-                            target_columns=self.data.columns, 
-                            volatile=True)
+                            target_columns=self.data.columns,
+                            volatile=self.volatile,
+                            persist=self.persist)
         
         cols_miss_val={}
         # Iterating over each row in the column summary result
@@ -705,11 +651,15 @@ class _FeatureEngineering:
             self.data_transform_dict['imputation_columns'] = self.imputation_cols
 
         if len(delete_rows) != 0:
+            rows = self.data.shape[0]
             self.data = self.data.dropna(subset=delete_rows)
             msg_val_found=1
             self._display_msg(msg='Deleting rows of these columns for handling missing values:',
                               col_lst=delete_rows,
                               progress_bar=self.progress_bar)
+            self._display_msg(msg=f'Sample of dataset after removing {rows-self.data.shape[0]} rows:',
+                              data=self.data,
+                              progress_bar=self.progress_bar)
             
         if len(drop_cols) != 0:
             self.data = self.data.drop(drop_cols, axis=1)
@@ -719,9 +669,12 @@ class _FeatureEngineering:
             self._display_msg(msg='Dropping these columns for handling missing values:',
                               col_lst=drop_cols,
                               progress_bar=self.progress_bar)
+            self._display_msg(msg=f'Sample of dataset after removing {len(drop_cols)} columns:',
+                              data=self.data,
+                              progress_bar=self.progress_bar)
         
         if len(self.imputation_cols) == 0 and msg_val_found ==0:
-            self._display_msg(inline_msg="No Missing Values Detected.",
+            self._display_msg(inline_msg="Analysis Completed. No Missing Values Detected.",
                               progress_bar=self.progress_bar)
             
         end_time = time.time()
@@ -787,21 +740,23 @@ class _FeatureEngineering:
             
             fit_obj = SimpleImputeFit(data=self.data, 
                                       stats_columns=col_stat, 
-                                      stats=stat, 
-                                      volatile=True)
+                                      stats=stat,
+                                      volatile=self.volatile,
+                                      persist=self.persist)
             
             # Storing fit object for imputation in data transform dictionary
             self.data_transform_dict['imputation_fit_object'] = fit_obj.output
             sm = SimpleImputeTransform(data=self.data, 
-                                        object=fit_obj, 
-                                        volatile=True)
+                                       object=fit_obj,
+                                       volatile=self.volatile,
+                                       persist=self.persist)
             
             self.data = sm.result
-            self._display_msg(msg="Sample of Data after Imputation:",
+            self._display_msg(msg="Sample of dataset after Imputation:",
                               data=self.data,
                               progress_bar=self.progress_bar)
         else:
-            self._display_msg(inline_msg="No imputation is Required.",
+            self._display_msg(inline_msg="Analysis completed. No imputation required.",
                               progress_bar=self.progress_bar)
               
         end_time = time.time()
@@ -827,6 +782,8 @@ class _FeatureEngineering:
                 drop_col_ind = missing_handling_param.get("DroppingColumnIndicator", False)
                 drop_row_ind = missing_handling_param.get("DroppingRowIndicator", False)
                 impute_ind = missing_handling_param.get("ImputeMissingIndicator", False)
+                volatile = missing_handling_param.pop("volatile", False)
+                persist = missing_handling_param.pop("persist", False)
                 # Checking for user input if all methods indicator are false or not
                 if not any([drop_col_ind, drop_row_ind, impute_ind]):  
                     self._display_msg(inline_msg="No method information provided for performing customized missing value handling. \
@@ -883,7 +840,9 @@ class _FeatureEngineering:
                             "stats_columns" : stat_list,
                             "stats" : stat_method,
                             "literals_columns" : literal_list,
-                            "literals" : literal_value
+                            "literals" : literal_value,
+                            "volatile" : volatile,
+                            "persist" : persist
                         }
                         # Fitting on dataset
                         fit_obj = SimpleImputeFit(**fit_param)
@@ -896,8 +855,18 @@ class _FeatureEngineering:
                             "object" : fit_obj.output,
                             "persist" : True
                         }
+                        # Disabling display table name if persist is True by default
+                        if not volatile and not persist:
+                            transform_param["display_table_name"] = False
+
+                        if volatile:
+                            transform_param["volatile"] = True
+                            transform_param["persist"] = False
                         # Updating dataset with transform result
                         self.data = SimpleImputeTransform(**transform_param).result
+                        if not volatile and not persist:
+                            # Adding transformed data containing table to garbage collector
+                            GarbageCollector._add_to_garbagecollector(self.data._table_name)                 
                         self._display_msg(msg="Updated dataset sample after performing customized missing value imputation:",
                                           data=self.data,
                                           progress_bar=self.progress_bar)
@@ -938,6 +907,8 @@ class _FeatureEngineering:
                 equal_width_bin_columns  = []
                 var_width_bin_list = []
                 var_width_bin_columns  = []
+                volatile = extracted_col.pop("volatile", False)
+                persist = extracted_col.pop("persist", False)
 
                 # Checking for column present in dataset or not
                 _Validators._validate_dataframe_has_argument_columns(list(extracted_col.keys()), "BincodeParam", self.data, "df")
@@ -971,7 +942,9 @@ class _FeatureEngineering:
                         "data" : self.data,
                         "target_columns": equal_width_bin_columns,
                         "method_type" : "Equal-Width",
-                        "nbins" : bins  
+                        "nbins" : bins,
+                        "volatile" : volatile,
+                        "persist" : persist
                     }
                     eql_bin_code_fit = BincodeFit(**fit_params)
                     # Storing fit object and column list for Equal-Width binning in data transform dictionary
@@ -984,9 +957,19 @@ class _FeatureEngineering:
                         "data" : self.data,
                         "object" : eql_bin_code_fit.output,
                         "accumulate" : accumulate_columns,
-                        "persist" : True,   
+                        "persist" : True
                     }
+                    # Disabling display table name if persist is True by default
+                    if not volatile and not persist:
+                        eql_transform_params["display_table_name"] = False
+                        
+                    if volatile:
+                        eql_transform_params["volatile"] = True
+                        eql_transform_params["persist"] = False
                     self.data = BincodeTransform(**eql_transform_params).result
+                    if not volatile and not persist:
+                        # Adding transformed data containing table to garbage collector
+                        GarbageCollector._add_to_garbagecollector(self.data._table_name)
                     self._display_msg(msg="\nUpdated dataset sample after performing Equal-Width binning :-",
                                       data=self.data,
                                       progress_bar=self.progress_bar)
@@ -1011,7 +994,9 @@ class _FeatureEngineering:
                         "maxvalue_column" : "MaxValue",
                         "label_column" : "Label",
                         "method_type" : "Variable-Width",
-                        "label_prefix" : "label_prefix"
+                        "label_prefix" : "label_prefix",
+                        "volatile" : volatile,
+                        "persist" : persist
                     }
                     var_bin_code_fit = BincodeFit(**fit_params)
                     # Storing fit object and column list for Variable-Width binning in data transform dictionary
@@ -1023,9 +1008,19 @@ class _FeatureEngineering:
                         "object" : var_bin_code_fit.output,
                         "object_order_column" : "TD_MinValue_BINFIT",
                         "accumulate" : accumulate_columns,
-                        "persist" : True   
+                        "persist" : True
                     }
+                    # Disabling display table name if persist is True by default
+                    if not volatile and not persist:
+                        var_transform_params["display_table_name"] = False
+                    
+                    if volatile:
+                        var_transform_params["volatile"] = True
+                        var_transform_params["persist"] = False
                     self.data = BincodeTransform(**var_transform_params).result
+                    if not volatile and not persist:
+                        # Adding transformed data containing table to garbage collector
+                        GarbageCollector._add_to_garbagecollector(self.data._table_name)
                     self._display_msg(msg="Updated dataset sample after performing Variable-Width binning:",
                                       data=self.data,
                                       progress_bar=self.progress_bar)
@@ -1049,11 +1044,13 @@ class _FeatureEngineering:
             # Storing custom string manipulation indicator in data transform dictionary
             self.data_transform_dict['custom_string_manipulation_ind'] = True
             # Fetching list required for performing operation.
-            extracted_col = self.custom_data.get("StringManipulationParam", None)
+            extracted_col = self.custom_data.get("StringManipulationParam", None).copy()
             if not extracted_col:
                 self._display_msg(inline_msg="No information provided for performing string manipulation.",
                                   progress_bar=self.progress_bar)
             else:
+                volatile = extracted_col.pop("volatile", False)
+                persist = extracted_col.pop("persist", False)
                 # Checking for column present in dataset or not
                 _Validators._validate_dataframe_has_argument_columns(list(extracted_col.keys()), "StringManipulationParam", self.data, "df")
 
@@ -1066,8 +1063,9 @@ class _FeatureEngineering:
                                   data=self.data,
                                   progress_bar=self.progress_bar)
         else:
-            self._display_msg(inline_msg="Skipping customized string manipulation.")
-                     
+            self._display_msg(inline_msg="Skipping customized string manipulation.",
+                              progress_bar=self.progress_bar)
+ 
     def _str_method_mapping(self,
                             target_col,
                             transform_val):
@@ -1096,7 +1094,11 @@ class _FeatureEngineering:
         
         # Fetching required parameters from json object
         string_operation = transform_val["StringOperation"]
-        
+
+        # Setting volatile and persist parameters for performing string manipulation
+        volatile, persist = self._set_generic_parameters(func_indicator="StringManipulationIndicator",
+                                                         param_name="StringManipulationParam")
+
         # Storing general parameters for performing string transformation
         fit_params = {
             "data" : self.data,
@@ -1106,6 +1108,14 @@ class _FeatureEngineering:
             "inplace" : True,
             "persist" : True
         }
+        # Disabling display table name if persist is True by default
+        if not volatile and not persist:
+            fit_params["display_table_name"] = False
+
+        if volatile:
+            fit_params["volatile"] = True
+            fit_params["persist"] = False
+
         # Adding additional parameters based on string operation type           
         if string_operation in ["StringCon", "StringTrim"]:
             string_argument = transform_val["String"] 
@@ -1125,11 +1135,15 @@ class _FeatureEngineering:
                           "string_length" : string_length}
         
         # returning dataset after performing string manipulation                 
-        return StrApply(**fit_params).result 
+        transform_output = StrApply(**fit_params).result
+        if not volatile and not persist:
+            # Adding transformed data containing table to garbage collector
+            GarbageCollector._add_to_garbagecollector(transform_output._table_name)
+        return transform_output
               
     def _one_hot_encoding(self, 
-                         one_hot_columns, 
-                         unique_counts):
+                          one_hot_columns, 
+                          unique_counts):
         """
         DESCRIPTION:
             Function performs the one hot encoding to categorcial columns/features in the dataset.
@@ -1143,12 +1157,16 @@ class _FeatureEngineering:
             unique_counts:
                   Required Argument.
                   Specifies the unique counts in the categorical columns.
-                  Types: int or list of integer (int)
-                  
+                  Types: int or list of integer (int)     
         """
         # TD function will add extra column_other in onehotEncoding, so 
         # initailizing this list to remove those extra columns 
         drop_lst = [ele + "_other" for ele in one_hot_columns]
+
+        # Setting volatile and persist parameters for performing encoding
+        volatile, persist = self._set_generic_parameters(func_indicator="CategoricalEncodingIndicator",
+                                                         param_name="CategoricalEncodingParam")
+
         # Adding fit parameters for performing encoding
         fit_params = {
             "data" : self.data,
@@ -1156,7 +1174,9 @@ class _FeatureEngineering:
             "is_input_dense" : True,
             "target_column" : one_hot_columns,
             "category_counts" : unique_counts,
-            "other_column" : "other"
+            "other_column" : "other",
+            "volatile" : volatile,
+            "persist" : persist
         }
         # Performing one hot encoding fit on target columns
         fit_obj = OneHotEncodingFit(**fit_params)
@@ -1172,9 +1192,22 @@ class _FeatureEngineering:
             "is_input_dense" : True,
             "persist" : True
         }
+        # Disabling display table name if persist is True by default
+        if not volatile and not persist:
+            transform_params["display_table_name"] = False
+        
+        # Setting persist to False if volatile is True
+        if volatile:
+            transform_params["volatile"] = True
+            transform_params["persist"] = False
+        
         # Performing one hot encoding transformation
-        transform_obj = OneHotEncodingTransform(**transform_params)
-        self.data = transform_obj.result.drop(drop_lst, axis=1)
+        transform_output = OneHotEncodingTransform(**transform_params).result
+
+        if not volatile and not persist:
+            # Adding transformed data containing table to garbage collector
+            GarbageCollector._add_to_garbagecollector(transform_output._table_name)
+        self.data = transform_output.drop(drop_lst, axis=1)
 
     def _ordinal_encoding(self,
                           ordinal_columns):
@@ -1188,11 +1221,16 @@ class _FeatureEngineering:
                 Specifies the categorical columns for which ordinal encoding will be performed.
                 Types: str or list of strings (str)
         """
+        # Setting volatile and persist parameters for performing encoding
+        volatile, persist = self._set_generic_parameters(func_indicator="CategoricalEncodingIndicator",
+                                                         param_name="CategoricalEncodingParam")
+
         # Adding fit parameters for performing encoding
         fit_params = {
             "data" : self.data,
             "target_column" : ordinal_columns,
-            "volatile" : True
+            "volatile" : volatile,
+            "persist" : persist
         }
         # Performing ordinal encoding fit on target columns
         ord_fit_obj = OrdinalEncodingFit(**fit_params)
@@ -1212,15 +1250,27 @@ class _FeatureEngineering:
             "accumulate" : accumulate_columns,
             "persist" : True
         }
+        # Disabling display table name if persist is True by default
+        if not volatile and not persist:
+            transform_params["display_table_name"] = False
+
+        # Setting persist to False if volatile is True
+        if volatile:
+            transform_params["volatile"] = True
+            transform_params["persist"] = False
         # Performing ordinal encoding transformation
         self.data = OrdinalEncodingTransform(**transform_params).result
+
+        if not volatile and not persist:
+            # Adding transformed data containing table to garbage collector
+            GarbageCollector._add_to_garbagecollector(self.data._table_name)
         
         if len(ordinal_columns) == 1 and ordinal_columns[0] == self.target_column:
             self.target_label = ord_fit_obj
         
 
     def _target_encoding(self,
-                          target_encoding_list):
+                         target_encoding_list):
         """
         DESCRIPTION:
             Function performs the target encoding to categorcial columns/features in the dataset.
@@ -1245,6 +1295,11 @@ class _FeatureEngineering:
         # Storing indicator and fit object for target encoding in data transform dictionary
         self.data_transform_dict["custom_target_encoding_ind"] = True
         self.data_transform_dict["custom_target_encoding_fit_obj"] = {}
+
+        # Setting volatile and persist parameters for performing encoding
+        volatile, persist = self._set_generic_parameters(func_indicator="CategoricalEncodingIndicator",
+                                                         param_name="CategoricalEncodingParam")
+        
         # Fetching required argument for performing target encoding
         for col,transform_val in target_encoding_list.items():
             encoder_method = transform_val["encoder_method"]
@@ -1255,7 +1310,9 @@ class _FeatureEngineering:
                 "category_data" : category_data,
                 "encoder_method" : encoder_method,
                 "target_columns" : col,
-                "response_column" : response_column
+                "response_column" : response_column,
+                "volatile" : volatile,
+                "persist" : persist
                 }
             if encoder_method == "CBM_DIRICHLET":
                 num_distinct_responses=transform_val["num_distinct_responses"]
@@ -1264,7 +1321,7 @@ class _FeatureEngineering:
             # Performing target encoding fit on target columns
             tar_fit_obj = TargetEncodingFit(**fit_params)
             # Storing each column fit object for target encoding in data transform dictionary
-            self.data_transform_dict["custom_target_encoding_fit_obj"].update({col : tar_fit_obj})
+            self.data_transform_dict["custom_target_encoding_fit_obj"].update({col : tar_fit_obj.result})
             # Extracting accumulate columns
             accumulate_columns = self._extract_list(self.data.columns, [col])
             # Adding transform parameters for performing encoding
@@ -1272,10 +1329,21 @@ class _FeatureEngineering:
                 "data" : self.data,
                 "object" : tar_fit_obj,
                 "accumulate" : accumulate_columns,
-                "persist" : True                
+                "persist" : True              
             }
+            
+            # Disabling display table name if persist is True by default
+            if not volatile and not persist:
+                transform_params["display_table_name"] = False
+            
+            if volatile:
+                transform_params["volatile"] = True
+                transform_params["persist"] = False
             # Performing ordinal encoding transformation
             self.data = TargetEncodingTransform(**transform_params).result
+            if not volatile and not persist:
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
     
     def _encoding_categorical_columns(self):
         """
@@ -1308,8 +1376,11 @@ class _FeatureEngineering:
             self._display_msg(msg="ONE HOT Encoding these Columns:",
                               col_lst=ohe_col,
                               progress_bar=self.progress_bar)
+            self._display_msg(msg="Sample of dataset after performing one hot encoding:",
+                              data=self.data,
+                              progress_bar=self.progress_bar)
         else:
-            self._display_msg(inline_msg="Encoding not required.",
+            self._display_msg(inline_msg="Analysis completed. No categorical columns were found.",
                               progress_bar=self.progress_bar)
         
         # List of columns after one hot
@@ -1337,8 +1408,10 @@ class _FeatureEngineering:
             # Storing custom categorical encoding indicator in data transform dictionary
             self.data_transform_dict["custom_categorical_encoding_ind"] = True
             # Fetching user input list for performing 
-            encoding_list = self.custom_data.get("CategoricalEncodingParam", None)
+            encoding_list = self.custom_data.get("CategoricalEncodingParam", None).copy()
             if encoding_list:
+                volatile = encoding_list.pop("volatile", False)
+                persist = encoding_list.pop("persist", False)
                 onehot_encode_ind = encoding_list.get("OneHotEncodingIndicator", False)
                 ordinal_encode_ind = encoding_list.get("OrdinalEncodingIndicator", False)
                 target_encode_ind = encoding_list.get("TargetEncodingIndicator", False)
@@ -1415,11 +1488,25 @@ class _FeatureEngineering:
         """
         DESCRIPTION:
             Function to perform different numerical transformations using NumApply on numerical features based on user input.
-                    
+
+        PARAMETERS:
+            target_col:
+                Required Argument.
+                Specifies the numerical column for which transformation will be performed.
+                Types: str
+
+            transform_val:
+                Required Argument.
+                Specifies different parameter require for applying numerical transformation.
+                Types: dict     
         """
         # Fetching columns for accumulation
         accumulate_columns = self._extract_list(self.data.columns, [target_col])
         apply_method = transform_val["apply_method"]
+
+        # Setting volatile and persist parameters for performing transformation
+        volatile, persist = self._set_generic_parameters(func_indicator="MathameticalTransformationIndicator",
+                                                         param_name="MathameticalTransformationParam")
         # Adding fit parameters for performing transformation
         fit_params={
             "data": self.data,
@@ -1429,14 +1516,25 @@ class _FeatureEngineering:
             "persist" :True,
             "accumulate" : accumulate_columns
         }
+        # Disabling display table name if persist is True by default
+        if not volatile and not persist:
+            fit_params["display_table_name"] = False
+        
+        if volatile:
+            fit_params["volatile"] = True
+            fit_params["persist"] = False
         # Adding addition details for fit parameters in case of SIGMOID transformation
         if apply_method == "sigmoid":
             sigmoid_style=transform_val["sigmoid_style"]
             fit_params = {**fit_params, "sigmoid_style" : sigmoid_style}
         # Performing transformation on target columns
-        return NumApply(**fit_params).result
+        transform_output = NumApply(**fit_params).result
+        if not volatile and not persist:
+            # Adding transformed data containing table to garbage collector
+            GarbageCollector._add_to_garbagecollector(transform_output._table_name)
+        return transform_output
   
-    def _numerical_transformation(self, target_columns, num_transform_data):
+    def _numerical_transformation(self, target_columns, num_transform_data, volatile, persist):
         """
         DESCRIPTION:
             Function to perform different numerical transformations using Fit and Transform on numerical features based on user input.
@@ -1446,7 +1544,9 @@ class _FeatureEngineering:
         fit_params={
             "data" : self.data,
             "object" : num_transform_data,
-            "object_order_column" : "TargetColumn"
+            "object_order_column" : "TargetColumn",
+            "volatile" : volatile,
+            "persist" : persist
         }
         # Peforming fit with all arguments.
         num_fit_obj = Fit(**fit_params)
@@ -1464,8 +1564,18 @@ class _FeatureEngineering:
             "id_columns" : id_columns,
             "persist" :True
         }
+        # Disabling display table name if persist is True by default
+        if not volatile and not persist:
+            transform_params["display_table_name"] = False
+        
+        if volatile:
+            transform_params["volatile"] = True
+            transform_params["persist"] = False
         # Peforming transformation on target columns
-        self.data = Transform(**transform_params).result  
+        self.data = Transform(**transform_params).result
+        if not volatile and not persist:
+            # Adding transformed data containing table to garbage collector
+            GarbageCollector._add_to_garbagecollector(self.data._table_name)
         self._display_msg(msg="Updated dataset sample after applying numerical transformation:",
                           data=self.data,
                           progress_bar=self.progress_bar)
@@ -1484,8 +1594,11 @@ class _FeatureEngineering:
         # Checking user input for mathematical transformations
         if mat_transform_input:
             # Extracting list required for mathematical transformations
-            mat_transform_list = self.custom_data.get("MathameticalTransformationParam", None)
+            mat_transform_list = self.custom_data.get("MathameticalTransformationParam", None).copy()
+            
             if mat_transform_list:
+                volatile = mat_transform_list.pop("volatile", False)
+                persist = mat_transform_list.pop("persist", False)
                 # Checking for column present in dataset or not
                 _Validators._validate_dataframe_has_argument_columns(list(mat_transform_list.keys()), 
                                                                      "MathameticalTransformationParam", self.data, "df")
@@ -1529,7 +1642,7 @@ class _FeatureEngineering:
                     copy_to_sql(df=transform_data, table_name="automl_num_transform_data", temporary=True)
                     num_transform_data = DataFrame.from_table("automl_num_transform_data") 
                     # Applying transformation using Fit/Transform functions
-                    self._numerical_transformation(target_columns, num_transform_data)
+                    self._numerical_transformation(target_columns, num_transform_data, volatile, persist)
                     # Storing custom numerical transformation parameters and column list in data transform dictionary
                     self.data_transform_dict['custom_numerical_transformation_col'] = target_columns
                     self.data_transform_dict['custom_numerical_transformation_params'] = num_transform_data
@@ -1555,6 +1668,8 @@ class _FeatureEngineering:
             nl_transform_list = self.custom_data.get("NonLinearTransformationParam", None)
             # Extracting list required for non-linear transformation
             if nl_transform_list:
+                volatile = nl_transform_list.pop("volatile", False)
+                persist = nl_transform_list.pop("persist", False)
                 total_combination = len(nl_transform_list)
                 # Generating all possible combination names
                 possible_combination = ["Combination_"+str(counter) for counter in range(1,total_combination+1)]
@@ -1581,12 +1696,14 @@ class _FeatureEngineering:
                             "data" : self.data,
                             "target_columns" : target_columns,
                             "formula" : formula,
-                            "result_column" : result_column
+                            "result_column" : result_column,
+                            "volatile" : volatile,
+                            "persist" : persist
                         }
                         # Performing fit on dataset
                         fit_obj = NonLinearCombineFit(**fit_param)
                         # Updating it for each non-linear combination
-                        self.data_transform_dict['custom_non_linear_transformation_fit_object'].update({comb:fit_obj})
+                        self.data_transform_dict['custom_non_linear_transformation_fit_object'].update({comb:fit_obj.result})
                         # Adding transform params for transformation   
                         transform_params = {
                             "data" : self.data,
@@ -1594,7 +1711,18 @@ class _FeatureEngineering:
                             "accumulate" : self.data.columns,
                             "persist" : True
                         }
+                        # Disabling display table name if persist is True by default
+                        if not volatile and not persist:
+                            transform_params["display_table_name"] = False
+                        
+                        if volatile:
+                            transform_params["volatile"] = True
+                            transform_params["persist"] = False
                         self.data = NonLinearCombineTransform(**transform_params).result
+
+                        if not volatile and not persist:
+                            # Adding transformed data containing table to garbage collector
+                            GarbageCollector._add_to_garbagecollector(self.data._table_name)
                     else:
                         self._display_msg(inline_msg="Combinations are not as per expectation.",
                                           progress_bar=self.progress_bar)
@@ -1620,29 +1748,64 @@ class _FeatureEngineering:
         anti_select_input = self.custom_data.get("AntiselectIndicator", False) 
         # Checking user input for anti-select columns    
         if anti_select_input:
-            # Extracting list required for anti-select columns
-            anti_select_list = self.custom_data.get("AntiselectParam", None)
-            if(anti_select_list):
-                if all(item in self.data.columns for item in anti_select_list):
-                    # Storing custom anti-select columns indicator and column list in data transform dictionary
-                    self.data_transform_dict['custom_anti_select_columns_ind'] = True
-                    self.data_transform_dict['custom_anti_select_columns'] = anti_select_list
-                    fit_params = {
-                        "data" : self.data,
-                        "exclude" : anti_select_list
-                    }
-                    # Performing transformation for given user input
-                    self.data = Antiselect(**fit_params).result
-                    self._display_msg(msg="Updated dataset sample after performing anti-select columns:",
-                                      data=self.data,
-                                      progress_bar=self.progress_bar)
-                else:
-                    self._display_msg(msg="Columns provided in list are not present in dataset:",
-                                      col_lst=anti_select_list,
-                                      progress_bar=self.progress_bar)         
+            anti_select_params = self.custom_data.get("AntiselectParam", None)
+            if anti_select_params:
+                # Extracting list required for anti-select columns
+                anti_select_list = anti_select_params.get("excluded_columns", None)
+                volatile = anti_select_params.get("volatile", False)
+                persist = anti_select_params.get("persist", False)
+                if(anti_select_list):
+                    if all(item in self.data.columns for item in anti_select_list):
+                        # Storing custom anti-select columns indicator and column list in data transform dictionary
+                        self.data_transform_dict['custom_anti_select_columns_ind'] = True
+                        self.data_transform_dict['custom_anti_select_columns'] = anti_select_list
+                        fit_params = {
+                            "data" : self.data,
+                            "exclude" : anti_select_list,
+                            "volatile" : volatile,
+                            "persist" : persist
+                        }
+                        # Performing transformation for given user input
+                        self.data = Antiselect(**fit_params).result
+                        self._display_msg(msg="Updated dataset sample after performing anti-select columns:",
+                                        data=self.data,
+                                        progress_bar=self.progress_bar)
+                    else:
+                        self._display_msg(msg="Columns provided in list are not present in dataset:",
+                                        col_lst=anti_select_list,
+                                        progress_bar=self.progress_bar)         
             else:
                 self._display_msg(inline_msg="No information provided for performing anti-select columns operation.",
                                   progress_bar=self.progress_bar)
         else:
             self._display_msg(inline_msg="Skipping customized anti-select columns.",
-                              progress_bar=self.progress_bar)
+                              progress_bar=self.progress_bar)
+            
+    def _set_generic_parameters(self,
+                                func_indicator=None,
+                                param_name=None):
+        """
+        DESCRIPTION:
+            Function to set generic parameters.
+        
+        PARAMETERS:     
+            func_indicator:
+                Optional Argument.
+                Specifies the name of function indicator.
+                Types: str
+                
+            param_name:
+                Optional Argument.
+                Specifies the name of the param which contains generic parameters.
+                Types: str
+
+        RETURNS:
+            Tuple containing volatile and persist parameters.
+        """
+        volatile = self.volatile
+        persist = self.persist
+        if self.custom_data is not None and self.custom_data.get(func_indicator, False):
+            volatile = self.custom_data[param_name].get("volatile", False)
+            persist = self.custom_data[param_name].get("persist", False)
+
+        return (volatile, persist)