teradataml 20.0.0.0__py3-none-any.whl → 20.0.0.2__py3-none-any.whl

teradataml/automl/data_preparation.py

@@ -19,7 +19,6 @@ import pandas as pd
 import random
 import time
 import warnings
-warnings.filterwarnings("ignore")
 
 # Teradata libraries
 from teradataml.dataframe.dataframe import DataFrame
@@ -27,10 +26,15 @@ from teradataml.dataframe.copy_to import copy_to_sql
 from teradataml import OutlierFilterFit, OutlierFilterTransform
 from teradataml import RoundColumns, TeradataMlException
 from teradataml import ScaleFit, ScaleTransform
-from teradataml import TrainTestSplit, UtilFuncs, TeradataConstants
+from teradataml import UtilFuncs, TeradataConstants
+from teradataml.common.garbagecollector import GarbageCollector
 from teradataml.common.messages import Messages, MessageCodes
 from teradataml.utils.validators import _Validators
+from teradataml import INTEGER
 
+# Control Randomnes
+random.seed(42)
+np.random.seed(42)
 
 class _DataPreparation:
     
@@ -41,7 +45,8 @@ class _DataPreparation:
                  excluded_columns=None,
                  custom_data=None,
                  data_transform_dict=None,
-                 task_type="Regression"):
+                 task_type="Regression",
+                 **kwargs):
         """
         DESCRIPTION:
             Function initializes the data, target column and columns datatypes
@@ -54,7 +59,7 @@ class _DataPreparation:
                 Types: teradataml Dataframe
             
             target_column:
-                Required Arugment.
+                Required Argument.
                 Specifies the name of the target column in "data".
                 Types: str
             
@@ -69,27 +74,49 @@ class _DataPreparation:
                 Types: int
             
             excluded_columns:
-                Required Arugment.
+                Required Argument.
                 Specifies the columns should be excluded from any processing.
                 Types: str or list of strings (str)
 
             custom_data:
-                Optional Arugment.
+                Optional Argument.
                 Specifies json object containing user customized input.
                 Types: json object
             
             data_transform_dict:
-                Optional Arugment.
+                Optional Argument.
                 Specifies the parameters for data transformation.
                 Types: dict
             
             task_type:
-                Required Arugment.
+                Required Argument.
                 Specifies the task type for AutoML, whether to apply regresion OR classification
                 on the provived dataset.
                 Default Value: "Regression"
                 Permitted Values: "Regression", "Classification"
                 Types: str
+
+            **kwargs:
+                Specifies the additional arguments for data preparation. Below
+                are the additional arguments:
+                    volatile:
+                        Optional Argument.
+                        Specifies whether to put the interim results of the
+                        functions in a volatile table or not. When set to
+                        True, results are stored in a volatile table,
+                        otherwise not.
+                        Default Value: False
+                        Types: bool
+
+                    persist:
+                        Optional Argument.
+                        Specifies whether to persist the interim results of the
+                        functions in a table or not. When set to True,
+                        results are persisted in a table; otherwise,
+                        results are garbage collected at the end of the
+                        session.
+                        Default Value: False
+                        Types: bool
         """
         self.data = data
         self.target_column = target_column
@@ -98,16 +125,15 @@ class _DataPreparation:
         self.data_transform_dict = data_transform_dict
         self.custom_data = custom_data
         self.task_type = task_type
+        self.volatile = kwargs.get("volatile", False)
+        self.persist = kwargs.get("persist", False)
         
         # Setting default value for auto run mode
-        self._train_size = 0.80
         self._data_sampling_method = "SMOTE"
         self._scale_method_reg = "STD"
         self._scale_method_cls = "RANGE"
         self.table_name_mapping = {}
         
-        random.seed(42)
-        np.random.seed(42)
         self.data_types = {key: value for key, value in self.data._column_names_and_types}
         
         
@@ -116,14 +142,13 @@ class _DataPreparation:
         """
         DESCRIPTION:
             Function to perform following tasks:-
-                1. Splits the given data into training and testing datasets.
-                2. Performs outlier processing on the training dataset and transformation on the testing dataset.
-                3. Performs feature selection using RFE, PCA, and Lasso.
-                4. Performs feature scaling.
+                1. Performs outlier processing and transformation on dataset.
+                2. Performs feature selection using RFE, PCA, and Lasso.
+                3. Performs feature scaling.
          
         PARAMETERS:  
             auto:
-                Optional Arugment.
+                Optional Argument.
                 Specifies whether to run AutoML in custom mode or auto mode.
                 When set to False, runs in custom mode. Otherwise, by default runs in auto mode.
                 Default Value: True
@@ -138,38 +163,36 @@ class _DataPreparation:
                           progress_bar=self.progress_bar)
         # Setting user value in case of custom running mode
         if not auto:
-            self._set_custom_train_test_split()
             self._set_custom_scaling_method()
             self._set_custom_sampling()
         
-        # Performing train test split   
-        self._train_test_split()
-        self.progress_bar.update()
-        
         # Handling ouliers in dataset
         self._handle_outliers(auto)
         self.progress_bar.update()
         
         # Handling float type features before processing with feature selection and scaling
-        train = self._handle_generated_features('train')
-        test = self._handle_generated_features('test')
+        training_data = self._handle_generated_features()
         self.progress_bar.update()
         
         # Temporary Pulling data for feature selection 
         # Will change after sto
         
         # Checking for data imbalance    
-        if self._check_data_imbalance(train):
-            train = self._data_sampling(train)
+        if self._check_data_imbalance(training_data):
+            training_data = self._data_sampling(training_data)
         self.progress_bar.update()
         
+        # Sorting the data based on id to 
+        # remove any shuffling done by sampling
+        training_data = training_data.sort_values(by='id')
+
         # Performing feature selection using lasso followed by scaling  
-        self._feature_selection_Lasso(train, test)
+        self._feature_selection_Lasso(training_data)
         self._scaling_features(feature_selection_mtd="lasso")
         self.progress_bar.update()
         
         # Performing feature selection using rfe followed by scaling      
-        self._feature_selection_RFE(train, test)
+        self._feature_selection_RFE(training_data)
         self._scaling_features(feature_selection_mtd="rfe")
         self.progress_bar.update()
         
@@ -180,85 +203,8 @@ class _DataPreparation:
         
         return [self.rfe_feature, self.lasso_feature, self.pca_feature], self.data_transform_dict
 
-    # Splits data into train and test
-    def _train_test_split(self):
-        
-        """
-        DESCRIPTION:
-            Function splits the data into training and testing datasets.
-
-        PARAMETERS:  
-            train_size:
-                Optional Argument.
-                Specifies the training size required for splitting dataset.
-                By Default, it takes 0.8 as training size.
-                Types: float
-        """
-        self._display_msg(msg="\nSpliting of dataset into training and testing ...",
-                          progress_bar=self.progress_bar,
-                          show_data=True)
-        self._display_msg(inline_msg="Training size : {}".format(self._train_size),
-                          progress_bar=self.progress_bar)
-        self._display_msg(inline_msg="Testing size  : {}".format(round((1-self._train_size),2)),
-                          progress_bar=self.progress_bar)
-        start_time = time.time()
-        # Applying TrainTestSplit function on data
-        # Regression
-        train_test_func_params = {
-            "data" : self.data, 
-            "id_column" : "id", 
-            "train_size" : self._train_size, 
-            "seed" : 42
-        }
-        if self.is_classification_type():
-            train_test_func_params["stratify_column"]=self.target_column
-        train_test_split_out = TrainTestSplit(**train_test_func_params)
-        train_test_split_out = train_test_split_out.result
-        
-        # Splitting the data into training and testing data
-        self.train_df = train_test_split_out[train_test_split_out['TD_IsTrainRow'] == 1].drop('TD_IsTrainRow', axis=1)
-        self.test_df = train_test_split_out[train_test_split_out['TD_IsTrainRow'] == 0].drop('TD_IsTrainRow', axis=1)
-
-        self._display_msg(msg="Training data sample",
-                          data=self.train_df,
-                          progress_bar=self.progress_bar)
-        
-        self._display_msg(msg="Testing data sample",
-                          data=self.test_df,
-                          progress_bar=self.progress_bar)
-        
-        end_time = time.time()
-        self._display_msg(msg="Time taken for spliting of data: {:.2f} sec  ".format(end_time - start_time),
-                          progress_bar=self.progress_bar,
-                          show_data=True)
-
-    def _set_custom_train_test_split(self):
-        """
-        DESCRIPTION:
-            Function to split dataset into training and testing based on user input.
-             
-        """ 
-        # Fetching user input for train test split
-        train_test_split_input = self.custom_data.get("TrainTestSplitIndicator", False) 
-        if train_test_split_input:
-            # Extracting training size
-            custom_train_size = self.custom_data.get("TrainingSize", None)
-            if custom_train_size is None:
-                self._display_msg(inline_msg="No information provided for training size. Proceeding with default option.",
-                                  progress_bar=self.progress_bar)
-            else:
-                if not isinstance(custom_train_size, float):
-                    err = Messages.get_message(MessageCodes.INVALID_COLUMN_TYPE,
-                                       'custom_train', type(custom_train_size).__name__, 
-                                       'float')
-                    raise TeradataMlException(err, MessageCodes.INVALID_COLUMN_TYPE)
-                self._train_size = custom_train_size
-        else:
-            self._display_msg(inline_msg="No information provided for performing customized train test split. Proceeding with default option.",
-                              progress_bar=self.progress_bar)
-    
     def _handle_outliers(self,
-                        auto):
+                         auto):
         """
         DESCRIPTION:
             Function to handle existing outliers in dataset based on running mode.
@@ -289,6 +235,12 @@ class _DataPreparation:
         DESCRIPTION:
             Function to handle data imbalance in dataset using sampling techniques 
             in case of classification.
+        
+        PARAMETERS:
+            data:
+                Required Argument.
+                Specifies the input teradataml DataFrame.
+                Types: pandas Dataframe.
         """
         pass
     
@@ -310,7 +262,7 @@ class _DataPreparation:
         outlier_method = "Tukey"
         
         # List of columns for outlier processing.
-        outlier_columns = [col for col in self.train_df.columns if col not in self.excluded_columns]
+        outlier_columns = [col for col in self.data.columns if col not in self.excluded_columns]
         
         # Detecting outlier percentage in each columns
         outlier_percentage_df = self._outlier_detection(outlier_method, outlier_columns)
@@ -360,26 +312,45 @@ class _DataPreparation:
             Pandas DataFrame containing, column name with outlier percentage.
 
         """
-        # Performing fit on train dataset for outlier handling
+
+        # Setting volatile and persist parameters for Outlier handling function
+        volatile, persist = self._set_generic_parameters(func_indicator='OutlierFilterIndicator',
+                                                         param_name='OutlierFilterParam')
+
+        # Performing fit on dataset for outlier handling
         fit_params = {
-            "data" : self.train_df,
+            "data" : self.data,
             "target_columns" : target_columns,
             "outlier_method" : outlier_method,
-            "replacement_value" : replacement_value
+            "replacement_value" : replacement_value,
+            "volatile" : volatile,
+            "persist" : persist
         }
         outlier_fit_out = OutlierFilterFit(**fit_params)
-        # Performing transform on train dataset for outlier handling
+        # Performing transform on dataset for outlier handling
         transform_params = {
-            "data" : self.train_df,
+            "data" : self.data,
             "object" : outlier_fit_out.result,
             "persist" : True
         }
-        self.train_df = OutlierFilterTransform(**transform_params).result
+
+        # Disabling print if persist is True by default
+        if not volatile and not persist:
+            transform_params["display_table_name"] = False
+
+        if volatile:
+            transform_params["volatile"] = True
+            transform_params["persist"] = False
+        self.data = OutlierFilterTransform(**transform_params).result
+
+        if not volatile and not persist:
+            # Adding transformed data containing table to garbage collector
+            GarbageCollector._add_to_garbagecollector(self.data._table_name)
 
     def _outlier_processing(self):
         """
         DESCRIPTION:
-            Function performs outlier processing on the training dataset. It identifies and handle outliers in the dataset.
+            Function performs outlier processing on dataset. It identifies and handle outliers in the dataset.
                 
         """
         self._display_msg(msg="\nOutlier preprocessing ...",
@@ -400,6 +371,9 @@ class _DataPreparation:
             target_columns=columns_to_drop_rows
             replacement_strategy = "DELETE"
             self._outlier_handling(target_columns, outlier_handling_method, replacement_strategy)
+            self._display_msg(msg="Sample of dataset after removing outlier rows:",
+                              data=self.data,
+                              progress_bar=self.progress_bar)
         
         # Imputing Median value in place of outliers
         if len(columns_to_impute) != 0:
@@ -409,6 +383,13 @@ class _DataPreparation:
             target_columns=columns_to_impute
             replacement_strategy = "MEDIAN"
             self._outlier_handling(target_columns, outlier_handling_method, replacement_strategy)
+            self._display_msg(msg="Sample of dataset after performing MEDIAN inplace:",
+                              data=self.data,
+                              progress_bar=self.progress_bar)
+
+        if len(columns_to_drop_rows) == 0 and len(columns_to_impute) == 0:
+            self._display_msg(msg='Analysis indicates not outlier in the dataset. No Action Taken.',
+                              progress_bar=self.progress_bar)
             
         end_time = time.time()
         self._display_msg("Time Taken by Outlier processing: {:.2f} sec ".format(end_time - start_time),
@@ -418,7 +399,7 @@ class _DataPreparation:
     def _custom_outlier_processing(self):
         """
         DESCRIPTION:
-            Function to perform outlier processing on the training dataset based on user input.
+            Function to perform outlier processing on dataset based on user input.
                 
         """
         self._display_msg(msg="\nStarting customized outlier processing ...",
@@ -428,7 +409,7 @@ class _DataPreparation:
         # Checking user input for outlier filtering   
         if outlier_filter_input:
             # List of columns for outlier processing.
-            target_columns = [col for col in self.train_df.columns if col not in self.excluded_columns]
+            target_columns = [col for col in self.data.columns if col not in self.excluded_columns]
             # Checking user input for outlier detection method
             outlier_method = self.custom_data.get("OutlierDetectionMethod", None)
             if outlier_method == 'PERCENTILE':
@@ -445,11 +426,13 @@ class _DataPreparation:
             # Checking for rows if outlier containing columns exist
             if outlier_df.shape[0]:
                 # Checking user input list for outlier handling
-                outlier_transform_list = self.custom_data.get("OutlierFilterParam", None)
+                outlier_transform_list = self.custom_data.get("OutlierFilterParam", None).copy()
                 if outlier_transform_list:
+                    volatile = outlier_transform_list.pop("volatile", False)
+                    persist = outlier_transform_list.pop("persist", False)
                     # Checking user input for outlier handling
                     _Validators._validate_dataframe_has_argument_columns(list(outlier_transform_list.keys()), "OutlierFilterParam",
-                                                                         self.train_df, "train")
+                                                                         self.data, "outlier_data")
 
                     for target_col, transform_val in outlier_transform_list.items():
                         # Fetching replacement value
@@ -498,14 +481,12 @@ class _DataPreparation:
         from sklearn.decomposition import PCA
         
         start_time = time.time()
-        # Training and testing data using pandas dataframe
+
         # Temporary Pulling data for feature selection
-        train = DataFrame.from_table(self.table_name_mapping['pca_train']).to_pandas()
-        test = DataFrame.from_table(self.table_name_mapping['pca_test']).to_pandas()
+        pca_train = DataFrame.from_table(self.table_name_mapping['pca_train']).to_pandas()
  
         # Drop unnecessary columns and store the result
-        train_data = train.drop(columns=['id', self.target_column], axis=1)
-        test_data = test.drop(columns=['id', self.target_column], axis=1)
+        train_data = pca_train.drop(columns=['id', self.target_column], axis=1)
 
         # Initialize and fit PCA
         pca = PCA()
@@ -518,16 +499,15 @@ class _DataPreparation:
         # Create a new instance of PCA with the optimal number of components
         pca = PCA(n_components=n, random_state=42)
 
-        # Apply PCA on training and testing dataset
+        # Apply PCA on dataset
         X_train_pca = pca.fit_transform(train_data)
-        X_test_pca = pca.transform(test_data)
         
         # storing instance of PCA in data transformation dictionary
         self.data_transform_dict["pca_fit_instance"] = pca
+        self.data_transform_dict["pca_fit_columns"] = train_data.columns.tolist()
         
         #converting the numarray into dataframes
         train_df = pd.DataFrame(X_train_pca)
-        test_df  = pd.DataFrame(X_test_pca)
         
         #creating names for combined columns 
         column_name = {col: 'col_'+str(i) for i,col in enumerate(train_df.columns)}
@@ -537,15 +517,12 @@ class _DataPreparation:
         
         #renaming them 
         train_df = train_df.rename(columns=column_name)
-        test_df = test_df.rename(columns=column_name)
         
         # adding the id column [PCA does not shuffle the dataset]
-        train_df = pd.concat([train.reset_index(drop=True)['id'], train_df.reset_index(drop=True)], axis=1)
-        test_df = pd.concat([test.reset_index(drop=True)['id'], test_df.reset_index(drop=True)], axis=1)
+        train_df = pd.concat([pca_train.reset_index(drop=True)['id'], train_df.reset_index(drop=True)], axis=1)
         
-        # merging target column with new training and testing data
-        train_df[self.target_column] = train[self.target_column].reset_index(drop=True)
-        test_df[self.target_column] = test[self.target_column].reset_index(drop=True)
+        # merging target column with new data
+        train_df[self.target_column] = pca_train[self.target_column].reset_index(drop=True)
 
         self.pca_feature = train_df.drop(columns=['id',self.target_column],axis=1).columns.tolist()
         
@@ -557,31 +534,21 @@ class _DataPreparation:
                           progress_bar=self.progress_bar,
                           show_data=True)
         
-        if self.is_classification_type():
-            train_df[self.target_column] = train_df[self.target_column].astype('int')
-            test_df[self.target_column] = test_df[self.target_column].astype('int')
-
         # Pushing the data in database
-        self.copy_dataframe_to_sql(train_df, test_df, 'pca')
+        self.copy_dataframe_to_sql(train_df, 'pca', self.persist)
  
-    def _feature_selection_RFE(self, 
-                              train=None,
-                              test=None):
+    def _feature_selection_RFE(self,
+                               data=None):
         """
         DESCRIPTION:
              Function performs Recursive Feature Elimination (RFE) for feature selection. 
              It identifies a subset of the most relevant features in the dataset.
              
         PARAMETERS:
-            train:
+            data:
                 Required Argument.
                 Specifies the input train pandas DataFrame.
-                Types: pandas Dataframe
-                
-            test:
-                Required Argument.
-                Specifies the input test pandas DataFrame.
-                Types: pandas Dataframe      
+                Types: pandas Dataframe   
         """
         self._display_msg(msg="\nFeature selection using rfe ...",
                           progress_bar=self.progress_bar,
@@ -590,57 +557,59 @@ class _DataPreparation:
         # Required imports for RFE   
         from sklearn.feature_selection import RFECV
         from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
-        from sklearn.model_selection import StratifiedKFold,KFold
+        from sklearn.model_selection import StratifiedKFold
 
         start_time = time.time()
         # Regression
         is_classification = self.is_classification_type()
         # Getting the value of k in k-fold cross-validation
-        folds = self._num_of_folds(train.shape[0])
+        folds = self._num_of_folds(data.shape[0])
 
-        # Random forest for RFE model
-        RFModel = RandomForestRegressor if not is_classification else RandomForestClassifier
-        rf = RFModel(n_estimators=100, random_state=42)
+        # Suppressing warnings generated by pandas and sklearn
+        with warnings.catch_warnings():
+            warnings.filterwarnings('ignore')
 
-        # Determine the scoring metric based on the number of unique classes
-        score = 'r2' if not self.is_classification_type() \
-                else 'roc_auc' if self.data.drop_duplicate(self.target_column).size == 2 else 'f1_macro'
+            # Random forest for RFE model
+            RFModel = RandomForestRegressor if not is_classification else RandomForestClassifier
+            rf = RFModel(n_estimators=100, random_state=42)
 
-        # Instantiate StratifiedKFold with shuffling for classification
-        cv = folds if not self.is_classification_type() \
-                   else StratifiedKFold(n_splits=folds, shuffle=True, random_state=42)
+            # Determine the scoring metric based on the number of unique classes
+            score = 'r2' if not self.is_classification_type() \
+                    else 'roc_auc' if self.data.drop_duplicate(self.target_column).size == 2 else 'f1_macro'
 
-        # Define the RFE with cross-validation
-        rfecv = RFECV(rf, cv=cv, scoring=score)
+            # # Instantiate StratifiedKFold with shuffling for classification
+            cv = folds if not self.is_classification_type() \
+                    else StratifiedKFold(n_splits=folds, shuffle=False)
 
-        # Prepare the training data
-        train_data = train.drop(columns=['id',self.target_column], axis=1)
-        train_target = train[self.target_column]
+            # Define the RFE with cross-validation
+            rfecv = RFECV(rf, cv=cv, scoring=score)
 
-        # Fit the RFE using cv
-        rfecv.fit(train_data, train_target)
+            # Prepare data
+            train_data = data.drop(columns=['id',self.target_column], axis=1)
+            train_target = data[self.target_column]
 
-        # Extract the features
-        features = train_data.columns[rfecv.support_].tolist()
+            # Fit the RFE using cv
+            rfecv.fit(train_data, train_target)
 
-        self._display_msg(msg="feature selected by RFE:",
-                          col_lst=features,
-                          progress_bar=self.progress_bar)
-        features.append(self.target_column)
-        features.insert(0,'id')
-        
-        train_df = train[features]
-        test_df = test[features]
-        
-        # storing the rfe selected features in data transformation dictionary
-        self.data_transform_dict['rfe_features'] = features
-        
-        columns_to_rename = [col for col in train_df.columns if col not in ['id', self.target_column]]
-        new_column = {col: f'r_{col}' for col in columns_to_rename}
-        self.excluded_columns.extend([new_column[key] for key in self.excluded_columns if key in new_column])
-        
-        train_df.rename(columns=new_column, inplace=True)
-        test_df.rename(columns=new_column, inplace=True)
+            # Extract the features
+            features = train_data.columns[rfecv.support_].tolist()
+
+            self._display_msg(msg="feature selected by RFE:",
+                            col_lst=features,
+                            progress_bar=self.progress_bar)
+            features.append(self.target_column)
+            features.insert(0,'id')
+            
+            selected_rfe_df = data[features]
+            
+            # storing the rfe selected features in data transformation dictionary
+            self.data_transform_dict['rfe_features'] = features
+            
+            columns_to_rename = [col for col in selected_rfe_df.columns if col not in ['id', self.target_column]]
+            new_column = {col: f'r_{col}' for col in columns_to_rename}
+            self.excluded_columns.extend([new_column[key] for key in self.excluded_columns if key in new_column])
+            
+            selected_rfe_df.rename(columns=new_column, inplace=True)
         
         # storing the rename column list in data transformation dictionary
         self.data_transform_dict['rfe_rename_column'] = columns_to_rename
@@ -649,29 +618,24 @@ class _DataPreparation:
         self._display_msg(msg="Total time taken by feature selection: {:.2f} sec  ".format( end_time - start_time),
                           progress_bar=self.progress_bar,
                           show_data=True)
-        self.rfe_feature = train_df.drop(columns=['id',self.target_column], axis=1).columns.tolist()
+        self.rfe_feature = selected_rfe_df.drop(columns=['id',self.target_column], axis=1).columns.tolist()
 
         # Pushing data into database
-        self.copy_dataframe_to_sql(train_df, test_df, 'rfe')
+        self.copy_dataframe_to_sql(selected_rfe_df, 'rfe', self.persist)
         
     def _feature_selection_Lasso(self,
-                                train=None,
-                                test=None):
+                                 data=None):
         """
         DESCRIPTION:
             Function performs Lasso Regression for feature selection.
             It helps in identifing and retaining the most important features while setting less important ones to zero.
         
         PARAMETERS:
-            train:
+            data:
                 Required Argument.
                 Specifies the input train pandas DataFrame.
                 Types: pandas Dataframe
 
-            test:
-                Required Argument.
-                Specifies the input test pandas DataFrame.
-                Types: pandas Dataframe 
         """
         start_time = time.time()
         self._display_msg(msg="\nFeature selection using lasso ...",
@@ -682,35 +646,46 @@ class _DataPreparation:
         from sklearn.model_selection import GridSearchCV
         from sklearn.linear_model import Lasso
         from sklearn.linear_model import LogisticRegression
-        
+        from sklearn.model_selection import StratifiedKFold
+
         # Getting the value k in k-fold cross-validation
-        num_folds = self._num_of_folds(train.shape[0])
+        num_folds = self._num_of_folds(data.shape[0])
 
-        # Prepare the training data
-        train_features = train.drop(columns=['id',self.target_column], axis=1)
-        train_target = train[self.target_column]
+        # Prepare data
+        train_features = data.drop(columns=['id',self.target_column], axis=1)
+        train_target = data[self.target_column]
 
-        # Determine the estimator and parameters based on the type of problem
-        if self.is_classification_type():
-            if self.data.drop_duplicate(self.target_column).size == 2:
-                scoring_metric = 'roc_auc'
+        # Suppressing warnings generated by pandas and sklearn
+        with warnings.catch_warnings():
+            warnings.filterwarnings('ignore')
+
+            # Determine the estimator and parameters based on the type of problem
+            if self.is_classification_type():
+                if self.data.drop_duplicate(self.target_column).size == 2:
+                    scoring_metric = 'roc_auc'
+                else:
+                    scoring_metric = 'f1_macro'
+                estimator = LogisticRegression(solver='saga', penalty='l2', multi_class='auto', random_state=42)
+                parameters = {'C':[0.00001,0.0001,0.001,0.01,0.05,0.1,10,100,1000], 'max_iter': [100, 500]}
             else:
-                scoring_metric = 'f1_macro'
-            estimator = LogisticRegression(penalty='l1', solver='liblinear', multi_class='auto')
-            parameters = {'C':[0.00001,0.0001,0.001,0.01,0.05,0.1,10,100,1000], 'max_iter': [100, 500]}
-        else:
-            estimator = Lasso()
-            parameters = {'alpha':[0.00001,0.0001,0.001,0.01,0.05,0.1,10,100,1000], 'max_iter': [100, 500]}
-            scoring_metric = "r2"
+                estimator = Lasso(random_state=42)
+                parameters = {'alpha':[0.00001,0.0001,0.001,0.01,0.05,0.1,10,100,1000], 'max_iter': [100, 500]}
+                scoring_metric = "r2"
+
+            if self.is_classification_type():
+                cv = StratifiedKFold(n_splits=5, shuffle=False)
+            else:
+                cv = num_folds
 
-        # Applying hyperparameter tuning and optimizing score
-        hyperparameter_search = GridSearchCV(estimator, parameters, cv=num_folds, scoring=scoring_metric, verbose=0)
+            # Applying hyperparameter tuning and optimizing score
+            hyperparameter_search = GridSearchCV(estimator, parameters, cv=cv, refit=True, 
+                                                scoring=scoring_metric, verbose=0)
 
-        # Fitting the best result from hyperparameter
-        hyperparameter_search.fit(train_features, train_target)
+            # Fitting the best result from hyperparameter
+            hyperparameter_search.fit(train_features, train_target)
 
-        # Extracting the important estimators
-        feature_importance = np.abs(hyperparameter_search.best_estimator_.coef_)
+            # Extracting the important estimators
+            feature_importance = np.abs(hyperparameter_search.best_estimator_.coef_)
 
         # Extracting feature using estimators whose importance > 0 
         if self.is_classification_type():
@@ -725,8 +700,7 @@ class _DataPreparation:
                         progress_bar=self.progress_bar)
 
         important_features = ['id'] + important_features + [self.target_column]
-        train_df = train[important_features]
-        test_df = test[important_features]
+        selected_lasso_df = data[important_features]
 
         # Storing the lasso selected features in data transformation dictionary
         self.data_transform_dict['lasso_features'] = important_features
@@ -736,61 +710,62 @@ class _DataPreparation:
         self._display_msg(msg="Total time taken by feature selection: {:.2f} sec  ".format( end_time - start_time),
                         progress_bar=self.progress_bar,
                         show_data=True)
-        self.lasso_feature = train_df.drop(columns=['id',self.target_column], axis=1).columns.tolist()
+        self.lasso_feature = selected_lasso_df.drop(columns=['id',self.target_column], axis=1).columns.tolist()
 
-        self.copy_dataframe_to_sql(train_df, test_df, 'lasso')
+        self.copy_dataframe_to_sql(selected_lasso_df, 'lasso', self.persist)
 
     def copy_dataframe_to_sql(self, 
-                              train, 
-                              test, 
-                              prefix):
+                              data,
+                              prefix,
+                              persist):
         """
         DESCRIPTION:
             Function to copy dataframe to SQL with generated table name.
 
         PARAMETERS:
-            train:
-                Required Argument.
-                Specifies the input train pandas DataFrame.
-                Types: pandas Dataframe
-
-            test:
+            data:
                 Required Argument.
-                Specifies the input test pandas DataFrame.
+                Specifies the input pandas DataFrame.
                 Types: pandas Dataframe
 
             prefix:
                 Required Argument.
                 Specifies the prefix for the table name.
                 Types: str
+
+            persist:
+                Required Argument.
+                Specifies whether to persist the results of the
+                function in a table or not. When set to True,
+                results are persisted in a table; otherwise,
+                results are garbage collected at the end of the
+                session.
+                Types: bool
         """
         # Generating table names
         train_table_name = UtilFuncs._generate_temp_table_name(prefix='{}_train'.format(prefix), 
-                                                               table_type = TeradataConstants.TERADATA_TABLE)
-        test_table_name = UtilFuncs._generate_temp_table_name(prefix='{}_test'.format(prefix), 
-                                                              table_type = TeradataConstants.TERADATA_TABLE)
-        
+                                                               table_type = TeradataConstants.TERADATA_TABLE,
+                                                               gc_on_quit=not persist)
         # Storing the table names in the table name mapping dictionary
         self.table_name_mapping['{}_train'.format(prefix)] = train_table_name
-        self.table_name_mapping['{}_test'.format(prefix)] = test_table_name
 
         # Pushing data into database
-        copy_to_sql(df=train, table_name=train_table_name, if_exists="replace")
-        copy_to_sql(df=test, table_name=test_table_name, if_exists="replace")
-
-
+        if self.is_classification_type():
+            copy_to_sql(df=data, table_name=train_table_name, if_exists="replace", types={f'{self.target_column}': INTEGER})
+        else:
+            copy_to_sql(df=data, table_name=train_table_name, if_exists="replace")
         
     def _scaling_features_helper(self, 
-                                train=None,  
-                                feature_selection_mtd=None):
+                                 data=None,  
+                                 feature_selection_mtd=None):
         """
         DESCRIPTION:
             This function selects the features on which feature scaling should be applied.
             
         PARAMETERS:
-            train:
+            data:
                 Required Argument.
-                Specifies the training data.
+                Specifies the data on which feature scaling will be applied.
                 Types: teradataml Dataframe
             
             feature_selection_mtd:
@@ -805,10 +780,10 @@ class _DataPreparation:
         columns_to_scale = []
         
         # Iterating over the columns
-        for col in train.columns:
+        for col in data.columns:
             # Selecting columns that will be scaled 
             # Exculding target_col and columns with single value
-            if col not in ['id', self.target_column] and train.drop_duplicate(col).size > 1:
+            if col not in ['id', self.target_column] and data.drop_duplicate(col).size > 1:
                 columns_to_scale.append(col)
         
         if feature_selection_mtd == "lasso":
@@ -822,7 +797,7 @@ class _DataPreparation:
         return columns_to_scale
 
     def _scaling_features(self,
-                        feature_selection_mtd=None):
+                          feature_selection_mtd=None):
         """
         DESCRIPTION:
             Function performs feature scaling on columns present inside the dataset 
@@ -832,7 +807,7 @@ class _DataPreparation:
             feature_selection_mtd:
                 Required Argument.
                 Specifies the feature selection algorithm used.
-                Types: str     
+                Types: str 
         """
 
         self._display_msg(msg="\nscaling Features of {} data ...".format(feature_selection_mtd),
@@ -840,8 +815,7 @@ class _DataPreparation:
                           show_data=True)
         
         start_time = time.time()
-        train = None
-        test = None
+        data_to_scale = None
         
         if self.is_classification_type():
             scale_method = self._scale_method_cls
@@ -850,17 +824,18 @@ class _DataPreparation:
         
         # Loading data for feature scaling based of feature selection method
         if feature_selection_mtd == 'rfe':
-            train = DataFrame(self.table_name_mapping['rfe_train'])
-            test = DataFrame(self.table_name_mapping['rfe_test'])
+            data_to_scale = DataFrame(self.table_name_mapping['rfe_train'])
         elif feature_selection_mtd == 'lasso':
-            train = DataFrame(self.table_name_mapping['lasso_train'])
-            test = DataFrame(self.table_name_mapping['lasso_test'])
+            data_to_scale = DataFrame(self.table_name_mapping['lasso_train'])
         else:
-            train = self.train_df
-            test = self.test_df
+            data_to_scale = self.data
+
+        # Setting volatile and persist parameters for ScaleFit and ScaleTransform functions
+        volatile, persist = self._set_generic_parameters(func_indicator='FeatureScalingIndicator',
+                                                         param_name='FeatureScalingParam')
 
         # List of columns that will be scaled
-        scale_col= self._scaling_features_helper(train, feature_selection_mtd)
+        scale_col= self._scaling_features_helper(data_to_scale, feature_selection_mtd)
         
         if len(scale_col) != 0:
             self._display_msg(msg="columns that will be scaled: ",
@@ -868,58 +843,38 @@ class _DataPreparation:
                               progress_bar=self.progress_bar)
 
             # Scale Fit
-            fit_obj = ScaleFit(data=train,
+            fit_obj = ScaleFit(data=data_to_scale,
                                target_columns=scale_col,
-                               scale_method=scale_method)
+                               scale_method=scale_method,
+                               volatile=volatile,
+                               persist=persist)
             
             # storing the scale fit object and columns in data transformation dictionary
-            self.data_transform_dict['{}_scale_fit_obj'.format(feature_selection_mtd)] = fit_obj
+            self.data_transform_dict['{}_scale_fit_obj'.format(feature_selection_mtd)] = fit_obj.output
             self.data_transform_dict['{}_scale_col'.format(feature_selection_mtd)] = scale_col
             
             # List of columns to copy to the output generated by scale transform
-            accumulate_cols = list(set(train.columns) - set(scale_col))
-            
-            # Scaling on training dataset
-            tr_obj = ScaleTransform(data=train,
-                      object=fit_obj,
-                      accumulate=accumulate_cols)
-            
-            # Scaling on testing dataset
-            ts_obj = ScaleTransform(data=test,
-                      object=fit_obj,
-                      accumulate=accumulate_cols)
+            accumulate_cols = list(set(data_to_scale.columns) - set(scale_col))
             
-            train = tr_obj.result
-            test = ts_obj.result
+            # Scaling dataset
+            transform_obj = ScaleTransform(data=data_to_scale,
+                                           object=fit_obj,
+                                           accumulate=accumulate_cols)
+            scaled_df = transform_obj.result
             
-            self._display_msg(msg="Training dataset sample after scaling:",
-                              data=train,
-                              progress_bar=self.progress_bar)
-            self._display_msg(msg="Testing dataset sample after scaling:",
-                              data=test,
+            self._display_msg(msg="Dataset sample after scaling:",
+                              data=scaled_df,
                               progress_bar=self.progress_bar)
         else:
             self._display_msg(msg="No columns to scale.",
                               progress_bar=self.progress_bar)
-            
-        if self.is_classification_type():
-            train, test = self._bigint_to_int(train, test)
 
-        self.copy_dataframe_to_sql(train, test, feature_selection_mtd)
+        self.copy_dataframe_to_sql(scaled_df, feature_selection_mtd, persist)
 
         end_time = time.time()
         self._display_msg(msg="Total time taken by feature scaling: {:.2f} sec".format( end_time - start_time),
                           progress_bar=self.progress_bar,
                           show_data=True)
-            
-    def _bigint_to_int(self, train, test):
-        tr = train.to_pandas()
-        tr[self.target_column] = tr[self.target_column].astype('int')
-        
-        ts = test.to_pandas()
-        ts[self.target_column] = ts[self.target_column].astype('int')
-        
-        return tr, ts
 
     def _set_custom_scaling_method(self):
         """
@@ -932,43 +887,32 @@ class _DataPreparation:
         # Checking user input for feature scaling    
         if feature_scaling_input:
             # Extracting scaling method
-            custom_scaling_method = self.custom_data.get("FeatureScalingMethod", None)
-            if custom_scaling_method is None:
-                self._display_msg(inline_msg="No information provided for customized scaling method. AutoML will continue with default option.",
-                                  progress_bar=self.progress_bar)
-            else:
-                if self.is_classification_type():
-                    self._scale_method_cls = custom_scaling_method
+            custom_scaling_params = self.custom_data.get("FeatureScalingParam", None)
+            if custom_scaling_params:
+                custom_scaling_method = custom_scaling_params.get("FeatureScalingMethod", None)
+                if custom_scaling_method is None:
+                    self._display_msg(inline_msg="No information provided for customized scaling method. AutoML will continue with default option.",
+                                    progress_bar=self.progress_bar)
                 else:
-                    self._scale_method_reg = custom_scaling_method
+                    if self.is_classification_type():
+                        self._scale_method_cls = custom_scaling_method
+                    else:
+                        self._scale_method_reg = custom_scaling_method
         else:
             self._display_msg(inline_msg="No information provided for performing customized feature scaling. Proceeding with default option.",
                               progress_bar=self.progress_bar)
 
     
-    def _handle_generated_features(self, 
-                                   label = None):
+    def _handle_generated_features(self):
         """
         DESCRIPTION:
             Function to handle newly generated float features. It will round them upto 4 digit after decimal point.
-            
-        PARAMETERS:
-            label:
-                Optional Argument.
-                Specifies label for dataset on which rounding up is getting done i.e., 'train' for training 
-                and 'test' for testing dataset.
-                By Default, it takes None and transformation is getting applied to whole dataset.
-                Types: str
-                
+        
+        RETURNS:
+            Pandas DataFrame containing, rounded up float columns.
         """
-        # Checking for label and accordingly deciding target dataset.
-        if label == 'train':
-            target_df = self.train_df
-        elif label == 'test':
-            target_df = self.test_df
-        else:
-            target_df=self.data
-            
+        # Assigning data to target dataframe
+        target_df = self.data            
         # Detecting list of float columns on target dataset
         float_columns =[col for col, d_type in target_df._column_names_and_types if d_type in ["float"]]
         
@@ -988,6 +932,19 @@ class _DataPreparation:
             "accumulate" : accumulate_columns,
             "persist" : True}
         
-        obj = RoundColumns(**fit_params).result
-        df = obj.to_pandas()
-        return df.reset_index()
+        # Disabling print if persist is True by default
+        if not self.volatile and not self.persist:
+            fit_params["display_table_name"] = False
+        
+        if self.volatile:
+            fit_params["volatile"] = True
+            fit_params["persist"] = False
+
+        transform_output = RoundColumns(**fit_params).result
+        if not self.volatile and not self.persist:
+            # Adding transformed data containing table to garbage collector
+            GarbageCollector._add_to_garbagecollector(transform_output._table_name)
+        cols = transform_output.columns
+        df = transform_output.to_pandas().reset_index()
+        df = df[cols]
+        return df