teradataml 20.0.0.6__py3-none-any.whl → 20.0.0.7__py3-none-any.whl

teradataml/automl/data_preparation.py

@@ -1,6 +1,6 @@
 # ################################################################## 
 # 
-# Copyright 2024 Teradata. All rights reserved.
+# Copyright 2025 Teradata. All rights reserved.
 # TERADATA CONFIDENTIAL AND TRADE SECRET
 # 
 # Primary Owner: Sweta Shaw
@@ -120,6 +120,26 @@ class _DataPreparation:
                         Specifies the random seed for reproducibility.
                         Default Value: 42
                         Types: int
+                    
+                    automl_phases:
+                        Optional Argument.
+                        Specifies the phase of AutoML to be executed.
+                        Default Value: None
+                        Types: str or list of str.
+                    
+                    cluster:
+                        Optional Argument.
+                        Specifies whether to run data preparation for handling clustering.
+                        Default Value: False
+                        Types: bool
+
+                    imbalance_handling_method:
+                        Optional Argument.
+                        Specifies which imbalance handling method to use.
+                        Default Value: "SMOTE"
+                        Permitted Values: "SMOTE", "ADASYN", "SMOTETomek", "NearMiss"
+                        Types: str
+
         """
         self.data = data
         self.target_column = target_column
@@ -131,11 +151,13 @@ class _DataPreparation:
         self.volatile = kwargs.get("volatile", False)
         self.persist = kwargs.get("persist", False)
         self.aml_phases = kwargs.get("automl_phases", None)
+        self.cluster = kwargs.get('cluster', False)
+        self._data_sampling_method = kwargs.get("imbalance_handling_method", "SMOTE")
         
         # Setting default value for auto run mode
-        self._data_sampling_method = "SMOTE"
         self._scale_method_reg = "STD"
         self._scale_method_cls = "RANGE"
+        self._scale_method_clust = "STD"
         
         self.data_types = {key: value for key, value in self.data._column_names_and_types}
         self.seed = kwargs.get("seed", 42)
@@ -147,9 +169,8 @@ class _DataPreparation:
   
         self.data_mapping = kwargs.get("data_mapping", {})
         
-        
     def data_preparation(self, 
-                         auto = True):
+                         auto=True):
         """
         DESCRIPTION:
             Function to perform following tasks:-
@@ -178,42 +199,50 @@ class _DataPreparation:
             self._set_custom_scaling_method()
             self._set_custom_sampling()
         
-        # Handling ouliers in dataset
-        self._handle_outliers(auto)
-        self.progress_bar.update()
-        
         # Handling float type features before processing with feature selection and scaling
         training_data = self._handle_generated_features()
         self.progress_bar.update()
-        
+
+        # Handling ouliers in dataset
+        self._handle_outliers(auto)
+        self.progress_bar.update()
+
         # Temporary Pulling data for feature selection 
         # Will change after sto
         
         # Checking for data imbalance    
-        if self._check_data_imbalance(training_data):
-            training_data = self._data_sampling(training_data)
+        if not self.cluster:
+            if self._check_data_imbalance(training_data):
+                training_data = self._data_sampling(training_data)
         self.progress_bar.update()
         
         # Sorting the data based on id to 
         # remove any shuffling done by sampling
         training_data = training_data.sort_values(by='id')
 
-        # Performing feature selection using lasso followed by scaling  
-        self._feature_selection_Lasso(training_data)
-        self._scaling_features(feature_selection_mtd="lasso")
-        self.progress_bar.update()
-        
-        # Performing feature selection using rfe followed by scaling      
-        self._feature_selection_RFE(training_data)
-        self._scaling_features(feature_selection_mtd="rfe")
-        self.progress_bar.update()
-        
-        # Performing scaling followed by feature selection using pca      
+        if not self.cluster:
+            # Performing feature selection using lasso followed by scaling  
+            self._feature_selection_Lasso(training_data)
+            self._scaling_features(feature_selection_mtd="lasso")
+            self.progress_bar.update()
+            
+            # Performing feature selection using rfe followed by scaling      
+            self._feature_selection_RFE(training_data)
+            self._scaling_features(feature_selection_mtd="rfe")
+            self.progress_bar.update()
+        else:
+            self._scaling_features(feature_selection_mtd="Non_pca")
+            self.progress_bar.update()
+
+        # Performing scaling followed by feature selection using pca     
         self._scaling_features(feature_selection_mtd="pca")
         self._feature_selection_PCA()
         self.progress_bar.update()
-        
-        return [self.rfe_feature, self.lasso_feature, self.pca_feature], self.data_transform_dict, self.data_mapping
+
+        if not self.cluster:
+            return [self.rfe_feature, self.lasso_feature, self.pca_feature], self.data_transform_dict, self.data_mapping
+        else:
+            return [self.non_pca_feature, self.pca_feature], self.data_transform_dict, self.data_mapping
 
     def _handle_outliers(self,
                          auto):
@@ -280,19 +309,23 @@ class _DataPreparation:
         if len(outlier_columns) != 0:
             # Detecting outlier percentage in each columns
             outlier_percentage_df = self._outlier_detection(outlier_method, outlier_columns)
-            
+
             # Outlier Handling techniques
             for i in outlier_percentage_df.itertuples():
                 # Column Name
                 col = i[0]
                 # Outlier value
                 value = i[1]
-                # Dropping rows
-                if value > 0.0  and value <= 8.0 :
-                    columns_to_drop_rows.append(col)
-                elif value> 8.0 and value <= 25.0:
-                    columns_to_impute.append(col)
-        
+                if self.cluster:
+                    if value > 0.0:
+                        columns_to_impute.append(col)
+                else:
+                    # Dropping rows
+                    if value > 0.0  and value <= 8.0 :
+                        columns_to_drop_rows.append(col)
+                    elif value> 8.0 and value <= 25.0:
+                        columns_to_impute.append(col)
+            
         return columns_to_drop_rows, columns_to_impute
 
     def _outlier_handling(self,
@@ -325,7 +358,7 @@ class _DataPreparation:
         """
 
         # Setting volatile and persist parameters for Outlier handling function
-        volatile, persist = self._set_generic_parameters(func_indicator='OutlierFilterIndicator',
+        volatile, persist = self._get_generic_parameters(func_indicator='OutlierFilterIndicator',
                                                          param_name='OutlierFilterParam')
 
         # Performing fit on dataset for outlier handling
@@ -431,17 +464,17 @@ class _DataPreparation:
             # List of columns for outlier processing.
             target_columns = [col for col in self.data.columns if col not in self.excluded_columns]
             # Checking user input for outlier detection method
-            outlier_method = self.custom_data.get("OutlierDetectionMethod", None)
+            outlier_method = self.custom_data.get("OutlierFilterMethod", None)
             if outlier_method == 'PERCENTILE':
                 lower_percentile = self.custom_data.get("OutlierLowerPercentile", None)
                 upper_percentile = self.custom_data.get("OutlierUpperPercentile", None)
                 if lower_percentile and upper_percentile:
                     # Detecting outlier percentage for each columns
-                    outlier_df = self._outlier_detection(outlier_method, target_columns, \
-                        lower_percentile, upper_percentile)
+                    outlier_df = self._outlier_detection(outlier_method=outlier_method, column_list=target_columns, \
+                        lower_percentile=lower_percentile, upper_percentile=upper_percentile)
             else:
                 # Detecting outlier percentage for each column in case of other than percentile method
-                outlier_df = self._outlier_detection(outlier_method, target_columns)
+                outlier_df = self._outlier_detection(outlier_method=outlier_method, column_list=target_columns)
                                               
             # Checking for rows if outlier containing columns exist
             if outlier_df.shape[0]:
@@ -462,6 +495,8 @@ class _DataPreparation:
                         self.data_mapping[f'fit_{target_col}_outlier_output'] = fit_obj.output_data._table_name
                         self.data_mapping[f'fit_{target_col}_outlier_result'] = fit_obj.result._table_name
                         self.data_mapping[f'{target_col}_outlier_treated_data'] = self.data._table_name
+                        self._display_msg(msg="Sample of dataset after performing custom outlier filtering",
+                                          data=self.data,progress_bar=self.progress_bar)
                 else:
                     self._display_msg(inline_msg="No information provided for feature transformation in outlier handling.",
                                       progress_bar=self.progress_bar)
@@ -472,7 +507,8 @@ class _DataPreparation:
             self._display_msg(inline_msg="No information provided for customized outlier processing. AutoML will proceed with default settings.",
                               progress_bar=self.progress_bar)
             # Performing default handling for outliers
-            self._outlier_processing()
+            if not self.cluster:
+                self._outlier_processing()
 
     # function for getting value of "K" in k folds cross validation
     def _num_of_folds(self, rows=None):
@@ -509,7 +545,10 @@ class _DataPreparation:
         pca_train = DataFrame.from_table(self.data_mapping['pca_train']).to_pandas()
  
         # Drop unnecessary columns and store the result
-        train_data = pca_train.drop(columns=['id', self.target_column], axis=1)
+        if not self.cluster:
+            train_data = pca_train.drop(columns=['id', self.target_column], axis=1)
+        else:
+            train_data = pca_train.drop(columns=['id'], axis=1)
 
         # Initialize and fit PCA
         pca = PCA(random_state=self.seed)
@@ -545,9 +584,11 @@ class _DataPreparation:
         train_df = pd.concat([pca_train.reset_index(drop=True)['id'], train_df.reset_index(drop=True)], axis=1)
         
         # merging target column with new data
-        train_df[self.target_column] = pca_train[self.target_column].reset_index(drop=True)
-
-        self.pca_feature = train_df.drop(columns=['id',self.target_column],axis=1).columns.tolist()
+        if not self.cluster:
+            train_df[self.target_column] = pca_train[self.target_column].reset_index(drop=True)
+            self.pca_feature = train_df.drop(columns=['id', self.target_column], axis=1).columns.tolist()
+        else:
+            self.pca_feature = train_df.drop(columns=['id'], axis=1).columns.tolist()
         
         self._display_msg(msg="PCA columns:",
                           col_lst=self.pca_feature,
@@ -820,8 +861,12 @@ class _DataPreparation:
             self.lasso_feature = columns_to_scale
         elif feature_selection_mtd == "rfe":
             self.rfe_feature = columns_to_scale
-        else:
+        elif feature_selection_mtd == "pca":
             self.pca_feature = columns_to_scale
+        elif feature_selection_mtd == "raw_scaled":
+            self.raw_scaled_feature = columns_to_scale
+        else:
+            self.non_pca_feature = columns_to_scale
         
         columns_to_scale = [col for col in columns_to_scale if col not in self.excluded_columns]
         return columns_to_scale
@@ -839,7 +884,8 @@ class _DataPreparation:
                 Specifies the feature selection algorithm used.
                 Types: str 
         """
-
+        
+        feature_selection_mtd = feature_selection_mtd.lower()
         self._display_msg(msg="\nscaling Features of {} data ...".format(feature_selection_mtd),
                           progress_bar=self.progress_bar,
                           show_data=True)
@@ -847,21 +893,26 @@ class _DataPreparation:
         start_time = time.time()
         data_to_scale = None
         
-        if self.is_classification_type():
-            scale_method = self._scale_method_cls
+        if not self.cluster:
+            if self.is_classification_type():
+                scale_method = self._scale_method_cls
+            else:
+                scale_method = self._scale_method_reg
         else:
-            scale_method = self._scale_method_reg
-        
+            scale_method = self._scale_method_clust
+
         # Loading data for feature scaling based of feature selection method
         if feature_selection_mtd == 'rfe':
             data_to_scale = DataFrame(self.data_mapping['rfe_train'])
         elif feature_selection_mtd == 'lasso':
             data_to_scale = DataFrame(self.data_mapping['lasso_train'])
+        elif feature_selection_mtd == 'raw_scaled':
+            data_to_scale = DataFrame(self.data_mapping['raw_scaled_train'])
         else:
             data_to_scale = self.data
 
         # Setting volatile and persist parameters for ScaleFit and ScaleTransform functions
-        volatile, persist = self._set_generic_parameters(func_indicator='FeatureScalingIndicator',
+        volatile, persist = self._get_generic_parameters(func_indicator='FeatureScalingIndicator',
                                                          param_name='FeatureScalingParam')
 
         # List of columns that will be scaled
@@ -881,14 +932,13 @@ class _DataPreparation:
             
             self.data_mapping[f'fit_scale_{feature_selection_mtd}_output'] = fit_obj.output_data._table_name
             self.data_mapping[f'fit_scale_{feature_selection_mtd}_result'] = fit_obj.output._table_name
-            
+
             # storing the scale fit object and columns in data transformation dictionary
             self.data_transform_dict['{}_scale_fit_obj'.format(feature_selection_mtd)] = fit_obj.output
             self.data_transform_dict['{}_scale_col'.format(feature_selection_mtd)] = scale_col
             
             # List of columns to copy to the output generated by scale transform
             accumulate_cols = list(set(data_to_scale.columns) - set(scale_col))
-
             
             # Scaling dataset
             transform_obj = ScaleTransform(data=data_to_scale,
@@ -907,6 +957,11 @@ class _DataPreparation:
 
         self.copy_dataframe_to_sql(scaled_df, feature_selection_mtd, persist)
 
+        if self.cluster and feature_selection_mtd == "non_pca":
+            self.data_mapping["non_pca_train"] = scaled_df._table_name
+        elif self.cluster and feature_selection_mtd == "raw_scaled":
+            self.data_mapping["raw_scaled_train"] = scaled_df._table_name
+
         end_time = time.time()
         self._display_msg(msg="Total time taken by feature scaling: {:.2f} sec".format( end_time - start_time),
                           progress_bar=self.progress_bar,
@@ -930,7 +985,9 @@ class _DataPreparation:
                     self._display_msg(inline_msg="No information provided for customized scaling method. AutoML will continue with default option.",
                                     progress_bar=self.progress_bar)
                 else:
-                    if self.is_classification_type():
+                    if self.cluster:
+                        self._scale_method_cluster = custom_scaling_method
+                    elif self.is_classification_type():
                         self._scale_method_cls = custom_scaling_method
                     else:
                         self._scale_method_reg = custom_scaling_method
@@ -943,7 +1000,7 @@ class _DataPreparation:
         """
         DESCRIPTION:
             Function to handle newly generated float features. It will round them upto 4 digit after decimal point.
-        
+
         RETURNS:
             Pandas DataFrame containing, rounded up float columns.
         """
@@ -951,7 +1008,7 @@ class _DataPreparation:
         target_df = self.data            
         # Detecting list of float columns on target dataset
         float_columns =[col for col, d_type in target_df._column_names_and_types if d_type in ["float", "decimal.Decimal"]]
-        
+
         if len(float_columns) == 0:
             cols = target_df.columns
             # Doing reset index to get index column
@@ -960,10 +1017,8 @@ class _DataPreparation:
             # Returning the dataframe with cols
             # to avoid extra columns generated by reset_index()
             return df[cols]
-        
         # storing the column details for round up in data transformation dictionary
         self.data_transform_dict["round_columns"] = float_columns
-        
         # Extracting accumulate columns
         accumulate_columns = self._extract_list(target_df.columns,float_columns)
         # Performing rounding up on target column upto 4 precision digit
@@ -973,11 +1028,11 @@ class _DataPreparation:
             "precision_digit" : 4,
             "accumulate" : accumulate_columns,
             "persist" : True}
-        
+
         # Disabling print if persist is True by default
         if not self.volatile and not self.persist:
             fit_params["display_table_name"] = False
-        
+       
         if self.volatile:
             fit_params["volatile"] = True
             fit_params["persist"] = False
@@ -990,4 +1045,4 @@ class _DataPreparation:
         cols = transform_output.columns
         df = transform_output.to_pandas().reset_index()
         df = df[cols]
-        return df
+        return df