teradataml 20.0.0.6__py3-none-any.whl → 20.0.0.7__py3-none-any.whl

teradataml/automl/feature_engineering.py

@@ -1,6 +1,6 @@
 # ################################################################## 
 # 
-# Copyright 2024 Teradata. All rights reserved.
+# Copyright 2025 Teradata. All rights reserved.
 # TERADATA CONFIDENTIAL AND TRADE SECRET
 # 
 # Primary Owner: Sweta Shaw
@@ -51,9 +51,9 @@ class _FeatureEngineering:
                  data,
                  target_column,
                  model_list,
-                 verbose = 0,
-                 task_type = "Regression",
-                 custom_data = None,
+                 verbose=0,
+                 task_type="Regression",
+                 custom_data=None,
                  **kwargs):
         """
         DESCRIPTION:
@@ -88,10 +88,10 @@ class _FeatureEngineering:
 
             task_type:
                 Required Argument.
-                Specifies the task type for AutoML, whether to apply regresion OR classification
+                Specifies the task type for AutoML, whether to apply regresion OR classification OR clustering
                 on the provived dataset.
                 Default Value: "Regression"
-                Permitted Values: "Regression", "Classification"
+                Permitted Values: "Regression", "Classification", "Clustering"
                 Types: str
 
             custom_data:
@@ -120,6 +120,30 @@ class _FeatureEngineering:
                         session.
                         Default Value: False
                         Types: bool
+
+                    cluster:
+                        Optional Argument.
+                        Specifies whether to apply clustering techniques.
+                        Default Value: False
+                        Types: bool
+                    
+                    progress_prefix:
+                        Optional Argument.
+                        Specifies the prefix for the progress bar messages.
+                        Default Value: None
+                        Types: str.
+
+                    automl_phases:
+                        Optional Argument.
+                        Specifies the phase of AutoML to be executed.
+                        Default Value: None
+                        Types: str or list of str.
+
+                    auto_dataprep:
+                        Optional Argument.
+                        Specifies whether to run AutoDataPrep workflow.
+                        Default Value: False
+                        Types: bool
         """
         # Instance variables
         self.data = data
@@ -131,16 +155,18 @@ class _FeatureEngineering:
         self.excluded_cols=[]
         self.data_types = {key: value for key, value in self.data._column_names_and_types} 
         self.target_label = None
-        self.data_transform_dict = {}
+        
         self.one_hot_obj_count = 0
         self.is_classification_type = lambda: self.task_type.upper() == 'CLASSIFICATION'
         self.persist = kwargs.get('persist', False)
         self.volatile = kwargs.get('volatile', False) or (configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE and self.persist is False)
+        self.cluster = kwargs.get('cluster', False)
 
         self.data_mapping = {}
         self.progress_prefix = kwargs.get('progress_prefix', None)
         self.aml_phases = kwargs.get('automl_phases', None)
-    
+        self.auto_dataprep = kwargs.get('auto_dataprep', False)
+
     # Method for doing feature engineering on data -> adding id, removing futile col, imputation, encoding(one hot)
     def feature_engineering(self, 
                             auto=True):
@@ -165,13 +191,25 @@ class _FeatureEngineering:
              second element represents list of columns which are not participating in outlier tranformation.
         """
         # Assigning number of base jobs for progress bar.
-        base_jobs = 12 if auto else 17
+        if self.cluster:
+            base_jobs = 11 if auto else 15
+        else:
+            base_jobs = 12 if auto else 17
         
         # Updating model list based on distinct value of target column for classification type
         if self.is_classification_type():
             if self.data.drop_duplicate(self.target_column).size > 2:
-                unsupported_models = ['svm', 'glm']
+                unsupported_models = ['svm', 'glm']  # Models that don't support multiclass
+                for model in unsupported_models:
+                    if model in self.model_list:
+                        self._display_msg(inline_msg="\nMulti-class classification is "
+                                          "not supported by {} model. Skipping {} model."
+                                          .format(model, model))
                 self.model_list = [model for model in self.model_list if model not in unsupported_models]
+        
+        # After filtering models like glm/svm due to multiclass
+        if not self.auto_dataprep:
+            _Validators._validate_non_empty_list_or_valid_selection(self.model_list, "List of models")
 
         # Updating number of jobs for progress bar based on number of models.
         jobs = base_jobs + len(self.model_list)
@@ -187,13 +225,24 @@ class _FeatureEngineering:
                           progress_bar=self.progress_bar)
         
         # Storing target column to data transform dictionary
-        self.data_transform_dict['data_target_column'] = self.target_column
+        # Setting target column for supervised learning, for clustering it will be None.
+        if not self.cluster:
+            self.data_transform_dict['data_target_column'] = self.target_column
+        else:
+            self.data_transform_dict['data_target_column'] = None
+    
         # Storing target column encoding indicator to data transform dictionary
-        self.data_transform_dict['target_col_encode_ind'] = False
+        if "target_col_encode_ind" not in self.data_transform_dict:
+            self.data_transform_dict["target_col_encode_ind"] = False
+        
+        
         # Storing task type to data transform dictionary
-        self.data_transform_dict['classification_type']=self.is_classification_type()
+        if not self.cluster:
+            self.data_transform_dict['classification_type'] = self.is_classification_type()
+        else:
+            self.data_transform_dict['classification_type'] = False
         # Storing params for performing one hot encoding
-        self.data_transform_dict['one_hot_encoding_fit_obj'] ={} 
+        self.data_transform_dict['one_hot_encoding_fit_obj'] = {}
         self.data_transform_dict['one_hot_encoding_drop_list'] = []
 
         if auto:
@@ -333,8 +382,8 @@ class _FeatureEngineering:
         # Removing id column, if exists
         if len(columns_to_be_removed) != 0:
             self.data = self.data.drop(columns_to_be_removed, axis=1)
-            # Storing irrelevent column list in data transform dictionary
-            self.data_transform_dict['drop_irrelevent_columns'] = columns_to_be_removed
+            # Storing irrelevant column list in data transform dictionary
+            self.data_transform_dict['drop_irrelevant_columns'] = columns_to_be_removed
             
         # Adding id columns
         obj = FillRowId(data=self.data, row_id_column='id')
@@ -355,18 +404,21 @@ class _FeatureEngineering:
         
         # Handling string type target column in classification
         # Performing Ordinal Encoding 
-        if self.data_types[self.target_column] in ['str']:
-            self._ordinal_encoding([self.target_column])
+        if not self.cluster:
+            if self.data_types[self.target_column] in ['str']:
+                self._ordinal_encoding([self.target_column])
     
         # Detecting categorical columns
         categorical_columns = [col for col, d_type in self.data._column_names_and_types if d_type == 'str']
 
         # Detecting and removing futile columns, if categorical_column exists
         if len(categorical_columns) != 0:
+            
             obj = CategoricalSummary(data=self.data, 
                                      target_columns=categorical_columns,
                                      volatile=self.volatile,
                                      persist=self.persist)
+            
             gfc_out = GetFutileColumns(data=self.data, 
                                        object=obj, 
                                        category_summary_column="ColumnName",
@@ -378,8 +430,8 @@ class _FeatureEngineering:
             f_cols = [row[0] for row in gfc_out.result.itertuples()]
 
             self.data_mapping['categorical_summary'] = obj.result._table_name
-            self.data_mapping['futile_columns'] = gfc_out.result._table_name 
-            
+            self.data_mapping['futile_columns'] = gfc_out.result._table_name
+
             if len(f_cols) == 0:
                 self._display_msg(inline_msg="Analysis indicates all categorical columns are significant. No action Needed.",
                                   progress_bar=self.progress_bar)
@@ -597,7 +649,8 @@ class _FeatureEngineering:
         """
         
         # Removing rows with missing target column value
-        self.data = self.data.dropna(subset=[self.target_column])
+        if not self.cluster:
+            self.data = self.data.dropna(subset=[self.target_column])
 
         params = {
             "data": self.data,
@@ -664,6 +717,11 @@ class _FeatureEngineering:
                 drop_cols.append(col)
                 continue
             
+            # For clustering tasks, all columns with missing values are sent directly to imputation
+            if self.cluster:
+                self.imputation_cols[col] = val
+                continue
+
             if self.data_types[col] in ['float', 'int']:
                 corr_df = self.data[col].corr(self.data[self.target_column])
                 corr_val = self.data.assign(True, corr_=corr_df)
@@ -674,7 +732,7 @@ class _FeatureEngineering:
                 if val < .02*d_size and related <= .25:
                     delete_rows.append(col)
                     continue
-                    
+
             elif self.data_types[col] in ['str']:
                 # Delete row, if count of missing value < 4%
                 if val < .04*d_size:
@@ -806,8 +864,7 @@ class _FeatureEngineering:
         self._display_msg(msg="Time taken to perform imputation: {:.2f} sec  ".format(end_time - start_time),
                           progress_bar=self.progress_bar,
                           show_data=True)
-
-    
+   
     def _custom_handling_missing_value(self):
         """
         DESCRIPTION:
@@ -1001,7 +1058,7 @@ class _FeatureEngineering:
                     # Extracting accumulate columns
                     accumulate_columns = self._extract_list(self.data.columns, equal_width_bin_columns)
                     # Adding transform parameters for performing binning with Equal-Width.
-                    eql_transform_params={
+                    eql_transform_params = {
                         "data" : self.data,
                         "object" : eql_bin_code_fit.output,
                         "accumulate" : accumulate_columns,
@@ -1021,7 +1078,7 @@ class _FeatureEngineering:
 
                     self.data_mapping['fit_eql_width'] = eql_bin_code_fit.output._table_name
                     self.data_mapping['eql_width_bincoded_data'] = self.data._table_name
-                    
+
                     self._display_msg(msg="\nUpdated dataset sample after performing Equal-Width binning :-",
                                       data=self.data,
                                       progress_bar=self.progress_bar)
@@ -1150,7 +1207,7 @@ class _FeatureEngineering:
         string_operation = transform_val["StringOperation"]
 
         # Setting volatile and persist parameters for performing string manipulation
-        volatile, persist = self._set_generic_parameters(func_indicator="StringManipulationIndicator",
+        volatile, persist = self._get_generic_parameters(func_indicator="StringManipulationIndicator",
                                                          param_name="StringManipulationParam")
 
         # Storing general parameters for performing string transformation
@@ -1219,7 +1276,7 @@ class _FeatureEngineering:
         drop_lst = [ele + "_other" for ele in one_hot_columns]
 
         # Setting volatile and persist parameters for performing encoding
-        volatile, persist = self._set_generic_parameters(func_indicator="CategoricalEncodingIndicator",
+        volatile, persist = self._get_generic_parameters(func_indicator="CategoricalEncodingIndicator",
                                                          param_name="CategoricalEncodingParam")
 
         # Adding fit parameters for performing encoding
@@ -1280,7 +1337,7 @@ class _FeatureEngineering:
                 Types: str or list of strings (str)
         """
         # Setting volatile and persist parameters for performing encoding
-        volatile, persist = self._set_generic_parameters(func_indicator="CategoricalEncodingIndicator",
+        volatile, persist = self._get_generic_parameters(func_indicator="CategoricalEncodingIndicator",
                                                          param_name="CategoricalEncodingParam")
 
         # Adding fit parameters for performing encoding
@@ -1326,11 +1383,10 @@ class _FeatureEngineering:
         self.data_mapping['fit_ordinal_output'] = ord_fit_obj.output_data._table_name
         self.data_mapping['fit_ordinal_result'] = ord_fit_obj.result._table_name
         self.data_mapping['ordinal_encoded_data'] = self.data._table_name
-        
+
         if len(ordinal_columns) == 1 and ordinal_columns[0] == self.target_column:
             self.target_label = ord_fit_obj
         
-
     def _target_encoding(self,
                          target_encoding_list):
         """
@@ -1344,22 +1400,22 @@ class _FeatureEngineering:
                   Types: str or list of strings (str)
         """
         # Fetching all columns on which target encoding will be performed.
-        target_columns= list(target_encoding_list.keys())
+        target_columns = list(target_encoding_list.keys())
         # Checking for column present in dataset or not
         _Validators._validate_dataframe_has_argument_columns(target_columns, "TargetEncodingList", self.data, "df")
         # Finding distinct values and counts for columns.
-        cat_sum = CategoricalSummary(data = self.data,
-                                    target_columns = target_columns)
-        category_data=cat_sum.result.groupby("ColumnName").count()
-        category_data = category_data.assign(drop_columns = True,
-                                            ColumnName = category_data.ColumnName,
-                                            CategoryCount = category_data.count_DistinctValue)
+        cat_sum = CategoricalSummary(data=self.data,
+                                    target_columns=target_columns)
+        category_data = cat_sum.result.groupby("ColumnName").count()
+        category_data = category_data.assign(drop_columns=True,
+                                            ColumnName=category_data.ColumnName,
+                                            CategoryCount=category_data.count_DistinctValue)
         # Storing indicator and fit object for target encoding in data transform dictionary
         self.data_transform_dict["custom_target_encoding_ind"] = True
         self.data_transform_dict["custom_target_encoding_fit_obj"] = {}
 
         # Setting volatile and persist parameters for performing encoding
-        volatile, persist = self._set_generic_parameters(func_indicator="CategoricalEncodingIndicator",
+        volatile, persist = self._get_generic_parameters(func_indicator="CategoricalEncodingIndicator",
                                                          param_name="CategoricalEncodingParam")
         
         # Fetching required argument for performing target encoding
@@ -1392,7 +1448,7 @@ class _FeatureEngineering:
                 "data" : self.data,
                 "object" : tar_fit_obj,
                 "accumulate" : accumulate_columns,
-                "persist" : True              
+                "persist" : True  
             }
             
             # Disabling display table name if persist is True by default
@@ -1422,7 +1478,7 @@ class _FeatureEngineering:
         start_time = time.time()
         
         ohe_col = []
-        unique_count  = []
+        unique_count = []
 
         # List of columns before one hot
         col_bf_ohe = self.data.columns
@@ -1487,7 +1543,7 @@ class _FeatureEngineering:
                                       progress_bar=self.progress_bar)
                 else:
                     if onehot_encode_ind:
-                        unique_count  = []
+                        unique_count = []
                         ohe_list = encoding_list.get("OneHotEncodingList", None)
                         # Checking for empty list
                         if not ohe_list:
@@ -1530,16 +1586,20 @@ class _FeatureEngineering:
                                               progress_bar=self.progress_bar)
 
                     if target_encode_ind:
-                        tar_list = encoding_list.get("TargetEncodingList", None)
-                        if not tar_list:
-                            self._display_msg(inline_msg="No information provided for customized target encoding technique.",
-                                              progress_bar=self.progress_bar)
-                        else:    
-                            # Performing target encoding
-                            self._target_encoding(tar_list)
-                            self._display_msg(msg="Updated dataset sample after performing target encoding:",
-                                              data=self.data,
+                        if self.cluster:
+                            self._display_msg(inline_msg="Target Encoding is not applicable for clustering. Skipping it.",
                                               progress_bar=self.progress_bar)
+                        else:
+                            tar_list = encoding_list.get("TargetEncodingList", None)
+                            if not tar_list:
+                                self._display_msg(inline_msg="No information provided for customized target encoding technique.",
+                                                  progress_bar=self.progress_bar)
+                            else:    
+                                # Performing target encoding
+                                self._target_encoding(tar_list)
+                                self._display_msg(msg="Updated dataset sample after performing target encoding:",
+                                                  data=self.data,
+                                                  progress_bar=self.progress_bar)
             else:
                 self._display_msg(inline_msg="No input provided for performing customized categorical encoding. AutoML will proceed with default encoding technique.",
                                   progress_bar=self.progress_bar)               
@@ -1571,7 +1631,7 @@ class _FeatureEngineering:
         apply_method = transform_val["apply_method"]
 
         # Setting volatile and persist parameters for performing transformation
-        volatile, persist = self._set_generic_parameters(func_indicator="MathameticalTransformationIndicator",
+        volatile, persist = self._get_generic_parameters(func_indicator="MathameticalTransformationIndicator",
                                                          param_name="MathameticalTransformationParam")
         # Adding fit parameters for performing transformation
         fit_params={
@@ -1855,7 +1915,7 @@ class _FeatureEngineering:
             self._display_msg(inline_msg="Skipping customized anti-select columns.",
                               progress_bar=self.progress_bar)
             
-    def _set_generic_parameters(self,
+    def _get_generic_parameters(self,
                                 func_indicator=None,
                                 param_name=None):
         """