teradataml 20.0.0.3__py3-none-any.whl → 20.0.0.5__py3-none-any.whl

teradataml/automl/feature_engineering.py

@@ -41,6 +41,8 @@ from teradataml.dataframe.sql_functions import case
 from teradataml.hyperparameter_tuner.utils import _ProgressBar
 from teradataml.utils.validators import _Validators
 from teradataml.common.utils import UtilFuncs
+from teradataml.common.constants import TeradataConstants
+from teradataml.options.configure import configure
 
 
 class _FeatureEngineering:
@@ -132,8 +134,12 @@ class _FeatureEngineering:
         self.data_transform_dict = {}
         self.one_hot_obj_count = 0
         self.is_classification_type = lambda: self.task_type.upper() == 'CLASSIFICATION'
-        self.volatile = kwargs.get('volatile', False)
         self.persist = kwargs.get('persist', False)
+        self.volatile = kwargs.get('volatile', False) or (configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE and self.persist is False)
+
+        self.data_mapping = {}
+        self.progress_prefix = kwargs.get('progress_prefix', None)
+        self.aml_phases = kwargs.get('automl_phases', None)
     
     # Method for doing feature engineering on data -> adding id, removing futile col, imputation, encoding(one hot)
     def feature_engineering(self, 
@@ -159,7 +165,7 @@ class _FeatureEngineering:
              second element represents list of columns which are not participating in outlier tranformation.
         """
         # Assigning number of base jobs for progress bar.
-        base_jobs = 13 if auto else 17
+        base_jobs = 12 if auto else 17
         
         # Updating model list based on distinct value of target column for classification type
         if self.is_classification_type():
@@ -169,10 +175,14 @@ class _FeatureEngineering:
 
         # Updating number of jobs for progress bar based on number of models.
         jobs = base_jobs + len(self.model_list)
-        self.progress_bar = _ProgressBar(jobs=jobs, verbose=2, prefix='Automl Running:')
+        self.progress_bar = _ProgressBar(jobs=jobs, 
+                                         verbose=2, 
+                                         prefix=self.progress_prefix) 
 
         self._display_heading(phase=1, 
-                              progress_bar=self.progress_bar)
+                              progress_bar=self.progress_bar,
+                              automl_phases=self.aml_phases)
+        
         self._display_msg(msg='Feature Engineering started ...', 
                           progress_bar=self.progress_bar)
         
@@ -236,7 +246,7 @@ class _FeatureEngineering:
             self._non_linear_transformation()
             self.progress_bar.update()
                
-        return self.data, self.excluded_cols, self.target_label, self.data_transform_dict
+        return self.data, self.excluded_cols, self.target_label, self.data_transform_dict, self.data_mapping
     
     def _extract_list(self,
                       list1,
@@ -260,6 +270,11 @@ class _FeatureEngineering:
             Returns extracted elements in form of list.
             
         """
+        # Ensure list1 and list2 are lists, default to empty list if None
+        if list1 is None:
+            list1 = []
+        if list2 is None:
+            list2 = []
         new_lst = list(set(list1) - set(list2))
         return new_lst
     
@@ -348,12 +363,10 @@ class _FeatureEngineering:
 
         # Detecting and removing futile columns, if categorical_column exists
         if len(categorical_columns) != 0:
-            
             obj = CategoricalSummary(data=self.data, 
                                      target_columns=categorical_columns,
                                      volatile=self.volatile,
                                      persist=self.persist)
-            
             gfc_out = GetFutileColumns(data=self.data, 
                                        object=obj, 
                                        category_summary_column="ColumnName",
@@ -363,6 +376,9 @@ class _FeatureEngineering:
             
             # Extracting Futile columns
             f_cols = [row[0] for row in gfc_out.result.itertuples()]
+
+            self.data_mapping['categorical_summary'] = obj.result._table_name
+            self.data_mapping['futile_columns'] = gfc_out.result._table_name 
             
             if len(f_cols) == 0:
                 self._display_msg(inline_msg="Analysis indicates all categorical columns are significant. No action Needed.",
@@ -372,6 +388,15 @@ class _FeatureEngineering:
                 self.data = self.data.drop(f_cols, axis=1)
                 # Storing futile column list in data transform dictionary
                 self.data_transform_dict['futile_columns'] = f_cols
+
+                if self.persist:
+                    table_name = UtilFuncs._generate_temp_table_name(table_type=TeradataConstants.TERADATA_TABLE,
+                                                                     gc_on_quit=False)
+                    self.data.to_sql(table_name)
+                else:
+                    self.data.materialize()
+
+                self.data_mapping['data_without_futile_columns'] = self.data._table_name
                 self._display_msg(msg='Removing Futile columns:',
                                   col_lst=f_cols,
                                   progress_bar=self.progress_bar)
@@ -547,6 +572,13 @@ class _FeatureEngineering:
             # Storing date column list in data transform dictionary
             self.data_transform_dict['date_columns'] = self.date_column_list
             self._handle_date_columns_helper()
+            if self.persist:
+                table_name = UtilFuncs._generate_temp_table_name(table_type=TeradataConstants.TERADATA_TABLE,
+                                                                 gc_on_quit=False)
+                self.data.to_sql(table_name)
+            else:
+                self.data.materialize()
+            self.data_mapping['data_after_date_handling'] = self.data._table_name
         
         end_time = time.time()
         self._display_msg(msg="Total time to handle date features: {:.2f} sec\n".format(end_time-start_time),
@@ -760,6 +792,9 @@ class _FeatureEngineering:
                                        persist=self.persist)
             
             self.data = sm.result
+            self.data_mapping['fit_simpleimpute_output'] = fit_obj.output_data._table_name 
+            self.data_mapping['fit_simpleimpute_result'] = fit_obj.output._table_name
+            self.data_mapping['data_without_missing_values'] = self.data._table_name
             self._display_msg(msg="Sample of dataset after Imputation:",
                               data=self.data,
                               progress_bar=self.progress_bar)
@@ -872,6 +907,11 @@ class _FeatureEngineering:
                             transform_param["persist"] = False
                         # Updating dataset with transform result
                         self.data = SimpleImputeTransform(**transform_param).result
+
+                        self.data_mapping['fit_simpleimpute_output'] = fit_obj.output_data._table_name 
+                        self.data_mapping['fit_simpleimpute_result'] = fit_obj.output._table_name
+                        self.data_mapping['data_without_missing_values'] = self.data._table_name
+
                         if not volatile and not persist:
                             # Adding transformed data containing table to garbage collector
                             GarbageCollector._add_to_garbagecollector(self.data._table_name)                 
@@ -978,6 +1018,10 @@ class _FeatureEngineering:
                     if not volatile and not persist:
                         # Adding transformed data containing table to garbage collector
                         GarbageCollector._add_to_garbagecollector(self.data._table_name)
+
+                    self.data_mapping['fit_eql_width'] = eql_bin_code_fit.output._table_name
+                    self.data_mapping['eql_width_bincoded_data'] = self.data._table_name
+                    
                     self._display_msg(msg="\nUpdated dataset sample after performing Equal-Width binning :-",
                                       data=self.data,
                                       progress_bar=self.progress_bar)
@@ -1026,6 +1070,8 @@ class _FeatureEngineering:
                         var_transform_params["volatile"] = True
                         var_transform_params["persist"] = False
                     self.data = BincodeTransform(**var_transform_params).result
+                    self.data_mapping['fit_var_width'] = var_bin_code_fit.output._table_name
+                    self.data_mapping['var_width_bincoded_data'] = self.data._table_name
                     if not volatile and not persist:
                         # Adding transformed data containing table to garbage collector
                         GarbageCollector._add_to_garbagecollector(self.data._table_name)
@@ -1147,6 +1193,7 @@ class _FeatureEngineering:
         if not volatile and not persist:
             # Adding transformed data containing table to garbage collector
             GarbageCollector._add_to_garbagecollector(transform_output._table_name)
+        self.data_mapping['string_manipulated_data'] = transform_output._table_name
         return transform_output
               
     def _one_hot_encoding(self, 
@@ -1216,6 +1263,9 @@ class _FeatureEngineering:
             # Adding transformed data containing table to garbage collector
             GarbageCollector._add_to_garbagecollector(transform_output._table_name)
         self.data = transform_output.drop(drop_lst, axis=1)
+        self.data.materialize()
+        self.data_mapping['one_hot_encoded_data'] = transform_output._table_name
+        self.data_mapping['fit_ohe_result'] = fit_obj.result._table_name
 
     def _ordinal_encoding(self,
                           ordinal_columns):
@@ -1273,6 +1323,10 @@ class _FeatureEngineering:
             # Adding transformed data containing table to garbage collector
             GarbageCollector._add_to_garbagecollector(self.data._table_name)
         
+        self.data_mapping['fit_ordinal_output'] = ord_fit_obj.output_data._table_name
+        self.data_mapping['fit_ordinal_result'] = ord_fit_obj.result._table_name
+        self.data_mapping['ordinal_encoded_data'] = self.data._table_name
+        
         if len(ordinal_columns) == 1 and ordinal_columns[0] == self.target_column:
             self.target_label = ord_fit_obj
         
@@ -1319,6 +1373,7 @@ class _FeatureEngineering:
                 "encoder_method" : encoder_method,
                 "target_columns" : col,
                 "response_column" : response_column,
+                "default_values": -1,
                 "volatile" : volatile,
                 "persist" : persist
                 }
@@ -1352,6 +1407,9 @@ class _FeatureEngineering:
             if not volatile and not persist:
                 # Adding transformed data containing table to garbage collector
                 GarbageCollector._add_to_garbagecollector(self.data._table_name)
+            self.data_mapping[f'fit_{col}_target_output'] = tar_fit_obj.output_data._table_name
+            self.data_mapping[f'fit_{col}_target_result'] = tar_fit_obj.result._table_name
+            self.data_mapping[f'{col}_target_encoded_data'] = self.data._table_name
     
     def _encoding_categorical_columns(self):
         """
@@ -1584,6 +1642,9 @@ class _FeatureEngineering:
         if not volatile and not persist:
             # Adding transformed data containing table to garbage collector
             GarbageCollector._add_to_garbagecollector(self.data._table_name)
+        
+        self.data_mapping['fit_numerical_result'] = num_fit_obj.result._table_name
+        self.data_mapping['numerical_transformed_data'] = self.data._table_name
         self._display_msg(msg="Updated dataset sample after applying numerical transformation:",
                           data=self.data,
                           progress_bar=self.progress_bar)
@@ -1624,6 +1685,7 @@ class _FeatureEngineering:
                     if apply_method in (["sininv","sigmoid"]):
                         # Applying numapply transformation
                         self.data = self._numapply_transformation(col,transform_val)
+                        self.data_mapping[f'{apply_method}_transformed_data'] = self.data._table_name
                         self._display_msg(msg="Updated dataset sample after applying numapply transformation:",
                                           data=self.data,
                                           progress_bar=self.progress_bar)
@@ -1728,6 +1790,10 @@ class _FeatureEngineering:
                             transform_params["persist"] = False
                         self.data = NonLinearCombineTransform(**transform_params).result
 
+                        self.data_mapping[f'fit_nonlinear_{comb}_output'] = fit_obj.output_data._table_name
+                        self.data_mapping[f'fit_nonlinear_{comb}_result'] = fit_obj.result._table_name
+                        self.data_mapping['non_linear_transformed_data'] = self.data._table_name
+
                         if not volatile and not persist:
                             # Adding transformed data containing table to garbage collector
                             GarbageCollector._add_to_garbagecollector(self.data._table_name)
@@ -1810,10 +1876,11 @@ class _FeatureEngineering:
         RETURNS:
             Tuple containing volatile and persist parameters.
         """
-        volatile = self.volatile
+        # Prioritizing persist argument and then volatile
         persist = self.persist
+        volatile = self.volatile or (configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE and persist is False)
         if self.custom_data is not None and self.custom_data.get(func_indicator, False):
             volatile = self.custom_data[param_name].get("volatile", False)
             persist = self.custom_data[param_name].get("persist", False)
 
-        return (volatile, persist)
+        return (volatile, persist)