teradataml 20.0.0.4__py3-none-any.whl → 20.0.0.5__py3-none-any.whl

teradataml/automl/feature_engineering.py

@@ -137,6 +137,9 @@ class _FeatureEngineering:
         self.persist = kwargs.get('persist', False)
         self.volatile = kwargs.get('volatile', False) or (configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE and self.persist is False)
 
+        self.data_mapping = {}
+        self.progress_prefix = kwargs.get('progress_prefix', None)
+        self.aml_phases = kwargs.get('automl_phases', None)
     
     # Method for doing feature engineering on data -> adding id, removing futile col, imputation, encoding(one hot)
     def feature_engineering(self, 
@@ -162,7 +165,7 @@ class _FeatureEngineering:
              second element represents list of columns which are not participating in outlier tranformation.
         """
         # Assigning number of base jobs for progress bar.
-        base_jobs = 13 if auto else 17
+        base_jobs = 12 if auto else 17
         
         # Updating model list based on distinct value of target column for classification type
         if self.is_classification_type():
@@ -172,10 +175,14 @@ class _FeatureEngineering:
 
         # Updating number of jobs for progress bar based on number of models.
         jobs = base_jobs + len(self.model_list)
-        self.progress_bar = _ProgressBar(jobs=jobs, verbose=2, prefix='Automl Running:')
+        self.progress_bar = _ProgressBar(jobs=jobs, 
+                                         verbose=2, 
+                                         prefix=self.progress_prefix) 
 
         self._display_heading(phase=1, 
-                              progress_bar=self.progress_bar)
+                              progress_bar=self.progress_bar,
+                              automl_phases=self.aml_phases)
+        
         self._display_msg(msg='Feature Engineering started ...', 
                           progress_bar=self.progress_bar)
         
@@ -239,7 +246,7 @@ class _FeatureEngineering:
             self._non_linear_transformation()
             self.progress_bar.update()
                
-        return self.data, self.excluded_cols, self.target_label, self.data_transform_dict
+        return self.data, self.excluded_cols, self.target_label, self.data_transform_dict, self.data_mapping
     
     def _extract_list(self,
                       list1,
@@ -369,6 +376,9 @@ class _FeatureEngineering:
             
             # Extracting Futile columns
             f_cols = [row[0] for row in gfc_out.result.itertuples()]
+
+            self.data_mapping['categorical_summary'] = obj.result._table_name
+            self.data_mapping['futile_columns'] = gfc_out.result._table_name 
             
             if len(f_cols) == 0:
                 self._display_msg(inline_msg="Analysis indicates all categorical columns are significant. No action Needed.",
@@ -378,6 +388,15 @@ class _FeatureEngineering:
                 self.data = self.data.drop(f_cols, axis=1)
                 # Storing futile column list in data transform dictionary
                 self.data_transform_dict['futile_columns'] = f_cols
+
+                if self.persist:
+                    table_name = UtilFuncs._generate_temp_table_name(table_type=TeradataConstants.TERADATA_TABLE,
+                                                                     gc_on_quit=False)
+                    self.data.to_sql(table_name)
+                else:
+                    self.data.materialize()
+
+                self.data_mapping['data_without_futile_columns'] = self.data._table_name
                 self._display_msg(msg='Removing Futile columns:',
                                   col_lst=f_cols,
                                   progress_bar=self.progress_bar)
@@ -553,6 +572,13 @@ class _FeatureEngineering:
             # Storing date column list in data transform dictionary
             self.data_transform_dict['date_columns'] = self.date_column_list
             self._handle_date_columns_helper()
+            if self.persist:
+                table_name = UtilFuncs._generate_temp_table_name(table_type=TeradataConstants.TERADATA_TABLE,
+                                                                 gc_on_quit=False)
+                self.data.to_sql(table_name)
+            else:
+                self.data.materialize()
+            self.data_mapping['data_after_date_handling'] = self.data._table_name
         
         end_time = time.time()
         self._display_msg(msg="Total time to handle date features: {:.2f} sec\n".format(end_time-start_time),
@@ -766,6 +792,9 @@ class _FeatureEngineering:
                                        persist=self.persist)
             
             self.data = sm.result
+            self.data_mapping['fit_simpleimpute_output'] = fit_obj.output_data._table_name 
+            self.data_mapping['fit_simpleimpute_result'] = fit_obj.output._table_name
+            self.data_mapping['data_without_missing_values'] = self.data._table_name
             self._display_msg(msg="Sample of dataset after Imputation:",
                               data=self.data,
                               progress_bar=self.progress_bar)
@@ -878,6 +907,11 @@ class _FeatureEngineering:
                             transform_param["persist"] = False
                         # Updating dataset with transform result
                         self.data = SimpleImputeTransform(**transform_param).result
+
+                        self.data_mapping['fit_simpleimpute_output'] = fit_obj.output_data._table_name 
+                        self.data_mapping['fit_simpleimpute_result'] = fit_obj.output._table_name
+                        self.data_mapping['data_without_missing_values'] = self.data._table_name
+
                         if not volatile and not persist:
                             # Adding transformed data containing table to garbage collector
                             GarbageCollector._add_to_garbagecollector(self.data._table_name)                 
@@ -984,6 +1018,10 @@ class _FeatureEngineering:
                     if not volatile and not persist:
                         # Adding transformed data containing table to garbage collector
                         GarbageCollector._add_to_garbagecollector(self.data._table_name)
+
+                    self.data_mapping['fit_eql_width'] = eql_bin_code_fit.output._table_name
+                    self.data_mapping['eql_width_bincoded_data'] = self.data._table_name
+                    
                     self._display_msg(msg="\nUpdated dataset sample after performing Equal-Width binning :-",
                                       data=self.data,
                                       progress_bar=self.progress_bar)
@@ -1032,6 +1070,8 @@ class _FeatureEngineering:
                         var_transform_params["volatile"] = True
                         var_transform_params["persist"] = False
                     self.data = BincodeTransform(**var_transform_params).result
+                    self.data_mapping['fit_var_width'] = var_bin_code_fit.output._table_name
+                    self.data_mapping['var_width_bincoded_data'] = self.data._table_name
                     if not volatile and not persist:
                         # Adding transformed data containing table to garbage collector
                         GarbageCollector._add_to_garbagecollector(self.data._table_name)
@@ -1153,6 +1193,7 @@ class _FeatureEngineering:
         if not volatile and not persist:
             # Adding transformed data containing table to garbage collector
             GarbageCollector._add_to_garbagecollector(transform_output._table_name)
+        self.data_mapping['string_manipulated_data'] = transform_output._table_name
         return transform_output
               
     def _one_hot_encoding(self, 
@@ -1222,6 +1263,9 @@ class _FeatureEngineering:
             # Adding transformed data containing table to garbage collector
             GarbageCollector._add_to_garbagecollector(transform_output._table_name)
         self.data = transform_output.drop(drop_lst, axis=1)
+        self.data.materialize()
+        self.data_mapping['one_hot_encoded_data'] = transform_output._table_name
+        self.data_mapping['fit_ohe_result'] = fit_obj.result._table_name
 
     def _ordinal_encoding(self,
                           ordinal_columns):
@@ -1279,6 +1323,10 @@ class _FeatureEngineering:
             # Adding transformed data containing table to garbage collector
             GarbageCollector._add_to_garbagecollector(self.data._table_name)
         
+        self.data_mapping['fit_ordinal_output'] = ord_fit_obj.output_data._table_name
+        self.data_mapping['fit_ordinal_result'] = ord_fit_obj.result._table_name
+        self.data_mapping['ordinal_encoded_data'] = self.data._table_name
+        
         if len(ordinal_columns) == 1 and ordinal_columns[0] == self.target_column:
             self.target_label = ord_fit_obj
         
@@ -1325,6 +1373,7 @@ class _FeatureEngineering:
                 "encoder_method" : encoder_method,
                 "target_columns" : col,
                 "response_column" : response_column,
+                "default_values": -1,
                 "volatile" : volatile,
                 "persist" : persist
                 }
@@ -1358,6 +1407,9 @@ class _FeatureEngineering:
             if not volatile and not persist:
                 # Adding transformed data containing table to garbage collector
                 GarbageCollector._add_to_garbagecollector(self.data._table_name)
+            self.data_mapping[f'fit_{col}_target_output'] = tar_fit_obj.output_data._table_name
+            self.data_mapping[f'fit_{col}_target_result'] = tar_fit_obj.result._table_name
+            self.data_mapping[f'{col}_target_encoded_data'] = self.data._table_name
     
     def _encoding_categorical_columns(self):
         """
@@ -1590,6 +1642,9 @@ class _FeatureEngineering:
         if not volatile and not persist:
             # Adding transformed data containing table to garbage collector
             GarbageCollector._add_to_garbagecollector(self.data._table_name)
+        
+        self.data_mapping['fit_numerical_result'] = num_fit_obj.result._table_name
+        self.data_mapping['numerical_transformed_data'] = self.data._table_name
         self._display_msg(msg="Updated dataset sample after applying numerical transformation:",
                           data=self.data,
                           progress_bar=self.progress_bar)
@@ -1630,6 +1685,7 @@ class _FeatureEngineering:
                     if apply_method in (["sininv","sigmoid"]):
                         # Applying numapply transformation
                         self.data = self._numapply_transformation(col,transform_val)
+                        self.data_mapping[f'{apply_method}_transformed_data'] = self.data._table_name
                         self._display_msg(msg="Updated dataset sample after applying numapply transformation:",
                                           data=self.data,
                                           progress_bar=self.progress_bar)
@@ -1734,6 +1790,10 @@ class _FeatureEngineering:
                             transform_params["persist"] = False
                         self.data = NonLinearCombineTransform(**transform_params).result
 
+                        self.data_mapping[f'fit_nonlinear_{comb}_output'] = fit_obj.output_data._table_name
+                        self.data_mapping[f'fit_nonlinear_{comb}_result'] = fit_obj.result._table_name
+                        self.data_mapping['non_linear_transformed_data'] = self.data._table_name
+
                         if not volatile and not persist:
                             # Adding transformed data containing table to garbage collector
                             GarbageCollector._add_to_garbagecollector(self.data._table_name)