teradataml 20.0.0.0__py3-none-any.whl → 20.0.0.1__py3-none-any.whl

teradataml/automl/data_transformation.py

@@ -31,6 +31,8 @@ from teradataml import ScaleTransform
 from teradataml import SimpleImputeTransform
 from teradataml import TargetEncodingTransform
 from teradataml import Transform, UtilFuncs, TeradataConstants
+from teradataml.common.garbagecollector import GarbageCollector
+from teradataml.hyperparameter_tuner.utils import _ProgressBar
 
 # AutoML Internal libraries
 from teradataml.automl.feature_exploration import _FeatureExplore
@@ -58,12 +60,12 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 Types: teradataml Dataframe
             
             data_transformation_params:
-                Required Arugment.
+                Required Argument.
                 Specifies the parameters for performing data transformation.
                 Types: dict
             
             auto:
-                Optional Arugment.
+                Optional Argument.
                 Specifies whether to run AutoML in custom mode or auto mode.
                 When set to False, runs in custom mode. Otherwise, by default runs in auto mode.
                 Default Value: True
@@ -80,7 +82,7 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 Types: int
             
             target_column_ind:
-                Optional Arugment.
+                Optional Argument.
                 Specifies whether target column is present in given dataset.
                 Default Value: False
                 Types: bool   
@@ -118,6 +120,11 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         # Extracting target column details and type whether it is classification or not
         self.data_target_column = self.data_transformation_params.get("data_target_column")
         self.classification_type = self.data_transformation_params.get("classification_type", False)
+        
+        # Setting number of jobs for progress bar based on mode of execution
+        jobs = 10 if self.auto else 15
+        self.progress_bar = _ProgressBar(jobs=jobs, verbose=2, prefix='Transformation Running:')
+        
         # Performing transformation carried out in feature engineering phase    
         self.feature_engineering_transformation()
         # Performing transformation carried out in data preparation phase
@@ -133,27 +140,52 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             on test data using parameters from data_transformation_params.
         """
         self._display_msg(msg="Performing transformation carried out in feature engineering phase ...", 
-                          show_data=True)
+                          show_data=True,
+                          progress_bar=self.progress_bar)
+
         # Performing default transformation for both auto and custom mode 
         self._preprocess_transformation()
+        self.progress_bar.update()
+        
         self._futile_column_handling_transformation()
+        self.progress_bar.update()
+        
         # Handling target column transformation
         if self.target_column_ind and self.classification_type:
             self._handle_target_column_transformation()
+        self.progress_bar.update()
+
         self._date_column_handling_transformation()
+        self.progress_bar.update()
 
         # Performing transformation according to run mode
         if self.auto:
             self._missing_value_handling_transformation()
+            self.progress_bar.update()
+            
             self._categorical_encoding_transformation()
+            self.progress_bar.update()
         else:
             self._custom_missing_value_handling_transformation()
+            self.progress_bar.update()
+            
             self._custom_bincode_column_transformation()
+            self.progress_bar.update()
+            
             self._custom_string_column_transformation()
+            self.progress_bar.update()
+            
             self._custom_categorical_encoding_transformation()
+            self.progress_bar.update()
+            
             self._custom_mathematical_transformation()
+            self.progress_bar.update()
+            
             self._custom_non_linear_transformation()
+            self.progress_bar.update()
+            
             self._custom_anti_select_column_transformation()
+            self.progress_bar.update()
 
     def data_preparation_transformation(self):
         """
@@ -162,15 +194,23 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             on test data using parameters from data_transformation_params.
         """
         self._display_msg(msg="Performing transformation carried out in data preparation phase ...", 
-                          show_data=True)
+                          show_data=True,
+                          progress_bar=self.progress_bar)
+        
         # Handling features transformed from feature engineering phase
         self._handle_generated_features_transformation()
+        self.progress_bar.update()
 
         # Performing transformation including feature selection using lasso, rfe and pca
         # followed by scaling
         self._feature_selection_lasso_transformation()
+        self.progress_bar.update()
+        
         self._feature_selection_rfe_transformation()
+        self.progress_bar.update()
+        
         self._feature_selection_pca_transformation()
+        self.progress_bar.update()
 
     def _preprocess_transformation(self):
         """
@@ -182,7 +222,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         if columns_to_be_removed:
             self.data = self.data.drop(columns_to_be_removed, axis=1)
             self._display_msg(msg="\nUpdated dataset after dropping irrelevent columns :", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
         # Adding id column
         self.data = FillRowId(data=self.data, row_id_column='id').result
@@ -197,7 +238,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         if futile_cols:
             self.data = self.data.drop(futile_cols, axis=1)
             self._display_msg(msg="\nUpdated dataset after dropping futile columns :", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _date_column_handling_transformation(self):
         """
@@ -205,47 +247,32 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             Function performs transformation on date columns and generates new columns.
         """
         # Extracting date columns
-        date_columns = self.data_transformation_params.get("date_columns",None)
-        if date_columns:
+        self.date_column_list = self.data_transformation_params.get("date_columns",None)
+        if self.date_column_list:
             # Dropping rows with null values in date columns
-            self.data = self.data.dropna(subset=date_columns)
+            self.data = self.data.dropna(subset=self.date_column_list)
             # Extracting unique date columns for dropping
             drop_unique_date_columns = self.data_transformation_params.get("drop_unique_date_columns",None)
             if drop_unique_date_columns:
                 self.data = self.data.drop(drop_unique_date_columns, axis=1)
+                # Updated date column list after dropping irrelevant date columns
+                self.date_column_list = [item for item in self.date_column_list if item not in drop_unique_date_columns]
 
-            # Extracting date components parameters for new columns generation
-            extract_date_comp_param = self.data_transformation_params.get("extract_date_comp_param",None)
-            extract_date_comp_col = self.data_transformation_params.get("extract_date_comp_col", None)
-            if extract_date_comp_param:
-                self.data=self.data.assign(**extract_date_comp_param)
-                self.data = self.data.drop(extract_date_comp_col, axis=1)
-
-            # Extracting irrelevant date component columns for dropping  
-            drop_extract_date_columns = self.data_transformation_params.get("drop_extract_date_columns", None)
-            if drop_extract_date_columns:
-                self.data = self.data.drop(drop_extract_date_columns, axis=1)
-
-            # Extracting date component fit objects for bincode transformation
-            day_component_fit_object = self.data_transformation_params.get("day_component_fit_object", None)
-            month_component_fit_object = self.data_transformation_params.get("month_component_fit_object", None)
-            year_diff_component_fit_object = self.data_transformation_params.get("year_diff_component_fit_object", None)
-
-            # Performing bincode transformation on day, month and year components
-            for fit_object in [day_component_fit_object, month_component_fit_object, year_diff_component_fit_object]:
-                if fit_object:
-                    for col, bin_code_fit in fit_object.items():
-                        accumulate_columns = self._extract_list(self.data.columns, [col])
-                        transform_params = {
-                            "data": self.data,
-                            "object": bin_code_fit,
-                            "accumulate": accumulate_columns,
-                            "persist": True
-                        }
-                        self.data = BincodeTransform(**transform_params).result
-
-            self._display_msg(msg="\nUpdated dataset after transforming date columns :", 
-                              data=self.data)
+            if len(self.date_column_list) != 0:
+                # Extracting date components parameters for new columns generation
+                new_columns=self._fetch_date_component()
+            
+                # Extracting irrelevant date component columns for dropping  
+                drop_extract_date_columns = self.data_transformation_params.get("drop_extract_date_columns", None)
+                if drop_extract_date_columns:
+                    self.data = self.data.drop(drop_extract_date_columns, axis=1)
+                    new_columns = [item for item in new_columns if item not in drop_extract_date_columns]
+                
+                self._display_msg(msg='Updated list of newly generated features from existing date features :',
+                                  col_lst=new_columns)
+                self._display_msg(msg="\nUpdated dataset after transforming date columns :", 
+                                  data=self.data,
+                                  progress_bar=self.progress_bar)
 
     def _missing_value_handling_transformation(self):
         """
@@ -257,7 +284,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         if drop_cols:
             self.data = self.data.drop(drop_cols, axis=1)
             self._display_msg(msg="\nUpdated dataset after dropping missing value containing columns : ", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
         # Extracting imputation columns and fit object for missing value imputation
         imputation_cols = self.data_transformation_params.get("imputation_columns", None)
@@ -265,20 +293,22 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             sm_fit_obj = self.data_transformation_params.get("imputation_fit_object")
             # imputing column using fit object
             self.data = SimpleImputeTransform(data=self.data, 
-                                            object=sm_fit_obj, 
-                                            volatile=True).result
+                                              object=sm_fit_obj).result
             self._display_msg(msg="\nUpdated dataset after imputing missing value containing columns :", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
         # Handling rest null, its temporary solution. It subjects to change based on input.
         dropped_data = self.data.dropna()
         dropped_count = self.data.shape[0] - dropped_data.shape[0]
         if dropped_count > 0:
-            self.data = dropped_data
             self._display_msg(msg="\nFound additional {} rows that contain missing values :".format(dropped_count),
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
+            self.data = dropped_data
             self._display_msg(msg="\nUpdated dataset after dropping additional missing value containing rows :", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _custom_missing_value_handling_transformation(self):
         """
@@ -291,7 +321,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         if drop_col_list:
             self.data = self.data.drop(drop_col_list, axis=1)
             self._display_msg(msg="\nUpdated dataset after dropping customized missing value containing columns :", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
         # Extracting custom imputation columns and fit object for missing value imputation
         custom_imp_ind = self.data_transformation_params.get("custom_imputation_ind", False)  
@@ -299,10 +330,10 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             sm_fit_obj = self.data_transformation_params.get("custom_imputation_fit_object")
             # imputing column using fit object 
             self.data = SimpleImputeTransform(data=self.data, 
-                                            object=sm_fit_obj, 
-                                            volatile=True).result
+                                              object=sm_fit_obj).result
             self._display_msg(msg="\nUpdated dataset after imputing customized missing value containing columns :",
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
         # Handling rest with default missing value handling    
         self._missing_value_handling_transformation()
 
@@ -328,8 +359,11 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     "persist" : True,   
                 }
                 self.data = BincodeTransform(**eql_transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
                 self._display_msg(msg="\nUpdated dataset after performing customized equal width bin-code transformation :",
-                                  data=self.data)
+                                  data=self.data,
+                                  progress_bar=self.progress_bar)
 
             # Hnadling bincode transformation for Variable-Width
             custom_var_bincode_col = self.data_transformation_params.get("custom_var_bincode_col", None)
@@ -346,8 +380,11 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     "persist" : True   
                 }
                 self.data = BincodeTransform(**var_transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
                 self._display_msg(msg="\nUpdated dataset after performing customized variable width bin-code transformation :",
-                                  data=self.data)
+                                  data=self.data,
+                                  progress_bar=self.progress_bar)
 
     def _custom_string_column_transformation(self):
         """
@@ -362,7 +399,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             for target_col,transform_val in custom_string_manipulation_param.items():            
                 self.data = self._str_method_mapping(target_col, transform_val)
             self._display_msg(msg="\nUpdated dataset after performing customized string manipulation :",
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _categorical_encoding_transformation(self):
         """
@@ -384,10 +422,13 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     }
                 # Performing one hot encoding transformation
                 self.data = OneHotEncodingTransform(**transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
             # Dropping old columns after encoding        
             self.data = self.data.drop(one_hot_encoding_drop_list, axis=1)  
             self._display_msg(msg="\nUpdated dataset after performing categorical encoding :",
-                            data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _custom_categorical_encoding_transformation(self):
         """
@@ -412,6 +453,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 }
                 # Performing ordinal encoding transformation
                 self.data = OrdinalEncodingTransform(**transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
             # Extracting parameters for target encoding
             custom_target_encoding_ind = self.data_transformation_params.get("custom_target_encoding_ind", False)
             custom_target_encoding_fit_obj = self.data_transformation_params.get("custom_target_encoding_fit_obj", None)
@@ -426,10 +469,13 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                         "accumulate" : accumulate_columns,
                         "persist" : True                
                     }
-                    # Performing ordinal encoding transformation
+                    # Performing target encoding transformation
                     self.data = TargetEncodingTransform(**transform_params).result
+                    # Adding transformed data containing table to garbage collector
+                    GarbageCollector._add_to_garbagecollector(self.data._table_name)
             self._display_msg(msg="\nUpdated dataset after performing customized categorical encoding :",
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
         # Handling rest with default categorical encoding transformation    
         self._categorical_encoding_transformation()
@@ -472,8 +518,11 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 }
                 # Peforming transformation on target columns
                 self.data = Transform(**transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
             self._display_msg(msg="\nUpdated dataset after performing customized mathematical transformation :",
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _custom_non_linear_transformation(self):
         """
@@ -495,8 +544,11 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 }
                 # Performing transformation
                 self.data = NonLinearCombineTransform(**transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
             self._display_msg(msg="\nUpdated dataset after performing customized non-linear transformation :",
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _custom_anti_select_column_transformation(self):
         """
@@ -516,7 +568,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 # Performing transformation for given user input
                 self.data = Antiselect(**fit_params).result
                 self._display_msg(msg="\nUpdated dataset after performing customized anti-selection :",
-                                  data=self.data)
+                                  data=self.data,
+                                  progress_bar=self.progress_bar)
 
     def _handle_generated_features_transformation(self):
         """
@@ -541,6 +594,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 "accumulate" : accumulate_columns,
                 "persist" : True}
             self.data = RoundColumns(**fit_params).result
+            # Adding transformed data containing table to garbage collector
+            GarbageCollector._add_to_garbagecollector(self.data._table_name)
             
     def _handle_target_column_transformation(self):
         """
@@ -565,7 +620,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 }
                 # Performing ordinal encoding transformation
                 self.data = OrdinalEncodingTransform(**transform_params).result
-                
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
         # Converting target column to integer datatype
         params = {
             "data" : self.data,
@@ -575,7 +631,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         }
         self.data = ConvertTo(**params).result
         self._display_msg(msg="\nUpdated dataset after performing target column transformation :",
-                          data=self.data)
+                          data=self.data,
+                          progress_bar=self.progress_bar)
 
     def _extract_and_display_features(self, feature_type, feature_list):
         """
@@ -605,7 +662,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         
         # Displaying feature dataframe
         self._display_msg(msg=f"\nUpdated dataset after performing {feature_type} feature selection:",
-                        data=feature_df)
+                          data=feature_df,
+                          progress_bar=self.progress_bar)
 
         # Returning feature dataframe
         return feature_df
@@ -631,7 +689,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     accumulate=accumulate_cols).result
         # Displaying scaled dataset
         self._display_msg(msg="\nUpdated dataset after performing scaling on Lasso selected features :",
-                          data=lasso_df)
+                          data=lasso_df,
+                          progress_bar=self.progress_bar)
 
         # Uploading lasso dataset to table for further use
         table_name = UtilFuncs._generate_temp_table_name(prefix="lasso_new_test", 
@@ -667,7 +726,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     accumulate=accumulate_cols).result
         # Displaying scaled dataset
         self._display_msg(msg="\nUpdated dataset after performing scaling on RFE selected features :",
-                          data=rfe_df)
+                          data=rfe_df,
+                          progress_bar=self.progress_bar)        
 
         # Uploading rfe dataset to table for further use
         table_name = UtilFuncs._generate_temp_table_name(prefix="rfe_new_test", 
@@ -691,7 +751,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     accumulate=accumulate_cols).result
         # Displaying scaled dataset
         self._display_msg(msg="\nUpdated dataset after performing scaling for PCA feature selection :",
-                          data=pca_scaled_df)
+                          data=pca_scaled_df,
+                          progress_bar=self.progress_bar)
 
         # Convert to pandas dataframe for applying pca
         pca_scaled_pd = pca_scaled_df.to_pandas()
@@ -718,7 +779,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             pca_df[self.data_target_column] = pca_scaled_pd[self.data_target_column].reset_index(drop=True)
         # Displaying pca dataframe
         self._display_msg(msg="\nUpdated dataset after performing PCA feature selection :",
-                          data=pca_df)
+                          data=pca_df.head(10),
+                          progress_bar=self.progress_bar)
 
         # Uploading pca dataset to table for further use
         table_name = UtilFuncs._generate_temp_table_name(prefix="pca_new_test",