teradataml 20.0.0.0__py3-none-any.whl → 20.0.0.2__py3-none-any.whl

teradataml/automl/data_transformation.py

@@ -31,6 +31,8 @@ from teradataml import ScaleTransform
 from teradataml import SimpleImputeTransform
 from teradataml import TargetEncodingTransform
 from teradataml import Transform, UtilFuncs, TeradataConstants
+from teradataml.common.garbagecollector import GarbageCollector
+from teradataml.hyperparameter_tuner.utils import _ProgressBar
 
 # AutoML Internal libraries
 from teradataml.automl.feature_exploration import _FeatureExplore
@@ -58,12 +60,12 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 Types: teradataml Dataframe
             
             data_transformation_params:
-                Required Arugment.
+                Required Argument.
                 Specifies the parameters for performing data transformation.
                 Types: dict
             
             auto:
-                Optional Arugment.
+                Optional Argument.
                 Specifies whether to run AutoML in custom mode or auto mode.
                 When set to False, runs in custom mode. Otherwise, by default runs in auto mode.
                 Default Value: True
@@ -80,7 +82,7 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 Types: int
             
             target_column_ind:
-                Optional Arugment.
+                Optional Argument.
                 Specifies whether target column is present in given dataset.
                 Default Value: False
                 Types: bool   
@@ -91,6 +93,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         self.verbose = verbose
         self.target_column_ind = target_column_ind
         self.table_name_mapping = table_name_mapping
+        self.data_node_id = data._nodeid
+        self.table_name_mapping[self.data_node_id] = {}
 
     def data_transformation(self):
         """
@@ -118,6 +122,11 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         # Extracting target column details and type whether it is classification or not
         self.data_target_column = self.data_transformation_params.get("data_target_column")
         self.classification_type = self.data_transformation_params.get("classification_type", False)
+        
+        # Setting number of jobs for progress bar based on mode of execution
+        jobs = 10 if self.auto else 15
+        self.progress_bar = _ProgressBar(jobs=jobs, verbose=2, prefix='Transformation Running:')
+        
         # Performing transformation carried out in feature engineering phase    
         self.feature_engineering_transformation()
         # Performing transformation carried out in data preparation phase
@@ -133,27 +142,52 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             on test data using parameters from data_transformation_params.
         """
         self._display_msg(msg="Performing transformation carried out in feature engineering phase ...", 
-                          show_data=True)
+                          show_data=True,
+                          progress_bar=self.progress_bar)
+
         # Performing default transformation for both auto and custom mode 
         self._preprocess_transformation()
+        self.progress_bar.update()
+        
         self._futile_column_handling_transformation()
+        self.progress_bar.update()
+        
         # Handling target column transformation
         if self.target_column_ind and self.classification_type:
             self._handle_target_column_transformation()
+        self.progress_bar.update()
+
         self._date_column_handling_transformation()
+        self.progress_bar.update()
 
         # Performing transformation according to run mode
         if self.auto:
             self._missing_value_handling_transformation()
+            self.progress_bar.update()
+            
             self._categorical_encoding_transformation()
+            self.progress_bar.update()
         else:
             self._custom_missing_value_handling_transformation()
+            self.progress_bar.update()
+            
             self._custom_bincode_column_transformation()
+            self.progress_bar.update()
+            
             self._custom_string_column_transformation()
+            self.progress_bar.update()
+            
             self._custom_categorical_encoding_transformation()
+            self.progress_bar.update()
+            
             self._custom_mathematical_transformation()
+            self.progress_bar.update()
+            
             self._custom_non_linear_transformation()
+            self.progress_bar.update()
+            
             self._custom_anti_select_column_transformation()
+            self.progress_bar.update()
 
     def data_preparation_transformation(self):
         """
@@ -162,15 +196,23 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             on test data using parameters from data_transformation_params.
         """
         self._display_msg(msg="Performing transformation carried out in data preparation phase ...", 
-                          show_data=True)
+                          show_data=True,
+                          progress_bar=self.progress_bar)
+        
         # Handling features transformed from feature engineering phase
         self._handle_generated_features_transformation()
+        self.progress_bar.update()
 
         # Performing transformation including feature selection using lasso, rfe and pca
         # followed by scaling
         self._feature_selection_lasso_transformation()
+        self.progress_bar.update()
+        
         self._feature_selection_rfe_transformation()
+        self.progress_bar.update()
+        
         self._feature_selection_pca_transformation()
+        self.progress_bar.update()
 
     def _preprocess_transformation(self):
         """
@@ -182,7 +224,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         if columns_to_be_removed:
             self.data = self.data.drop(columns_to_be_removed, axis=1)
             self._display_msg(msg="\nUpdated dataset after dropping irrelevent columns :", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
         # Adding id column
         self.data = FillRowId(data=self.data, row_id_column='id').result
@@ -197,7 +240,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         if futile_cols:
             self.data = self.data.drop(futile_cols, axis=1)
             self._display_msg(msg="\nUpdated dataset after dropping futile columns :", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _date_column_handling_transformation(self):
         """
@@ -205,47 +249,32 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             Function performs transformation on date columns and generates new columns.
         """
         # Extracting date columns
-        date_columns = self.data_transformation_params.get("date_columns",None)
-        if date_columns:
+        self.date_column_list = self.data_transformation_params.get("date_columns",None)
+        if self.date_column_list:
             # Dropping rows with null values in date columns
-            self.data = self.data.dropna(subset=date_columns)
+            self.data = self.data.dropna(subset=self.date_column_list)
             # Extracting unique date columns for dropping
             drop_unique_date_columns = self.data_transformation_params.get("drop_unique_date_columns",None)
             if drop_unique_date_columns:
                 self.data = self.data.drop(drop_unique_date_columns, axis=1)
+                # Updated date column list after dropping irrelevant date columns
+                self.date_column_list = [item for item in self.date_column_list if item not in drop_unique_date_columns]
 
-            # Extracting date components parameters for new columns generation
-            extract_date_comp_param = self.data_transformation_params.get("extract_date_comp_param",None)
-            extract_date_comp_col = self.data_transformation_params.get("extract_date_comp_col", None)
-            if extract_date_comp_param:
-                self.data=self.data.assign(**extract_date_comp_param)
-                self.data = self.data.drop(extract_date_comp_col, axis=1)
-
-            # Extracting irrelevant date component columns for dropping  
-            drop_extract_date_columns = self.data_transformation_params.get("drop_extract_date_columns", None)
-            if drop_extract_date_columns:
-                self.data = self.data.drop(drop_extract_date_columns, axis=1)
-
-            # Extracting date component fit objects for bincode transformation
-            day_component_fit_object = self.data_transformation_params.get("day_component_fit_object", None)
-            month_component_fit_object = self.data_transformation_params.get("month_component_fit_object", None)
-            year_diff_component_fit_object = self.data_transformation_params.get("year_diff_component_fit_object", None)
-
-            # Performing bincode transformation on day, month and year components
-            for fit_object in [day_component_fit_object, month_component_fit_object, year_diff_component_fit_object]:
-                if fit_object:
-                    for col, bin_code_fit in fit_object.items():
-                        accumulate_columns = self._extract_list(self.data.columns, [col])
-                        transform_params = {
-                            "data": self.data,
-                            "object": bin_code_fit,
-                            "accumulate": accumulate_columns,
-                            "persist": True
-                        }
-                        self.data = BincodeTransform(**transform_params).result
-
-            self._display_msg(msg="\nUpdated dataset after transforming date columns :", 
-                              data=self.data)
+            if len(self.date_column_list) != 0:
+                # Extracting date components parameters for new columns generation
+                new_columns=self._fetch_date_component()
+            
+                # Extracting irrelevant date component columns for dropping  
+                drop_extract_date_columns = self.data_transformation_params.get("drop_extract_date_columns", None)
+                if drop_extract_date_columns:
+                    self.data = self.data.drop(drop_extract_date_columns, axis=1)
+                    new_columns = [item for item in new_columns if item not in drop_extract_date_columns]
+                
+                self._display_msg(msg='Updated list of newly generated features from existing date features :',
+                                  col_lst=new_columns)
+                self._display_msg(msg="\nUpdated dataset after transforming date columns :", 
+                                  data=self.data,
+                                  progress_bar=self.progress_bar)
 
     def _missing_value_handling_transformation(self):
         """
@@ -257,7 +286,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         if drop_cols:
             self.data = self.data.drop(drop_cols, axis=1)
             self._display_msg(msg="\nUpdated dataset after dropping missing value containing columns : ", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
         # Extracting imputation columns and fit object for missing value imputation
         imputation_cols = self.data_transformation_params.get("imputation_columns", None)
@@ -265,20 +295,22 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             sm_fit_obj = self.data_transformation_params.get("imputation_fit_object")
             # imputing column using fit object
             self.data = SimpleImputeTransform(data=self.data, 
-                                            object=sm_fit_obj, 
-                                            volatile=True).result
+                                              object=sm_fit_obj).result
             self._display_msg(msg="\nUpdated dataset after imputing missing value containing columns :", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
         # Handling rest null, its temporary solution. It subjects to change based on input.
         dropped_data = self.data.dropna()
         dropped_count = self.data.shape[0] - dropped_data.shape[0]
         if dropped_count > 0:
-            self.data = dropped_data
             self._display_msg(msg="\nFound additional {} rows that contain missing values :".format(dropped_count),
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
+            self.data = dropped_data
             self._display_msg(msg="\nUpdated dataset after dropping additional missing value containing rows :", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _custom_missing_value_handling_transformation(self):
         """
@@ -291,7 +323,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         if drop_col_list:
             self.data = self.data.drop(drop_col_list, axis=1)
             self._display_msg(msg="\nUpdated dataset after dropping customized missing value containing columns :", 
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
         # Extracting custom imputation columns and fit object for missing value imputation
         custom_imp_ind = self.data_transformation_params.get("custom_imputation_ind", False)  
@@ -299,10 +332,10 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             sm_fit_obj = self.data_transformation_params.get("custom_imputation_fit_object")
             # imputing column using fit object 
             self.data = SimpleImputeTransform(data=self.data, 
-                                            object=sm_fit_obj, 
-                                            volatile=True).result
+                                              object=sm_fit_obj).result
             self._display_msg(msg="\nUpdated dataset after imputing customized missing value containing columns :",
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
         # Handling rest with default missing value handling    
         self._missing_value_handling_transformation()
 
@@ -325,11 +358,15 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     "data" : self.data,
                     "object" : custom_eql_bincode_fit_object,
                     "accumulate" : accumulate_columns,
-                    "persist" : True,   
+                    "persist" : True,
+                    "display_table_name" : False   
                 }
                 self.data = BincodeTransform(**eql_transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
                 self._display_msg(msg="\nUpdated dataset after performing customized equal width bin-code transformation :",
-                                  data=self.data)
+                                  data=self.data,
+                                  progress_bar=self.progress_bar)
 
             # Hnadling bincode transformation for Variable-Width
             custom_var_bincode_col = self.data_transformation_params.get("custom_var_bincode_col", None)
@@ -343,11 +380,15 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     "object" : custom_var_bincode_fit_object,
                     "object_order_column" : "TD_MinValue_BINFIT",
                     "accumulate" : accumulate_columns,
-                    "persist" : True   
+                    "persist" : True,
+                    "display_table_name" : False   
                 }
                 self.data = BincodeTransform(**var_transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
                 self._display_msg(msg="\nUpdated dataset after performing customized variable width bin-code transformation :",
-                                  data=self.data)
+                                  data=self.data,
+                                  progress_bar=self.progress_bar)
 
     def _custom_string_column_transformation(self):
         """
@@ -362,7 +403,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             for target_col,transform_val in custom_string_manipulation_param.items():            
                 self.data = self._str_method_mapping(target_col, transform_val)
             self._display_msg(msg="\nUpdated dataset after performing customized string manipulation :",
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _categorical_encoding_transformation(self):
         """
@@ -380,14 +422,18 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                         "data" : self.data, 
                         "object" : fit_obj, 
                         "is_input_dense" : True,
-                        "persist" : True
+                        "persist" : True,
+                        "display_table_name" : False
                     }
                 # Performing one hot encoding transformation
                 self.data = OneHotEncodingTransform(**transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
             # Dropping old columns after encoding        
             self.data = self.data.drop(one_hot_encoding_drop_list, axis=1)  
             self._display_msg(msg="\nUpdated dataset after performing categorical encoding :",
-                            data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _custom_categorical_encoding_transformation(self):
         """
@@ -408,10 +454,13 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     "data" : self.data,
                     "object" : custom_ord_encoding_fit_obj,
                     "accumulate" : accumulate_columns,
-                    "persist" : True
+                    "persist" : True,
+                    "display_table_name" : False
                 }
                 # Performing ordinal encoding transformation
                 self.data = OrdinalEncodingTransform(**transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
             # Extracting parameters for target encoding
             custom_target_encoding_ind = self.data_transformation_params.get("custom_target_encoding_ind", False)
             custom_target_encoding_fit_obj = self.data_transformation_params.get("custom_target_encoding_fit_obj", None)
@@ -424,12 +473,16 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                         "data" : self.data,
                         "object" : tar_fit_obj,
                         "accumulate" : accumulate_columns,
-                        "persist" : True                
+                        "persist" : True,
+                        "display_table_name" : False                
                     }
-                    # Performing ordinal encoding transformation
+                    # Performing target encoding transformation
                     self.data = TargetEncodingTransform(**transform_params).result
+                    # Adding transformed data containing table to garbage collector
+                    GarbageCollector._add_to_garbagecollector(self.data._table_name)
             self._display_msg(msg="\nUpdated dataset after performing customized categorical encoding :",
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
         # Handling rest with default categorical encoding transformation    
         self._categorical_encoding_transformation()
@@ -468,12 +521,16 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     "data" : self.data,
                     "object" : custom_numerical_transformation_fit_object,
                     "id_columns" : custom_numerical_transformation_id_columns,
-                    "persist" :True
+                    "persist" :True,
+                    "display_table_name" : False
                 }
                 # Peforming transformation on target columns
                 self.data = Transform(**transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
             self._display_msg(msg="\nUpdated dataset after performing customized mathematical transformation :",
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _custom_non_linear_transformation(self):
         """
@@ -491,12 +548,16 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     "data" : self.data,
                     "object" : fit_obj,
                     "accumulate" : self.data.columns,
-                    "persist" : True
+                    "persist" : True,
+                    "display_table_name" : False
                 }
                 # Performing transformation
                 self.data = NonLinearCombineTransform(**transform_params).result
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
             self._display_msg(msg="\nUpdated dataset after performing customized non-linear transformation :",
-                              data=self.data)
+                              data=self.data,
+                              progress_bar=self.progress_bar)
 
     def _custom_anti_select_column_transformation(self):
         """
@@ -516,7 +577,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 # Performing transformation for given user input
                 self.data = Antiselect(**fit_params).result
                 self._display_msg(msg="\nUpdated dataset after performing customized anti-selection :",
-                                  data=self.data)
+                                  data=self.data,
+                                  progress_bar=self.progress_bar)
 
     def _handle_generated_features_transformation(self):
         """
@@ -539,8 +601,11 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                 "target_columns" : round_columns,
                 "precision_digit" : 4,
                 "accumulate" : accumulate_columns,
-                "persist" : True}
+                "persist" : True,
+                "display_table_name" : False}
             self.data = RoundColumns(**fit_params).result
+            # Adding transformed data containing table to garbage collector
+            GarbageCollector._add_to_garbagecollector(self.data._table_name)
             
     def _handle_target_column_transformation(self):
         """
@@ -561,11 +626,13 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     "data" : self.data,
                     "object" : target_col_ord_encoding_fit_obj,
                     "accumulate" : accumulate_columns,
-                    "persist" : True
+                    "persist" : True,
+                    "display_table_name" : False
                 }
                 # Performing ordinal encoding transformation
                 self.data = OrdinalEncodingTransform(**transform_params).result
-                
+                # Adding transformed data containing table to garbage collector
+                GarbageCollector._add_to_garbagecollector(self.data._table_name)
         # Converting target column to integer datatype
         params = {
             "data" : self.data,
@@ -575,7 +642,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         }
         self.data = ConvertTo(**params).result
         self._display_msg(msg="\nUpdated dataset after performing target column transformation :",
-                          data=self.data)
+                          data=self.data,
+                          progress_bar=self.progress_bar)
 
     def _extract_and_display_features(self, feature_type, feature_list):
         """
@@ -605,7 +673,8 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         
         # Displaying feature dataframe
         self._display_msg(msg=f"\nUpdated dataset after performing {feature_type} feature selection:",
-                        data=feature_df)
+                          data=feature_df,
+                          progress_bar=self.progress_bar)
 
         # Returning feature dataframe
         return feature_df
@@ -631,12 +700,14 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     accumulate=accumulate_cols).result
         # Displaying scaled dataset
         self._display_msg(msg="\nUpdated dataset after performing scaling on Lasso selected features :",
-                          data=lasso_df)
+                          data=lasso_df,
+                          progress_bar=self.progress_bar)
 
         # Uploading lasso dataset to table for further use
         table_name = UtilFuncs._generate_temp_table_name(prefix="lasso_new_test", 
                                                          table_type = TeradataConstants.TERADATA_TABLE)
-        self.table_name_mapping["lasso_new_test"] = table_name
+        # Storing table name mapping for lasso dataset
+        self.table_name_mapping[self.data_node_id]["lasso_new_test"] = table_name
         copy_to_sql(df = lasso_df, table_name= table_name, if_exists="replace")
 
     def _feature_selection_rfe_transformation(self):
@@ -667,12 +738,14 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     accumulate=accumulate_cols).result
         # Displaying scaled dataset
         self._display_msg(msg="\nUpdated dataset after performing scaling on RFE selected features :",
-                          data=rfe_df)
+                          data=rfe_df,
+                          progress_bar=self.progress_bar)        
 
         # Uploading rfe dataset to table for further use
         table_name = UtilFuncs._generate_temp_table_name(prefix="rfe_new_test", 
                                                          table_type = TeradataConstants.TERADATA_TABLE)
-        self.table_name_mapping["rfe_new_test"] = table_name
+        # Storing table name mapping for rfe dataset
+        self.table_name_mapping[self.data_node_id]["rfe_new_test"] = table_name
         copy_to_sql(df = rfe_df, table_name= table_name, if_exists="replace")
 
     def _feature_selection_pca_transformation(self):
@@ -691,18 +764,25 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     accumulate=accumulate_cols).result
         # Displaying scaled dataset
         self._display_msg(msg="\nUpdated dataset after performing scaling for PCA feature selection :",
-                          data=pca_scaled_df)
+                          data=pca_scaled_df,
+                          progress_bar=self.progress_bar)
 
         # Convert to pandas dataframe for applying pca
         pca_scaled_pd = pca_scaled_df.to_pandas()
         # Extracting pca fit instance for applying pca
         pca_fit_instance = self.data_transformation_params.get("pca_fit_instance", None)
+        # Extracting columns for applying pca  
+        pca_fit_columns = self.data_transformation_params.get("pca_fit_columns", None)
 
         # drop id column and target column if present
         drop_col = ['id']
         if self.target_column_ind:
             drop_col.append(self.data_target_column)
         pca_df = pca_scaled_pd.drop(columns=drop_col, axis=1)
+        
+        # Rearranging columns to match the order used during PCA fitting to 
+        # avoid issues during PCA transformation.
+        pca_df = pca_df[pca_fit_columns]
 
         # Applying pca on scaled dataset
         pca_df = pca_fit_instance.transform(pca_df)
@@ -718,10 +798,12 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             pca_df[self.data_target_column] = pca_scaled_pd[self.data_target_column].reset_index(drop=True)
         # Displaying pca dataframe
         self._display_msg(msg="\nUpdated dataset after performing PCA feature selection :",
-                          data=pca_df)
+                          data=pca_df.head(10),
+                          progress_bar=self.progress_bar)
 
         # Uploading pca dataset to table for further use
         table_name = UtilFuncs._generate_temp_table_name(prefix="pca_new_test", 
                                                          table_type = TeradataConstants.TERADATA_TABLE)
-        self.table_name_mapping["pca_new_test"] = table_name
+        # Storing table name mapping for pca dataset
+        self.table_name_mapping[self.data_node_id]["pca_new_test"] = table_name
         copy_to_sql(df = pca_df, table_name=table_name, if_exists="replace")