teradataml 20.0.0.3__py3-none-any.whl → 20.0.0.5__py3-none-any.whl

teradataml/automl/data_preparation.py

@@ -16,7 +16,6 @@
 # Python libraries
 import numpy as np
 import pandas as pd
-import random
 import time
 import warnings
 
@@ -30,11 +29,9 @@ from teradataml import UtilFuncs, TeradataConstants
 from teradataml.common.garbagecollector import GarbageCollector
 from teradataml.common.messages import Messages, MessageCodes
 from teradataml.utils.validators import _Validators
-from teradataml import INTEGER
+from teradataml import configure, INTEGER
+from teradataml.common.constants import TeradataConstants
 
-# Control Randomnes
-random.seed(42)
-np.random.seed(42)
 
 class _DataPreparation:
     
@@ -117,6 +114,12 @@ class _DataPreparation:
                         session.
                         Default Value: False
                         Types: bool
+
+                    seed:
+                        Optional Argument.
+                        Specifies the random seed for reproducibility.
+                        Default Value: 42
+                        Types: int
         """
         self.data = data
         self.target_column = target_column
@@ -127,14 +130,22 @@ class _DataPreparation:
         self.task_type = task_type
         self.volatile = kwargs.get("volatile", False)
         self.persist = kwargs.get("persist", False)
+        self.aml_phases = kwargs.get("automl_phases", None)
         
         # Setting default value for auto run mode
         self._data_sampling_method = "SMOTE"
         self._scale_method_reg = "STD"
         self._scale_method_cls = "RANGE"
-        self.table_name_mapping = {}
         
         self.data_types = {key: value for key, value in self.data._column_names_and_types}
+        self.seed = kwargs.get("seed", 42)
+        # np.random.seed() affects the random number generation in numpy and sklearn
+        # setting this changes the global state of the random number generator
+        # hence, setting the seed only if it is not None
+        if kwargs.get("seed") is not None:
+            np.random.seed(self.seed)
+  
+        self.data_mapping = kwargs.get("data_mapping", {})
         
         
     def data_preparation(self, 
@@ -158,7 +169,8 @@ class _DataPreparation:
              list of lists containing, feature selected by rfe, pca and lasso.  
         """
         self._display_heading(phase=2,
-                              progress_bar=self.progress_bar)
+                              progress_bar=self.progress_bar,
+                              automl_phases=self.aml_phases)
         self._display_msg(msg='Data preparation started ...',
                           progress_bar=self.progress_bar)
         # Setting user value in case of custom running mode
@@ -201,7 +213,7 @@ class _DataPreparation:
         self._feature_selection_PCA()
         self.progress_bar.update()
         
-        return [self.rfe_feature, self.lasso_feature, self.pca_feature], self.data_transform_dict
+        return [self.rfe_feature, self.lasso_feature, self.pca_feature], self.data_transform_dict, self.data_mapping
 
     def _handle_outliers(self,
                          auto):
@@ -262,25 +274,24 @@ class _DataPreparation:
         outlier_method = "Tukey"
         
         # List of columns for outlier processing.
-        outlier_columns = [col for col in self.data.columns if col not in self.excluded_columns]
+        # Excluding target column and excluded columns from outlier processing
+        outlier_columns = [col for col in self.data.columns if col not in self.excluded_columns + ['id', self.target_column]]
         
-        # Detecting outlier percentage in each columns
-        outlier_percentage_df = self._outlier_detection(outlier_method, outlier_columns)
-        
-        # Outlier Handling techniques
-        for i in outlier_percentage_df.itertuples():
-            # Column Name
-            col = i[0]
-            # Outlier value
-            value = i[1]
-
-            if col == self.target_column:
-                if value < 5.0 and value > 0.0:
+        if len(outlier_columns) != 0:
+            # Detecting outlier percentage in each columns
+            outlier_percentage_df = self._outlier_detection(outlier_method, outlier_columns)
+            
+            # Outlier Handling techniques
+            for i in outlier_percentage_df.itertuples():
+                # Column Name
+                col = i[0]
+                # Outlier value
+                value = i[1]
+                # Dropping rows
+                if value > 0.0  and value <= 8.0 :
                     columns_to_drop_rows.append(col)
-            elif value > 0.0  and value <= 8.0 :
-                columns_to_drop_rows.append(col)
-            elif value> 8.0 and value <= 25.0:
-                columns_to_impute.append(col)
+                elif value> 8.0 and value <= 25.0:
+                    columns_to_impute.append(col)
         
         return columns_to_drop_rows, columns_to_impute
 
@@ -347,6 +358,9 @@ class _DataPreparation:
             # Adding transformed data containing table to garbage collector
             GarbageCollector._add_to_garbagecollector(self.data._table_name)
 
+        # Returning outlier fit object to store in data mapping dictionary
+        return outlier_fit_out
+
     def _outlier_processing(self):
         """
         DESCRIPTION:
@@ -370,7 +384,10 @@ class _DataPreparation:
                               progress_bar=self.progress_bar)
             target_columns=columns_to_drop_rows
             replacement_strategy = "DELETE"
-            self._outlier_handling(target_columns, outlier_handling_method, replacement_strategy)
+            fit_obj = self._outlier_handling(target_columns, outlier_handling_method, replacement_strategy)
+            self.data_mapping['fit_outlier_delete_output'] = fit_obj.output_data._table_name
+            self.data_mapping['fit_outlier_delete_result'] = self.data._table_name
+            self.data_mapping['outlier_filtered_data'] = self.data._table_name
             self._display_msg(msg="Sample of dataset after removing outlier rows:",
                               data=self.data,
                               progress_bar=self.progress_bar)
@@ -382,7 +399,10 @@ class _DataPreparation:
                               progress_bar=self.progress_bar)
             target_columns=columns_to_impute
             replacement_strategy = "MEDIAN"
-            self._outlier_handling(target_columns, outlier_handling_method, replacement_strategy)
+            fit_obj = self._outlier_handling(target_columns, outlier_handling_method, replacement_strategy)
+            self.data_mapping['fit_outlier_impute_output'] = fit_obj.output_data._table_name
+            self.data_mapping['fit_outlier_impute_result'] = fit_obj.result._table_name
+            self.data_mapping['outlier_imputed_data'] = self.data._table_name
             self._display_msg(msg="Sample of dataset after performing MEDIAN inplace:",
                               data=self.data,
                               progress_bar=self.progress_bar)
@@ -438,7 +458,10 @@ class _DataPreparation:
                         # Fetching replacement value
                         replacement_value = transform_val["replacement_value"]
                         # Performing outlier handling
-                        self._outlier_handling(target_col, outlier_method, replacement_value)
+                        fit_obj = self._outlier_handling(target_col, outlier_method, replacement_value)
+                        self.data_mapping[f'fit_{target_col}_outlier_output'] = fit_obj.output_data._table_name
+                        self.data_mapping[f'fit_{target_col}_outlier_result'] = fit_obj.result._table_name
+                        self.data_mapping[f'{target_col}_outlier_treated_data'] = self.data._table_name
                 else:
                     self._display_msg(inline_msg="No information provided for feature transformation in outlier handling.",
                                       progress_bar=self.progress_bar)
@@ -483,13 +506,13 @@ class _DataPreparation:
         start_time = time.time()
 
         # Temporary Pulling data for feature selection
-        pca_train = DataFrame.from_table(self.table_name_mapping['pca_train']).to_pandas()
+        pca_train = DataFrame.from_table(self.data_mapping['pca_train']).to_pandas()
  
         # Drop unnecessary columns and store the result
         train_data = pca_train.drop(columns=['id', self.target_column], axis=1)
 
         # Initialize and fit PCA
-        pca = PCA()
+        pca = PCA(random_state=self.seed)
         pca.fit(train_data)
 
         # Find the number of components for PCA
@@ -497,7 +520,7 @@ class _DataPreparation:
         n = np.argmax(np.cumsum(variance) >= 0.95) + 1
 
         # Create a new instance of PCA with the optimal number of components
-        pca = PCA(n_components=n, random_state=42)
+        pca = PCA(n_components=n, random_state=self.seed)
 
         # Apply PCA on dataset
         X_train_pca = pca.fit_transform(train_data)
@@ -571,7 +594,7 @@ class _DataPreparation:
 
             # Random forest for RFE model
             RFModel = RandomForestRegressor if not is_classification else RandomForestClassifier
-            rf = RFModel(n_estimators=100, random_state=42)
+            rf = RFModel(n_estimators=100, random_state=self.seed)
 
             # Determine the scoring metric based on the number of unique classes
             score = 'r2' if not self.is_classification_type() \
@@ -665,10 +688,10 @@ class _DataPreparation:
                     scoring_metric = 'roc_auc'
                 else:
                     scoring_metric = 'f1_macro'
-                estimator = LogisticRegression(solver='saga', penalty='l2', multi_class='auto', random_state=42)
+                estimator = LogisticRegression(solver='saga', penalty='l2', multi_class='auto', random_state=self.seed)
                 parameters = {'C':[0.00001,0.0001,0.001,0.01,0.05,0.1,10,100,1000], 'max_iter': [100, 500]}
             else:
-                estimator = Lasso(random_state=42)
+                estimator = Lasso(random_state=self.seed)
                 parameters = {'alpha':[0.00001,0.0001,0.001,0.01,0.05,0.1,10,100,1000], 'max_iter': [100, 500]}
                 scoring_metric = "r2"
 
@@ -679,7 +702,7 @@ class _DataPreparation:
 
             # Applying hyperparameter tuning and optimizing score
             hyperparameter_search = GridSearchCV(estimator, parameters, cv=cv, refit=True, 
-                                                scoring=scoring_metric, verbose=0)
+                                                 scoring=scoring_metric, verbose=0)
 
             # Fitting the best result from hyperparameter
             hyperparameter_search.fit(train_features, train_target)
@@ -746,14 +769,20 @@ class _DataPreparation:
         train_table_name = UtilFuncs._generate_temp_table_name(prefix='{}_train'.format(prefix), 
                                                                table_type = TeradataConstants.TERADATA_TABLE,
                                                                gc_on_quit=not persist)
+        # If configure.temp_object_type="VT", _generate_temp_table_name() retruns the
+        # table name in fully qualified format.
+        train_table_name = UtilFuncs._extract_table_name(train_table_name)
+
         # Storing the table names in the table name mapping dictionary
-        self.table_name_mapping['{}_train'.format(prefix)] = train_table_name
+        self.data_mapping['{}_train'.format(prefix)] = train_table_name
 
+        # In the case of the VT option, the table was being persisted, so the VT condition is being checked. 
+        is_temporary = configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE
         # Pushing data into database
         if self.is_classification_type():
-            copy_to_sql(df=data, table_name=train_table_name, if_exists="replace", types={f'{self.target_column}': INTEGER})
+            copy_to_sql(df=data, table_name=train_table_name, temporary=is_temporary, if_exists="replace", types={f'{self.target_column}': INTEGER})
         else:
-            copy_to_sql(df=data, table_name=train_table_name, if_exists="replace")
+            copy_to_sql(df=data, table_name=train_table_name, if_exists="replace", temporary=is_temporary)
         
     def _scaling_features_helper(self, 
                                  data=None,  
@@ -825,9 +854,9 @@ class _DataPreparation:
         
         # Loading data for feature scaling based of feature selection method
         if feature_selection_mtd == 'rfe':
-            data_to_scale = DataFrame(self.table_name_mapping['rfe_train'])
+            data_to_scale = DataFrame(self.data_mapping['rfe_train'])
         elif feature_selection_mtd == 'lasso':
-            data_to_scale = DataFrame(self.table_name_mapping['lasso_train'])
+            data_to_scale = DataFrame(self.data_mapping['lasso_train'])
         else:
             data_to_scale = self.data
 
@@ -850,12 +879,16 @@ class _DataPreparation:
                                volatile=volatile,
                                persist=persist)
             
+            self.data_mapping[f'fit_scale_{feature_selection_mtd}_output'] = fit_obj.output_data._table_name
+            self.data_mapping[f'fit_scale_{feature_selection_mtd}_result'] = fit_obj.output._table_name
+            
             # storing the scale fit object and columns in data transformation dictionary
             self.data_transform_dict['{}_scale_fit_obj'.format(feature_selection_mtd)] = fit_obj.output
             self.data_transform_dict['{}_scale_col'.format(feature_selection_mtd)] = scale_col
             
             # List of columns to copy to the output generated by scale transform
             accumulate_cols = list(set(data_to_scale.columns) - set(scale_col))
+
             
             # Scaling dataset
             transform_obj = ScaleTransform(data=data_to_scale,
@@ -867,6 +900,8 @@ class _DataPreparation:
                               data=scaled_df,
                               progress_bar=self.progress_bar)
         else:
+            # No columns to scale, Original data will be used
+            scaled_df = data_to_scale
             self._display_msg(msg="No columns to scale.",
                               progress_bar=self.progress_bar)
 
@@ -915,10 +950,16 @@ class _DataPreparation:
         # Assigning data to target dataframe
         target_df = self.data            
         # Detecting list of float columns on target dataset
-        float_columns =[col for col, d_type in target_df._column_names_and_types if d_type in ["float"]]
+        float_columns =[col for col, d_type in target_df._column_names_and_types if d_type in ["float", "decimal.Decimal"]]
         
         if len(float_columns) == 0:
-            return target_df.to_pandas()
+            cols = target_df.columns
+            # Doing reset index to get index column
+            df = target_df.to_pandas().reset_index()
+
+            # Returning the dataframe with cols
+            # to avoid extra columns generated by reset_index()
+            return df[cols]
         
         # storing the column details for round up in data transformation dictionary
         self.data_transform_dict["round_columns"] = float_columns
@@ -942,6 +983,7 @@ class _DataPreparation:
             fit_params["persist"] = False
 
         transform_output = RoundColumns(**fit_params).result
+        self.data_mapping['round_columns_data'] = transform_output._table_name
         if not self.volatile and not self.persist:
             # Adding transformed data containing table to garbage collector
             GarbageCollector._add_to_garbagecollector(transform_output._table_name)

teradataml/automl/data_transformation.py

@@ -15,6 +15,7 @@
 
 # Python libraries
 import pandas as pd
+import warnings
 
 # Teradata libraries
 from teradataml.dataframe.dataframe import DataFrame
@@ -31,8 +32,11 @@ from teradataml import ScaleTransform
 from teradataml import SimpleImputeTransform
 from teradataml import TargetEncodingTransform
 from teradataml import Transform, UtilFuncs, TeradataConstants
+from teradataml import execute_sql
 from teradataml.common.garbagecollector import GarbageCollector
 from teradataml.hyperparameter_tuner.utils import _ProgressBar
+from teradataml.options.configure import configure
+from teradataml.common.constants import TeradataConstants
 
 # AutoML Internal libraries
 from teradataml.automl.feature_exploration import _FeatureExplore
@@ -219,11 +223,11 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         DESCRIPTION:
             Function drops irrelevent columns and adds id column.
         """
-        # Extracting irrelevent column list
+        # Extracting irrelevant column list
         columns_to_be_removed = self.data_transformation_params.get("drop_irrelevent_columns", None)
         if columns_to_be_removed:
             self.data = self.data.drop(columns_to_be_removed, axis=1)
-            self._display_msg(msg="\nUpdated dataset after dropping irrelevent columns :", 
+            self._display_msg(msg="\nUpdated dataset after dropping irrelevant columns :",
                               data=self.data,
                               progress_bar=self.progress_bar)
 
@@ -465,6 +469,7 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
             custom_target_encoding_ind = self.data_transformation_params.get("custom_target_encoding_ind", False)
             custom_target_encoding_fit_obj = self.data_transformation_params.get("custom_target_encoding_fit_obj", None)
             if custom_target_encoding_ind:
+                warn_cols = []
                 for col, tar_fit_obj in custom_target_encoding_fit_obj.items():
                     # Extracting accumulate columns
                     accumulate_columns = self._extract_list(self.data.columns, [col])
@@ -480,6 +485,15 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
                     self.data = TargetEncodingTransform(**transform_params).result
                     # Adding transformed data containing table to garbage collector
                     GarbageCollector._add_to_garbagecollector(self.data._table_name)
+                    if self.data[self.data[col] == -1].shape[0] > 0:
+                        warn_cols.append(col)
+                
+                # Checking for unseen values in target encoding columns
+                if len(warn_cols) > 0:
+                    warnings.warn(message=f"Unseen categorical values found in test data column(s): {warn_cols}. \
+                                  This may cause inaccurate predictions. Consider retraining the model with updated data.",
+                                  stacklevel=0)
+                
             self._display_msg(msg="\nUpdated dataset after performing customized categorical encoding :",
                               data=self.data,
                               progress_bar=self.progress_bar)
@@ -693,22 +707,28 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         lasso_scale_fit_obj = self.data_transformation_params.get("lasso_scale_fit_obj", None)
         lasso_scale_col = self.data_transformation_params.get("lasso_scale_col", None)
         # Extracting accumulate columns
-        accumulate_cols = self._extract_list(lasso_df.columns, lasso_scale_col)
-        # Scaling dataset
-        lasso_df = ScaleTransform(data=lasso_df,
-                    object=lasso_scale_fit_obj,
-                    accumulate=accumulate_cols).result
-        # Displaying scaled dataset
-        self._display_msg(msg="\nUpdated dataset after performing scaling on Lasso selected features :",
-                          data=lasso_df,
-                          progress_bar=self.progress_bar)
+        if lasso_scale_fit_obj is not None:
+            accumulate_cols = self._extract_list(lasso_df.columns, lasso_scale_col)
+            # Scaling dataset
+            lasso_df = ScaleTransform(data=lasso_df,
+                        object=lasso_scale_fit_obj,
+                        accumulate=accumulate_cols).result
+            # Displaying scaled dataset
+            self._display_msg(msg="\nUpdated dataset after performing scaling on Lasso selected features :",
+                            data=lasso_df,
+                            progress_bar=self.progress_bar)
 
         # Uploading lasso dataset to table for further use
         table_name = UtilFuncs._generate_temp_table_name(prefix="lasso_new_test", 
                                                          table_type = TeradataConstants.TERADATA_TABLE)
+        # If configure.temp_object_type="VT", _generate_temp_table_name() retruns the
+        # table name in fully qualified format.
+        table_name = UtilFuncs._extract_table_name(table_name)
         # Storing table name mapping for lasso dataset
         self.table_name_mapping[self.data_node_id]["lasso_new_test"] = table_name
-        copy_to_sql(df = lasso_df, table_name= table_name, if_exists="replace")
+        # In the case of the VT option, the table was being persisted, so the VT condition is being checked. 
+        is_temporary = configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE
+        copy_to_sql(df = lasso_df, table_name= table_name, if_exists="replace", temporary=is_temporary)
 
     def _feature_selection_rfe_transformation(self):
         """
@@ -730,23 +750,30 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         # Extracting fit object and columns for scaling
         rfe_scale_fit_obj = self.data_transformation_params.get("rfe_scale_fit_obj", None)
         rfe_scale_col = self.data_transformation_params.get("rfe_scale_col", None)
-        # Extracting accumulate columns
-        accumulate_cols = self._extract_list(rfe_df.columns, rfe_scale_col)
-        # Scaling on rfe dataset
-        rfe_df = ScaleTransform(data=rfe_df,
-                    object=rfe_scale_fit_obj,
-                    accumulate=accumulate_cols).result
-        # Displaying scaled dataset
-        self._display_msg(msg="\nUpdated dataset after performing scaling on RFE selected features :",
-                          data=rfe_df,
-                          progress_bar=self.progress_bar)        
+
+        if rfe_scale_fit_obj is not None:
+            # Extracting accumulate columns
+            accumulate_cols = self._extract_list(rfe_df.columns, rfe_scale_col)
+            # Scaling on rfe dataset
+            rfe_df = ScaleTransform(data=rfe_df,
+                                    object=rfe_scale_fit_obj,
+                                    accumulate=accumulate_cols).result
+            # Displaying scaled dataset
+            self._display_msg(msg="\nUpdated dataset after performing scaling on RFE selected features :",
+                            data=rfe_df,
+                            progress_bar=self.progress_bar)        
 
         # Uploading rfe dataset to table for further use
         table_name = UtilFuncs._generate_temp_table_name(prefix="rfe_new_test", 
                                                          table_type = TeradataConstants.TERADATA_TABLE)
+        # If configure.temp_object_type="VT", _generate_temp_table_name() retruns the
+        # table name in fully qualified format.
+        table_name = UtilFuncs._extract_table_name(table_name)
         # Storing table name mapping for rfe dataset
         self.table_name_mapping[self.data_node_id]["rfe_new_test"] = table_name
-        copy_to_sql(df = rfe_df, table_name= table_name, if_exists="replace")
+        # In the case of the VT option, the table was being persisted, so the VT condition is being checked. 
+        is_temporary = configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE
+        copy_to_sql(df = rfe_df, table_name= table_name, if_exists="replace", temporary=is_temporary)
 
     def _feature_selection_pca_transformation(self):
         """
@@ -758,17 +785,20 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         pca_scale_col = self.data_transformation_params.get("pca_scale_col", None)
         # Extracting accumulate columns
         accumulate_cols = self._extract_list(self.data.columns, pca_scale_col)
-        # Scaling on pca dataset
-        pca_scaled_df = ScaleTransform(data=self.data,
-                    object=pca_scale_fit_obj,
-                    accumulate=accumulate_cols).result
-        # Displaying scaled dataset
-        self._display_msg(msg="\nUpdated dataset after performing scaling for PCA feature selection :",
-                          data=pca_scaled_df,
-                          progress_bar=self.progress_bar)
+
+        pca_scaled_df = self.data
+        if pca_scale_fit_obj is not None:
+            # Scaling on pca dataset
+            pca_scaled_df = ScaleTransform(data=self.data,
+                                           object=pca_scale_fit_obj,
+                                           accumulate=accumulate_cols).result
+            # Displaying scaled dataset
+            self._display_msg(msg="\nUpdated dataset after performing scaling for PCA feature selection :",
+                            data=pca_scaled_df,
+                            progress_bar=self.progress_bar)
 
         # Convert to pandas dataframe for applying pca
-        pca_scaled_pd = pca_scaled_df.to_pandas()
+        pca_scaled_pd = pca_scaled_df.to_pandas().reset_index()
         # Extracting pca fit instance for applying pca
         pca_fit_instance = self.data_transformation_params.get("pca_fit_instance", None)
         # Extracting columns for applying pca  
@@ -804,6 +834,12 @@ class _DataTransformation(_FeatureExplore, _FeatureEngineering):
         # Uploading pca dataset to table for further use
         table_name = UtilFuncs._generate_temp_table_name(prefix="pca_new_test", 
                                                          table_type = TeradataConstants.TERADATA_TABLE)
+        # If configure.temp_object_type="VT", _generate_temp_table_name() retruns the
+        # table name in fully qualified format.
+        table_name = UtilFuncs._extract_table_name(table_name)
         # Storing table name mapping for pca dataset
         self.table_name_mapping[self.data_node_id]["pca_new_test"] = table_name
-        copy_to_sql(df = pca_df, table_name=table_name, if_exists="replace")
+        # In the case of the VT option, the table was being persisted, so the VT condition is being checked. 
+        is_temporary = configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE
+        copy_to_sql(df = pca_df, table_name=table_name, if_exists="replace", temporary=is_temporary)
+