teradataml 20.0.0.0__py3-none-any.whl → 20.0.0.1__py3-none-any.whl

teradataml/automl/data_preparation.py

@@ -28,9 +28,14 @@ from teradataml import OutlierFilterFit, OutlierFilterTransform
 from teradataml import RoundColumns, TeradataMlException
 from teradataml import ScaleFit, ScaleTransform
 from teradataml import TrainTestSplit, UtilFuncs, TeradataConstants
+from teradataml.common.garbagecollector import GarbageCollector
 from teradataml.common.messages import Messages, MessageCodes
 from teradataml.utils.validators import _Validators
+from teradataml import INTEGER
 
+# Control Randomnes
+random.seed(42)
+np.random.seed(42)
 
 class _DataPreparation:
     
@@ -54,7 +59,7 @@ class _DataPreparation:
                 Types: teradataml Dataframe
             
             target_column:
-                Required Arugment.
+                Required Argument.
                 Specifies the name of the target column in "data".
                 Types: str
             
@@ -69,22 +74,22 @@ class _DataPreparation:
                 Types: int
             
             excluded_columns:
-                Required Arugment.
+                Required Argument.
                 Specifies the columns should be excluded from any processing.
                 Types: str or list of strings (str)
 
             custom_data:
-                Optional Arugment.
+                Optional Argument.
                 Specifies json object containing user customized input.
                 Types: json object
             
             data_transform_dict:
-                Optional Arugment.
+                Optional Argument.
                 Specifies the parameters for data transformation.
                 Types: dict
             
             task_type:
-                Required Arugment.
+                Required Argument.
                 Specifies the task type for AutoML, whether to apply regresion OR classification
                 on the provived dataset.
                 Default Value: "Regression"
@@ -106,8 +111,6 @@ class _DataPreparation:
         self._scale_method_cls = "RANGE"
         self.table_name_mapping = {}
         
-        random.seed(42)
-        np.random.seed(42)
         self.data_types = {key: value for key, value in self.data._column_names_and_types}
         
         
@@ -123,7 +126,7 @@ class _DataPreparation:
          
         PARAMETERS:  
             auto:
-                Optional Arugment.
+                Optional Argument.
                 Specifies whether to run AutoML in custom mode or auto mode.
                 When set to False, runs in custom mode. Otherwise, by default runs in auto mode.
                 Default Value: True
@@ -163,6 +166,10 @@ class _DataPreparation:
             train = self._data_sampling(train)
         self.progress_bar.update()
         
+        # Sorting the data based on id to 
+        # remove any shuffling done by sampling
+        train = train.sort_values(by='id')
+
         # Performing feature selection using lasso followed by scaling  
         self._feature_selection_Lasso(train, test)
         self._scaling_features(feature_selection_mtd="lasso")
@@ -375,6 +382,8 @@ class _DataPreparation:
             "persist" : True
         }
         self.train_df = OutlierFilterTransform(**transform_params).result
+        # Adding transformed data containing table to garbage collector
+        GarbageCollector._add_to_garbagecollector(self.train_df._table_name)
 
     def _outlier_processing(self):
         """
@@ -400,6 +409,9 @@ class _DataPreparation:
             target_columns=columns_to_drop_rows
             replacement_strategy = "DELETE"
             self._outlier_handling(target_columns, outlier_handling_method, replacement_strategy)
+            self._display_msg(msg="Sample of training dataset after removing outlier rows:",
+                              data=self.train_df,
+                              progress_bar=self.progress_bar)
         
         # Imputing Median value in place of outliers
         if len(columns_to_impute) != 0:
@@ -409,6 +421,13 @@ class _DataPreparation:
             target_columns=columns_to_impute
             replacement_strategy = "MEDIAN"
             self._outlier_handling(target_columns, outlier_handling_method, replacement_strategy)
+            self._display_msg(msg="Sample of training dataset after performing MEDIAN inplace:",
+                              data=self.train_df,
+                              progress_bar=self.progress_bar)
+
+        if len(columns_to_drop_rows) == 0 and len(columns_to_impute) == 0:
+            self._display_msg(msg='Analysis indicates not outlier in the dataset. No Action Taken.',
+                              progress_bar=self.progress_bar)
             
         end_time = time.time()
         self._display_msg("Time Taken by Outlier processing: {:.2f} sec ".format(end_time - start_time),
@@ -557,10 +576,6 @@ class _DataPreparation:
                           progress_bar=self.progress_bar,
                           show_data=True)
         
-        if self.is_classification_type():
-            train_df[self.target_column] = train_df[self.target_column].astype('int')
-            test_df[self.target_column] = test_df[self.target_column].astype('int')
-
         # Pushing the data in database
         self.copy_dataframe_to_sql(train_df, test_df, 'pca')
  
@@ -590,7 +605,7 @@ class _DataPreparation:
         # Required imports for RFE   
         from sklearn.feature_selection import RFECV
         from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
-        from sklearn.model_selection import StratifiedKFold,KFold
+        from sklearn.model_selection import StratifiedKFold
 
         start_time = time.time()
         # Regression
@@ -606,9 +621,9 @@ class _DataPreparation:
         score = 'r2' if not self.is_classification_type() \
                 else 'roc_auc' if self.data.drop_duplicate(self.target_column).size == 2 else 'f1_macro'
 
-        # Instantiate StratifiedKFold with shuffling for classification
+        # # Instantiate StratifiedKFold with shuffling for classification
         cv = folds if not self.is_classification_type() \
-                   else StratifiedKFold(n_splits=folds, shuffle=True, random_state=42)
+                   else StratifiedKFold(n_splits=folds, shuffle=False)
 
         # Define the RFE with cross-validation
         rfecv = RFECV(rf, cv=cv, scoring=score)
@@ -682,7 +697,8 @@ class _DataPreparation:
         from sklearn.model_selection import GridSearchCV
         from sklearn.linear_model import Lasso
         from sklearn.linear_model import LogisticRegression
-        
+        from sklearn.model_selection import StratifiedKFold
+
         # Getting the value k in k-fold cross-validation
         num_folds = self._num_of_folds(train.shape[0])
 
@@ -696,15 +712,21 @@ class _DataPreparation:
                 scoring_metric = 'roc_auc'
             else:
                 scoring_metric = 'f1_macro'
-            estimator = LogisticRegression(penalty='l1', solver='liblinear', multi_class='auto')
+            estimator = LogisticRegression(solver='saga', penalty='l2', multi_class='auto', random_state=42)
             parameters = {'C':[0.00001,0.0001,0.001,0.01,0.05,0.1,10,100,1000], 'max_iter': [100, 500]}
         else:
-            estimator = Lasso()
+            estimator = Lasso(random_state=42)
             parameters = {'alpha':[0.00001,0.0001,0.001,0.01,0.05,0.1,10,100,1000], 'max_iter': [100, 500]}
             scoring_metric = "r2"
 
+        if self.is_classification_type():
+            cv = StratifiedKFold(n_splits=5, shuffle=False)
+        else:
+            cv = num_folds
+
         # Applying hyperparameter tuning and optimizing score
-        hyperparameter_search = GridSearchCV(estimator, parameters, cv=num_folds, scoring=scoring_metric, verbose=0)
+        hyperparameter_search = GridSearchCV(estimator, parameters, cv=cv, refit=True, 
+                                             scoring=scoring_metric, verbose=0)
 
         # Fitting the best result from hyperparameter
         hyperparameter_search.fit(train_features, train_target)
@@ -775,8 +797,12 @@ class _DataPreparation:
         self.table_name_mapping['{}_test'.format(prefix)] = test_table_name
 
         # Pushing data into database
-        copy_to_sql(df=train, table_name=train_table_name, if_exists="replace")
-        copy_to_sql(df=test, table_name=test_table_name, if_exists="replace")
+        if self.is_classification_type():
+            copy_to_sql(df=train, table_name=train_table_name, if_exists="replace", types={f'{self.target_column}': INTEGER})
+            copy_to_sql(df=test, table_name=test_table_name, if_exists="replace", types={f'{self.target_column}': INTEGER})
+        else:
+            copy_to_sql(df=train, table_name=train_table_name, if_exists="replace")
+            copy_to_sql(df=test, table_name=test_table_name, if_exists="replace")
 
 
         
@@ -901,9 +927,6 @@ class _DataPreparation:
         else:
             self._display_msg(msg="No columns to scale.",
                               progress_bar=self.progress_bar)
-            
-        if self.is_classification_type():
-            train, test = self._bigint_to_int(train, test)
 
         self.copy_dataframe_to_sql(train, test, feature_selection_mtd)
 
@@ -911,15 +934,6 @@ class _DataPreparation:
         self._display_msg(msg="Total time taken by feature scaling: {:.2f} sec".format( end_time - start_time),
                           progress_bar=self.progress_bar,
                           show_data=True)
-            
-    def _bigint_to_int(self, train, test):
-        tr = train.to_pandas()
-        tr[self.target_column] = tr[self.target_column].astype('int')
-        
-        ts = test.to_pandas()
-        ts[self.target_column] = ts[self.target_column].astype('int')
-        
-        return tr, ts
 
     def _set_custom_scaling_method(self):
         """
@@ -987,7 +1001,11 @@ class _DataPreparation:
             "precision_digit" : 4,
             "accumulate" : accumulate_columns,
             "persist" : True}
-        
-        obj = RoundColumns(**fit_params).result
-        df = obj.to_pandas()
-        return df.reset_index()
+
+        transform_output = RoundColumns(**fit_params).result
+        # Adding transformed data containing table to garbage collector
+        GarbageCollector._add_to_garbagecollector(transform_output._table_name)
+        cols = transform_output.columns
+        df = transform_output.to_pandas().reset_index()
+        df = df[cols]
+        return df