teradataml 20.0.0.0__py3-none-any.whl → 20.0.0.2__py3-none-any.whl

teradataml/dataframe/dataframe_utils.py

@@ -35,6 +35,7 @@ from teradataml.utils.utils import execute_sql
 from teradatasqlalchemy.types import FLOAT, NUMBER, DECIMAL, PERIOD_TIMESTAMP
 from teradatasqlalchemy.dialect import preparer, dialect as td_dialect
 import teradataml.dataframe as tdmldf
+from teradataml.dataframe.sql_interfaces import ColumnExpression
 
 from sqlalchemy.sql import select
 from sqlalchemy.sql.expression import text
@@ -792,6 +793,8 @@ class DataFrameUtils():
                             2. List of functions
                             3. Dictionary containing column name as key and aggregate
                                function name (string or list of strings) as value
+                            4. ColumnExpression built using the aggregate functions. 
+                            5. List of ColumnExpression built using the aggregate functions.
 
             percentile:
                 Optional Argument.
@@ -911,7 +914,6 @@ class DataFrameUtils():
                 DataFrameUtils._generate_aggregate_column_expression(df=df, column=column, operation=func,
                                                                      describe_op=describe_op, percentile=percentile,
                                                                      tdp=tdp, **kwargs)
-
             if column_supported:
                 all_unsupported_columns = False
                 new_column_names.append(new_column_name)
@@ -985,6 +987,8 @@ class DataFrameUtils():
                             2. List of functions
                             3. Dictionary containing column name as key and aggregate
                                function name (string or list of strings) as value
+                            4. ColumnExpression built using the aggregate functions. 
+                            5. List of ColumnExpression built using the aggregate functions.
 
             percentile:
                 Optional Argument.
@@ -1064,6 +1068,28 @@ class DataFrameUtils():
         #   ['bottom', 'bottom with ties', 'delta_t', 'mad', 'top', 'top with ties']
         # Thus, no extra processing is required for time series aggregates over here.
 
+        if isinstance(func, ColumnExpression) or (isinstance(func, list) and isinstance(func[0], ColumnExpression)):
+            column_agg_expr = []
+            new_column_names = []
+            new_column_types = []
+            if isinstance(func, ColumnExpression):
+                func= UtilFuncs._as_list(func)
+
+            # validate that func is a list of ColumnExpression
+            for expr in func:
+                if not isinstance(expr, ColumnExpression):
+                    raise TeradataMlException(Messages.get_message(MessageCodes.UNSUPPORTED_DATATYPE,
+                                              'func', ['str, dict, ColumnExpression or list of values of type(s): str, ColumnExpression']),
+                                              MessageCodes.UNSUPPORTED_DATATYPE)
+
+            for operations in func:
+                alias = operations.alias_name
+                column_agg_expr.append(operations.compile_label(alias))
+                new_column_names.append(alias)
+                new_column_types.append(operations.type)
+            aggregate_expr = ", ".join(column_agg_expr)
+            return aggregate_expr, new_column_names, new_column_types
+
         # 'operations' contains dict of columns -> list of aggregate operations
         operations = DataFrameUtils._validate_agg_function(func, column_names)
 
@@ -1588,16 +1614,17 @@ class DataFrameUtils():
         return col_names, col_types
 
     @staticmethod
-    def _insert_all_from_table(to_table_name, from_table_name, column_list, schema_name,
-                               temporary=False):
+    def _insert_all_from_table(to_table_name, from_table_name, column_list, to_schema_name=None,
+                               from_schema_name=None, temporary=False):
         """
         Inserts all records from one table into the second, using columns ordered by column list.
 
         PARAMETERS:
             to_table_name - String specifying name of the SQL Table to insert to.
-            insert_from_table_name - String specifying name of the SQL Table to insert from.
+            from_table_name - String specifying name of the SQL Table to insert from.
             column_list - List of strings specifying column names used in the insertion.
-            schema_name - Name of the database schema to insert table data into.
+            to_schema_name - Name of the database schema to insert table data into.
+            from_schema_name - Name of the database schema to insert table data from.
             temporary - Specifies whether to create Vantage tables as permanent or volatile.
                         Default: False
                         Note: When True:
@@ -1618,16 +1645,25 @@ class DataFrameUtils():
         # Construct INSERT command.
         column_order_string = ', '.join([tdp.quote("{0}".format(element)) for element in column_list])
 
-        if schema_name:
-            full_to_table_name = tdp.quote(schema_name) + "." + tdp.quote(to_table_name)
+        # Generate full name of the destination table.
+        if to_schema_name:
+            full_to_table_name = tdp.quote(to_schema_name) + "." + tdp.quote(to_table_name)
         elif temporary:
             full_to_table_name = tdp.quote(to_table_name)
         else:
             full_to_table_name = tdp.quote(_get_current_databasename()) + "." + tdp.quote(
                 to_table_name)
 
-        insert_sql = SQLBundle._build_insert_from_table_query(full_to_table_name, from_table_name, column_order_string)
+        # Generate full name of source table.
+        if from_schema_name:
+            full_from_table_name = tdp.quote(from_schema_name) + "." + tdp.quote(from_table_name)
+        else:
+            full_from_table_name = tdp.quote(_get_current_databasename()) + "." + tdp.quote(
+                from_table_name)
 
+        insert_sql = SQLBundle._build_insert_from_table_query(full_to_table_name,
+                                                              full_from_table_name,
+                                                              column_order_string)
         # Execute INSERT command.
         return UtilFuncs._execute_ddl_statement(insert_sql)
 
@@ -1797,7 +1833,9 @@ class DataFrameUtils():
         aed_utils = AedUtils()
         if len(dfs) == 1:
             operation = aed_utils._aed_get_node_query_type(dfs[0]._nodeid)
-            if operation == "table":
+            if operation in ["table", "assign"]:
+                # Assign might have removed some columns and if it is only one dataframe,
+                # then return the same dataframe.
                 # Return the same dataframe if it is DataFrame object from table.
                 return dfs[0]
 

teradataml/dataframe/fastload.py

@@ -16,6 +16,8 @@ import pandas as pd
 
 from sqlalchemy import MetaData, Table, Column
 from sqlalchemy.exc import OperationalError as sqlachemyOperationalError
+
+from teradataml.context.context import _get_current_databasename
 from teradataml.dataframe import dataframe
 from teradataml.context.context import *
 from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils
@@ -28,13 +30,15 @@ from teradataml.dataframe.copy_to import copy_to_sql, \
                                         _create_pti_table_object, _extract_column_info, \
                                         _check_columns_insertion_compatible
 from teradataml.dataframe.data_transfer import _DataTransferUtils
-from teradatasqlalchemy.telemetry.queryband import collect_queryband
+from teradataml.telemetry_utils.queryband import collect_queryband
 
 
 @collect_queryband(queryband="fstLd")
 def fastload(df, table_name, schema_name=None, if_exists='replace', index=False, 
              index_label=None, primary_index=None, types=None, batch_size=None, 
-             save_errors=False, open_sessions=None):
+             save_errors=False, open_sessions=None, err_tbl_1_suffix=None,
+             err_tbl_2_suffix=None, err_tbl_name=None, warn_tbl_name=None,
+             err_staging_db=None):
     """
     The fastload() API writes records from a Pandas DataFrame to Teradata Vantage 
     using Fastload. FastLoad API can be used to quickly load large amounts of data
@@ -49,13 +53,24 @@ def fastload(df, table_name, schema_name=None, if_exists='replace', index=False,
        loaded.
     4. If there are any incorrect rows i.e. due to constraint violations, data type 
        conversion errors, etc., FastLoad protocol ignores those rows and inserts 
-       all valid rows. 
+       all valid rows.
     5. Rows in the DataFrame that failed to get inserted are categorized into errors 
        and warnings by FastLoad protocol and these errors and warnings are stored 
-       into respective error and warning tables by FastLoad API. 
-    6. If save_errors argument is True, the names of error and warning tables are 
-       shown once the fastload operation is complete. These tables will be persisted
-       using copy_to_sql.
+       into respective error and warning tables by FastLoad API.
+    6. fastload() creates 2 error tables when data is erroneous. These error tables are
+       refered as ERR_1 and ERR_2 tables.
+       * ERR_1 table is used to capture rows that violate the constraints or have format
+         errors. It typically contains information about rows that could not be inserted
+         into the target table due to data conversion errors, constraint violations, etc.
+       * ERR_2 table is used to log any duplicate rows found during the load process and
+         which are not loaded in target table, since fastLoad does not allow duplicate
+         rows to be loaded into the target table.
+    7. When "save_errors" argument is set to True, ERR_1 and ERR_2 tables are presisted.
+       The fully qualified names of ERR_1, ERR_2 and warning tables are shown once the
+       fastload operation is complete.
+    8. If user wants both error and warnings information from pandas dataframe to be
+       persisted rather than that from ERR_1 and ERR_2 tables, then "save_errors" should
+       be set to True and "err_tbl_name" must be provided.
 
     For additional information about FastLoad protocol through teradatasql driver, 
     please refer the FASTLOAD section of https://pypi.org/project/teradatasql/#FastLoad
@@ -165,10 +180,19 @@ def fastload(df, table_name, schema_name=None, if_exists='replace', index=False,
         save_errors:
             Optional Argument.
             Specifies whether to persist the error/warning information in Vantage 
-            or not. If save_errors is set to False, error/warnings are not persisted
-            as tables. If argument is set to True, the error and warnings information
-            are presisted and names of error and warning tables are returned. Otherwise,
-            the function returns None for the names of the tables.
+            or not.
+            Notes:
+               *  When "save_errors" is set to True, ERR_1 and ERR_2 tables are presisted.
+                 The fully qualified names of ERR_1, ERR_2 and warning table are returned
+                 in a dictionary containing keys named as "ERR_1_table", "ERR_2_table",
+                 "warnings_table" respectively.
+               * When "save_errors" is set to True and "err_tbl_name" is also provided,
+                 "err_tbl_name" takes precedence and error information is persisted into
+                  a single table using pandas dataframe rather than in ERR_1 and ERR_2 tables.
+               * When "save_errors" is set to False, errors and warnings information is
+                 not persisted as tables, but it is returned as pandas dataframes in a
+                 dictionary containing keys named as "errors_dataframe" and "warnings_dataframe"
+                 respectively.
             Default Value: False
             Types: bool
 
@@ -183,59 +207,182 @@ def fastload(df, table_name, schema_name=None, if_exists='replace', index=False,
             Default Value: None
             Types: int
 
+        err_tbl_1_suffix:
+            Optional Argument.
+            Specifies the suffix for error table 1 created by fastload job.
+            Default Value: "_ERR_1"
+            Types: String
+
+        err_tbl_2_suffix:
+            Optional Argument.
+            Specifies the suffix for error table 2 created by fastload job.
+            Default Value: "_ERR_2"
+            Types: String
+
+        err_tbl_name:
+            Optional Argument.
+            Specifies the name for error table. This argument takes precedence
+            over "save_errors" and saves error information in single table,
+            rather than ERR_1 and ERR_2 error tables.
+            Default value: "td_fl_<table_name>_err_<unique_id>" where table_name
+                           is name of target/staging table and unique_id is logon
+                           sequence number of fastload job.
+            Types: String
+
+        warn_tbl_name:
+            Optional Argument.
+            Specifies the name for warning table.
+            Default value: "td_fl_<table_name>_warn_<unique_id>" where table_name
+                           is name of target/staging table and unique_id is logon
+                           sequence number of fastload job.
+            Types: String
+
+        err_staging_db:
+            Optional Argument.
+            Specifies the name of the database to be used for creating staging
+            table and error/warning tables.
+            Note:
+                Current session user must have CREATE, DROP and INSERT table
+                permissions on err_staging_db database.
+            Types: String
+
     RETURNS:
         A dict containing the following attributes:
             1. errors_dataframe: It is a Pandas DataFrame containing error messages
-               thrown by fastload. DataFrame is empty if there are no errors.
+               thrown by fastload. DataFrame is empty if there are no errors or
+               "save_errors" is set to True.
             2. warnings_dataframe: It is a Pandas DataFrame containing warning messages 
                thrown by fastload. DataFrame is empty if there are no warnings.
-            3. errors_table: Name of the table containing errors. It is None, if 
-               argument save_errors is False.
-            4. warnings_table: Name of the table containing warnings. It is None, if 
-               argument save_errors is False.
+            3. errors_table: Fully qualified name of the table containing errors. It is
+               an empty string (''), if argument "save_errors" is set to False.
+            4. warnings_table: Fully qualified name of the table containing warnings. It is
+               an empty string (''), if argument "save_errors" is set to False.
+            5. ERR_1_table: Fully qualified name of the ERR 1 table created by fastload
+               job. It is an empty string (''), if argument "save_errors" is set to False.
+            6. ERR_2_table: Fully qualified name of the ERR 2 table created by fastload
+               job. It is an empty string (''), if argument "save_errors" is set to False.
 
     RAISES:
         TeradataMlException
 
     EXAMPLES:
         Saving a Pandas DataFrame using Fastload:
-            >>> from teradataml.dataframe.fastload import fastload
-            >>> from teradatasqlalchemy.types import *
-
-            >>> df = {'emp_name': ['A1', 'A2', 'A3', 'A4'],
-                'emp_sage': [100, 200, 300, 400],
-                'emp_id': [133, 144, 155, 177],
-                'marks': [99.99, 97.32, 94.67, 91.00]
-                }
-
-            >>> pandas_df = pd.DataFrame(df)
-
-            # a) Default execution
-            >>> fastload(df = pandas_df, table_name = 'my_table')
-
-            # b) Save a Pandas DataFrame with primary_index
-            >>> pandas_df = pandas_df.set_index(['emp_id'])
-            >>> fastload(df = pandas_df, table_name = 'my_table_1', primary_index='emp_id')
-
-            # c) Save a Pandas DataFrame using fastload() with index and primary_index
-            >>> fastload(df = pandas_df, table_name = 'my_table_2', index=True,
-                         primary_index='index_label')
-
-            # d) Save a Pandas DataFrame using types, appending to the table if it already exists
-            >>> fastload(df = pandas_df, table_name = 'my_table_3', schema_name = 'alice',
-                         index = True, index_label = 'my_index_label', 
-                         primary_index = ['emp_id'], if_exists = 'append',
-                         types = {'emp_name': VARCHAR, 'emp_sage':INTEGER,
-                        'emp_id': BIGINT, 'marks': DECIMAL})
-
-            # e) Save a Pandas DataFrame using levels in index of type MultiIndex
-            >>> pandas_df = pandas_df.set_index(['emp_id', 'emp_name'])
-            >>> fastload(df = pandas_df, table_name = 'my_table_4', schema_name = 'alice',
-                         index = True, index_label = ['index1', 'index2'], 
-                         primary_index = ['index1'], if_exists = 'replace')
-
-            # f) Save a Pandas DataFrame by opening spcified number of teradata data transfer sessions
-            >>> fastload(df = pandas_df, table_name = 'my_table_5', open_sessions = 2)
+        >>> from teradataml.dataframe.fastload import fastload
+        >>> from teradatasqlalchemy.types import *
+
+        >>> df = {'emp_name': ['A1', 'A2', 'A3', 'A4'],
+                  'emp_sage': [100, 200, 300, 400],
+                  'emp_id': [133, 144, 155, 177],
+                  'marks': [99.99, 97.32, 94.67, 91.00]
+                  }
+
+        >>> pandas_df = pd.DataFrame(df)
+
+        # Example 1: Default execution.
+        >>> fastload(df = pandas_df, table_name = 'my_table')
+
+        # Example 2: Save a Pandas DataFrame with primary_index.
+        >>> pandas_df = pandas_df.set_index(['emp_id'])
+        >>> fastload(df = pandas_df, table_name = 'my_table_1', primary_index='emp_id')
+
+        # Example 3: Save a Pandas DataFrame using fastload() with index and primary_index.
+        >>> fastload(df = pandas_df, table_name = 'my_table_2', index=True,
+                     primary_index='index_label')
+
+        # Example 4: Save a Pandas DataFrame using types, appending to the table if it already exists.
+        >>> fastload(df = pandas_df, table_name = 'my_table_3', schema_name = 'alice',
+                     index = True, index_label = 'my_index_label',
+                     primary_index = ['emp_id'], if_exists = 'append',
+                     types = {'emp_name': VARCHAR, 'emp_sage':INTEGER,
+                    'emp_id': BIGINT, 'marks': DECIMAL})
+
+        # Example 5: Save a Pandas DataFrame using levels in index of type MultiIndex.
+        >>> pandas_df = pandas_df.set_index(['emp_id', 'emp_name'])
+        >>> fastload(df = pandas_df, table_name = 'my_table_4', schema_name = 'alice',
+                     index = True, index_label = ['index1', 'index2'],
+                     primary_index = ['index1'], if_exists = 'replace')
+
+        # Example 6: Save a Pandas DataFrame by opening specified number of teradata data transfer sessions.
+        >>> fastload(df = pandas_df, table_name = 'my_table_5', open_sessions = 2)
+
+        # Example 7: Save a Pandas Dataframe to a table in specified target database "schema_name".
+        #            Save errors and warnings to database specified with "err_staging_db".
+        #            Save errors to table named as "err_tbl_name" and warnings to "warn_tbl_name".
+        #            Given that, user is connected to a database different from "schema_name"
+        #            and "err_staging_db".
+
+        # Create a pandas dataframe having one duplicate and one fualty row.
+        >>>> data_dict = {"C_ID": [301, 301, 302, 303, 304, 305, 306, 307, 308],
+                         "C_timestamp": ['2014-01-06 09:01:25', '2014-01-06 09:01:25',
+                                         '2015-01-06 09:01:25.25.122200', '2017-01-06 09:01:25.11111',
+                                         '2013-01-06 09:01:25', '2019-03-06 10:15:28',
+                                         '2014-01-06 09:01:25.1098', '2014-03-06 10:01:02',
+                                         '2014-03-06 10:01:20.0000']}
+        >>> my_df = pd.DataFrame(data_dict)
+
+        # Fastlaod data in non-default schema "target_db" and save erors and warnings in given tables.
+        >>> fastload(df=my_df, table_name='fastload_with_err_warn_tbl_stag_db',
+                    if_exists='replace', primary_index='C_ID', schema_name='target_db',
+                    types={'C_ID': INTEGER, 'C_timestamp': TIMESTAMP(6)},
+                    err_tbl_name='fld_errors', warn_tbl_name='fld_warnings',
+                    err_staging_db='stage_db')
+        Processed 9 rows in batch 1.
+        {'errors_dataframe':    batch_no                                      error_message
+        0         1   [Session 14527] [Teradata Database] [Error 26...,
+        'warnings_dataframe':         batch_no                                      error_message
+        0  batch_summary   [Session 14526] [Teradata SQL Driver] [Warnin...,
+        'errors_table': 'stage_db.fld_errors',
+        'warnings_table': 'stage_db.fld_warnings',
+        'ERR_1_table': '',
+        'ERR_2_table': ''}
+
+        # Validate loaded data table.
+        >>>  DataFrame(in_schema("target_db", "fastload_with_err_warn_tbl_stag_db"))
+        C_ID	C_timestamp
+        303	2017-01-06 09:01:25.111110
+        306	2014-01-06 09:01:25.109800
+        304	2013-01-06 09:01:25.000000
+        307	2014-03-06 10:01:02.000000
+        305	2019-03-06 10:15:28.000000
+        301	2014-01-06 09:01:25.000000
+        308	2014-03-06 10:01:20.000000
+
+        # Validate error and warning tables.
+        >>> DataFrame(in_schema("stage_db", "fld_errors"))
+        batch_no      error_message
+        1             [Session 14527] [Teradata Database] [Error 2673] FastLoad failed to insert 1 of 9 batched rows. Batched row 3 failed to insert because of Teradata Database error 2673 in "target_db"."fastload_with_err_warn_tbl_stag_db"."C_timestamp"
+
+        >>> DataFrame(in_schema("stage_db", "fld_warnings"))
+        batch_no        error_message
+        batch_summary   [Session 14526] [Teradata SQL Driver] [Warning 518] Found 1 duplicate or faulty row(s) while ending FastLoad of database table "target_db"."fastload_with_err_warn_tbl_stag_db": expected a row count of 8, got a row count of 7
+
+        # Example 8: Save a Pandas Dataframe to a table in specified target database "schema_name".
+        #            Save errors in ERR_1 and ERR_2 tables having user defined suffixes provided
+        #            in "err_tbl_1_suffix" and "err_tbl_2_suffix".
+        #            Source Pandas dataframe is same as Example 7.
+
+        >>> fastload(df=my_df, table_name = 'fastload_with_err_warn_tbl_stag_db',
+                     schema_name = 'target_db', if_exists = 'append',
+                     types={'C_ID': INTEGER, 'C_timestamp': TIMESTAMP(6)},
+                     err_staging_db='stage_db', save_errors=True,
+                     err_tbl_1_suffix="_user_err_1", err_tbl_2_suffix="_user_err_2")
+        {'errors_dataframe': Empty DataFrame
+         Columns: []
+         Index: [],
+         'warnings_dataframe':         batch_no                                      error_message
+         0  batch_summary   [Session 14699] [Teradata SQL Driver] [Warnin...,
+         'errors_table': '',
+         'warnings_table': 'stage_db.td_fl_fastload_with_err_warn_tbl_stag_db_warn_1730',
+         'ERR_1_table': 'stage_db.ml__fl_stag_1716272404181579_user_err_1',
+         'ERR_2_table': 'stage_db.ml__fl_stag_1716272404181579_user_err_2'}
+
+        # Validate ERR_1 and ERR_2 tables.
+        >>> DataFrame(in_schema("stage_db", "ml__fl_stag_1716270574550744_user_err_1"))
+        ErrorCode	ErrorFieldName	DataParcel
+        2673	F_C_timestamp	b'12E...'
+
+        >>> DataFrame(in_schema("stage_db", "ml__fl_stag_1716270574550744_user_err_2"))
+        C_ID	C_timestamp
 
     """
     # Deriving global connection using get_connection()
@@ -253,7 +400,10 @@ def fastload(df, table_name, schema_name=None, if_exists='replace', index=False,
                                     index=index, index_label=index_label, primary_index=primary_index,
                                     types=types, batch_size=batch_size,
                                     save_errors=save_errors, api_name='fastload',
-                                    use_fastload=True, open_sessions=open_sessions)
+                                    use_fastload=True, open_sessions=open_sessions,
+                                    err_tbl_1_suffix=err_tbl_1_suffix, err_tbl_2_suffix=err_tbl_2_suffix,
+                                    err_tbl_name=err_tbl_name, warn_tbl_name=warn_tbl_name,
+                                    err_staging_db=err_staging_db)
         # Validate DataFrame & related flags; Proceed only when True
         dt_obj._validate()
 
@@ -316,7 +466,8 @@ def fastload(df, table_name, schema_name=None, if_exists='replace', index=False,
         if not table_exists or if_exists.lower() == 'replace':
             dt_obj._create_or_replace_table(con, table_exists=table_exists)
 
-            fl_dict = _insert_from_dataframe(dt_obj, table_name, batch_size)
+            # Insert data to target table using fastload.
+            fl_dict = _insert_from_pd_dataframe_with_fastload(dt_obj, table_name, batch_size)
             
         # Check column compatibility for insertion when table exists and if_exists = 'append'
         if table_exists and if_exists.lower() == 'append':
@@ -332,28 +483,37 @@ def fastload(df, table_name, schema_name=None, if_exists='replace', index=False,
             try:
                 # Create staging table and use FastLoad to load data.
                 # Then copy all the rows from staging table to target table using insert_into sql.
-                stag_table_name = UtilFuncs._generate_temp_table_name(prefix="fl_stag",
+                # If err_staging_db is not provided, create staging table
+                # object in default connected DB.
+                if err_staging_db is None:
+                    err_staging_db = _get_current_databasename()
+                stag_table_name = UtilFuncs._generate_temp_table_name(databasename=err_staging_db,
+                                                                      prefix="fl_stag",
                                                                       gc_on_quit=False,
                                                                       quote=False,
                                                                       table_type=TeradataConstants.TERADATA_TABLE)
 
-                # Get the table name without schema name for further steps
-                stag_table_name = stag_table_name.split('.')[-1].replace('"', '')
-                # Create staging table object
-                dt_obj._create_table(con, table_name=stag_table_name)
+                # Get the table name without schema name for further steps.
+                stag_table_name = UtilFuncs._extract_table_name(stag_table_name)
+                # Create staging table object.
+                dt_obj._create_table(con, table_name=stag_table_name,
+                                     schema_name=err_staging_db)
 
-                # Insert data to staging table using faslload
-                fl_dict = _insert_from_dataframe(dt_obj, stag_table_name, batch_size)
+                # Insert data to staging table using fastload.
+                fl_dict = _insert_from_pd_dataframe_with_fastload(dt_obj, stag_table_name, batch_size, err_staging_db)
 
-                # Insert data from staging table to target data.
+                # Insert data from staging table to target table.
                 df_utils._insert_all_from_table(table_name,
-                                                dt_obj._get_fully_qualified_table_name(table_name=stag_table_name),
-                                                cols[0], schema_name)
+                                                stag_table_name,
+                                                cols[0],
+                                                schema_name,
+                                                err_staging_db)
             except:
                 raise
             finally:
+                # Drop the staging table.
                 if stag_table_name:
-                    UtilFuncs._drop_table(dt_obj._get_fully_qualified_table_name(stag_table_name))
+                    UtilFuncs._drop_table(dt_obj._get_fully_qualified_table_name(stag_table_name, err_staging_db))
 
     except (TeradataMlException, ValueError, TypeError):
         raise
@@ -363,10 +523,10 @@ def fastload(df, table_name, schema_name=None, if_exists='replace', index=False,
     return fl_dict
 
 
-def _insert_from_dataframe(dt_obj, table_name, batch_size):
+def _insert_from_pd_dataframe_with_fastload(dt_obj, table_name, batch_size, to_schema_name=None):
     """
-    This is an internal function used to to sequentially extract column info from DataFrame,
-    iterate rows, and insert rows manually. Used for Insertions to Tables with Pandas index.
+    This is an internal function used to sequentially extract column info from pandas DataFrame,
+    iterate rows, and insert rows manually. Used for insertions to Tables with Pandas index.
     This uses DBAPI's escape functions for Fastload which is a batch insertion method.
 
     PARAMETERS:
@@ -382,6 +542,10 @@ def _insert_from_dataframe(dt_obj, table_name, batch_size):
             Specifies the number of rows to be inserted in a batch.
             Types: Int
 
+        to_schema_name:
+            Optional Argument.
+            Specifies name of the database schema where target table needs to be created.
+
     RETURNS:
         dict
 
@@ -389,7 +553,7 @@ def _insert_from_dataframe(dt_obj, table_name, batch_size):
         Exception
 
     EXAMPLES:
-        _insert_from_dataframe(dt_obj, table_name, batch_size=100)
+        _insert_from_pd_dataframe_with_fastload(dt_obj, table_name, batch_size=100)
     """
     conn = get_connection().connection
     # Create a cursor from connection object
@@ -430,9 +594,9 @@ def _insert_from_dataframe(dt_obj, table_name, batch_size):
         # FastLoad. Returns an empty string if the request is not a FastLoad.
 
         # Quoted, schema-qualified table name.
-        table = dt_obj._get_fully_qualified_table_name(table_name)
+        table = dt_obj._get_fully_qualified_table_name(table_name, to_schema_name)
 
-        # Form the INSERT query for fastlod.
+        # Form the INSERT query for fastload.
         ins = dt_obj._form_insert_query(table)
 
         #  Turn off autocommit before the Fastload insertion
@@ -440,20 +604,20 @@ def _insert_from_dataframe(dt_obj, table_name, batch_size):
                                         DriverEscapeFunctions.AUTOCOMMIT_OFF)
 
         # Initialize dict template for saving error/warning information
-        err_dict = {key:[] for key in ['batch_no', 'error_message']}
-        warn_dict = {key:[] for key in ['batch_no', 'error_message']}
+        err_dict = {key: [] for key in ['batch_no', 'error_message']}
+        warn_dict = {key: [] for key in ['batch_no', 'error_message']}
 
         batch_number = 1
         num_batches = int(dt_obj.df.shape[0]/batch_size)
 
-
         # Empty queryband buffer before SQL call.
         UtilFuncs._set_queryband()
+
         for i in range(0, dt_obj.df.shape[0], batch_size):
             # Add the remaining rows to last batch after second last batch
             if (batch_number == num_batches) :
                 last_elem = dt_obj.df.shape[0]
-            else :
+            else:
                 last_elem = i + batch_size
 
             pdf = dt_obj.df.iloc[i:last_elem]
@@ -466,10 +630,10 @@ def _insert_from_dataframe(dt_obj, table_name, batch_size):
                 if dt_obj.index is True:
                     insert_list2.extend(row[0]) if is_multi_index else insert_list2.append(row[0])
                 insert_list.append(insert_list2)
-            # Execute insert statement
-            cur.execute (ins, insert_list)
+            # Execute insert statement.
+            cur.execute(ins, insert_list)
 
-            # Get error and warning information
+            # Get error and warning information from cursor.
             err, _ = dt_obj._process_fastexport_errors_warnings(ins)
             if len(err) != 0:
                 err_dict['batch_no'].extend([batch_number] * len(err))
@@ -487,7 +651,6 @@ def _insert_from_dataframe(dt_obj, table_name, batch_size):
         logon_seq_number = dt_obj._process_escape_functions(cur, escape_function= \
                                                             DriverEscapeFunctions.LOGON_SEQ_NUM,
                                                             insert_query=ins)
-
         # Commit the rows
         conn.commit()
 
@@ -497,19 +660,38 @@ def _insert_from_dataframe(dt_obj, table_name, batch_size):
         if len(warn) != 0:
             warn_dict['batch_no'].extend(['batch_summary'] * len(warn))
             warn_dict['error_message'].extend(warn)
-        
-        # Get error and warning informations for error and warning tables, persist
+
+        # Get error and warning information for error and warning tables, persist
         # error and warning tables to Vantage if user has specified save_error as True
         # else show it as pandas dataframe on console.
         pd_err_df = dt_obj._get_pandas_df_from_errors_warnings(err_dict)
-        if not pd_err_df.empty:
-            msg_type = "err"
-            error_tablename = dt_obj._create_error_warnings_table(pd_err_df, msg_type, logon_seq_number[0][0])
-
         pd_warn_df = dt_obj._get_pandas_df_from_errors_warnings(warn_dict)
-        if not pd_warn_df.empty:
-            msg_type = "warn"
-            warn_tablename = dt_obj._create_error_warnings_table(pd_warn_df, msg_type, logon_seq_number[0][0])
+
+        # Create persistent tables using pandas df if
+        # save_errors=True or
+        # tables names for errors or warning are provided by user.
+        if dt_obj.save_errors or dt_obj.err_tbl_name:
+            if not pd_err_df.empty:
+                error_tablename = dt_obj._create_error_warnings_table(pd_err_df, "err", logon_seq_number[0][0],
+                                                                      dt_obj.err_tbl_name)
+        if dt_obj.save_errors or dt_obj.warn_tbl_name:
+            if not pd_warn_df.empty:
+                warn_tablename = dt_obj._create_error_warnings_table(pd_warn_df, "warn", logon_seq_number[0][0],
+                                                                     dt_obj.warn_tbl_name)
+
+        # Generate ERR_1 and ERR_2 table names if save_errors=True and
+        # errors are not stored in user provided error table name.
+        if dt_obj.save_errors and not dt_obj.err_tbl_name:
+            err_1_table = "{}.{}{}".format(dt_obj.err_staging_db if dt_obj.err_staging_db else _get_current_databasename(),
+                                           table_name,
+                                           dt_obj.err_tbl_1_suffix if dt_obj.err_tbl_1_suffix else "_ERR_1")
+            err_2_table = "{}.{}{}".format(dt_obj.err_staging_db if dt_obj.err_staging_db else _get_current_databasename(),
+                                           table_name,
+                                           dt_obj.err_tbl_2_suffix if dt_obj.err_tbl_2_suffix else "_ERR_2")
+
+        else:
+            err_1_table = ""
+            err_2_table = ""
 
     except Exception:
         conn.rollback()
@@ -520,7 +702,8 @@ def _insert_from_dataframe(dt_obj, table_name, batch_size):
         cur.close()
 
     return {"errors_dataframe": pd_err_df, "warnings_dataframe": pd_warn_df,
-            "errors_table": error_tablename, "warnings_table": warn_tablename}
+            "errors_table": error_tablename, "warnings_table": warn_tablename,
+            "ERR_1_table": err_1_table, "ERR_2_table": err_2_table}
 
 
 def _get_batchsize(df):