teradataml 20.0.0.3__py3-none-any.whl → 20.0.0.4__py3-none-any.whl

teradataml/dbutils/dbutils.py

@@ -9,35 +9,45 @@ teradataml db utilities
 ----------
 A teradataml database utility functions provide interface to Teradata Vantage common tasks such as drop_table, drop_view, create_table etc.
 """
-import concurrent.futures, json, os, tempfile, shutil
+import concurrent.futures
+import json
+import os
+import re
+import shutil
+import tempfile
 from datetime import datetime
+
 import pandas as pd
+from sqlalchemy import (CheckConstraint, Column, ForeignKeyConstraint,
+                        MetaData, PrimaryKeyConstraint, Table,
+                        UniqueConstraint)
 from sqlalchemy.sql.functions import Function
-from teradataml.context import context as tdmlctx
-from teradataml.common.utils import UtilFuncs
-from teradataml.common.messages import Messages
-from teradataml.common.messagecodes import MessageCodes
+from teradatasql import OperationalError
+from teradatasqlalchemy.dialect import TDCreateTablePost as post
+from teradatasqlalchemy.dialect import dialect as td_dialect
+from teradatasqlalchemy.dialect import preparer
+
+import teradataml.dataframe as tdmldf
+from teradataml.common.constants import (SessionParamsPythonNames,
+                                         SessionParamsSQL, SQLConstants,
+                                         TableOperatorConstants,
+                                         TeradataTableKindConstants)
 from teradataml.common.exceptions import TeradataMlException
-from teradataml.common.constants import TeradataTableKindConstants
+from teradataml.common.messagecodes import MessageCodes
+from teradataml.common.messages import Messages
 from teradataml.common.sqlbundle import SQLBundle
-from teradataml.common.constants import SQLConstants, SessionParamsSQL, SessionParamsPythonNames
-from teradataml.common.constants import TableOperatorConstants
-import teradataml.dataframe as tdmldf
+from teradataml.common.utils import UtilFuncs
+from teradataml.context import context as tdmlctx
 from teradataml.options.configure import configure
-from teradataml.utils.utils import execute_sql
-from teradataml.utils.validators import _Validators
-from teradataml.utils.internal_buffer import _InternalBuffer
-from teradatasql import OperationalError
-from teradatasqlalchemy.dialect import preparer, dialect as td_dialect
-from teradatasqlalchemy.dialect import TDCreateTablePost as post
 from teradataml.telemetry_utils.queryband import collect_queryband
-from sqlalchemy import Table, Column, MetaData, CheckConstraint, \
-    PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint
 from teradataml.utils.internal_buffer import _InternalBuffer
+from teradataml.utils.utils import execute_sql
+from teradataml.utils.validators import _Validators
 
 
 @collect_queryband(queryband='DrpTbl')
-def db_drop_table(table_name, schema_name=None, suppress_error=False):
+def db_drop_table(table_name, schema_name=None, suppress_error=False,
+                  datalake_name=None, purge=None):
     """
     DESCRIPTION:
         Drops the table from the given schema.
@@ -61,6 +71,22 @@ def db_drop_table(table_name, schema_name=None, suppress_error=False):
             Default Value: False
             Types: str
 
+        datalake_name:
+            Optional Argument
+            Specifies name of the datalake to drop table from.
+            Note:
+                 "schema_name" must be provided while using this argument.
+            Default Value: None
+            Types: str
+
+        purge:
+            Optional Argument
+            Specifies whether to use purge clause or not while dropping datalake table.
+            It is only applicable when "datalake_name" argument is used. When "datalake_name" is specified,
+            but "purge" is not specified, data is purged by default.
+            Default Value: None
+            Types: bool
+
     RETURNS:
         True - if the operation is successful.
 
@@ -70,25 +96,44 @@ def db_drop_table(table_name, schema_name=None, suppress_error=False):
     EXAMPLES:
         >>> load_example_data("dataframe", "admissions_train")
 
-        # Drop table in current database
+        # Example 1: Drop table in current database.
         >>> db_drop_table(table_name = 'admissions_train')
 
-        # Drop table from the given schema
+        # Example 2: Drop table from the given schema.
         >>> db_drop_table(table_name = 'admissions_train', schema_name = 'alice')
+
+        #Example 3: Drop a table from datalake and purge the data.
+        >>> db_drop_table(table_name = 'datalake_table', schema_name = 'datalake_db',
+        ...               datalake_name='datalake', purge=True)
+
     """
     # Argument validations
     awu_matrix = []
     awu_matrix.append(["schema_name", schema_name, True, (str), True])
     awu_matrix.append(["table_name", table_name, False, (str), True])
-
+    awu_matrix.append(["datalake_name", datalake_name, True, (str), True])
+    awu_matrix.append(["purge", purge, True, (bool, type(None)), True])
     # Validate argument types
     _Validators._validate_function_arguments(awu_matrix)
 
+    # Process datalake related arguments.
+    purge_clause = None
+    if datalake_name is not None:
+        if schema_name is None:
+            err_ = Messages.get_message(MessageCodes.DEPENDENT_ARG_MISSING, "schema_name",
+                                        "datalake_name")
+            raise TeradataMlException(err_, MessageCodes.DEPENDENT_ARG_MISSING)
+
+        if purge is False:
+            purge_clause = "NO PURGE"
+        else:
+            purge_clause = "PURGE ALL"
+
     # Joining view and schema names in the format "schema_name"."view_name"
-    table_name = _get_quoted_object_name(schema_name, table_name)
+    table_name = _get_quoted_object_name(schema_name, table_name, datalake_name)
 
     try:
-        return UtilFuncs._drop_table(table_name)
+        return UtilFuncs._drop_table(table_name, purge_clause=purge_clause)
     except (TeradataMlException, OperationalError):
         if suppress_error:
             pass
@@ -162,7 +207,7 @@ def db_drop_view(view_name, schema_name=None):
 
 
 @collect_queryband(queryband='LstTbls')
-def db_list_tables(schema_name=None, object_name=None, object_type='all'):
+def db_list_tables(schema_name=None, object_name=None, object_type='all', datalake_name=None):
     """
     DESCRIPTION:
         Lists the Vantage objects(table/view) names for the specified schema name.
@@ -179,10 +224,10 @@ def db_list_tables(schema_name=None, object_name=None, object_type='all'):
             Optional Argument.
             Specifies a table/view name or pattern to be used for filtering them from the database.
             Pattern may contain '%' or '_' as pattern matching characters.
-            A '%' represents any string of zero or more arbitrary characters. Any string of characters is acceptable as
-            a replacement for the percent.
-            A '_' represents exactly one arbitrary character. Any single character is acceptable in the position in
-            which the underscore character appears.
+            - '%' represents any string of zero or more arbitrary characters. Any string of characters is acceptable as
+              a replacement for the percent.
+            - '_' represents exactly one arbitrary character. Any single character is acceptable in the position in
+              which the underscore character appears.
             Note:
                 * If '%' is specified in 'object_name', then the '_' character is not evaluated for an arbitrary character.
             Default Value: None
@@ -203,6 +248,14 @@ def db_list_tables(schema_name=None, object_name=None, object_type='all'):
             Default Value: 'all'
             Types: str
 
+        datalake_name:
+            Optional Argument.
+            Specifies the name of datalake to list tables from.
+            Note:
+                "schema_name" must be provided while using this argument.
+            Default Value: None
+            Types: str
+
     RETURNS:
         Pandas DataFrame
 
@@ -211,38 +264,40 @@ def db_list_tables(schema_name=None, object_name=None, object_type='all'):
         OperationalError    - If any errors are raised from Vantage.
 
     EXAMPLES:
-        # Example 1 - List all object types in the default schema
+        # Example 1: List all object types in the default schema
         >>> load_example_data("dataframe", "admissions_train")
         >>> db_list_tables()
 
-        # Example 2 - List all the views in the default schema
+        # Example 2: List all the views in the default schema
         >>> execute_sql("create view temporary_view as (select 1 as dummy_col1, 2 as dummy_col2);")
         >>> db_list_tables(None , None, 'view')
 
-        # Example 3 - List all the object types in the default schema whose names begin with 'abc' followed by any number
+        # Example 3: List all the object types in the default schema whose names begin with 'abc' followed by any number
         # of characters in the end.
         >>> execute_sql("create view abcd123 as (select 1 as dummy_col1, 2 as dummy_col2);")
         >>> db_list_tables(None, 'abc%', None)
 
-        # Example 4 - List all the tables in the default schema whose names begin with 'adm' followed by any number of
+        # Example 4: List all the tables in the default schema whose names begin with 'adm' followed by any number of
         # characters and ends with 'train'.
         >>> load_example_data("dataframe", "admissions_train")
         >>> db_list_tables(None, 'adm%train', 'table')
 
-        # Example 5 - List all the views in the default schema whose names begin with any character but ends with 'abc'
+        # Example 5: List all the views in the default schema whose names begin with any character but ends with 'abc'
         >>> execute_sql("create view view_abc as (select 1 as dummy_col1, 2 as dummy_col2);")
         >>> db_list_tables(None, '%abc', 'view')
 
-        # Example 6 - List all the volatile tables in the default schema whose names begin with 'abc' and ends with any
+        # Example 6: List all the volatile tables in the default schema whose names begin with 'abc' and ends with any
         # arbitrary character and has a length of 4
         >>> execute_sql("CREATE volatile TABLE abcd(col0 int, col1 float) NO PRIMARY INDEX;")
         >>> db_list_tables(None, 'abc_', 'volatile')
 
-        # Example 7 - List all the temporary objects created by teradataml in the default schema whose names begins and
+        # Example 7: List all the temporary objects created by teradataml in the default schema whose names begins and
         # ends with any number of arbitrary characters but contains 'filter' in between.
         >>> db_list_tables(None, '%filter%', 'temp')
-    """
 
+        # Example 8: List all the tables in datalake's database.
+        >>> db_list_tables(schema_name='datalake_db_name', datalake_name='datalake_name')
+    """
     if tdmlctx.get_connection() is None:
         raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_CONTEXT_CONNECTION),
                                   MessageCodes.INVALID_CONTEXT_CONNECTION)
@@ -257,12 +312,18 @@ def db_list_tables(schema_name=None, object_name=None, object_type='all'):
                               TeradataTableKindConstants.VOLATILE.value,
                               TeradataTableKindConstants.TEMP.value]
     awu_matrix.append(["object_type", object_type, True, (str), True, permitted_object_types])
-
+    awu_matrix.append(["datalake_name", datalake_name, True, (str), True])
     # Validate argument types
     _Validators._validate_function_arguments(awu_matrix)
 
+    # 'schema_name' must be provided while using 'datalake_name'.
+    _Validators._validate_dependent_argument(dependent_arg='datalake_name',
+                                             dependent_arg_value=datalake_name,
+                                             independent_arg='schema_name',
+                                             independent_arg_value=schema_name)
+
     try:
-        return _get_select_table_kind(schema_name, object_name, object_type)
+        return _get_select_table_kind(schema_name, object_name, object_type, datalake_name)
     except TeradataMlException:
         raise
     except OperationalError:
@@ -272,21 +333,49 @@ def db_list_tables(schema_name=None, object_name=None, object_type='all'):
                                   MessageCodes.LIST_DB_TABLES_FAILED) from err
 
 
-def _get_select_table_kind(schema_name, table_name, table_kind):
+def _convert_sql_search_string_to_regex(sql_str):
+    """Internal function to convert SQL string matching patterns to python regex."""
+    if sql_str:
+        # sql_str[1:-1] Removes single quotes from sql_str.
+        sql_str = sql_str[1:-1]
+
+        # If '%' is specified in 'sql_str',
+        # then the '_' character is not evaluated for an arbitrary character.
+        if '%' in sql_str:
+            # Replace % with .* if not preceded by a backslash.
+            sql_str = re.sub(r'(?<!\\)%', r'.*', sql_str, flags=re.IGNORECASE)
+            # Remove the escape character for the replacements.
+            sql_str = sql_str.replace(r'\%', '%')
+        else:
+            # Replace _ with . if not preceded by a backslash.
+            sql_str = re.sub(r'(?<!\\)_', r'.', sql_str, flags=re.IGNORECASE)
+            # Remove the escape character for the replacements.
+            sql_str = sql_str.replace(r'\_', '_')
+
+        # Add boundaries if the string doesn't start or end with '.*' i.e. SQL '%'.
+        if not sql_str.startswith('.*'):
+            sql_str = '^' + sql_str  # Anchor to the start of the string.
+        if not sql_str.endswith('.*'):
+            sql_str = sql_str + '$'  # Anchor to the end of the string.
+    return sql_str
+
+
+def _get_select_table_kind(schema_name, table_name, table_kind, datalake_name):
     """
-    Get the list of the table names from the specified schema name.
+    Get the list of the table names from the specified schema name and datalake.
 
     PARAMETERS:
         schema_name - The Name of schema in the database. The default value is the current database name.
         table_name -  The pattern to be used to filtering the table names from the database.
-                      The table name argument can contain '%' as pattern matching charecter.For example '%abc'
-                      will return all table names starting with any charecters and ending with abc.
+                      The table name argument can contain '%' as pattern matching character.For example '%abc'
+                      will return all table names starting with any characters and ending with abc.
         table_kind -  The table kind to apply the filter. The valid values are 'all','table','view','volatile','temp'.
                       all - list the all the table kinds.
                       table - list only tables.
                       view - list only views.
                       volatile - list only volatile temp.
                       temp - list all teradata ml temporary objects created in the specified database.
+        datalake_name - The name of datalake to search schema in.
     RETURNS:
         Panda's DataFrame - if the operation is successful.
 
@@ -301,60 +390,106 @@ def _get_select_table_kind(schema_name, table_name, table_kind):
         object_name_str = "'{0}'".format(table_name)
     object_table_kind = None
 
-    # Check the schema name.
-    if schema_name is None:
-        schema_name = tdmlctx._get_current_databasename()
-
-    # Check the table kind.
-    if (table_kind == TeradataTableKindConstants.VOLATILE.value):
-        query = SQLBundle._build_help_volatile_table()
-    else:
-        # Tablekind:
-        # 'O' - stands for Table with no primary index and no partitioning
-        # 'Q' - stands for Queue table
-        # 'T' - stands for a Table with a primary index or primary AMP index, partitioning, or both.
-        #       Or a partitioned table with NoPI
-        # 'V' - stands for View
-        if (table_kind == TeradataTableKindConstants.TABLE.value):
-            object_table_kind = "'{0}','{1}','{2}'".format('O', 'Q', 'T')
-        elif (table_kind == TeradataTableKindConstants.VIEW.value):
-            object_table_kind = "'{0}'".format('V')
-        elif (table_kind == TeradataTableKindConstants.TEMP.value):
-            if table_name is None:
-                object_name_str = "'{0}'".format(TeradataTableKindConstants.ML_PATTERN.value)
-            else:
-                object_name_str = "'{0}','{1}'".format(table_name,
-                                                       TeradataTableKindConstants.ML_PATTERN.value)
+    # Tablekind:
+    # 'O' - stands for Table with no primary index and no partitioning
+    # 'Q' - stands for Queue table
+    # 'T' - stands for a Table with a primary index or primary AMP index, partitioning, or both.
+    #       Or a partitioned table with NoPI
+    # 'V' - stands for View
+    if (table_kind == TeradataTableKindConstants.TABLE.value):
+        object_table_kind = ['O', 'Q', 'T']
+    elif (table_kind == TeradataTableKindConstants.VIEW.value):
+        object_table_kind = ['V']
+    elif (table_kind == TeradataTableKindConstants.TEMP.value):
+        if table_name is None:
+            object_name_str = "'{0}'".format(TeradataTableKindConstants.ML_PATTERN.value)
         else:
-            object_table_kind = "'{0}','{1}','{2}','{3}'".format('O', 'Q', 'T', 'V')
-        query = SQLBundle._build_select_table_kind(schema_name, object_name_str, object_table_kind)
-
-    try:
-        pddf = pd.read_sql(query, tdmlctx.td_connection.connection)
-        # Check if all table kind is requested and add also volatile tables to the pdf.
-        if (table_kind == TeradataTableKindConstants.ALL.value):
+            object_name_str = "'{0}','{1}'".format(table_name,
+                                                   TeradataTableKindConstants.ML_PATTERN.value)
+    else:
+        object_table_kind = ['O', 'Q', 'T', 'V']
+
+    if datalake_name is None:
+        # Check the schema name.
+        if schema_name is None:
+            schema_name = tdmlctx._get_current_databasename()
+
+        # Create an empty dataframe with desired column name.
+        pddf = pd.DataFrame(columns=[TeradataTableKindConstants.REGULAR_TABLE_NAME.value])
+
+        # Check the table kind.
+        if table_kind != TeradataTableKindConstants.VOLATILE.value:
+            if object_table_kind is not None:
+                object_table_kind = ', '.join([f"'{value}'" for value in object_table_kind])
+            query = SQLBundle._build_select_table_kind(schema_name, object_name_str, object_table_kind)
+            pddf = pd.read_sql(query, tdmlctx.td_connection.connection)
+
+        # Check if all table kind or volatile table kind is requested.
+        # If so,add volatile tables to the pddf.
+        if table_kind == TeradataTableKindConstants.ALL.value or \
+                table_kind == TeradataTableKindConstants.VOLATILE.value:
+            # Create list of volatile tables.
             try:
-                # Add volatile tables to all dataframe.
                 vtquery = SQLBundle._build_help_volatile_table()
                 vtdf = pd.read_sql(vtquery, tdmlctx.td_connection.connection)
                 if not vtdf.empty:
+                    # Volatile table query returns different column names.
+                    # So, rename its column names to match with normal
+                    # 'SELECT TABLENAME FROM DBC.TABLESV' query results.
                     columns_dict = {TeradataTableKindConstants.VOLATILE_TABLE_NAME.value:
                                         TeradataTableKindConstants.REGULAR_TABLE_NAME.value}
                     vtdf.rename(columns=columns_dict, inplace=True)
+                    # Volatile table names might contain leading whitespaces. Remove those.
+                    vtdf[TeradataTableKindConstants.REGULAR_TABLE_NAME.value] = vtdf[TeradataTableKindConstants.REGULAR_TABLE_NAME.value].str.strip()
+                    # Filter volatile tables using table name pattern.
+                    if object_name_str and (object_name_str := _convert_sql_search_string_to_regex(object_name_str)):
+                        name_filter = vtdf[TeradataTableKindConstants.REGULAR_TABLE_NAME.value].str.strip().str.match(
+                            object_name_str,
+                            na=False,
+                            flags=re.IGNORECASE)
+                        vtdf = vtdf[name_filter]
+                    # Concat existing list with volatile tables list.
                     frames = [pddf, vtdf[[TeradataTableKindConstants.REGULAR_TABLE_NAME.value]]]
                     pddf = pd.concat(frames)
                     pddf.reset_index(drop=True, inplace=True)
             except Exception as err:
-                # No volatle tables exist.
+                # No volatile tables exist.
                 pass
-        if (table_kind == TeradataTableKindConstants.VOLATILE.value):
-            columns_dict = {TeradataTableKindConstants.VOLATILE_TABLE_NAME.value:
-                                TeradataTableKindConstants.REGULAR_TABLE_NAME.value}
-            pddf.rename(columns=columns_dict, inplace=True)
-            return pddf[[TeradataTableKindConstants.REGULAR_TABLE_NAME.value]]
         else:
             return pddf
-    except Exception as err:
+    else:
+        # TODO: when OTF team enables VSD support for datalake tables
+        #  with epic: https://teradata-pe.atlassian.net/browse/OTF-454,
+        #  this can be changed to use VSD_tablesV table which is
+        #  similar to DBC.TABLESV.
+        # For datalake tables' information we need to use help database and
+        # then apply filter for table kind and table substring.
+        # We can't use select from DBC.TABLESV.
+        sqlbundle = SQLBundle()
+        help_db_sql = sqlbundle._get_sql_query(SQLConstants.SQL_HELP_DATABASE)
+        pddf = pd.read_sql(help_db_sql.format(_get_quoted_object_name(schema_name=datalake_name,
+                                                                      object_name=schema_name)),
+                           tdmlctx.td_connection.connection)
+
+        if object_name_str:
+            object_name_str = _convert_sql_search_string_to_regex(object_name_str)
+            if object_name_str:
+                name_filter = pddf['Table/View/Macro Name'].str.strip().str.match(object_name_str, na=False,
+                                                                      flags=re.IGNORECASE)
+                pddf = pddf[name_filter]
+
+        if object_table_kind is not None:
+            object_filter = pddf['Kind'].isin(object_table_kind)
+            pddf = pddf[object_filter]
+
+        columns_dict = {'Table/View/Macro Name':
+                            TeradataTableKindConstants.REGULAR_TABLE_NAME.value}
+        pddf.rename(columns=columns_dict, inplace=True)
+
+    # Return only filtered columns.
+    if not pddf.empty:
+        return pddf[[TeradataTableKindConstants.REGULAR_TABLE_NAME.value]]
+    else:
         return pd.DataFrame()
 
 
@@ -444,6 +579,7 @@ def db_transaction(func):
         True
         >>>
     """
+
     def execute_transaction(*args, **kwargs):
         auto_commit_off = "{fn teradata_nativesql}{fn teradata_autocommit_off}"
         auto_commit_on = "{fn teradata_nativesql}{fn teradata_autocommit_on}"
@@ -479,6 +615,7 @@ def db_transaction(func):
 
     return execute_transaction
 
+
 def _execute_stored_procedure(function_call, fetchWarnings=True, expect_none_result=False):
     """
     DESCRIPTION:
@@ -576,7 +713,7 @@ def _get_function_call_as_string(sqlcFuncObj):
     return str(sqlcFuncObj.compile(**kw))
 
 
-def _get_quoted_object_name(schema_name, object_name):
+def _get_quoted_object_name(schema_name, object_name, datalake=None):
     """
     DESCRIPTION:
         This function quotes and joins schema name to the object name which can either be table or a view.
@@ -585,12 +722,18 @@ def _get_quoted_object_name(schema_name, object_name):
         schema_name
             Required Argument.
             Specifies the schema name.
-            Types: str
+            Type: str
 
         object_name
             Required Argument.
             Specifies the object name either table or view.
-            Types: str
+            Type: str
+
+        datalake
+            Optional Argument.
+            Specifies the datalake name.
+            Default value: None
+            Type: str
 
     RAISES:
         None
@@ -612,6 +755,8 @@ def _get_quoted_object_name(schema_name, object_name):
         schema_name = tdp.quote(tdmlctx._get_current_databasename())
 
     quoted_object_name = "{0}.{1}".format(schema_name, tdp.quote(object_name))
+    if datalake is not None:
+        quoted_object_name = "{}.{}".format(tdp.quote(datalake), quoted_object_name)
     return quoted_object_name
 
 
@@ -712,7 +857,6 @@ def view_log(log_type="script", num_lines=1000, query_id=None, log_dir=None):
         # Validate num_lines is a positive integer.
         _Validators._validate_positive_int(num_lines, "num_lines")
 
-
         # Query for viewing last n lines of script log.
         view_log_query = TableOperatorConstants.SCRIPT_LOG_QUERY.value \
             .format(num_lines, configure.default_varchar_size)
@@ -733,8 +877,9 @@ def view_log(log_type="script", num_lines=1000, query_id=None, log_dir=None):
                 err_msg = 'Please provide directory path instead of file path.'.format(
                     log_dir)
                 raise TeradataMlException(err_msg, MessageCodes.INPUT_FILE_NOT_FOUND)
-        from teradataml.scriptmgmt.UserEnv import _get_auth_token, _get_ues_url, \
-            _process_ues_response
+        from teradataml.scriptmgmt.UserEnv import (_get_auth_token,
+                                                   _get_ues_url,
+                                                   _process_ues_response)
         ues_url = _get_ues_url(logs=True, query_id=query_id)
         response = UtilFuncs._http_request(ues_url, headers=_get_auth_token())
         resp = _process_ues_response(api_name="view_log", response=response)
@@ -798,9 +943,10 @@ def _fetch_url_and_save(url, file_path):
 def _check_if_python_packages_installed():
     """
     DESCRIPTION:
-        Function to set global variable 'python_packages_installed' to True
-        or False based on whether the Vantage node has Python and add-on
-        packages including pip3 installed.
+        Function to set the following global variables based on whether the Vantage node
+        has Python and add-on packages including pip3 installed.
+        - 'python_packages_installed' to True or False 
+        - 'python_version_vantage' to the version of Python installed on Vantage.
 
     PARAMETERS:
         None.
@@ -814,14 +960,21 @@ def _check_if_python_packages_installed():
     EXAMPLES:
         _check_if_python_packages_installed()
     """
+    if tdmlctx.python_packages_installed:
+        # Skip check if Python and add-on packages are already installed and checked.
+        return
+
     # Check if Python interpreter and add-ons packages are installed or not.
     try:
         query = TableOperatorConstants.CHECK_PYTHON_INSTALLED.value.format(configure.indb_install_location)
-        UtilFuncs._execute_query(query=query)
+        opt = UtilFuncs._execute_query(query=query)
+
+        python_version = opt[0][0].split(" -- ")[1].split(" ")[1].strip()
 
         # If query execution is successful, then Python and add-on packages are
         # present.
         tdmlctx.python_packages_installed = True
+        tdmlctx.python_version_vantage = python_version
     except Exception as err:
         # Raise Exception if the error message does not contain
         # "bash: pip3: command not found".
@@ -932,6 +1085,203 @@ def db_python_package_details(names=None):
     return ret_val
 
 
+def _db_python_package_version_diff(packages=None, only_diff=True):
+    """
+    DESCRIPTION:
+        Internal function to get the pandas dataframe containing the difference in the Python
+        packages installed on Vantage and the packages mentioned in the argument "packages".
+        Note:
+            * Using this function is valid only when Python interpreter and add-on packages
+                are installed on the Vantage node.
+            * This function also checks for differences in Python packages versions given
+                part of package name as string.
+            * Returns pandas dataframe of only differences when the argument `only_diff` is set to
+                True. Otherwise, returns all the packages.
+
+    PARAMETERS:
+        packages:
+            Required Argument.
+            Specifies the name(s) of the Python package(s) for which the difference
+            in the versions is to be fetched from Vantage.
+            Note:
+            * If this argument is None, all the packages installed on Vantage are considered.
+            * If any package is present in Vantage but not in the current environment, then None
+                is shown as the version of the package in the current environment.
+            Types: str or list of str
+
+        only_diff:
+            Optional Argument.
+            Specifies whether to return only the differences in the versions of the packages
+            installed on Vantage and the packages mentioned in the argument "packages".
+            Default Value: True
+
+    RETURNS:
+        pandas DataFrame
+
+    RAISES:
+        TeradataMlException.
+
+    EXAMPLES:
+        # Note:
+        #   These examples will work only when the Python packages are installed on Vantage.
+
+        # Example 1: Get the difference in the versions of Python packages 'dill' and 'matplotlib' 
+        # installed on Vantage.
+        >>> _db_python_package_version_diff(["dill", "matplotlib"])
+          package vantage  local
+        0    dill   0.3.6  0.3.7
+
+        # Example 2: Get the difference in the versions of Python packages 'dill' and 'matplotlib' 
+        # installed on Vantage and 'only_diff' argument set to False.
+        >>> _db_python_package_version_diff(["dill", "matplotlib"], only_diff=False)
+                    package vantage  local
+        0  matplotlib-inline   0.1.6  0.1.6
+        1               dill   0.3.6  0.3.7
+        2         matplotlib   3.6.2  3.6.2
+    """
+    # Check if Python interpreter and add-on packages are installed or not.
+    _check_if_python_packages_installed()
+
+    # Raise error if Python and add-on packages are not installed.
+    if not tdmlctx.python_packages_installed:
+        raise TeradataMlException(Messages.get_message(MessageCodes.PYTHON_NOT_INSTALLED),
+                                  MessageCodes.PYTHON_NOT_INSTALLED)
+
+    # Installed packages dictionary.
+    db_pkg_df = db_python_package_details(packages)
+    if db_pkg_df is None:
+        return None
+
+    pkgs_dict = {row.package: row.version for row in db_pkg_df.itertuples()}
+
+    from importlib.metadata import PackageNotFoundError, version
+    diff_list = []
+
+    for pkg in pkgs_dict.keys():
+        vantage_version = pkgs_dict.get(pkg)
+        try:
+            local_version = version(pkg)
+        except PackageNotFoundError:
+            # If package is not found in the current environment, then the local version is set to None.
+            local_version = None
+        except Exception as e:
+            # Any other exception is raised.
+            raise
+
+        if only_diff:
+            if vantage_version != local_version:
+                # Add to list only when the versions are different.
+                diff_list.append([pkg, vantage_version, local_version])
+        else:
+            # Add to list all the packages and versions irrespective of the differences.
+            diff_list.append([pkg, vantage_version, local_version])
+
+    return pd.DataFrame(diff_list, columns=["package", "vantage", "local"])
+
+
+@collect_queryband(queryband='PythonDiff')
+def db_python_version_diff():
+    """
+    DESCRIPTION:
+        Function to get the difference of the Python intepreter major version installed on Vantage
+        and the Python version used in the current environment.
+
+        Note:
+            * Using this function is valid only when Python interpreter and add-on packages
+              are installed on the Vantage node.
+
+    RETURNS:
+        Empty dictionary when Python major version is same on Vantage and the current environment.
+        Otherwise, returns a dictionary with the following keys:
+            - 'vantage_version': Python major version installed on Vantage.
+            - 'local_version': Python major version used in the current environment.
+
+    RAISES:
+        TeradataMlException.
+
+    EXAMPLES:
+        # Note:
+        # These examples will work only when the Python packages are installed on Vantage.
+
+        # Example 1: Get the difference in the Python version installed on Vantage and the current environment.
+        >>> db_python_version_diff()
+        {"vantage_version": "3.7", "local_version": "3.8"}
+    """
+    # Check if Python interpretor and add-on packages are installed or not.
+    _check_if_python_packages_installed()
+
+    # Raise error if Python and add-on packages are not installed.
+    if not tdmlctx.python_packages_installed:
+        raise TeradataMlException(Messages.get_message(MessageCodes.PYTHON_NOT_INSTALLED),
+                                  MessageCodes.PYTHON_NOT_INSTALLED)
+
+    # Get major version of python installed on Vantage and the current environment.
+    python_local = tdmlctx.python_version_local.rsplit(".", 1)[0]
+    python_vantage = tdmlctx.python_version_vantage.rsplit(".", 1)[0]
+
+    if python_local != python_vantage:
+        return {"vantage_version": python_vantage, "local_version": python_local}
+
+    return {}
+
+
+@collect_queryband(queryband='PkgDiff')
+def db_python_package_version_diff(packages=None):
+    """
+    DESCRIPTION:
+        Function to get the difference of the Python packages installed on Vantage and
+        in the current environment mentioned in the argument "packages".
+
+        Notes:
+            * Using this function is valid only when Python interpreter and add-on packages
+              are installed on the Vantage node.
+            * This function also checks for differences in Python packages versions given
+              part of package name as string.
+
+    PARAMETERS:
+        packages:
+            Optional Argument.
+            Specifies the name(s) of the Python package(s) for which the difference
+            in the versions is to be fetched from Vantage.
+            Notes:
+                * If this argument is None, all the packages installed on Vantage are considered.
+                * If any package is present in Vantage but not in the current environment, then None
+                  is shown as the version of the package in the current environment.
+            Types: str or list of str
+
+    RETURNS:
+        pandas DataFrame
+
+    RAISES:
+        TeradataMlException.
+
+    EXAMPLES:
+        # Note:
+        # These examples will work only when the Python packages are installed on Vantage.
+
+        # Example 1: Get the difference in the versions of Python package 'dill' installed on Vantage.
+        >>> db_python_package_version_diff("dill")
+                  package   vantage    local
+        0            dill    0.10.0   0.11.2
+
+        # Example 2: Get the difference in the versions of all Python packages installed on Vantage.
+        >>> db_python_package_version_diff()
+                  package   vantage    local
+        0    scikit-learn     1.3.3   0.24.2
+        1            dill    0.10.0   0.11.2
+        ...
+        532         attrs    18.2.0   17.0.0
+
+    """
+    # Validate arguments.
+    __arg_info_matrix = []
+    __arg_info_matrix.append(["packages", packages, True, (str, list), True])
+
+    _Validators._validate_function_arguments(arg_list=__arg_info_matrix)
+
+    return _db_python_package_version_diff(packages=packages)
+
+
 def _create_table(table_name,
                   columns,
                   primary_index=None,
@@ -1060,7 +1410,7 @@ def _create_table(table_name,
         else:
             pti = pti.no_primary_index()
 
-        con_form=[]
+        con_form = []
         foreign_constraints = []
         for c_name, parameters in kwargs.items():
             _Validators._validate_function_arguments([["constraint_type", c_name, True, str,
@@ -1097,7 +1447,7 @@ def _create_table(table_name,
                     " columns.items()),{} teradatasql_post_create=pti,prefixes=prefix," \
                     "schema=schema_name)".format("" if con_form is None else ",".join(con_form))
 
-        table=eval(table_str)
+        table = eval(table_str)
         for foreign_constraint in foreign_constraints:
             table.append_constraint(foreign_constraint)
         table.create(bind=tdmlctx.get_context())
@@ -1107,7 +1457,8 @@ def _create_table(table_name,
         raise TeradataMlException(Messages.get_message(msg_code, "create table", str(err)), msg_code)
 
 
-def _create_database(schema_name, size='10e6', spool_size=None):
+def _create_database(schema_name, size='10e6', spool_size=None,
+                     datalake=None, **kwargs):
     """
     DESCRIPTION:
         Internal function to create a database with the specified name and size.
@@ -1133,6 +1484,16 @@ def _create_database(schema_name, size='10e6', spool_size=None):
                 Exponential notation can also be used.
             Types: str or int
 
+        datalake:
+            Optional Argument.
+            Specifies the name of datalake to create database in.
+            Types: str
+
+        kwargs:
+            Optional Argument.
+            Specifies keyword arguments which are used in DBPROPERTIES
+            clause as key-value pair while creating datalake database.
+
     RETURNS:
         bool
 
@@ -1140,11 +1501,29 @@ def _create_database(schema_name, size='10e6', spool_size=None):
         TeradataMlException.
 
     EXAMPLES:
-       >>> from teradataml.dbutils.dbutils import _create_database
-       >>> _create_database("db_name1", "10e5")
+        >>> from teradataml.dbutils.dbutils import _create_database
+        # Example 1: Create database.
+        >>> _create_database("db_name1", "10e5")
+
+        # Example 2: Create database in datalake.
+        >>> _create_database("otf_db_1", datalake="datalake_iceberg_glue")
+
+        # Example 3: Create database in datalake having DBPROPERTIES.
+        >>> _create_database("otf_db", datalake="datalake_iceberg_glue",
+        ...                  owner='tdml_user', other_property='some_value',
+        ...                  other_property2=20, comment='Created by tdml_user')
     """
-    sql = "CREATE DATABASE {} FROM {} AS PERM = {}".format(
-        schema_name, tdmlctx._get_database_username(), size)
+    if datalake:
+        db_properties = []
+        for key, val in kwargs.items():
+            db_properties.append("'{}'='{}'".format(key, val))
+
+        sql = "CREATE DATABASE {}.{}{};".format(datalake, schema_name,
+                                                ' DBPROPERTIES({})'.format(','.join(db_properties))
+                                                if db_properties else '')
+
+    else:
+        sql = "CREATE DATABASE {} FROM {} AS PERM = {}".format(schema_name, tdmlctx._get_database_username(), size)
 
     # If user pass spool size, create it with specified space.
     if spool_size:
@@ -1203,7 +1582,7 @@ def _update_data(update_columns_values, table_name, schema_name, datalake_name=N
 
     # If key_columns_values is passed, then prepare the SQL with where clause.
     # Else, simply update every thing.
-    schema_name = "{}.{}".format(datalake_name, schema_name) if datalake_name else schema_name
+    qualified_table_name = _get_quoted_object_name(schema_name, table_name, datalake_name)
 
     get_str_ = lambda val: "'{}'".format(val) if isinstance(val, str) else val
     if update_conditions:
@@ -1220,14 +1599,14 @@ def _update_data(update_columns_values, table_name, schema_name, datalake_name=N
 
         where_clause = " AND ".join(where_)
 
-        sql = f"""UPDATE {schema_name}.{table_name} SET {update_clause}
+        sql = f"""UPDATE {qualified_table_name} SET {update_clause}
                  WHERE {where_clause}
             """
 
-        execute_sql(sql, (*update_values, ))
+        execute_sql(sql, (*update_values,))
 
     else:
-        sql = f"""UPDATE {schema_name}.{table_name} SET {update_clause}"""
+        sql = f"""UPDATE {qualified_table_name} SET {update_clause}"""
 
         execute_sql(sql, update_values)
     return True
@@ -1276,10 +1655,7 @@ def _insert_data(table_name, values, columns=None, schema_name=None, datalake_na
        >>> _insert_data("tbl", (1, 2, 3))
     """
     # Prepare the update clause.
-    if schema_name:
-        table_name = '"{}"."{}"'.format(schema_name, table_name)
-        if datalake_name:
-            table_name = '"{}"."{}"'.format(datalake_name, table_name)
+    qualified_table_name = _get_quoted_object_name(schema_name, table_name, datalake_name)
 
     values = UtilFuncs._as_list(values)
 
@@ -1292,7 +1668,7 @@ def _insert_data(table_name, values, columns=None, schema_name=None, datalake_na
         columns = ""
         _q_marks = ["?"] * (len(values[0]))
 
-    sql = "insert into {} {} values ({});".format(table_name, columns, ", ".join(_q_marks))
+    sql = "insert into {} {} values ({});".format(qualified_table_name, columns, ", ".join(_q_marks))
     execute_sql(sql, values)
 
     return True
@@ -1339,6 +1715,8 @@ def _upsert_data(update_columns_values,
         datalake_name:
             Optional Argument.
             Specifies the name of the datalake to look for "schema_name".
+            Note:
+                "schema_name" must be provided while using this argument.
             Types: str
 
     RETURNS:
@@ -1357,8 +1735,7 @@ def _upsert_data(update_columns_values,
                         )
     """
     # If user passes datalake name, then append the same to schema name.
-    if datalake_name:
-        schema_name = "{}.{}".format(datalake_name, schema_name)
+    qualified_table_name = _get_quoted_object_name(schema_name, table_name, datalake_name)
 
     # Prepare the update clause.
     update_clause = ", ".join(("{} = ?".format(col) for col in update_columns_values))
@@ -1373,12 +1750,13 @@ def _upsert_data(update_columns_values,
     insert_clause = "({}) values ({})".format(", ".join(insert_columns_values), insert_values_clause)
     insert_values = tuple((_value for _value in insert_columns_values.values()))
 
-    sql = f"""UPDATE {schema_name}.{table_name} SET {update_clause}
+    sql = f"""UPDATE {qualified_table_name} SET {update_clause}
          WHERE {where_clause}
-       ELSE INSERT {schema_name}.{table_name} {insert_clause}
+       ELSE INSERT {qualified_table_name} {insert_clause}
     """
     execute_sql(sql, (*update_values, *where_values, *insert_values))
 
+
 def _delete_data(table_name, schema_name=None, datalake_name=None, delete_conditions=None):
     """
     DESCRIPTION:
@@ -1403,8 +1781,9 @@ def _delete_data(table_name, schema_name=None, datalake_name=None, delete_condit
 
         delete_conditions:
             Optional Argument.
-            Specifies the ColumnExpression to use for removing the data.
-            Types: ColumnExpression
+            Specifies the ColumnExpression or dictionary containing key values
+            pairs to use for removing the data.
+            Types: ColumnExpression, dict
 
     RETURNS:
         int, specifies the number of records those are deleted.
@@ -1416,24 +1795,34 @@ def _delete_data(table_name, schema_name=None, datalake_name=None, delete_condit
        >>> from teradataml.dbutils.dbutils import _delete_data
        >>> _delete_data("tbl", "db_name1", delete_conditions={"column1": "value1"})
     """
-    if schema_name:
-        table_name = '"{}"."{}"'.format(schema_name, table_name)
-
-    if datalake_name:
-        table_name = "{}.{}".format(datalake_name, table_name)
-
+    qualified_table_name = _get_quoted_object_name(schema_name, table_name, datalake_name)
     sqlbundle = SQLBundle()
 
-    sql = sqlbundle._get_sql_query(SQLConstants.SQL_DELETE_ALL_ROWS).format(table_name)
+    sql = sqlbundle._get_sql_query(SQLConstants.SQL_DELETE_ALL_ROWS).format(qualified_table_name)
 
     # If condition exist, the prepare where clause.
     if delete_conditions:
-        where_clause = delete_conditions.compile()
-        sql = sqlbundle._get_sql_query(SQLConstants.SQL_DELETE_SPECIFIC_ROW).format(table_name, where_clause)
+        from teradataml.dataframe.sql import _SQLColumnExpression
+        if isinstance(delete_conditions, _SQLColumnExpression):
+            where_clause = delete_conditions.compile()
+        elif isinstance(delete_conditions, dict):
+            get_str_ = lambda val: "'{}'".format(val) if isinstance(val, str) else val
+            where_ = []
+            for column, col_value in delete_conditions.items():
+                if isinstance(col_value, list):
+                    col_value = ", ".join(get_str_(val) for val in col_value)
+                    col_value = "({})".format(col_value)
+                    where_.append("{} IN {}".format(column, col_value))
+                else:
+                    where_.append("{} = {}".format(column, col_value))
+            where_clause = " AND ".join(where_)
+
+        sql = sqlbundle._get_sql_query(SQLConstants.SQL_DELETE_SPECIFIC_ROW).format(qualified_table_name, where_clause)
 
     res = execute_sql(sql)
     return res.rowcount
 
+
 @collect_queryband(queryband='LstKwrds')
 def list_td_reserved_keywords(key=None, raise_error=False):
     """
@@ -1498,6 +1887,7 @@ def list_td_reserved_keywords(key=None, raise_error=False):
     """
 
     from teradataml.dataframe.dataframe import DataFrame, in_schema
+
     # Get the reserved keywords from the table
     reserved_keys = DataFrame(in_schema("SYSLIB", "SQLRestrictedWords"))
 
@@ -1515,10 +1905,10 @@ def list_td_reserved_keywords(key=None, raise_error=False):
     # Check if key contains Teradata reserved keyword or not.
     res_key = (k.upper() for k in key if k.upper() in reservered_words)
     res_key = list(res_key)
-    if len(res_key)>0:
+    if len(res_key) > 0:
         if raise_error:
             raise TeradataMlException(Messages.get_message(MessageCodes.RESERVED_KEYWORD, res_key),
-                                                          MessageCodes.RESERVED_KEYWORD)
+                                      MessageCodes.RESERVED_KEYWORD)
         return True
     return False
 
@@ -1608,6 +1998,7 @@ def _execute_query_and_generate_pandas_df(query, index=None, **kwargs):
 
     return pandas_df
 
+
 class _TDSessionParams:
     """
     A successfull connection through teradataml establishes a session with Vantage.
@@ -1615,6 +2006,7 @@ class _TDSessionParams:
     for parameter 'Session Time Zone'.
     This is an internal utility to store all session related parameters.
     """
+
     def __init__(self, data):
         """
         Constructor to store columns and rows of session params.
@@ -1641,6 +2033,7 @@ class _TDSessionParams:
             return self.__session_params[parameter]
         raise AttributeError("'TDSessionParams' object has no attribute '{}'".format(parameter))
 
+
 def set_session_param(name, value):
     """
     DESCRIPTION:
@@ -1816,15 +2209,16 @@ def set_session_param(name, value):
         [param[0] for param in result.description],
         [value for value in next(result)]
     ))
-    _InternalBuffer.add(session_params = _TDSessionParams(data))
+    _InternalBuffer.add(session_params=_TDSessionParams(data))
     # Store function name of 'DEBUG_FUNCTION' used.
-    _InternalBuffer.add(function_name = value[0] if name.upper() == 'DEBUG_FUNCTION' else '')
+    _InternalBuffer.add(function_name=value[0] if name.upper() == 'DEBUG_FUNCTION' else '')
 
     # Set the session parameter.
     execute_sql(getattr(SessionParamsSQL, name.upper()).format(*value))
 
     return True
 
+
 def unset_session_param(name):
     """
     DESCRIPTION:
@@ -1868,7 +2262,7 @@ def unset_session_param(name):
     # unset_values stores params which are not available in _InternalBuffer, to unset create a dictionary
     # with param as key and unset param as value
     unset_values = {"CHARACTER_SET_UNICODE": "OFF", "DEBUG_FUNCTION": [_InternalBuffer.get('function_name'), "OFF"],
-                    "ISOLATED_LOADING":"NO", "FUNCTION_TRACE":"SET SESSION FUNCTION TRACE OFF",
+                    "ISOLATED_LOADING": "NO", "FUNCTION_TRACE": "SET SESSION FUNCTION TRACE OFF",
                     "JSON_IGNORE_ERRORS": "OFF", "QUERY_BAND": ["", "SESSION"]}
 
     # If 'name' in unset_values unset the params
@@ -1882,14 +2276,16 @@ def unset_session_param(name):
         return True
 
     previous_value = "{}".format(session_params[getattr(SessionParamsPythonNames, name.upper())]) \
-        if name.upper() != 'TIMEZONE' else "'{}'".format(session_params[getattr(SessionParamsPythonNames, name.upper())])
-    
+        if name.upper() != 'TIMEZONE' else "'{}'".format(
+        session_params[getattr(SessionParamsPythonNames, name.upper())])
+
     if name.upper() == "ACCOUNT":
         previous_value = [previous_value, 'SESSION']
     set_session_param(name, previous_value)
 
     return True
 
+
 class _Authorize:
     """ Parent class to either provide or revoke access on table(s). """
     _property = None