teradataml 20.0.0.3__py3-none-any.whl → 20.0.0.5__py3-none-any.whl

teradataml/common/utils.py

@@ -13,50 +13,48 @@ by other classes which can be reused according to the need.
 Add all the common functions in this class like creating temporary table names, getting
 the datatypes etc.
 """
-from inspect import getsource
 import json
+import os
+import re
+import time
 import uuid
+import warnings
+from functools import reduce
+from inspect import getsource
 from math import floor
-import os, itertools
-import time
-import re, requests
+
+import requests
 import sqlalchemy
-from pathlib import Path
 from numpy import number
 from sqlalchemy import Column, MetaData, Table
-
-from teradataml.context.context import get_connection
+from sqlalchemy.exc import OperationalError as sqlachemyOperationalError
+from teradatasql import OperationalError
+from teradatasqlalchemy.dialect import dialect as td_dialect
+from teradatasqlalchemy.dialect import preparer
+from teradatasqlalchemy.types import (BIGINT, BLOB, BYTE, BYTEINT, CHAR, CLOB,
+                                      DATE, DECIMAL, FLOAT, INTEGER, NUMBER,
+                                      SMALLINT, TIME, TIMESTAMP, VARBYTE,
+                                      VARCHAR, _TDType)
 
 from teradataml import _version
-from teradataml.context import context as tdmlctx
+from teradataml.common import td_coltype_code_to_tdtype
+from teradataml.common.constants import (HTTPRequest, PTITableConstants,
+                                         PythonTypes, TeradataConstants,
+                                         TeradataReservedKeywords,
+                                         TeradataTypes)
 from teradataml.common.exceptions import TeradataMlException
-from teradataml.common.messages import Messages
+from teradataml.common.garbagecollector import GarbageCollector
 from teradataml.common.messagecodes import MessageCodes
+from teradataml.common.messages import Messages
 from teradataml.common.sqlbundle import SQLBundle
-from teradataml.common import td_coltype_code_to_tdtype
-from teradataml.common.constants import PythonTypes
-from teradataml.common.constants import TeradataTypes
-from teradataml.common.garbagecollector import GarbageCollector
-from teradataml.common.constants import TeradataConstants, PTITableConstants, \
-    TableOperatorConstants, HTTPRequest
-from teradataml.common.warnings import VantageRuntimeWarning
+from teradataml.common.warnings import (OneTimeUserWarning,
+                                        VantageRuntimeWarning)
+from teradataml.context import context as tdmlctx
 from teradataml.options.configure import configure
 from teradataml.options.display import display
-from teradataml.common.constants import TeradataReservedKeywords, TeradataConstants
-
-from teradataml.utils.internal_buffer import _InternalBuffer
-from teradatasqlalchemy.types import _TDType
-from teradatasqlalchemy.types import (INTEGER, SMALLINT, BIGINT, BYTEINT,
-                                      DECIMAL, FLOAT, NUMBER)
-from teradatasqlalchemy.types import (DATE, TIME, TIMESTAMP)
-from teradatasqlalchemy.types import (BYTE, VARBYTE, BLOB)
-from teradatasqlalchemy.types import (CHAR, VARCHAR, CLOB)
-from functools import reduce
-import warnings
 from teradataml.telemetry_utils.queryband import collect_queryband
 from teradataml.utils.utils import execute_sql
 from teradataml.utils.validators import _Validators
-from sqlalchemy.exc import OperationalError as sqlachemyOperationalError
 
 
 class UtilFuncs():
@@ -304,13 +302,17 @@ class UtilFuncs():
             tabname = "{}_{}".format(tabname, prefix)
 
         tabname = "{}_{}".format(tabname, random_string)
-        
+
         # ELE-6710 - Use database user associated with the current context for volatile tables.
         if table_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
             from teradataml.context.context import _get_user
             tabname = "\"{}\".\"{}\"".format(_get_user(), tabname)
             return tabname
 
+        if configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
+            from teradataml.context.context import _get_user
+            return "\"{}\".\"{}_{}\"".format(_get_user(), "vt", tabname)
+
         if use_default_database and databasename is None:
             tabname = "\"{}\".\"{}\"".format(tdmlctx._get_context_temp_databasename(
                 table_type=table_type), tabname)
@@ -450,9 +452,10 @@ class UtilFuncs():
             >>> def udf(a, b): return a + b
             >>> func = UtilFuncs._serialize_and_encode(udf)
         """
-        from dill import dumps as dill_dumps
         from base64 import b64encode as base64_b64encode
 
+        from dill import dumps as dill_dumps
+
         return base64_b64encode(dill_dumps(obj, recurse=True))
 
     @staticmethod
@@ -694,6 +697,7 @@ class UtilFuncs():
         EXAMPLES:
             UtilFuncs._create_table('"dbname"."table_name"', "select * from table_name")
         """
+
         crt_table = SQLBundle._build_create_table_with_data(table_name, query)
         if volatile:
             crt_table = SQLBundle._build_create_volatile_table_with_data(table_name, query)
@@ -757,13 +761,14 @@ class UtilFuncs():
         return vtab_name
 
     @staticmethod
-    def _drop_table(table_name, check_table_exist=True):
+    def _drop_table(table_name, check_table_exist=True, purge_clause=None):
         """
         Drops a table.
 
         PARAMETERS:
             table_name - The table to drop.
             check_table_exist - Checks if the table exist before dropping the table.
+            purge_clause - Specifies string representing purge clause to be appended to drop table query.
 
         RETURNS:
             True - if the table is dropped.
@@ -776,9 +781,11 @@ class UtilFuncs():
             UtilFuncs._drop_table('mytab', check_table_exist = False)
             UtilFuncs._drop_table('mydb.mytab', check_table_exist = False)
             UtilFuncs._drop_table("mydb"."mytab", check_table_exist = True)
+            UtilFuncs._drop_table("my_lake"."my_db"."my_tab", purge_clause='PURGE ALL')
+            UtilFuncs._drop_table("my_lake"."my_db"."my_tab", purge_clause='NO PURGE')
 
         """
-        drop_tab = SQLBundle._build_drop_table(table_name)
+        drop_tab = SQLBundle._build_drop_table(table_name, purge_clause)
         if check_table_exist is True:
             helptable = UtilFuncs._get_help_tablename(table_name)
             if helptable:
@@ -958,6 +965,22 @@ class UtilFuncs():
         """
         return UtilFuncs._execute_query(SQLBundle._build_help_table(table_name))
 
+    @staticmethod
+    def _get_help_datalakename(datalake_name):
+        """
+        Function to get help of the datalake.
+
+        PARAMETERS:
+            datalake_name - The name of the datalake.
+
+        RETURNS:
+            The help information of the datalake specified by datalake_name.
+
+        EXAMPLES:
+            UtilFuncs._get_help_datalakename(mydatalake)
+        """
+        return UtilFuncs._execute_query(SQLBundle._build_help_datalake(datalake_name))
+
     @staticmethod
     def _get_select_table(table_name):
         """
@@ -1251,7 +1274,7 @@ class UtilFuncs():
             return UtilFuncs._teradata_quote_arg(keyword, "\"", False)
 
         return keyword
-    
+
     def _contains_space(item):
         """
         Check if the specified string in item has spaces or tabs in it.
@@ -1283,46 +1306,74 @@ class UtilFuncs():
         # If the input is a list, check each element
         if isinstance(item, list):
             # Check each item in the list
-            return any(UtilFuncs._contains_space(col) for col in item)  
+            return any(UtilFuncs._contains_space(col) for col in item)
 
-        return False  
+        return False
 
-    @staticmethod
-    def _in_schema(schema_name, table_name):
+    def _is_ascii(col_lst):
         """
-        Function takes a schema name and a table name and creates a database
-        object name in the format "schema"."table_name".
-        Note:
-            teradataml recommends to use this function to access table(s)/view(s),
-            from the database other than the default database.
+        Description:
+            Check if the specified string in col_lst has non-ASCII characters in it.
 
         PARAMETERS:
-            schema_name:
-                Required Argument
-                Specifies the schema where the table resides in.
-                Types: str
+            col_lst:
+                Required Argument.
+                Specifies a list of strings to check for non-ASCII characters.
+                Types: list
 
-            table_name:
-                Required Argument
-                Specifies the table name or view name in Vantage.
-                Types: str
+        RETURNS:
+            True, if the specified string has non-ASCII characters in it, else False.
 
+        RAISES:
+            None.
+        
         EXAMPLES:
-            from teradataml.dataframe.dataframe import in_schema, DataFrame
+            # Passing column name with non-ASCII characters returns True.
+            >>> is_non_ascii = UtilFuncs._is_ascii(["col name", "がく片の長さ@name"])
+            >>> print(is_non_ascii)
+            >>> True
+            # Passing column name without non-ASCII characters returns False.
+            >>> is_non_ascii = UtilFuncs._is_ascii(["colname", "col_name"])
+            >>> print(is_non_ascii)
+            >>> False
+        """
+        if isinstance(col_lst, str):
+            # Check if the input is a string and look for non-ASCII characters
+            return not col_lst.isascii()
+
+        if isinstance(col_lst, list):
+            for item in col_lst:
+                if isinstance(item, str) and not item.isascii():
+                    return True
+        return False
 
-            # Example 1: The following example creates a DataFrame from the
-            #            existing Vantage table "dbcinfo" in the non-default
-            #            database "dbc" using the in_schema() function.
-            df = DataFrame(in_schema("dbc", "dbcinfo"))
+    @staticmethod
+    def __get_dot_separated_names(full_qualified_name):
+        """
+        Takes in fully qualified name of the table/view (db.table), and returns
+        a dot separated name from the same.
 
-            # Example 2: The following example uses from_table() function, existing
-            #            Vantage table "dbcinfo" and non-default database "dbc" to
-            #            create a teradataml DataFrame.
-            df = DataFrame.from_table(in_schema("dbc","dbcinfo"))
+        PARAMETERS:
+            full_qualified_name - Name of the table/view
+
+        EXAMPLES:
+            UtilFuncs._extract_db_name('"db1"."tablename"')
+            UtilFuncs._extract_db_name('"Icebergs"."db1"."tablename"')
+
+        RETURNS:
+            List of dot separated name from the provided name.
 
         """
-        return "{0}.{1}".format(UtilFuncs._teradata_quote_arg(schema_name, "\"", False),
-                                UtilFuncs._teradata_quote_arg(table_name, "\"", False))
+        # If condition to handle the quoted name.
+        if '"' in full_qualified_name:
+            # Extract the double quoted strings.
+            names = re.findall(r'["](.*?)["]', full_qualified_name)
+            # Remove quotes around the string.
+            names = [i.replace('"', '') for i in names]
+        # Handle non-quoted string with dot separated name.
+        else:
+            names = full_qualified_name.split(".")
+        return names
 
     @staticmethod
     def _extract_db_name(full_qualified_name):
@@ -1334,19 +1385,19 @@ class UtilFuncs():
             full_qualified_name - Name of the table/view
 
         EXAMPLES:
-            UtilFuncs._extract_db_name("db1"."tablename")
+            UtilFuncs._extract_db_name('"db1"."tablename"')
+            UtilFuncs._extract_db_name('"Icebergs"."db1"."tablename"')
 
         RETURNS:
             Database name from the provided name.
 
         """
-        # Extract the double quoted strings.
-        names = re.findall(r'["](.*?)["]', full_qualified_name)
-        # Remove quotes around the string.
-        names = [i.replace('"', '') for i in names]
+        names = UtilFuncs.__get_dot_separated_names(full_qualified_name)
         if names:
             if len(names) == 2:
                 return names[0]
+            elif len(names) == 3:
+                return names[1]
         else:
             return None
 
@@ -1360,23 +1411,45 @@ class UtilFuncs():
             full_qualified_name - Name of the table/view
 
         EXAMPLES:
-            UtilFuncs._extract_db_name("db1"."tablename")
+            UtilFuncs._extract_table_name('"db1"."tablename"')
+            UtilFuncs._extract_table_name('"Icebergs"."db1"."tablename"')
 
         RETURNS:
             Table/View name from the provided name.
 
         """
-        # Extract the double quoted strings.
-        names = re.findall(r'["](.*?)["]', full_qualified_name)
-        # Remove quotes around the string.
-        names = [i.replace('"', '') for i in names]
+        names = UtilFuncs.__get_dot_separated_names(full_qualified_name)
         if names:
             if len(names) == 2:
                 return names[1]
+            elif len(names) == 3:
+                return names[2]
             else:
                 return names[0]
         return full_qualified_name
 
+    @staticmethod
+    def _extract_datalake_name(full_qualified_name):
+        """
+        Takes in fully qualified name of the table/view (db.table), and returns
+        a datalake name from the same.
+
+        PARAMETERS:
+            full_qualified_name - Name of the table/view
+
+        EXAMPLES:
+            UtilFuncs._extract_datalake_name('"db1"."tablename"')
+            UtilFuncs._extract_datalake_name('"Icebergs"."db1"."tablename"')
+
+        RETURNS:
+            Database name from the provided name.
+
+        """
+        names = UtilFuncs.__get_dot_separated_names(full_qualified_name)
+        if names and len(names) == 3:
+            return names[0]
+        return None
+
     @staticmethod
     def _teradata_quote_arg(args, quote="'", call_from_wrapper=True):
         """
@@ -2402,7 +2475,14 @@ class UtilFuncs():
         # and determine the system type accordingly.
         if tbl_operator is None:
             from teradataml.context.context import _get_database_version
-            return int(_get_database_version().split(".")[0]) >= 20
+            if int(_get_database_version().split(".")[0]) < 20:
+                return False
+            # If the database version is 20 or higher, check if the system is VCL or not.
+            try:
+                res = UtilFuncs._execute_query("SELECT 1 WHERE TD_GetSystemType('PRODUCT') = 'VCL';")
+                return True if res else False
+            except OperationalError:
+                return True
 
         return tbl_operator == "apply"
 
@@ -2429,7 +2509,6 @@ class UtilFuncs():
         return "python" if UtilFuncs._is_lake() else \
             '{}/bin/python3'.format(configure.indb_install_location)
 
-
     def _is_view(tablename):
         """
         DESCRIPTION:
@@ -2515,7 +2594,7 @@ class UtilFuncs():
                 raise tdml_e
         except Exception as exc:
             raise exc
-        
+
     def _get_env_name(col=None):
         """
         DESCRIPTION:
@@ -2630,6 +2709,219 @@ class UtilFuncs():
         """
         return '"{}"."{}"'.format(schema_name, table_name)
 
+    def _check_python_version_diff(env = None):
+        """
+        DESCRIPTION:
+            Internal function to check the python version difference between Vantage and local.
+            
+        PARAMETERS:
+            env:
+                Optional Argument.
+                Specifies the user environment for Vantage Cloud Lake.
+                Types: str, object of class UserEnv
+                Default Value: None
+
+        RAISES:
+            TeradataMlException
+
+        RETURNS:
+            None.
+
+        EXAMPLES:
+            >>> self._check_python_version_diff(env)
+        """
+        if env:
+            # Get the Python interpreter version of the user environment.
+            from teradataml.scriptmgmt.lls_utils import list_user_envs
+            from teradataml.scriptmgmt.UserEnv import UserEnv
+            env_list = list_user_envs()
+            user_env_name = env.env_name if isinstance(env, UserEnv) else env
+            env_base_version = env_list[env_list['env_name'] == user_env_name].base_env_name.values
+            # Check if the user environment is not found, then return.
+            if len(env_base_version) == 0:
+                return
+            python_env = env_base_version[0].split("_")[1]
+
+            # Get the Python interpreter version of the local environment.
+            from teradataml.context import context as tdmlctx
+            python_local = tdmlctx.python_version_local.rsplit(".", 1)[0]
+            # Check if the Python interpreter major versions are consistent between Lake user environment and local.
+            # If not, raise an exception.
+            if python_env != python_local:
+                raise TeradataMlException(Messages.get_message(MessageCodes.PYTHON_VERSION_MISMATCH_OAF,
+                                                                python_env, python_local),
+                                            MessageCodes.PYTHON_VERSION_MISMATCH_OAF)
+        else:
+            from teradataml.context import context as tdmlctx
+            from teradataml.dbutils.dbutils import (db_python_version_diff, set_session_param)
+            set_session_param("searchuifdbpath",
+                              UtilFuncs._get_dialect_quoted_name(tdmlctx._get_current_databasename()))
+            if len(db_python_version_diff()) > 0:
+                # Raise exception when python versions don't match between Vantage and local.
+                py_major_vantage_version = tdmlctx.python_version_vantage.rsplit(".", 1)[0]
+                raise TeradataMlException(Messages.get_message(MessageCodes.PYTHON_VERSION_MISMATCH,
+                                                                tdmlctx.python_version_vantage, py_major_vantage_version),
+                                            MessageCodes.PYTHON_VERSION_MISMATCH)
+
+    def _check_package_version_diff(func, packages, env=None):
+        """
+        DESCRIPTION:
+            Internal function to process packages differences between Vantage and local.
+            Note:
+                * Raises a warning if the versions of certain Python packages are not consistent between Vantage and local.
+
+        PARAMETERS:
+            func:
+                Required Argument.
+                Specifies the function name.
+                Types: str
+
+            packages:
+                Required Argument.
+                Specifies the list of package names.
+                Types: list of str
+
+            env:
+                Optional Argument.
+                Specifies the user environment for Vantage Cloud Lake.
+                Types: str, object of class UserEnv
+                Default Value: None
+
+        RETURNS:
+            None
+
+        RAISES:
+            OneTimeUserWarning
+
+        EXAMPLES:
+            self._process_package_differences("apply", ["dill"], env)
+        """
+
+        # Check if OSML required packages are verified or not.
+        from teradataml.opensource._constants import _packages_verified_in_vantage
+        _is_packages_verfied_in_vantage = _packages_verified_in_vantage.get(
+            func, None)
+        if _is_packages_verfied_in_vantage:
+            return
+
+        if env:
+            from teradataml.scriptmgmt.lls_utils import get_env
+            from teradataml.scriptmgmt.UserEnv import UserEnv
+            env = env if isinstance(env, UserEnv) else get_env(env)
+            env_pkg_df = env.libs
+            pkgs_dict = dict(zip(env_pkg_df['name'], env_pkg_df['version']))
+
+            from importlib.metadata import version
+            warning_raised = False
+            strr = []
+            for pkg in packages:
+                env_version = pkgs_dict.get(pkg)
+                local_version = version(pkg)
+                # Write the requirements file listing all the related packages and their versions
+                # if the versions Python packages are not consistent between Vantage and local.
+                if env_version != local_version:
+                    warning_raised = True
+                    strr.append(f"{pkg}=={local_version}")
+
+            # If there are differences in package versions, display a warning message to the user.
+            # about the package differences and the requirements file created for the user to install the packages
+            if warning_raised:
+                file_name = f"requirements_{func}.txt"
+                req_file = os.path.join(GarbageCollector._get_temp_dir_name(), file_name)
+                with open(req_file, "w") as f:
+                    f.write("\n".join(strr))
+
+                packages = "{}".format(packages[0]) if len(packages) == 1 else\
+                           "', '".join(packages[:-1]) + "' and '" + packages[-1]
+
+                if func == "apply":
+                    warning_msg = f"The version of certain Python packages are not consistent between Lake "\
+                        f"user environment and local. Teradata recommends to maintain same version of '{packages}' "\
+                        f"between Lake user environment and local for '{func}'."
+                else:
+                    _packages_verified_in_vantage[func] = True
+                    warning_msg = "The versions of certain Python packages are not consistent between "\
+                        "Lake user environment and local. OpenSourceML compares the versions of '{}' "\
+                        f"(and also matches the patterns of these packages) used by 'td_{func}'. "\
+                        "Teradata recommends same versions for all the Python packages between Lake "\
+                        "user environment and local."
+
+                req = f"\nA requirements file listing all '{func}' " + \
+                    f"related packages and their versions has been written to '{req_file}'. "+ \
+                    "Update the Lake user environment with the required packages.\n"
+
+                warning_msg += req
+                warnings.warn(warning_msg.format(packages), category=OneTimeUserWarning)
+
+        else:
+            # Check if the versions of Python packages are consistent between Vantage and local.
+            from teradataml.dbutils.dbutils import _db_python_package_version_diff
+            all_package_versions = _db_python_package_version_diff(packages, only_diff=False)
+            package_difference = \
+                all_package_versions[all_package_versions.vantage != all_package_versions.local]
+            # If there are differences in package versions, raise a warning.
+            if package_difference.shape[0] > 0:
+                strr = []
+                # Write the requirements file listing all the related packages and their versions.
+                for rec in all_package_versions.to_records():
+                    strr.append(f"{rec[1]}=={rec[2]}")
+                file_name = f"requirements_{func}.txt"
+                req_file = os.path.join(GarbageCollector._get_temp_dir_name(), file_name)
+                with open(req_file, "w") as f:
+                    f.write("\n".join(strr))
+
+                packages = "{}".format(packages[0]) if len(packages) == 1 else\
+                           "', '".join(packages[:-1]) + "' and '" + packages[-1]
+
+                if func in ["map_row", "map_partition"]:
+                    warning_msg = f"The version of certain Python packages are not consistent between "\
+                        "Vantage and local. User can identify them using db_python_package_version_diff() "\
+                        f"function. Teradata recommends to maintain same version of '{packages}' "\
+                        f"between Vantage and local for '{func}'."
+                else:
+                    _packages_verified_in_vantage[func] = True
+                    warning_msg = "The versions of certain Python packages are not consistent between "\
+                        "Vantage and local. User can identify them using db_python_package_version_diff() "\
+                        "function. OpenSourceML compares the versions of '{}' (and also matches the "\
+                        f"patterns of these packages) used by 'td_{func}'. Teradata "\
+                        "recommends to maintain same versions for all the Python packages between Vantage "\
+                        "and local."
+
+                # Display a warning message to the user about the package differences
+                # and the requirements file created for the user to install the packages.
+                req = f"\nA requirements file listing all '{func}' " + \
+                    f"related packages and their versions has been written to '{req_file}'.\n"
+
+                warning_msg += req
+                warnings.warn(warning_msg.format(packages), category=OneTimeUserWarning)
+
+    @staticmethod
+    def _get_dialect_quoted_name(object_name):
+        """
+        DESCRIPTION:
+            Function to quote the SQL identifiers as per teradatasqlalchemy's quoting rules.
+
+        PARAMETERS:
+            object_name
+                Required Argument.
+                Specifies the name of the SQL identifier to be quoted.
+                Type: str
+
+        RAISES:
+            None
+
+        RETURNS:
+            Quoted object name.
+
+        EXAMPLES:
+            _get_dialect_quoted_name(object_name = "tdml.alice")
+
+        OUTPUT:
+            '"tdml.alice"'
+        """
+        tdp = preparer(td_dialect)
+        return tdp.quote(object_name)
+
 
 from teradataml.common.aed_utils import AedUtils
 from teradataml.dbutils.filemgr import remove_file

teradataml/common/warnings.py

@@ -17,9 +17,20 @@ class VantageRuntimeWarning(RuntimeWarning):
     """
     pass
 
+
 class TeradataMlRuntimeWarning(RuntimeWarning):
     """
     The TeradataMlRuntimeWarning is thrown whenever there are Warnings in runtime execution of objects in teradataml.
     """
     # For future purpose
     pass
+
+
+
+class OneTimeUserWarning(UserWarning):
+    """
+    The OneTimeUserWarning is thrown when the warning should only be displayed once to the user.
+    This can be useful for deprecation warnings, configuration issues, or other
+    non-critical warnings that do not need to be repeated multiple times.
+    """
+    pass