teradataml 20.0.0.4__py3-none-any.whl → 20.0.0.6__py3-none-any.whl

teradataml/common/utils.py

@@ -13,50 +13,50 @@ by other classes which can be reused according to the need.
 Add all the common functions in this class like creating temporary table names, getting
 the datatypes etc.
 """
-from inspect import getsource
+import datetime
 import json
+import os
+import re
+import time
 import uuid
+import warnings
+from functools import reduce
+from inspect import getsource
 from math import floor
-import os, itertools
-import time
-import re, requests
+
+import requests
 import sqlalchemy
-from pathlib import Path
 from numpy import number
 from sqlalchemy import Column, MetaData, Table
-
-from teradataml.context.context import get_connection
+from sqlalchemy.exc import OperationalError as sqlachemyOperationalError
+from teradatasql import OperationalError
+from teradatasqlalchemy.dialect import dialect as td_dialect
+from teradatasqlalchemy.dialect import preparer
+from teradatasqlalchemy.types import (BIGINT, BLOB, BYTE, BYTEINT, CHAR, CLOB,
+                                      DATE, DECIMAL, FLOAT, INTEGER, NUMBER,
+                                      SMALLINT, TIME, TIMESTAMP, VARBYTE,
+                                      VARCHAR, _TDType)
 
 from teradataml import _version
-from teradataml.context import context as tdmlctx
+from teradataml.common import td_coltype_code_to_tdtype
+from teradataml.common.constants import (HTTPRequest, PTITableConstants,
+                                         PythonTypes, SQLConstants,
+                                         TeradataConstants,
+                                         TeradataReservedKeywords,
+                                         TeradataTypes)
 from teradataml.common.exceptions import TeradataMlException
-from teradataml.common.messages import Messages
+from teradataml.common.garbagecollector import GarbageCollector
 from teradataml.common.messagecodes import MessageCodes
+from teradataml.common.messages import Messages
 from teradataml.common.sqlbundle import SQLBundle
-from teradataml.common import td_coltype_code_to_tdtype
-from teradataml.common.constants import PythonTypes
-from teradataml.common.constants import TeradataTypes
-from teradataml.common.garbagecollector import GarbageCollector
-from teradataml.common.constants import TeradataConstants, PTITableConstants, \
-    TableOperatorConstants, HTTPRequest
-from teradataml.common.warnings import VantageRuntimeWarning
+from teradataml.common.warnings import (OneTimeUserWarning,
+                                        VantageRuntimeWarning)
+from teradataml.context import context as tdmlctx
 from teradataml.options.configure import configure
 from teradataml.options.display import display
-from teradataml.common.constants import TeradataReservedKeywords, TeradataConstants
-
-from teradataml.utils.internal_buffer import _InternalBuffer
-from teradatasqlalchemy.types import _TDType
-from teradatasqlalchemy.types import (INTEGER, SMALLINT, BIGINT, BYTEINT,
-                                      DECIMAL, FLOAT, NUMBER)
-from teradatasqlalchemy.types import (DATE, TIME, TIMESTAMP)
-from teradatasqlalchemy.types import (BYTE, VARBYTE, BLOB)
-from teradatasqlalchemy.types import (CHAR, VARCHAR, CLOB)
-from functools import reduce
-import warnings
 from teradataml.telemetry_utils.queryband import collect_queryband
 from teradataml.utils.utils import execute_sql
 from teradataml.utils.validators import _Validators
-from sqlalchemy.exc import OperationalError as sqlachemyOperationalError
 
 
 class UtilFuncs():
@@ -282,36 +282,48 @@ class UtilFuncs():
         RAISES:
 
         EXAMPLES:
-            new_table_name = UtilFuncs._generate_temp_table_name(user='tdqg', prefix="from_pandas")
-            new_table_name = UtilFuncs._generate_temp_table_name(user='tdqg', prefix="from_pandas",
+            >>> new_table_name = UtilFuncs._generate_temp_table_name(user='tdqg', prefix="from_pandas")
+            >>> new_table_name = UtilFuncs._generate_temp_table_name(user='tdqg', prefix="from_pandas",
                                                                  table_type = TeradataConstants.TERADATA_VIEW)
-            new_table_name = UtilFuncs._generate_temp_table_name(user='tdqg', prefix="from_pandas",
+            >>> new_table_name = UtilFuncs._generate_temp_table_name(user='tdqg', prefix="from_pandas",
                                                                  table_type = TeradataConstants.TERADATA_TABLE)
+            # Example when use_short_object_name is set to True
+            >>> from teradataml.options.configure import configure
+            >>> configure.use_short_object_name = True
+            >>> new_table_name = UtilFuncs._generate_temp_table_name(user='tdqg', prefix="from_pandas")
 
         Output:
             tdml_temp_table__1517501990393350 (or)
             tdqg.tdml_temp_table__1517501990393350 (or)
             tdml_temp_table__from_pandas_1517501990393350 (or)
-            tdqg.tdml_temp_table__from_pandas_1517501990393350
+            tdqg.tdml_temp_table__from_pandas_1517501990393350 (or)
+            ml__1749637109887272
         """
         # Number of seconds since  Jan 1, 1970 00:00:00
         timestamp = time.time()
+        use_short_name = configure.use_short_object_name
         tabname = "ml_"
         random_string = "{}{}".format(floor(timestamp / 1000000),
                                       floor(timestamp % 1000000 * 1000000 +
                                             int(str(uuid.uuid4().fields[-1])[:10])))
-        if prefix is not None:
+        
+        # Append prefix only if use_short_object_name is False and prefix is provided.
+        if (not use_short_name) and (prefix is not None):
             tabname = "{}_{}".format(tabname, prefix)
+        # Append prefix "tdml" when use_short_object_name is True and random string is of length 15.
+        elif use_short_name and (len(random_string)==15):
+            tabname = "tdml"
 
         tabname = "{}_{}".format(tabname, random_string)
-        
+
         # ELE-6710 - Use database user associated with the current context for volatile tables.
         if table_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
             from teradataml.context.context import _get_user
             tabname = "\"{}\".\"{}\"".format(_get_user(), tabname)
             return tabname
 
-        if configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
+        if (not use_short_name) and (configure.temp_object_type == TeradataConstants.
+                                     TERADATA_VOLATILE_TABLE):
             from teradataml.context.context import _get_user
             return "\"{}\".\"{}_{}\"".format(_get_user(), "vt", tabname)
 
@@ -454,9 +466,10 @@ class UtilFuncs():
             >>> def udf(a, b): return a + b
             >>> func = UtilFuncs._serialize_and_encode(udf)
         """
-        from dill import dumps as dill_dumps
         from base64 import b64encode as base64_b64encode
 
+        from dill import dumps as dill_dumps
+
         return base64_b64encode(dill_dumps(obj, recurse=True))
 
     @staticmethod
@@ -1275,7 +1288,7 @@ class UtilFuncs():
             return UtilFuncs._teradata_quote_arg(keyword, "\"", False)
 
         return keyword
-    
+
     def _contains_space(item):
         """
         Check if the specified string in item has spaces or tabs in it.
@@ -1307,11 +1320,12 @@ class UtilFuncs():
         # If the input is a list, check each element
         if isinstance(item, list):
             # Check each item in the list
-            return any(UtilFuncs._contains_space(col) for col in item)  
+            return any(UtilFuncs._contains_space(col) for col in item)
 
-        return False  
-    
-    def _is_ascii(col_lst):
+        return False
+
+    @staticmethod
+    def _is_non_ascii(col_lst):
         """
         Description:
             Check if the specified string in col_lst has non-ASCII characters in it.
@@ -2476,7 +2490,14 @@ class UtilFuncs():
         # and determine the system type accordingly.
         if tbl_operator is None:
             from teradataml.context.context import _get_database_version
-            return int(_get_database_version().split(".")[0]) >= 20
+            if int(_get_database_version().split(".")[0]) < 20:
+                return False
+            # If the database version is 20 or higher, check if the system is VCL or not.
+            try:
+                res = UtilFuncs._execute_query("SELECT 1 WHERE TD_GetSystemType('PRODUCT') = 'VCL';")
+                return True if res else False
+            except OperationalError:
+                return True
 
         return tbl_operator == "apply"
 
@@ -2503,7 +2524,6 @@ class UtilFuncs():
         return "python" if UtilFuncs._is_lake() else \
             '{}/bin/python3'.format(configure.indb_install_location)
 
-
     def _is_view(tablename):
         """
         DESCRIPTION:
@@ -2589,7 +2609,7 @@ class UtilFuncs():
                 raise tdml_e
         except Exception as exc:
             raise exc
-        
+
     def _get_env_name(col=None):
         """
         DESCRIPTION:
@@ -2665,7 +2685,10 @@ class UtilFuncs():
         user_function_code = ""
         for func in user_functions:
             # Get the source code of the user function.
-            func = getsource(func)
+            # Note that, checking for lambda function is required for teradatamlspk UDFs
+            # If the function is a lambda function, get the source code from __source__.
+            func = getsource(func) if func.__code__.co_name != "<lambda>" else func.__source__
+
             # If the function have any extra space in the beginning remove it.
             func = func.lstrip()
             # Function can have decorator,e.g. udf as decorator, remove it.
@@ -2704,6 +2727,283 @@ class UtilFuncs():
         """
         return '"{}"."{}"'.format(schema_name, table_name)
 
+    def _check_python_version_diff(env = None):
+        """
+        DESCRIPTION:
+            Internal function to check the python version difference between Vantage and local.
+            
+        PARAMETERS:
+            env:
+                Optional Argument.
+                Specifies the user environment for Vantage Cloud Lake.
+                Types: str, object of class UserEnv
+                Default Value: None
+
+        RAISES:
+            TeradataMlException
+
+        RETURNS:
+            None.
+
+        EXAMPLES:
+            >>> self._check_python_version_diff(env)
+        """
+        if env:
+            # Get the Python interpreter version of the user environment.
+            from teradataml.scriptmgmt.lls_utils import list_user_envs
+            from teradataml.scriptmgmt.UserEnv import UserEnv
+            env_list = list_user_envs()
+            user_env_name = env.env_name if isinstance(env, UserEnv) else env
+            env_base_version = env_list[env_list['env_name'] == user_env_name].base_env_name.values
+            # Check if the user environment is not found, then return.
+            if len(env_base_version) == 0:
+                return
+            python_env = env_base_version[0].split("_")[1]
+
+            # Get the Python interpreter version of the local environment.
+            from teradataml.context import context as tdmlctx
+            python_local = tdmlctx.python_version_local.rsplit(".", 1)[0]
+            # Check if the Python interpreter major versions are consistent between Lake user environment and local.
+            # If not, raise an exception.
+            if python_env != python_local:
+                raise TeradataMlException(Messages.get_message(MessageCodes.PYTHON_VERSION_MISMATCH_OAF,
+                                                                python_env, python_local),
+                                            MessageCodes.PYTHON_VERSION_MISMATCH_OAF)
+        else:
+            from teradataml.context import context as tdmlctx
+            from teradataml.dbutils.dbutils import (db_python_version_diff,
+                                                    set_session_param)
+            set_session_param("searchuifdbpath",
+                              UtilFuncs._get_dialect_quoted_name(tdmlctx._get_current_databasename()))
+            if len(db_python_version_diff()) > 0:
+                # Raise exception when python versions don't match between Vantage and local.
+                py_major_vantage_version = tdmlctx.python_version_vantage.rsplit(".", 1)[0]
+                raise TeradataMlException(Messages.get_message(MessageCodes.PYTHON_VERSION_MISMATCH,
+                                                                tdmlctx.python_version_vantage, py_major_vantage_version),
+                                            MessageCodes.PYTHON_VERSION_MISMATCH)
+
+    def _check_package_version_diff(func, packages, env=None):
+        """
+        DESCRIPTION:
+            Internal function to process packages differences between Vantage and local.
+            Note:
+                * Raises a warning if the versions of certain Python packages are not consistent between Vantage and local.
+
+        PARAMETERS:
+            func:
+                Required Argument.
+                Specifies the function name.
+                Types: str
+
+            packages:
+                Required Argument.
+                Specifies the list of package names.
+                Types: list of str
+
+            env:
+                Optional Argument.
+                Specifies the user environment for Vantage Cloud Lake.
+                Types: str, object of class UserEnv
+                Default Value: None
+
+        RETURNS:
+            None
+
+        RAISES:
+            OneTimeUserWarning
+
+        EXAMPLES:
+            self._process_package_differences("apply", ["dill"], env)
+        """
+
+        # Check if OSML required packages are verified or not.
+        from teradataml.opensource._constants import \
+            _packages_verified_in_vantage
+        _is_packages_verfied_in_vantage = _packages_verified_in_vantage.get(
+            func, None)
+        if _is_packages_verfied_in_vantage:
+            return
+
+        if env:
+            from teradataml.scriptmgmt.lls_utils import get_env
+            from teradataml.scriptmgmt.UserEnv import UserEnv
+            env = env if isinstance(env, UserEnv) else get_env(env)
+            env_pkg_df = env.libs
+            pkgs_dict = dict(zip(env_pkg_df['name'], env_pkg_df['version']))
+
+            from importlib.metadata import version
+            warning_raised = False
+            strr = []
+            for pkg in packages:
+                env_version = pkgs_dict.get(pkg)
+                local_version = version(pkg)
+                # Write the requirements file listing all the related packages and their versions
+                # if the versions Python packages are not consistent between Vantage and local.
+                if env_version != local_version:
+                    warning_raised = True
+                    strr.append(f"{pkg}=={local_version}")
+
+            # If there are differences in package versions, display a warning message to the user.
+            # about the package differences and the requirements file created for the user to install the packages
+            if warning_raised:
+                file_name = f"requirements_{func}.txt"
+                req_file = os.path.join(GarbageCollector._get_temp_dir_name(), file_name)
+                with open(req_file, "w") as f:
+                    f.write("\n".join(strr))
+
+                packages = "{}".format(packages[0]) if len(packages) == 1 else\
+                           "', '".join(packages[:-1]) + "' and '" + packages[-1]
+
+                if func == "apply":
+                    warning_msg = f"The version of certain Python packages are not consistent between Lake "\
+                        f"user environment and local. Teradata recommends to maintain same version of '{packages}' "\
+                        f"between Lake user environment and local for '{func}'."
+                else:
+                    _packages_verified_in_vantage[func] = True
+                    warning_msg = "The versions of certain Python packages are not consistent between "\
+                        "Lake user environment and local. OpenSourceML compares the versions of '{}' "\
+                        f"(and also matches the patterns of these packages) used by 'td_{func}'. "\
+                        "Teradata recommends same versions for all the Python packages between Lake "\
+                        "user environment and local."
+
+                req = f"\nA requirements file listing all '{func}' " + \
+                    f"related packages and their versions has been written to '{req_file}'. "+ \
+                    "Update the Lake user environment with the required packages.\n"
+
+                warning_msg += req
+                warnings.warn(warning_msg.format(packages), category=OneTimeUserWarning)
+
+        else:
+            # Check if the versions of Python packages are consistent between Vantage and local.
+            from teradataml.dbutils.dbutils import \
+                _db_python_package_version_diff
+            all_package_versions = _db_python_package_version_diff(packages, only_diff=False)
+            package_difference = \
+                all_package_versions[all_package_versions.vantage != all_package_versions.local]
+            # If there are differences in package versions, raise a warning.
+            if package_difference.shape[0] > 0:
+                strr = []
+                # Write the requirements file listing all the related packages and their versions.
+                for rec in all_package_versions.to_records():
+                    strr.append(f"{rec[1]}=={rec[2]}")
+                file_name = f"requirements_{func}.txt"
+                req_file = os.path.join(GarbageCollector._get_temp_dir_name(), file_name)
+                with open(req_file, "w") as f:
+                    f.write("\n".join(strr))
+
+                packages = "{}".format(packages[0]) if len(packages) == 1 else\
+                           "', '".join(packages[:-1]) + "' and '" + packages[-1]
+
+                if func in ["map_row", "map_partition"]:
+                    warning_msg = f"The version of certain Python packages are not consistent between "\
+                        "Vantage and local. User can identify them using db_python_package_version_diff() "\
+                        f"function. Teradata recommends to maintain same version of '{packages}' "\
+                        f"between Vantage and local for '{func}'."
+                else:
+                    _packages_verified_in_vantage[func] = True
+                    warning_msg = "The versions of certain Python packages are not consistent between "\
+                        "Vantage and local. User can identify them using db_python_package_version_diff() "\
+                        "function. OpenSourceML compares the versions of '{}' (and also matches the "\
+                        f"patterns of these packages) used by 'td_{func}'. Teradata "\
+                        "recommends to maintain same versions for all the Python packages between Vantage "\
+                        "and local."
+
+                # Display a warning message to the user about the package differences
+                # and the requirements file created for the user to install the packages.
+                req = f"\nA requirements file listing all '{func}' " + \
+                    f"related packages and their versions has been written to '{req_file}'.\n"
+
+                warning_msg += req
+                warnings.warn(warning_msg.format(packages), category=OneTimeUserWarning)
+
+    @staticmethod
+    def _get_dialect_quoted_name(object_name):
+        """
+        DESCRIPTION:
+            Function to quote the SQL identifiers as per teradatasqlalchemy's quoting rules.
+
+        PARAMETERS:
+            object_name
+                Required Argument.
+                Specifies the name of the SQL identifier to be quoted.
+                Type: str
+
+        RAISES:
+            None
+
+        RETURNS:
+            Quoted object name.
+
+        EXAMPLES:
+            _get_dialect_quoted_name(object_name = "tdml.alice")
+
+        OUTPUT:
+            '"tdml.alice"'
+        """
+        tdp = preparer(td_dialect)
+        return tdp.quote(object_name)
+
+    @staticmethod
+    def _get_http_status_phrases_description():
+        """
+        DESCRIPTION:
+            Function to get phrases and description for all HTTP status codes.
+
+        PARAMETERS:
+            None
+
+        RETURNS:
+            dict
+
+        EXAMPLES:
+            >>> UtilFuncs._get_http_status_phrases_description()
+        """
+        from http import HTTPStatus
+        return {status.value: {"phrase": status.phrase, "description": status.description} \
+                for status in HTTPStatus}
 
+    @staticmethod
+    def _get_time_formatted_string(period):
+        """
+        DESCRIPTION:
+            Converts a string representing Period to the formatted TIMESTAMP/DATE string for snapshot queries.
+
+        PARAMETERS:
+            period:
+                Required Argument.
+                Specifies the period string to be converted.
+                Types: str
+
+        RETURNS:
+            The formatted TIMESTAMP/DATE string.
+
+        RAISES:
+            ValueError.
+
+        EXAMPLES:
+        >>> UtilFuncs._get_time_formatted_string('2025-06-01 12:00:00.123')
+        """
+        # Try to parse as datetime string
+        try:
+            for fmt in ["%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d"]:
+                try:
+                    dt = datetime.datetime.strptime(period, fmt)
+                    # If input had microseconds, preserve them
+                    if "%f" in fmt and "." in period:
+                        # Remove trailing zeros and dot if needed
+                        result = "TIMESTAMP'{}'".format(dt.strftime("%Y-%m-%d %H:%M:%S.%f").rstrip("0").rstrip("."))
+                    elif "%S" in fmt:
+                        result = "TIMESTAMP'{}'".format(dt.strftime("%Y-%m-%d %H:%M:%S"))
+                    else:
+                        result = "DATE'{}'".format(dt.strftime("%Y-%m-%d"))
+                    return result
+                except ValueError:
+                    continue
+            raise ValueError(f"Unrecognized period format: {period}")
+        except Exception as e:
+            raise ValueError(f"Could not convert period: {period}") from e
+
+
+# Keeping at the end to avoid circular dependency
 from teradataml.common.aed_utils import AedUtils
 from teradataml.dbutils.filemgr import remove_file