teradataml 20.0.0.0__py3-none-any.whl → 20.0.0.2__py3-none-any.whl

teradataml/options/__init__.py

@@ -1,11 +1,13 @@
+from teradataml.common.deprecations import argument_deprecation
 from teradataml.common.exceptions import TeradataMlException
 from teradataml.common.messagecodes import MessageCodes
 from teradataml.common.messages import Messages
 from teradataml.options.configure import configure
 from teradataml.utils.internal_buffer import _InternalBuffer
-from teradatasqlalchemy.telemetry.queryband import collect_queryband
+from teradataml.telemetry_utils.queryband import collect_queryband
 
 
+@argument_deprecation("future", ["auth_token", "ues_url"], False, None)
 @collect_queryband(queryband="StCnfgPrms")
 def set_config_params(**kwargs):
     """
@@ -21,9 +23,11 @@ def set_config_params(**kwargs):
              auth_token:
                 Optional Parameter.
                 Specifies the authentication token to connect to VantageCloud Lake.
-                Note:
-                    Authentication token will expire after a specific time.
-                    One can get the new authentication token and set it again.
+                Notes:
+                     * Authentication token will expire after a specific time.
+                       One can get the new authentication token and set it again.
+                     * if "auth_token" is set through this function, then this function
+                       should always be used only after create_context.
                 Types: str
 
             ues_url:
@@ -77,6 +81,11 @@ def set_config_params(**kwargs):
                     The default value is the installation location of In-DB 2.0.0 packages.
                     Older versions of In-DB packages are installed at
                     "/opt/teradata/languages/Python/".
+
+            local_storage:
+                Specifies the location on client where garbage collector folder will be created.
+                Types: str
+
     RETURNS:
         bool
 
@@ -93,7 +102,8 @@ def set_config_params(**kwargs):
         ...                   val_install_location="VAL_USER",
         ...                   read_nos_function_mapping="read_nos_fm",
         ...                   write_nos_function_mapping="write_nos_fm",
-        ...                   indb_install_location="/opt/teradata/languages/Python")
+        ...                   indb_install_location="/opt/teradata/languages/Python",
+        ...                   local_storage="/Users/gc")
         True
 
         # Example 2: Alternatively, set configuration parameters without using set_config_params() function.
@@ -106,6 +116,7 @@ def set_config_params(**kwargs):
         >>> configure.read_nos_function_mapping="read_nos_fm"
         >>> configure.write_nos_function_mapping="write_nos_fm"
         >>> configure.indb_install_location="/opt/teradata/languages/Python"
+        >>> configure.local_storage = "/Users/gc/"
     """
     for option in kwargs:
         try:

teradataml/options/configure.py

@@ -13,7 +13,6 @@ from teradataml.common.exceptions import TeradataMlException
 from teradataml.common.messages import Messages
 from teradataml.common.messagecodes import MessageCodes
 
-
 class _ConfigureSuper(object):
 
     def __init__(self):
@@ -58,6 +57,8 @@ class _Configure(_ConfigureSuper):
     inline_plot = _create_property('inline_plot')
     indb_install_location = _create_property('indb_install_location')
     openml_user_env = _create_property('openml_user_env')
+    local_storage = _create_property('local_storage')
+    stored_procedure_install_location = _create_property('stored_procedure_install_location')
 
     def __init__(self, default_varchar_size=1024, column_casesensitive_handler = False,
                  vantage_version="vantage1.1", val_install_location=None,
@@ -66,7 +67,7 @@ class _Configure(_ConfigureSuper):
                  read_nos_function_mapping="read_nos", write_nos_function_mapping="write_nos",
                  cran_repositories=None, inline_plot=True, 
                  indb_install_location="/var/opt/teradata/languages/sles12sp3/Python/",
-                 openml_user_env=None):
+                 openml_user_env=None, local_storage=None, stored_procedure_install_location="SYSLIB"):
 
         """
         PARAMETERS:
@@ -163,6 +164,23 @@ class _Configure(_ConfigureSuper):
                     # Set the environment to be used for OpenML.
                     _env_name = "OpenAF" # Name of the user defined environment.
                     teradataml.options.configure.openml_user_env = get_env(_env_name)
+
+            local_storage:
+                Specifies the location on client where garbage collector folder will be created.
+                Types: string
+                Example:
+                    # Set the garbage collector location to "/Users/gc/"
+                    teradataml.options.configure.local_storage = "/Users/gc/"
+
+            stored_procedure_install_location:
+                Specifies the name of the database where stored procedures
+                are installed.
+                Types: string
+                Example:
+                    # Set the Stored Procedure install location to 'SYSLIB'
+                    # when stored procedures are installed in 'SYSLIB'.
+                    teradataml.options.configure.stored_procedure_install_location = "SYSLIB"
+
         """
         super().__init__()
         super().__setattr__('default_varchar_size', default_varchar_size)
@@ -179,6 +197,8 @@ class _Configure(_ConfigureSuper):
         super().__setattr__('inline_plot', True)
         super().__setattr__('indb_install_location', indb_install_location)
         super().__setattr__('openml_user_env', openml_user_env)
+        super().__setattr__('local_storage', local_storage)
+        super().__setattr__('stored_procedure_install_location', stored_procedure_install_location)
 
         # internal configurations
         # These configurations are internal and should not be
@@ -221,6 +241,12 @@ class _Configure(_ConfigureSuper):
         super().__setattr__('_oauth_client_id', None)
         # Internal parameter, that is used for specifying the Authentication token expiry time.
         super().__setattr__('_auth_token_expiry_time', None)
+        # Internal parameter, that is used for specifying the OAuth authentication.
+        super().__setattr__('_oauth', None)
+        # Internal parameter, that is used for specifying the current database associated with current connection.
+        super().__setattr__('_current_database_name', None)
+        # Internal parameter, that is used for specifying the database username associated with current connection.
+        super().__setattr__('_database_username', None)
 
     def __setattr__(self, name, value):
         if hasattr(self, name):
@@ -243,7 +269,7 @@ class _Configure(_ConfigureSuper):
                                                                    "greater than or equal to"),
                                               MessageCodes.TDMLDF_POSITIVE_INT)
             elif name in ['column_casesensitive_handler', '_validate_metaexpression',
-                          '_validate_gc', 'inline_plot']:
+                          '_validate_gc', 'inline_plot', '_oauth']:
 
                 if not isinstance(value, bool):
                     raise TeradataMlException(Messages.get_message(MessageCodes.UNSUPPORTED_DATATYPE, name,
@@ -283,15 +309,21 @@ class _Configure(_ConfigureSuper):
                                                                    "a value in {}".format(valid_versions)),
                                               MessageCodes.INVALID_ARG_VALUE)
 
-            elif name in ['val_install_location', 'byom_install_location', 'database_version',
+            elif name in ['val_install_location', 'byom_install_location',
                           'read_nos_function_mapping', 'write_nos_function_mapping',
                           '_byom_model_catalog_database', '_byom_model_catalog_table',
                           '_byom_model_catalog_license', '_byom_model_catalog_license_source',
-                          'indb_install_location']:
+                          'indb_install_location', 'local_storage', 'stored_procedure_install_location']:
                 if not isinstance(value, str):
                     raise TeradataMlException(Messages.get_message(MessageCodes.UNSUPPORTED_DATATYPE, name,
                                                                    'str'),
                                               MessageCodes.UNSUPPORTED_DATATYPE)
+                if name == 'local_storage':
+                    # Validate if path exists.
+                    if not os.path.exists(value):
+                        raise TeradataMlException(
+                            Messages.get_message(MessageCodes.PATH_NOT_FOUND).format(value),
+                            MessageCodes.PATH_NOT_FOUND)
 
             elif name in {'ues_url', '_oauth_end_point', '_oauth_client_id'}:
 
@@ -305,7 +337,8 @@ class _Configure(_ConfigureSuper):
                     value = value[: -1] if value.endswith("/") else value
 
             elif name in ['temp_table_database', 'temp_view_database',
-                          "_byom_model_catalog_license_table", "_byom_model_catalog_license_database"]:
+                          "_byom_model_catalog_license_table", "_byom_model_catalog_license_database",
+                          "_current_database_name", "_database_username", "database_version"]:
                 if not isinstance(value, str) and not isinstance(value, type(None)):
                     raise TeradataMlException(Messages.get_message(MessageCodes.UNSUPPORTED_DATATYPE, name,
                                                                    'str or None'),

teradataml/options/display.py

@@ -58,7 +58,7 @@ class _Display(_DisplaySuper):
                  byte_encoding = 'base16',
                  print_sqlmr_query = False,
                  blob_length=10,
-                 suppress_vantage_runtime_warnings=False,
+                 suppress_vantage_runtime_warnings=True,
                  geometry_column_length=30):
         """
         PARAMETERS:
@@ -111,7 +111,7 @@ class _Display(_DisplaySuper):
                 Specifies whether to display the warnings raised by the Vantage or not.
                 When set to True, warnings raised by Vantage are not displayed.
                 Otherwise, warnings are displayed.
-                Default Value: False
+                Default Value: True
                 Types: bool
                 Example:
                     display.suppress_vantage_runtime_warnings = True

teradataml/plot/axis.py

@@ -951,7 +951,7 @@ class Axis:
             # Execute the node and create the table in Vantage.
             if self.__y_axis_data[0]._parent_df._table_name is None:
                 # Assuming all the columns are from same DataFrame.
-                repr(self.__y_axis_data[0]._parent_df)
+                self.__y_axis_data[0]._parent_df.materialize()
 
             series = TDSeries(data=_df if self.ignore_nulls else self.__x_axis_data[0]._parent_df,
                               id=self.__series_identifier,
@@ -994,7 +994,7 @@ class Axis:
         # Remove null values from DataFrame
         if self.ignore_nulls:
             _df = _df.dropna()
-        repr(_df)
+        _df.materialize()
         series = TDSeries(data=_df,
                           id="id",
                           row_index="x",
@@ -1049,7 +1049,7 @@ class Axis:
 
             # Execute the node and create the table/view in Vantage.
             if self.__y_axis_data[0]._parent_df._table_name is None:
-                repr(self.__y_axis_data[0]._parent_df)
+                self.__y_axis_data[0]._parent_df.materialize()
 
             matrix = TDMatrix(data=_df if self.ignore_nulls else self.__x_axis_data[0]._parent_df,
                               id=self.__series_identifier,
@@ -1077,7 +1077,7 @@ class Axis:
         # Remove null values from DataFrame
         if self.ignore_nulls:
             _df = _df.dropna()
-        repr(_df)
+        _df.materialize()
         matrix = TDMatrix(data=_df,
                           id="id",
                           row_index="x",

teradataml/scriptmgmt/UserEnv.py

@@ -23,6 +23,7 @@ from teradataml import configure
 from teradataml.utils.internal_buffer import _InternalBuffer
 from concurrent.futures import ThreadPoolExecutor, wait
 from teradataml.clients.pkce_client import _DAWorkflow
+from teradataml.clients.auth_client import _AuthWorkflow
 from teradataml.context.context import _get_user
 from teradataml.common.constants import HTTPRequest, CloudProvider
 from teradataml.common.exceptions import TeradataMlException
@@ -31,7 +32,7 @@ from teradataml.common.messagecodes import MessageCodes
 from teradataml.common.utils import UtilFuncs
 from teradataml.utils.validators import _Validators
 from urllib.parse import urlparse
-from teradatasqlalchemy.telemetry.queryband import collect_queryband
+from teradataml.telemetry_utils.queryband import collect_queryband
 
 
 def _get_ues_url(env_type="users", **kwargs):
@@ -219,22 +220,24 @@ def _get_auth_token():
         >>>_get_auth_token()
     """
     # Check the current time. If token is expiring, get another one from refresh token.
-    if configure._auth_token_expiry_time and time.time() > configure._auth_token_expiry_time:
-        # Extract the base URL from "ues_url".
-        ues_url = configure.ues_url
-        client_id = configure._oauth_client_id
+    if configure._oauth:
+        if configure._auth_token_expiry_time and time.time() > configure._auth_token_expiry_time:
+            # Extract the base URL from "ues_url".
+            ues_url = configure.ues_url
+            client_id = configure._oauth_client_id
 
-        url_parser = urlparse(ues_url)
-        base_url = "{}://{}".format(url_parser.scheme, url_parser.netloc)
+            url_parser = urlparse(ues_url)
+            base_url = "{}://{}".format(url_parser.scheme, url_parser.netloc)
 
-        # Get the JWT Token details.
-        da_wf = _DAWorkflow(base_url, client_id)
-        token_data = da_wf._get_token_data()
+            # Get the JWT Token details.
+            da_wf = _DAWorkflow(base_url, client_id)
+            token_data = da_wf._get_token_data()
 
-        # Replace the options with new values.
-        configure._auth_token_expiry_time = time.time() + token_data["expires_in"] - 15
-        # Store the jwt token in internal class attribute.
-        _InternalBuffer.add(auth_token=_AuthToken(token=token_data["access_token"]))
+            # Replace the options with new values.
+            configure._auth_token_expiry_time = time.time() + token_data["expires_in"] - 15
+
+            # Store the jwt token in internal class attribute.
+            _InternalBuffer.add(auth_token=_AuthToken(token=token_data["access_token"]))
 
     return {"Authorization": "Bearer {}".format(_InternalBuffer.get("auth_token").value)}
 
@@ -532,6 +535,8 @@ class UserEnv:
 
         # Check if file exists or not.
         _Validators._validate_file_exists(file_path)
+        # Check if file is empty or not.
+        _Validators._check_empty_file(file_path)
 
         try:
             # If file size is more than 10 MB, upload the file to cloud and export it to UES.
@@ -678,11 +683,11 @@ class UserEnv:
                        "x-ms-blob-type": CloudProvider.X_MS_BLOB_TYPE.value}
 
         # Initiate file upload to cloud.
-        response = UtilFuncs._http_request(cloud_storage_url,
-                                           HTTPRequest.PUT,
-                                           data=UtilFuncs._get_file_contents(file_path,
-                                                                             read_in_binary_mode=True),
-                                           headers=headers)
+        with open(file_path, 'rb') as fp:
+            response = UtilFuncs._http_request(cloud_storage_url,
+                                               HTTPRequest.PUT,
+                                               data=fp,
+                                               headers=headers)
 
         # Since the API is not for UES, it is better to validate and raise error separately.
         if not (200 <= response.status_code < 300):
@@ -3518,6 +3523,8 @@ class UserEnv:
             user environment created in Vantage Languages Ecosystem. If
             model with same name already exists in the remote user
             environment, error is thrown.
+            Note:
+                Maximum size of the model should be less than or equal to 5GB.
 
         PARAMETERS:
             model_path:

teradataml/scriptmgmt/lls_utils.py

@@ -21,13 +21,14 @@ import requests
 
 from json.decoder import JSONDecodeError
 from teradataml import configure
-from teradataml.context.context import _get_user
+from teradataml.context.context import _get_user, get_connection
 from teradataml.common.constants import HTTPRequest, AsyncStatusColumns
 from teradataml.common.exceptions import TeradataMlException
 from teradataml.common.messages import Messages
 from teradataml.common.messagecodes import MessageCodes
 from teradataml.common.utils import UtilFuncs
 from teradataml.clients.pkce_client import _DAWorkflow
+from teradataml.clients.auth_client import _AuthWorkflow
 from teradataml.utils.internal_buffer import _InternalBuffer
 from teradataml.scriptmgmt.UserEnv import UserEnv, _get_auth_token, \
     _process_ues_response, _get_ues_url, _AuthToken
@@ -37,7 +38,7 @@ import warnings
 import webbrowser
 from urllib.parse import parse_qs, urlparse
 from teradataml.utils.utils import _async_run_id_info
-from teradatasqlalchemy.telemetry.queryband import collect_queryband
+from teradataml.telemetry_utils.queryband import collect_queryband
 
 
 @collect_queryband(queryband="LstBsEnv")
@@ -1120,7 +1121,8 @@ def get_env(env_name):
         # Get environments created by the current logged in user.
         user_envs_df = list_user_envs()
 
-        if env_name not in user_envs_df.env_name.values:
+        if (user_envs_df is None or
+                (not user_envs_df.empty and env_name not in user_envs_df.env_name.values)):
             msg_code = MessageCodes.FUNC_EXECUTION_FAILED
             error_msg = Messages.get_message(msg_code, "get_env()", "User environment '{}' not found."
                                                                     " Use 'create_env()' function to create"
@@ -1548,15 +1550,17 @@ def get_user_env():
 
 
 @collect_queryband(queryband="StAthTkn")
-def set_auth_token(ues_url, client_id=None):
+def set_auth_token(ues_url, client_id=None, pat_token=None, pem_file=None, **kwargs):
     """
     DESCRIPTION:
         Function to set the Authentication token to connect to User Environment Service
         in VantageCloud Lake.
         Note:
-            User must have a privilage to login with a NULL password to use set_auth_token().
+            User must have a privilege to login with a NULL password to use set_auth_token().
             Please refer to GRANT LOGON section in Teradata Documentation for more details.
-
+            If ues_url and client_id are specified then authentication is through OAuth.
+            If ues_url, pat_token, pem_file are specified then authentication is through PAT.
+            Refresh token still works but only for OAuth authentication.
 
     PARAMETERS:
         ues_url:
@@ -1570,6 +1574,32 @@ def set_auth_token(ues_url, client_id=None):
             VantageCloud Lake.
             Types: str
 
+        pat_token:
+            Required, if PAT authentication is to be used, optional otherwise.
+            Specifies the PAT token generated from VantageCloud Lake Console.
+            Types: str
+
+        pem_file:
+            Required, if PAT authentication is to be used, optional otherwise.
+            Specifies the path to private key file which is generated from VantageCloud Lake Console.
+            Types: str
+
+        **kwargs:
+            username:
+                Specifies the user for which authentication is to be requested.
+                If not specified, then user associated with current connection is used.
+                Note:
+                    1. Use this option only if name of the database username has lower case letters.
+                    2. This option is used only for PAT and not for OAuth.
+                Types: str
+
+            expiration_time:
+                Specifies the expiration time of the token in seconds. After expiry time JWT token expires and
+                UserEnv methods does not work, user should regenerate the token.
+                Note:
+                    This option is used only for PAT and not for OAuth.
+                Default Value: 31536000
+                Types: int
 
     RETURNS:
         True, if the operation is successful.
@@ -1586,31 +1616,105 @@ def set_auth_token(ues_url, client_id=None):
         # Example 2: Set the Authentication token by specifying the client_id.
         >>> set_auth_token(ues_url=getpass.getpass("ues_url : "),
         ...                client_id=getpass.getpass("client_id : "))
+
+        # Example 3: Set the Authentication token by specifying the "pem_file" and "pat_token"
+        #            without specifying "username".
+        >>> import getpass
+        >>> set_auth_token(ues_url=getpass.getpass("ues_url : "),
+        ...                pat_token=getpass.getpass("pat_token : "),
+        ...                pem_file=getpass.getpass("pem_file : "))
+        True
+
+        # Example 4: Set the Authentication token by specifying the "pem_file" and "pat_token"
+        #            and "username".
+        >>> import getpass
+        >>> set_auth_token(ues_url=getpass.getpass("ues_url : "),
+        ...                pat_token=getpass.getpass("pat_token : "),
+        ...                pem_file=getpass.getpass("pem_file : "))
+        ...                username = "alice")
+        True
     """
+    # Deriving global connection using get_connection().
+    con = get_connection()
+    if con is None:
+        raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_CONTEXT_CONNECTION), 
+                                  MessageCodes.INVALID_CONTEXT_CONNECTION)
+
     __arg_info_matrix = []
     __arg_info_matrix.append(["ues_url", ues_url, False, (str), True])
     __arg_info_matrix.append(["client_id", client_id, True, (str), True])
+    __arg_info_matrix.append(["pat_token", pat_token, True, (str), True])
+    __arg_info_matrix.append(["pem_file", pem_file, True, (str), True])
+
+    username = kwargs.get("username", None)
+    __arg_info_matrix.append((["username", username, True, (str), True]))
+
+    expiration_time = kwargs.get("expiration_time", 31536000)
+    __arg_info_matrix.append((["expiration_time", expiration_time, True, (int), True]))
 
     # Validate arguments.
     _Validators._validate_function_arguments(__arg_info_matrix)
 
+    if client_id and any([pat_token, pem_file]):
+        message = Messages.get_message(MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT,
+                                       "client_id", "pat_token' and 'pem_file")
+        raise TeradataMlException(message, MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT)
+
+    if client_id is None:
+        if (pat_token and pem_file is None) or (pem_file and pat_token is None):
+            message = Messages.get_message(MessageCodes.MUST_PASS_ARGUMENT,
+                                           "pat_token", "pem_file")
+            raise TeradataMlException(message, MessageCodes.MUST_PASS_ARGUMENT)
+
+    # Check if pem file exists.
+    if pem_file is not None:
+        _Validators._validate_file_exists(pem_file)
+
     # Extract the base URL from "ues_url".
     url_parser = urlparse(ues_url)
     base_url = "{}://{}".format(url_parser.scheme, url_parser.netloc)
+    netloc = url_parser.netloc.split('.')[0]
 
-    if client_id is None:
-        netloc = url_parser.netloc
-        client_id = "{}-oaf-device".format(netloc.split('.')[0])
+    # Check if the authentication is PAT based or OAuth.
+    if all(arg is None for arg in [pat_token, pem_file]):
+        configure._oauth = True
+        client_id = "{}-oaf-device".format(netloc) if client_id is None else client_id
+        da_wf = _DAWorkflow(base_url, client_id)
+        token_data = da_wf._get_token_data()
+
+        # Set Open AF parameters.
+        configure._oauth_client_id = client_id
+        configure._oauth_end_point = da_wf.device_auth_end_point
+        configure._auth_token_expiry_time = time() + token_data["expires_in"] - 15
+
+        # Store the jwt token in internal class attribute.
+        _InternalBuffer.add(auth_token=_AuthToken(token=token_data["access_token"]))
+
+    else:
+        configure._oauth = False
+
+        if username is None:
+            # If username is not specified then the database username associated with the current context will be
+            # considered.
+            username = _get_user()
+
+        org_id = netloc
+
+        # Construct a dictionary to be passed to _AuthWorkflow().
+        state_dict = {}
+        state_dict["base_url"] = base_url
+        state_dict["org_id"] = org_id
+        state_dict["pat_token"] = pat_token
+        state_dict["pem_file"] = pem_file
+        state_dict["username"] = username
+        state_dict["expiration_time"] = expiration_time
 
-    da_wf = _DAWorkflow(base_url, client_id)
-    token_data = da_wf._get_token_data()
+        auth_wf = _AuthWorkflow(state_dict)
+        token_data = auth_wf._proxy_jwt()
+        # Store the jwt token in internal class attribute.
+        _InternalBuffer.add(auth_token=_AuthToken(token=token_data))
 
     # Set Open AF parameters.
-    configure._oauth_client_id = client_id
     configure.ues_url = ues_url
-    configure._oauth_end_point = da_wf.device_auth_end_point
-    configure._auth_token_expiry_time = time() + token_data["expires_in"] - 15
-    # Store the jwt token in internal class attribute.
-    _InternalBuffer.add(auth_token=_AuthToken(token=token_data["access_token"]))
 
     return True

teradataml/table_operators/Script.py

@@ -431,8 +431,8 @@ class Script(TableOperator):
             from teradataml import list_td_reserved_keywords
             if get_connection():
                 # Checking for reserved keywords and raising error if present.
-                for column_name in self.returns:
-                    list_td_reserved_keywords(key=column_name, raise_error=True)
+                columns = self.returns
+                list_td_reserved_keywords(key=columns, raise_error=True)
 
     def __validate(self):
         """
@@ -728,7 +728,7 @@ class Script(TableOperator):
             5          1           1
 
             # Example 2 -
-            # Script is tested using test_script and executed on Vantage.
+            # Input data is barrier_new and script is executed on Vantage.
             # use set_data() to reset arguments.
             # Create teradataml DataFrame objects.
             >>> load_example_data("Script", ["barrier_new"])
@@ -751,7 +751,7 @@ class Script(TableOperator):
             ...              sort_ascending=False,
             ...              charset='latin',
             ...              returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
-            # Script is tested using test_script and executed on Vantage.
+            # Script is executed on Vantage.
             >>> sto.execute_script()
             ############ STDOUT Output ############
                word count_input
@@ -786,7 +786,6 @@ class Script(TableOperator):
             5          1           1
 
             # Example 3
-            # Script is tested using test_script and executed on Vantage.
             # In order to run the script with same dataset but different data related
             # arguments, use set_data() to reset arguments.
             # Note: