teradataml 20.0.0.5__py3-none-any.whl → 20.0.0.6__py3-none-any.whl

teradataml/hyperparameter_tuner/utils.py

@@ -276,6 +276,8 @@ class _ProgressBar:
                progress_precent=_progress_percent,
                completed_jobs=self.completed_jobs,
                total_jobs=self.total_jobs)
+        # Add padding to clear any leftover characters from the previous message.
+        padded_msg = _msg.ljust(self.blank_space_len)
         # Display the formatted bar.
-        print(_msg, end='\r', file=self.STDOUT, flush=True)
-        self.blank_space_len = len(_msg)
+        print(padded_msg, end='\r', file=self.STDOUT, flush=True)
+        self.blank_space_len = len(padded_msg)

teradataml/opensource/_base.py

@@ -51,6 +51,7 @@ from teradataml.opensource._wrapper_utils import (_generate_new_name,
                                                   _validate_df_query_type)
 from teradataml.options.configure import configure
 from teradataml.utils.validators import _Validators
+from teradataml.utils.internal_buffer import _InternalBuffer
 
 logger = pylogger.getLogger()
 
@@ -88,9 +89,13 @@ class _GenericObjectWrapper:
                 self._env = configure.openml_user_env
             else:
                 self._env = UtilFuncs._create_or_get_env("open_source_ml.json")
+        else:
+            set_session_param("searchuifdbpath", self._db_name)
 
         # Check if the Python interpreter major versions are consistent between Vantage and local.
-        UtilFuncs._check_python_version_diff(self._env)
+        _check_py_version = _InternalBuffer.get('_check_py_version')
+        if _check_py_version:
+            UtilFuncs._check_python_version_diff(self._env)
             
         # Raise warning when python package versions don't match between Vantage and local.
         # OPENSOURCE_PACKAGE_NAME is set for each opensource package, but not for the base class.
@@ -1257,6 +1262,7 @@ class _OpenSourceObjectWrapper(_GenericObjectWrapper):
 
 class _FunctionWrapper(_GenericObjectWrapper):
     def __init__(self, module_name, func_name, file_type, template_file):
+
         super().__init__()
         self._module_name = module_name
         self._func_name = func_name

teradataml/options/configure.py

@@ -62,6 +62,7 @@ class _Configure(_ConfigureSuper):
     stored_procedure_install_location = _create_property('stored_procedure_install_location')
     table_operator = _create_property('table_operator')
     temp_object_type = _create_property('temp_object_type')
+    use_short_object_name = _create_property('use_short_object_name')
 
     def __init__(self, default_varchar_size=1024, column_casesensitive_handler=False,
                  vantage_version="vantage1.1", val_install_location=None,
@@ -71,7 +72,7 @@ class _Configure(_ConfigureSuper):
                  cran_repositories=None, inline_plot=True,
                  indb_install_location=None,
                  openml_user_env=None, local_storage=None, stored_procedure_install_location="SYSLIB",
-                 table_operator=None, temp_object_type=None):
+                 table_operator=None, temp_object_type=None, use_short_object_name=False):
 
         """
         PARAMETERS:
@@ -140,8 +141,8 @@ class _Configure(_ConfigureSuper):
 
             indb_install_location:
                 Specifies the installation location of In-DB Python package.
-                Types: string
                 Default Value: "/var/opt/teradata/languages/sles12sp3/Python/"
+                Types: string
                 Example:
                     # Set the installation location for older versions.
                     teradataml.options.configure.indb_install_location = "/opt/teradata/languages/Python/"
@@ -182,9 +183,9 @@ class _Configure(_ConfigureSuper):
                 Specifies the type of temporary database objects created internally by teradataml.
                 Permitted Values:
                     * "VT" - Volatile tables.
-                Types: String
                 Default Value: None
-                Note:
+                Types: String
+                Notes:
                     * If this option is set to "VT" and "persist" argument of analytic functions is
                       set to True, then volatile tables are not created as volatile table can't be
                       persisted and "persist" argument takes precedence.
@@ -193,6 +194,16 @@ class _Configure(_ConfigureSuper):
                     # Set the type of temporary database objects to "VT" to create volatile internal
                     # tables.
                     teradataml.options.configure.temp_object_type = "VT"
+
+            use_short_object_name:
+                Specifies whether to use shorter names for temporary tables created internally by teradataml.
+                When set to True, teradataml generates internal temporary table names with a maximum length 
+                of 20 characters. Otherwise, there is no restriction on the length of these table names.
+                Default Value: False
+                Types: bool
+                Example:
+                    # Set the option to use short names for temporary tables.
+                    teradataml.options.configure.use_short_object_name = True
         """
         super().__init__()
         super().__setattr__('default_varchar_size', default_varchar_size)
@@ -213,6 +224,7 @@ class _Configure(_ConfigureSuper):
         super().__setattr__('table_operator', table_operator)
         super().__setattr__('_indb_install_location', indb_install_location)
         super().__setattr__('temp_object_type', self.__get_temp_object_type(temp_object_type))
+        super().__setattr__('use_short_object_name', use_short_object_name)
 
         # internal configurations
         # These configurations are internal and should not be
@@ -447,6 +459,10 @@ class _Configure(_ConfigureSuper):
 
                 value = self.__get_temp_object_type(value)
 
+            elif name == 'use_short_object_name':
+                if not isinstance(value, bool):
+                    raise TypeError(Messages.get_message(MessageCodes.UNSUPPORTED_DATATYPE, name, 'bool'))
+
             super().__setattr__(name, value)
         else:
             raise AttributeError("'{}' object has no attribute '{}'".format(self.__class__.__name__, name))

teradataml/scriptmgmt/UserEnv.py

@@ -20,11 +20,10 @@ import time
 from concurrent.futures import ThreadPoolExecutor, wait
 from json.decoder import JSONDecodeError
 from urllib.parse import urlparse
+
 import pandas as pd
-import requests
 
 from teradataml import configure
-from teradataml.clients.auth_client import _AuthWorkflow
 from teradataml.clients.pkce_client import _DAWorkflow
 from teradataml.common.constants import (AsyncOpStatus, CloudProvider,
                                          HTTPRequest)
@@ -4046,6 +4045,14 @@ class _AuthToken:
     def value(self):
         return self.__value
 
+    @value.setter
+    def value(self, token_value):
+        """
+        DESCRIPTION:
+            Sets value of _AuthToken.
+        """
+        self.__value = token_value
+
     @property
     def auth_type(self):
         return self.__auth_type
@@ -4060,3 +4067,7 @@ class _AuthToken:
         elif self.auth_type.lower() == "bearer":
             # Form the Authorization header value by prepending 'Bearer ' to the JWT token.
             return {"Authorization": "Bearer {}".format(self.value)}
+        elif self.auth_type.lower() == "keycloak":
+            # Get valid token value for current time.
+            self.value = _InternalBuffer.get("keycloak_manager").get_token()
+            return {"Authorization": "Bearer {}".format(self.value)}

teradataml/scriptmgmt/lls_utils.py

@@ -24,27 +24,25 @@ from urllib.parse import urlparse
 import pandas as pd
 
 from teradataml import configure
-from teradataml.context.context import _get_user, get_connection
-from teradataml.common.constants import HTTPRequest, AsyncStatusColumns, AuthMechs
 from teradataml.clients.auth_client import _AuthWorkflow
+from teradataml.clients.keycloak_client import _KeycloakManager
 from teradataml.clients.pkce_client import _DAWorkflow
 from teradataml.common.constants import (AsyncOpStatus, AsyncStatusColumns,
-                                         HTTPRequest)
+                                         AuthMechs, HTTPRequest, TDServices)
 from teradataml.common.exceptions import TeradataMlException
 from teradataml.common.messagecodes import MessageCodes
 from teradataml.common.messages import Messages
 from teradataml.common.utils import UtilFuncs
 from teradataml.context.context import _get_user, get_connection
 from teradataml.scriptmgmt.UserEnv import (UserEnv, _AuthToken,
-                                           _get_auth_token, _get_ues_url,
-                                           _process_ues_response, _get_ccp_url)
+                                           _get_auth_token, _get_ccp_url,
+                                           _get_ues_url, _process_ues_response)
 from teradataml.telemetry_utils.queryband import collect_queryband
 from teradataml.utils.internal_buffer import _InternalBuffer
 from teradataml.utils.utils import _async_run_id_info
 from teradataml.utils.validators import _Validators
 
 
-
 @collect_queryband(queryband="LstBsEnv")
 def list_base_envs():
     """
@@ -1778,6 +1776,18 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
                 Specifies the password for database user to be used for Basic authentication.
                 Types: str
 
+            auth_url:
+                Optional Argument.
+                Specifies the endpoint URL for a keycloak server.
+                Types: str
+
+            rest_client:
+                Optional Argument.
+                Specifies the service for which keycloak token is to be generated.
+                Permitted values: "VECTORSTORE"
+                Default value: "VECTORSTORE"
+                Types: str
+
             auth_mech:
                 Optional Argument.
                 Specifies the mechanism to be used for generating authentication token.
@@ -1787,14 +1797,31 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
                         * OAuth: Token generation is done through OAuth by using client id
                                  which can be sepcified by user in "client_id" argument or
                                  can be derived internally from "base_url".
-                        * PAT  : Token generation is done using "pat_token" and "pem_file".
+                        * PAT: Token generation is done using "pat_token" and "pem_file".
                         * BASIC: Authentication is done via Basic authentication mechanism
                                  using user credentials passed in "username" and "password"
                                  arguments.
-                        * JWT  : Readily available token in "auth_token" argument is used.
-                Permitted Values: "OAuth", "PAT", "BASIC", "JWT".
+                        * JWT: Readily available token in "auth_token" argument is used.
+                        * KEYCLOAK: Token generation is done using keycloak.
+                Permitted Values: "OAuth", "PAT", "BASIC", "JWT", "KEYCLOAK".
                 Types: str
 
+            validate_jwt:
+                Optional Argument.
+                Specifies whether to validate generated JWT token or not.
+                Note:
+                    * Applicable only when "auth_mech" is "PAT".
+                Default value: True
+                Types: boolean
+
+            valid_from:
+                Optional Argument.
+                Specifies epoch seconds representing time from which JWT token will be valid.
+                Note:
+                    * Applicable only when "auth_mech" is "PAT".
+                Default value: None
+                Types: int
+
     RETURNS:
         True, if the operation is successful.
 
@@ -1861,6 +1888,16 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
         Authentication token is generated and set for the session.
         True
 
+        # Example 8: Set the authentication token for by specifying "base_url", "auth_url"
+        #            "password" and "rest_client" and generating keycloak token internally.
+        >>> import getpass
+        >>> set_auth_token(base_url=getpass.getpass("base_url : "),
+        ...                auth_url=getpass.getpass("auth_url : "),
+        ...                password=getpass.getpass("password : "),
+        ...                rest_client=getpass.getpass("rest_client : "))
+        Authentication token is generated and set for the session.
+        True
+
     """
 
     # Deriving global connection using get_connection().
@@ -1886,22 +1923,34 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
     __arg_info_matrix.append(["ues_url", ues_url, True, (str), True])
 
     username = kwargs.get("username", _get_user())
-    __arg_info_matrix.append((["username", username, True, (str), True]))
+    __arg_info_matrix.append(["username", username, True, (str), True])
 
     password = kwargs.get("password", None)
     __arg_info_matrix.append(["password", password, True, (str), True])
 
     auth_token = kwargs.get("auth_token")
-    __arg_info_matrix.append((["auth_token", auth_token, True, (str), True]))
+    __arg_info_matrix.append(["auth_token", auth_token, True, (str), True])
 
-    expiration_time = kwargs.get("expiration_time", 31536000)
-    __arg_info_matrix.append((["expiration_time", expiration_time, True, (int), True]))
+    expiration_time = kwargs.get("expiration_time", 31536000) # 31536000 seconds meaning 365 days.
+    __arg_info_matrix.append(["expiration_time", expiration_time, True, (int), True])
 
     kid = kwargs.get("kid")
-    __arg_info_matrix.append((["kid", kid, True, (str), True]))
+    __arg_info_matrix.append(["kid", kid, True, (str), True])
+
+    auth_url = kwargs.get("auth_url", None)
+    __arg_info_matrix.append(["auth_url", auth_url, True, (str), True])
+
+    rest_client = kwargs.get("rest_client", "VECTORSTORE")
+    __arg_info_matrix.append(["rest_client", rest_client, True, (str), True, [svc.name for svc in TDServices]])
 
     auth_mech = kwargs.get("auth_mech", None)
-    __arg_info_matrix.append((["auth_mech", auth_mech, True, (str), True, [mech.value for mech in AuthMechs]]))
+    __arg_info_matrix.append(["auth_mech", auth_mech, True, (str), True, [mech.name for mech in AuthMechs]])
+
+    validate_jwt = kwargs.get("validate_jwt", True)
+    __arg_info_matrix.append(["validate_jwt", validate_jwt, True, (bool)])
+
+    valid_from = kwargs.get("valid_from", None)
+    __arg_info_matrix.append(["valid_from", valid_from, True, int])
 
     # Validate arguments.
     _Validators._validate_function_arguments(__arg_info_matrix)
@@ -1950,13 +1999,15 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
     if auth_mech:
         auth_mech = auth_mech.lower()
         if auth_mech == 'oauth':
-            pat_token = pem_file = password = auth_token = None
+            pat_token = pem_file = password = auth_token = auth_url = None
         elif auth_mech == 'jwt':
-            pat_token = pem_file = password = client_id = None
+            pat_token = pem_file = password = client_id = auth_url = None
         elif auth_mech == 'basic':
-            pat_token = pem_file = auth_token = client_id = None
+            pat_token = pem_file = auth_token = client_id = auth_url = None
         elif auth_mech == 'pat':
-            password = client_id = auth_token = None
+            password = client_id = auth_token = auth_url = None
+        elif auth_mech == 'keycloak':
+            pat_token = pem_file = auth_token = client_id = None
 
     # Validate arguments for mutual exclusiveness.
     all_groups_none = \
@@ -1964,7 +2015,8 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
                                                                  {"auth_token": auth_token},
                                                                  {"pat_token": pat_token,
                                                                   "pem_file": pem_file},
-                                                                 {"password": password},
+                                                                 {"password": password} if not auth_url else
+                                                                 {"password": password, "auth_url": auth_url},
                                                                  return_all_falsy_status=True)
 
     # Determine authentication mechanism from availability of supportive arguments.
@@ -1973,6 +2025,8 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
             auth_mech = 'jwt'
         elif any([pat_token, pem_file]):
             auth_mech = 'pat'
+        elif auth_url:
+            auth_mech = 'keycloak'
         elif password:
             # Authentication is done via Basic authentication mechanism
             # by passing 'basic' field in header.
@@ -1995,7 +2049,6 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
         _InternalBuffer.add(auth_token=_AuthToken(token=auth_token,
                                                   auth_type='bearer'))
     elif auth_mech == 'oauth':
-        # TODO: Finalize need to set this flag to False in other scenarios.
         configure._oauth = True
         client_id = "{}-oaf-device".format(org_id) if client_id is None else client_id
         da_wf = _DAWorkflow(parsed_base_url, client_id)
@@ -2028,11 +2081,13 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
                                  "pem_file": pem_file,
                                  "username": username,
                                  "expiration_time": expiration_time,
-                                 "kid": kid})
+                                 "kid": kid,
+                                 "valid_from": valid_from})
         token_data = auth_wf._proxy_jwt()
 
-        # Validate generated JWT token.
-        token_validated = _validate_jwt_token(base_url, token_data)
+        if validate_jwt:
+            # Validate generated JWT token.
+            token_validated = _validate_jwt_token(base_url, token_data)
 
         # Store the jwt token in internal class attribute.
         _InternalBuffer.add(auth_token=_AuthToken(token=token_data,
@@ -2047,6 +2102,26 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
         # Store the header data in internal class attribute.
         _InternalBuffer.add(auth_token=_AuthToken(token=encoded_credentials,
                                                   auth_type='basic'))
+    elif auth_mech == 'keycloak':
+        _Validators._validate_missing_required_arguments([["password", password, False, (str), True],
+                                                          ["auth_url", auth_url, False, (str), True]
+                                                          ])
+        token_generator = _KeycloakManager(auth_url=auth_url,
+                                           client_id=TDServices[rest_client].value)
+
+        # Store manager object in _InternalBuffer in order to generate token after expiry time.
+        _InternalBuffer.add(keycloak_manager=token_generator)
+        try:
+            token_data = token_generator.generate_token(username=username,
+                                                        password=password)
+        except:
+            raise TeradataMlException(Messages.get_message(MessageCodes.FUNC_EXECUTION_FAILED,
+                                                           "set_auth_token",
+                                                           "Failed to generate keycloak token."),
+                                      MessageCodes.FUNC_EXECUTION_FAILED)
+
+        _InternalBuffer.add(auth_token=_AuthToken(token=token_data,
+                                                  auth_type='keycloak'))
 
     if token_validated:
         print("Authentication token is generated, authenticated and set for the session.")

teradataml/sdk/README.md

@@ -0,0 +1,79 @@
+## How to create Software Development Kit (SDK) using OpenAPI spec
+
+All SDKs should be in module `teradataml.sdk`. If a module name `<module_name>` is to be created with OpenAPI JSON file, 
+- The json file (say `openapi_spec.json`) should be placed in the directory `teradataml/data/sdk/<module_name>"` after creating required folders to maintain the folder structure.
+- Create a folder `<module_name>` in `teradataml/sdk/` and place `__init__.py` file.
+- Copy and paste the code from `teradataml/sdk/modelops/__init__.py` to newly created `__init__.py` file.
+- Update the constants `SdkNames` and `SdkPackagePaths` in `teradataml/sdk/constants.py` to name the module and locate to appropriate location of OpenAPI JSON spec.
+- Update the values to the constants `SdkNames` and `SdkPackagePaths` in `__init__.py`.
+- Create `teradataml/sdk/<module_name>/_constants.py`.
+- Copy and paste the code from `teradataml/sdk/modelops/_constants.py`. Keep empty dictionaries if no change in class names and function names is required. Fill in the dictionary similar to `teradataml/sdk/modelops/_constants.py` with intended class and function names.
+- If customization is needed in client object, copy and paste the file `teradataml/sdk/modelops/_client.py` to `teradataml/sdk/<module_name>/_client.py` and update corresponding imports (for new client) in `teradataml/sdk/<module_name>/__init__.py`. Otherwise, user can use `BaseClient` from `teradataml/sdk/api_client.py`.
+- Generate Pydantic models for OpenAPI schema using below commands which will create `teradataml/sdk/<module_name>/models.py`. Add unit test cases similar to `teradataml/tests/unit/sdk/test_modelops_schema.py` which import models and use helper class to bulk test the classes.
+```shell
+pip install datamodel-code-generator==0.27.3
+# Run below command from pyTeradata directory.
+datamodel-codegen --input teradataml/data/sdk/<module_name>/openapi_spec.json --input-file-type openapi --output teradataml/sdk/<module_name>/models.py --output-model-type pydantic_v2.BaseModel --use-field-description --keep-model-order --target-python-version=3.8 --use-schema-description  --field-constraints --allow-population-by-field-name --strict-types bool int float # From windows use datamodel-codegen.exe
+```
+- If there are not schema in OpenAPI spec, create empty file `teradataml/sdk/<module_name>/models.py`.
+
+
+## What is created dynamically
+
+When the user runs `from teradataml.sdk.<module_name> import *` or `from teradataml.sdk.<module_name> import AnyClass`, it creates classes and class methods dynamically.
+
+How are these classes created?
+- Tags from OpenAPI spec are converted to Camel case and used as classes.
+- If any of the API endpoint methods does not have tags associated with them, such functions are kept in class name `DefaultApi`.
+
+How are class methods created?
+- All the API endpoint methods form each function of the class.
+- The function names are taken from `OperationId` field of function in OpenAPI JSON spec. The value of `OperationId` is converted to Snake case along with replacing hyphens (`-`) with underscores (`_`). This new value is used as API function name for the user to call by passing parameters.
+
+
+## Example - Vantage ModelOps SDK
+
+Couple of the tags in ModelOps OpenAPI spec is `Alert rules`, `Models`. These tags are converted to classes `AlertRules` and `Models` respectively.
+
+The API end point `/api/models` contains `get` method with `operationId` as `getCollectionResource-model-get`. The framework creates `get_collection_resource_model_get(...)` function in `Models` class. 
+
+Similarly, the API end point `/api/models/{id}/alertRules` contains `get` method with `operationId` as `getAlertRules`. The framework creates `get_alert_rules(...)` functions in `AlertRules` class.
+
+So, these API endpoints can be accessed through SDK as follows:
+```python
+# Authenticate and connect to client.
+from teradataml.sdk.modelops import ModelOpsClient
+from teradataml.sdk import ClientCredentialsAuth # Three different authentication mechanisms are supported.
+auth_obj = ClientCredentialsAuth(
+    auth_client_id="<client_id>",
+    auth_client_secret="<client_secret>",
+    auth_token_url="https://10.27.117.175/sso/realms/teradata/protocol/openid-connect/token"
+)
+client = ModelOpsCleint(
+    base_url="https://10.27.117.175/core",
+    auth=auth_obj,
+    ssl_verify=False
+)
+
+# Import required classes.
+from teradataml.sdk.modelops import AlertRules, Models
+
+# Some APIs are tied with Project ID which needs to be set to the client.
+client.set_project_id("<uuid_project_id>")
+
+# Create Models object and get all the models.
+mod = Models(client=client)
+models = mod.get_collection_resource_model_get()
+
+# Get alert rules for any one of the model.
+ar = AlertRules(client=client)
+ar.get_alert_rules(id="<uuid_model_id>")
+```
+
+## Note:
+- All the argument to the API functions are read as `**kwargs`. So, the user has to pass keyword arguments i.e., positional argument usage is not supported yet.
+- The following validations are done for arguments which accepts basic data types:
+  - Missing required arguments.
+  - Invalid type passed to arguments.
+  - Invalid (non-permitted) value passed to arguments.
+- Validations for dict type objects (like input for `requestBody`) are yet to be added.

teradataml/sdk/__init__.py

@@ -0,0 +1,4 @@
+from teradataml.sdk._auth_modes import (BearerAuth, ClientCredentialsAuth,
+                                        DeviceCodeAuth)
+from teradataml.sdk.api_client import Client
+from teradataml.sdk.constants import SdkPackagePaths