teradataml 20.0.0.5__py3-none-any.whl → 20.0.0.7__py3-none-any.whl

teradataml/scriptmgmt/lls_utils.py

@@ -24,27 +24,25 @@ from urllib.parse import urlparse
 import pandas as pd
 
 from teradataml import configure
-from teradataml.context.context import _get_user, get_connection
-from teradataml.common.constants import HTTPRequest, AsyncStatusColumns, AuthMechs
 from teradataml.clients.auth_client import _AuthWorkflow
+from teradataml.clients.keycloak_client import _KeycloakManager
 from teradataml.clients.pkce_client import _DAWorkflow
 from teradataml.common.constants import (AsyncOpStatus, AsyncStatusColumns,
-                                         HTTPRequest)
+                                         AuthMechs, HTTPRequest, TDServices)
 from teradataml.common.exceptions import TeradataMlException
 from teradataml.common.messagecodes import MessageCodes
 from teradataml.common.messages import Messages
 from teradataml.common.utils import UtilFuncs
 from teradataml.context.context import _get_user, get_connection
 from teradataml.scriptmgmt.UserEnv import (UserEnv, _AuthToken,
-                                           _get_auth_token, _get_ues_url,
-                                           _process_ues_response, _get_ccp_url)
+                                           _get_auth_token, _get_ccp_url,
+                                           _get_ues_url, _process_ues_response)
 from teradataml.telemetry_utils.queryband import collect_queryband
 from teradataml.utils.internal_buffer import _InternalBuffer
 from teradataml.utils.utils import _async_run_id_info
 from teradataml.utils.validators import _Validators
 
 
-
 @collect_queryband(queryband="LstBsEnv")
 def list_base_envs():
     """
@@ -269,7 +267,7 @@ def list_user_envs(env_name=None, **kwargs):
 
         # Example 8: List all user environments where python 3 environment release version has 
         #            odd number. For e.g. python_3.7.x. 
-        >>> list_user_envs(base_env="\.\d*[13579]\.")
+        >>> list_user_envs(base_env="\\.\\d*[13579]\\.")
                   env_name                        env_description  base_env_name language
         1  Customer_Trends                Analyse customer trends        r_4.1.3        R
         2  Fraud_Detection  Fraud detection through time matching  python_3.7.13   Python
@@ -476,29 +474,51 @@ def __create_envs(template):
     return last_successful_env
 
 
-def __get_default_base_env():
+def __get_default_base_env(lang="python"):
     """
-    Function returns the latest python environment available with
-    Open Analytics Framework.
+    DESCRIPTION:
+        Function returns the name of latest environment available with
+        Open Analytics Framework for given programming language.
+
+    PARAMETERS:
+        lang:
+            Optional Argument.
+            Specifies the language for which latest base env is to be retrieved.
+            Default value: "python"
+            Permitted values: "python", "r", "PYTHON", "R"
+            Types: str
+
+    RETURNS:
+        Base environment name.
+
+    RAISES:
+        None.
+
+    EXAMPLES:
+        # Get default R base environment.
+        >>> __get_default_base_env(lang="R")
     """
+    lang = lang.lower()
+    default_env_key = "default_base_env_{}".format(lang)
     # Check if the default base environment is already available.
-    if _InternalBuffer.get('default_base_env') is not None:
-        return _InternalBuffer.get('default_base_env')
+    if _InternalBuffer.get(default_env_key) is not None:
+        return _InternalBuffer.get(default_env_key)
 
     try:
         base_envs = list_base_envs()
-        python_versions = base_envs[base_envs.language == 'Python']['version']
+        versions = base_envs[base_envs.language.str.lower() == lang]['version']
         # Convert version strings to tuples of integers for comparison
-        version_tuples = [tuple(map(int, version.split('.'))) for version in python_versions]
+        version_tuples = [tuple(map(int, version.split('.'))) for version in versions]
         # Find the latest version tuple using max() function
         latest_version_tuple = max(version_tuples)
         # Convert the latest version tuple back to a string
         latest_version = '.'.join(map(str, latest_version_tuple))
-        # Get the base environment name for the latest version
-        _InternalBuffer.add(default_base_env=base_envs[base_envs.version == latest_version]['base_name'].to_list()[0])
-        return _InternalBuffer.get('default_base_env')
+        # Get the base environment name for the latest version and add in internal buffer.
+        _InternalBuffer.add(**{default_env_key:
+                                   base_envs[base_envs.version == latest_version]['base_name'].to_list()[0]})
+        return _InternalBuffer.get(default_env_key)
     except Exception as base_env_err:
-        raise Exception("Failed to obtain default base environment.", str(base_env_err.exception))
+        raise Exception("Failed to obtain default base environment.", str(base_env_err))
 
 
 def __install_files(env, directory):
@@ -801,12 +821,14 @@ def create_env(env_name=None, base_env=None, desc=None, template=None, conda_env
                 # the list of base envs. If not available, set base_env to the default python base env.
                 if not base_env or \
                         base_env.lower() not in list_base_envs()['base_name'].str.lower().to_list():
+                    lang = "python"
                     # Print warning message if base_env provided is not available.
                     if base_env:
                         print(f"Note: The specified base environment '{base_env}' is unavailable. " \
                               "Using the default base environment as specified in the documentation.")
+                        lang = base_env.split('_')[0].lower() # Extract language for given base_env.
                     # Set base_env to the default
-                    base_env = __get_default_base_env()
+                    base_env = __get_default_base_env(lang=lang)
         if not desc:
             desc = "This env '{}' is created with base env '{}'.".format(env_name, base_env)
         try:
@@ -1507,7 +1529,7 @@ def _remove_all_envs(env_type, **kwargs):
                             claim_id_list.append(future_result)
 
                     except (TeradataMlException, RuntimeError, Exception) as emsg:
-                        # Catching exceptions by remove_env if occured in any thread.
+                        # Catching exceptions by remove_env if occurred in any thread.
                         failed_envs[env] = emsg
 
             # Negative case - Failed to remove env.
@@ -1778,6 +1800,18 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
                 Specifies the password for database user to be used for Basic authentication.
                 Types: str
 
+            auth_url:
+                Optional Argument.
+                Specifies the endpoint URL for a keycloak server.
+                Types: str
+
+            rest_client:
+                Optional Argument.
+                Specifies the service for which keycloak token is to be generated.
+                Permitted values: "VECTORSTORE"
+                Default value: "VECTORSTORE"
+                Types: str
+
             auth_mech:
                 Optional Argument.
                 Specifies the mechanism to be used for generating authentication token.
@@ -1787,14 +1821,31 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
                         * OAuth: Token generation is done through OAuth by using client id
                                  which can be sepcified by user in "client_id" argument or
                                  can be derived internally from "base_url".
-                        * PAT  : Token generation is done using "pat_token" and "pem_file".
+                        * PAT: Token generation is done using "pat_token" and "pem_file".
                         * BASIC: Authentication is done via Basic authentication mechanism
                                  using user credentials passed in "username" and "password"
                                  arguments.
-                        * JWT  : Readily available token in "auth_token" argument is used.
-                Permitted Values: "OAuth", "PAT", "BASIC", "JWT".
+                        * JWT: Readily available token in "auth_token" argument is used.
+                        * KEYCLOAK: Token generation is done using keycloak.
+                Permitted Values: "OAuth", "PAT", "BASIC", "JWT", "KEYCLOAK".
                 Types: str
 
+            validate_jwt:
+                Optional Argument.
+                Specifies whether to validate generated JWT token or not.
+                Note:
+                    * Applicable only when "auth_mech" is "PAT".
+                Default value: True
+                Types: boolean
+
+            valid_from:
+                Optional Argument.
+                Specifies epoch seconds representing time from which JWT token will be valid.
+                Note:
+                    * Applicable only when "auth_mech" is "PAT".
+                Default value: 0
+                Types: int
+
     RETURNS:
         True, if the operation is successful.
 
@@ -1861,6 +1912,16 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
         Authentication token is generated and set for the session.
         True
 
+        # Example 8: Set the authentication token for by specifying "base_url", "auth_url"
+        #            "password" and "rest_client" and generating keycloak token internally.
+        >>> import getpass
+        >>> set_auth_token(base_url=getpass.getpass("base_url : "),
+        ...                auth_url=getpass.getpass("auth_url : "),
+        ...                password=getpass.getpass("password : "),
+        ...                rest_client=getpass.getpass("rest_client : "))
+        Authentication token is generated and set for the session.
+        True
+
     """
 
     # Deriving global connection using get_connection().
@@ -1869,7 +1930,7 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
                                   MessageCodes.INVALID_CONTEXT_CONNECTION)
 
     # Remove keys from _InternalBuffer which are interrelated to base_url and authentication token.
-    _InternalBuffer.remove_keys(['list_base_envs', 'default_base_env',
+    _InternalBuffer.remove_keys(['list_base_envs', 'default_base_env_python', 'default_base_env_r',
                                  'vs_session_id', 'vs_header'])
 
     # ---------------------------------ARGUMENT VALIDATION------------------------------------------------------
@@ -1886,22 +1947,34 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
     __arg_info_matrix.append(["ues_url", ues_url, True, (str), True])
 
     username = kwargs.get("username", _get_user())
-    __arg_info_matrix.append((["username", username, True, (str), True]))
+    __arg_info_matrix.append(["username", username, True, (str), True])
 
     password = kwargs.get("password", None)
     __arg_info_matrix.append(["password", password, True, (str), True])
 
     auth_token = kwargs.get("auth_token")
-    __arg_info_matrix.append((["auth_token", auth_token, True, (str), True]))
+    __arg_info_matrix.append(["auth_token", auth_token, True, (str), True])
 
-    expiration_time = kwargs.get("expiration_time", 31536000)
-    __arg_info_matrix.append((["expiration_time", expiration_time, True, (int), True]))
+    expiration_time = kwargs.get("expiration_time", 31536000) # 31536000 seconds meaning 365 days.
+    __arg_info_matrix.append(["expiration_time", expiration_time, True, (int), True])
 
     kid = kwargs.get("kid")
-    __arg_info_matrix.append((["kid", kid, True, (str), True]))
+    __arg_info_matrix.append(["kid", kid, True, (str), True])
+
+    auth_url = kwargs.get("auth_url", None)
+    __arg_info_matrix.append(["auth_url", auth_url, True, (str), True])
+
+    rest_client = kwargs.get("rest_client", "VECTORSTORE")
+    __arg_info_matrix.append(["rest_client", rest_client, True, (str), True, [svc.name for svc in TDServices]])
 
     auth_mech = kwargs.get("auth_mech", None)
-    __arg_info_matrix.append((["auth_mech", auth_mech, True, (str), True, [mech.value for mech in AuthMechs]]))
+    __arg_info_matrix.append(["auth_mech", auth_mech, True, (str), True, [mech.name for mech in AuthMechs]])
+
+    validate_jwt = kwargs.get("validate_jwt", True)
+    __arg_info_matrix.append(["validate_jwt", validate_jwt, True, (bool)])
+
+    valid_from = kwargs.get("valid_from", 0) # This sets iat to UTC beginning.
+    __arg_info_matrix.append(["valid_from", valid_from, True, int])
 
     # Validate arguments.
     _Validators._validate_function_arguments(__arg_info_matrix)
@@ -1919,6 +1992,8 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
 
         # Set the vector_store_base_url. This should only be done if base_url is set.
         # In case ues_url is set, vector_store_base_url should not be set.
+        # Remove trailing forward slash from base_url if present.
+        base_url = base_url[: -1] if base_url.endswith("/") else base_url
         configure._vector_store_base_url = f'{base_url}/data-insights'
 
     if ues_url:
@@ -1950,13 +2025,15 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
     if auth_mech:
         auth_mech = auth_mech.lower()
         if auth_mech == 'oauth':
-            pat_token = pem_file = password = auth_token = None
+            pat_token = pem_file = password = auth_token = auth_url = None
         elif auth_mech == 'jwt':
-            pat_token = pem_file = password = client_id = None
+            pat_token = pem_file = password = client_id = auth_url = None
         elif auth_mech == 'basic':
-            pat_token = pem_file = auth_token = client_id = None
+            pat_token = pem_file = auth_token = client_id = auth_url = None
         elif auth_mech == 'pat':
-            password = client_id = auth_token = None
+            password = client_id = auth_token = auth_url = None
+        elif auth_mech == 'keycloak':
+            pat_token = pem_file = auth_token = client_id = None
 
     # Validate arguments for mutual exclusiveness.
     all_groups_none = \
@@ -1964,7 +2041,8 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
                                                                  {"auth_token": auth_token},
                                                                  {"pat_token": pat_token,
                                                                   "pem_file": pem_file},
-                                                                 {"password": password},
+                                                                 {"password": password} if not auth_url else
+                                                                 {"password": password, "auth_url": auth_url},
                                                                  return_all_falsy_status=True)
 
     # Determine authentication mechanism from availability of supportive arguments.
@@ -1973,6 +2051,8 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
             auth_mech = 'jwt'
         elif any([pat_token, pem_file]):
             auth_mech = 'pat'
+        elif auth_url:
+            auth_mech = 'keycloak'
         elif password:
             # Authentication is done via Basic authentication mechanism
             # by passing 'basic' field in header.
@@ -1995,7 +2075,6 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
         _InternalBuffer.add(auth_token=_AuthToken(token=auth_token,
                                                   auth_type='bearer'))
     elif auth_mech == 'oauth':
-        # TODO: Finalize need to set this flag to False in other scenarios.
         configure._oauth = True
         client_id = "{}-oaf-device".format(org_id) if client_id is None else client_id
         da_wf = _DAWorkflow(parsed_base_url, client_id)
@@ -2028,11 +2107,13 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
                                  "pem_file": pem_file,
                                  "username": username,
                                  "expiration_time": expiration_time,
-                                 "kid": kid})
+                                 "kid": kid,
+                                 "valid_from": valid_from})
         token_data = auth_wf._proxy_jwt()
 
-        # Validate generated JWT token.
-        token_validated = _validate_jwt_token(base_url, token_data)
+        if validate_jwt:
+            # Validate generated JWT token.
+            token_validated = _validate_jwt_token(base_url, token_data)
 
         # Store the jwt token in internal class attribute.
         _InternalBuffer.add(auth_token=_AuthToken(token=token_data,
@@ -2047,6 +2128,26 @@ def set_auth_token(base_url=None, client_id=None, pat_token=None, pem_file=None,
         # Store the header data in internal class attribute.
         _InternalBuffer.add(auth_token=_AuthToken(token=encoded_credentials,
                                                   auth_type='basic'))
+    elif auth_mech == 'keycloak':
+        _Validators._validate_missing_required_arguments([["password", password, False, (str), True],
+                                                          ["auth_url", auth_url, False, (str), True]
+                                                          ])
+        token_generator = _KeycloakManager(auth_url=auth_url,
+                                           client_id=TDServices[rest_client].value)
+
+        # Store manager object in _InternalBuffer in order to generate token after expiry time.
+        _InternalBuffer.add(keycloak_manager=token_generator)
+        try:
+            token_data = token_generator.generate_token(username=username,
+                                                        password=password)
+        except:
+            raise TeradataMlException(Messages.get_message(MessageCodes.FUNC_EXECUTION_FAILED,
+                                                           "set_auth_token",
+                                                           "Failed to generate keycloak token."),
+                                      MessageCodes.FUNC_EXECUTION_FAILED)
+
+        _InternalBuffer.add(auth_token=_AuthToken(token=token_data,
+                                                  auth_type='keycloak'))
 
     if token_validated:
         print("Authentication token is generated, authenticated and set for the session.")

teradataml/sdk/README.md

@@ -0,0 +1,79 @@
+## How to create Software Development Kit (SDK) using OpenAPI spec
+
+All SDKs should be in module `teradataml.sdk`. If a module name `<module_name>` is to be created with OpenAPI JSON file, 
+- The json file (say `openapi_spec.json`) should be placed in the directory `teradataml/data/sdk/<module_name>"` after creating required folders to maintain the folder structure.
+- Create a folder `<module_name>` in `teradataml/sdk/` and place `__init__.py` file.
+- Copy and paste the code from `teradataml/sdk/modelops/__init__.py` to newly created `__init__.py` file.
+- Update the constants `SdkNames` and `SdkPackagePaths` in `teradataml/sdk/constants.py` to name the module and locate to appropriate location of OpenAPI JSON spec.
+- Update the values to the constants `SdkNames` and `SdkPackagePaths` in `__init__.py`.
+- Create `teradataml/sdk/<module_name>/_constants.py`.
+- Copy and paste the code from `teradataml/sdk/modelops/_constants.py`. Keep empty dictionaries if no change in class names and function names is required. Fill in the dictionary similar to `teradataml/sdk/modelops/_constants.py` with intended class and function names.
+- If customization is needed in client object, copy and paste the file `teradataml/sdk/modelops/_client.py` to `teradataml/sdk/<module_name>/_client.py` and update corresponding imports (for new client) in `teradataml/sdk/<module_name>/__init__.py`. Otherwise, user can use `BaseClient` from `teradataml/sdk/api_client.py`.
+- Generate Pydantic models for OpenAPI schema using below commands which will create `teradataml/sdk/<module_name>/models.py`. Add unit test cases similar to `teradataml/tests/unit/sdk/test_modelops_schema.py` which import models and use helper class to bulk test the classes.
+```shell
+pip install datamodel-code-generator==0.27.3
+# Run below command from pyTeradata directory.
+datamodel-codegen --input teradataml/data/sdk/<module_name>/openapi_spec.json --input-file-type openapi --output teradataml/sdk/<module_name>/models.py --output-model-type pydantic_v2.BaseModel --use-field-description --keep-model-order --target-python-version=3.8 --use-schema-description  --field-constraints --allow-population-by-field-name --strict-types bool int float # From windows use datamodel-codegen.exe
+```
+- If there are not schema in OpenAPI spec, create empty file `teradataml/sdk/<module_name>/models.py`.
+
+
+## What is created dynamically
+
+When the user runs `from teradataml.sdk.<module_name> import *` or `from teradataml.sdk.<module_name> import AnyClass`, it creates classes and class methods dynamically.
+
+How are these classes created?
+- Tags from OpenAPI spec are converted to Camel case and used as classes.
+- If any of the API endpoint methods does not have tags associated with them, such functions are kept in class name `DefaultApi`.
+
+How are class methods created?
+- All the API endpoint methods form each function of the class.
+- The function names are taken from `OperationId` field of function in OpenAPI JSON spec. The value of `OperationId` is converted to Snake case along with replacing hyphens (`-`) with underscores (`_`). This new value is used as API function name for the user to call by passing parameters.
+
+
+## Example - Vantage ModelOps SDK
+
+Couple of the tags in ModelOps OpenAPI spec is `Alert rules`, `Models`. These tags are converted to classes `AlertRules` and `Models` respectively.
+
+The API end point `/api/models` contains `get` method with `operationId` as `getCollectionResource-model-get`. The framework creates `get_collection_resource_model_get(...)` function in `Models` class. 
+
+Similarly, the API end point `/api/models/{id}/alertRules` contains `get` method with `operationId` as `getAlertRules`. The framework creates `get_alert_rules(...)` functions in `AlertRules` class.
+
+So, these API endpoints can be accessed through SDK as follows:
+```python
+# Authenticate and connect to client.
+from teradataml.sdk.modelops import ModelOpsClient
+from teradataml.sdk import ClientCredentialsAuth # Three different authentication mechanisms are supported.
+auth_obj = ClientCredentialsAuth(
+    auth_client_id="<client_id>",
+    auth_client_secret="<client_secret>",
+    auth_token_url="https://10.27.117.175/sso/realms/teradata/protocol/openid-connect/token"
+)
+client = ModelOpsCleint(
+    base_url="https://10.27.117.175/core",
+    auth=auth_obj,
+    ssl_verify=False
+)
+
+# Import required classes.
+from teradataml.sdk.modelops import AlertRules, Models
+
+# Some APIs are tied with Project ID which needs to be set to the client.
+client.set_project_id("<uuid_project_id>")
+
+# Create Models object and get all the models.
+mod = Models(client=client)
+models = mod.get_collection_resource_model_get()
+
+# Get alert rules for any one of the model.
+ar = AlertRules(client=client)
+ar.get_alert_rules(id="<uuid_model_id>")
+```
+
+## Note:
+- All the argument to the API functions are read as `**kwargs`. So, the user has to pass keyword arguments i.e., positional argument usage is not supported yet.
+- The following validations are done for arguments which accepts basic data types:
+  - Missing required arguments.
+  - Invalid type passed to arguments.
+  - Invalid (non-permitted) value passed to arguments.
+- Validations for dict type objects (like input for `requestBody`) are yet to be added.

teradataml/sdk/__init__.py

@@ -0,0 +1,4 @@
+from teradataml.sdk._auth_modes import (BearerAuth, ClientCredentialsAuth,
+                                        DeviceCodeAuth)
+from teradataml.sdk.api_client import Client
+from teradataml.sdk.constants import SdkPackagePaths