teradataml 17.20.0.6__py3-none-any.whl → 20.0.0.0__py3-none-any.whl

teradataml/opensource/sklearn/_wrapper_utils.py

@@ -0,0 +1,268 @@
+# ################################################################## 
+# 
+# Copyright 2023 Teradata. All rights reserved.
+# TERADATA CONFIDENTIAL AND TRADE SECRET
+# 
+# Primary Owner: Adithya Avvaru (adithya.avvaru@teradata.com)
+# Secondary Owner: Pankaj Purandare (pankajvinod.purandare@teradata.com)
+# 
+# Version: 1.0
+# Function Version: 1.0
+#
+# This file contains helper functions for opensource wrapper and sklearn wrapper class.
+# 
+# ##################################################################
+
+import time
+import functools
+import uuid
+
+from math import floor
+from teradataml import TeradataMlException
+from teradataml.common.messagecodes import MessageCodes
+from teradataml.common.messages import Messages
+from teradataml.common.aed_utils import AedUtils
+from teradataml.dataframe.dataframe_utils import DataFrameUtils
+from teradataml.dataframe.dataframe import DataFrame, in_schema
+from teradataml.common.utils import UtilFuncs
+from teradataml.utils.validators import _Validators
+from teradataml.dataframe.sql_interfaces import ColumnExpression
+
+aed_utils = AedUtils()
+df_utils = DataFrameUtils()
+
+def _validate_fit_run(func):
+    """
+    Internal function to validate if the model is fitted before calling function specified
+    by "func" parameter.
+
+    PARAMETERS:
+        func    - Specifies the function to be called if the model is fitted.
+
+    RETURNS:
+        function call
+
+    RAISES:
+        TeradataMlException if model is not fitted and function is called.
+    """
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        func_name = kwargs["name"]
+        if not self._is_model_installed:
+            raise TeradataMlException(Messages.get_message(MessageCodes.MODEL_NOT_FITTED,
+                                                           func_name),
+                                      MessageCodes.MODEL_NOT_FITTED)
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+def _generate_new_name(type=None, extension=None):
+    """
+    Internal function to generate new column name or file name.
+
+    PARAMETERS:
+        type    - Specifies the type of the name to be generated.
+                  Permitted Values: 'column', 'file'
+        extension   - Specifies the extension to be added to the random file name.
+                      Applicable only when type is file'
+
+    RETURNS:
+        New name as string
+
+    RAISES:
+        None
+    """
+    timestamp = time.time()
+    tmp = "{}{}".format(floor(timestamp / 1000000),
+                        floor(timestamp % 1000000 * 1000000 +
+                              int(str(uuid.uuid4().fields[-1])[:10])))
+    if type:
+        tmp = f"{type}_{tmp}_"
+    if extension:
+        tmp = f"{tmp}.{extension}"
+    return tmp
+
+def _derive_df_and_required_columns(X=None, y=None, groups=None, kwargs={},
+                                    fit_partition_cols=None):
+    """
+    Internal function to get parent teradataml DataFrame from X, y and groups and corresponding
+    feature columns, label columns and group columns along with partition_columns passed
+    in kwargs.
+
+    PARAMETERS:
+        X   - Specifies the teradataml DataFrame containing data related to feature columns.
+        y   - Specifies the teradataml DataFrame containing data related to label columns.
+        groups  - Specifies the teradataml DataFrame containing data related to group columns.
+        kwargs  - Specifies the dictionary of arguments with keys:
+                    - data
+                    - feature_columns
+                    - label_columns
+                    - group_columns
+                    - partition_columns
+        fit_partition_cols - Specifies the partition columns fitted to the model.
+
+    RETURNS:
+        parent DataFrame, feature columns, label columns, group columns, data partition columns
+
+    RAISES:
+        TeradataMlException if columns are not from the given DataFrame "data".
+    """
+    data = kwargs.get("data", None)
+    feature_columns = kwargs.get("feature_columns", None)
+    feature_columns = UtilFuncs._as_list(feature_columns) if feature_columns else []
+    label_columns = kwargs.get("label_columns", None)
+    label_columns = UtilFuncs._as_list(label_columns) if label_columns else []
+    group_columns = kwargs.get("group_columns", None)
+    group_columns = UtilFuncs._as_list(group_columns) if group_columns else []
+    partition_columns = kwargs.get("partition_columns", None)
+    if partition_columns:
+        partition_columns = UtilFuncs._as_list(partition_columns)
+    elif fit_partition_cols:
+        partition_columns = fit_partition_cols
+    else:
+        partition_columns = []
+
+    if X:
+        # If X is passed, then data, feature_columns, label_columns, group_columns are
+        # ignored and derived from X, y, groups arguments.
+        feature_columns = [col for col in X.columns if col not in partition_columns]
+        all_dfs = [X]
+        label_columns = []
+        group_columns = []
+        if y:
+            all_dfs.append(y)
+            label_columns = y.columns
+        if groups:
+            all_dfs.append(groups)
+            group_columns = groups.columns
+
+        data = df_utils._get_common_parent_df_from_dataframes(all_dfs)
+
+    if data:
+        # Execute node, if not executed already to get table name.
+        if aed_utils._aed_is_node_executed(data._nodeid):
+            _ = df_utils._execute_node_return_db_object_name(data._nodeid)
+
+    # Validate if columns in "feature_columns", "label_columns" are different or not.
+    if set(feature_columns).intersection(set(label_columns)):
+        raise TeradataMlException(Messages.get_message(MessageCodes.ARGS_WITH_SAME_COLUMNS,
+                                                       "feature_columns/X DataFrame",
+                                                       "label_columns/y DataFrame",
+                                                       " not"),
+                                  MessageCodes.ARGS_WITH_SAME_COLUMNS)
+
+    return data, feature_columns, label_columns, group_columns, partition_columns
+
+def _validate_df_query_type(df, node_query_type, arg_name):
+    """
+    Internal function to validate if the DataFrame's node type is same as the type specified.
+
+    PARAMETERS:
+        df  - Specifies the teradataml DataFrame to be validated for node type.
+        node_query_type - Specifies the type of the node to be compared with.
+        arg_name - Specifies the name of the argument in which the node id is passed.
+
+    RETURNS:
+        None
+
+    RAISES:
+        TeradataMlException if node with given id is not same as the type specified.
+    """
+    if df and aed_utils._aed_get_node_query_type(df._nodeid) != node_query_type:
+        raise TeradataMlException(Messages.get_message(MessageCodes.NODE_NOT_GIVEN_TYPE,
+                                                       arg_name, node_query_type),
+                                  MessageCodes.NODE_NOT_GIVEN_TYPE)
+
+def _validate_opensource_func_args(X=None, y=None, groups=None, fit_partition_cols=None,
+                                   kwargs={}, skip_either_or_that=False):
+    """
+    Internal function to validate arguments passed to exposed opensource APIs.
+
+    PARAMETERS:
+        X   - Specifies the teradataml DataFrame containing data related to feature columns.
+        y   - Specifies the teradataml DataFrame containing data related to label columns.
+        groups  - Specifies the teradataml DataFrame containing data related to group columns.
+        fit_partition_cols - Specifies the partition columns fitted to the model.
+        kwargs  - Specifies the dictionary of arguments with keys:
+                    - data
+                    - feature_columns
+                    - label_columns
+                    - group_columns
+                    - partition_columns
+        skip_either_or_that - Specifies whether to skip validation of either or that arguments.
+    RETURNS:
+        None
+
+    RAISES:
+        TeradataMlException if arguments' validations fail.
+    """
+    data = kwargs.get("data", None)
+    feature_columns = kwargs.get("feature_columns", None)
+    label_columns = kwargs.get("label_columns", None)
+    group_columns = kwargs.get("group_columns", None)
+    partition_columns = kwargs.get("partition_columns", None)
+
+    # Argument validations
+    arg_info_matrix = []
+    arg_info_matrix.append(["X", X, True, (DataFrame)])
+    arg_info_matrix.append(["y", y, True, (DataFrame)])
+    arg_info_matrix.append(["groups", groups, True, (DataFrame)])
+    arg_info_matrix.append(["partition_columns", partition_columns, True, (str, list)])
+    arg_info_matrix.append(["data", data, True, (DataFrame)])
+    arg_info_matrix.append(["feature_columns", feature_columns, True, (str, list)])
+    arg_info_matrix.append(["label_columns", label_columns, True, (str, list)])
+    arg_info_matrix.append(["group_columns", group_columns, True, (str, list)])
+
+    # Validate argument types
+    _Validators._validate_function_arguments(arg_info_matrix)
+
+    if not skip_either_or_that and not X and not data and not feature_columns:
+        raise TeradataMlException(Messages.get_message(MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT,
+                                                       "X", "data and feature_columns"),
+                                    MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT)
+
+    if X:
+        # Validate if "X", "y" and "groups" are from select() call.
+        # Whether they are from same parent or not, will be validated while parent is being
+        # extracted.
+        if y or groups:
+            # If any one of "y" or "groups" exists, then "X" must be from select() call.
+            # Otherwise, it can be from any parent or direct DataFrame.
+            _validate_df_query_type(X, "select", "X")
+
+        _validate_df_query_type(y, "select", "y")
+        _validate_df_query_type(groups, "select", "groups")
+
+        # Validate if columns in "partition_columns" argument are present in "X".
+        _Validators._validate_column_exists_in_dataframe(columns=partition_columns,
+                                                         metaexpr=X._metaexpr,
+                                                         column_arg="partition_columns",
+                                                         data_arg="X")
+
+        if fit_partition_cols and not partition_columns and \
+            not all([col in X.columns for col in fit_partition_cols]):
+            # Check if fitted partition columns are present in "X" if "partition_columns" is not
+            # not passed.
+            msg = Messages.get_message(MessageCodes.PARTITIONING_COLS_DIFFERENT, "X")
+            raise TeradataMlException(msg, MessageCodes.PARTITIONING_COLS_DIFFERENT)
+
+    if data and not X:
+        # Thse validations are required only when "X" is not passed because "data" is ignored when
+        # "X" is passed.
+
+        all_cols_list = [feature_columns, label_columns, group_columns, partition_columns]
+        arg_name_list = ["feature_columns", "label_columns", "group_columns", "partition_columns"]
+
+        for cols, arg_name in zip(all_cols_list, arg_name_list):
+            # Validate if columns in these arguments are present in "data".
+            _Validators._validate_column_exists_in_dataframe(columns=cols,
+                                                             metaexpr=data._metaexpr,
+                                                             column_arg=arg_name,
+                                                             data_arg="data")
+
+        if fit_partition_cols and not partition_columns and \
+            not all([col in data.columns for col in fit_partition_cols]):
+            # Check if fitted partition columns are present in "data" if "partition_columns" is not
+            # not passed.
+            msg = Messages.get_message(MessageCodes.PARTITIONING_COLS_DIFFERENT, "data")
+            raise TeradataMlException(msg, MessageCodes.PARTITIONING_COLS_DIFFERENT)

teradataml/opensource/sklearn/constants.py

@@ -0,0 +1,54 @@
+# ################################################################## 
+# 
+# Copyright 2023 Teradata. All rights reserved.
+# TERADATA CONFIDENTIAL AND TRADE SECRET
+# 
+# Primary Owner: Adithya Avvaru (adithya.avvaru@teradata.com)
+# Secondary Owner: Pankaj Purandare (pankajvinod.purandare@teradata.com)
+# 
+# Version: 1.0
+# Function Version: 1.0
+#
+# This file contains constants needed for the opensource packages.
+# 
+# ################################################################## 
+
+from enum import Enum
+from teradataml import VARCHAR, BLOB
+from dataclasses import dataclass, field
+from typing import Any, Dict, Tuple, Optional, List
+
+
+_MODULES = ["sklearn.calibration", "sklearn.cluster", "sklearn.compose", "sklearn.covariance",
+            "sklearn.decomposition", "sklearn.discriminant_analysis",
+            "sklearn.dummy", "sklearn.ensemble", "sklearn.feature_extraction", "sklearn.feature_selection",
+            "sklearn.gaussian_process", "sklearn.impute", "sklearn.isotonic", "sklearn.kernel_approximation",
+            "sklearn.kernel_ridge", "sklearn.linear_model", "sklearn.manifold", "sklearn.mixture",
+            "sklearn.model_selection", "sklearn.multiclass", "sklearn.multioutput", "sklearn.naive_bayes",
+            "sklearn.neighbors", "sklearn.neural_network", "sklearn.preprocessing", "sklearn.random_projection",
+            "sklearn.semi_supervised", "sklearn.svm", "sklearn.tree", "sklearn.pipeline", "sklearn.cross_decomposition",
+            "sklearn.gaussian_process.kernels", "sklearn.metrics"]
+
+class OpenSourcePackage(Enum):
+    SKLEARN = "sklearn"
+
+    @classmethod
+    def values(cls):
+        return [item.value for item in cls]
+
+
+@dataclass
+class OpensourceModels:
+    """Dataclass for Opensource Models details."""
+    is_default_partition_value: bool  # Whether partition value is default or not.
+    partition_file_prefix: str
+    model: Any # Either individual model or pandas dataframe of models with partition columns.
+    pos_args: Tuple[Any] = tuple() # Positional arguments used for model creation.
+    key_args: Dict[str, Any] = field(default_factory=dict) # Keyword arguments used for model creation.
+    fit_partition_columns_non_default: Optional[str] = None  # Columns used for partitioning.
+
+# Model table details used by opensource BYOM.
+_OSML_MODELS_TABLE_NAME = "opensourceml_models"
+_OSML_MODELS_PRIMARY_INDEX = "model_id"
+_OSML_ADDITIONAL_COLUMN_TYPES = {"package": VARCHAR(128)} # sklearn or keras etc
+_OSML_MODELS_TABLE_COLUMNS_TYPE_DICT = {"model_id": VARCHAR(128), "model": BLOB()}

teradataml/options/__init__.py

@@ -3,7 +3,10 @@ from teradataml.common.messagecodes import MessageCodes
 from teradataml.common.messages import Messages
 from teradataml.options.configure import configure
 from teradataml.utils.internal_buffer import _InternalBuffer
+from teradatasqlalchemy.telemetry.queryband import collect_queryband
 
+
+@collect_queryband(queryband="StCnfgPrms")
 def set_config_params(**kwargs):
     """
     DESCRIPTION:
@@ -54,10 +57,6 @@ def set_config_params(**kwargs):
                 are installed.
                 Types: str
 
-            sandbox_container_id:
-                Specifies the id of sandbox container that will be used by Script.test_script() method.
-                Types: string
-
             database_version:
                 Specifies the database version of the system teradataml is connected to.
                 Types: str
@@ -92,7 +91,6 @@ def set_config_params(**kwargs):
         ...                   certificate_file="cert.crt",
         ...                   default_varchar_size=512,
         ...                   val_install_location="VAL_USER",
-        ...                   sandbox_container_id="bgf1233csdh123",
         ...                   read_nos_function_mapping="read_nos_fm",
         ...                   write_nos_function_mapping="write_nos_fm",
         ...                   indb_install_location="/opt/teradata/languages/Python")
@@ -105,7 +103,6 @@ def set_config_params(**kwargs):
         >>> configure.certificate_file="cert.crt"
         >>> configure.default_varchar_size=512
         >>> configure.val_install_location="VAL_USER"
-        >>> configure.sandbox_container_id="bgf1233csdh123"
         >>> configure.read_nos_function_mapping="read_nos_fm"
         >>> configure.write_nos_function_mapping="write_nos_fm"
         >>> configure.indb_install_location="/opt/teradata/languages/Python"

teradataml/options/configure.py

@@ -50,7 +50,6 @@ class _Configure(_ConfigureSuper):
     vantage_version = _create_property('vantage_version')
     val_install_location = _create_property('VAL_install_location')
     byom_install_location = _create_property('BYOM_install_location')
-    sandbox_container_id = _create_property('sandbox_container_id')
     temp_table_database = _create_property('temp_table_database')
     temp_view_database = _create_property('temp_view_database')
     read_nos_function_mapping = _create_property('read_nos_function_mapping')
@@ -58,14 +57,16 @@ class _Configure(_ConfigureSuper):
     cran_repositories = _create_property('cran_repositories')
     inline_plot = _create_property('inline_plot')
     indb_install_location = _create_property('indb_install_location')
+    openml_user_env = _create_property('openml_user_env')
 
     def __init__(self, default_varchar_size=1024, column_casesensitive_handler = False,
                  vantage_version="vantage1.1", val_install_location=None,
-                 byom_install_location=None, sandbox_container_id=None,
-                 temp_table_database=None, temp_view_database=None, database_version=None,
+                 byom_install_location=None, temp_table_database=None,
+                 temp_view_database=None, database_version=None,
                  read_nos_function_mapping="read_nos", write_nos_function_mapping="write_nos",
                  cran_repositories=None, inline_plot=True, 
-                 indb_install_location="/var/opt/teradata/languages/sles12sp3/Python/"):
+                 indb_install_location="/var/opt/teradata/languages/sles12sp3/Python/",
+                 openml_user_env=None):
 
         """
         PARAMETERS:
@@ -113,13 +114,6 @@ class _Configure(_ConfigureSuper):
                     # when BYOM functions are installed in 'SYSLIB'.
                     teradataml.options.configure.byom_install_location = "SYSLIB"
 
-            sandbox_container_id:
-                Specifies the id of sandbox container that will be used by test_script method.
-                Types: string
-                Example:
-                    # Set the sandbox_container_id.
-                    teradataml.options.configure.sandbox_container_id = '734rfjsls3'
-
             database_version:
                 Specifies the actual database version of the system teradataml is connected to.
                 Types: string
@@ -161,6 +155,14 @@ class _Configure(_ConfigureSuper):
                 Example:
                     # Set the installation location for older versions.
                     teradataml.options.configure.indb_install_location = "/opt/teradata/languages/Python/"
+
+            openml_user_env:
+                Specifies the user environment to be used for OpenML.
+                Types: UserEnv
+                Example:
+                    # Set the environment to be used for OpenML.
+                    _env_name = "OpenAF" # Name of the user defined environment.
+                    teradataml.options.configure.openml_user_env = get_env(_env_name)
         """
         super().__init__()
         super().__setattr__('default_varchar_size', default_varchar_size)
@@ -168,7 +170,6 @@ class _Configure(_ConfigureSuper):
         super().__setattr__('vantage_version', vantage_version)
         super().__setattr__('val_install_location', val_install_location)
         super().__setattr__('byom_install_location', byom_install_location)
-        super().__setattr__('sandbox_container_id', sandbox_container_id)
         super().__setattr__('temp_table_database', temp_table_database)
         super().__setattr__('temp_view_database', temp_view_database)
         super().__setattr__('database_version', database_version)
@@ -177,6 +178,7 @@ class _Configure(_ConfigureSuper):
         super().__setattr__('cran_repositories', cran_repositories)
         super().__setattr__('inline_plot', True)
         super().__setattr__('indb_install_location', indb_install_location)
+        super().__setattr__('openml_user_env', openml_user_env)
 
         # internal configurations
         # These configurations are internal and should not be
@@ -185,11 +187,6 @@ class _Configure(_ConfigureSuper):
         # Internal parameter, that should be used while testing to validate whether
         # Garbage collection is being done or not.
         super().__setattr__('_validate_gc', False)
-        # Internal parameter, that is used for checking if sto sandbox image exists on user's system
-        super().__setattr__('_latest_sandbox_exists', False)
-        # Internal parameter, that is used for checking whether a container was started by
-        # teradataml.
-        super().__setattr__('_container_started_by_teradataml', None)
         # Internal parameter, that is used for specifying the global model cataloging schema name which
         # will be used by the byom APIs.
         super().__setattr__('_byom_model_catalog_database', None)
@@ -246,7 +243,7 @@ class _Configure(_ConfigureSuper):
                                                                    "greater than or equal to"),
                                               MessageCodes.TDMLDF_POSITIVE_INT)
             elif name in ['column_casesensitive_handler', '_validate_metaexpression',
-                          '_validate_gc', '_latest_sandbox_exists', 'inline_plot']:
+                          '_validate_gc', 'inline_plot']:
 
                 if not isinstance(value, bool):
                     raise TeradataMlException(Messages.get_message(MessageCodes.UNSUPPORTED_DATATYPE, name,
@@ -307,8 +304,7 @@ class _Configure(_ConfigureSuper):
                 if name == 'ues_url':
                     value = value[: -1] if value.endswith("/") else value
 
-            elif name in ['sandbox_container_id', '_container_started_by_teradataml',
-                          'temp_table_database', 'temp_view_database',
+            elif name in ['temp_table_database', 'temp_view_database',
                           "_byom_model_catalog_license_table", "_byom_model_catalog_license_database"]:
                 if not isinstance(value, str) and not isinstance(value, type(None)):
                     raise TeradataMlException(Messages.get_message(MessageCodes.UNSUPPORTED_DATATYPE, name,
@@ -328,6 +324,11 @@ class _Configure(_ConfigureSuper):
                             raise TypeError(
                                 Messages.get_message(MessageCodes.UNSUPPORTED_DATATYPE, name, 'str, list of str or None'))
 
+            elif name == 'openml_user_env':
+                from teradataml.scriptmgmt.UserEnv import UserEnv
+                if not isinstance(value, UserEnv) and not isinstance(value, type(None)):
+                    raise TypeError(Messages.get_message(MessageCodes.UNSUPPORTED_DATATYPE, name, 'UserEnv or None'))
+
             super().__setattr__(name, value)
         else:
             raise AttributeError("'{}' object has no attribute '{}'".format(self.__class__.__name__, name))