teradataml 20.0.0.3__py3-none-any.whl → 20.0.0.5__py3-none-any.whl

teradataml/opensource/_class.py

@@ -15,10 +15,14 @@
 # ################################################################## 
 
 from importlib import import_module
-from teradataml.opensource.sklearn._sklearn_wrapper import _SkLearnObjectWrapper, _SKLearnFunctionWrapper
-from teradataml.opensource._lightgbm import _LightgbmDatasetWrapper, \
-    _LightgbmFunctionWrapper, _LightgbmBoosterWrapper, _LighgbmSklearnWrapper
-from teradataml.opensource.constants import _SKL_MODULES, _LIGHTGBM_MODULES
+
+from teradataml.opensource._constants import _LIGHTGBM_MODULES, _SKL_MODULES
+from teradataml.opensource._lightgbm import (_LightgbmSklearnWrapper,
+                                             _LightgbmBoosterWrapper,
+                                             _LightgbmDatasetWrapper,
+                                             _LightgbmFunctionWrapper)
+from teradataml.opensource._sklearn import (_SKLearnFunctionWrapper,
+                                            _SkLearnObjectWrapper)
 
 
 class _OpenSource:
@@ -82,6 +86,7 @@ class _OpenSource:
                 Specifies the teradataml supported opensource model object that is to be deployed.
                 Currently supported models are:
                     - sklearn
+                    - lightgbm
                 Types: object
 
             replace_if_exists:
@@ -100,6 +105,9 @@ class _OpenSource:
             "replace_if_exists" is set to False.
 
         EXAMPLES:
+            ## sklearn examples.
+
+            # Import required packages and create LinearRegression sklearn object.
             >>> from teradataml import td_sklearn
             >>> from sklearn.linear_model import LinearRegression
             >>> model = LinearRegression(normalize=True)
@@ -118,6 +126,53 @@ class _OpenSource:
             >>> lin_reg
             LinearRegression(normalize=True)
 
+            ## lightgbm examples.
+
+            # Import required packages and create LGBMClassifier lightGBM object.
+            >>> from teradataml import td_lightgbm
+            >>> import lightgbm as lgb
+            >>> model = lgb.LGBMClassifier()
+
+            # Example 1: Deploy the LightGBM model to Vantage.
+            >>> lgb_model = td_lightgbm.deploy("lgb_model_ver_1", model)
+            Model is saved.
+            >>> lgb_model
+            LGBMClassifier()
+
+            # Example 2: Deploy the LightGBM model to Vantage with the name same as that of model that
+            #            already existed in Vantage.
+            >>> lgb_model = td_lightgbm.deploy("lgb_model_ver_1", model, replace_if_exists=True)
+            Model is deleted.
+            Model is saved.
+            >>> lgb_model
+            LGBMClassifier()
+
+            # Example 3: Deploy LightGBM model trained locally using train() function to Vantage.
+            # Create Dataset object locally, assuming pdf_x and pdf_y are the feature and label pandas
+            # DataFrames.
+            >>> lgbm_data = lgb.Dataset(data=pdf_x, label=pdf_y, free_raw_data=False)
+            >>> lgbm_data
+            <lightgbm.basic.Dataset object at ....>
+
+            # Train the model using train() function.
+            >>> model = lgb.train(params={}, train_set=lgbm_data, num_boost_round=30, valid_sets=[lgbm_data])
+            [LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000043 seconds.
+            You can set `force_row_wise=true` to remove the overhead.
+            And if memory is not enough, you can set `force_col_wise=true`.
+            [LightGBM] [Info] Total Bins 532
+            [LightGBM] [Info] Number of data points in the train set: 400, number of used features: 4
+            [1]	valid_0's l2: 0.215811
+            [2]	valid_0's l2: 0.188138
+            [3]	valid_0's l2: 0.166146
+            ...
+            ...
+            [29]	valid_0's l2: 0.042255
+            [30]	valid_0's l2: 0.0416953
+
+            # Deploy the model to Vantage.
+            >>> lgb_model = td_lightgbm.deploy("lgb_model_ver_2", model)
+            >>> lgb_model
+            <lightgbm.basic.Booster object at ...>
 
         """
         return self._object_wrapper._deploy(model_name=model_name,
@@ -145,6 +200,7 @@ class _OpenSource:
             TeradataMlException if model with name "model_name" does not exist.
 
         EXAMPLE:
+            # sklearn example.
             >>> from teradataml import td_sklearn
             >>> # Load the model saved in Vantage. Note that the model is saved using
             >>> # `deploy()` of exposed interface object (like `td_sklearn`) or
@@ -152,6 +208,15 @@ class _OpenSource:
             >>> model = td_sklearn.load("linreg_model_ver_1")
             >>> model
             LinearRegression(normalize=True)
+
+            # lightgbm example.
+            >>> from teradataml import td_lightgbm
+            >>> # Load the model saved in Vantage. Note that the model is saved using
+            >>> # `deploy()` of exposed interface object (like `td_lightgbm`) or
+            >>> # `_OpenSourceObjectWrapper` Object.
+            >>> model = td_lightgbm.load("lgb_model_ver_1")
+            >>> model
+            LGBMClassifier()
         """
         return self._object_wrapper._load(model_name)
 
@@ -290,18 +355,73 @@ class Lightgbm(_OpenSource):
         >>> df_x = df.select(df.columns[:-1])
         >>> df_y = df.select(df.columns[-1])
 
+        >>> from teradataml import td_lightgbm
+
+        # Example 1: Train the model using train() function.
         # Create lightgbm Dataset object.
         >>> lgbm_data = td_lightgbm.Dataset(data=df_x, label=df_y, free_raw_data=False)
         >>> lgbm_data
-        <lightgbm.basic.Dataset object at 0x7f33f0656820>
+        <lightgbm.basic.Dataset object at ...>
+
+        # Train the model.
+        >>> model = td_lightgbm.train(params={}, train_set=lgbm_data, num_boost_round=30, valid_sets=[lgbm_data])
+        [LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000043 seconds.
+        You can set `force_row_wise=true` to remove the overhead.
+        And if memory is not enough, you can set `force_col_wise=true`.
+        [LightGBM] [Info] Total Bins 532
+        [LightGBM] [Info] Number of data points in the train set: 400, number of used features: 4
+        [1]	valid_0's l2: 0.215811
+        [2]	valid_0's l2: 0.188138
+        [3]	valid_0's l2: 0.166146
+        ...
+        ...
+        [29]	valid_0's l2: 0.042255
+        [30]	valid_0's l2: 0.0416953
+        >>> model
+        <lightgbm.basic.Booster object at ...>
+
+        # Example 2: Train the model using LGBMClassifier sklearn object.
+        # Create lightgbm sklearn object.
+        >>> lgbm_cl = td_lightgbm.LGBMClassifier()
+        >>> lgbm_cl
+        LGBMClassifier()
+
+        # Fit/train the model using fit() function.
+        >>> lgbm_cl.fit(df_x, df_y)
+        LGBMClassifier()
+
+        # Perform prediction.
+        >>> lgbm_cl.predict(df_x).head(3)
+               col1      col2      col3      col4  lgbmclassifier_predict_1
+        0  1.105026 -1.949894 -1.537164  0.073171                         1
+        1  1.878349  0.577289  1.795746  2.762539                         1
+        2 -1.130582 -0.020296 -0.710234 -1.440991                         0
+
+        # Access attributes.
+        >>> lgbm_cl.feature_importances_
+        array([ 0, 20, 10, 10])
     """
 
     def __init__(self):
         super().__init__()
         self._modules = _LIGHTGBM_MODULES
-        self._object_wrapper = _LightgbmDatasetWrapper
+        self._object_wrapper = _LightgbmBoosterWrapper
         self._function_wrapper = _LightgbmFunctionWrapper
 
+    def _assign_object_wrapper(self, module, class_name):
+        """
+        Assigns the appropriate object wrapper based on the module and class name.
+        """
+
+        if module == "lightgbm.basic" and class_name == "Booster":
+            self._object_wrapper = _LightgbmBoosterWrapper
+
+        if module == "lightgbm.basic" and class_name == "Dataset":
+            self._object_wrapper = _LightgbmDatasetWrapper
+
+        if module == "lightgbm.sklearn":
+            self._object_wrapper = _LightgbmSklearnWrapper
+
     def __getattr__(self, name):
 
         def __get_module(*c, **kwargs):
@@ -317,23 +437,27 @@ class Lightgbm(_OpenSource):
                 return self._function_wrapper(module_name=module, func_name=name)(**kwargs)
 
             kwargs.update(zip(class_instance.__init__.__code__.co_varnames[1:], c))
-            if module == "lightgbm.basic" and name == "Booster":
-                return _LightgbmBoosterWrapper(module_name=module, class_name=name, kwargs=kwargs)
-            
-            if module == "lightgbm.sklearn":
-                return _LighgbmSklearnWrapper(module_name=module, class_name=name, kwargs=kwargs)
 
-            return self._object_wrapper(module_name=module, class_name=name, kwargs=kwargs)
+            all_args = {"module_name": module, "class_name": name, "kwargs": kwargs}
+            self._assign_object_wrapper(module, name)
+
+            return self._object_wrapper(**all_args)
 
         return __get_module
 
     def deploy(self, model_name, model, replace_if_exists=False):
-        raise NotImplementedError("The deploy() function is not yet supported for td_lightgbm. \
-                                  Support will be added in future releases.")
+        # Docstring of parent class also contain examples of lightgbm.
+        module = model.__module__ if hasattr(model, "__module__") else None
+        class_name = model.__class__.__name__ if hasattr(model, "__class__") else None
 
-    def load(self, model_name):
-            raise NotImplementedError("The load() function is not yet supported for td_lightgbm. \
-                                    Support will be added in future releases.")
+        if module is None or class_name is None:
+            raise ValueError("The model object is not supported for deployment.")
+
+        self._assign_object_wrapper(module, class_name)
+
+        return self._object_wrapper._deploy(model_name=model_name,
+                                            model=model,
+                                            replace_if_exists=replace_if_exists)
 
 
 td_sklearn = Sklearn()

teradataml/opensource/{constants.py → _constants.py}

@@ -13,11 +13,11 @@
 # 
 # ################################################################## 
 
-from enum import Enum
-from teradataml import VARCHAR, BLOB
 from dataclasses import dataclass, field
-from typing import Any, Dict, Tuple, Optional
+from enum import Enum
+from typing import Any, Dict, Optional, Tuple
 
+from teradataml import BLOB, VARCHAR
 
 _SKL_MODULES = ["sklearn.calibration", "sklearn.cluster", "sklearn.compose", "sklearn.covariance",
                 "sklearn.decomposition", "sklearn.discriminant_analysis",
@@ -40,6 +40,8 @@ class OpenSourcePackage(Enum):
         return [item.value for item in cls]
 
 
+_packages_verified_in_vantage = {} # Used to ensure check for python and python packages done only once per package.
+
 @dataclass
 class OpensourceModels:
     """Dataclass for Opensource Models details."""
@@ -49,6 +51,8 @@ class OpensourceModels:
     pos_args: Tuple[Any] = tuple() # Positional arguments used for model creation.
     key_args: Dict[str, Any] = field(default_factory=dict) # Keyword arguments used for model creation.
     fit_partition_columns_non_default: Optional[str] = None  # Columns used for partitioning.
+    osml_module: Optional[str] = None # Module of corresponding wrapper class.
+    osml_class: Optional[str] = None # Corresponding wrapper class name.
 
 # Model table details used by opensource BYOM.
 _OSML_MODELS_TABLE_NAME = "opensourceml_models"

teradataml/opensource/_lightgbm.py

@@ -19,19 +19,20 @@ import json
 import os
 import pickle
 import warnings
-
 from collections import OrderedDict
 from importlib import import_module
 
-
+import numpy
 import pandas as pd
 from teradatasqlalchemy import BLOB, CLOB, FLOAT
 
-from teradataml import _TDML_DIRECTORY, UtilFuncs, execute_sql, TeradataMlException, Messages, MessageCodes, DataFrame
+from teradataml import (_TDML_DIRECTORY, MessageCodes, Messages,
+                        TeradataMlException, UtilFuncs, execute_sql)
+from teradataml.opensource._base import (_FunctionWrapper,
+                                         _OpenSourceObjectWrapper)
+from teradataml.opensource._constants import OpenSourcePackage
+from teradataml.opensource._sklearn import _SkLearnObjectWrapper
 from teradataml.opensource._wrapper_utils import _generate_new_name
-from teradataml.opensource.constants import OpenSourcePackage
-from teradataml.opensource.sklearn._sklearn_wrapper import (
-    _FunctionWrapper, _OpenSourceObjectWrapper, _SkLearnObjectWrapper)
 
 
 class _LightgbmDatasetWrapper(_OpenSourceObjectWrapper):
@@ -43,6 +44,7 @@ class _LightgbmDatasetWrapper(_OpenSourceObjectWrapper):
 
         file_type = "file_fn_lightgbm"
         self._template_file = "dataset.template"
+        self._pkgs = ["lightgbm", "scikit-learn", "numpy", "scipy"]
         super().__init__(model=model, module_name=module_name, class_name=class_name, kwargs=kwargs)
 
         self._scripts_path = os.path.join(_TDML_DIRECTORY, "data", "scripts", "lightgbm")
@@ -221,12 +223,16 @@ class _LightgbmDatasetWrapper(_OpenSourceObjectWrapper):
 
         return self
 
+    def deploy(self, model_name, replace_if_exists=False):
+        raise ValueError("lightgbm Dataset object is not the model object that can be trained. "
+                         "Hence, not deployable.")
 
 class _LightgbmFunctionWrapper(_FunctionWrapper):
     OPENSOURCE_PACKAGE_NAME = OpenSourcePackage.LIGHTGBM
     def __init__(self, module_name=None, func_name=None):
         file_type = "file_fn_lightgbm"
         template_file = "lightgbm_function.template"
+        self._pkgs = ["lightgbm", "scikit-learn", "numpy", "scipy"]
         self._script_file_name = _generate_new_name(type=file_type, extension="py")
         super().__init__(module_name, func_name, file_type=file_type, template_file=template_file)
         self._scripts_path = os.path.join(_TDML_DIRECTORY, "data", "scripts", "lightgbm")
@@ -462,10 +468,9 @@ class _LightgbmBoosterWrapper(_SkLearnObjectWrapper):
     OPENSOURCE_PACKAGE_NAME = OpenSourcePackage.LIGHTGBM
     def __init__(self, model=None, module_name=None, class_name=None, kwargs=None, model_column_name=None):
         file_type = "file_fn_lightgbm_booster"
-
         self._model_column_name = model_column_name
-
         self.record_evaluation_result = None
+        self._pkgs = ["lightgbm", "scikit-learn", "numpy", "scipy"]
 
         if model is not None and isinstance(model, dict) and self._model_column_name in model.keys():
             self.record_evaluation_result = model["record_evaluation_result"]
@@ -500,10 +505,6 @@ class _LightgbmBoosterWrapper(_SkLearnObjectWrapper):
             class_obj = getattr(import_module(self.module_name), self.class_name)
             self.modelObj = class_obj(**self.kwargs)
 
-    def deploy(self, model_name, replace_if_exists=False):
-        raise NotImplementedError("The deploy() function is not yet supported for lightgbm OpensourceML objects. \
-                                  Support will be added in future releases.")
-
     @property
     def model_info(self):
         """
@@ -662,42 +663,6 @@ class _LightgbmBoosterWrapper(_SkLearnObjectWrapper):
         """
         return self.modelObj
 
-    def _convert_arguments_to_modelObj(self, args, idx_multi_model=None):
-        """
-        Internal function to convert all OpensourceML related objects in arguments to
-        underlying model objects.
-        """
-        if isinstance(args, dict):
-            new_args = args.copy() # To avoid updating 
-            for k, v in new_args.items():
-                if isinstance(v, type(self)) or isinstance(v, _LightgbmDatasetWrapper):
-                    if idx_multi_model is None:
-                        # single model. This argument (idx_multi_model) is set only when modelObj
-                        # is multi model.
-                        new_args[k] = v.modelObj
-                    else:
-                        # multi-model. Get appropriate model from modelObj.
-                        new_args[k] = v.modelObj.iloc[idx_multi_model][self._model_column_name]
-                else:
-                    new_args[k] = v
-            return new_args
-
-        # If args is tuple, convert all elements to underlying model object.
-        elif isinstance(args, tuple):
-            new_args = tuple()
-            for arg in args:
-                if isinstance(arg, type(self)) or isinstance(arg, _LightgbmDatasetWrapper):
-                    if idx_multi_model is None:
-                        # single model. This argument is set only when modelObj is single model.
-                        new_args += (arg.modelObj,)
-                    else:
-                        # multi-model. Get appropriate model from modelObj.
-                        new_args += (arg.modelObj.iloc[idx_multi_model][self._model_column_name],)
-                else:
-                    new_args += (arg,)
-            return new_args
-        return args
-
     def __getattr__(self, name):
         def __run_transform(*c, **kwargs):
             # Lightgbm predict method takes other keyword arguments along with data related arguments.
@@ -729,6 +694,43 @@ class _LightgbmBoosterWrapper(_SkLearnObjectWrapper):
             return __run_transform
         return super().__getattr__(name)
 
+    def _execute_function_locally(self, ten_row_data, feature_columns, label_columns, openml_obj,
+                                  func_name, **kwargs):
+        """
+        Function which overrides the existing _execute_function_locally method to handle ValueError
+        as argument names are different in lightgbm compared to scikit-learn.
+        """
+        X = numpy.array(ten_row_data)
+
+        if label_columns:
+            n_f = len(feature_columns)
+            n_c = len(label_columns)
+            y = X[:,n_f : n_f + n_c]
+            X = X[:,:n_f]
+            # predict() now takes 'y' ("label" lightgbm argument) also for it to return the labels
+            # from script. Skipping 'y' in local run if passed.
+            # Generally, 'y' is passed to return y along with actual output.
+            # Since actual lightgbm predict() does not have "label" argument and have other arguments like
+            # "start_iteration" etc, local run in try block is resulting into ValueError as
+            # "ValueError: The truth value of an array with more than one element is ambiguous. 
+            # Use a.any() or a.all()" for "start_iteration" argument because the value for "y" is
+            # taken for "start_iteration" positional argument. Hence, skipping y in local run.
+            try:
+                trans_opt = getattr(openml_obj, func_name)(X, y, **kwargs)
+            except TypeError as _:
+                # Function which does not accept 'y' like predict_proba() raises error like
+                # "predict_proba() takes 2 positional arguments but 3 were given".
+                trans_opt = getattr(openml_obj, func_name)(X, **kwargs)
+            except ValueError as _:
+                trans_opt = getattr(openml_obj, func_name)(X, **kwargs)
+        else:
+            trans_opt = getattr(openml_obj, func_name)(X, **kwargs)
+
+        if isinstance(trans_opt, numpy.ndarray) and trans_opt.shape == (X.shape[0],):
+            trans_opt = trans_opt.reshape(X.shape[0], 1)
+
+        return trans_opt
+
     def _transform(self, **kwargs):
         # Overwriting existing _transform method to handle data related arguments and other
         # keyword arguments.
@@ -773,16 +775,13 @@ class _LightgbmBoosterWrapper(_SkLearnObjectWrapper):
         return self.modelObj.__repr__()
 
 
-class _LighgbmSklearnWrapper(_SkLearnObjectWrapper):
+class _LightgbmSklearnWrapper(_SkLearnObjectWrapper):
     OPENSOURCE_PACKAGE_NAME = OpenSourcePackage.LIGHTGBM
     def __init__(self, model=None, module_name=None, class_name=None, kwargs=None):
+        self._pkgs = ["lightgbm", "scikit-learn", "numpy", "scipy"]
         super().__init__(model=model, module_name=module_name, class_name=class_name, kwargs=kwargs)
         self._scripts_path = os.path.join(_TDML_DIRECTORY, "data", "scripts", "lightgbm")
 
-    def deploy(self, model_name, replace_if_exists=False):
-        raise NotImplementedError("The deploy() function is not yet supported for lightgbm OpensourceML objects. \
-                                  Support will be added in future releases.")
-
     def set_params(self, **params):
         """
         Please check the description in Docs/OpensourceML/sklearn.py.