teradataml 20.0.0.8__py3-none-any.whl

teradataml/data/score_category.csv

@@ -0,0 +1,101 @@
+"id","score","stratum"
+61,6,"fair"
+17,9,"fair"
+40,47,"fair"
+19,25,"fair"
+38,22,"fair"
+34,34,"fair"
+80,95,"excellent"
+59,89,"very good"
+78,91,"excellent"
+13,87,"very good"
+57,75,"fair"
+99,44,"fair"
+76,100,"excellent"
+32,3,"fair"
+55,74,"fair"
+36,15,"fair"
+93,67,"fair"
+11,27,"fair"
+95,84,"very good"
+97,20,"fair"
+26,54,"fair"
+51,55,"fair"
+72,79,"fair"
+15,22,"fair"
+5,90,"excellent"
+9,65,"fair"
+7,40,"fair"
+74,7,"fair"
+24,11,"fair"
+49,8,"fair"
+22,25,"fair"
+53,87,"very good"
+3,95,"excellent"
+28,90,"excellent"
+62,74,"fair"
+30,6,"fair"
+43,49,"fair"
+89,31,"fair"
+39,50,"fair"
+91,82,"very good"
+1,5,"fair"
+68,30,"fair"
+37,14,"fair"
+70,95,"excellent"
+41,70,"fair"
+87,18,"fair"
+77,51,"fair"
+47,79,"fair"
+20,85,"very good"
+66,44,"fair"
+35,94,"excellent"
+45,21,"fair"
+60,97,"excellent"
+64,41,"fair"
+54,43,"fair"
+85,79,"fair"
+18,67,"fair"
+83,81,"very good"
+94,76,"fair"
+100,18,"fair"
+79,71,"fair"
+81,13,"fair"
+12,93,"excellent"
+14,52,"fair"
+58,92,"excellent"
+16,19,"fair"
+52,51,"fair"
+31,79,"fair"
+98,15,"fair"
+33,85,"very good"
+92,32,"fair"
+29,75,"fair"
+56,22,"fair"
+10,27,"fair"
+50,12,"fair"
+69,18,"fair"
+96,71,"fair"
+71,59,"fair"
+27,44,"fair"
+6,55,"fair"
+75,66,"fair"
+48,11,"fair"
+67,29,"fair"
+46,19,"fair"
+73,43,"fair"
+88,98,"excellent"
+4,95,"excellent"
+86,6,"fair"
+90,100,"excellent"
+65,53,"fair"
+44,70,"fair"
+23,14,"fair"
+8,57,"fair"
+63,52,"fair"
+42,39,"fair"
+84,1,"fair"
+25,17,"fair"
+82,56,"fair"
+21,5,"fair"
+2,83,"very good"

teradataml/data/score_summary.csv

@@ -0,0 +1,4 @@
+"stratum","stratum_count"
+"very good","9"
+"excellent","14"
+"fair","77"

teradataml/data/script_example.json

@@ -0,0 +1,10 @@
+{
+    "barrier" : {
+        "Id" : "integer",
+        "Name" : "varchar(25)"
+    },
+    "barrier_new" : {
+        "Id" : "integer",
+        "Name" : "varchar(25)"
+    }
+}

teradataml/data/scripts/deploy_script.py

@@ -0,0 +1,84 @@
+import sys
+import numpy as np
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.svm import SVC
+import base64, pickle
+
+DELIMITER = "\t"
+
+def get_value(value):
+    ret_val = value
+    try:
+        ret_val = float(value.replace(' ', ''))
+    except Exception as ex:
+        # If the value can't be converted to float, then it is string.
+        pass
+    return ret_val
+
+
+def get_values_list(values, ignore_none=True):
+    ret_vals = []
+    for val in values:
+        if val == "" and ignore_none:
+            # Empty cell value in the database table.
+            continue
+        ret_vals.append(get_value(val))
+
+    return ret_vals
+
+if len(sys.argv) != 2:
+    sys.exit("Script command format: python deploy_script.py <enterprise/lake>")
+
+vantage_type = sys.argv[1]
+
+data_partition_column_values = []
+data_partition_column_indices = [5, 6]
+
+features = []
+labels = []
+
+while 1:
+    try:
+        line = input()
+        if line == '':  # Exit if user provides blank line
+            break
+        else:
+            values = line.split(DELIMITER)
+            
+            features.append(get_values_list(values[:4]))
+            labels.append(get_values_list([values[4]]))
+            if not data_partition_column_values:
+                # Partition column values is same for all rows. Hence, only read once.
+                for i, val in enumerate(data_partition_column_indices):
+                    data_partition_column_values.append(int(values[val]))
+
+    except EOFError:  # Exit if reached EOF or CTRL-D
+        break
+
+if not len(features):
+    sys.exit(0)
+
+X = np.array(features)
+y = np.array(labels).ravel()
+
+clf = make_pipeline(StandardScaler(), SVC(gamma='auto'))
+clf.fit(X, y)
+
+model_str = pickle.dumps(clf)
+
+# Prepare the corresponding model file name and extract model.
+partition_join = "_".join([str(x) for x in data_partition_column_values])
+# Replace '-' with '_' as '-' because partition_columns can be negative.
+partition_join = partition_join.replace("-", "_")
+
+if vantage_type == "lake":
+    model = f"/tmp/sklearn_model_{partition_join}.pickle"
+    with open(model, "wb") as fp:
+        fp.write(model_str)
+elif vantage_type == "enterprise":
+    model = base64.b64encode(model_str)
+else:
+    sys.exit("Invalid vantage type. Use either 'lake' or 'enterprise'.")
+
+print(*(data_partition_column_values + [model]), sep=DELIMITER)

teradataml/data/scripts/lightgbm/dataset.template

@@ -0,0 +1,175 @@
+import pandas as pd
+import pickle
+import json
+import numpy as np
+import ast
+import sys
+from collections import OrderedDict
+import base64
+from importlib import import_module
+import sys
+import os
+from contextlib import contextmanager
+
+DELIMITER = "\t"
+
+@contextmanager
+def suppress_stderr():
+    """
+    Function to suppress the warnings(lake systems treats warnings as errors).
+    """
+    with open(os.devnull, "w") as devnull:
+        old_stderr = sys.stderr
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stderr = old_stderr
+
+## On Lake system warnings raised by script are treated as a errors.
+## Hence, to suppress it putting the under suppress_stderr().
+with suppress_stderr():
+    def convert_to_type(val, typee):
+        if typee == 'int':
+            return int(val) if val != "" else np.nan
+        if typee == 'float':
+            if isinstance(val, str):
+                val = val.replace(' ', '')
+            return float(val) if val != "" else np.nan
+        if typee == 'bool':
+            return eval(val) if val != "" else None
+        return str(val) if val != "" else None
+
+    def splitter(strr, delim=",", convert_to="str"):
+        """
+        Split the string based on delimiter and convert to the type specified.
+        """
+        if strr == "None":
+            return []
+        return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+
+    is_lake_system = eval(sys.argv[2])
+    model_file_prefix = sys.argv[1]
+    if not is_lake_system:
+        db = sys.argv[0].split("/")[1]
+
+    ### Start of data related arguments processing
+    data_partition_column_values = []
+    data_present = False
+    partition_join = ""
+    model = None
+
+    # Data related arguments information of indices and types.
+    data_args_indices_types = OrderedDict()
+
+    func_name = <func_name>
+    module_name = <module_name>
+    class_name = <class_name>
+    all_col_names = <all_col_names>
+    all_col_types = <types_of_data_cols>
+    data_partition_column_indices = <partition_cols_indices>
+    data_partition_column_types = [all_col_types[idx] for idx in data_partition_column_indices]
+
+    # Data related arguments values - prepare dictionary and populate data later.
+    data_args_values = {}
+
+    data_args_info_str = <data_args_info_str>
+
+    for data_arg in data_args_info_str.split("--"):
+        _arg_name, _indices, _types = data_arg.split("-")
+        _indices = splitter(_indices, convert_to="int")
+        types = [type_ for idx, type_ in enumerate(all_col_types) if idx in _indices]
+
+        data_args_indices_types[_arg_name] = {"indices": _indices, "types": types}
+        data_args_values[_arg_name] = [] # Keeping empty for each data arg name and populate data later.
+
+    ### End of data related arguments processing
+
+
+    ### Start of other arguments processing
+    params = json.loads('<params>')
+    ### End of other arguments processing
+
+
+    # Read data - columns information is passed as command line argument and stored in
+    # data_args_indices_types dictionary.
+    while 1:
+        try:
+            line = input()
+            if line == '':  # Exit if user provides blank line
+                break
+            else:
+                data_present = True
+                values = line.split(DELIMITER)
+                if not data_partition_column_values:
+                    # Partition column values is same for all rows. Hence, only read once.
+                    for i, val in enumerate(data_partition_column_indices):
+                        data_partition_column_values.append(
+                            convert_to_type(values[val], typee=data_partition_column_types[i])
+                            )
+
+                    # Prepare the corresponding model file name and extract model.
+                    partition_join = "_".join([str(x) for x in data_partition_column_values])
+                    # Replace '-' with '_' as '-' because partition_columns can be negative.
+                    partition_join = partition_join.replace("-", "_")
+
+                    model_file_path = f"{model_file_prefix}_{partition_join}"\
+                        if is_lake_system else \
+                        f"./{db}/{model_file_prefix}_{partition_join}"
+
+                # Prepare data dictionary containing only arguments related to data.
+                for arg_name in data_args_values:
+                    data_indices = data_args_indices_types[arg_name]["indices"]
+                    types = data_args_indices_types[arg_name]["types"]
+                    cur_row = []
+                    for idx, data_idx in enumerate(data_indices):
+                        cur_row.append(convert_to_type(values[data_idx], types[idx]))
+                    data_args_values[arg_name].append(cur_row)
+        except EOFError:  # Exit if reached EOF or CTRL-D
+            break
+
+    if not data_present:
+        sys.exit(0)
+
+    for key, value in data_args_values.items():
+        col_names = [all_col_names[idx] for idx in data_args_indices_types[key]["indices"]]
+        data_args_values[key] = pd.DataFrame(value, columns=col_names)
+
+    # If reference argument (is a Dataset object) present in params, then it contains
+    # the prefix of the file path which contains the reference Dataset object.
+    if "reference" in params.keys() and params["reference"] is not None:
+        reference_dataset_file_prefix = params["reference"]
+        reference_arg_file_path = f"{reference_dataset_file_prefix}_{partition_join}"\
+                        if is_lake_system else \
+                        f"./{db}/{reference_dataset_file_prefix}_{partition_join}"
+        with open(reference_arg_file_path, "rb") as f:
+            params["reference"] = pickle.load(f)
+
+    if not func_name:
+        # Create DataSet object if no function of Dataset class is called.
+        lib = import_module(module_name)
+        class_instance = getattr(lib, class_name)
+        obj = class_instance(**{**data_args_values, **params})
+    else:
+        # If function of Dataset object is called, then call the function on model object.
+        with open(model_file_path, "rb") as fp:
+            model = pickle.loads(fp.read())
+
+        if not model:
+            sys.exit("Model file is not installed in Vantage.")
+
+        obj = getattr(model, func_name)(**{**data_args_values, **params})
+
+    model_str = pickle.dumps(obj)
+
+    if is_lake_system:
+        model_file_path = f"/tmp/{model_file_prefix}_{partition_join}.pickle"
+
+    # Save DataSet object to binary file
+    with open(model_file_path, "wb") as f:
+        f.write(model_str)
+
+    model_data = model_file_path if is_lake_system else base64.b64encode(model_str)
+
+    print(*(data_partition_column_values + [model_data]), sep=DELIMITER)

teradataml/data/scripts/lightgbm/lightgbm_class_functions.template

@@ -0,0 +1,264 @@
+import base64
+import io
+import math
+import os
+import pickle
+import sys
+import os
+from contextlib import contextmanager
+import numpy as np
+
+DELIMITER = '\t'
+
+@contextmanager
+def suppress_stderr():
+    """
+    Function to suppress the warnings(lake systems treats warnings as errors).
+    """
+    with open(os.devnull, "w") as devnull:
+        old_stderr = sys.stderr
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stderr = old_stderr
+
+## On Lake system warnings raised by script are treated as a errors.
+## Hence, to suppress it putting the under suppress_stderr().
+with suppress_stderr():
+    def get_values_list(values, types):
+        ret_vals = []
+        for i, val in enumerate(values):
+            ret_vals.append(convert_to_type(val, types[i]))
+        return ret_vals
+
+    def convert_to_type(val, typee):
+        if typee == 'int':
+            return int(val) if val != "" else np.nan
+        if typee == 'float':
+            if isinstance(val, str):
+                val = val.replace(' ', '')
+            return float(val) if val != "" else np.nan
+        if typee == 'bool':
+            return eval(val) if val != "" else None
+        return str(val) if val != "" else None
+
+    def splitter(strr, delim=",", convert_to="str"):
+        """
+        Split the string based on delimiter and convert to the type specified.
+        """
+        if strr == "None":
+            return []
+        return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+    def should_convert(t_val, py_type):
+        """
+        Function to check type of value and whether value is nan and infinity.
+        """
+        return not isinstance(t_val, eval(py_type)) and not math.isinf(t_val) and not math.isnan(t_val)
+
+    def convert_value(t_val, py_type):
+        """
+        Function to convert value to specified python type.
+        """
+        return convert_to_type(t_val, py_type) if should_convert(t_val, py_type) else t_val
+
+    # Process output returned by sklearn function.
+    def get_output_data(trans_values, func_name, n_c_labels, n_out_columns):
+        # Converting    sparse matrix to dense array as sparse matrices are NOT
+        # supported in Vantage.
+        # module_name = model_obj.__module__.split("._")[0]
+
+        # Converting the translated values into corresponding the return column's
+        # python type.
+        if (return_columns_python_types is None or not isinstance(trans_values, np.ndarray)):
+            trans_values_list = trans_values
+        else:
+            # Conversion.
+            trans_values_list = []
+            for trans_value in trans_values.tolist():
+                if not isinstance(trans_value, list):
+                    trans_value = [trans_value]
+
+                converted_list = []
+                if len(return_columns_python_types) == len(trans_value):
+                    for t_val, py_type in zip(trans_value, return_columns_python_types):
+                        converted_list.append(convert_value(t_val, py_type))
+                ## transform() is having only 1 python return type, But it actually returns more than 1 column.
+                else:
+                    for t_val in trans_value:
+                        converted_list.append(convert_value(t_val, return_columns_python_types[0]))
+
+                trans_values_list.append(converted_list)
+
+        if type(trans_values_list).__name__ in ["csr_matrix", "csc_matrix"]:
+            trans_values_list = trans_values_list.toarray()
+
+        if isinstance(trans_values_list[0], np.ndarray) \
+                or isinstance(trans_values_list[0], list) \
+                or isinstance(trans_values_list[0], tuple):
+            # Here, the value returned by sklearn function is list type.
+            opt_list = list(trans_values_list[0])
+
+            if len(opt_list) < n_out_columns:
+                # If the output list is less than the required number of columns, append
+                # empty strings to the list.
+                opt_list += [""] * (n_out_columns - len(opt_list))
+
+            return opt_list
+
+        # Only one element is returned by the function.
+        return [trans_values_list[0]]
+
+    # Arguments to the Script
+    if len(sys.argv) != 10:
+        # 10 arguments command line arguments should be passed to this file.
+        # 1: file to be run
+        # 2. function name (Eg. predict, fit etc)
+        # 3. No of feature columns.
+        # 4. No of class labels.
+        # 5. Comma separated indices of partition columns.
+        # 6. Comma separated types of all the data columns.
+        # 7. Model file prefix to generated model file using partition columns.
+        # 8. Number of columns to be returned by the sklearn's transform function.
+        # 9. Flag to check the system type. True, means Lake, Enterprise otherwise.
+        # 10. Python types of returned/transfromed columns.
+        sys.exit("10 arguments should be passed to this file - file to be run, function name, "\
+                     "no of feature columns, no of class labels, comma separated indices of partition "\
+                     "columns, comma separated types of all columns, model file prefix to generate model "\
+                     "file using partition columns, number of columns to be returnd by sklearn's "\
+                     "transform function, flag to check lake or enterprise and Python types of "\
+                     "returned/transfromed columns.")
+
+    is_lake_system = eval(sys.argv[8])
+    if not is_lake_system:
+        db = sys.argv[0].split("/")[1]
+    func_name = sys.argv[1]
+    n_f_cols = int(sys.argv[2])
+    n_c_labels = int(sys.argv[3])
+    data_column_types = splitter(sys.argv[5], delim="--")
+    data_partition_column_indices = splitter(sys.argv[4], convert_to="int") # indices are integers.
+    model_file_prefix = sys.argv[6]
+    # sys.argv[9] will contain a string of python datatypes with '--'
+    # separator OR a single datatype OR None in string format.
+    ret_col_argv = sys.argv[9]
+    if ret_col_argv == "None":
+        return_columns_python_types = eval(ret_col_argv)
+    else:
+        return_columns_python_types = splitter(ret_col_argv, delim="--")
+
+    no_of_output_columns = int(sys.argv[7])
+
+    data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
+    model = None
+    data_partition_column_values = []
+
+    all_x_rows = []
+    all_y_rows = []
+
+    # Data Format:
+    # feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ...,
+    # data_partition_columnn.
+    # label is optional (it is present when label_exists is not "None")
+
+    model_name = ""
+    while 1:
+        try:
+            line = input()
+            if line == '':  # Exit if user provides blank line
+                break
+            else:
+                values = line.split(DELIMITER)
+                values = get_values_list(values, data_column_types)
+                if not data_partition_column_values:
+                    # Partition column values is same for all rows. Hence, only read once.
+                    for i, val in enumerate(data_partition_column_indices):
+                        data_partition_column_values.append(
+                            convert_to_type(values[val], typee=data_partition_column_types[i])
+                            )
+
+                    # Prepare the corresponding model file name and extract model.
+                    partition_join = "_".join([str(x) for x in data_partition_column_values])
+                    # Replace '-' with '_' as '-' because partition_columns can be negative.
+                    partition_join = partition_join.replace("-", "_")
+
+                    model_file_path = f"{model_file_prefix}_{partition_join}" \
+                        if is_lake_system else \
+                        f"./{db}/{model_file_prefix}_{partition_join}"
+
+                    with open(model_file_path, "rb") as fp:
+                        model = pickle.loads(fp.read())
+
+                    if not model:
+                        sys.exit("Model file is not installed in Vantage.")
+
+                f_ = values[:n_f_cols]
+                f__ = np.array([f_])
+
+                if n_c_labels > 0:
+                    l_ = values[n_f_cols:n_f_cols+n_c_labels]
+                    l__ = np.array([l_])
+
+                if func_name == "refit":
+                    # refit() needs all data at once. Hence, read all data at once and call refit().
+                    all_x_rows.append(f_)
+                    all_y_rows.append(l_)
+                    continue
+
+                # Because `predict` function does not accept 'y' as input, we need to handle it separately.
+                if n_c_labels > 0 and func_name not in ["predict"]:
+                    # Labels are present in last column.
+                    trans_values = getattr(model, func_name)(f__, l__, **params)
+                else:
+                    # If class labels do not exist in data, don't read labels, read just features.
+                    trans_values = getattr(model, func_name)(f__, **params)
+
+                result_list = f_
+                if n_c_labels > 0 and func_name in ["predict", "decision_function"]:
+                    result_list += l_
+                result_list += get_output_data(trans_values=trans_values, func_name=func_name,
+                                               n_c_labels=n_c_labels, n_out_columns=no_of_output_columns)
+
+                for i, val in enumerate(result_list):
+                    if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val)))):
+                        result_list[i] = ""
+                    elif val == False:
+                        result_list[i] = 0
+                    elif val == True:
+                        result_list[i] = 1
+
+                print(*(data_partition_column_values + result_list), sep=DELIMITER)
+
+        except EOFError:  # Exit if reached EOF or CTRL-D
+            break
+
+
+    if func_name == "refit":
+        result = ""
+        stdout = None
+        try:
+            stdout = sys.stdout
+            new_stdout = io.StringIO()
+            sys.stdout = new_stdout
+            trained_model = getattr(model, func_name)(all_x_rows, all_y_rows, **params)
+            result = new_stdout.getvalue()
+        except Exception:
+            raise
+        finally:
+            sys.stdout = stdout
+
+        model_str = pickle.dumps(trained_model)
+
+
+        if is_lake_system:
+            model_file_path = f"/tmp/{model_file_prefix}_{partition_join}.pickle"
+
+        # Write to trained model file in Vantage.
+        with open(model_file_path, "wb") as fp:
+            fp.write(model_str)
+
+        model_data = model_file_path if is_lake_system else base64.b64encode(model_str)
+        console_output = base64.b64encode(result.encode())
+
+        print(*(data_partition_column_values + [model_data, console_output]), sep="..")