teradataml 20.0.0.8__py3-none-any.whl

teradataml/data/scripts/sklearn/sklearn_fit_predict.py

@@ -0,0 +1,148 @@
+import sys
+import numpy as np
+import pickle
+import math
+import os
+from contextlib import contextmanager
+
+DELIMITER = '\t'
+
+@contextmanager
+def suppress_stderr():
+    """
+    Function to suppress the warnings(lake systems treats warnings as errors).
+    """
+    with open(os.devnull, "w") as devnull:
+        old_stderr = sys.stderr
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stderr = old_stderr
+
+## On Lake system warnings raised by script are treated as a errors. 
+## Hence, to suppress it putting the under suppress_stderr().
+with suppress_stderr():
+    def get_values_list(values, types):
+        ret_vals = []
+        for i, val in enumerate(values):
+            ret_vals.append(convert_to_type(val, types[i]))
+        return ret_vals
+
+    def convert_to_type(val, typee):
+        if typee == 'int':
+            return int(val) if val != "" else np.nan
+        if typee == 'float':
+            if isinstance(val, str):
+                val = val.replace(' ', '')
+            return float(val) if val != "" else np.nan
+        if typee == 'bool':
+            return eval(val) if val != "" else None
+        return str(val) if val != "" else None
+
+    def splitter(strr, delim=",", convert_to="str"):
+        """
+        Split the string based on delimiter and convert to the type specified.
+        """
+        if strr == "None":
+            return []
+        return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+    # Arguments to the Script
+    if len(sys.argv) != 7:
+        # 6 arguments command line arguments should be passed to this file.
+        # 1: file to be run
+        # 2. No of feature columns.
+        # 3. No of class labels.
+        # 4. Comma separated indices of partition columns.
+        # 5. Comma separated types of all the data columns.
+        # 6. Model file prefix to generated model file using partition columns.
+        # 7. Flag to check the system type. True, means Lake, Enterprise otherwise.
+        sys.exit("7 arguments should be passed to this file - file to be run, "\
+                 "no of feature columns, no of class labels, comma separated indices of partition "
+                 "columns, comma separated types of all columns, model file prefix to generate model "
+                 "file using partition columns and flag to check lake or enterprise.")
+
+    is_lake_system = eval(sys.argv[6])
+    if not is_lake_system:
+        db = sys.argv[0].split("/")[1]
+    n_f_cols = int(sys.argv[1])
+    n_c_labels = int(sys.argv[2])
+    model_file_prefix = sys.argv[5]
+    data_column_types = splitter(sys.argv[4], delim="--")
+    data_partition_column_indices = splitter(sys.argv[3], convert_to="int") # indices are integers.
+
+    data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
+    model = None
+
+    # Data Format (n_features, k_labels, one data_partition_columns):
+    # feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ...,
+    # data_partition_columnn.
+    # There can be no labels also.
+
+    # Read data from table through STO and build features and labels.
+    features = []
+    labels = []
+    data_partition_column_values = []
+
+
+    while 1:
+        try:
+            line = input()
+            if line == '':  # Exit if user provides blank line
+                break
+            else:
+                values = line.split(DELIMITER)
+                values = get_values_list(values, data_column_types)
+                features.append(values[:n_f_cols])
+                if n_c_labels > 0:
+                    labels.append(values[n_f_cols:(n_f_cols+n_c_labels)])
+                if not data_partition_column_values:
+                    # Partition column values is same for all rows. Hence, only read once.
+                    for i, val in enumerate(data_partition_column_indices):
+                        data_partition_column_values.append(
+                            convert_to_type(values[val], typee=data_partition_column_types[i])
+                            )
+
+                    # Prepare the corresponding model file name and extract model.
+                    partition_join = "_".join([str(x) for x in data_partition_column_values])
+                    # Replace '-' with '_' as '-' because partition_columns can be negative.
+                    partition_join = partition_join.replace("-", "_")
+
+                    model_file_path = f"{model_file_prefix}_{partition_join}" \
+                        if is_lake_system else \
+                        f"./{db}/{model_file_prefix}_{partition_join}"
+
+                    with open(model_file_path, "rb") as fp:
+                        model = pickle.loads(fp.read())
+
+                    if model is None:
+                        sys.exit("Model file is not installed in Vantage.")
+
+        except EOFError:  # Exit if reached EOF or CTRL-D
+            break
+
+    if not len(features):
+        sys.exit(0)
+
+    # write code to call fit_predict with features and labels when n_c_labels > 0
+    model_name = model.__class__.__name__
+    if n_c_labels > 0:
+        if model_name in ["SelectFromModel"]:
+            labels = np.array(labels).ravel()
+        predictions = model.fit_predict(features, labels)
+    else:
+        predictions = model.fit_predict(features)
+
+    # Export results to to the Databse through standard output
+    for i in range(len(predictions)):
+        if n_c_labels > 0:
+            # Add labels into output, if user passes it.
+            result_list = features[i] + labels[i] + [predictions[i]]
+        else:
+            result_list = features[i] + [predictions[i]]
+        print(*(data_partition_column_values +
+                ['' if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val))))
+                 else val for val in result_list]),
+                 sep= DELIMITER)

teradataml/data/scripts/sklearn/sklearn_function.template

@@ -0,0 +1,144 @@
+import sys, json
+import pickle, base64, importlib, numpy as np
+from collections import OrderedDict
+import os
+from contextlib import contextmanager
+
+func_name = "<func_name>"
+module_name = "<module_name>"
+params = json.loads('<params>')
+
+DELIMITER = '\t'
+
+@contextmanager
+def suppress_stderr():
+    """
+    Function to suppress the warnings(lake systems treats warnings as errors).
+    """
+    with open(os.devnull, "w") as devnull:
+        old_stderr = sys.stderr
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stderr = old_stderr
+
+## On Lake system warnings raised by script are treated as a errors.
+## Hence, to suppress it putting the under suppress_stderr().
+with suppress_stderr():
+    def convert_to_type(val, typee):
+        if typee == 'int':
+            return int(val) if val != "" else np.nan
+        if typee == 'float':
+            if isinstance(val, str):
+                val = val.replace(' ', '')
+            return float(val) if val != "" else np.nan
+        if typee == 'bool':
+            return eval(val) if val != "" else None
+        return str(val) if val != "" else None
+
+    def splitter(strr, delim=",", convert_to="str"):
+        """
+        Split the string based on delimiter and convert to the type specified.
+        """
+        if strr == "None":
+            return []
+        return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+    # Arguments to the Script.
+    if len(sys.argv) != 3:
+        # 3 command line arguments should be passed to this file.
+        # 1: file to be run
+        # 2. Model file prefix for lake system, None otherwise.
+        # 3. Flag to check the system type. True, means Lake, Enterprise otherwise.
+        sys.exit("3 arguments command line arguments should be passed: file to be run,"
+                 " model file prefix used only for lake system and flag to check lake or enterprise.")
+
+    is_lake_system = eval(sys.argv[2])
+    if not is_lake_system:
+        db = sys.argv[0].split("/")[1]
+    else:
+        model_file_prefix = sys.argv[1]
+
+    data_partition_column_indices = <partition_cols_indices>
+    data_column_types = <types_of_data_cols>
+
+    data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
+    # Data related arguments information of indices and types.
+    data_args_indices_types = OrderedDict()
+
+    # Data related arguments values - prepare dictionary and populate data later.
+    data_args_values = {}
+
+    data_args_info_str = <data_args_info_str>
+    for data_arg in data_args_info_str.split("--"):
+        arg_name, indices, types = data_arg.split("-")
+        indices = splitter(indices, convert_to="int")
+        types = splitter(types)
+
+        data_args_indices_types[arg_name] = {"indices": indices, "types": types}
+        data_args_values[arg_name] = [] # Keeping empty for each data arg name and populate data later.
+
+    data_partition_column_values = []
+    data_present = False
+
+    # Read data - columns information is passed as command line argument and stored in
+    # data_args_indices_types dictionary.
+    while 1:
+        try:
+            line = input()
+            if line == '':  # Exit if user provides blank line
+                break
+            else:
+                data_present = True
+                values = line.split(DELIMITER)
+                if not data_partition_column_values:
+                    # Partition column values is same for all rows. Hence, only read once.
+                    for i, val in enumerate(data_partition_column_indices):
+                        data_partition_column_values.append(
+                            convert_to_type(values[val], typee=data_partition_column_types[i])
+                            )
+
+                    # Prepare the corresponding model file name and extract model.
+                    partition_join = "_".join([str(x) for x in data_partition_column_values])
+                    # Replace '-' with '_' as '-' because partition_columns can be negative.
+                    partition_join = partition_join.replace("-", "_")
+
+                # Prepare data dictionary containing only arguments related to data.
+                for arg_name in data_args_values:
+                    data_indices = data_args_indices_types[arg_name]["indices"]
+                    types = data_args_indices_types[arg_name]["types"]
+                    cur_row = []
+                    for idx, data_idx in enumerate(data_indices):
+                        cur_row.append(convert_to_type(values[data_idx], types[idx]))
+                    data_args_values[arg_name].append(cur_row)
+        except EOFError:  # Exit if reached EOF or CTRL-D
+            break
+
+    if not data_present:
+        sys.exit(0)
+
+    # Update data as numpy arrays.
+    for arg_name in data_args_values:
+        np_values = np.array(data_args_values[arg_name])
+        data_args_values[arg_name] = np_values
+
+    # Combine all arguments.
+    all_args = {**data_args_values, **params}
+
+    module_ = importlib.import_module(module_name)
+    sklearn_model = getattr(module_, func_name)(**all_args)
+
+    model_str = pickle.dumps(sklearn_model)
+
+    if is_lake_system:
+        model_file_path = f"/tmp/{model_file_prefix}_{partition_join}.pickle"
+
+        # Write to file in Vantage, to be used in predict/scoring.
+        with open(model_file_path, "wb") as fp:
+            fp.write(model_str)
+
+    model_data = model_file_path if is_lake_system else base64.b64encode(model_str)
+
+    print(*(data_partition_column_values + [model_data]), sep=DELIMITER)

teradataml/data/scripts/sklearn/sklearn_model_selection_split.py

@@ -0,0 +1,166 @@
+import pickle
+import math
+import sys
+import numpy as np
+import base64
+from contextlib import contextmanager
+import os
+
+DELIMITER = '\t'
+
+@contextmanager
+def suppress_stderr():
+    """
+    Function to suppress the warnings(lake systems treats warnings as errors).
+    """
+    with open(os.devnull, "w") as devnull:
+        old_stderr = sys.stderr
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stderr = old_stderr
+
+## On Lake system warnings raised by script are treated as a errors. 
+## Hence, to suppress it putting the under suppress_stderr().
+with suppress_stderr():
+    def get_values_list(values, types):
+        ret_vals = []
+        for i, val in enumerate(values):
+            ret_vals.append(convert_to_type(val, types[i]))
+        return ret_vals
+
+    def convert_to_type(val, typee):
+        if typee == 'int':
+            return int(val) if val != "" else np.nan
+        if typee == 'float':
+            if isinstance(val, str):
+                val = val.replace(' ', '')
+            return float(val) if val != "" else np.nan
+        if typee == 'bool':
+            return eval(val) if val != "" else None
+        return str(val) if val != "" else None
+
+    def splitter(strr, delim=",", convert_to="str"):
+        """
+        Split the string based on delimiter and convert to the type specified.
+        """
+        if strr == "None":
+            return []
+        return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+
+    # Arguments to the Script
+    if len(sys.argv) != 9:
+        # 9 arguments command line arguments should be passed to this file.
+        # 1: file to be run
+        # 2. function name
+        # 3. No of feature columns.
+        # 4. No of class labels.
+        # 5. No of group columns.
+        # 6. Comma separated indices of partition columns.
+        # 7. Comma separated types of all the data columns.
+        # 8. Model file prefix to generated model file using partition columns.
+        # 9. Flag to check the system type. True, means Lake, Enterprise otherwise.
+        sys.exit("9 arguments command line arguments should be passed: file to be run,"
+                 " function name, no of feature columns, no of class labels, no of group columns,"
+                 " comma separated indices of partition columns, comma separated types of all columns,"
+                 " model file prefix to generated model file using partition columns and flag to check"
+                 " lake or enterprise.")
+
+
+    is_lake_system = eval(sys.argv[8])
+    if not is_lake_system:
+        db = sys.argv[0].split("/")[1]
+    function_name = sys.argv[1]
+    n_f_cols = int(sys.argv[2])
+    n_c_labels = int(sys.argv[3])
+    n_g_cols = int(sys.argv[4])
+    data_column_types = splitter(sys.argv[6], delim="--")
+    data_partition_column_indices = splitter(sys.argv[5], convert_to="int") # indices are integers.
+    model_file_prefix = sys.argv[7]
+
+    data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
+    model = None
+    data_partition_column_values = []
+
+    # Data Format (n_features, k_labels, one data_partition_column):
+    # feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ...,
+    # data_partition_columnn.
+    # labels are optional.
+
+    features = []
+    labels = []
+    groups = []
+    while 1:
+        try:
+            line = input()
+            if line == '':  # Exit if user provides blank line
+                break
+            else:
+                values = line.split(DELIMITER)
+                values = get_values_list(values, data_column_types)
+                if not data_partition_column_values:
+                    # Partition column values is same for all rows. Hence, only read once.
+                    for i, val in enumerate(data_partition_column_indices):
+                        data_partition_column_values.append(
+                            convert_to_type(values[val], typee=data_partition_column_types[i])
+                            )
+
+                    # Prepare the corresponding model file name and extract model.
+                    partition_join = "_".join([str(x) for x in data_partition_column_values])
+                    # Replace '-' with '_' as '-' because partition_columns can be negative.
+                    partition_join = partition_join.replace("-", "_")
+
+                    model_file_path = f"{model_file_prefix}_{partition_join}" \
+                        if is_lake_system else \
+                        f"./{db}/{model_file_prefix}_{partition_join}"
+
+                    with open(model_file_path, "rb") as fp:
+                        model = pickle.loads(fp.read())
+
+                    if not model:
+                        sys.exit("Model file is not installed in Vantage.")
+
+                start = 0
+                if n_f_cols > 0:
+                    features.append(values[:n_f_cols])
+                    start = start + n_f_cols
+                if n_c_labels > 0:
+                    labels.append(values[start:(start+n_c_labels)])
+                    start = start + n_c_labels
+                if n_g_cols > 0:
+                    groups.append(values[start:(start+n_g_cols)])
+
+        except EOFError:  # Exit if reached EOF or CTRL-D
+            break
+
+    if len(features) == 0:
+        sys.exit(0)
+
+    features = np.array(features) if len(features) > 0 else None
+    labels = np.array(labels).flatten() if len(labels) > 0 else None
+    groups = np.array(groups).flatten() if len(groups) > 0 else None
+
+    if function_name == "split":
+        # Printing both train and test data instead of just indices unlike sklearn.
+        # Generator is created based on split_id and type of split (train/test) in client.
+        split_id = 1
+        for train_idx, test_idx in model.split(features, labels, groups):
+            X_train, X_test = features[train_idx], features[test_idx]
+            y_train, y_test = labels[train_idx], labels[test_idx]
+            for X, y in zip(X_train, y_train):
+                print(*(data_partition_column_values + [split_id, "train"] +
+                        ['' if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val)))) else val
+                         for val in X] + [y]
+                        ), sep=DELIMITER)
+            for X, y in zip(X_test, y_test):
+                print(*(data_partition_column_values + [split_id, "test"] +
+                        ['' if (val is None or (not isinstance(val, str) and (math.isnan(val) or math.isinf(val)))) else val
+                         for val in X] + [y]
+                        ), sep=DELIMITER)
+            split_id += 1
+    else:
+        val = getattr(model, function_name)(features, labels, groups)
+        print(*(data_partition_column_values + [val]), sep=DELIMITER)

teradataml/data/scripts/sklearn/sklearn_neighbors.py

@@ -0,0 +1,161 @@
+import pickle
+import math
+import sys
+import numpy as np
+import os
+from contextlib import contextmanager
+
+# The below import is needed to convert sparse matrix to dense array as sparse matrices are NOT
+# supported in Vantage.
+# This is in scipy 1.6.x. Might vary based on scipy version.
+from scipy.sparse.csr import csr_matrix
+
+
+DELIMITER = '\t'
+
+
+@contextmanager
+def suppress_stderr():
+    """
+    Function to suppress the warnings(lake systems treats warnings as errors).
+    """
+    with open(os.devnull, "w") as devnull:
+        old_stderr = sys.stderr
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stderr = old_stderr
+
+## On Lake system warnings raised by script are treated as a errors.
+## Hence, to suppress it putting the under suppress_stderr().
+with suppress_stderr():
+    def get_values_list(values, types):
+        ret_vals = []
+        for i, val in enumerate(values):
+            ret_vals.append(convert_to_type(val, types[i]))
+        return ret_vals
+
+    def convert_to_type(val, typee):
+        if typee == 'int':
+            return int(val) if val != "" else np.nan
+        if typee == 'float':
+            if isinstance(val, str):
+                val = val.replace(' ', '')
+            return float(val) if val != "" else np.nan
+        if typee == 'bool':
+            return eval(val) if val != "" else None
+        return str(val) if val != "" else None
+
+    def splitter(strr, delim=",", convert_to="str"):
+        """
+        Split the string based on delimiter and convert to the type specified.
+        """
+        if strr == "None":
+            return []
+        return [convert_to_type(i, convert_to) for i in strr.split(delim)]
+
+
+    # Arguments to the Script
+    if len(sys.argv) < 7:
+        # At least 7 arguments command line arguments should be passed to this file.
+        # 1: file to be run
+        # 2. function name.
+        # 3. No of feature columns.
+        # 4. Comma separated indices of partition columns.
+        # 5. Comma separated types of all the data columns.
+        # 6. Model file prefix to generate model file using partition columns.
+        # 7. Flag to check the system type. True, means Lake, Enterprise otherwise.
+        # 8. OPTIONAL - Arguments in string format like "return_distance True-bool",
+        #    "n_neighbors 3-int", "radius 3.4-float" etc.
+        sys.exit("At least 7 arguments should be passed to this file - file to be run, function name, "\
+                 "no of feature columns, comma separated indices of partition columns, comma "\
+                 "separated types of all columns, model file prefix to generate model file using "\
+                 "partition columns, flag to check lake or enterprise and optional arguments in string format.")
+
+    convert_to_int = lambda x: int(x) if x != "None" else None
+
+    is_lake_system = eval(sys.argv[6])
+    if not is_lake_system:
+        db = sys.argv[0].split("/")[1]
+    func_name = sys.argv[1]
+    n_f_cols = convert_to_int(sys.argv[2])
+    data_column_types = splitter(sys.argv[4], delim="--")
+    data_partition_column_indices = splitter(sys.argv[3], convert_to="int") # indices are integers.
+    model_file_prefix = sys.argv[5]
+    # Extract arguments from string.
+    arguments = {}
+    for i in range(7, len(sys.argv), 2):
+        value = sys.argv[i + 1].split("-", 1)
+        arguments[sys.argv[i]] = convert_to_type(value[0], value[1])
+
+    data_partition_column_types = [data_column_types[idx] for idx in data_partition_column_indices]
+
+    model = None
+    data_partition_column_values = []
+
+    # Data Format:
+    # feature1, feature2, ..., featuren, label1, label2, ... labelk, data_partition_column1, ...,
+    # data_partition_columnn.
+    # label is optional (it is present when label_exists is not "None")
+
+    # `return_distance` is needed as the result is a tuple of two arrays when it is True.
+    return_distance = arguments.get("return_distance", True) # Default value is True.
+
+    while 1:
+        try:
+            line = input()
+            if line == '':  # Exit if user provides blank line
+                break
+            else:
+                values = line.split(DELIMITER)
+                values = get_values_list(values, data_column_types)
+                if not data_partition_column_values:
+                    # Partition column values is same for all rows. Hence, only read once.
+                    for i, val in enumerate(data_partition_column_indices):
+                        data_partition_column_values.append(
+                            convert_to_type(values[val], typee=data_partition_column_types[i])
+                            )
+
+                    # Prepare the corresponding model file name and extract model.
+                    partition_join = "_".join([str(x) for x in data_partition_column_values])
+                    # Replace '-' with '_' as '-' because partition_columns can be negative.
+                    partition_join = partition_join.replace("-", "_")
+
+                    model_file_path = f"{model_file_prefix}_{partition_join}" \
+                        if is_lake_system else \
+                        f"./{db}/{model_file_prefix}_{partition_join}"
+
+                    with open(model_file_path, "rb") as fp:
+                        model = pickle.loads(fp.read())
+
+                    if not model:
+                        sys.exit("Model file is not installed in Vantage.")
+
+                f_ = values[:n_f_cols]
+                if f_:
+                    output = getattr(model, func_name)([f_], **arguments)
+                else:
+                    output = getattr(model, func_name)(**arguments)
+                result_list = f_
+
+                if func_name in ['kneighbors', 'radius_neighbors']:
+                    if return_distance:
+                        result_list += [str(output[0][0].tolist()), str(output[1][0].tolist())]
+                    else:
+                        result_list += [str(output[0].tolist())]
+                else:
+                    # cases like 'kneighbors_graph', 'radius_neighbors_graph' and other functions.
+                    if isinstance(output, csr_matrix):
+                        # 'kneighbors_graph', 'radius_neighbors_graph' return sparse matrix.
+                        output = output.toarray()
+                    result_list += [str(output[0].tolist())]
+
+                print(*(data_partition_column_values +
+                        ['' if (val is None or (not isinstance(val, str) and
+                                                (math.isnan(val) or math.isinf(val))))
+                            else val
+                        for val in result_list]), sep=DELIMITER)
+
+        except EOFError:  # Exit if reached EOF or CTRL-D
+            break